From e90569905ecdfe3225314f0e5f122263aa68efb2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 2 Sep 2013 15:18:15 +0200
Subject: [PATCH 0001/2196] Initial version of nix-repl

This program interactively reads a Nix expression from the user,
evaluates it, and prints the result.
---
 default.nix | 10 ++++++++++
 nix-repl.cc | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 default.nix
 create mode 100644 nix-repl.cc

diff --git a/default.nix b/default.nix
new file mode 100644
index 00000000000..55c9ed9e3af
--- /dev/null
+++ b/default.nix
@@ -0,0 +1,10 @@
+with import <nixpkgs> { };
+
+runCommand "nix-repl"
+  { buildInputs = [ readline nixUnstable boehmgc ]; }
+  ''
+    mkdir -p $out/bin
+    g++ -O3 -Wall -o $out/bin/nix-repl ${./nix-repl.cc} \
+      -I${nixUnstable}/include/nix -L${nixUnstable}/lib/nix \
+      -lexpr -lmain -lreadline
+  ''
diff --git a/nix-repl.cc b/nix-repl.cc
new file mode 100644
index 00000000000..ea248b5ca15
--- /dev/null
+++ b/nix-repl.cc
@@ -0,0 +1,57 @@
+#include <iostream>
+#include <cstdlib>
+
+#include <readline/readline.h>
+#include <readline/history.h>
+
+#include "shared.hh"
+#include "eval.hh"
+
+using namespace std;
+using namespace nix;
+
+
+string programId = "nix-repl";
+
+
+void printHelp()
+{
+    std::cout << "Usage: nix-repl\n";
+}
+
+
+bool getLine(string & line)
+{
+    char * s = readline ("nix-repl> ");
+    if (!s) return false;
+    line = chomp(string(s));
+    free(s);
+    if (line != "") add_history(line.c_str());
+    return true;
+}
+
+
+void run(nix::Strings args)
+{
+    EvalState state;
+    Path curDir = absPath(".");
+
+    while (true) {
+        string line;
+        if (!getLine(line)) break;
+
+        try {
+            Expr * e = state.parseExprFromString(line, curDir);
+            Value v;
+            state.eval(e, v);
+            state.strictForceValue(v);
+            std::cout << v << std::endl;
+        } catch (Error & e) {
+            printMsg(lvlError, e.msg());
+        }
+
+        std::cout << std::endl;
+    }
+
+    std::cout << std::endl;
+}

From 287c88ca59c5eae2b33874acc6271ca30b7b7e52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 2 Sep 2013 17:53:58 +0200
Subject: [PATCH 0002/2196] Support adding variables to the scope

The command ":a <expr>" evaluates <expr> and adds the attributes in
the resulting attribute set to the interpreter scope.  For instance:

nix-repl> :a import <nixpkgs> {}

nix-repl> lib.range 0 10
[ 0 1 2 3 4 5 6 7 8 9 10 ]
---
 nix-repl.cc | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 86 insertions(+), 9 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index ea248b5ca15..10e91ec5328 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -6,6 +6,8 @@
 
 #include "shared.hh"
 #include "eval.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
 
 using namespace std;
 using namespace nix;
@@ -14,6 +16,23 @@ using namespace nix;
 string programId = "nix-repl";
 
 
+struct NixRepl
+{
+    string curDir;
+    EvalState state;
+
+    StaticEnv staticEnv;
+    Env * env;
+    int displ;
+
+    NixRepl();
+    void mainLoop();
+    void processLine(string line);
+    void addVar(const Symbol & name, Value * v);
+    Expr * parseString(string s);
+};
+
+
 void printHelp()
 {
     std::cout << "Usage: nix-repl\n";
@@ -22,7 +41,7 @@ void printHelp()
 
 bool getLine(string & line)
 {
-    char * s = readline ("nix-repl> ");
+    char * s = readline("nix-repl> ");
     if (!s) return false;
     line = chomp(string(s));
     free(s);
@@ -31,21 +50,33 @@ bool getLine(string & line)
 }
 
 
-void run(nix::Strings args)
+NixRepl::NixRepl()
+    : staticEnv(false, &state.staticBaseEnv)
 {
-    EvalState state;
-    Path curDir = absPath(".");
+    curDir = absPath(".");
+
+    env = &state.allocEnv(32768);
+    env->up = &state.baseEnv;
+    displ = 0;
+
+    store = openStore();
+}
+
+
+void NixRepl::mainLoop()
+{
+    std::cerr << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
     while (true) {
         string line;
         if (!getLine(line)) break;
 
+        /* Remove preceeding whitespace. */
+        size_t n = line.find_first_not_of(" \n\r\t");
+        if (n != string::npos) line = string(line, n);
+
         try {
-            Expr * e = state.parseExprFromString(line, curDir);
-            Value v;
-            state.eval(e, v);
-            state.strictForceValue(v);
-            std::cout << v << std::endl;
+            processLine(line);
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         }
@@ -55,3 +86,49 @@ void run(nix::Strings args)
 
     std::cout << std::endl;
 }
+
+
+void NixRepl::processLine(string line)
+{
+    if (string(line, 0, 2) == ":a") {
+        Expr * e = parseString(string(line, 2));
+        Value v;
+        e->eval(state, *env, v);
+        state.forceAttrs(v);
+        foreach (Bindings::iterator, i, *v.attrs)
+            addVar(i->name, i->value);
+    }
+
+    else if (string(line, 0, 1) == ":") {
+        throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
+    }
+
+    else {
+        Expr * e = parseString(line);
+        Value v;
+        e->eval(state, *env, v);
+        state.strictForceValue(v);
+        std::cout << v << std::endl;
+    }
+}
+
+
+void NixRepl::addVar(const Symbol & name, Value * v)
+{
+    staticEnv.vars[name] = displ;
+    env->values[displ++] = v;
+}
+
+
+Expr * NixRepl::parseString(string s)
+{
+    Expr * e = state.parseExprFromString(s, curDir, staticEnv);
+    return e;
+}
+
+
+void run(nix::Strings args)
+{
+    NixRepl repl;
+    repl.mainLoop();
+}

From 3202206d1d906ea6279dadfe608ea92ea0aaf927 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 2 Sep 2013 16:00:48 +0000
Subject: [PATCH 0003/2196] Add a command :t for showing the type of an
 expression

---
 nix-repl.cc | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 10e91ec5328..9858a034d72 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -30,6 +30,7 @@ struct NixRepl
     void processLine(string line);
     void addVar(const Symbol & name, Value * v);
     Expr * parseString(string s);
+    void evalString(string s, Value & v);
 };
 
 
@@ -91,22 +92,26 @@ void NixRepl::mainLoop()
 void NixRepl::processLine(string line)
 {
     if (string(line, 0, 2) == ":a") {
-        Expr * e = parseString(string(line, 2));
         Value v;
-        e->eval(state, *env, v);
+        evalString(string(line, 2), v);
         state.forceAttrs(v);
         foreach (Bindings::iterator, i, *v.attrs)
             addVar(i->name, i->value);
     }
 
+    else if (string(line, 0, 2) == ":t") {
+        Value v;
+        evalString(string(line, 2), v);
+        std::cout << showType(v) << std::endl;
+    }
+
     else if (string(line, 0, 1) == ":") {
         throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
     }
 
     else {
-        Expr * e = parseString(line);
         Value v;
-        e->eval(state, *env, v);
+        evalString(line, v);
         state.strictForceValue(v);
         std::cout << v << std::endl;
     }
@@ -127,6 +132,14 @@ Expr * NixRepl::parseString(string s)
 }
 
 
+void NixRepl::evalString(string s, Value & v)
+{
+    Expr * e = parseString(s);
+    e->eval(state, *env, v);
+    state.forceValue(v);
+}
+
+
 void run(nix::Strings args)
 {
     NixRepl repl;

From 0f6279d87421f19cd2c1e286163d7567f13dc77f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 2 Sep 2013 18:18:27 +0200
Subject: [PATCH 0004/2196] Add a command :l for loading a file into scope

Example:

nix-repl> :l <nixpkgs>

nix-repl> lib.range 0 10
[ 0 1 2 3 4 5 6 7 8 9 10 ]

nix-repl> :l <nixos>

nix-repl> config.boot.kernelModules
[ "vboxdrv" "vboxnetadp" ... ]
---
 nix-repl.cc | 43 +++++++++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 9858a034d72..0b84834491e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -8,6 +8,7 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
+#include "common-opts.hh"
 
 using namespace std;
 using namespace nix;
@@ -28,7 +29,8 @@ struct NixRepl
     NixRepl();
     void mainLoop();
     void processLine(string line);
-    void addVar(const Symbol & name, Value * v);
+    void addAttrsToScope(Value & attrs);
+    void addVarToScope(const Symbol & name, Value * v);
     Expr * parseString(string s);
     void evalString(string s, Value & v);
 };
@@ -51,6 +53,15 @@ bool getLine(string & line)
 }
 
 
+string removeWhitespace(string s)
+{
+    s = chomp(s);
+    size_t n = s.find_first_not_of(" \n\r\t");
+    if (n != string::npos) s = string(s, n);
+    return s;
+}
+
+
 NixRepl::NixRepl()
     : staticEnv(false, &state.staticBaseEnv)
 {
@@ -72,12 +83,8 @@ void NixRepl::mainLoop()
         string line;
         if (!getLine(line)) break;
 
-        /* Remove preceeding whitespace. */
-        size_t n = line.find_first_not_of(" \n\r\t");
-        if (n != string::npos) line = string(line, n);
-
         try {
-            processLine(line);
+            processLine(removeWhitespace(line));
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
         }
@@ -94,9 +101,17 @@ void NixRepl::processLine(string line)
     if (string(line, 0, 2) == ":a") {
         Value v;
         evalString(string(line, 2), v);
-        state.forceAttrs(v);
-        foreach (Bindings::iterator, i, *v.attrs)
-            addVar(i->name, i->value);
+        addAttrsToScope(v);
+    }
+
+    else if (string(line, 0, 2) == ":l") {
+        state.resetFileCache();
+        Path path = lookupFileArg(state, removeWhitespace(string(line, 2)));
+        Value v, v2;
+        state.evalFile(path, v);
+        Bindings bindings;
+        state.autoCallFunction(bindings, v, v2);
+        addAttrsToScope(v2);
     }
 
     else if (string(line, 0, 2) == ":t") {
@@ -118,7 +133,15 @@ void NixRepl::processLine(string line)
 }
 
 
-void NixRepl::addVar(const Symbol & name, Value * v)
+void NixRepl::addAttrsToScope(Value & attrs)
+{
+    state.forceAttrs(attrs);
+    foreach (Bindings::iterator, i, *attrs.attrs)
+        addVarToScope(i->name, i->value);
+}
+
+
+void NixRepl::addVarToScope(const Symbol & name, Value * v)
 {
     staticEnv.vars[name] = displ;
     env->values[displ++] = v;

From 504563ea4431f765028dc0ccacd5ee834d0d8a91 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 11:54:49 +0200
Subject: [PATCH 0005/2196] Fix build

---
 default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/default.nix b/default.nix
index 55c9ed9e3af..635ead71ea5 100644
--- a/default.nix
+++ b/default.nix
@@ -6,5 +6,5 @@ runCommand "nix-repl"
     mkdir -p $out/bin
     g++ -O3 -Wall -o $out/bin/nix-repl ${./nix-repl.cc} \
       -I${nixUnstable}/include/nix -L${nixUnstable}/lib/nix \
-      -lexpr -lmain -lreadline
+      -lexpr -lmain -lreadline -lgc
   ''

From 4fb82d3d800be9a53631c3c8a3321a359306d835 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 13:01:02 +0200
Subject: [PATCH 0006/2196] Handle SIGINT to cancel the current line

---
 nix-repl.cc | 42 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 0b84834491e..bb205211793 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -1,6 +1,8 @@
 #include <iostream>
 #include <cstdlib>
 
+#include <setjmp.h>
+
 #include <readline/readline.h>
 #include <readline/history.h>
 
@@ -42,13 +44,41 @@ void printHelp()
 }
 
 
+/* Apparently, the only way to get readline() to return on Ctrl-C
+   (SIGINT) is to use siglongjmp().  That's fucked up... */
+static sigjmp_buf sigintJmpBuf;
+
+
+static void sigintHandler(int signo)
+{
+    siglongjmp(sigintJmpBuf, 1);
+}
+
+
 bool getLine(string & line)
 {
-    char * s = readline("nix-repl> ");
-    if (!s) return false;
-    line = chomp(string(s));
-    free(s);
-    if (line != "") add_history(line.c_str());
+    struct sigaction act, old;
+    act.sa_handler = sigintHandler;
+    sigfillset(&act.sa_mask);
+    act.sa_flags = 0;
+    if (sigaction(SIGINT, &act, &old))
+        throw SysError("installing handler for SIGINT");
+
+    if (sigsetjmp(sigintJmpBuf, 1))
+        line = "";
+    else {
+        char * s = readline("nix-repl> ");
+        if (!s) return false;
+        line = chomp(string(s));
+        free(s);
+        if (line != "") add_history(line.c_str());
+    }
+
+    _isInterrupted = 0;
+
+    if (sigaction(SIGINT, &old, 0))
+        throw SysError("restoring handler for SIGINT");
+
     return true;
 }
 
@@ -98,6 +128,8 @@ void NixRepl::mainLoop()
 
 void NixRepl::processLine(string line)
 {
+    if (line == "") return;
+
     if (string(line, 0, 2) == ":a") {
         Value v;
         evalString(string(line, 2), v);

From 0b419c048b206e2fe68758ea1bd5fa7b1c29c521 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 13:14:28 +0200
Subject: [PATCH 0007/2196] Use readline history file

---
 nix-repl.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index bb205211793..bb18359c155 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -71,7 +71,10 @@ bool getLine(string & line)
         if (!s) return false;
         line = chomp(string(s));
         free(s);
-        if (line != "") add_history(line.c_str());
+        if (line != "") {
+            add_history(line.c_str());
+            append_history(1, 0);
+        }
     }
 
     _isInterrupted = 0;
@@ -109,6 +112,9 @@ void NixRepl::mainLoop()
 {
     std::cerr << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
+    using_history();
+    read_history(0);
+
     while (true) {
         string line;
         if (!getLine(line)) break;

From ad0dd359b4434db84bf6458715440cc15f896ddc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 13:20:35 +0200
Subject: [PATCH 0008/2196] Don't exit on SIGINT during evaluation

However, this may leave thunks in black-holed state, so it's not
really safe.
---
 nix-repl.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/nix-repl.cc b/nix-repl.cc
index bb18359c155..45d7433396e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -123,6 +123,8 @@ void NixRepl::mainLoop()
             processLine(removeWhitespace(line));
         } catch (Error & e) {
             printMsg(lvlError, e.msg());
+        } catch (Interrupted & e) {
+            printMsg(lvlError, e.msg());
         }
 
         std::cout << std::endl;
@@ -176,6 +178,7 @@ void NixRepl::addAttrsToScope(Value & attrs)
     state.forceAttrs(attrs);
     foreach (Bindings::iterator, i, *attrs.attrs)
         addVarToScope(i->name, i->value);
+    printMsg(lvlError, format("added %1% variables") % attrs.attrs->size());
 }
 
 

From c6712a007fc55398893995a3466d35ae0697db05 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 14:58:53 +0200
Subject: [PATCH 0009/2196] Add a command :b to build a derivation

---
 nix-repl.cc | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 45d7433396e..f41cf4d292d 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -11,6 +11,8 @@
 #include "eval-inline.hh"
 #include "store-api.hh"
 #include "common-opts.hh"
+#include "get-drvs.hh"
+#include "derivations.hh"
 
 using namespace std;
 using namespace nix;
@@ -122,9 +124,9 @@ void NixRepl::mainLoop()
         try {
             processLine(removeWhitespace(line));
         } catch (Error & e) {
-            printMsg(lvlError, e.msg());
+            printMsg(lvlError, "error: " + e.msg());
         } catch (Interrupted & e) {
-            printMsg(lvlError, e.msg());
+            printMsg(lvlError, "error: " + e.msg());
         }
 
         std::cout << std::endl;
@@ -160,10 +162,29 @@ void NixRepl::processLine(string line)
         std::cout << showType(v) << std::endl;
     }
 
-    else if (string(line, 0, 1) == ":") {
-        throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
+    else if (string(line, 0, 2) == ":b") {
+        Value v;
+        evalString(string(line, 2), v);
+        DrvInfo drvInfo;
+        if (!getDerivation(state, v, drvInfo, false))
+            throw Error("expression does not evaluation to a derivation, so I can't build it");
+        Path drvPath = drvInfo.queryDrvPath(state);
+        if (drvPath == "" || !store->isValidPath(drvPath))
+            throw Error("expression did not evaluate to a valid derivation");
+        /* We could do the build in this process using buildPaths(),
+           but doing it in a child makes it easier to recover from
+           problems / SIGINT. */
+        if (system(("nix-store -r " + drvPath + " > /dev/null").c_str()) == -1)
+            throw SysError("starting nix-store");
+        Derivation drv = parseDerivation(readFile(drvPath));
+        std::cout << "this derivation produced the following outputs:" << std::endl;
+        foreach (DerivationOutputs::iterator, i, drv.outputs)
+            std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
     }
 
+    else if (string(line, 0, 1) == ":")
+        throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
+
     else {
         Value v;
         evalString(line, v);
@@ -178,7 +199,7 @@ void NixRepl::addAttrsToScope(Value & attrs)
     state.forceAttrs(attrs);
     foreach (Bindings::iterator, i, *attrs.attrs)
         addVarToScope(i->name, i->value);
-    printMsg(lvlError, format("added %1% variables") % attrs.attrs->size());
+    std::cout << format("added %1% variables") % attrs.attrs->size() << std::endl;
 }
 
 

From b5944ac4ffffde7f52d0bbe3f4a2d53bba70bb66 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 15:05:18 +0200
Subject: [PATCH 0010/2196] Add a command :s to start a nix-shell for a
 derivation

---
 nix-repl.cc | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index f41cf4d292d..838928b35dd 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -140,13 +140,15 @@ void NixRepl::processLine(string line)
 {
     if (line == "") return;
 
-    if (string(line, 0, 2) == ":a") {
+    string command = string(line, 0, 2);
+
+    if (command == ":a") {
         Value v;
         evalString(string(line, 2), v);
         addAttrsToScope(v);
     }
 
-    else if (string(line, 0, 2) == ":l") {
+    else if (command == ":l") {
         state.resetFileCache();
         Path path = lookupFileArg(state, removeWhitespace(string(line, 2)));
         Value v, v2;
@@ -156,13 +158,13 @@ void NixRepl::processLine(string line)
         addAttrsToScope(v2);
     }
 
-    else if (string(line, 0, 2) == ":t") {
+    else if (command == ":t") {
         Value v;
         evalString(string(line, 2), v);
         std::cout << showType(v) << std::endl;
     }
 
-    else if (string(line, 0, 2) == ":b") {
+    else if (command == ":b" || command == ":s") {
         Value v;
         evalString(string(line, 2), v);
         DrvInfo drvInfo;
@@ -171,15 +173,21 @@ void NixRepl::processLine(string line)
         Path drvPath = drvInfo.queryDrvPath(state);
         if (drvPath == "" || !store->isValidPath(drvPath))
             throw Error("expression did not evaluate to a valid derivation");
-        /* We could do the build in this process using buildPaths(),
-           but doing it in a child makes it easier to recover from
-           problems / SIGINT. */
-        if (system(("nix-store -r " + drvPath + " > /dev/null").c_str()) == -1)
-            throw SysError("starting nix-store");
-        Derivation drv = parseDerivation(readFile(drvPath));
-        std::cout << "this derivation produced the following outputs:" << std::endl;
-        foreach (DerivationOutputs::iterator, i, drv.outputs)
-            std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
+
+        if (command == ":b") {
+            /* We could do the build in this process using buildPaths(),
+               but doing it in a child makes it easier to recover from
+               problems / SIGINT. */
+            if (system(("nix-store -r " + drvPath + " > /dev/null").c_str()) == -1)
+                throw SysError("starting nix-store");
+            Derivation drv = parseDerivation(readFile(drvPath));
+            std::cout << "this derivation produced the following outputs:" << std::endl;
+            foreach (DerivationOutputs::iterator, i, drv.outputs)
+                std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
+        } else {
+            if (system(("nix-shell " + drvPath).c_str()) == -1)
+                throw SysError("starting nix-shell");
+        }
     }
 
     else if (string(line, 0, 1) == ":")

From cf4c29d90a1b0ec7f8cd7f0c5258be63a2e02058 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 15:20:06 +0200
Subject: [PATCH 0011/2196] Load files specified on the command line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For example:

  $ nix-repl '<nixpkgs>' '<nixos>'
  Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.

  Loading ‘<nixpkgs>’...
  Added 3337 variables.

  Loading ‘<nixos>’...
  Added 7 variables.

  nix-repl>
---
 nix-repl.cc | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 838928b35dd..2ef23faa585 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -31,8 +31,9 @@ struct NixRepl
     int displ;
 
     NixRepl();
-    void mainLoop();
+    void mainLoop(const Strings & args);
     void processLine(string line);
+    void loadFile(const Path & path);
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol & name, Value * v);
     Expr * parseString(string s);
@@ -110,9 +111,15 @@ NixRepl::NixRepl()
 }
 
 
-void NixRepl::mainLoop()
+void NixRepl::mainLoop(const Strings & args)
 {
-    std::cerr << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
+    std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
+
+    foreach (Strings::const_iterator, i, args) {
+        std::cout << format("Loading ‘%1%’...") % *i << std::endl;
+        loadFile(*i);
+        std::cout << std::endl;
+    }
 
     using_history();
     read_history(0);
@@ -150,12 +157,7 @@ void NixRepl::processLine(string line)
 
     else if (command == ":l") {
         state.resetFileCache();
-        Path path = lookupFileArg(state, removeWhitespace(string(line, 2)));
-        Value v, v2;
-        state.evalFile(path, v);
-        Bindings bindings;
-        state.autoCallFunction(bindings, v, v2);
-        addAttrsToScope(v2);
+        loadFile(removeWhitespace(string(line, 2)));
     }
 
     else if (command == ":t") {
@@ -202,12 +204,22 @@ void NixRepl::processLine(string line)
 }
 
 
+void NixRepl::loadFile(const Path & path)
+{
+    Value v, v2;
+    state.evalFile(lookupFileArg(state, path), v);
+    Bindings bindings;
+    state.autoCallFunction(bindings, v, v2);
+    addAttrsToScope(v2);
+}
+
+
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state.forceAttrs(attrs);
     foreach (Bindings::iterator, i, *attrs.attrs)
         addVarToScope(i->name, i->value);
-    std::cout << format("added %1% variables") % attrs.attrs->size() << std::endl;
+    std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
 }
 
 
@@ -233,8 +245,8 @@ void NixRepl::evalString(string s, Value & v)
 }
 
 
-void run(nix::Strings args)
+void run(Strings args)
 {
     NixRepl repl;
-    repl.mainLoop();
+    repl.mainLoop(args);
 }

From 0abdf4beaaa64c01ed3cfb4cc5fd78997116fac8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 19:51:59 +0200
Subject: [PATCH 0012/2196] Add basic variable name completion

---
 nix-repl.cc | 129 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 87 insertions(+), 42 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 2ef23faa585..f85eac1246b 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -29,9 +29,15 @@ struct NixRepl
     StaticEnv staticEnv;
     Env * env;
     int displ;
+    StringSet varNames;
+
+    StringSet completions;
+    StringSet::iterator curCompletion;
 
     NixRepl();
     void mainLoop(const Strings & args);
+    void completePrefix(string prefix);
+    bool getLine(string & line);
     void processLine(string line);
     void loadFile(const Path & path);
     void addAttrsToScope(Value & attrs);
@@ -47,48 +53,6 @@ void printHelp()
 }
 
 
-/* Apparently, the only way to get readline() to return on Ctrl-C
-   (SIGINT) is to use siglongjmp().  That's fucked up... */
-static sigjmp_buf sigintJmpBuf;
-
-
-static void sigintHandler(int signo)
-{
-    siglongjmp(sigintJmpBuf, 1);
-}
-
-
-bool getLine(string & line)
-{
-    struct sigaction act, old;
-    act.sa_handler = sigintHandler;
-    sigfillset(&act.sa_mask);
-    act.sa_flags = 0;
-    if (sigaction(SIGINT, &act, &old))
-        throw SysError("installing handler for SIGINT");
-
-    if (sigsetjmp(sigintJmpBuf, 1))
-        line = "";
-    else {
-        char * s = readline("nix-repl> ");
-        if (!s) return false;
-        line = chomp(string(s));
-        free(s);
-        if (line != "") {
-            add_history(line.c_str());
-            append_history(1, 0);
-        }
-    }
-
-    _isInterrupted = 0;
-
-    if (sigaction(SIGINT, &old, 0))
-        throw SysError("restoring handler for SIGINT");
-
-    return true;
-}
-
-
 string removeWhitespace(string s)
 {
     s = chomp(s);
@@ -143,6 +107,86 @@ void NixRepl::mainLoop(const Strings & args)
 }
 
 
+/* Apparently, the only way to get readline() to return on Ctrl-C
+   (SIGINT) is to use siglongjmp().  That's fucked up... */
+static sigjmp_buf sigintJmpBuf;
+
+
+static void sigintHandler(int signo)
+{
+    siglongjmp(sigintJmpBuf, 1);
+}
+
+
+/* Oh, if only g++ had nested functions... */
+NixRepl * curRepl;
+
+char * completerThunk(const char * s, int state)
+{
+    string prefix(s);
+
+    /* If the prefix has a slash in it, use readline's builtin filename
+       completer. */
+    if (prefix.find('/') != string::npos)
+        return rl_filename_completion_function(s, state);
+
+    /* Otherwise, return all symbols that start with the prefix. */
+    if (state == 0) {
+        curRepl->completePrefix(s);
+        curRepl->curCompletion = curRepl->completions.begin();
+    }
+    if (curRepl->curCompletion == curRepl->completions.end()) return 0;
+    return strdup((curRepl->curCompletion++)->c_str());
+}
+
+
+bool NixRepl::getLine(string & line)
+{
+    struct sigaction act, old;
+    act.sa_handler = sigintHandler;
+    sigfillset(&act.sa_mask);
+    act.sa_flags = 0;
+    if (sigaction(SIGINT, &act, &old))
+        throw SysError("installing handler for SIGINT");
+
+    if (sigsetjmp(sigintJmpBuf, 1))
+        line = "";
+    else {
+        curRepl = this;
+        rl_completion_entry_function = completerThunk;
+
+        char * s = readline("nix-repl> ");
+        if (!s) return false;
+        line = chomp(string(s));
+        free(s);
+        if (line != "") {
+            add_history(line.c_str());
+            append_history(1, 0);
+        }
+    }
+
+    _isInterrupted = 0;
+
+    if (sigaction(SIGINT, &old, 0))
+        throw SysError("restoring handler for SIGINT");
+
+    return true;
+}
+
+
+void NixRepl::completePrefix(string prefix)
+{
+    completions.clear();
+
+    StringSet::iterator i = varNames.lower_bound(prefix);
+    while (i != varNames.end()) {
+        if (string(*i, 0, prefix.size()) != prefix) break;
+        completions.insert(*i);
+        i++;
+    }
+}
+
+
 void NixRepl::processLine(string line)
 {
     if (line == "") return;
@@ -227,6 +271,7 @@ void NixRepl::addVarToScope(const Symbol & name, Value * v)
 {
     staticEnv.vars[name] = displ;
     env->values[displ++] = v;
+    varNames.insert((string) name);
 }
 
 

From c6f2b89c0e3091e5020983c9a02f36b9c33c3f81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 6 Sep 2013 21:00:36 +0200
Subject: [PATCH 0013/2196] Restore affinity

---
 default.nix |  3 ++-
 nix-repl.cc | 33 ++++++++++++++++++++++++++-------
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/default.nix b/default.nix
index 635ead71ea5..eae1d56d0a7 100644
--- a/default.nix
+++ b/default.nix
@@ -4,7 +4,8 @@ runCommand "nix-repl"
   { buildInputs = [ readline nixUnstable boehmgc ]; }
   ''
     mkdir -p $out/bin
-    g++ -O3 -Wall -o $out/bin/nix-repl ${./nix-repl.cc} \
+    g++ -O3 -Wall -std=c++0x \
+      -o $out/bin/nix-repl ${./nix-repl.cc} \
       -I${nixUnstable}/include/nix -L${nixUnstable}/lib/nix \
       -lexpr -lmain -lreadline -lgc
   ''
diff --git a/nix-repl.cc b/nix-repl.cc
index f85eac1246b..15ac42dcc77 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -13,6 +13,7 @@
 #include "common-opts.hh"
 #include "get-drvs.hh"
 #include "derivations.hh"
+#include "affinity.hh"
 
 using namespace std;
 using namespace nix;
@@ -187,6 +188,27 @@ void NixRepl::completePrefix(string prefix)
 }
 
 
+static int runProgram(const string & program, const Strings & args)
+{
+    std::vector<const char *> cargs; /* careful with c_str()! */
+    cargs.push_back(program.c_str());
+    for (Strings::const_iterator i = args.begin(); i != args.end(); ++i)
+        cargs.push_back(i->c_str());
+    cargs.push_back(0);
+
+    Pid pid;
+    pid = fork();
+    if (pid == -1) throw SysError("forking");
+    if (pid == 0) {
+        restoreAffinity();
+        execvp(program.c_str(), (char * *) &cargs[0]);
+        _exit(1);
+    }
+
+    return pid.wait(true);
+}
+
+
 void NixRepl::processLine(string line)
 {
     if (line == "") return;
@@ -224,16 +246,13 @@ void NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (system(("nix-store -r " + drvPath + " > /dev/null").c_str()) == -1)
-                throw SysError("starting nix-store");
+            if (runProgram("nix-store", Strings{"-r", drvPath}) != 0) return;
             Derivation drv = parseDerivation(readFile(drvPath));
-            std::cout << "this derivation produced the following outputs:" << std::endl;
+            std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
             foreach (DerivationOutputs::iterator, i, drv.outputs)
                 std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
-        } else {
-            if (system(("nix-shell " + drvPath).c_str()) == -1)
-                throw SysError("starting nix-shell");
-        }
+        } else
+            runProgram("nix-shell", Strings{drvPath});
     }
 
     else if (string(line, 0, 1) == ":")

From 7e3625f924825db2ffa5e58d4b414d93d9af2465 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Sat, 7 Sep 2013 00:35:54 +0200
Subject: [PATCH 0014/2196] Improved value display

By default, we don't recurse into attribute sets or lists when
printing a value.  However, the new :p command does recurse.
---
 nix-repl.cc | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 116 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 15ac42dcc77..d2179d77a8e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -45,6 +45,7 @@ struct NixRepl
     void addVarToScope(const Symbol & name, Value * v);
     Expr * parseString(string s);
     void evalString(string s, Value & v);
+    std::ostream &  printValue(std::ostream & str, Value & v, unsigned int maxDepth);
 };
 
 
@@ -255,14 +256,19 @@ void NixRepl::processLine(string line)
             runProgram("nix-shell", Strings{drvPath});
     }
 
+    else if (command == ":p") {
+        Value v;
+        evalString(string(line, 2), v);
+        printValue(std::cout, v, 1000000000) << std::endl;
+    }
+
     else if (string(line, 0, 1) == ":")
         throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
 
     else {
         Value v;
         evalString(line, v);
-        state.strictForceValue(v);
-        std::cout << v << std::endl;
+        printValue(std::cout, v, 1) << std::endl;
     }
 }
 
@@ -309,6 +315,114 @@ void NixRepl::evalString(string s, Value & v)
 }
 
 
+// FIXME: lot of cut&paste from Nix's eval.cc.
+std::ostream &  NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
+{
+    str.flush();
+    checkInterrupt();
+
+    state.forceValue(v);
+
+    switch (v.type) {
+
+    case tInt:
+        str << v.integer;
+        break;
+
+    case tBool:
+        str << (v.boolean ? "true" : "false");
+        break;
+
+    case tString:
+        str << "\"";
+        for (const char * i = v.string.s; *i; i++)
+            if (*i == '\"' || *i == '\\') str << "\\" << *i;
+            else if (*i == '\n') str << "\\n";
+            else if (*i == '\r') str << "\\r";
+            else if (*i == '\t') str << "\\t";
+            else str << *i;
+        str << "\"";
+        break;
+
+    case tPath:
+        str << v.path; // !!! escaping?
+        break;
+
+    case tNull:
+        str << "null";
+        break;
+
+    case tAttrs: {
+        bool isDrv = state.isDerivation(v);
+        if (isDrv) str << "(derivation ";
+        str << "{ ";
+
+        if (maxDepth > 0) {
+            typedef std::map<string, Value *> Sorted;
+            Sorted sorted;
+            foreach (Bindings::iterator, i, *v.attrs)
+                sorted[i->name] = i->value;
+
+            /* If this is a derivation, then don't show the
+               self-references ("all", "out", etc.). */
+            StringSet hidden;
+            if (isDrv) {
+                hidden.insert("all");
+                Bindings::iterator i = v.attrs->find(state.sOutputs);
+                if (i == v.attrs->end())
+                    hidden.insert("out");
+                else {
+                    state.forceList(*i->value);
+                    for (unsigned int j = 0; j < i->value->list.length; ++j)
+                        hidden.insert(state.forceStringNoCtx(*i->value->list.elems[j]));
+                }
+            }
+
+            foreach (Sorted::iterator, i, sorted)
+                if (hidden.find(i->first) == hidden.end())
+                    printValue(str << i->first << " = ", *i->second, maxDepth - 1) << "; ";
+                else
+                    str << i->first << " = ...; ";
+
+        } else
+            str << "... ";
+
+        str << "}";
+        if (isDrv) str << ")";
+        break;
+    }
+
+    case tList:
+        str << "[ ";
+        if (maxDepth > 0)
+            for (unsigned int n = 0; n < v.list.length; ++n)
+                printValue(str, *v.list.elems[n], maxDepth - 1) << " ";
+        else
+            str << "... ";
+        str << "]";
+        break;
+
+    case tLambda:
+        str << "«lambda»";
+        break;
+
+    case tPrimOp:
+        str << "«primop»";
+        break;
+
+    case tPrimOpApp:
+        str << "«primop-app»";
+        break;
+
+    default:
+        str << "«unknown»";
+        break;
+    }
+
+    return str;
+}
+
+
 void run(Strings args)
 {
     NixRepl repl;

From e587aec1235d7834d04d6d9f7997bc010ef99925 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 11:14:43 +0200
Subject: [PATCH 0015/2196] printValue: Don't show lists/attribute sets twice

---
 nix-repl.cc | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index d2179d77a8e..a6e5f78ec9c 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -45,7 +45,10 @@ struct NixRepl
     void addVarToScope(const Symbol & name, Value * v);
     Expr * parseString(string s);
     void evalString(string s, Value & v);
+
+    typedef set<Value *> ValuesSeen;
     std::ostream &  printValue(std::ostream & str, Value & v, unsigned int maxDepth);
+    std::ostream &  printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
 };
 
 
@@ -315,8 +318,15 @@ void NixRepl::evalString(string s, Value & v)
 }
 
 
+std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
+{
+    ValuesSeen seen;
+    return printValue(str, v, maxDepth, seen);
+}
+
+
 // FIXME: lot of cut&paste from Nix's eval.cc.
-std::ostream &  NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
+std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
 {
     str.flush();
     checkInterrupt();
@@ -353,6 +363,8 @@ std::ostream &  NixRepl::printValue(std::ostream & str, Value & v, unsigned int
         break;
 
     case tAttrs: {
+        seen.insert(&v);
+
         bool isDrv = state.isDerivation(v);
         if (isDrv) str << "(derivation ";
         str << "{ ";
@@ -379,10 +391,12 @@ std::ostream &  NixRepl::printValue(std::ostream & str, Value & v, unsigned int
             }
 
             foreach (Sorted::iterator, i, sorted)
-                if (hidden.find(i->first) == hidden.end())
-                    printValue(str << i->first << " = ", *i->second, maxDepth - 1) << "; ";
+                if (hidden.find(i->first) != hidden.end())
+                    str << i->first << " = «...»; ";
+                else if (seen.find(i->second) != seen.end())
+                    str << i->first << " = «repeated»; ";
                 else
-                    str << i->first << " = ...; ";
+                    printValue(str << i->first << " = ", *i->second, maxDepth - 1, seen) << "; ";
 
         } else
             str << "... ";
@@ -393,10 +407,16 @@ std::ostream &  NixRepl::printValue(std::ostream & str, Value & v, unsigned int
     }
 
     case tList:
+        seen.insert(&v);
+
         str << "[ ";
         if (maxDepth > 0)
-            for (unsigned int n = 0; n < v.list.length; ++n)
-                printValue(str, *v.list.elems[n], maxDepth - 1) << " ";
+            for (unsigned int n = 0; n < v.list.length; ++n) {
+                if (seen.find(v.list.elems[n]) != seen.end())
+                    str << "«repeated» ";
+                else
+                    printValue(str, *v.list.elems[n], maxDepth - 1, seen) << " ";
+            }
         else
             str << "... ";
         str << "]";

From 8e765b8876ff67879a6bd1a067bad526b14a4045 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 11:34:54 +0200
Subject: [PATCH 0016/2196] printValue: Show assertion errors inline

---
 nix-repl.cc | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index a6e5f78ec9c..f7eb6ab5c0d 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -390,13 +390,20 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                 }
             }
 
-            foreach (Sorted::iterator, i, sorted)
+            foreach (Sorted::iterator, i, sorted) {
+                str << i->first << " = ";
                 if (hidden.find(i->first) != hidden.end())
-                    str << i->first << " = «...»; ";
+                    str << "«...»";
                 else if (seen.find(i->second) != seen.end())
-                    str << i->first << " = «repeated»; ";
+                    str << "«repeated»";
                 else
-                    printValue(str << i->first << " = ", *i->second, maxDepth - 1, seen) << "; ";
+                    try {
+                        printValue(str, *i->second, maxDepth - 1, seen);
+                    } catch (AssertionError & e) {
+                        str << "«error: " << e.msg() << "»";
+                    }
+                str << "; ";
+            }
 
         } else
             str << "... ";
@@ -413,9 +420,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         if (maxDepth > 0)
             for (unsigned int n = 0; n < v.list.length; ++n) {
                 if (seen.find(v.list.elems[n]) != seen.end())
-                    str << "«repeated» ";
+                    str << "«repeated»";
                 else
-                    printValue(str, *v.list.elems[n], maxDepth - 1, seen) << " ";
+                    try {
+                        printValue(str, *v.list.elems[n], maxDepth - 1, seen);
+                    } catch (AssertionError & e) {
+                        str << "«error: " << e.msg() << "»";
+                    }
+                str << " ";
             }
         else
             str << "... ";

From e133e91410d9486e022a1bc0372b822152b6654e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 12:00:33 +0200
Subject: [PATCH 0017/2196] Support tab-completion on attribute sets

Example:

$ nix-repl '<nixos>'

> config.services.xserver.desktop<TAB>

comletes to

> config.services.xserver.desktopManager

You also get suggestions if there are multiple matches:

> config.services.xserver.desktopManager.kde4
config.services.xserver.desktopManager.kde4.enable
config.services.xserver.desktopManager.kde4.phononBackends
---
 nix-repl.cc | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index f7eb6ab5c0d..9c22abbfacf 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -183,11 +183,40 @@ void NixRepl::completePrefix(string prefix)
 {
     completions.clear();
 
-    StringSet::iterator i = varNames.lower_bound(prefix);
-    while (i != varNames.end()) {
-        if (string(*i, 0, prefix.size()) != prefix) break;
-        completions.insert(*i);
-        i++;
+    size_t dot = prefix.rfind('.');
+
+    if (dot == string::npos) {
+        /* This is a variable name; look it up in the current scope. */
+        StringSet::iterator i = varNames.lower_bound(prefix);
+        while (i != varNames.end()) {
+            if (string(*i, 0, prefix.size()) != prefix) break;
+            completions.insert(*i);
+            i++;
+        }
+    } else {
+        try {
+            /* This is an expression that should evaluate to an
+               attribute set.  Evaluate it to get the names of the
+               attributes. */
+            string expr(prefix, 0, dot);
+            string prefix2 = string(prefix, dot + 1);
+
+            Expr * e = parseString(expr);
+            Value v;
+            e->eval(state, *env, v);
+            state.forceAttrs(v);
+
+            foreach (Bindings::iterator, i, *v.attrs) {
+                string name = i->name;
+                if (string(name, 0, prefix2.size()) != prefix2) continue;
+                completions.insert(expr + "." + name);
+            }
+
+        } catch (ParseError & e) {
+            // Quietly ignore parse errors.
+        }catch (EvalError & e) {
+            // Quietly ignore evaluation errors.
+        }
     }
 }
 

From 4b33c2dd4cac95151ca5d99ceb3161fdb460ec0a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 13:22:33 +0200
Subject: [PATCH 0018/2196] Add help (:?)

---
 nix-repl.cc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 9c22abbfacf..72099f46e62 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -248,7 +248,19 @@ void NixRepl::processLine(string line)
 
     string command = string(line, 0, 2);
 
-    if (command == ":a") {
+    if (command == ":?") {
+        cout << "The following commands are available:\n"
+             << "\n"
+             << "  <expr>     Evaluate and print expression\n"
+             << "  :a <expr>  Add attributes from resulting set to scope\n"
+             << "  :b <expr>  Build derivation\n"
+             << "  :l <path>  Load Nix expression and add it to scope\n"
+             << "  :p <expr>  Evaluate and print expression recursively\n"
+             << "  :s <expr>  Build dependencies of derivation, then start nix-shell\n"
+             << "  :t <expr>  Describe result of evaluation\n";
+    }
+
+    else if (command == ":a") {
         Value v;
         evalString(string(line, 2), v);
         addAttrsToScope(v);

From 3c67df928f8d311d7dfd565e8c3a16db9d6a0278 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 13:56:53 +0200
Subject: [PATCH 0019/2196] Add sugar for defining a variable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

‘x = <expr>’ is short for ‘:a { x = <expr>; }’.  Note that the
right-hand side refers to the original scope, so you get:

  nix-repl> x = 1
  nix-repl> x = x + 1
  nix-repl> x
  2

rather than an infinite recursion.
---
 nix-repl.cc | 55 +++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 14 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 72099f46e62..bcd80ffc756 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -42,7 +42,7 @@ struct NixRepl
     void processLine(string line);
     void loadFile(const Path & path);
     void addAttrsToScope(Value & attrs);
-    void addVarToScope(const Symbol & name, Value * v);
+    void addVarToScope(const Symbol & name, Value & v);
     Expr * parseString(string s);
     void evalString(string s, Value & v);
 
@@ -242,6 +242,19 @@ static int runProgram(const string & program, const Strings & args)
 }
 
 
+bool isVarName(const string & s)
+{
+    // FIXME: not quite correct.
+    foreach (string::const_iterator, i, s)
+        if (!((*i >= 'a' && *i <= 'z') ||
+              (*i >= 'A' && *i <= 'Z') ||
+              (*i >= '0' && *i <= '9') ||
+              *i == '_' || *i == '\''))
+            return false;
+    return true;
+}
+
+
 void NixRepl::processLine(string line)
 {
     if (line == "") return;
@@ -251,13 +264,14 @@ void NixRepl::processLine(string line)
     if (command == ":?") {
         cout << "The following commands are available:\n"
              << "\n"
-             << "  <expr>     Evaluate and print expression\n"
-             << "  :a <expr>  Add attributes from resulting set to scope\n"
-             << "  :b <expr>  Build derivation\n"
-             << "  :l <path>  Load Nix expression and add it to scope\n"
-             << "  :p <expr>  Evaluate and print expression recursively\n"
-             << "  :s <expr>  Build dependencies of derivation, then start nix-shell\n"
-             << "  :t <expr>  Describe result of evaluation\n";
+             << "  <expr>        Evaluate and print expression\n"
+             << "  <x> = <expr>  Bind expression to variable\n"
+             << "  :a <expr>     Add attributes from resulting set to scope\n"
+             << "  :b <expr>     Build derivation\n"
+             << "  :l <path>     Load Nix expression and add it to scope\n"
+             << "  :p <expr>     Evaluate and print expression recursively\n"
+             << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
+             << "  :t <expr>     Describe result of evaluation\n";
     }
 
     else if (command == ":a") {
@@ -310,9 +324,22 @@ void NixRepl::processLine(string line)
         throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
 
     else {
-        Value v;
-        evalString(line, v);
-        printValue(std::cout, v, 1) << std::endl;
+        size_t p = line.find('=');
+        string name;
+        if (p != string::npos &&
+            isVarName(name = removeWhitespace(string(line, 0, p))))
+        {
+            Expr * e = parseString(string(line, p + 1));
+            Value & v(*state.allocValue());
+            v.type = tThunk;
+            v.thunk.env = env;
+            v.thunk.expr = e;
+            addVarToScope(state.symbols.create(name), v);
+        } else {
+            Value v;
+            evalString(line, v);
+            printValue(std::cout, v, 1) << std::endl;
+        }
     }
 }
 
@@ -331,15 +358,15 @@ void NixRepl::addAttrsToScope(Value & attrs)
 {
     state.forceAttrs(attrs);
     foreach (Bindings::iterator, i, *attrs.attrs)
-        addVarToScope(i->name, i->value);
+        addVarToScope(i->name, *i->value);
     std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
 }
 
 
-void NixRepl::addVarToScope(const Symbol & name, Value * v)
+void NixRepl::addVarToScope(const Symbol & name, Value & v)
 {
     staticEnv.vars[name] = displ;
-    env->values[displ++] = v;
+    env->values[displ++] = &v;
     varNames.insert((string) name);
 }
 

From a5dffb3d3dcfff3b1d6c166451268ebb1cbde991 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 14:03:28 +0200
Subject: [PATCH 0020/2196] Temporary hack to parameterize nixpkgs

---
 default.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/default.nix b/default.nix
index eae1d56d0a7..97162c6e8b8 100644
--- a/default.nix
+++ b/default.nix
@@ -1,4 +1,6 @@
-with import <nixpkgs> { };
+{ nixpkgs ? <nixpkgs> }:
+
+with import nixpkgs { };
 
 runCommand "nix-repl"
   { buildInputs = [ readline nixUnstable boehmgc ]; }

From 3567bdb514e6ba4e460b76dc1d1b1dd466214286 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 15:02:56 +0200
Subject: [PATCH 0021/2196] Add :quit command

---
 nix-repl.cc | 55 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index bcd80ffc756..a4d74055d5f 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -39,7 +39,7 @@ struct NixRepl
     void mainLoop(const Strings & args);
     void completePrefix(string prefix);
     bool getLine(string & line);
-    void processLine(string line);
+    bool processLine(string line);
     void loadFile(const Path & path);
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol & name, Value & v);
@@ -95,10 +95,13 @@ void NixRepl::mainLoop(const Strings & args)
 
     while (true) {
         string line;
-        if (!getLine(line)) break;
+        if (!getLine(line)) {
+            std::cout << std::endl;
+            break;
+        }
 
         try {
-            processLine(removeWhitespace(line));
+            if (!processLine(removeWhitespace(line))) return;
         } catch (Error & e) {
             printMsg(lvlError, "error: " + e.msg());
         } catch (Interrupted & e) {
@@ -108,7 +111,6 @@ void NixRepl::mainLoop(const Strings & args)
         std::cout << std::endl;
     }
 
-    std::cout << std::endl;
 }
 
 
@@ -255,11 +257,19 @@ bool isVarName(const string & s)
 }
 
 
-void NixRepl::processLine(string line)
+bool NixRepl::processLine(string line)
 {
-    if (line == "") return;
+    if (line == "") return true;
+
+    string command, arg;
 
-    string command = string(line, 0, 2);
+    if (line[0] == ':') {
+        size_t p = line.find(' ');
+        command = string(line, 0, p);
+        if (p != string::npos) arg = removeWhitespace(string(line, p));
+    } else {
+        arg = line;
+    }
 
     if (command == ":?") {
         cout << "The following commands are available:\n"
@@ -270,30 +280,31 @@ void NixRepl::processLine(string line)
              << "  :b <expr>     Build derivation\n"
              << "  :l <path>     Load Nix expression and add it to scope\n"
              << "  :p <expr>     Evaluate and print expression recursively\n"
+             << "  :q            Exit nix-repl\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n";
     }
 
     else if (command == ":a") {
         Value v;
-        evalString(string(line, 2), v);
+        evalString(arg, v);
         addAttrsToScope(v);
     }
 
     else if (command == ":l") {
         state.resetFileCache();
-        loadFile(removeWhitespace(string(line, 2)));
+        loadFile(arg);
     }
 
     else if (command == ":t") {
         Value v;
-        evalString(string(line, 2), v);
+        evalString(arg, v);
         std::cout << showType(v) << std::endl;
     }
 
     else if (command == ":b" || command == ":s") {
         Value v;
-        evalString(string(line, 2), v);
+        evalString(arg, v);
         DrvInfo drvInfo;
         if (!getDerivation(state, v, drvInfo, false))
             throw Error("expression does not evaluation to a derivation, so I can't build it");
@@ -305,23 +316,27 @@ void NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram("nix-store", Strings{"-r", drvPath}) != 0) return;
-            Derivation drv = parseDerivation(readFile(drvPath));
-            std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-            foreach (DerivationOutputs::iterator, i, drv.outputs)
-                std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
+            if (runProgram("nix-store", Strings{"-r", drvPath}) == 0) {
+                Derivation drv = parseDerivation(readFile(drvPath));
+                std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
+                foreach (DerivationOutputs::iterator, i, drv.outputs)
+                    std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
+            }
         } else
             runProgram("nix-shell", Strings{drvPath});
     }
 
     else if (command == ":p") {
         Value v;
-        evalString(string(line, 2), v);
+        evalString(arg, v);
         printValue(std::cout, v, 1000000000) << std::endl;
     }
 
-    else if (string(line, 0, 1) == ":")
-        throw Error(format("unknown command ‘%1%’") % string(line, 0, 2));
+    else if (command == ":q" || command == ":quit")
+        return false;
+
+    else if (command != "")
+        throw Error(format("unknown command ‘%1%’") % command);
 
     else {
         size_t p = line.find('=');
@@ -341,6 +356,8 @@ void NixRepl::processLine(string line)
             printValue(std::cout, v, 1) << std::endl;
         }
     }
+
+    return true;
 }
 
 

From 03ef6b69be4711309fa04fa32681e03fde3d234f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 15:17:08 +0200
Subject: [PATCH 0022/2196] Add system argument

---
 default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/default.nix b/default.nix
index 97162c6e8b8..4c9cbcc0cb4 100644
--- a/default.nix
+++ b/default.nix
@@ -1,6 +1,6 @@
-{ nixpkgs ? <nixpkgs> }:
+{ nixpkgs ? <nixpkgs>, system ? builtins.currentSystem }:
 
-with import nixpkgs { };
+with import nixpkgs { inherit system; };
 
 runCommand "nix-repl"
   { buildInputs = [ readline nixUnstable boehmgc ]; }

From ddd22c37c58aa9d39a632c746a0617f6602d1815 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 16:02:35 +0200
Subject: [PATCH 0023/2196] Build on Mac OS X

---
 default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/default.nix b/default.nix
index 4c9cbcc0cb4..c4a967bf528 100644
--- a/default.nix
+++ b/default.nix
@@ -9,5 +9,5 @@ runCommand "nix-repl"
     g++ -O3 -Wall -std=c++0x \
       -o $out/bin/nix-repl ${./nix-repl.cc} \
       -I${nixUnstable}/include/nix -L${nixUnstable}/lib/nix \
-      -lexpr -lmain -lreadline -lgc
+      -lformat -lutil -lstore -lexpr -lmain -lreadline -lgc
   ''

From adde4f0c8d720b9c9c43f6d0a2e789d7c68798bd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 16:02:46 +0200
Subject: [PATCH 0024/2196] Add :reload command

---
 nix-repl.cc | 41 +++++++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index a4d74055d5f..c0ba3a58831 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -27,6 +27,8 @@ struct NixRepl
     string curDir;
     EvalState state;
 
+    Strings loadedFiles;
+
     StaticEnv staticEnv;
     Env * env;
     int displ;
@@ -41,6 +43,7 @@ struct NixRepl
     bool getLine(string & line);
     bool processLine(string line);
     void loadFile(const Path & path);
+    void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol & name, Value & v);
     Expr * parseString(string s);
@@ -84,9 +87,11 @@ void NixRepl::mainLoop(const Strings & args)
 {
     std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
-    foreach (Strings::const_iterator, i, args) {
-        std::cout << format("Loading ‘%1%’...") % *i << std::endl;
-        loadFile(*i);
+    foreach (Strings::const_iterator, i, args)
+        loadedFiles.push_back(*i);
+
+    if (!loadedFiles.empty()) {
+        reloadFiles();
         std::cout << std::endl;
     }
 
@@ -110,7 +115,6 @@ void NixRepl::mainLoop(const Strings & args)
 
         std::cout << std::endl;
     }
-
 }
 
 
@@ -271,7 +275,7 @@ bool NixRepl::processLine(string line)
         arg = line;
     }
 
-    if (command == ":?") {
+    if (command == ":?" || command == ":help") {
         cout << "The following commands are available:\n"
              << "\n"
              << "  <expr>        Evaluate and print expression\n"
@@ -281,21 +285,27 @@ bool NixRepl::processLine(string line)
              << "  :l <path>     Load Nix expression and add it to scope\n"
              << "  :p <expr>     Evaluate and print expression recursively\n"
              << "  :q            Exit nix-repl\n"
+             << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n";
     }
 
-    else if (command == ":a") {
+    else if (command == ":a" || command == ":add") {
         Value v;
         evalString(arg, v);
         addAttrsToScope(v);
     }
 
-    else if (command == ":l") {
+    else if (command == ":l" || command == ":load") {
         state.resetFileCache();
         loadFile(arg);
     }
 
+    else if (command == ":r" || command == ":reload") {
+        state.resetFileCache();
+        reloadFiles();
+    }
+
     else if (command == ":t") {
         Value v;
         evalString(arg, v);
@@ -326,7 +336,7 @@ bool NixRepl::processLine(string line)
             runProgram("nix-shell", Strings{drvPath});
     }
 
-    else if (command == ":p") {
+    else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
         printValue(std::cout, v, 1000000000) << std::endl;
@@ -363,6 +373,8 @@ bool NixRepl::processLine(string line)
 
 void NixRepl::loadFile(const Path & path)
 {
+    loadedFiles.remove(path);
+    loadedFiles.push_back(path);
     Value v, v2;
     state.evalFile(lookupFileArg(state, path), v);
     Bindings bindings;
@@ -371,6 +383,19 @@ void NixRepl::loadFile(const Path & path)
 }
 
 
+void NixRepl::reloadFiles()
+{
+    Strings old = loadedFiles;
+    loadedFiles.clear();
+
+    foreach (Strings::iterator, i, old) {
+        if (i != old.begin()) std::cout << std::endl;
+        std::cout << format("Loading ‘%1%’...") % *i << std::endl;
+        loadFile(*i);
+    }
+}
+
+
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state.forceAttrs(attrs);

From 498f8b048513bf3eee810c80f6795ab1ef32793f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 16:17:45 +0200
Subject: [PATCH 0025/2196] Add license

---
 COPYING | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 674 insertions(+)
 create mode 100644 COPYING

diff --git a/COPYING b/COPYING
new file mode 100644
index 00000000000..94a9ed024d3
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

From e91160021f992169228bc59cfa509cfb66335e8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 17:06:14 +0200
Subject: [PATCH 0026/2196] On reload, wipe the environment

---
 nix-repl.cc | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index c0ba3a58831..012fdfe1e6c 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -29,6 +29,7 @@ struct NixRepl
 
     Strings loadedFiles;
 
+    const static int envSize = 32768;
     StaticEnv staticEnv;
     Env * env;
     int displ;
@@ -43,6 +44,7 @@ struct NixRepl
     bool getLine(string & line);
     bool processLine(string line);
     void loadFile(const Path & path);
+    void initEnv();
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol & name, Value & v);
@@ -75,10 +77,6 @@ NixRepl::NixRepl()
 {
     curDir = absPath(".");
 
-    env = &state.allocEnv(32768);
-    env->up = &state.baseEnv;
-    displ = 0;
-
     store = openStore();
 }
 
@@ -90,10 +88,8 @@ void NixRepl::mainLoop(const Strings & args)
     foreach (Strings::const_iterator, i, args)
         loadedFiles.push_back(*i);
 
-    if (!loadedFiles.empty()) {
-        reloadFiles();
-        std::cout << std::endl;
-    }
+    reloadFiles();
+    if (!loadedFiles.empty()) std::cout << std::endl;
 
     using_history();
     read_history(0);
@@ -383,8 +379,20 @@ void NixRepl::loadFile(const Path & path)
 }
 
 
+void NixRepl::initEnv()
+{
+    env = &state.allocEnv(envSize);
+    env->up = &state.baseEnv;
+    displ = 0;
+    varNames.clear();
+    staticEnv.vars.clear();
+}
+
+
 void NixRepl::reloadFiles()
 {
+    initEnv();
+
     Strings old = loadedFiles;
     loadedFiles.clear();
 
@@ -407,6 +415,8 @@ void NixRepl::addAttrsToScope(Value & attrs)
 
 void NixRepl::addVarToScope(const Symbol & name, Value & v)
 {
+    if (displ >= envSize)
+        throw Error("environment full; cannot add more variables");
     staticEnv.vars[name] = displ;
     env->values[displ++] = &v;
     varNames.insert((string) name);

From dc670a173ab22d344c5d77d502798f0296f572cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 17:22:42 +0200
Subject: [PATCH 0027/2196] Make tab-completion work on builtins

---
 nix-repl.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 012fdfe1e6c..98d27b435ef 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -384,8 +384,11 @@ void NixRepl::initEnv()
     env = &state.allocEnv(envSize);
     env->up = &state.baseEnv;
     displ = 0;
-    varNames.clear();
     staticEnv.vars.clear();
+
+    varNames.clear();
+    foreach (StaticEnv::Vars::iterator, i, state.staticBaseEnv.vars)
+        varNames.insert(i->first);
 }
 
 

From ae50a5e7bee19cb6de331e847936ea8afa0ba8b6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 18:09:46 +0200
Subject: [PATCH 0028/2196] Add a README

---
 README.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..af2d27a2067
--- /dev/null
+++ b/README.md
@@ -0,0 +1,104 @@
+nix-repl
+========
+
+`nix-repl` is a simple read–eval–print loop (REPL) for the Nix package
+manager.
+
+Installation
+------------
+
+Assuming you have Nix installed, just do
+
+  $ git clone https://github.com/edolstra/nix-repl.git
+  $ cd nix-repl
+  $ nix-env -f . -i nix-repl
+
+Example
+-------
+
+Here is a typical `nix-repl` session:
+
+  $ nix-repl
+  Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
+
+  nix-repl> 3 * 4
+  12
+
+  nix-repl> :l <nixpkgs>
+  Added 3337 variables.
+
+  nix-repl> lib.range 1 5
+  [ 1 2 3 4 5 ]
+
+  nix-repl> :a lib
+  Added 299 variables.
+
+  nix-repl> range 1 5
+  [ 1 2 3 4 5 ]
+
+  nix-repl> xs = range 1 5
+
+  nix-repl> map (x: x * x) xs
+  [ 1 4 9 16 25 ]
+
+  nix-repl> :l <nixos>
+  Added 7 variables.
+
+  nix-repl> config.services.dhcpd
+  { configFile = null; enable = false; extraConfig = ""; interfaces = [ ... ]; machines = [ ... ]; }
+
+  nix-repl> :p config.services.dhcpd
+  { configFile = null; enable = false; extraConfig = ""; interfaces = [ "eth0" ]; machines = [ ]; }
+
+  nix-repl> config.fileSystems
+  { "/" = { ... }; "/boot" = { ... }; }
+
+  nix-repl> mapAttrsToList (n: v: v.device) config.fileSystems
+  [ "/dev/disk/by-label/nixos" "/dev/disk/by-label/boot" ]
+
+  nix-repl> :b libjson
+  these derivations will be built:
+    /nix/store/h910xqb36pysxcxkayb1zkr1zcvvk1zy-libjson_7.6.1.zip.drv
+    /nix/store/iv0rdx08di0fg704zyxklkvdz6i96lm8-libjson-7.6.1.drv
+  ...
+  this derivation produced the following outputs:
+    out -> /nix/store/ys6bvgfia81rjwqxjlgkwnx6jhsml8h9-libjson-7.6.1
+
+  nix-repl> :t makeFontsConf
+  a function
+
+  nix-repl> :b makeFontsConf { fontDirectories = [ "${freefont_ttf}/share/fonts/truetype" ]; }
+  ...
+  this derivation produced the following outputs:
+    out -> /nix/store/jkw848xj0gkbhmvxi0hwpnhzn2716v3c-fonts.conf
+
+  nix-repl> :s pan
+  # Builds dependencies of the ‘pan’ derivation, then starts a shell
+  # in which the environment variables of the derivation are set
+
+  [nix-shell:/tmp/nix-repl]$ echo $src
+  /nix/store/0ibx15r02nnkwiclmfbpzrzjm2y204fh-pan-0.139.tar.bz2
+
+  [nix-shell:/tmp/nix-repl]$ exit
+
+  nix-repl>
+
+Tab completion works on variables in scope and on attribute sets.  For
+example:
+
+  $ nix-repl  '<nixpkgs>' '<nixos>'
+  Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
+
+  nix-repl> thunder<TAB> => thunderbird
+
+  nix-repl> <TAB>
+  Display all 3634 possibilities? (y or n)
+
+  nix-repl> lib<TAB>
+  Display all 291 possibilities? (y or n)
+
+  nix-repl> xorg.libX<TAB>
+  xorg.libXdamage  xorg.libXdmcp
+
+  nix-repl> config.networking.use<TAB>
+  config.networking.useDHCP   config.networking.usePredictableInterfaceNames

From 853d2e0aa42b1a7e5de6111c86dbd4a16e0fe411 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 18:10:57 +0200
Subject: [PATCH 0029/2196] Fix markdown

---
 README.md | 112 +++++++++++++++++++++++++++---------------------------
 1 file changed, 56 insertions(+), 56 deletions(-)

diff --git a/README.md b/README.md
index af2d27a2067..01967e80034 100644
--- a/README.md
+++ b/README.md
@@ -18,87 +18,87 @@ Example
 
 Here is a typical `nix-repl` session:
 
-  $ nix-repl
-  Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
+    $ nix-repl
+    Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
 
-  nix-repl> 3 * 4
-  12
+    nix-repl> 3 * 4
+    12
 
-  nix-repl> :l <nixpkgs>
-  Added 3337 variables.
+    nix-repl> :l <nixpkgs>
+    Added 3337 variables.
 
-  nix-repl> lib.range 1 5
-  [ 1 2 3 4 5 ]
+    nix-repl> lib.range 1 5
+    [ 1 2 3 4 5 ]
 
-  nix-repl> :a lib
-  Added 299 variables.
+    nix-repl> :a lib
+    Added 299 variables.
 
-  nix-repl> range 1 5
-  [ 1 2 3 4 5 ]
+    nix-repl> range 1 5
+    [ 1 2 3 4 5 ]
 
-  nix-repl> xs = range 1 5
+    nix-repl> xs = range 1 5
 
-  nix-repl> map (x: x * x) xs
-  [ 1 4 9 16 25 ]
+    nix-repl> map (x: x * x) xs
+    [ 1 4 9 16 25 ]
 
-  nix-repl> :l <nixos>
-  Added 7 variables.
+    nix-repl> :l <nixos>
+    Added 7 variables.
 
-  nix-repl> config.services.dhcpd
-  { configFile = null; enable = false; extraConfig = ""; interfaces = [ ... ]; machines = [ ... ]; }
+    nix-repl> config.services.dhcpd
+    { configFile = null; enable = false; extraConfig = ""; interfaces = [ ... ]; machines = [ ... ]; }
 
-  nix-repl> :p config.services.dhcpd
-  { configFile = null; enable = false; extraConfig = ""; interfaces = [ "eth0" ]; machines = [ ]; }
+    nix-repl> :p config.services.dhcpd
+    { configFile = null; enable = false; extraConfig = ""; interfaces = [ "eth0" ]; machines = [ ]; }
 
-  nix-repl> config.fileSystems
-  { "/" = { ... }; "/boot" = { ... }; }
+    nix-repl> config.fileSystems
+    { "/" = { ... }; "/boot" = { ... }; }
 
-  nix-repl> mapAttrsToList (n: v: v.device) config.fileSystems
-  [ "/dev/disk/by-label/nixos" "/dev/disk/by-label/boot" ]
+    nix-repl> mapAttrsToList (n: v: v.device) config.fileSystems
+    [ "/dev/disk/by-label/nixos" "/dev/disk/by-label/boot" ]
 
-  nix-repl> :b libjson
-  these derivations will be built:
-    /nix/store/h910xqb36pysxcxkayb1zkr1zcvvk1zy-libjson_7.6.1.zip.drv
-    /nix/store/iv0rdx08di0fg704zyxklkvdz6i96lm8-libjson-7.6.1.drv
-  ...
-  this derivation produced the following outputs:
-    out -> /nix/store/ys6bvgfia81rjwqxjlgkwnx6jhsml8h9-libjson-7.6.1
+    nix-repl> :b libjson
+    these derivations will be built:
+      /nix/store/h910xqb36pysxcxkayb1zkr1zcvvk1zy-libjson_7.6.1.zip.drv
+      /nix/store/iv0rdx08di0fg704zyxklkvdz6i96lm8-libjson-7.6.1.drv
+    ...
+    this derivation produced the following outputs:
+      out -> /nix/store/ys6bvgfia81rjwqxjlgkwnx6jhsml8h9-libjson-7.6.1
 
-  nix-repl> :t makeFontsConf
-  a function
+    nix-repl> :t makeFontsConf
+    a function
 
-  nix-repl> :b makeFontsConf { fontDirectories = [ "${freefont_ttf}/share/fonts/truetype" ]; }
-  ...
-  this derivation produced the following outputs:
-    out -> /nix/store/jkw848xj0gkbhmvxi0hwpnhzn2716v3c-fonts.conf
+    nix-repl> :b makeFontsConf { fontDirectories = [ "${freefont_ttf}/share/fonts/truetype" ]; }
+    ...
+    this derivation produced the following outputs:
+      out -> /nix/store/jkw848xj0gkbhmvxi0hwpnhzn2716v3c-fonts.conf
 
-  nix-repl> :s pan
-  # Builds dependencies of the ‘pan’ derivation, then starts a shell
-  # in which the environment variables of the derivation are set
+    nix-repl> :s pan
+    # Builds dependencies of the ‘pan’ derivation, then starts a shell
+    # in which the environment variables of the derivation are set
 
-  [nix-shell:/tmp/nix-repl]$ echo $src
-  /nix/store/0ibx15r02nnkwiclmfbpzrzjm2y204fh-pan-0.139.tar.bz2
+    [nix-shell:/tmp/nix-repl]$ echo $src
+    /nix/store/0ibx15r02nnkwiclmfbpzrzjm2y204fh-pan-0.139.tar.bz2
 
-  [nix-shell:/tmp/nix-repl]$ exit
+    [nix-shell:/tmp/nix-repl]$ exit
 
-  nix-repl>
+    nix-repl>
 
 Tab completion works on variables in scope and on attribute sets.  For
 example:
 
-  $ nix-repl  '<nixpkgs>' '<nixos>'
-  Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
+    $ nix-repl  '<nixpkgs>' '<nixos>'
+    Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
 
-  nix-repl> thunder<TAB> => thunderbird
+    nix-repl> thunder<TAB> => thunderbird
 
-  nix-repl> <TAB>
-  Display all 3634 possibilities? (y or n)
+    nix-repl> <TAB>
+    Display all 3634 possibilities? (y or n)
 
-  nix-repl> lib<TAB>
-  Display all 291 possibilities? (y or n)
+    nix-repl> lib<TAB>
+    Display all 291 possibilities? (y or n)
 
-  nix-repl> xorg.libX<TAB>
-  xorg.libXdamage  xorg.libXdmcp
+    nix-repl> xorg.libX<TAB>
+    xorg.libXdamage  xorg.libXdmcp
 
-  nix-repl> config.networking.use<TAB>
-  config.networking.useDHCP   config.networking.usePredictableInterfaceNames
+    nix-repl> config.networking.use<TAB>
+    config.networking.useDHCP   config.networking.usePredictableInterfaceNames

From 81d658fe4afda234028cd4551e12491db4303957 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 9 Sep 2013 18:11:47 +0200
Subject: [PATCH 0030/2196] Fix readme

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 01967e80034..c5964fd5ea1 100644
--- a/README.md
+++ b/README.md
@@ -9,9 +9,9 @@ Installation
 
 Assuming you have Nix installed, just do
 
-  $ git clone https://github.com/edolstra/nix-repl.git
-  $ cd nix-repl
-  $ nix-env -f . -i nix-repl
+    $ git clone https://github.com/edolstra/nix-repl.git
+    $ cd nix-repl
+    $ nix-env -f . -i nix-repl
 
 Example
 -------

From 3beb6f6e763f56d791db0a99baf285e0ba745bc8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 28 Jan 2014 10:40:02 +0100
Subject: [PATCH 0031/2196] Show derivations more concisely

---
 nix-repl.cc | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 98d27b435ef..78be0f98c42 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -489,10 +489,18 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         seen.insert(&v);
 
         bool isDrv = state.isDerivation(v);
-        if (isDrv) str << "(derivation ";
-        str << "{ ";
 
-        if (maxDepth > 0) {
+        if (isDrv) {
+            str << "«derivation ";
+            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+            PathSet context;
+            Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->value, context) : "???";
+            str << drvPath << "»";
+        }
+
+        else if (maxDepth > 0) {
+            str << "{ ";
+
             typedef std::map<string, Value *> Sorted;
             Sorted sorted;
             foreach (Bindings::iterator, i, *v.attrs)
@@ -528,11 +536,10 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                 str << "; ";
             }
 
+            str << "}";
         } else
-            str << "... ";
+            str << "{ ... }";
 
-        str << "}";
-        if (isDrv) str << ")";
         break;
     }
 

From 6a4a8208be10462b1051c689b26577dc36495632 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 28 Jan 2014 10:42:05 +0100
Subject: [PATCH 0032/2196] Fix building against current Nix

---
 nix-repl.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 78be0f98c42..1c7b2f5142e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -311,10 +311,10 @@ bool NixRepl::processLine(string line)
     else if (command == ":b" || command == ":s") {
         Value v;
         evalString(arg, v);
-        DrvInfo drvInfo;
+        DrvInfo drvInfo(state);
         if (!getDerivation(state, v, drvInfo, false))
             throw Error("expression does not evaluation to a derivation, so I can't build it");
-        Path drvPath = drvInfo.queryDrvPath(state);
+        Path drvPath = drvInfo.queryDrvPath();
         if (drvPath == "" || !store->isValidPath(drvPath))
             throw Error("expression did not evaluate to a valid derivation");
 

From 22a47ab03c8a1bf81df3aa0d29a4d6a29aacc36f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 11 Apr 2014 12:50:46 +0200
Subject: [PATCH 0033/2196] Fix building against Nix 1.7

---
 default.nix | 7 ++++---
 nix-repl.cc | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/default.nix b/default.nix
index c4a967bf528..435c16c0915 100644
--- a/default.nix
+++ b/default.nix
@@ -3,11 +3,12 @@
 with import nixpkgs { inherit system; };
 
 runCommand "nix-repl"
-  { buildInputs = [ readline nixUnstable boehmgc ]; }
+  { buildInputs = [ readline nix boehmgc ]; }
   ''
     mkdir -p $out/bin
     g++ -O3 -Wall -std=c++0x \
       -o $out/bin/nix-repl ${./nix-repl.cc} \
-      -I${nixUnstable}/include/nix -L${nixUnstable}/lib/nix \
-      -lformat -lutil -lstore -lexpr -lmain -lreadline -lgc
+      -I${nix}/include/nix \
+      -lnixformat -lnixutil -lnixstore -lnixexpr -lnixmain -lreadline -lgc \
+      -DNIX_VERSION=${(builtins.parseDrvName nix.name).version}
   ''
diff --git a/nix-repl.cc b/nix-repl.cc
index 1c7b2f5142e..ef9be417acd 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -323,7 +323,7 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             if (runProgram("nix-store", Strings{"-r", drvPath}) == 0) {
-                Derivation drv = parseDerivation(readFile(drvPath));
+                Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 foreach (DerivationOutputs::iterator, i, drv.outputs)
                     std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
@@ -494,7 +494,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
             str << "«derivation ";
             Bindings::iterator i = v.attrs->find(state.sDrvPath);
             PathSet context;
-            Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->value, context) : "???";
+            Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???";
             str << drvPath << "»";
         }
 

From 1734e8a1491ef831c83c2620b6b0f4a590b67c1f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 11 Apr 2014 12:51:15 +0200
Subject: [PATCH 0034/2196] Fix crash in tab completion

Fixes #1. Patch by Maxdamantus.
---
 nix-repl.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index ef9be417acd..8049008d14e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -216,8 +216,10 @@ void NixRepl::completePrefix(string prefix)
 
         } catch (ParseError & e) {
             // Quietly ignore parse errors.
-        }catch (EvalError & e) {
+        } catch (EvalError & e) {
             // Quietly ignore evaluation errors.
+        } catch (UndefinedVarError & e) {
+            // Quietly ignore undefined variable errors.
         }
     }
 }

From 66b2d18243d055bcbc6c3f7708960575d02db09c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 16 Jun 2014 10:05:09 -0400
Subject: [PATCH 0035/2196] Don't parse 'var == expr' as an assignment

---
 nix-repl.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8049008d14e..ea188a1c75e 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -350,6 +350,8 @@ bool NixRepl::processLine(string line)
         size_t p = line.find('=');
         string name;
         if (p != string::npos &&
+            p < line.size() &&
+            line[p + 1] != '=' &&
             isVarName(name = removeWhitespace(string(line, 0, p))))
         {
             Expr * e = parseString(string(line, p + 1));

From 2cf0e67761121a4ddceb69a932bc2e3c0cd6cb6c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 24 Jul 2014 17:46:58 +0200
Subject: [PATCH 0036/2196] Handle non-numeric version strings

Fixes #2.
---
 default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/default.nix b/default.nix
index 435c16c0915..e7cdc23580e 100644
--- a/default.nix
+++ b/default.nix
@@ -10,5 +10,5 @@ runCommand "nix-repl"
       -o $out/bin/nix-repl ${./nix-repl.cc} \
       -I${nix}/include/nix \
       -lnixformat -lnixutil -lnixstore -lnixexpr -lnixmain -lreadline -lgc \
-      -DNIX_VERSION=${(builtins.parseDrvName nix.name).version}
+      -DNIX_VERSION=\"${(builtins.parseDrvName nix.name).version}\"
   ''

From 02b66e97ba08ed68a73654556734aadfc9f41c89 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 24 Jul 2014 17:53:32 +0200
Subject: [PATCH 0037/2196] Fix building against current Nix master

---
 nix-repl.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index ea188a1c75e..fadaf079a8b 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -73,7 +73,8 @@ string removeWhitespace(string s)
 
 
 NixRepl::NixRepl()
-    : staticEnv(false, &state.staticBaseEnv)
+    : state(Strings()) 
+    , staticEnv(false, &state.staticBaseEnv)
 {
     curDir = absPath(".");
 

From 89f9c0d41b29d18e2804677da856e84c86c83b45 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 26 Aug 2014 20:03:12 +0200
Subject: [PATCH 0038/2196] Fix building against current Nix master

---
 default.nix |  2 ++
 nix-repl.cc | 30 +++++++++++++++++++++++-------
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/default.nix b/default.nix
index e7cdc23580e..8690325e528 100644
--- a/default.nix
+++ b/default.nix
@@ -2,6 +2,8 @@
 
 with import nixpkgs { inherit system; };
 
+let nix = nixUnstable; in
+
 runCommand "nix-repl"
   { buildInputs = [ readline nix boehmgc ]; }
   ''
diff --git a/nix-repl.cc b/nix-repl.cc
index fadaf079a8b..bff54aa14f3 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -39,7 +39,7 @@ struct NixRepl
     StringSet::iterator curCompletion;
 
     NixRepl();
-    void mainLoop(const Strings & args);
+    void mainLoop(const Strings & files);
     void completePrefix(string prefix);
     bool getLine(string & line);
     bool processLine(string line);
@@ -73,7 +73,7 @@ string removeWhitespace(string s)
 
 
 NixRepl::NixRepl()
-    : state(Strings()) 
+    : state(Strings())
     , staticEnv(false, &state.staticBaseEnv)
 {
     curDir = absPath(".");
@@ -82,11 +82,11 @@ NixRepl::NixRepl()
 }
 
 
-void NixRepl::mainLoop(const Strings & args)
+void NixRepl::mainLoop(const Strings & files)
 {
     std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
-    foreach (Strings::const_iterator, i, args)
+    foreach (Strings::const_iterator, i, files)
         loadedFiles.push_back(*i);
 
     reloadFiles();
@@ -590,8 +590,24 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 }
 
 
-void run(Strings args)
+int main(int argc, char * * argv)
 {
-    NixRepl repl;
-    repl.mainLoop(args);
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
+        Strings files;
+
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--version")
+                printVersion("nix-repl");
+            else if (*arg != "" && arg->at(0) == '-')
+                return false;
+            else
+                files.push_back(*arg);
+            return true;
+        });
+
+        NixRepl repl;
+        repl.mainLoop(files);
+    });
 }

From 71d61508f203e8d926a0365332ff218c1314f734 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 26 Aug 2014 20:05:08 +0200
Subject: [PATCH 0039/2196] Support -I flag

---
 nix-repl.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index bff54aa14f3..51d297d1d17 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -38,7 +38,7 @@ struct NixRepl
     StringSet completions;
     StringSet::iterator curCompletion;
 
-    NixRepl();
+    NixRepl(const Strings & searchPath);
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
     bool getLine(string & line);
@@ -72,8 +72,8 @@ string removeWhitespace(string s)
 }
 
 
-NixRepl::NixRepl()
-    : state(Strings())
+NixRepl::NixRepl(const Strings & searchPath)
+    : state(searchPath)
     , staticEnv(false, &state.staticBaseEnv)
 {
     curDir = absPath(".");
@@ -595,11 +595,13 @@ int main(int argc, char * * argv)
     return handleExceptions(argv[0], [&]() {
         initNix();
 
-        Strings files;
+        Strings files, searchPath;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--version")
                 printVersion("nix-repl");
+            else if (parseSearchPathArg(arg, end, searchPath))
+                ;
             else if (*arg != "" && arg->at(0) == '-')
                 return false;
             else
@@ -607,7 +609,7 @@ int main(int argc, char * * argv)
             return true;
         });
 
-        NixRepl repl;
+        NixRepl repl(searchPath);
         repl.mainLoop(files);
     });
 }

From f92408136ed08804bab14b3e2a2def9b8effd7eb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 1 Dec 2014 10:07:10 +0100
Subject: [PATCH 0040/2196] Fix building against current Nix master

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 51d297d1d17..343d3f9f3bf 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -378,7 +378,7 @@ void NixRepl::loadFile(const Path & path)
     loadedFiles.push_back(path);
     Value v, v2;
     state.evalFile(lookupFileArg(state, path), v);
-    Bindings bindings;
+    Bindings & bindings(*state.allocBindings(0));
     state.autoCallFunction(bindings, v, v2);
     addAttrsToScope(v2);
 }

From 45c6405a30bd1b2cb8ad6a94b23be8b10cf52069 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Tue, 2 Jun 2015 13:23:53 +0200
Subject: [PATCH 0041/2196] Fix building against latest Nix

Fixes #8.
Fixes #9.
---
 nix-repl.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix-repl.cc b/nix-repl.cc
index 343d3f9f3bf..8cfbfeff9f4 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -594,6 +594,7 @@ int main(int argc, char * * argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+        initGC();
 
         Strings files, searchPath;
 

From 57aeef0b6a3d3c9506e35f57f5b6db33019967e5 Mon Sep 17 00:00:00 2001
From: Susan Potter <me@susanpotter.net>
Date: Mon, 6 Jul 2015 08:26:17 -0500
Subject: [PATCH 0042/2196] Fix nix-repl does not support '--help'

According to popular practice and convention `nix-repl` now supports `--help`
like a good POSIX citizen[1].

[1] https://www.gnu.org/prep/standards/html_node/Command_002dLine-Interfaces.html
---
 nix-repl.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8cfbfeff9f4..43e8c4a6cc6 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -59,7 +59,8 @@ struct NixRepl
 
 void printHelp()
 {
-    std::cout << "Usage: nix-repl\n";
+    std::cout << "Usage: nix-repl [--help|--version]";
+    std::cout << std::endl;
 }
 
 
@@ -601,6 +602,11 @@ int main(int argc, char * * argv)
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--version")
                 printVersion("nix-repl");
+            else if (*arg == "--help") {
+                printHelp();
+                // exit with 0 since user asked for help
+                _exit(0);
+            }
             else if (parseSearchPathArg(arg, end, searchPath))
                 ;
             else if (*arg != "" && arg->at(0) == '-')

From 8a2f5f0607540ffe56b56d52db544373e1efb980 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 7 Sep 2015 13:05:58 +0200
Subject: [PATCH 0043/2196] Fix building against Nix 1.10

Fixes #12.
---
 nix-repl.cc | 68 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 36 insertions(+), 32 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 43e8c4a6cc6..474ad4fc9af 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -87,8 +87,8 @@ void NixRepl::mainLoop(const Strings & files)
 {
     std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
-    foreach (Strings::const_iterator, i, files)
-        loadedFiles.push_back(*i);
+    for (auto & i : files)
+        loadedFiles.push_back(i);
 
     reloadFiles();
     if (!loadedFiles.empty()) std::cout << std::endl;
@@ -210,8 +210,8 @@ void NixRepl::completePrefix(string prefix)
             e->eval(state, *env, v);
             state.forceAttrs(v);
 
-            foreach (Bindings::iterator, i, *v.attrs) {
-                string name = i->name;
+            for (auto & i : *v.attrs) {
+                string name = i.name;
                 if (string(name, 0, prefix2.size()) != prefix2) continue;
                 completions.insert(expr + "." + name);
             }
@@ -251,11 +251,11 @@ static int runProgram(const string & program, const Strings & args)
 bool isVarName(const string & s)
 {
     // FIXME: not quite correct.
-    foreach (string::const_iterator, i, s)
-        if (!((*i >= 'a' && *i <= 'z') ||
-              (*i >= 'A' && *i <= 'Z') ||
-              (*i >= '0' && *i <= '9') ||
-              *i == '_' || *i == '\''))
+    for (auto & i : s)
+        if (!((i >= 'a' && i <= 'z') ||
+              (i >= 'A' && i <= 'Z') ||
+              (i >= '0' && i <= '9') ||
+              i == '_' || i == '\''))
             return false;
     return true;
 }
@@ -329,8 +329,8 @@ bool NixRepl::processLine(string line)
             if (runProgram("nix-store", Strings{"-r", drvPath}) == 0) {
                 Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                foreach (DerivationOutputs::iterator, i, drv.outputs)
-                    std::cout << format("  %1% -> %2%") % i->first % i->second.path << std::endl;
+                for (auto & i : drv.outputs)
+                    std::cout << format("  %1% -> %2%") % i.first % i.second.path << std::endl;
             }
         } else
             runProgram("nix-shell", Strings{drvPath});
@@ -393,8 +393,8 @@ void NixRepl::initEnv()
     staticEnv.vars.clear();
 
     varNames.clear();
-    foreach (StaticEnv::Vars::iterator, i, state.staticBaseEnv.vars)
-        varNames.insert(i->first);
+    for (auto & i : state.staticBaseEnv.vars)
+        varNames.insert(i.first);
 }
 
 
@@ -405,10 +405,12 @@ void NixRepl::reloadFiles()
     Strings old = loadedFiles;
     loadedFiles.clear();
 
-    foreach (Strings::iterator, i, old) {
-        if (i != old.begin()) std::cout << std::endl;
-        std::cout << format("Loading ‘%1%’...") % *i << std::endl;
-        loadFile(*i);
+    bool first = true;
+    for (auto & i : old) {
+        if (!first) std::cout << std::endl;
+        first = false;
+        std::cout << format("Loading ‘%1%’...") % i << std::endl;
+        loadFile(i);
     }
 }
 
@@ -416,8 +418,8 @@ void NixRepl::reloadFiles()
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state.forceAttrs(attrs);
-    foreach (Bindings::iterator, i, *attrs.attrs)
-        addVarToScope(i->name, *i->value);
+    for (auto & i : *attrs.attrs)
+        addVarToScope(i.name, *i.value);
     std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
 }
 
@@ -509,8 +511,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 
             typedef std::map<string, Value *> Sorted;
             Sorted sorted;
-            foreach (Bindings::iterator, i, *v.attrs)
-                sorted[i->name] = i->value;
+            for (auto & i : *v.attrs)
+                sorted[i.name] = i.value;
 
             /* If this is a derivation, then don't show the
                self-references ("all", "out", etc.). */
@@ -522,20 +524,20 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                     hidden.insert("out");
                 else {
                     state.forceList(*i->value);
-                    for (unsigned int j = 0; j < i->value->list.length; ++j)
-                        hidden.insert(state.forceStringNoCtx(*i->value->list.elems[j]));
+                    for (unsigned int j = 0; j < i->value->listSize(); ++j)
+                        hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j]));
                 }
             }
 
-            foreach (Sorted::iterator, i, sorted) {
-                str << i->first << " = ";
-                if (hidden.find(i->first) != hidden.end())
+            for (auto & i : sorted) {
+                str << i.first << " = ";
+                if (hidden.find(i.first) != hidden.end())
                     str << "«...»";
-                else if (seen.find(i->second) != seen.end())
+                else if (seen.find(i.second) != seen.end())
                     str << "«repeated»";
                 else
                     try {
-                        printValue(str, *i->second, maxDepth - 1, seen);
+                        printValue(str, *i.second, maxDepth - 1, seen);
                     } catch (AssertionError & e) {
                         str << "«error: " << e.msg() << "»";
                     }
@@ -549,17 +551,19 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
     }
 
-    case tList:
+    case tList1:
+    case tList2:
+    case tListN:
         seen.insert(&v);
 
         str << "[ ";
         if (maxDepth > 0)
-            for (unsigned int n = 0; n < v.list.length; ++n) {
-                if (seen.find(v.list.elems[n]) != seen.end())
+            for (unsigned int n = 0; n < v.listSize(); ++n) {
+                if (seen.find(v.listElems()[n]) != seen.end())
                     str << "«repeated»";
                 else
                     try {
-                        printValue(str, *v.list.elems[n], maxDepth - 1, seen);
+                        printValue(str, *v.listElems()[n], maxDepth - 1, seen);
                     } catch (AssertionError & e) {
                         str << "«error: " << e.msg() << "»";
                     }

From f7980b471273d695aa0b28f2f73e6ee443dfe9eb Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Sun, 14 Feb 2016 01:16:30 -0600
Subject: [PATCH 0044/2196] Parse `foo-bar = expr` as an assignment.

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 474ad4fc9af..25f84a3082a 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -255,7 +255,7 @@ bool isVarName(const string & s)
         if (!((i >= 'a' && i <= 'z') ||
               (i >= 'A' && i <= 'Z') ||
               (i >= '0' && i <= '9') ||
-              i == '_' || i == '\''))
+              i == '_' || i == '-' || i == '\''))
             return false;
     return true;
 }

From 2111098a3a26d11edf4452f021245b55287c45b8 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Sun, 14 Feb 2016 01:29:48 -0600
Subject: [PATCH 0045/2196] Don't consider strings starting with - or ' as
 variable names.

---
 nix-repl.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 25f84a3082a..5c319f06865 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -250,7 +250,8 @@ static int runProgram(const string & program, const Strings & args)
 
 bool isVarName(const string & s)
 {
-    // FIXME: not quite correct.
+    if (s.size() > 0 && (s[0] == '-' || s[0] == '\''))
+        return false;
     for (auto & i : s)
         if (!((i >= 'a' && i <= 'z') ||
               (i >= 'A' && i <= 'Z') ||

From f30fd9c47b1ae7a48f4854c86b3ad5e038845aa3 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Sun, 14 Feb 2016 01:50:47 -0600
Subject: [PATCH 0046/2196] Don't consider empty strings or strings beginning
 with numbers as variable names.

---
 nix-repl.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 5c319f06865..1077f5d8f61 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -250,8 +250,9 @@ static int runProgram(const string & program, const Strings & args)
 
 bool isVarName(const string & s)
 {
-    if (s.size() > 0 && (s[0] == '-' || s[0] == '\''))
-        return false;
+    if (s.size() == 0) return false;
+    char c = s[0];
+    if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
     for (auto & i : s)
         if (!((i >= 'a' && i <= 'z') ||
               (i >= 'A' && i <= 'Z') ||

From 3cfb8d15846238d79b36de7e52a90a4d3afa4268 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Mon, 15 Feb 2016 19:16:24 -0600
Subject: [PATCH 0047/2196] Remove unused global variable.

---
 nix-repl.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 1077f5d8f61..8569b5eb67b 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -19,9 +19,6 @@ using namespace std;
 using namespace nix;
 
 
-string programId = "nix-repl";
-
-
 struct NixRepl
 {
     string curDir;

From 30a7bfbebe8582ab02f2a9b659403a5b3e1c097b Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Mon, 15 Feb 2016 23:11:26 -0600
Subject: [PATCH 0048/2196] Fix grammar.

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8569b5eb67b..e753c637a75 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -316,7 +316,7 @@ bool NixRepl::processLine(string line)
         evalString(arg, v);
         DrvInfo drvInfo(state);
         if (!getDerivation(state, v, drvInfo, false))
-            throw Error("expression does not evaluation to a derivation, so I can't build it");
+            throw Error("expression does not evaluate to a derivation, so I can't build it");
         Path drvPath = drvInfo.queryDrvPath();
         if (drvPath == "" || !store->isValidPath(drvPath))
             throw Error("expression did not evaluate to a valid derivation");

From 82aca33899a0348736604e6b6a601f9c7b4e0633 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Tue, 16 Feb 2016 00:24:50 -0600
Subject: [PATCH 0049/2196] Add :i command to install a package to the current
 profile.

It works by running `nix-env -i <derivation path>`.

Fixes #15.
---
 nix-repl.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index e753c637a75..ca221b03338 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -281,6 +281,7 @@ bool NixRepl::processLine(string line)
              << "  <x> = <expr>  Bind expression to variable\n"
              << "  :a <expr>     Add attributes from resulting set to scope\n"
              << "  :b <expr>     Build derivation\n"
+             << "  :i <expr>     Build derivation, then install result into current profile\n"
              << "  :l <path>     Load Nix expression and add it to scope\n"
              << "  :p <expr>     Evaluate and print expression recursively\n"
              << "  :q            Exit nix-repl\n"
@@ -311,7 +312,7 @@ bool NixRepl::processLine(string line)
         std::cout << showType(v) << std::endl;
     }
 
-    else if (command == ":b" || command == ":s") {
+    else if (command == ":b" || command == ":i" || command == ":s") {
         Value v;
         evalString(arg, v);
         DrvInfo drvInfo(state);
@@ -331,8 +332,11 @@ bool NixRepl::processLine(string line)
                 for (auto & i : drv.outputs)
                     std::cout << format("  %1% -> %2%") % i.first % i.second.path << std::endl;
             }
-        } else
+        } else if (command == ":i") {
+            runProgram("nix-env", Strings{"-i", drvPath});
+        } else {
             runProgram("nix-shell", Strings{drvPath});
+        }
     }
 
     else if (command == ":p" || command == ":print") {

From cfc874ee52008f523a86b5079243deabaecb62e4 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Tue, 16 Feb 2016 18:40:45 -0600
Subject: [PATCH 0050/2196] Open the store before constructing EvalState.

EvalState requires the `store` global to be initialized before it is constructed
in some cases, e.g. when it needs to download a tarball for something in
NIX_PATH. Hence, this fixes #13.
---
 nix-repl.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 1077f5d8f61..cdf6a79e920 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -78,8 +78,6 @@ NixRepl::NixRepl(const Strings & searchPath)
     , staticEnv(false, &state.staticBaseEnv)
 {
     curDir = absPath(".");
-
-    store = openStore();
 }
 
 
@@ -622,6 +620,7 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        store = openStore();
         NixRepl repl(searchPath);
         repl.mainLoop(files);
     });

From 287dfee35edc6c324ee8829f37a8ed6b07c35ffa Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Thu, 18 Feb 2016 04:05:11 -0600
Subject: [PATCH 0051/2196] Expand the help message printed from --help.

Fixes #10. I consider this a temporary measure, however, until nix-repl has a
manpage (see #14). Then it can just open its manpage on --help like the other
nix tools do.

Much of the text in this commit was copied from nix-build's manpage.
---
 nix-repl.cc | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 1077f5d8f61..1b098405fc8 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -59,8 +59,35 @@ struct NixRepl
 
 void printHelp()
 {
-    std::cout << "Usage: nix-repl [--help|--version]";
-    std::cout << std::endl;
+    cout << "Usage: nix-repl [--help] [--version] [-I path] paths...\n"
+         << "\n"
+         << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n"
+         << "\n"
+         << "Options:\n"
+         << "    --help\n"
+         << "        Prints out a summary of the command syntax and exits.\n"
+         << "\n"
+         << "    --version\n"
+         << "        Prints out the Nix version number on standard output and exits.\n"
+         << "\n"
+         << "    -I path\n"
+         << "        Add a path to the Nix expression search path. This option may be given\n"
+         << "        multiple times. See the NIX_PATH environment variable for information on\n"
+         << "        the semantics of the Nix search path. Paths added through -I take\n"
+         << "        precedence over NIX_PATH.\n"
+         << "\n"
+         << "    paths...\n"
+         << "        A list of paths to files containing Nix expressions which nix-repl will\n"
+         << "        load and add to its scope.\n"
+         << "\n"
+         << "        A path surrounded in < and > will be looked up in the Nix expression search\n"
+         << "        path, as in the Nix language itself.\n"
+         << "\n"
+         << "        If an element of paths starts with http:// or https://, it is interpreted\n"
+         << "        as the URL of a tarball that will be downloaded and unpacked to a temporary\n"
+         << "        location. The tarball must include a single top-level directory containing\n"
+         << "        at least a file named default.nix.\n"
+         << flush;
 }
 
 

From 2d729e4f6f45c079ddf149610357e648e805f42c Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Thu, 18 Feb 2016 06:27:39 -0600
Subject: [PATCH 0052/2196] Support multiline input by detecting "unfinished"
 parse errors.

Fixes #4,
---
 nix-repl.cc | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 1077f5d8f61..9dddb26037a 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -41,7 +41,7 @@ struct NixRepl
     NixRepl(const Strings & searchPath);
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
-    bool getLine(string & line);
+    bool getLine(string & line, const char * prompt);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -96,21 +96,39 @@ void NixRepl::mainLoop(const Strings & files)
     using_history();
     read_history(0);
 
+    string input;
+
     while (true) {
+        // When continuing input from a previous, don't print a prompt, just align to the same
+        // number of chars as the prompt.
+        const char * prompt = input.empty() ? "nix-repl> " : "          ";
         string line;
-        if (!getLine(line)) {
+        if (!getLine(line, prompt)) {
             std::cout << std::endl;
             break;
         }
 
+        input.append(removeWhitespace(line));
+        input.push_back('\n');
+
         try {
-            if (!processLine(removeWhitespace(line))) return;
+            if (!processLine(input)) return;
+        } catch (ParseError & e) {
+            if (e.msg().find("unexpected $end") != std::string::npos) {
+                // For parse errors on incomplete input, we continue waiting for the next line of
+                // input without clearing the input so far.
+                continue;
+            } else {
+                printMsg(lvlError, "error: " + e.msg());
+            }
         } catch (Error & e) {
             printMsg(lvlError, "error: " + e.msg());
         } catch (Interrupted & e) {
             printMsg(lvlError, "error: " + e.msg());
         }
 
+        // We handled the current input fully, so we should clear it and read brand new input.
+        input.clear();
         std::cout << std::endl;
     }
 }
@@ -149,7 +167,7 @@ char * completerThunk(const char * s, int state)
 }
 
 
-bool NixRepl::getLine(string & line)
+bool NixRepl::getLine(string & line, const char * prompt)
 {
     struct sigaction act, old;
     act.sa_handler = sigintHandler;
@@ -164,7 +182,7 @@ bool NixRepl::getLine(string & line)
         curRepl = this;
         rl_completion_entry_function = completerThunk;
 
-        char * s = readline("nix-repl> ");
+        char * s = readline(prompt);
         if (!s) return false;
         line = chomp(string(s));
         free(s);

From 64080d26fe9364bc0ea0893f357386ed3121878f Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Thu, 18 Feb 2016 06:50:52 -0600
Subject: [PATCH 0053/2196] Cancel multiline input on Ctrl-C.

---
 nix-repl.cc | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 9dddb26037a..8d2fbd919c9 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -41,7 +41,7 @@ struct NixRepl
     NixRepl(const Strings & searchPath);
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
-    bool getLine(string & line, const char * prompt);
+    bool getLine(string & input, const char * prompt);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -102,15 +102,11 @@ void NixRepl::mainLoop(const Strings & files)
         // When continuing input from a previous, don't print a prompt, just align to the same
         // number of chars as the prompt.
         const char * prompt = input.empty() ? "nix-repl> " : "          ";
-        string line;
-        if (!getLine(line, prompt)) {
+        if (!getLine(input, prompt)) {
             std::cout << std::endl;
             break;
         }
 
-        input.append(removeWhitespace(line));
-        input.push_back('\n');
-
         try {
             if (!processLine(input)) return;
         } catch (ParseError & e) {
@@ -167,7 +163,7 @@ char * completerThunk(const char * s, int state)
 }
 
 
-bool NixRepl::getLine(string & line, const char * prompt)
+bool NixRepl::getLine(string & input, const char * prompt)
 {
     struct sigaction act, old;
     act.sa_handler = sigintHandler;
@@ -176,15 +172,17 @@ bool NixRepl::getLine(string & line, const char * prompt)
     if (sigaction(SIGINT, &act, &old))
         throw SysError("installing handler for SIGINT");
 
-    if (sigsetjmp(sigintJmpBuf, 1))
-        line = "";
-    else {
+    if (sigsetjmp(sigintJmpBuf, 1)) {
+        input.clear();
+    } else {
         curRepl = this;
         rl_completion_entry_function = completerThunk;
 
         char * s = readline(prompt);
         if (!s) return false;
-        line = chomp(string(s));
+        string line = chomp(string(s));
+        input.append(removeWhitespace(line));
+        input.push_back('\n');
         free(s);
         if (line != "") {
             add_history(line.c_str());

From 60ba98242f7c976e4e14113d28bced03b32db4f5 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Thu, 18 Feb 2016 06:59:51 -0600
Subject: [PATCH 0054/2196] Fix recognition of REPL commands.

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8d2fbd919c9..e52c3b257a7 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -286,7 +286,7 @@ bool NixRepl::processLine(string line)
     string command, arg;
 
     if (line[0] == ':') {
-        size_t p = line.find(' ');
+        size_t p = line.find_first_of(" \n\r\t");
         command = string(line, 0, p);
         if (p != string::npos) arg = removeWhitespace(string(line, p));
     } else {

From 56c7f0e8c581c66a968fdae681e9c417817e28d0 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Thu, 18 Feb 2016 07:04:55 -0600
Subject: [PATCH 0055/2196] Fix typo in comment.

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index e52c3b257a7..3834e572a2d 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -99,7 +99,7 @@ void NixRepl::mainLoop(const Strings & files)
     string input;
 
     while (true) {
-        // When continuing input from a previous, don't print a prompt, just align to the same
+        // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         const char * prompt = input.empty() ? "nix-repl> " : "          ";
         if (!getLine(input, prompt)) {

From 97da6d62f2a1580dfdae9575c92418d5f45a29af Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Sat, 20 Feb 2016 01:10:06 -0600
Subject: [PATCH 0056/2196] Print syntactially invalid attribute names as
 strings.

---
 nix-repl.cc | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index f4850271dc6..577efa8e2ba 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -472,6 +472,19 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 }
 
 
+std::ostream & printStringValue(std::ostream & str, const char * string) {
+    str << "\"";
+    for (const char * i = string; *i; i++)
+        if (*i == '\"' || *i == '\\') str << "\\" << *i;
+        else if (*i == '\n') str << "\\n";
+        else if (*i == '\r') str << "\\r";
+        else if (*i == '\t') str << "\\t";
+        else str << *i;
+    str << "\"";
+    return str;
+}
+
+
 // FIXME: lot of cut&paste from Nix's eval.cc.
 std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
 {
@@ -491,14 +504,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case tString:
-        str << "\"";
-        for (const char * i = v.string.s; *i; i++)
-            if (*i == '\"' || *i == '\\') str << "\\" << *i;
-            else if (*i == '\n') str << "\\n";
-            else if (*i == '\r') str << "\\r";
-            else if (*i == '\t') str << "\\t";
-            else str << *i;
-        str << "\"";
+        printStringValue(str, v.string.s);
         break;
 
     case tPath:
@@ -546,7 +552,11 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
             }
 
             for (auto & i : sorted) {
-                str << i.first << " = ";
+                if (isVarName(i.first))
+                    str << i.first;
+                else
+                    printStringValue(str, i.first.c_str());
+                str << " = ";
                 if (hidden.find(i.first) != hidden.end())
                     str << "«...»";
                 else if (seen.find(i.second) != seen.end())

From 103c46abc273266afcd5dbffa40151114234a02b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ole=20J=C3=B8rgen=20Br=C3=B8nner?= <olejorgenb@yahoo.no>
Date: Tue, 23 Feb 2016 23:19:49 +0100
Subject: [PATCH 0057/2196] Preserve readline history across sessions. Add
 rl_readline_name.

---
 README.md   | 4 ++++
 nix-repl.cc | 7 ++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c5964fd5ea1..acdcd367fc9 100644
--- a/README.md
+++ b/README.md
@@ -102,3 +102,7 @@ example:
 
     nix-repl> config.networking.use<TAB>
     config.networking.useDHCP   config.networking.usePredictableInterfaceNames
+
+Input history is preserved by readline in ~/.nix-repl-history
+The readline "application name" is nix-repl. This allows for nix-repl specific
+settings in ~/.inputrc
diff --git a/nix-repl.cc b/nix-repl.cc
index 577efa8e2ba..1c878cd0e23 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -20,6 +20,7 @@ using namespace nix;
 
 
 string programId = "nix-repl";
+const string historyFile = string(getenv("HOME")) + "/.nix-repl-history";
 
 
 struct NixRepl
@@ -91,8 +92,10 @@ void NixRepl::mainLoop(const Strings & files)
     reloadFiles();
     if (!loadedFiles.empty()) std::cout << std::endl;
 
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
     using_history();
-    read_history(0);
+    read_history(historyFile.c_str());
 
     string input;
 
@@ -649,5 +652,7 @@ int main(int argc, char * * argv)
         store = openStore();
         NixRepl repl(searchPath);
         repl.mainLoop(files);
+
+        write_history(historyFile.c_str());
     });
 }

From 87e6649fc30a37adabf384a68f588de946bc3468 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Tue, 23 Feb 2016 18:29:56 -0600
Subject: [PATCH 0058/2196] Fix handling of whitespace.

Whitespace will no longer be removed from input lines, which fixes pasting
multiline strings containing end-of-line or beginning-of-line whitespace.
---
 nix-repl.cc | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 577efa8e2ba..1cdff20fd41 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -178,14 +178,13 @@ bool NixRepl::getLine(string & input, const char * prompt)
 
         char * s = readline(prompt);
         if (!s) return false;
-        string line = chomp(string(s));
-        input.append(removeWhitespace(line));
+        input.append(s);
         input.push_back('\n');
-        free(s);
-        if (line != "") {
-            add_history(line.c_str());
+        if (!removeWhitespace(s).empty()) {
+            add_history(s);
             append_history(1, 0);
         }
+        free(s);
     }
 
     _isInterrupted = 0;

From 38816759fc7f70605ecfd73304b9d442db388b78 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Tue, 23 Feb 2016 18:30:21 -0600
Subject: [PATCH 0059/2196] Ignore blank inputs.

Previously, nix-repl would consider this an incomplete parse and wait for the
next line as if it was a multiline input.

Blank lines in the middle of a multiline input will continue to work.
---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 1cdff20fd41..6f4287a2b69 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -106,7 +106,7 @@ void NixRepl::mainLoop(const Strings & files)
         }
 
         try {
-            if (!processLine(input)) return;
+            if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {
             if (e.msg().find("unexpected $end") != std::string::npos) {
                 // For parse errors on incomplete input, we continue waiting for the next line of

From ff8d0698c76914f30d5311e1ceb0ab7d18a543b6 Mon Sep 17 00:00:00 2001
From: Fabian Schmitthenner <development@schmitthenner.eu>
Date: Sun, 28 Feb 2016 22:41:16 +0000
Subject: [PATCH 0060/2196] fix nix-repl after we don't have a global store
 variable anymore (cf nix@c10c61449f954702ae6d8092120321744acd82ff)

---
 nix-repl.cc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 6f4287a2b69..89c35c311db 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -38,7 +38,7 @@ struct NixRepl
     StringSet completions;
     StringSet::iterator curCompletion;
 
-    NixRepl(const Strings & searchPath);
+    NixRepl(const Strings & searchPath, nix::ref<Store> store);
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
     bool getLine(string & input, const char * prompt);
@@ -73,8 +73,8 @@ string removeWhitespace(string s)
 }
 
 
-NixRepl::NixRepl(const Strings & searchPath)
-    : state(searchPath)
+NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
+    : state(searchPath, store)
     , staticEnv(false, &state.staticBaseEnv)
 {
     curDir = absPath(".");
@@ -334,7 +334,7 @@ bool NixRepl::processLine(string line)
         if (!getDerivation(state, v, drvInfo, false))
             throw Error("expression does not evaluation to a derivation, so I can't build it");
         Path drvPath = drvInfo.queryDrvPath();
-        if (drvPath == "" || !store->isValidPath(drvPath))
+        if (drvPath == "" || !state.store->isValidPath(drvPath))
             throw Error("expression did not evaluate to a valid derivation");
 
         if (command == ":b") {
@@ -645,8 +645,7 @@ int main(int argc, char * * argv)
             return true;
         });
 
-        store = openStore();
-        NixRepl repl(searchPath);
+        NixRepl repl(searchPath, openStore());
         repl.mainLoop(files);
     });
 }

From 86e93b9f61cf53cfd766e1724e65507aca952f55 Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Wed, 17 Feb 2016 20:31:30 -0600
Subject: [PATCH 0061/2196] Add :x command which works like `nix-shell -p`.

---
 nix-repl.cc | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8c4f4017470..adf4186975b 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -39,6 +39,7 @@ struct NixRepl
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
     bool getLine(string & input, const char * prompt);
+    Path getDerivationPath(Value & v);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -300,6 +301,17 @@ bool isVarName(const string & s)
 }
 
 
+Path NixRepl::getDerivationPath(Value & v) {
+    DrvInfo drvInfo(state);
+    if (!getDerivation(state, v, drvInfo, false))
+        throw Error("expression does not evaluate to a derivation, so I can't build it");
+    Path drvPath = drvInfo.queryDrvPath();
+    if (drvPath == "" || !state.store->isValidPath(drvPath))
+        throw Error("expression did not evaluate to a valid derivation");
+    return drvPath;
+}
+
+
 bool NixRepl::processLine(string line)
 {
     if (line == "") return true;
@@ -327,7 +339,8 @@ bool NixRepl::processLine(string line)
              << "  :q            Exit nix-repl\n"
              << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
-             << "  :t <expr>     Describe result of evaluation\n";
+             << "  :t <expr>     Describe result of evaluation\n"
+             << "  :x <expr>     Build derivation, then start nix-shell\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -350,17 +363,21 @@ bool NixRepl::processLine(string line)
         Value v;
         evalString(arg, v);
         std::cout << showType(v) << std::endl;
+
+    } else if (command == ":x") {
+        Value v, f, result;
+        evalString(arg, v);
+        evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
+        state.callFunction(f, v, result, Pos());
+
+        Path drvPath = getDerivationPath(result);
+        runProgram("nix-shell", Strings{drvPath});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
         Value v;
         evalString(arg, v);
-        DrvInfo drvInfo(state);
-        if (!getDerivation(state, v, drvInfo, false))
-            throw Error("expression does not evaluate to a derivation, so I can't build it");
-        Path drvPath = drvInfo.queryDrvPath();
-        if (drvPath == "" || !state.store->isValidPath(drvPath))
-            throw Error("expression did not evaluate to a valid derivation");
+        Path drvPath = getDerivationPath(v);
 
         if (command == ":b") {
             /* We could do the build in this process using buildPaths(),

From e2ff27da07f30c598c88e8e7552be0d126e2b4da Mon Sep 17 00:00:00 2001
From: Scott Olson <scott@solson.me>
Date: Fri, 19 Feb 2016 04:00:36 -0600
Subject: [PATCH 0062/2196] Rename :x to :u, for 'use'.

---
 nix-repl.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index adf4186975b..a9a21ae766d 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -340,7 +340,7 @@ bool NixRepl::processLine(string line)
              << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n"
-             << "  :x <expr>     Build derivation, then start nix-shell\n";
+             << "  :u <expr>     Build derivation, then start nix-shell\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -364,7 +364,7 @@ bool NixRepl::processLine(string line)
         evalString(arg, v);
         std::cout << showType(v) << std::endl;
 
-    } else if (command == ":x") {
+    } else if (command == ":u") {
         Value v, f, result;
         evalString(arg, v);
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);

From 8bec2c07a1e25e3471ee5f1b7aedb30a5d5b03cd Mon Sep 17 00:00:00 2001
From: Fabian Schmitthenner <development@schmitthenner.eu>
Date: Sat, 5 Mar 2016 16:47:17 +0000
Subject: [PATCH 0063/2196] When showing a lambda, also show the position of
 the definition

---
 nix-repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index 8c4f4017470..e17f73ae0ca 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -628,7 +628,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case tLambda:
-        str << "«lambda»";
+        str << "«lambda defined at " << v.lambda.fun->pos << "»";
         break;
 
     case tPrimOp:

From 828cf7b0582220ce568b69e9cc51af794bdd2416 Mon Sep 17 00:00:00 2001
From: Fabian Schmitthenner <development@schmitthenner.eu>
Date: Sat, 19 Mar 2016 13:52:39 +0000
Subject: [PATCH 0064/2196] show trace of errors when using --show-trace

---
 nix-repl.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index a9a21ae766d..d1c67276c2f 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -14,6 +14,7 @@
 #include "get-drvs.hh"
 #include "derivations.hh"
 #include "affinity.hh"
+#include "globals.hh"
 
 using namespace std;
 using namespace nix;
@@ -108,6 +109,7 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
 
 void NixRepl::mainLoop(const Strings & files)
 {
+    string error = ANSI_RED "error:" ANSI_NORMAL " ";
     std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
 
     for (auto & i : files)
@@ -138,12 +140,12 @@ void NixRepl::mainLoop(const Strings & files)
                 // input without clearing the input so far.
                 continue;
             } else {
-                printMsg(lvlError, "error: " + e.msg());
+              printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
             }
         } catch (Error & e) {
-            printMsg(lvlError, "error: " + e.msg());
+            printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         } catch (Interrupted & e) {
-            printMsg(lvlError, "error: " + e.msg());
+            printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         }
 
         // We handled the current input fully, so we should clear it and read brand new input.

From eaabcba1c39a22c588b8c9336ec7ba98aefce86e Mon Sep 17 00:00:00 2001
From: Emery Hemingway <emery@vfemail.net>
Date: Sat, 25 Jun 2016 13:25:31 +0200
Subject: [PATCH 0065/2196] Colorize

Berlin NixOS meetup
---
 nix-repl.cc | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index a9a21ae766d..e28c543035f 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -18,6 +18,13 @@
 using namespace std;
 using namespace nix;
 
+#define ESC_RED "\033[31m"
+#define ESC_GRE "\033[32m"
+#define ESC_YEL "\033[33m"
+#define ESC_BLU "\033[34m"
+#define ESC_MAG "\033[35m"
+#define ESC_CYA "\033[36m"
+#define ESC_END "\033[0m"
 
 struct NixRepl
 {
@@ -540,23 +547,25 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     switch (v.type) {
 
     case tInt:
-        str << v.integer;
+        str << ESC_CYA << v.integer << ESC_END;
         break;
 
     case tBool:
-        str << (v.boolean ? "true" : "false");
+        str << ESC_CYA << (v.boolean ? "true" :  "false") << ESC_END;
         break;
 
     case tString:
+        str << ESC_YEL;
         printStringValue(str, v.string.s);
+        str << ESC_END;
         break;
 
     case tPath:
-        str << v.path; // !!! escaping?
+        str << ESC_GRE << v.path << ESC_END; // !!! escaping?
         break;
 
     case tNull:
-        str << "null";
+        str << ESC_CYA "null" ESC_END;
         break;
 
     case tAttrs: {
@@ -609,7 +618,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                     try {
                         printValue(str, *i.second, maxDepth - 1, seen);
                     } catch (AssertionError & e) {
-                        str << "«error: " << e.msg() << "»";
+                        str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
                     }
                 str << "; ";
             }
@@ -635,7 +644,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                     try {
                         printValue(str, *v.listElems()[n], maxDepth - 1, seen);
                     } catch (AssertionError & e) {
-                        str << "«error: " << e.msg() << "»";
+                        str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
                     }
                 str << " ";
             }
@@ -645,19 +654,19 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case tLambda:
-        str << "«lambda»";
+        str << ESC_BLU "«lambda»" ESC_END;
         break;
 
     case tPrimOp:
-        str << "«primop»";
+        str << ESC_MAG "«primop»" ESC_END;
         break;
 
     case tPrimOpApp:
-        str << "«primop-app»";
+        str << ESC_BLU "«primop-app»" ESC_END;
         break;
 
     default:
-        str << "«unknown»";
+        str << "ESC_RED «unknown»" ESC_END;
         break;
     }
 

From 34ec98176e4644c8c6ec45e4e932b83cbddc6cce Mon Sep 17 00:00:00 2001
From: Emery Hemingway <emery@vfemail.net>
Date: Sat, 25 Jun 2016 13:40:50 +0200
Subject: [PATCH 0066/2196] fixup "Colorize"

Do not quote escape sequence macro
---
 nix-repl.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index e28c543035f..fc0c3791334 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -551,7 +551,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case tBool:
-        str << ESC_CYA << (v.boolean ? "true" :  "false") << ESC_END;
+        str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END;
         break;
 
     case tString:
@@ -666,7 +666,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     default:
-        str << "ESC_RED «unknown»" ESC_END;
+        str << ESC_RED "«unknown»" ESC_END;
         break;
     }
 

From 8b6052923167f19c1f4728c46e8cfc97b9f029fb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 21 Jul 2016 11:21:59 +0200
Subject: [PATCH 0067/2196] Strip ANSI escapes from file names

Also, use bright blue for lambdas, otherwise my eyes hurt.
---
 nix-repl.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/nix-repl.cc b/nix-repl.cc
index ab943f7bac1..0e8c67cf7f2 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -22,7 +22,7 @@ using namespace nix;
 #define ESC_RED "\033[31m"
 #define ESC_GRE "\033[32m"
 #define ESC_YEL "\033[33m"
-#define ESC_BLU "\033[34m"
+#define ESC_BLU "\033[34;1m"
 #define ESC_MAG "\033[35m"
 #define ESC_CYA "\033[36m"
 #define ESC_END "\033[0m"
@@ -655,9 +655,12 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         str << "]";
         break;
 
-    case tLambda:
-        str << ESC_BLU "«lambda defined at " << v.lambda.fun->pos << "»" ESC_END;
+    case tLambda: {
+        std::ostringstream s;
+        s << v.lambda.fun->pos;
+        str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END;
         break;
+    }
 
     case tPrimOp:
         str << ESC_MAG "«primop»" ESC_END;

From 5476e987d5d605e8fc8e50d76912e342c722bbdb Mon Sep 17 00:00:00 2001
From: Kjetil Orbekk <kjetil.orbekk@gmail.com>
Date: Sun, 1 Jan 2017 16:13:11 -0500
Subject: [PATCH 0068/2196] Update <nixpkgs/nixos> path in documentation.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c5964fd5ea1..57613cc19e9 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ Here is a typical `nix-repl` session:
     nix-repl> map (x: x * x) xs
     [ 1 4 9 16 25 ]
 
-    nix-repl> :l <nixos>
+    nix-repl> :l <nixpkgs/nixos>
     Added 7 variables.
 
     nix-repl> config.services.dhcpd
@@ -86,7 +86,7 @@ Here is a typical `nix-repl` session:
 Tab completion works on variables in scope and on attribute sets.  For
 example:
 
-    $ nix-repl  '<nixpkgs>' '<nixos>'
+    $ nix-repl  '<nixpkgs>' '<nixpkgs/nixos>'
     Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
 
     nix-repl> thunder<TAB> => thunderbird

From bfa41eb6714a7e7c3956389ee063e898bd1f37ff Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 10 Aug 2016 10:44:39 -0400
Subject: [PATCH 0069/2196] nix-copy-closure: Implement in C++.

Tests fail currently because the database is not given proper hashes in the VM
---
 .gitignore                               |   2 +
 Makefile                                 |   1 +
 scripts/local.mk                         |   7 +-
 scripts/nix-copy-closure.in              | 103 -----------------------
 src/build-remote/build-remote.cc         |   1 -
 src/build-remote/local.mk                |   2 +-
 src/libstore/ssh-store.cc                |   8 +-
 src/libstore/store-api.cc                |  24 +++++-
 src/libstore/store-api.hh                |   2 +-
 src/nix-copy-closure/local.mk            |   7 ++
 src/nix-copy-closure/nix-copy-closure.cc |  60 +++++++++++++
 11 files changed, 101 insertions(+), 116 deletions(-)
 delete mode 100755 scripts/nix-copy-closure.in
 create mode 100644 src/nix-copy-closure/local.mk
 create mode 100755 src/nix-copy-closure/nix-copy-closure.cc

diff --git a/.gitignore b/.gitignore
index 92f95fe1fcb..a8abc64ada5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,6 +81,8 @@ Makefile.config
 # /src/nix-build/
 /src/nix-build/nix-build
 
+/src/nix-copy-closure/nix-copy-closure
+
 /src/build-remote/build-remote
 
 # /tests/
diff --git a/Makefile b/Makefile
index 14be271bb10..8390c867d8b 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,7 @@ makefiles = \
   src/nix-env/local.mk \
   src/nix-daemon/local.mk \
   src/nix-collect-garbage/local.mk \
+  src/nix-copy-closure/local.mk \
   src/nix-prefetch-url/local.mk \
   src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \
diff --git a/scripts/local.mk b/scripts/local.mk
index ee8ae6845dc..9f666bde4d0 100644
--- a/scripts/local.mk
+++ b/scripts/local.mk
@@ -1,8 +1,3 @@
-nix_bin_scripts := \
-  $(d)/nix-copy-closure \
-
-bin-scripts += $(nix_bin_scripts)
-
 nix_noinst_scripts := \
   $(d)/build-remote.pl \
   $(d)/nix-http-export.cgi \
@@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
 $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
 
-clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)
+clean-files += $(nix_noinst_scripts)
diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in
deleted file mode 100755
index af1d3091926..00000000000
--- a/scripts/nix-copy-closure.in
+++ /dev/null
@@ -1,103 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use utf8;
-use strict;
-use Nix::SSH;
-use Nix::Config;
-use Nix::Store;
-use Nix::CopyClosure;
-use List::Util qw(sum);
-
-binmode STDERR, ":encoding(utf8)";
-
-if (scalar @ARGV < 1) {
-    print STDERR <<EOF
-Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
-EOF
-    ;
-    exit 1;
-}
-
-
-# Get the target host.
-my $sshHost;
-my $toMode = 1;
-my $includeOutputs = 0;
-my $dryRun = 0;
-my $useSubstitutes = 0;
-my $verbosity = 1;
-
-
-# !!! Copied from nix-pack-closure, should put this in a module.
-my @storePaths = ();
-
-while (@ARGV) {
-    my $arg = shift @ARGV;
-
-    if ($arg eq "--help") {
-        exec "man nix-copy-closure" or die;
-    }
-    elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
-        warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip";
-        push @globalSshOpts, "-C";
-    }
-    elsif ($arg eq "--from") {
-        $toMode = 0;
-    }
-    elsif ($arg eq "--to") {
-        $toMode = 1;
-    }
-    elsif ($arg eq "--include-outputs") {
-        $includeOutputs = 1;
-    }
-    elsif ($arg eq "--show-progress") {
-        warn "$0: ‘$arg’ is not implemented\n";
-    }
-    elsif ($arg eq "--dry-run") {
-        $dryRun = 1;
-    }
-    elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
-        $useSubstitutes = 1;
-    }
-    elsif ($arg eq "-v") {
-        $verbosity++;
-        setVerbosity($verbosity);
-    }
-    elsif (!defined $sshHost) {
-        $sshHost = $arg;
-    }
-    else {
-        push @storePaths, $arg;
-    }
-}
-
-die "$0: you did not specify a host name\n" unless defined $sshHost;
-
-
-if ($toMode) { # Copy TO the remote machine.
-    Nix::CopyClosure::copyTo(
-        $sshHost, [ @storePaths ],
-        $includeOutputs, $dryRun, $useSubstitutes);
-}
-
-else { # Copy FROM the remote machine.
-
-    my ($from, $to) = connectToRemoteNix($sshHost, []);
-
-    # Query the closure of the given store paths on the remote
-    # machine.  Paths are assumed to be store paths; there is no
-    # resolution (following of symlinks).
-    syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
-    writeStrings(\@storePaths, $to);
-    my @missing = grep { !isValidPath($_) } readStrings($from);
-
-    # Export the store paths on the remote machine and import them locally.
-    if (scalar @missing > 0) {
-        print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
-        writeInt(5, $to); # == cmdExportPaths
-        writeInt(0, $to); # obsolete
-        writeStrings(\@missing, $to);
-        importPaths(fileno($from), 1);
-    }
-
-}
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 98ccc3ddc37..1ac9711a110 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -9,7 +9,6 @@
 #include "shared.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
-#include "serve-protocol.hh"
 #include "serialise.hh"
 #include "store-api.hh"
 #include "derivations.hh"
diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
index 05b8cb45143..62d5a010c24 100644
--- a/src/build-remote/local.mk
+++ b/src/build-remote/local.mk
@@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
 
 build-remote_SOURCES := $(d)/build-remote.cc
 
-build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
+build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 3d01594009a..cce0458c69f 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -39,6 +39,8 @@ class SSHStore : public RemoteStore
     string uri;
 
     Path key;
+
+    bool compress;
 };
 
 SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
@@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
     , socketPath((Path) tmpDir + "/ssh.sock")
     , uri(std::move(uri))
     , key(get(params, "ssh-key", ""))
+    , compress(get(params, "compress", "") == "true")
 {
     /* open a connection and perform the handshake to verify all is well */
     connections->get();
@@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     if ((pid_t) sshMaster == -1) {
+        auto flags = compress ? "-NMCS" : "-NMS";
         sshMaster = startProcess([&]() {
             if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
             else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
             throw SysError("starting ssh master");
         });
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8fdd6277155..c8ca00f0069 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
+#include "derivations.hh"
 
 #include <future>
 
@@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
-{
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
+{
+    if (substitute) {
+        /* Filter out .drv files (we don't want to build anything). */
+        PathSet paths2;
+        for (auto & path : storePaths)
+            if (!isDerivation(path)) paths2.insert(path);
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
+        /* FIXME: should use ensurePath(), but it only
+           does one path at a time. */
+        if (!willSubstitute.empty())
+            try {
+                to->buildPaths(willSubstitute);
+            } catch (Error & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+    }
+
     std::string copiedLabel = "copied";
 
     logger->setExpected(copiedLabel, storePaths.size());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ec3bf5a6fd8..30ee433bf07 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -625,7 +625,7 @@ void removeTempRoots();
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,
diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk
new file mode 100644
index 00000000000..42bb34dd820
--- /dev/null
+++ b/src/nix-copy-closure/local.mk
@@ -0,0 +1,7 @@
+programs += nix-copy-closure
+
+nix-copy-closure_DIR := $(d)
+
+nix-copy-closure_LIBS = libmain libutil libformat libstore
+
+nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
new file mode 100755
index 00000000000..b7e997ca4b0
--- /dev/null
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -0,0 +1,60 @@
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+int main(int argc, char ** argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        auto gzip = false;
+        auto toMode = true;
+        auto includeOutputs = false;
+        auto dryRun = false;
+        auto useSubstitutes = false;
+        auto sshHost = string{};
+        auto storePaths = PathSet{};
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-copy-closure");
+            else if (*arg == "--version")
+                printVersion("nix-copy-closure");
+            else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
+                if (*arg != "--gzip")
+                    printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
+                gzip = true;
+            } else if (*arg == "--from")
+                toMode = false;
+            else if (*arg == "--to")
+                toMode = true;
+            else if (*arg == "--include-outputs")
+                includeOutputs = true;
+            else if (*arg == "--show-progress")
+                printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
+            else if (*arg == "--dry-run")
+                dryRun = true;
+            else if (*arg == "--use-substitutes" || *arg == "-s")
+                useSubstitutes = true;
+            else if (sshHost.empty())
+                sshHost = *arg;
+            else
+                storePaths.insert(*arg);
+            return true;
+        });
+        if (sshHost.empty())
+            throw UsageError("no host name specified");
+
+        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto to = toMode ? openStore(remoteUri) : openStore();
+        auto from = toMode ? openStore() : openStore(remoteUri);
+        if (includeOutputs) {
+            auto newPaths = PathSet{};
+            for (const auto & p : storePaths) {
+                auto outputs = from->queryDerivationOutputs(p);
+                newPaths.insert(outputs.begin(), outputs.end());
+            }
+            storePaths.insert(newPaths.begin(), newPaths.end());
+        }
+        copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
+    });
+}

From 4425a5c5470e4f26213cfecac404fed88d0f35aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Jan 2017 17:18:11 +0100
Subject: [PATCH 0070/2196] Move exportReferencesGraph into a separate method

startBuilder() is getting rather obese.
---
 src/libstore/build.cc | 99 ++++++++++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 44 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 7fc6ff0df0f..607601e2aac 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -864,6 +864,9 @@ class DerivationGoal : public Goal
     /* Start building a derivation. */
     void startBuilder();
 
+    /* Handle the exportReferencesGraph attribute. */
+    void doExportReferencesGraph();
+
     /* Run the builder's process. */
     void runChild();
 
@@ -1791,51 +1794,9 @@ void DerivationGoal::startBuilder()
     for (auto & output : drv->outputs)
         inputRewrites[hashPlaceholder(output.first)] = output.second.path;
 
-    /* The `exportReferencesGraph' feature allows the references graph
-       to be passed to a builder.  This attribute should be a list of
-       pairs [name1 path1 name2 path2 ...].  The references graph of
-       each `pathN' will be stored in a text file `nameN' in the
-       temporary build directory.  The text files have the format used
-       by `nix-store --register-validity'.  However, the deriver
-       fields are left empty. */
-    string s = get(drv->env, "exportReferencesGraph");
-    Strings ss = tokenizeString<Strings>(s);
-    if (ss.size() % 2 != 0)
-        throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s);
-    for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-        string fileName = *i++;
-        checkStoreName(fileName); /* !!! abuse of this function */
 
-        /* Check that the store path is valid. */
-        Path storePath = *i++;
-        if (!worker.store.isInStore(storePath))
-            throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’")
-                % storePath);
-        storePath = worker.store.toStorePath(storePath);
-        if (!worker.store.isValidPath(storePath))
-            throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’")
-                % storePath);
-
-        /* If there are derivations in the graph, then include their
-           outputs as well.  This is useful if you want to do things
-           like passing all build-time dependencies of some path to a
-           derivation that builds a NixOS DVD image. */
-        PathSet paths, paths2;
-        worker.store.computeFSClosure(storePath, paths);
-        paths2 = paths;
-
-        for (auto & j : paths2) {
-            if (isDerivation(j)) {
-                Derivation drv = worker.store.derivationFromPath(j);
-                for (auto & k : drv.outputs)
-                    worker.store.computeFSClosure(k.second.path, paths);
-            }
-        }
-
-        /* Write closure info to `fileName'. */
-        writeFile(tmpDir + "/" + fileName,
-            worker.store.makeValidityRegistration(paths, false, false));
-    }
+    /* Handle exportReferencesGraph(), if set. */
+    doExportReferencesGraph();
 
 
     /* If `build-users-group' is not empty, then we have to build as
@@ -2242,6 +2203,56 @@ void DerivationGoal::startBuilder()
 }
 
 
+void DerivationGoal::doExportReferencesGraph()
+{
+    /* The `exportReferencesGraph' feature allows the references graph
+       to be passed to a builder.  This attribute should be a list of
+       pairs [name1 path1 name2 path2 ...].  The references graph of
+       each `pathN' will be stored in a text file `nameN' in the
+       temporary build directory.  The text files have the format used
+       by `nix-store --register-validity'.  However, the deriver
+       fields are left empty. */
+    string s = get(drv->env, "exportReferencesGraph");
+    Strings ss = tokenizeString<Strings>(s);
+    if (ss.size() % 2 != 0)
+        throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s);
+    for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
+        string fileName = *i++;
+        checkStoreName(fileName); /* !!! abuse of this function */
+
+        /* Check that the store path is valid. */
+        Path storePath = *i++;
+        if (!worker.store.isInStore(storePath))
+            throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’")
+                % storePath);
+        storePath = worker.store.toStorePath(storePath);
+        if (!worker.store.isValidPath(storePath))
+            throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’")
+                % storePath);
+
+        /* If there are derivations in the graph, then include their
+           outputs as well.  This is useful if you want to do things
+           like passing all build-time dependencies of some path to a
+           derivation that builds a NixOS DVD image. */
+        PathSet paths, paths2;
+        worker.store.computeFSClosure(storePath, paths);
+        paths2 = paths;
+
+        for (auto & j : paths2) {
+            if (isDerivation(j)) {
+                Derivation drv = worker.store.derivationFromPath(j);
+                for (auto & k : drv.outputs)
+                    worker.store.computeFSClosure(k.second.path, paths);
+            }
+        }
+
+        /* Write closure info to `fileName'. */
+        writeFile(tmpDir + "/" + fileName,
+            worker.store.makeValidityRegistration(paths, false, false));
+    }
+}
+
+
 void DerivationGoal::runChild()
 {
     /* Warning: in the child we should absolutely not make any SQLite

From e8c43abd9aed4d88f2cce9ebc97de73930dffdfb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Jan 2017 20:13:19 +0100
Subject: [PATCH 0071/2196] On HTTP errors, also show the curl error

This is a hopefully temporary measure to diagnose the intermittent
"HTTP error 200" failures.
---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8030e83b0dd..074e0ca6642 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -276,7 +276,7 @@ struct CurlDownloader : public Downloader
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
                     ? DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri)
                     : httpStatus != 0
-                      ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d") % request.uri % httpStatus)
+                      ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code))
                       : DownloadError(err, format("unable to download ‘%s’: %s (%d)") % request.uri % curl_easy_strerror(code) % code);
 
                 /* If this is a transient error, then maybe retry the

From a529c740d28859201a3a4b245b88ade96fb89fb0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 12:00:28 +0100
Subject: [PATCH 0072/2196] Moving more code out of
 DerivationGoal::startBuilder()

---
 src/libstore/build.cc | 211 ++++++++++++++++++++++--------------------
 1 file changed, 111 insertions(+), 100 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 607601e2aac..1d039d33849 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -780,6 +780,7 @@ class DerivationGoal : public Goal
     };
     typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
     DirsInChroot dirsInChroot;
+
     typedef map<string, string> Environment;
     Environment env;
 
@@ -817,6 +818,8 @@ class DerivationGoal : public Goal
     const uid_t sandboxUid = 1000;
     const gid_t sandboxGid = 100;
 
+    const static Path homeDir;
+
 public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
@@ -864,6 +867,12 @@ class DerivationGoal : public Goal
     /* Start building a derivation. */
     void startBuilder();
 
+    /* Fill in the environment for the builder. */
+    void initEnv();
+
+    /* Make a file owned by the builder. */
+    void chownToBuilder(const Path & path);
+
     /* Handle the exportReferencesGraph attribute. */
     void doExportReferencesGraph();
 
@@ -907,6 +916,9 @@ class DerivationGoal : public Goal
 };
 
 
+const Path DerivationGoal::homeDir = "/homeless-shelter";
+
+
 DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
     Worker & worker, BuildMode buildMode)
     : Goal(worker)
@@ -1672,11 +1684,7 @@ void DerivationGoal::startBuilder()
     additionalSandboxProfile = get(drv->env, "__sandboxProfile");
 #endif
 
-    /* Are we doing a chroot build?  Note that fixed-output
-       derivations are never done in a chroot, mainly so that
-       functions like fetchurl (which needs a proper /etc/resolv.conf)
-       work properly.  Purity checking for fixed-output derivations
-       is somewhat pointless anyway. */
+    /* Are we doing a chroot build? */
     {
         string x = settings.get("build-use-sandbox",
             /* deprecated alias */
@@ -1703,31 +1711,15 @@ void DerivationGoal::startBuilder()
     if (worker.store.storeDir != worker.store.realStoreDir)
         useChroot = true;
 
-    /* Construct the environment passed to the builder. */
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    Path homeDir = "/homeless-shelter";
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
+    /* If `build-users-group' is not empty, then we have to build as
+       one of the members of that group. */
+    if (settings.buildUsersGroup != "" && getuid() == 0) {
+        buildUser.acquire();
 
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+        /* Make sure that no other processes are executing under this
+           uid. */
+        buildUser.kill();
+    }
 
     /* Create a temporary directory where the build will take
        place. */
@@ -1737,86 +1729,18 @@ void DerivationGoal::startBuilder()
     /* In a sandbox, for determinism, always use the same temporary
        directory. */
     tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
+    chownToBuilder(tmpDir);
 
-    /* Add all bindings specified in the derivation via the
-       environments, except those listed in the passAsFile
-       attribute. Those are passed as file names pointing to
-       temporary files containing the contents. */
-    PathSet filesToChown;
-    StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
-    int fileNr = 0;
-    for (auto & i : drv->env) {
-        if (passAsFile.find(i.first) == passAsFile.end()) {
-            env[i.first] = i.second;
-        } else {
-            string fn = ".attr-" + std::to_string(fileNr++);
-            Path p = tmpDir + "/" + fn;
-            writeFile(p, i.second);
-            filesToChown.insert(p);
-            env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
-        }
-    }
-
-    /* For convenience, set an environment pointing to the top build
-       directory. */
-    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
-    /* Also set TMPDIR and variants to point to this directory. */
-    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
-    /* Explicitly set PWD to prevent problems with chroot builds.  In
-       particular, dietlibc cannot figure out the cwd because the
-       inode of the current directory doesn't appear in .. (because
-       getdents returns the inode of the mount point). */
-    env["PWD"] = tmpDirInSandbox;
-
-    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
-       derivation, tell the builder, so that for instance `fetchurl'
-       can skip checking the output.  On older Nixes, this environment
-       variable won't be set, so `fetchurl' will do the check. */
-    if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
-
-    /* *Only* if this is a fixed-output derivation, propagate the
-       values of the environment variables specified in the
-       `impureEnvVars' attribute to the builder.  This allows for
-       instance environment variables for proxy configuration such as
-       `http_proxy' to be easily passed to downloaders like
-       `fetchurl'.  Passing such environment variables from the caller
-       to the builder is generally impure, but the output of
-       fixed-output derivations is by definition pure (since we
-       already know the cryptographic hash of the output). */
-    if (fixedOutput) {
-        Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
-        for (auto & i : varNames) env[i] = getEnv(i);
-    }
+    /* Construct the environment passed to the builder. */
+    initEnv();
 
     /* Substitute output placeholders with the actual output paths. */
     for (auto & output : drv->outputs)
         inputRewrites[hashPlaceholder(output.first)] = output.second.path;
 
-
     /* Handle exportReferencesGraph(), if set. */
     doExportReferencesGraph();
 
-
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-        buildUser.acquire();
-
-        /* Make sure that no other processes are executing under this
-           uid. */
-        buildUser.kill();
-
-        /* Change ownership of the temporary build directory. */
-        filesToChown.insert(tmpDir);
-
-        for (auto & p : filesToChown)
-            if (chown(p.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
-                throw SysError(format("cannot change ownership of ‘%1%’") % p);
-    }
-
-
     if (useChroot) {
 
         string defaultChrootDirs;
@@ -2203,6 +2127,93 @@ void DerivationGoal::startBuilder()
 }
 
 
+void DerivationGoal::initEnv()
+{
+    env.clear();
+
+    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
+       PATH is not set.  We don't want this, so we fill it in with some dummy
+       value. */
+    env["PATH"] = "/path-not-set";
+
+    /* Set HOME to a non-existing path to prevent certain programs from using
+       /etc/passwd (or NIS, or whatever) to locate the home directory (for
+       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
+       if HOME is not set, but they will just assume that the settings file
+       they are looking for does not exist if HOME is set but points to some
+       non-existing path. */
+    env["HOME"] = homeDir;
+
+    /* Tell the builder where the Nix store is.  Usually they
+       shouldn't care, but this is useful for purity checking (e.g.,
+       the compiler or linker might only want to accept paths to files
+       in the store or in the build directory). */
+    env["NIX_STORE"] = worker.store.storeDir;
+
+    /* The maximum number of cores to utilize for parallel building. */
+    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+
+    /* Add all bindings specified in the derivation via the
+       environments, except those listed in the passAsFile
+       attribute. Those are passed as file names pointing to
+       temporary files containing the contents. */
+    StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+    int fileNr = 0;
+    for (auto & i : drv->env) {
+        if (passAsFile.find(i.first) == passAsFile.end()) {
+            env[i.first] = i.second;
+        } else {
+            string fn = ".attr-" + std::to_string(fileNr++);
+            Path p = tmpDir + "/" + fn;
+            writeFile(p, i.second);
+            chownToBuilder(p);
+            env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+        }
+    }
+
+    /* For convenience, set an environment pointing to the top build
+       directory. */
+    env["NIX_BUILD_TOP"] = tmpDirInSandbox;
+
+    /* Also set TMPDIR and variants to point to this directory. */
+    env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
+
+    /* Explicitly set PWD to prevent problems with chroot builds.  In
+       particular, dietlibc cannot figure out the cwd because the
+       inode of the current directory doesn't appear in .. (because
+       getdents returns the inode of the mount point). */
+    env["PWD"] = tmpDirInSandbox;
+
+    /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
+       derivation, tell the builder, so that for instance `fetchurl'
+       can skip checking the output.  On older Nixes, this environment
+       variable won't be set, so `fetchurl' will do the check. */
+    if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
+
+    /* *Only* if this is a fixed-output derivation, propagate the
+       values of the environment variables specified in the
+       `impureEnvVars' attribute to the builder.  This allows for
+       instance environment variables for proxy configuration such as
+       `http_proxy' to be easily passed to downloaders like
+       `fetchurl'.  Passing such environment variables from the caller
+       to the builder is generally impure, but the output of
+       fixed-output derivations is by definition pure (since we
+       already know the cryptographic hash of the output). */
+    if (fixedOutput) {
+        Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
+        for (auto & i : varNames) env[i] = getEnv(i);
+    }
+}
+
+
+void DerivationGoal::chownToBuilder(const Path & path)
+{
+    if (!buildUser.enabled()) return;
+    if (chown(path.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
+        throw SysError(format("cannot change ownership of ‘%1%’") % path);
+}
+
+
 void DerivationGoal::doExportReferencesGraph()
 {
     /* The `exportReferencesGraph' feature allows the references graph

From c0f2f4eeeffd9c62ee2c59b42e6824d297d210f1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 12:45:38 +0100
Subject: [PATCH 0073/2196] UserLock: Make more RAII-ish

---
 src/libstore/build.cc | 91 ++++++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 53 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1d039d33849..e0859269dce 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -440,16 +440,14 @@ class UserLock
     AutoCloseFD fdUserLock;
 
     string user;
-    uid_t uid = 0;
-    gid_t gid = 0;
+    uid_t uid;
+    gid_t gid;
     std::vector<gid_t> supplementaryGIDs;
 
 public:
+    UserLock();
     ~UserLock();
 
-    void acquire();
-    void release();
-
     void kill();
 
     string getUser() { return user; }
@@ -465,16 +463,8 @@ class UserLock
 PathSet UserLock::lockedPaths;
 
 
-UserLock::~UserLock()
-{
-    release();
-}
-
-
-void UserLock::acquire()
+UserLock::UserLock()
 {
-    assert(uid == 0);
-
     assert(settings.buildUsersGroup != "");
 
     /* Get the members of the build-users-group. */
@@ -551,20 +541,15 @@ void UserLock::acquire()
 }
 
 
-void UserLock::release()
+UserLock::~UserLock()
 {
-    if (uid == 0) return;
-    fdUserLock = -1; /* releases lock */
-    assert(lockedPaths.find(fnUserLock) != lockedPaths.end());
+    assert(lockedPaths.count(fnUserLock));
     lockedPaths.erase(fnUserLock);
-    fnUserLock = "";
-    uid = 0;
 }
 
 
 void UserLock::kill()
 {
-    assert(enabled());
     killUser(uid);
 }
 
@@ -720,7 +705,7 @@ class DerivationGoal : public Goal
     PathSet missingPaths;
 
     /* User selected for running the builder. */
-    UserLock buildUser;
+    std::unique_ptr<UserLock> buildUser;
 
     /* The process ID of the builder. */
     Pid pid;
@@ -966,7 +951,7 @@ void DerivationGoal::killChild()
     if (pid != -1) {
         worker.childTerminated(this);
 
-        if (buildUser.enabled()) {
+        if (buildUser) {
             /* If we're using a build user, then there is a tricky
                race condition: if we kill the build user before the
                child has done its setuid() to the build user uid, then
@@ -974,7 +959,7 @@ void DerivationGoal::killChild()
                pid.wait().  So also send a conventional kill to the
                child. */
             ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser.kill();
+            buildUser->kill();
             pid.wait();
         } else
             pid.kill();
@@ -1395,7 +1380,7 @@ void DerivationGoal::tryToBuild()
     } catch (BuildError & e) {
         printError(e.msg());
         outputLocks.unlock();
-        buildUser.release();
+        buildUser.reset();
         worker.permanentFailure = true;
         done(BuildResult::InputRejected, e.msg());
         return;
@@ -1429,6 +1414,11 @@ void DerivationGoal::buildDone()
 {
     trace("build done");
 
+    /* Release the build user at the end of this function. We don't do
+       it right away because we don't want another build grabbing this
+       uid and then messing around with our output. */
+    Finally releaseBuildUser([&]() { buildUser.reset(); });
+
     /* Since we got an EOF on the logger pipe, the builder is presumed
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe, so just to be sure,
@@ -1458,7 +1448,7 @@ void DerivationGoal::buildDone()
        malicious user from leaving behind a process that keeps files
        open and modifies them after they have been chown'ed to
        root. */
-    if (buildUser.enabled()) buildUser.kill();
+    if (buildUser) buildUser->kill();
 
     bool diskFull = false;
 
@@ -1528,7 +1518,6 @@ void DerivationGoal::buildDone()
         /* Repeat the build if necessary. */
         if (curRound++ < nrRounds) {
             outputLocks.unlock();
-            buildUser.release();
             state = &DerivationGoal::tryToBuild;
             worker.wakeUp(shared_from_this());
             return;
@@ -1545,7 +1534,6 @@ void DerivationGoal::buildDone()
         if (!hook)
             printError(e.msg());
         outputLocks.unlock();
-        buildUser.release();
 
         BuildResult::Status st = BuildResult::MiscFailure;
 
@@ -1567,9 +1555,6 @@ void DerivationGoal::buildDone()
         return;
     }
 
-    /* Release the build user, if applicable. */
-    buildUser.release();
-
     done(BuildResult::Built);
 }
 
@@ -1714,11 +1699,11 @@ void DerivationGoal::startBuilder()
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
     if (settings.buildUsersGroup != "" && getuid() == 0) {
-        buildUser.acquire();
+        buildUser = std::make_unique<UserLock>();
 
         /* Make sure that no other processes are executing under this
            uid. */
-        buildUser.kill();
+        buildUser->kill();
     }
 
     /* Create a temporary directory where the build will take
@@ -1831,7 +1816,7 @@ void DerivationGoal::startBuilder()
         if (mkdir(chrootRootDir.c_str(), 0750) == -1)
             throw SysError(format("cannot create ‘%1%’") % chrootRootDir);
 
-        if (buildUser.enabled() && chown(chrootRootDir.c_str(), 0, buildUser.getGID()) == -1)
+        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
             throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need
@@ -1875,7 +1860,7 @@ void DerivationGoal::startBuilder()
         createDirs(chrootStoreDir);
         chmod_(chrootStoreDir, 01775);
 
-        if (buildUser.enabled() && chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1)
+        if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
             throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir);
 
         for (auto & i : inputPaths) {
@@ -2085,8 +2070,8 @@ void DerivationGoal::startBuilder()
         /* Set the UID/GID mapping of the builder's user namespace
            such that the sandbox user maps to the build user, or to
            the calling user (if build users are disabled). */
-        uid_t hostUid = buildUser.enabled() ? buildUser.getUID() : getuid();
-        uid_t hostGid = buildUser.enabled() ? buildUser.getGID() : getgid();
+        uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
+        uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
 
         writeFile("/proc/" + std::to_string(pid) + "/uid_map",
             (format("%d %d 1") % sandboxUid % hostUid).str());
@@ -2104,7 +2089,7 @@ void DerivationGoal::startBuilder()
     } else
 #endif
     {
-        options.allowVfork = !buildUser.enabled() && !drv->isBuiltin();
+        options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
         }, options);
@@ -2208,8 +2193,8 @@ void DerivationGoal::initEnv()
 
 void DerivationGoal::chownToBuilder(const Path & path)
 {
-    if (!buildUser.enabled()) return;
-    if (chown(path.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
+    if (!buildUser) return;
+    if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
         throw SysError(format("cannot change ownership of ‘%1%’") % path);
 }
 
@@ -2497,22 +2482,22 @@ void DerivationGoal::runChild()
            descriptors except std*, so that's safe.  Also note that
            setuid() when run as root sets the real, effective and
            saved UIDs. */
-        if (setUser && buildUser.enabled()) {
+        if (setUser && buildUser) {
             /* Preserve supplementary groups of the build user, to allow
                admins to specify groups such as "kvm".  */
-            if (!buildUser.getSupplementaryGIDs().empty() &&
-                setgroups(buildUser.getSupplementaryGIDs().size(),
-                          buildUser.getSupplementaryGIDs().data()) == -1)
+            if (!buildUser->getSupplementaryGIDs().empty() &&
+                setgroups(buildUser->getSupplementaryGIDs().size(),
+                          buildUser->getSupplementaryGIDs().data()) == -1)
                 throw SysError("cannot set supplementary groups of build user");
 
-            if (setgid(buildUser.getGID()) == -1 ||
-                getgid() != buildUser.getGID() ||
-                getegid() != buildUser.getGID())
+            if (setgid(buildUser->getGID()) == -1 ||
+                getgid() != buildUser->getGID() ||
+                getegid() != buildUser->getGID())
                 throw SysError("setgid failed");
 
-            if (setuid(buildUser.getUID()) == -1 ||
-                getuid() != buildUser.getUID() ||
-                geteuid() != buildUser.getUID())
+            if (setuid(buildUser->getUID()) == -1 ||
+                getuid() != buildUser->getUID() ||
+                geteuid() != buildUser->getUID())
                 throw SysError("setuid failed");
         }
 
@@ -2752,7 +2737,7 @@ void DerivationGoal::registerOutputs()
            build.  Also, the output should be owned by the build
            user. */
         if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
-            (buildUser.enabled() && st.st_uid != buildUser.getUID()))
+            (buildUser && st.st_uid != buildUser->getUID()))
             throw BuildError(format("suspicious ownership or permission on ‘%1%’; rejecting this build output") % path);
 #endif
 
@@ -2764,7 +2749,7 @@ void DerivationGoal::registerOutputs()
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
                something like that. */
-            canonicalisePathMetaData(actualPath, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen);
+            canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
 
             /* FIXME: this is in-memory. */
             StringSink sink;
@@ -2822,7 +2807,7 @@ void DerivationGoal::registerOutputs()
         /* Get rid of all weird permissions.  This also checks that
            all files are owned by the build user, if applicable. */
         canonicalisePathMetaData(actualPath,
-            buildUser.enabled() && !rewritten ? buildUser.getUID() : -1, inodesSeen);
+            buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
 
         /* For this output path, find the references to other paths
            contained in it.  Compute the SHA-256 NAR hash at the same

From a55f589720e6499ed8ca1e3dd63ae18c52782150 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 12:51:35 +0100
Subject: [PATCH 0074/2196] openLockFile: Return an AutoCloseFD

---
 src/build-remote/build-remote.cc | 4 ++--
 src/libstore/pathlocks.cc        | 4 ++--
 src/libstore/pathlocks.hh        | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 1daf0b80ba7..acbd308f84e 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -109,7 +109,7 @@ static std::vector<machine> read_conf()
 
 static string currentLoad;
 
-static int openSlotLock(const machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const machine & m, unsigned long long slot)
 {
     std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out);
     fn_stream << "/";
@@ -187,7 +187,7 @@ int main (int argc, char * * argv)
                         AutoCloseFD free;
                         unsigned long long load = 0;
                         for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
-                            AutoCloseFD slotLock = openSlotLock(m, slot);
+                            auto slotLock = openSlotLock(m, slot);
                             if (lockFile(slotLock.get(), ltWrite, false)) {
                                 if (!free) {
                                     free = std::move(slotLock);
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 620c9a6b752..55973799279 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -13,7 +13,7 @@
 namespace nix {
 
 
-int openLockFile(const Path & path, bool create)
+AutoCloseFD openLockFile(const Path & path, bool create)
 {
     AutoCloseFD fd;
 
@@ -21,7 +21,7 @@ int openLockFile(const Path & path, bool create)
     if (!fd && (create || errno != ENOENT))
         throw SysError(format("opening lock file ‘%1%’") % path);
 
-    return fd.release();
+    return fd;
 }
 
 
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 40103c393f6..2a7de611446 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "types.hh"
+#include "util.hh"
 
 
 namespace nix {
@@ -9,7 +9,7 @@ namespace nix {
 /* Open (possibly create) a lock file and return the file descriptor.
    -1 is returned if create is false and the lock could not be opened
    because it doesn't exist.  Any other error throws an exception. */
-int openLockFile(const Path & path, bool create);
+AutoCloseFD openLockFile(const Path & path, bool create);
 
 /* Delete an open lock file. */
 void deleteLockFile(const Path & path, int fd);
@@ -19,7 +19,7 @@ enum LockType { ltRead, ltWrite, ltNone };
 bool lockFile(int fd, LockType lockType, bool wait);
 
 
-class PathLocks 
+class PathLocks
 {
 private:
     typedef std::pair<int, Path> FDPair;

From 951357e5fb4cd0804e729866f204b635add926a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 13:00:38 +0100
Subject: [PATCH 0075/2196] UserLock: Fix multi-threaded access to a global
 variable

---
 src/libstore/build.cc | 70 ++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 30 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e0859269dce..6250de13cbb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -434,7 +434,7 @@ class UserLock
        close that file again (without closing the original file
        descriptor), we lose the lock.  So we have to be *very* careful
        not to open a lock file on which we are holding a lock. */
-    static PathSet lockedPaths; /* !!! not thread-safe */
+    static Sync<PathSet> lockedPaths_;
 
     Path fnUserLock;
     AutoCloseFD fdUserLock;
@@ -460,7 +460,7 @@ class UserLock
 };
 
 
-PathSet UserLock::lockedPaths;
+Sync<PathSet> UserLock::lockedPaths_;
 
 
 UserLock::UserLock()
@@ -499,39 +499,48 @@ UserLock::UserLock()
 
         fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
 
-        if (lockedPaths.find(fnUserLock) != lockedPaths.end())
-            /* We already have a lock on this one. */
-            continue;
+        {
+            auto lockedPaths(lockedPaths_.lock());
+            if (lockedPaths->count(fnUserLock))
+                /* We already have a lock on this one. */
+                continue;
+            lockedPaths->insert(fnUserLock);
+        }
+
+        try {
 
-        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-        if (!fd)
-            throw SysError(format("opening user lock ‘%1%’") % fnUserLock);
+            AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+            if (!fd)
+                throw SysError(format("opening user lock ‘%1%’") % fnUserLock);
 
-        if (lockFile(fd.get(), ltWrite, false)) {
-            fdUserLock = std::move(fd);
-            lockedPaths.insert(fnUserLock);
-            user = i;
-            uid = pw->pw_uid;
+            if (lockFile(fd.get(), ltWrite, false)) {
+                fdUserLock = std::move(fd);
+                user = i;
+                uid = pw->pw_uid;
 
-            /* Sanity check... */
-            if (uid == getuid() || uid == geteuid())
-                throw Error(format("the Nix user should not be a member of ‘%1%’")
-                    % settings.buildUsersGroup);
+                /* Sanity check... */
+                if (uid == getuid() || uid == geteuid())
+                    throw Error(format("the Nix user should not be a member of ‘%1%’")
+                        % settings.buildUsersGroup);
 
 #if __linux__
-            /* Get the list of supplementary groups of this build user.  This
-               is usually either empty or contains a group such as "kvm".  */
-            supplementaryGIDs.resize(10);
-            int ngroups = supplementaryGIDs.size();
-            int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                supplementaryGIDs.data(), &ngroups);
-            if (err == -1)
-                throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name);
-
-            supplementaryGIDs.resize(ngroups);
+                /* Get the list of supplementary groups of this build user.  This
+                   is usually either empty or contains a group such as "kvm".  */
+                supplementaryGIDs.resize(10);
+                int ngroups = supplementaryGIDs.size();
+                int err = getgrouplist(pw->pw_name, pw->pw_gid,
+                    supplementaryGIDs.data(), &ngroups);
+                if (err == -1)
+                    throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name);
+
+                supplementaryGIDs.resize(ngroups);
 #endif
 
-            return;
+                return;
+            }
+
+        } catch (...) {
+            lockedPaths_.lock()->erase(fnUserLock);
         }
     }
 
@@ -543,8 +552,9 @@ UserLock::UserLock()
 
 UserLock::~UserLock()
 {
-    assert(lockedPaths.count(fnUserLock));
-    lockedPaths.erase(fnUserLock);
+    auto lockedPaths(lockedPaths_.lock());
+    assert(lockedPaths->count(fnUserLock));
+    lockedPaths->erase(fnUserLock);
 }
 
 

From 83ae6503e87c7f5237fb0f1602793c126436495a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 13:37:02 +0100
Subject: [PATCH 0076/2196] Fix interrupt handling

---
 src/libmain/shared.cc     |  9 +++++++++
 src/libutil/monitor-fd.hh |  3 +--
 src/libutil/util.cc       | 28 ++++++++++++++++------------
 src/libutil/util.hh       | 16 ++++++++++++++++
 4 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 12f083c7f79..d564e03853e 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -97,6 +97,9 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
 }
 
 
+static void sigHandler(int signo) { }
+
+
 void initNix()
 {
     /* Turn on buffering for cerr. */
@@ -130,6 +133,10 @@ void initNix()
     if (sigaction(SIGCHLD, &act, 0))
         throw SysError("resetting SIGCHLD");
 
+    /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */
+    act.sa_handler = sigHandler;
+    if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
+
     /* Register a SIGSEGV handler to detect stack overflows. */
     detectStackOverflow();
 
@@ -253,6 +260,8 @@ void showManPage(const string & name)
 
 int handleExceptions(const string & programName, std::function<void()> fun)
 {
+    ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
+
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh
index 6f01ccd91a4..e0ec66c0180 100644
--- a/src/libutil/monitor-fd.hh
+++ b/src/libutil/monitor-fd.hh
@@ -27,8 +27,7 @@ public:
             fds[0].events = 0;
             if (poll(fds, 1, -1) == -1) abort(); // can't happen
             assert(fds[0].revents & POLLHUP);
-            /* We got POLLHUP, so send an INT signal to the main thread. */
-            kill(getpid(), SIGINT);
+            triggerInterrupt();
         });
     };
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 52608ac2a01..ca4edc2cd6c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1197,18 +1197,22 @@ static void signalHandlerThread(sigset_t set)
         int signal = 0;
         sigwait(&set, &signal);
 
-        if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) {
-            _isInterrupted = 1;
-
-            {
-                auto interruptCallbacks(_interruptCallbacks.lock());
-                for (auto & callback : *interruptCallbacks) {
-                    try {
-                        callback();
-                    } catch (...) {
-                        ignoreException();
-                    }
-                }
+        if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
+            triggerInterrupt();
+    }
+}
+
+void triggerInterrupt()
+{
+    _isInterrupted = 1;
+
+    {
+        auto interruptCallbacks(_interruptCallbacks.lock());
+        for (auto & callback : *interruptCallbacks) {
+            try {
+                callback();
+            } catch (...) {
+                ignoreException();
             }
         }
     }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b68d48582b3..07141ffed6b 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -433,5 +433,21 @@ struct InterruptCallback
 std::unique_ptr<InterruptCallback> createInterruptCallback(
     std::function<void()> callback);
 
+void triggerInterrupt();
+
+/* A RAII class that causes the current thread to receive SIGUSR1 when
+   the signal handler thread receives SIGINT. That is, this allows
+   SIGINT to be multiplexed to multiple threads. */
+struct ReceiveInterrupts
+{
+    pthread_t target;
+    std::unique_ptr<InterruptCallback> callback;
+
+    ReceiveInterrupts()
+        : target(pthread_self())
+        , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); }))
+    { }
+};
+
 
 }

From b1f001538e41a4f28e315baeede93a8fe70d6d62 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 13:42:07 +0100
Subject: [PATCH 0077/2196] Fix assertion failure when a path is locked

Fixes:

  nix-store: src/libstore/build.cc:3649: void nix::Worker::run(const Goals&): Assertion `!awake.empty()' failed.
---
 src/libstore/pathlocks.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 55973799279..bf7ad3d2185 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -136,6 +136,7 @@ bool PathLocks::lockPaths(const PathSet & _paths,
                         /* Failed to lock this path; release all other
                            locks. */
                         unlock();
+                        lockedPaths_.lock()->erase(lockPath);
                         return false;
                     }
                 }

From 54801ed6ad4e0ea8faa67b0b4ba10debeb824d3b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 16:06:50 +0100
Subject: [PATCH 0078/2196] Bindings: Add a method for iterating in
 lexicographically sorted order

---
 src/libexpr/attr-set.hh | 13 +++++++++++++
 src/libexpr/eval.cc     | 10 +++-------
 src/libexpr/get-drvs.cc | 24 +++++++++---------------
 src/libexpr/primops.cc  |  9 +++------
 4 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 7cf6a9c5808..e1fc2bf6d79 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -75,6 +75,19 @@ public:
 
     size_t capacity() { return capacity_; }
 
+    /* Returns the attributes in lexicographically sorted order. */
+    std::vector<const Attr *> lexicographicOrder() const
+    {
+        std::vector<const Attr *> res;
+        res.reserve(size_);
+        for (size_t n = 0; n < size_; n++)
+            res.emplace_back(&attrs[n]);
+        std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) {
+            return (string) a->name < (string) b->name;
+        });
+        return res;
+    }
+
     friend class EvalState;
 };
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 64f3874db61..5f9f5bac1ed 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -91,13 +91,9 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
         break;
     case tAttrs: {
         str << "{ ";
-        typedef std::map<string, Value *> Sorted;
-        Sorted sorted;
-        for (auto & i : *v.attrs)
-            sorted[i.name] = i.value;
-        for (auto & i : sorted) {
-            str << i.first << " = ";
-            printValue(str, active, *i.second);
+        for (auto & i : v.attrs->lexicographicOrder()) {
+            str << i->name << " = ";
+            printValue(str, active, *i->value);
             str << "; ";
         }
         str << "}";
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index dc5def911ca..5342739c53c 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -284,25 +284,19 @@ static void getDerivations(EvalState & state, Value & vIn,
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        typedef std::map<string, Symbol> SortedSymbols;
-        SortedSymbols attrs;
-        for (auto & i : *v.attrs)
-            attrs.insert(std::pair<string, Symbol>(i.name, i.name));
-
-        for (auto & i : attrs) {
-            Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first);
-            string pathPrefix2 = addToPath(pathPrefix, i.first);
-            Value & v2(*v.attrs->find(i.second)->value);
+        for (auto & i : v.attrs->lexicographicOrder()) {
+            Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name);
+            string pathPrefix2 = addToPath(pathPrefix, i->name);
             if (combineChannels)
-                getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-            else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
+                getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
                 /* If the value of this attribute is itself a set,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
-                if (v2.type == tAttrs) {
-                    Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations"));
-                    if (j != v2.attrs->end() && state.forceBool(*j->value, *j->pos))
-                        getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                if (i->value->type == tAttrs) {
+                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
+                    if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
+                        getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }
             }
         }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 59623874c3f..d8dbedcaf89 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -998,12 +998,9 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
 
     state.mkList(v, args[0]->attrs->size());
 
-    unsigned int n = 0;
-    for (auto & i : *args[0]->attrs)
-        mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
-
-    std::sort(v.listElems(), v.listElems() + n,
-        [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
+    size_t n = 0;
+    for (auto & i : args[0]->attrs->lexicographicOrder())
+        mkString(*(v.listElems()[n++] = state.allocValue()), i->name);
 }
 
 

From 6de33a9c675b187437a2e1abbcb290981a89ecb1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Jan 2017 16:42:07 +0100
Subject: [PATCH 0079/2196] Add support for passing structured data to builders

Previously, all derivation attributes had to be coerced into strings
so that they could be passed via the environment. This is lossy
(e.g. lists get flattened, necessitating configureFlags
vs. configureFlagsArray, of which the latter cannot be specified as an
attribute), doesn't support attribute sets at all, and has size
limitations (necessitating hacks like passAsFile).

This patch adds a new mode for passing attributes to builders, namely
encoded as a JSON file ".attrs.json" in the current directory of the
builder. This mode is activated via the special attribute

  __structuredAttrs = true;

(The idea is that one day we can set this in stdenv.mkDerivation.)

For example,

  stdenv.mkDerivation {
    __structuredAttrs = true;
    name = "foo";
    buildInputs = [ pkgs.hello pkgs.cowsay ];
    doCheck = true;
    hardening.format = false;
  }

results in a ".attrs.json" file containing (sans the indentation):

  {
    "buildInputs": [],
    "builder": "/nix/store/ygl61ycpr2vjqrx775l1r2mw1g2rb754-bash-4.3-p48/bin/bash",
    "configureFlags": [
      "--with-foo",
      "--with-bar=1 2"
    ],
    "doCheck": true,
    "hardening": {
      "format": false
    },
    "name": "foo",
    "nativeBuildInputs": [
      "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
      "/nix/store/4jnvjin0r6wp6cv1hdm5jbkx3vinlcvk-cowsay-3.03"
    ],
    "propagatedBuildInputs": [],
    "propagatedNativeBuildInputs": [],
    "stdenv": "/nix/store/f3hw3p8armnzy6xhd4h8s7anfjrs15n2-stdenv",
    "system": "x86_64-linux"
  }

"passAsFile" is ignored in this mode because it's not needed - large
strings are included directly in the JSON representation.

It is up to the builder to do something with the JSON
representation. For example, in bash-based builders, lists/attrsets of
string values could be mapped to bash (associative) arrays.
---
 src/libexpr/eval.cc    |   2 +
 src/libexpr/eval.hh    |   2 +-
 src/libexpr/primops.cc | 127 ++++++++++++++++++++++++++++-------------
 src/libstore/build.cc  |  57 ++++++++++++------
 4 files changed, 130 insertions(+), 58 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5f9f5bac1ed..d418ab4e43a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -291,6 +291,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sToString(symbols.create("__toString"))
     , sRight(symbols.create("right"))
     , sWrong(symbols.create("wrong"))
+    , sStructuredAttrs(symbols.create("__structuredAttrs"))
+    , sBuilder(symbols.create("builder"))
     , store(store)
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 195cb0db3ac..46d5a1cc866 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -68,7 +68,7 @@ public:
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong;
+        sRight, sWrong, sStructuredAttrs, sBuilder;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d8dbedcaf89..5a570cefb2f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -8,6 +8,7 @@
 #include "names.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "json.hh"
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
@@ -474,6 +475,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         throw;
     }
 
+    /* Check whether attributes should be passed as a JSON file. */
+    std::ostringstream jsonBuf;
+    std::unique_ptr<JSONObject> jsonObject;
+    attr = args[0]->attrs->find(state.sStructuredAttrs);
+    if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
+        jsonObject = std::make_unique<JSONObject>(jsonBuf);
+
     /* Check whether null attributes should be ignored. */
     bool ignoreNulls = false;
     attr = args[0]->attrs->find(state.sIgnoreNulls);
@@ -491,24 +499,48 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     StringSet outputs;
     outputs.insert("out");
 
-    for (auto & i : *args[0]->attrs) {
-        if (i.name == state.sIgnoreNulls) continue;
-        string key = i.name;
+    for (auto & i : args[0]->attrs->lexicographicOrder()) {
+        if (i->name == state.sIgnoreNulls) continue;
+        string key = i->name;
         Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key);
 
+        auto handleHashMode = [&](const std::string & s) {
+            if (s == "recursive") outputHashRecursive = true;
+            else if (s == "flat") outputHashRecursive = false;
+            else throw EvalError("invalid value ‘%s’ for ‘outputHashMode’ attribute, at %s", s, posDrvName);
+        };
+
+        auto handleOutputs = [&](const Strings & ss) {
+            outputs.clear();
+            for (auto & j : ss) {
+                if (outputs.find(j) != outputs.end())
+                    throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName);
+                /* !!! Check whether j is a valid attribute
+                   name. */
+                /* Derivations cannot be named ‘drv’, because
+                   then we'd have an attribute ‘drvPath’ in
+                   the resulting set. */
+                if (j == "drv")
+                    throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName);
+                outputs.insert(j);
+            }
+            if (outputs.empty())
+                throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName);
+        };
+
         try {
 
             if (ignoreNulls) {
-                state.forceValue(*i.value);
-                if (i.value->type == tNull) continue;
+                state.forceValue(*i->value);
+                if (i->value->type == tNull) continue;
             }
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
             if (key == "args") {
-                state.forceList(*i.value, pos);
-                for (unsigned int n = 0; n < i.value->listSize(); ++n) {
-                    string s = state.coerceToString(posDrvName, *i.value->listElems()[n], context, true);
+                state.forceList(*i->value, pos);
+                for (unsigned int n = 0; n < i->value->listSize(); ++n) {
+                    string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -516,39 +548,51 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             /* All other attributes are passed to the builder through
                the environment. */
             else {
-                string s = state.coerceToString(posDrvName, *i.value, context, true);
-                drv.env[key] = s;
-                if (key == "builder") drv.builder = s;
-                else if (i.name == state.sSystem) drv.platform = s;
-                else if (i.name == state.sName) {
-                    drvName = s;
-                    printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName);
-                }
-                else if (key == "outputHash") outputHash = s;
-                else if (key == "outputHashAlgo") outputHashAlgo = s;
-                else if (key == "outputHashMode") {
-                    if (s == "recursive") outputHashRecursive = true;
-                    else if (s == "flat") outputHashRecursive = false;
-                    else throw EvalError(format("invalid value ‘%1%’ for ‘outputHashMode’ attribute, at %2%") % s % posDrvName);
-                }
-                else if (key == "outputs") {
-                    Strings tmp = tokenizeString<Strings>(s);
-                    outputs.clear();
-                    for (auto & j : tmp) {
-                        if (outputs.find(j) != outputs.end())
-                            throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName);
-                        /* !!! Check whether j is a valid attribute
-                           name. */
-                        /* Derivations cannot be named ‘drv’, because
-                           then we'd have an attribute ‘drvPath’ in
-                           the resulting set. */
-                        if (j == "drv")
-                            throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName);
-                        outputs.insert(j);
+
+                if (jsonObject) {
+
+                    if (i->name == state.sStructuredAttrs) continue;
+
+                    auto placeholder(jsonObject->placeholder(key));
+                    printValueAsJSON(state, true, *i->value, placeholder, context);
+
+                    if (i->name == state.sBuilder)
+                        drv.builder = state.forceString(*i->value, context, posDrvName);
+                    else if (i->name == state.sSystem)
+                        drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sName)
+                        drvName = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (key == "outputHash")
+                        outputHash = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (key == "outputHashAlgo")
+                        outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (key == "outputHashMode")
+                        handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
+                    else if (key == "outputs") {
+                        /* Require ‘outputs’ to be a list of strings. */
+                        state.forceList(*i->value, posDrvName);
+                        Strings ss;
+                        for (unsigned int n = 0; n < i->value->listSize(); ++n)
+                            ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
+                        handleOutputs(ss);
+                    }
+
+                } else {
+                    auto s = state.coerceToString(posDrvName, *i->value, context, true);
+                    drv.env.emplace(key, s);
+                    if (i->name == state.sBuilder) drv.builder = s;
+                    else if (i->name == state.sSystem) drv.platform = s;
+                    else if (i->name == state.sName) {
+                        drvName = s;
+                        printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName);
                     }
-                    if (outputs.empty())
-                        throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName);
+                    else if (key == "outputHash") outputHash = s;
+                    else if (key == "outputHashAlgo") outputHashAlgo = s;
+                    else if (key == "outputHashMode") handleHashMode(s);
+                    else if (key == "outputs")
+                        handleOutputs(tokenizeString<Strings>(s));
                 }
+
             }
 
         } catch (Error & e) {
@@ -558,6 +602,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         }
     }
 
+    if (jsonObject) {
+        jsonObject.reset();
+        drv.env.emplace("__json", jsonBuf.str());
+    }
+
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6250de13cbb..d76c8d1727f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -865,6 +865,9 @@ class DerivationGoal : public Goal
     /* Fill in the environment for the builder. */
     void initEnv();
 
+    /* Write a JSON file containing the derivation attributes. */
+    void writeStructuredAttrs();
+
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
 
@@ -1726,13 +1729,15 @@ void DerivationGoal::startBuilder()
     tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
     chownToBuilder(tmpDir);
 
-    /* Construct the environment passed to the builder. */
-    initEnv();
-
     /* Substitute output placeholders with the actual output paths. */
     for (auto & output : drv->outputs)
         inputRewrites[hashPlaceholder(output.first)] = output.second.path;
 
+    /* Construct the environment passed to the builder. */
+    initEnv();
+
+    writeStructuredAttrs();
+
     /* Handle exportReferencesGraph(), if set. */
     doExportReferencesGraph();
 
@@ -2148,22 +2153,29 @@ void DerivationGoal::initEnv()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
-    /* Add all bindings specified in the derivation via the
-       environments, except those listed in the passAsFile
-       attribute. Those are passed as file names pointing to
-       temporary files containing the contents. */
-    StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
-    int fileNr = 0;
-    for (auto & i : drv->env) {
-        if (passAsFile.find(i.first) == passAsFile.end()) {
-            env[i.first] = i.second;
-        } else {
-            string fn = ".attr-" + std::to_string(fileNr++);
-            Path p = tmpDir + "/" + fn;
-            writeFile(p, i.second);
-            chownToBuilder(p);
-            env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+    /* In non-structured mode, add all bindings specified in the
+       derivation via the environments, except those listed in the
+       passAsFile attribute. Those are passed as file names pointing
+       to temporary files containing the contents. Note that
+       passAsFile is ignored in structure mode because it's not
+       needed (attributes are not passed through the environment, so
+       there is no size constraint). */
+    if (!drv->env.count("__json")) {
+
+        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+        int fileNr = 0;
+        for (auto & i : drv->env) {
+            if (passAsFile.find(i.first) == passAsFile.end()) {
+                env[i.first] = i.second;
+            } else {
+                string fn = ".attr-" + std::to_string(fileNr++);
+                Path p = tmpDir + "/" + fn;
+                writeFile(p, i.second);
+                chownToBuilder(p);
+                env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+            }
         }
+
     }
 
     /* For convenience, set an environment pointing to the top build
@@ -2201,6 +2213,15 @@ void DerivationGoal::initEnv()
 }
 
 
+void DerivationGoal::writeStructuredAttrs()
+{
+    auto json = drv->env.find("__json");
+    if (json == drv->env.end()) return;
+
+    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites));
+}
+
+
 void DerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;

From c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 26 Jan 2017 20:36:20 +0100
Subject: [PATCH 0080/2196] exportReferencesGraph: Export more complete info in
 JSON format
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This writes info about every path in the closure in the same format as
‘nix path-info --json’. Thus it also includes NAR hashes and sizes.

Example:

  [
    {
      "path": "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
      "narHash": "sha256:0ckdc4z20kkmpqdilx0wl6cricxv90lh85xpv2qljppcmz6vzcxl",
      "narSize": 197648,
      "references": [
        "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
        "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
      ],
      "closureSize": 20939776
    },
    {
      "path": "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24",
      "narHash": "sha256:1nfn3m3p98y1c0kd0brp80dn9n5mycwgrk183j17rajya0h7gax3",
      "narSize": 20742128,
      "references": [
        "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
      ],
      "closureSize": 20742128
    }
  ]

Fixes #1134.
---
 src/libstore/build.cc     | 12 +++++++-
 src/libstore/store-api.cc | 59 +++++++++++++++++++++++++++++++++++++++
 src/libstore/store-api.hh | 14 ++++++++++
 src/nix/path-info.cc      | 55 ++++--------------------------------
 4 files changed, 90 insertions(+), 50 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d76c8d1727f..7fb5271f4a8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -10,6 +10,7 @@
 #include "builtins.hh"
 #include "finally.hh"
 #include "compression.hh"
+#include "json.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -2273,9 +2274,18 @@ void DerivationGoal::doExportReferencesGraph()
             }
         }
 
-        /* Write closure info to `fileName'. */
+        /* Write closure info to <fileName>. */
         writeFile(tmpDir + "/" + fileName,
             worker.store.makeValidityRegistration(paths, false, false));
+
+        /* Write a more comprehensive JSON serialisation to
+           <fileName>.json. */
+        std::ostringstream str;
+        {
+            JSONPlaceholder jsonRoot(str, true);
+            worker.store.pathInfoToJSON(jsonRoot, paths, false, true);
+        }
+        writeFile(tmpDir + "/" + fileName + ".json", str.str());
     }
 }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8fdd6277155..a42d1183405 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
+#include "json.hh"
 
 #include <future>
 
@@ -439,6 +440,64 @@ string Store::makeValidityRegistration(const PathSet & paths,
 }
 
 
+void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
+    bool includeImpureInfo, bool showClosureSize)
+{
+    auto jsonList = jsonOut.list();
+
+    for (auto storePath : storePaths) {
+        auto info = queryPathInfo(storePath);
+        storePath = info->path;
+
+        auto jsonPath = jsonList.object();
+        jsonPath
+            .attr("path", storePath)
+            .attr("narHash", info->narHash.to_string())
+            .attr("narSize", info->narSize);
+
+        {
+            auto jsonRefs = jsonPath.list("references");
+            for (auto & ref : info->references)
+                jsonRefs.elem(ref);
+        }
+
+        if (info->ca != "")
+            jsonPath.attr("ca", info->ca);
+
+        if (showClosureSize)
+            jsonPath.attr("closureSize", getClosureSize(storePath));
+
+        if (!includeImpureInfo) continue;
+
+        if (info->deriver != "")
+            jsonPath.attr("deriver", info->deriver);
+
+        if (info->registrationTime)
+            jsonPath.attr("registrationTime", info->registrationTime);
+
+        if (info->ultimate)
+            jsonPath.attr("ultimate", info->ultimate);
+
+        if (!info->sigs.empty()) {
+            auto jsonSigs = jsonPath.list("signatures");
+            for (auto & sig : info->sigs)
+                jsonSigs.elem(sig);
+        }
+    }
+}
+
+
+unsigned long long Store::getClosureSize(const Path & storePath)
+{
+    unsigned long long totalSize = 0;
+    PathSet closure;
+    computeFSClosure(storePath, closure, false, false);
+    for (auto & p : closure)
+        totalSize += queryPathInfo(p)->narSize;
+    return totalSize;
+}
+
+
 const Store::Stats & Store::getStats()
 {
     {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ec3bf5a6fd8..3fee999072f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -22,6 +22,7 @@ struct Derivation;
 class FSAccessor;
 class NarInfoDiskCache;
 class Store;
+class JSONPlaceholder;
 
 
 /* Size of the hash part of store paths, in base-32 characters. */
@@ -469,6 +470,19 @@ public:
     string makeValidityRegistration(const PathSet & paths,
         bool showDerivers, bool showHash);
 
+    /* Write a JSON representation of store path metadata, such as the
+       hash and the references. If ‘includeImpureInfo’ is true,
+       variable elements such as the registration time are
+       included. If ‘showClosureSize’ is true, the closure size of
+       each path is included. */
+    void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
+        bool includeImpureInfo, bool showClosureSize);
+
+    /* Return the size of the closure of the specified path, that is,
+       the sum of the size of the NAR serialisation of each path in
+       the closure. */
+    unsigned long long getClosureSize(const Path & storePath);
+
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
     virtual void optimiseStore() = 0;
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index a9b33e1877d..0f9a1125f2e 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -65,55 +65,12 @@ struct CmdPathInfo : StorePathsCommand
         for (auto & storePath : storePaths)
             pathLen = std::max(pathLen, storePath.size());
 
-        auto getClosureSize = [&](const Path & storePath) -> unsigned long long {
-            unsigned long long totalSize = 0;
-            PathSet closure;
-            store->computeFSClosure(storePath, closure, false, false);
-            for (auto & p : closure)
-                totalSize += store->queryPathInfo(p)->narSize;
-            return totalSize;
-        };
-
         if (json) {
-            JSONList jsonRoot(std::cout, true);
-
-            for (auto storePath : storePaths) {
-                auto info = store->queryPathInfo(storePath);
-                storePath = info->path;
-
-                auto jsonPath = jsonRoot.object();
-                jsonPath
-                    .attr("path", storePath)
-                    .attr("narHash", info->narHash.to_string())
-                    .attr("narSize", info->narSize);
-
-                if (showClosureSize)
-                    jsonPath.attr("closureSize", getClosureSize(storePath));
-
-                if (info->deriver != "")
-                    jsonPath.attr("deriver", info->deriver);
-
-                {
-                    auto jsonRefs = jsonPath.list("references");
-                    for (auto & ref : info->references)
-                        jsonRefs.elem(ref);
-                }
-
-                if (info->registrationTime)
-                    jsonPath.attr("registrationTime", info->registrationTime);
-
-                if (info->ultimate)
-                    jsonPath.attr("ultimate", info->ultimate);
-
-                if (info->ca != "")
-                    jsonPath.attr("ca", info->ca);
-
-                if (!info->sigs.empty()) {
-                    auto jsonSigs = jsonPath.list("signatures");
-                    for (auto & sig : info->sigs)
-                        jsonSigs.elem(sig);
-                }
-            }
+            JSONPlaceholder jsonRoot(std::cout, true);
+            store->pathInfoToJSON(jsonRoot,
+                // FIXME: preserve order?
+                PathSet(storePaths.begin(), storePaths.end()),
+                true, showClosureSize);
         }
 
         else {
@@ -128,7 +85,7 @@ struct CmdPathInfo : StorePathsCommand
                     std::cout << '\t' << std::setw(11) << info->narSize;
 
                 if (showClosureSize)
-                    std::cout << '\t' << std::setw(11) << getClosureSize(storePath);
+                    std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath);
 
                 if (showSigs) {
                     std::cout << '\t';

From f57a38b109dbea26239a1cc001ff4b608006af6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Jan 2017 12:57:49 +0100
Subject: [PATCH 0081/2196] Remove unused NARExistence table

---
 src/libstore/nar-info-disk-cache.cc | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index d28ff42c7f2..ff4bd651aa5 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -36,15 +36,6 @@ create table if not exists NARs (
     foreign key (cache) references BinaryCaches(id) on delete cascade
 );
 
-create table if not exists NARExistence (
-    cache            integer not null,
-    storePath        text not null,
-    exist            integer not null,
-    timestamp        integer not null,
-    primary key (cache, storePath),
-    foreign key (cache) references BinaryCaches(id) on delete cascade
-);
-
 )sql";
 
 class NarInfoDiskCacheImpl : public NarInfoDiskCache
@@ -155,7 +146,6 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
         auto queryNAR(state->queryNAR.use()(cache.id)(hashPart));
 
         if (!queryNAR.next())
-            // FIXME: check NARExistence
             return {oUnknown, 0};
 
         if (!queryNAR.getInt(13))

From 211bc7f0e6daa65fe4083334e2411bc441067904 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Jan 2017 13:17:08 +0100
Subject: [PATCH 0082/2196] Implement TTL for binary cache lookups

---
 src/libstore/nar-info-disk-cache.cc | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index ff4bd651aa5..ed2f18ffe82 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -42,8 +42,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 {
 public:
 
-    /* How long negative lookups are valid. */
+    /* How long negative and positive lookups are valid. */
     const int ttlNegative = 3600;
+    const int ttlPositive = 30 * 24 * 3600;
 
     struct Cache
     {
@@ -94,7 +95,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
 
         state->queryNAR.create(state->db,
-            "select * from NARs where cache = ? and hashPart = ?");
+            "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
     }
 
     Cache & getCache(State & state, const std::string & uri)
@@ -143,7 +144,13 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
         auto & cache(getCache(*state, uri));
 
-        auto queryNAR(state->queryNAR.use()(cache.id)(hashPart));
+        auto now = time(0);
+
+        auto queryNAR(state->queryNAR.use()
+            (cache.id)
+            (hashPart)
+            (now - ttlNegative)
+            (now - ttlPositive));
 
         if (!queryNAR.next())
             return {oUnknown, 0};
@@ -153,8 +160,6 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
         auto narInfo = make_ref<NarInfo>();
 
-        // FIXME: implement TTL.
-
         auto namePart = queryNAR.getStr(2);
         narInfo->path = cache.storeDir + "/" +
             hashPart + (namePart.empty() ? "" : "-" + namePart);

From cb1951e746cb04f331b126c746758c96bb9fdc81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Jan 2017 15:19:33 +0100
Subject: [PATCH 0083/2196] Periodically purge binary-cache.sqlite

---
 src/libstore/nar-info-disk-cache.cc | 31 ++++++++++++++++++++++++++++-
 src/libstore/sqlite.hh              |  1 +
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index ed2f18ffe82..13b67b81f35 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -36,6 +36,11 @@ create table if not exists NARs (
     foreign key (cache) references BinaryCaches(id) on delete cascade
 );
 
+create table if not exists LastPurge (
+    dummy            text primary key,
+    value            integer
+);
+
 )sql";
 
 class NarInfoDiskCacheImpl : public NarInfoDiskCache
@@ -46,6 +51,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
     const int ttlNegative = 3600;
     const int ttlPositive = 30 * 24 * 3600;
 
+    /* How often to purge expired entries from the cache. */
+    const int purgeInterval = 24 * 3600;
+
     struct Cache
     {
         int id;
@@ -57,7 +65,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
     struct State
     {
         SQLite db;
-        SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR;
+        SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
         std::map<std::string, Cache> caches;
     };
 
@@ -96,6 +104,27 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
         state->queryNAR.create(state->db,
             "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
+
+        /* Periodically purge expired entries from the database. */
+        auto now = time(0);
+
+        SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
+        auto queryLastPurge_(queryLastPurge.use());
+
+        if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) {
+            SQLiteStmt(state->db,
+                "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
+                .use()
+                (now - ttlNegative)
+                (now - ttlPositive)
+                .exec();
+
+            debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
+
+            SQLiteStmt(state->db,
+                "insert or replace into LastPurge(dummy, value) values ('', ?)")
+                .use()(now).exec();
+        }
     }
 
     Cache & getCache(State & state, const std::string & uri)
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 7c1ed538215..4d347a2e56a 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -31,6 +31,7 @@ struct SQLiteStmt
     sqlite3 * db = 0;
     sqlite3_stmt * stmt = 0;
     SQLiteStmt() { }
+    SQLiteStmt(sqlite3 * db, const std::string & s) { create(db, s); }
     void create(sqlite3 * db, const std::string & s);
     ~SQLiteStmt();
     operator sqlite3_stmt * () { return stmt; }

From 583ff4ec46fe1fa758f0fa4df1d8b37d9192736c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Jan 2017 16:13:22 +0100
Subject: [PATCH 0084/2196] release.nix: Drop nix-shell references

---
 release.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index d825dd58376..ace0f9cc89a 100644
--- a/release.nix
+++ b/release.nix
@@ -20,7 +20,7 @@ let
         name = "nix-tarball";
         version = builtins.readFile ./version;
         versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = if lib.inNixShell then null else nix;
+        src = nix;
         inherit officialRelease;
 
         buildInputs =
@@ -28,7 +28,8 @@ let
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
-          ] ++ lib.optional (!lib.inNixShell) git;
+            git
+          ];
 
         configureFlags = ''
           --with-dbi=${perlPackages.DBI}/${perl.libPrefix}

From 7a65b2470eb53a320749d76746fbf65790183d9d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Feb 2017 13:00:21 +0100
Subject: [PATCH 0085/2196] Restore default signal handling in child processes

In particular, this fixes Ctrl-C in nix-shell sessions.
---
 src/download-via-ssh/download-via-ssh.cc |  1 +
 src/libmain/shared.cc                    | 29 ++++++++----------------
 src/libstore/build.cc                    |  4 ++--
 src/libstore/ssh-store.cc                |  1 +
 src/libutil/util.cc                      | 24 ++++++++++++--------
 src/libutil/util.hh                      |  7 +++---
 src/nix-build/nix-build.cc               |  2 ++
 7 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc
index ff28a60ff4b..4a1ba9a1123 100644
--- a/src/download-via-ssh/download-via-ssh.cc
+++ b/src/download-via-ssh/download-via-ssh.cc
@@ -30,6 +30,7 @@ static std::pair<FdSink, FdSource> connect(const string & conn)
             throw SysError("dupping stdin");
         if (dup2(from.writeSide, STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
+        restoreSignals();
         execlp("ssh", "ssh", "-x", "-T", conn.c_str(), "nix-store --serve", NULL);
         throw SysError("executing ssh");
     });
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d564e03853e..52cb2312826 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -119,15 +119,9 @@ void initNix()
 
     startSignalHandlerThread();
 
-    /* Ignore SIGPIPE. */
+    /* Reset SIGCHLD to its default. */
     struct sigaction act;
     sigemptyset(&act.sa_mask);
-    act.sa_handler = SIG_IGN;
-    act.sa_flags = 0;
-    if (sigaction(SIGPIPE, &act, 0))
-        throw SysError("ignoring SIGPIPE");
-
-    /* Reset SIGCHLD to its default. */
     act.sa_handler = SIG_DFL;
     act.sa_flags = 0;
     if (sigaction(SIGCHLD, &act, 0))
@@ -252,7 +246,7 @@ void printVersion(const string & programName)
 
 void showManPage(const string & name)
 {
-    restoreSIGPIPE();
+    restoreSignals();
     execlp("man", "man", name.c_str(), NULL);
     throw SysError(format("command ‘man %1%’ failed") % name.c_str());
 }
@@ -305,16 +299,6 @@ RunPager::RunPager()
     if (!pager) pager = getenv("PAGER");
     if (pager && ((string) pager == "" || (string) pager == "cat")) return;
 
-    /* Ignore SIGINT. The pager will handle it (and we'll get
-       SIGPIPE). */
-    struct sigaction act;
-    act.sa_handler = SIG_IGN;
-    act.sa_flags = 0;
-    sigemptyset(&act.sa_mask);
-    if (sigaction(SIGINT, &act, 0)) throw SysError("ignoring SIGINT");
-
-    restoreSIGPIPE();
-
     Pipe toPager;
     toPager.create();
 
@@ -323,6 +307,7 @@ RunPager::RunPager()
             throw SysError("dupping stdin");
         if (!getenv("LESS"))
             setenv("LESS", "FRSXMK", 1);
+        restoreSignals();
         if (pager)
             execl("/bin/sh", "sh", "-c", pager, NULL);
         execlp("pager", "pager", NULL);
@@ -331,6 +316,8 @@ RunPager::RunPager()
         throw SysError(format("executing ‘%1%’") % pager);
     });
 
+    pid.setKillSignal(SIGINT);
+
     if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
         throw SysError("dupping stdout");
 }
@@ -345,7 +332,11 @@ RunPager::~RunPager()
             pid.wait();
         }
     } catch (...) {
-        ignoreException();
+        try {
+            pid.kill(true);
+        } catch (...) {
+            ignoreException();
+        }
     }
 }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 7fb5271f4a8..40927c06326 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -400,6 +400,8 @@ void Goal::trace(const format & f)
 /* Common initialisation performed in child processes. */
 static void commonChildInit(Pipe & logPipe)
 {
+    restoreSignals();
+
     /* Put the child in a separate session (and thus a separate
        process group) so that it has no controlling terminal (meaning
        that e.g. ssh cannot open /dev/tty) and it doesn't receive
@@ -2662,8 +2664,6 @@ void DerivationGoal::runChild()
         for (auto & i : drv->args)
             args.push_back(rewriteStrings(i, inputRewrites));
 
-        restoreSIGPIPE();
-
         /* Indicate that we managed to set up the build environment. */
         writeFull(STDERR_FILENO, string("\1\n"));
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 3d01594009a..f5d0a270438 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -91,6 +91,7 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     if ((pid_t) sshMaster == -1) {
         sshMaster = startProcess([&]() {
+            restoreSignals();
             if (key.empty())
                 execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
             else
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index ca4edc2cd6c..6c4c5c969d8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -860,6 +860,8 @@ string runProgram(Path program, bool searchPath, const Strings & args,
         Strings args_(args);
         args_.push_front(program);
 
+        restoreSignals();
+
         if (searchPath)
             execvp(program.c_str(), stringsToCharPtrs(args_).data());
         else
@@ -909,16 +911,6 @@ void closeOnExec(int fd)
 }
 
 
-void restoreSIGPIPE()
-{
-    struct sigaction act;
-    act.sa_handler = SIG_DFL;
-    act.sa_flags = 0;
-    sigemptyset(&act.sa_mask);
-    if (sigaction(SIGPIPE, &act, 0)) throw SysError("resetting SIGPIPE");
-}
-
-
 //////////////////////////////////////////////////////////////////////
 
 
@@ -1218,19 +1210,31 @@ void triggerInterrupt()
     }
 }
 
+static sigset_t savedSignalMask;
+
 void startSignalHandlerThread()
 {
+    if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
+        throw SysError("quering signal mask");
+
     sigset_t set;
     sigemptyset(&set);
     sigaddset(&set, SIGINT);
     sigaddset(&set, SIGTERM);
     sigaddset(&set, SIGHUP);
+    sigaddset(&set, SIGPIPE);
     if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
         throw SysError("blocking signals");
 
     std::thread(signalHandlerThread, set).detach();
 }
 
+void restoreSignals()
+{
+    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+        throw SysError("restoring signals");
+}
+
 /* RAII helper to automatically deregister a callback. */
 struct InterruptCallbackImpl : InterruptCallback
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 07141ffed6b..cfaaf1486e9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -256,10 +256,6 @@ void closeMostFDs(const set<int> & exceptions);
 /* Set the close-on-exec flag for the given file descriptor. */
 void closeOnExec(int fd);
 
-/* Restore default handling of SIGPIPE, otherwise some programs will
-   randomly say "Broken pipe". */
-void restoreSIGPIPE();
-
 
 /* User interruption. */
 
@@ -423,6 +419,9 @@ void callSuccess(
    on the current thread (and thus any threads created by it). */
 void startSignalHandlerThread();
 
+/* Restore default signal handling. */
+void restoreSignals();
+
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 3eb2d2c0b7a..ee030c57b6b 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -452,6 +452,8 @@ int main(int argc, char ** argv)
 
                 auto argPtrs = stringsToCharPtrs(args);
 
+                restoreSignals();
+
                 execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), argPtrs.data());
 
                 throw SysError("executing shell");

From 1351b0df87a0984914769c5dc76489618b3a3fec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Feb 2017 12:20:28 +0100
Subject: [PATCH 0086/2196] exportReferencesGraph: Only export in JSON format
 when in structured mode

This prevents breaking compatibility with builders that read
"closure.*", since they would accidentally pick up the new JSON files.
---
 src/libstore/build.cc | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 40927c06326..5d6fff4e349 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2276,18 +2276,24 @@ void DerivationGoal::doExportReferencesGraph()
             }
         }
 
-        /* Write closure info to <fileName>. */
-        writeFile(tmpDir + "/" + fileName,
-            worker.store.makeValidityRegistration(paths, false, false));
+        if (!drv->env.count("__json")) {
+
+            /* Write closure info to <fileName>. */
+            writeFile(tmpDir + "/" + fileName,
+                worker.store.makeValidityRegistration(paths, false, false));
+
+        } else {
+
+            /* Write a more comprehensive JSON serialisation to
+               <fileName>. */
+            std::ostringstream str;
+            {
+                JSONPlaceholder jsonRoot(str, true);
+                worker.store.pathInfoToJSON(jsonRoot, paths, false, true);
+            }
+            writeFile(tmpDir + "/" + fileName, str.str());
 
-        /* Write a more comprehensive JSON serialisation to
-           <fileName>.json. */
-        std::ostringstream str;
-        {
-            JSONPlaceholder jsonRoot(str, true);
-            worker.store.pathInfoToJSON(jsonRoot, paths, false, true);
         }
-        writeFile(tmpDir + "/" + fileName + ".json", str.str());
     }
 }
 

From e6e74f987f0fa284d220432d426eb965269a97d6 Mon Sep 17 00:00:00 2001
From: Renzo Carbonara <gnuk0001@gmail.com>
Date: Wed, 1 Feb 2017 13:37:34 +0100
Subject: [PATCH 0087/2196] Add netrc-file support

---
 doc/manual/command-ref/conf-file.xml | 15 +++++++++++++++
 src/libstore/download.cc             |  5 +++++
 2 files changed, 20 insertions(+)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 6c0af39ecda..a7d60538c21 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -430,6 +430,21 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>netrc-file</literal></term>
+
+    <listitem><para>If set to an absolute path to a <filename>netrc</filename>
+    file, Nix will use the HTTP authentication credentials in this file when
+    trying to download from a remote host through HTTP or HTTPS. Defaults to
+    <filename>$NIX_CONF_DIR/netrc</filename>.</para>
+
+    <para>The <filename>netrc</filename> file consists of zero or more lines
+    like: <literal>machine <replaceable>my-machine</replaceable> login
+    <replaceable>my-username</replaceable> password
+    <replaceable>my-password</replaceable></literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 074e0ca6642..8f2387b6325 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -230,6 +230,11 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
+            Path netrcFile = settings.get("netrc-file",
+               (format("%1%/%2%") % settings.nixConfDir % "netrc").str());
+            curl_easy_setopt(req, CURLOPT_NETRC_FILE, netrcFile.c_str());
+            curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
+
             result.data = std::make_shared<std::string>();
         }
 

From 27dc76c1a5dbe654465245ff5f6bc22e2c8902da Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 18:49:17 +0100
Subject: [PATCH 0088/2196] Remove build-remote.pl.in

---
 .gitignore                 |   1 -
 local.mk                   |   2 +-
 scripts/build-remote.pl.in | 275 -------------------------------------
 scripts/local.mk           |   2 -
 4 files changed, 1 insertion(+), 279 deletions(-)
 delete mode 100755 scripts/build-remote.pl.in

diff --git a/.gitignore b/.gitignore
index 92f95fe1fcb..4d0ac32b16f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,7 +35,6 @@ Makefile.config
 # /scripts/
 /scripts/nix-profile.sh
 /scripts/nix-copy-closure
-/scripts/build-remote.pl
 /scripts/nix-reduce-build
 /scripts/nix-http-export.cgi
 
diff --git a/local.mk b/local.mk
index 2541f3f3229..eebd7196119 100644
--- a/local.mk
+++ b/local.mk
@@ -10,7 +10,7 @@ clean-files += Makefile.config
 GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr \
   -Wno-unneeded-internal-declaration
 
-$(foreach i, config.h $(call rwildcard, src/lib*, *.hh) src/nix-store/serve-protocol.hh, \
+$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644)))
diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in
deleted file mode 100755
index b5fc629eb49..00000000000
--- a/scripts/build-remote.pl.in
+++ /dev/null
@@ -1,275 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use utf8;
-use Fcntl qw(:DEFAULT :flock);
-use English '-no_match_vars';
-use IO::Handle;
-use Nix::Config;
-use Nix::SSH;
-use Nix::CopyClosure;
-use Nix::Store;
-use Encode;
-no warnings('once');
-
-STDERR->autoflush(1);
-binmode STDERR, ":encoding(utf8)";
-
-my $debug = defined $ENV{NIX_DEBUG_HOOK};
-
-
-# General operation:
-#
-# Try to find a free machine of type $neededSystem.  We do this as
-# follows:
-# - We acquire an exclusive lock on $currentLoad/main-lock.
-# - For each machine $machine of type $neededSystem and for each $slot
-#   less than the maximum load for that machine, we try to get an
-#   exclusive lock on $currentLoad/$machine-$slot (without blocking).
-#   If we get such a lock, we send "accept" to the caller.  Otherwise,
-#   we send "postpone" and exit.
-# - We release the exclusive lock on $currentLoad/main-lock.
-# - We perform the build on $neededSystem.
-# - We release the exclusive lock on $currentLoad/$machine-$slot.
-#
-# The nice thing about this scheme is that if we die prematurely, the
-# locks are released automatically.
-
-
-# Make sure that we don't get any SSH passphrase or host key popups -
-# if there is any problem it should fail, not do something
-# interactive.
-$ENV{"DISPLAY"} = "";
-$ENV{"SSH_ASKPASS"} = "";
-
-
-sub sendReply {
-    my $reply = shift;
-    print STDERR "# $reply\n";
-}
-
-sub all { $_ || return 0 for @_; 1 }
-
-
-# Initialisation.
-my $loadIncreased = 0;
-
-my ($localSystem, $maxSilentTime, $buildTimeout) = @ARGV;
-
-my $currentLoad = $ENV{"NIX_CURRENT_LOAD"} // "/run/nix/current-load";
-my $conf = $ENV{"NIX_REMOTE_SYSTEMS"} // "@sysconfdir@/nix/machines";
-
-
-sub openSlotLock {
-    my ($machine, $slot) = @_;
-    my $slotLockFn = "$currentLoad/" . (join '+', @{$machine->{systemTypes}}) . "-" . $machine->{hostName} . "-$slot";
-    my $slotLock = new IO::Handle;
-    sysopen $slotLock, "$slotLockFn", O_RDWR|O_CREAT, 0600 or die;
-    return $slotLock;
-}
-
-
-# Read the list of machines.
-my @machines;
-if (defined $conf && -e $conf) {
-    open CONF, "<$conf" or die;
-    while (<CONF>) {
-        chomp;
-        s/\#.*$//g;
-        next if /^\s*$/;
-        my @tokens = split /\s/, $_;
-        my @supportedFeatures = split(/,/, $tokens[5] || "");
-        my @mandatoryFeatures = split(/,/, $tokens[6] || "");
-        push @machines,
-            { hostName => $tokens[0]
-            , systemTypes => [ split(/,/, $tokens[1]) ]
-            , sshKey => $tokens[2]
-            , maxJobs => int($tokens[3])
-            , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1)
-            , supportedFeatures => [ @supportedFeatures, @mandatoryFeatures ]
-            , mandatoryFeatures => [ @mandatoryFeatures ]
-            , enabled => 1
-            };
-    }
-    close CONF;
-}
-
-
-
-# Wait for the calling process to ask us whether we can build some derivation.
-my ($drvPath, $hostName, $slotLock);
-my ($from, $to);
-
-REQ: while (1) {
-    $_ = <STDIN> || exit 0;
-    (my $amWilling, my $neededSystem, $drvPath, my $requiredFeatures) = split;
-    my @requiredFeatures = split /,/, $requiredFeatures;
-
-    my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
-
-    if (!defined $currentLoad) {
-        sendReply "decline";
-        next;
-    }
-
-    # Acquire the exclusive lock on $currentLoad/main-lock.
-    mkdir $currentLoad, 0777 or die unless -d $currentLoad;
-    my $mainLock = "$currentLoad/main-lock";
-    sysopen MAINLOCK, "$mainLock", O_RDWR|O_CREAT, 0600 or die;
-    flock(MAINLOCK, LOCK_EX) or die;
-
-
-    while (1) {
-        # Find all machine that can execute this build, i.e., that
-        # support builds for the given platform and features, and are
-        # not at their job limit.
-        my $rightType = 0;
-        my @available = ();
-        LOOP: foreach my $cur (@machines) {
-            if ($cur->{enabled}
-                && (grep { $neededSystem eq $_ } @{$cur->{systemTypes}})
-                && all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{supportedFeatures}} } (@requiredFeatures, @mandatoryFeatures))
-                && all(map { my $f = $_; 0 != grep { $f eq $_ } @requiredFeatures } @{$cur->{mandatoryFeatures}})
-                )
-            {
-                $rightType = 1;
-
-                # We have a machine of the right type.  Determine the load on
-                # the machine.
-                my $slot = 0;
-                my $load = 0;
-                my $free;
-                while ($slot < $cur->{maxJobs}) {
-                    my $slotLock = openSlotLock($cur, $slot);
-                    if (flock($slotLock, LOCK_EX | LOCK_NB)) {
-                        $free = $slot unless defined $free;
-                        flock($slotLock, LOCK_UN) or die;
-                    } else {
-                        $load++;
-                    }
-                    close $slotLock;
-                    $slot++;
-                }
-
-                push @available, { machine => $cur, load => $load, free => $free }
-                if $load < $cur->{maxJobs};
-            }
-        }
-
-        if ($debug) {
-            print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
-                foreach @available;
-        }
-
-
-        # Didn't find any available machine?  Then decline or postpone.
-        if (scalar @available == 0) {
-            # Postpone if we have a machine of the right type, except
-            # if the local system can and wants to do the build.
-            if ($rightType && !$canBuildLocally) {
-                sendReply "postpone";
-            } else {
-                sendReply "decline";
-            }
-            close MAINLOCK;
-            next REQ;
-        }
-
-
-        # Prioritise the available machines as follows:
-        # - First by load divided by speed factor, rounded to the nearest
-        #   integer.  This causes fast machines to be preferred over slow
-        #   machines with similar loads.
-        # - Then by speed factor.
-        # - Finally by load.
-        sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
-        @available = sort
-            { lf($a) <=> lf($b)
-                  || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
-                  || $a->{load} <=> $b->{load}
-            } @available;
-
-
-        # Select the best available machine and lock a free slot.
-        my $selected = $available[0];
-        my $machine = $selected->{machine};
-
-        $slotLock = openSlotLock($machine, $selected->{free});
-        flock($slotLock, LOCK_EX | LOCK_NB) or die;
-        utime undef, undef, $slotLock;
-
-        close MAINLOCK;
-
-
-        # Connect to the selected machine.
-        my @sshOpts = ("-i", $machine->{sshKey});
-        $hostName = $machine->{hostName};
-        eval {
-            ($from, $to) = connectToRemoteNix($hostName, \@sshOpts, "2>&4");
-            # FIXME: check if builds are inhibited.
-        };
-        last REQ unless $@;
-        print STDERR "$@";
-        warn "unable to open SSH connection to ‘$hostName’, trying other available machines...\n";
-        $from = undef;
-        $to = undef;
-        $machine->{enabled} = 0;
-    }
-}
-
-
-# Tell Nix we've accepted the build.
-sendReply "accept";
-my @inputs = split /\s/, readline(STDIN);
-my @outputs = split /\s/, readline(STDIN);
-
-
-# Copy the derivation and its dependencies to the build machine.  This
-# is guarded by an exclusive lock per machine to prevent multiple
-# build-remote instances from copying to a machine simultaneously.
-# That's undesirable because we may end up with N instances uploading
-# the same missing path simultaneously, causing the effective network
-# bandwidth and target disk speed to be divided by N.
-my $uploadLock = "$currentLoad/$hostName.upload-lock";
-sysopen UPLOADLOCK, "$uploadLock", O_RDWR|O_CREAT, 0600 or die;
-eval {
-    local $SIG{ALRM} = sub { die "alarm\n" };
-    # Don't wait forever, so that a process that gets stuck while
-    # holding the lock doesn't block everybody else indefinitely.
-    # It's safe to continue after a timeout, just (potentially)
-    # inefficient.
-    alarm 15 * 60;
-    flock(UPLOADLOCK, LOCK_EX);
-    alarm 0;
-};
-if ($@) {
-    die unless $@ eq "alarm\n";
-    print STDERR "somebody is hogging $uploadLock, continuing...\n";
-    unlink $uploadLock;
-}
-Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0);
-close UPLOADLOCK;
-
-
-# Perform the build.
-print STDERR "building ‘$drvPath’ on ‘$hostName’\n";
-writeInt(6, $to) or die; # == cmdBuildPaths
-writeStrings([$drvPath], $to);
-writeInt($maxSilentTime, $to);
-writeInt($buildTimeout, $to);
-my $res = readInt($from);
-if ($res != 0) {
-    my $msg = decode("utf-8", readString($from));
-    print STDERR "error: $msg on ‘$hostName’\n";
-    exit $res;
-}
-
-
-# Copy the output from the build machine.
-my @outputs2 = grep { !isValidPath($_) } @outputs;
-if (scalar @outputs2 > 0) {
-    writeInt(5, $to); # == cmdExportPaths
-    writeInt(0, $to); # don't sign
-    writeStrings(\@outputs2, $to);
-    $ENV{'NIX_HELD_LOCKS'} = "@outputs2"; # FIXME: ugly
-    importPaths(fileno($from), 1);
-}
diff --git a/scripts/local.mk b/scripts/local.mk
index ee8ae6845dc..365d72086bf 100644
--- a/scripts/local.mk
+++ b/scripts/local.mk
@@ -4,7 +4,6 @@ nix_bin_scripts := \
 bin-scripts += $(nix_bin_scripts)
 
 nix_noinst_scripts := \
-  $(d)/build-remote.pl \
   $(d)/nix-http-export.cgi \
   $(d)/nix-profile.sh \
   $(d)/nix-reduce-build
@@ -14,6 +13,5 @@ noinst-scripts += $(nix_noinst_scripts)
 profiledir = $(sysconfdir)/profile.d
 
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
-$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
 
 clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)

From c54814b175793f0e7a53fdeba55d1149342ec82a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 18:54:33 +0100
Subject: [PATCH 0089/2196] Remove download-via-ssh

Replaced by SSHStore.
---
 .gitignore                               |   3 -
 Makefile                                 |   1 -
 src/download-via-ssh/download-via-ssh.cc | 142 -----------------------
 src/download-via-ssh/local.mk            |  11 --
 4 files changed, 157 deletions(-)
 delete mode 100644 src/download-via-ssh/download-via-ssh.cc
 delete mode 100644 src/download-via-ssh/local.mk

diff --git a/.gitignore b/.gitignore
index 4d0ac32b16f..336967d15eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,9 +71,6 @@ Makefile.config
 # /src/nix-channel/
 /src/nix-channel/nix-channel
 
-# /src/download-via-ssh/
-/src/download-via-ssh/download-via-ssh
-
 # /src/buildenv/
 /src/buildenv/buildenv
 
diff --git a/Makefile b/Makefile
index 14be271bb10..67dc14f8e48 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,6 @@ makefiles = \
   misc/emacs/local.mk \
   doc/manual/local.mk \
   tests/local.mk
-  #src/download-via-ssh/local.mk \
 
 GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
 
diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc
deleted file mode 100644
index 4a1ba9a1123..00000000000
--- a/src/download-via-ssh/download-via-ssh.cc
+++ /dev/null
@@ -1,142 +0,0 @@
-#include "shared.hh"
-#include "util.hh"
-#include "serialise.hh"
-#include "archive.hh"
-#include "affinity.hh"
-#include "globals.hh"
-#include "serve-protocol.hh"
-#include "worker-protocol.hh"
-#include "store-api.hh"
-
-#include <iostream>
-#include <cstdlib>
-#include <unistd.h>
-
-using namespace nix;
-
-// !!! TODO:
-// * Respect more than the first host
-// * use a database
-// * show progress
-
-
-static std::pair<FdSink, FdSource> connect(const string & conn)
-{
-    Pipe to, from;
-    to.create();
-    from.create();
-    startProcess([&]() {
-        if (dup2(to.readSide, STDIN_FILENO) == -1)
-            throw SysError("dupping stdin");
-        if (dup2(from.writeSide, STDOUT_FILENO) == -1)
-            throw SysError("dupping stdout");
-        restoreSignals();
-        execlp("ssh", "ssh", "-x", "-T", conn.c_str(), "nix-store --serve", NULL);
-        throw SysError("executing ssh");
-    });
-    // If child exits unexpectedly, we'll EPIPE or EOF early.
-    // If we exit unexpectedly, child will EPIPE or EOF early.
-    // So no need to keep track of it.
-
-    return std::pair<FdSink, FdSource>(to.writeSide.borrow(), from.readSide.borrow());
-}
-
-
-static void substitute(std::pair<FdSink, FdSource> & pipes, Path storePath, Path destPath)
-{
-    pipes.first << cmdDumpStorePath << storePath;
-    pipes.first.flush();
-    restorePath(destPath, pipes.second);
-    std::cout << std::endl;
-}
-
-
-static void query(std::pair<FdSink, FdSource> & pipes)
-{
-    for (string line; getline(std::cin, line);) {
-        Strings tokenized = tokenizeString<Strings>(line);
-        string cmd = tokenized.front();
-        tokenized.pop_front();
-        if (cmd == "have") {
-            pipes.first
-                << cmdQueryValidPaths
-                << 0 // don't lock
-                << 0 // don't substitute
-                << tokenized;
-            pipes.first.flush();
-            PathSet paths = readStrings<PathSet>(pipes.second);
-            for (auto & i : paths)
-                std::cout << i << std::endl;
-        } else if (cmd == "info") {
-            pipes.first << cmdQueryPathInfos << tokenized;
-            pipes.first.flush();
-            while (1) {
-                Path path = readString(pipes.second);
-                if (path.empty()) break;
-                assertStorePath(path);
-                std::cout << path << std::endl;
-                string deriver = readString(pipes.second);
-                if (!deriver.empty()) assertStorePath(deriver);
-                std::cout << deriver << std::endl;
-                PathSet references = readStorePaths<PathSet>(pipes.second);
-                std::cout << references.size() << std::endl;
-                for (auto & i : references)
-                    std::cout << i << std::endl;
-                std::cout << readLongLong(pipes.second) << std::endl;
-                std::cout << readLongLong(pipes.second) << std::endl;
-            }
-        } else
-            throw Error(format("unknown substituter query ‘%1%’") % cmd);
-        std::cout << std::endl;
-    }
-}
-
-
-int main(int argc, char * * argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        if (argc < 2)
-            throw UsageError("download-via-ssh requires an argument");
-
-        initNix();
-
-        settings.update();
-
-        if (settings.sshSubstituterHosts.empty())
-            return;
-
-        std::cout << std::endl;
-
-        /* Pass on the location of the daemon client's SSH
-           authentication socket. */
-        string sshAuthSock = settings.get("ssh-auth-sock", string(""));
-        if (sshAuthSock != "") setenv("SSH_AUTH_SOCK", sshAuthSock.c_str(), 1);
-
-        string host = settings.sshSubstituterHosts.front();
-        std::pair<FdSink, FdSource> pipes = connect(host);
-
-        /* Exchange the greeting */
-        pipes.first << SERVE_MAGIC_1;
-        pipes.first.flush();
-        unsigned int magic = readInt(pipes.second);
-        if (magic != SERVE_MAGIC_2)
-            throw Error("protocol mismatch");
-        readInt(pipes.second); // Server version, unused for now
-        pipes.first << SERVE_PROTOCOL_VERSION;
-        pipes.first.flush();
-
-        string arg = argv[1];
-        if (arg == "--query")
-            query(pipes);
-        else if (arg == "--substitute") {
-            if (argc != 4)
-                throw UsageError("download-via-ssh: --substitute takes exactly two arguments");
-            Path storePath = argv[2];
-            Path destPath = argv[3];
-            printError(format("downloading ‘%1%’ via SSH from ‘%2%’...") % storePath % host);
-            substitute(pipes, storePath, destPath);
-        }
-        else
-            throw UsageError(format("download-via-ssh: unknown command ‘%1%’") % arg);
-    });
-}
diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk
deleted file mode 100644
index 80f4c385acb..00000000000
--- a/src/download-via-ssh/local.mk
+++ /dev/null
@@ -1,11 +0,0 @@
-programs += download-via-ssh
-
-download-via-ssh_DIR := $(d)
-
-download-via-ssh_SOURCES := $(d)/download-via-ssh.cc
-
-download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters
-
-download-via-ssh_CXXFLAGS = -Isrc/nix-store
-
-download-via-ssh_LIBS = libmain libstore libutil libformat

From 612aeb2df5e02000a02def849b94b8342385974e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:16:25 +0100
Subject: [PATCH 0090/2196] Cleanup

---
 src/libstore/binary-cache-store.hh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 31878bbb247..7b228422d1b 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -138,8 +138,6 @@ public:
 
     ref<FSAccessor> getFSAccessor() override;
 
-public:
-
     void addSignatures(const Path & storePath, const StringSet & sigs) override
     { notImpl(); }
 

From 7a58ad0ef5d32f130163276d49f962b4be92e6fd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:20:15 +0100
Subject: [PATCH 0091/2196] SSHStore: uri -> host

---
 src/libstore/ssh-store.cc | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index f5d0a270438..1b44af07bcc 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -7,11 +7,13 @@
 
 namespace nix {
 
+static std::string uriScheme = "ssh://";
+
 class SSHStore : public RemoteStore
 {
 public:
 
-    SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+    SSHStore(string host, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
 
     std::string getUri() override;
 
@@ -36,17 +38,17 @@ class SSHStore : public RemoteStore
 
     Pid sshMaster;
 
-    string uri;
+    string host;
 
     Path key;
 };
 
-SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
+SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
     : Store(params)
     , RemoteStore(params, maxConnections)
     , tmpDir(createTempDir("", "nix", true, true, 0700))
     , socketPath((Path) tmpDir + "/ssh.sock")
-    , uri(std::move(uri))
+    , host(std::move(host))
     , key(get(params, "ssh-key", ""))
 {
     /* open a connection and perform the handshake to verify all is well */
@@ -55,7 +57,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
 
 string SSHStore::getUri()
 {
-    return "ssh://" + uri;
+    return uriScheme + host;
 }
 
 class ForwardSource : public Source
@@ -93,9 +95,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
         sshMaster = startProcess([&]() {
             restoreSignals();
             if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), host.c_str(), NULL);
             else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), host.c_str(), NULL);
             throw SysError("starting ssh master");
         });
     }
@@ -109,7 +111,7 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
             throw SysError("duping over STDIN");
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("duping over STDOUT");
-        execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL);
+        execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-daemon", "--stdio", NULL);
         throw SysError("executing nix-daemon --stdio over ssh");
     });
     in.readSide = -1;
@@ -126,8 +128,8 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    if (std::string(uri, 0, 6) != "ssh://") return 0;
-    return std::make_shared<SSHStore>(uri.substr(6), params);
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
 });
 
 }

From ce4d8e3ef83505560db787876beb30d20ca6faf2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:20:41 +0100
Subject: [PATCH 0092/2196] Remove unnecessary call to topoSortPaths()

exportPaths() already does this.
---
 src/nix-store/nix-store.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index c1e6afef0e5..0aabe66c562 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -922,9 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                Paths sorted = store->topoSortPaths(readStorePaths<PathSet>(*store, in));
-                reverse(sorted.begin(), sorted.end());
-                store->exportPaths(sorted, out);
+                store->exportPaths(readStorePaths<Paths>(*store, in), out);
                 break;
             }
 

From ddb5577f2eb158279ee40219de409dcf1230aacc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:21:54 +0100
Subject: [PATCH 0093/2196] Move SavingSourceAdapter to serialise.hh

---
 src/libutil/serialise.hh     | 15 +++++++++++++++
 src/nix-daemon/nix-daemon.cc | 15 ---------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index f12f02543bc..5646d08c131 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -139,6 +139,21 @@ struct StringSource : Source
 };
 
 
+/* Adapter class of a Source that saves all data read to `s'. */
+struct SavingSourceAdapter : Source
+{
+    Source & orig;
+    string s;
+    SavingSourceAdapter(Source & orig) : orig(orig) { }
+    size_t read(unsigned char * data, size_t len)
+    {
+        size_t n = orig.read(data, len);
+        s.append((const char *) data, n);
+        return n;
+    }
+};
+
+
 void writePadding(size_t len, Sink & sink);
 void writeString(const unsigned char * buf, size_t len, Sink & sink);
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 90a7301873c..f3ee0afc11e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -168,21 +168,6 @@ struct RetrieveRegularNARSink : ParseSink
 };
 
 
-/* Adapter class of a Source that saves all data read to `s'. */
-struct SavingSourceAdapter : Source
-{
-    Source & orig;
-    string s;
-    SavingSourceAdapter(Source & orig) : orig(orig) { }
-    size_t read(unsigned char * data, size_t len)
-    {
-        size_t n = orig.read(data, len);
-        s.append((const char *) data, n);
-        return n;
-    }
-};
-
-
 static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVersion,
     Source & from, Sink & to, unsigned int op)
 {

From fa07558a069b974769e22ac944cfe21a69fb4485 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:22:48 +0100
Subject: [PATCH 0094/2196] Provide default implementations for a couple of
 Store methods

---
 src/libstore/binary-cache-store.hh            | 19 -------------------
 src/{nix-store => libstore}/serve-protocol.hh |  0
 src/libstore/store-api.cc                     | 13 +++++++++++++
 src/libstore/store-api.hh                     | 14 +++++++-------
 4 files changed, 20 insertions(+), 26 deletions(-)
 rename src/{nix-store => libstore}/serve-protocol.hh (100%)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 7b228422d1b..a70d50d4949 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -71,9 +71,6 @@ public:
         PathSet & referrers) override
     { notImpl(); }
 
-    PathSet queryValidDerivers(const Path & path) override
-    { return {}; }
-
     PathSet queryDerivationOutputs(const Path & path) override
     { notImpl(); }
 
@@ -83,13 +80,6 @@ public:
     Path queryPathFromHashPart(const string & hashPart) override
     { notImpl(); }
 
-    PathSet querySubstitutablePaths(const PathSet & paths) override
-    { return {}; }
-
-    void querySubstitutablePathInfos(const PathSet & paths,
-        SubstitutablePathInfos & infos) override
-    { }
-
     bool wantMassQuery() override { return wantMassQuery_; }
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
@@ -121,21 +111,12 @@ public:
     void addIndirectRoot(const Path & path) override
     { notImpl(); }
 
-    void syncWithGC() override
-    { }
-
     Roots findRoots() override
     { notImpl(); }
 
     void collectGarbage(const GCOptions & options, GCResults & results) override
     { notImpl(); }
 
-    void optimiseStore() override
-    { }
-
-    bool verifyStore(bool checkContents, bool repair) override
-    { return true; }
-
     ref<FSAccessor> getFSAccessor() override;
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override
diff --git a/src/nix-store/serve-protocol.hh b/src/libstore/serve-protocol.hh
similarity index 100%
rename from src/nix-store/serve-protocol.hh
rename to src/libstore/serve-protocol.hh
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a42d1183405..f98ba38406d 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -285,6 +285,19 @@ bool Store::isValidPath(const Path & storePath)
 }
 
 
+/* Default implementation for stores that only implement
+   queryPathInfoUncached(). */
+bool Store::isValidPathUncached(const Path & path)
+{
+    try {
+        queryPathInfo(path);
+        return true;
+    } catch (InvalidPath &) {
+        return false;
+    }
+}
+
+
 ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 {
     std::promise<ref<ValidPathInfo>> promise;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3fee999072f..8058daf149b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -320,7 +320,7 @@ public:
 
 protected:
 
-    virtual bool isValidPathUncached(const Path & path) = 0;
+    virtual bool isValidPathUncached(const Path & path);
 
 public:
 
@@ -360,7 +360,7 @@ public:
        output.  (Note that the result of `queryDeriver()' is the
        derivation that was actually used to produce `path', which may
        not exist anymore.) */
-    virtual PathSet queryValidDerivers(const Path & path) = 0;
+    virtual PathSet queryValidDerivers(const Path & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
     virtual PathSet queryDerivationOutputs(const Path & path) = 0;
@@ -373,13 +373,13 @@ public:
     virtual Path queryPathFromHashPart(const string & hashPart) = 0;
 
     /* Query which of the given paths have substitutes. */
-    virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
+    virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; };
 
     /* Query substitute info (i.e. references, derivers and download
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const PathSet & paths,
-        SubstitutablePathInfos & infos) = 0;
+        SubstitutablePathInfos & infos) { return; };
 
     virtual bool wantMassQuery() { return false; }
 
@@ -454,7 +454,7 @@ public:
          permanent root and sees our's.
 
        In either case the permanent root is seen by the collector. */
-    virtual void syncWithGC() = 0;
+    virtual void syncWithGC() { };
 
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
@@ -485,11 +485,11 @@ public:
 
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
-    virtual void optimiseStore() = 0;
+    virtual void optimiseStore() { };
 
     /* Check the integrity of the Nix store.  Returns true if errors
        remain. */
-    virtual bool verifyStore(bool checkContents, bool repair) = 0;
+    virtual bool verifyStore(bool checkContents, bool repair) { return false; };
 
     /* Return an object to access files in the Nix store. */
     virtual ref<FSAccessor> getFSAccessor() = 0;

From f38224e924bc38ea2b94930f8d12e29c7c8df7a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:23:16 +0100
Subject: [PATCH 0095/2196] copyStorePath(): Don't require signatures for
 "trusted" stores

For example, SSH stores could be trusted.
---
 src/libstore/store-api.cc | 9 +++++++++
 src/libstore/store-api.hh | 4 ++++
 2 files changed, 13 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f98ba38406d..11c2f4b02b2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -529,6 +529,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
+    if (srcStore->isTrusted())
+        dontCheckSigs = true;
+
+    if (!info->narHash && dontCheckSigs) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->narHash = hashString(htSHA256, *sink.s);
+        info = info2;
+    }
+
     dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
 }
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8058daf149b..39132be893c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -562,6 +562,10 @@ public:
 
     const Stats & getStats();
 
+    /* Whether this store paths from this store can be imported even
+       if they lack a signature. */
+    virtual bool isTrusted() { return false; }
+
 protected:
 
     Stats stats;

From caa5793b4a74049ee37dd88eb1c5b785456ce40d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 19:28:40 +0100
Subject: [PATCH 0096/2196] Add a LegacySSHStore that uses nix-store --serve

This is useful for nix-copy-closure.
---
 src/libstore/legacy-ssh-store.cc | 247 +++++++++++++++++++++++++++++++
 src/libstore/remote-store.cc     |   1 +
 2 files changed, 248 insertions(+)
 create mode 100644 src/libstore/legacy-ssh-store.cc

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
new file mode 100644
index 00000000000..5d9e5aad6e0
--- /dev/null
+++ b/src/libstore/legacy-ssh-store.cc
@@ -0,0 +1,247 @@
+#include "archive.hh"
+#include "pool.hh"
+#include "remote-store.hh"
+#include "serve-protocol.hh"
+#include "store-api.hh"
+#include "worker-protocol.hh"
+
+namespace nix {
+
+static std::string uriScheme = "legacy-ssh://";
+
+struct LegacySSHStore : public Store
+{
+    string host;
+
+    struct Connection
+    {
+        Pid sshPid;
+        AutoCloseFD out;
+        AutoCloseFD in;
+        FdSink to;
+        FdSource from;
+    };
+
+    AutoDelete tmpDir;
+
+    Path socketPath;
+
+    Pid sshMaster;
+
+    ref<Pool<Connection>> connections;
+
+    Path key;
+
+    LegacySSHStore(const string & host, const Params & params,
+        size_t maxConnections = std::numeric_limits<size_t>::max())
+        : Store(params)
+        , host(host)
+        , tmpDir(createTempDir("", "nix", true, true, 0700))
+        , socketPath((Path) tmpDir + "/ssh.sock")
+        , connections(make_ref<Pool<Connection>>(
+            maxConnections,
+            [this]() { return openConnection(); },
+            [](const ref<Connection> & r) { return true; }
+            ))
+        , key(get(params, "ssh-key", ""))
+    {
+    }
+
+    ref<Connection> openConnection()
+    {
+        if ((pid_t) sshMaster == -1) {
+            sshMaster = startProcess([&]() {
+                restoreSignals();
+                Strings args{ "ssh", "-M", "-S", socketPath, "-N", "-x", "-a", host };
+                if (!key.empty())
+                    args.insert(args.end(), {"-i", key});
+                execvp("ssh", stringsToCharPtrs(args).data());
+                throw SysError("starting SSH master connection to host ‘%s’", host);
+            });
+        }
+
+        auto conn = make_ref<Connection>();
+        Pipe in, out;
+        in.create();
+        out.create();
+        conn->sshPid = startProcess([&]() {
+            if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
+                throw SysError("duping over STDIN");
+            if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+                throw SysError("duping over STDOUT");
+            execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-store", "--serve", "--write", nullptr);
+            throw SysError("executing ‘nix-store --serve’ on remote host ‘%s’", host);
+        });
+        in.readSide = -1;
+        out.writeSide = -1;
+        conn->out = std::move(out.readSide);
+        conn->in = std::move(in.writeSide);
+        conn->to = FdSink(conn->in.get());
+        conn->from = FdSource(conn->out.get());
+
+        int remoteVersion;
+
+        try {
+            conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
+            conn->to.flush();
+
+            unsigned int magic = readInt(conn->from);
+            if (magic != SERVE_MAGIC_2)
+                throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host);
+            remoteVersion = readInt(conn->from);
+            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
+                throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host);
+
+        } catch (EndOfFile & e) {
+            throw Error("cannot connect to ‘%1%’", host);
+        }
+
+        return conn;
+    };
+
+    string getUri() override
+    {
+        return uriScheme + host;
+    }
+
+    void queryPathInfoUncached(const Path & path,
+        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
+        std::function<void(std::exception_ptr exc)> failure) override
+    {
+        sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
+            auto conn(connections->get());
+
+            debug("querying remote host ‘%s’ for info on ‘%s’", host, path);
+
+            conn->to << cmdQueryPathInfos << PathSet{path};
+            conn->to.flush();
+
+            auto info = std::make_shared<ValidPathInfo>();
+            conn->from >> info->path;
+            if (info->path.empty()) return nullptr;
+            assert(path == info->path);
+
+            PathSet references;
+            conn->from >> info->deriver;
+            info->references = readStorePaths<PathSet>(*this, conn->from);
+            readLongLong(conn->from); // download size
+            info->narSize = readLongLong(conn->from);
+
+            auto s = readString(conn->from);
+            assert(s == "");
+
+            return info;
+        });
+    }
+
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+        bool repair, bool dontCheckSigs,
+        std::shared_ptr<FSAccessor> accessor) override
+    {
+        debug("adding path ‘%s’ to remote host ‘%s’", info.path, host);
+
+        auto conn(connections->get());
+
+        conn->to
+            << cmdImportPaths
+            << 1;
+        conn->to(*nar);
+        conn->to
+            << exportMagic
+            << info.path
+            << info.references
+            << info.deriver
+            << 0
+            << 0;
+        conn->to.flush();
+
+        if (readInt(conn->from) != 1)
+            throw Error("failed to add path ‘%s’ to remote host ‘%s’, info.path, host");
+
+    }
+
+    void narFromPath(const Path & path, Sink & sink) override
+    {
+        auto conn(connections->get());
+
+        conn->to << cmdDumpStorePath << path;
+        conn->to.flush();
+
+        /* FIXME: inefficient. */
+        ParseSink parseSink; /* null sink; just parse the NAR */
+        SavingSourceAdapter savedNAR(conn->from);
+        parseDump(parseSink, savedNAR);
+        sink(savedNAR.s);
+    }
+
+    /* Unsupported methods. */
+    [[noreturn]] void unsupported()
+    {
+        throw Error("operation not supported on SSH stores");
+    }
+
+    PathSet queryAllValidPaths() override { unsupported(); }
+
+    void queryReferrers(const Path & path, PathSet & referrers) override
+    { unsupported(); }
+
+    PathSet queryDerivationOutputs(const Path & path) override
+    { unsupported(); }
+
+    StringSet queryDerivationOutputNames(const Path & path) override
+    { unsupported(); }
+
+    Path queryPathFromHashPart(const string & hashPart) override
+    { unsupported(); }
+
+    Path addToStore(const string & name, const Path & srcPath,
+        bool recursive, HashType hashAlgo,
+        PathFilter & filter, bool repair) override
+    { unsupported(); }
+
+    Path addTextToStore(const string & name, const string & s,
+        const PathSet & references, bool repair) override
+    { unsupported(); }
+
+    void buildPaths(const PathSet & paths, BuildMode buildMode) override
+    { unsupported(); }
+
+    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override
+    { unsupported(); }
+
+    void ensurePath(const Path & path) override
+    { unsupported(); }
+
+    void addTempRoot(const Path & path) override
+    { unsupported(); }
+
+    void addIndirectRoot(const Path & path) override
+    { unsupported(); }
+
+    Roots findRoots() override
+    { unsupported(); }
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override
+    { unsupported(); }
+
+    ref<FSAccessor> getFSAccessor()
+    { unsupported(); }
+
+    void addSignatures(const Path & storePath, const StringSet & sigs) override
+    { unsupported(); }
+
+    bool isTrusted() override
+    { return true; }
+
+};
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
+});
+
+}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 816d95ba607..42c09ec7e0b 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -37,6 +37,7 @@ template<class T> T readStorePaths(Store & store, Source & from)
 }
 
 template PathSet readStorePaths(Store & store, Source & from);
+template Paths readStorePaths(Store & store, Source & from);
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params, size_t maxConnections)

From 4724903c78e80481fc63d627081fac6a98e4205d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Feb 2017 20:55:47 +0100
Subject: [PATCH 0097/2196] nix-copy-closure: Use computeFSClosure() and
 LegacySSHStore

---
 src/nix-copy-closure/nix-copy-closure.cc | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index b7e997ca4b0..4340443b5cc 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -7,13 +7,15 @@ int main(int argc, char ** argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+
         auto gzip = false;
         auto toMode = true;
         auto includeOutputs = false;
         auto dryRun = false;
         auto useSubstitutes = false;
-        auto sshHost = string{};
-        auto storePaths = PathSet{};
+        std::string sshHost;
+        PathSet storePaths;
+
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
                 showManPage("nix-copy-closure");
@@ -41,20 +43,17 @@ int main(int argc, char ** argv)
                 storePaths.insert(*arg);
             return true;
         });
+
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
-        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto remoteUri = "legacy-ssh://" + sshHost + (gzip ? "?compress=true" : "");
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
-        if (includeOutputs) {
-            auto newPaths = PathSet{};
-            for (const auto & p : storePaths) {
-                auto outputs = from->queryDerivationOutputs(p);
-                newPaths.insert(outputs.begin(), outputs.end());
-            }
-            storePaths.insert(newPaths.begin(), newPaths.end());
-        }
-        copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
+
+        PathSet closure;
+        from->computeFSClosure(storePaths, closure, false, includeOutputs);
+
+        copyPaths(from, to, Paths(closure.begin(), closure.end()), useSubstitutes);
     });
 }

From 418a837897e597c5dc2eb4e458462bd04b2abde7 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Fri, 12 Aug 2016 06:38:08 -0400
Subject: [PATCH 0098/2196] Remove perl dependency.

Fixes #341
---
 Makefile                                      |  1 -
 configure.ac                                  | 52 -------------------
 .../advanced-topics/distributed-builds.xml    | 10 ++--
 release.nix                                   | 22 +++-----
 src/nix-store/nix-store.cc                    |  2 +-
 tests/optimise-store.sh                       | 12 ++---
 6 files changed, 19 insertions(+), 80 deletions(-)

diff --git a/Makefile b/Makefile
index 62a4850d854..960685b8f48 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,6 @@ makefiles = \
   src/nix-channel/local.mk \
   src/nix-build/local.mk \
   src/build-remote/local.mk \
-  perl/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
   misc/systemd/local.mk \
diff --git a/configure.ac b/configure.ac
index e6b11be2df1..5d5f1d2be79 100644
--- a/configure.ac
+++ b/configure.ac
@@ -121,7 +121,6 @@ AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(perl, perl)
 NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 NEED_PROG(bzip2, bzip2)
@@ -131,23 +130,6 @@ AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
 
 
-# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
-AC_MSG_CHECKING([whether Perl is recent enough])
-if ! $perl -e 'open(FOO, "-|", "true"); while (<FOO>) { print; }; close FOO or die;'; then
-    AC_MSG_RESULT(no)
-    AC_MSG_ERROR([Your Perl version is too old.  Nix requires Perl 5.8.0 or newer.])
-fi
-AC_MSG_RESULT(yes)
-
-
-# Figure out where to install Perl modules.
-AC_MSG_CHECKING([for the Perl installation prefix])
-perlversion=$($perl -e 'use Config; print $Config{version};')
-perlarchname=$($perl -e 'use Config; print $Config{archname};')
-AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
-AC_MSG_RESULT($perllibdir)
-
-
 NEED_PROG(cat, cat)
 NEED_PROG(tr, tr)
 AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
@@ -213,40 +195,6 @@ if test "$gc" = yes; then
 fi
 
 
-# Check for the required Perl dependencies (DBI, DBD::SQLite).
-perlFlags="-I$perllibdir"
-
-AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
-  [prefix of the Perl DBI library]),
-  perlFlags="$perlFlags -I$withval")
-
-AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
-  [prefix of the Perl DBD::SQLite library]),
-  perlFlags="$perlFlags -I$withval")
-
-AC_MSG_CHECKING([whether DBD::SQLite works])
-if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
-    AC_MSG_RESULT(no)
-    AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.])
-fi
-AC_MSG_RESULT(yes)
-
-AC_SUBST(perlFlags)
-
-
-# Whether to build the Perl bindings
-AC_MSG_CHECKING([whether to build the Perl bindings])
-AC_ARG_ENABLE(perl-bindings, AC_HELP_STRING([--enable-perl-bindings],
-  [whether to build the Perl bindings (recommended) [default=yes]]),
-  perlbindings=$enableval, perlbindings=yes)
-if test "$enable_shared" = no; then
-   # Perl bindings require shared libraries.
-   perlbindings=no
-fi
-AC_SUBST(perlbindings)
-AC_MSG_RESULT($perlbindings)
-
-
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
   [do not initialise DB etc. in `make install']),
   init_state=$enableval, init_state=yes)
diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index f8583700393..d5bc1c59255 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -42,7 +42,7 @@ purposes.  It uses <command>ssh</command> and
 <command>nix-copy-closure</command> to copy the build inputs and
 outputs and perform the remote build.  To use it, you should set
 <envar>NIX_BUILD_HOOK</envar> to
-<filename><replaceable>prefix</replaceable>/libexec/nix/build-remote.pl</filename>.
+<filename><replaceable>prefix</replaceable>/libexec/nix/build-remote</filename>.
 You should also define a list of available build machines and point
 the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to
 it. <envar>NIX_REMOTE_SYSTEMS</envar> must be an absolute path. An
@@ -68,7 +68,7 @@ bits of information:
   should not have a passphrase!</para></listitem>
 
   <listitem><para>The maximum number of builds that
-  <filename>build-remote.pl</filename> will execute in parallel on the
+  <filename>build-remote</filename> will execute in parallel on the
   machine.  Typically this should be equal to the number of CPU cores.
   For instance, the machine <literal>itchy</literal> in the example
   will execute up to 8 builds in parallel.</para></listitem>
@@ -80,7 +80,7 @@ bits of information:
   <listitem><para>A comma-separated list of <emphasis>supported
   features</emphasis>.  If a derivation has the
   <varname>requiredSystemFeatures</varname> attribute, then
-  <filename>build-remote.pl</filename> will only perform the
+  <filename>build-remote</filename> will only perform the
   derivation on a machine that has the specified features.  For
   instance, the attribute
 
@@ -106,11 +106,11 @@ requiredSystemFeatures = [ "kvm" ];
 You should also set up the environment variable
 <envar>NIX_CURRENT_LOAD</envar> to point at a directory (e.g.,
 <filename>/var/run/nix/current-load</filename>) that
-<filename>build-remote.pl</filename> uses to remember how many builds
+<filename>build-remote</filename> uses to remember how many builds
 it is currently executing remotely.  It doesn't look at the actual
 load on the remote machine, so if you have multiple instances of Nix
 running, they should use the same <envar>NIX_CURRENT_LOAD</envar>
-file.  Maybe in the future <filename>build-remote.pl</filename> will
+file.  Maybe in the future <filename>build-remote</filename> will
 look at the actual remote load.</para>
 
 </chapter>
diff --git a/release.nix b/release.nix
index ace0f9cc89a..7bfde71a644 100644
--- a/release.nix
+++ b/release.nix
@@ -24,18 +24,14 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison flex perl libxml2 libxslt bzip2 xz
+          [ curl bison flex libxml2 libxslt bzip2 xz
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
             git
           ];
 
-        configureFlags = ''
-          --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
-          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
-          --enable-gc
-        '';
+	configureFlags = "--enable-gc";
 
         postUnpack = ''
           # Clean up when building from a working tree.
@@ -73,7 +69,7 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ]
+          [ curl bzip2 xz openssl pkgconfig sqlite boehmgc ]
           ++ lib.optional stdenv.isLinux libsodium
           ++ lib.optional stdenv.isLinux
             (aws-sdk-cpp.override {
@@ -83,8 +79,6 @@ let
 
         configureFlags = ''
           --disable-init-state
-          --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
-          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
           --enable-gc
           --sysconfdir=/etc
         '';
@@ -147,15 +141,13 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl perl bzip2 openssl pkgconfig sqlite xz libsodium
+          [ curl bzip2 openssl pkgconfig sqlite xz libsodium
             # These are for "make check" only:
             graphviz libxml2 libxslt
           ];
 
         configureFlags = ''
           --disable-init-state
-          --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
-          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
         '';
 
         dontInstall = false;
@@ -282,7 +274,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
@@ -303,14 +295,14 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" "libssl1.0.0" "liblzma5" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 0aabe66c562..7dda6d208df 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -926,7 +926,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
-            case cmdBuildPaths: { /* Used by build-remote.pl. */
+            case cmdBuildPaths: {
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
                 PathSet paths = readStorePaths<PathSet>(*store, in);
diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh
index ea4478693e7..bd88662bc37 100644
--- a/tests/optimise-store.sh
+++ b/tests/optimise-store.sh
@@ -5,14 +5,14 @@ clearStore
 outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true)
 outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true)
 
-inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")"
-inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")"
+inode1="$(stat --format=%i $outPath1/foo)"
+inode2="$(stat --format=%i $outPath2/foo)"
 if [ "$inode1" != "$inode2" ]; then
     echo "inodes do not match"
     exit 1
 fi
 
-nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")"
+nlink="$(stat --format=%h $outPath1/foo)"
 if [ "$nlink" != 3 ]; then
     echo "link count incorrect"
     exit 1
@@ -20,7 +20,7 @@ fi
 
 outPath3=$(echo 'with import ./config.nix; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link)
 
-inode3="$(perl -e "print ((lstat('$outPath3/foo'))[1])")"
+inode3="$(stat --format=%i $outPath3/foo)"
 if [ "$inode1" = "$inode3" ]; then
     echo "inodes match unexpectedly"
     exit 1
@@ -28,8 +28,8 @@ fi
 
 nix-store --optimise
 
-inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")"
-inode3="$(perl -e "print ((lstat('$outPath3/foo'))[1])")"
+inode1="$(stat --format=%i $outPath1/foo)"
+inode3="$(stat --format=%i $outPath3/foo)"
 if [ "$inode1" != "$inode3" ]; then
     echo "inodes do not match"
     exit 1

From f7b7df8d1fcb41156fe92ef2ecf5b17074b88446 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Fri, 12 Aug 2016 09:44:01 -0400
Subject: [PATCH 0099/2196] Add nix-perl package for the perl bindings

---
 .gitignore                |   1 +
 Makefile.config.in        |   3 -
 corepkgs/config.nix.in    |   3 +
 local.mk                  |   2 +-
 perl/Makefile             |  14 +++++
 perl/Makefile.config.in   |  19 +++++++
 perl/configure.ac         | 117 ++++++++++++++++++++++++++++++++++++++
 perl/lib/Nix/Config.pm.in |  14 ++---
 perl/local.mk             |  25 ++++----
 release.nix               |  27 +++++++++
 10 files changed, 201 insertions(+), 24 deletions(-)
 create mode 100644 perl/Makefile
 create mode 100644 perl/Makefile.config.in
 create mode 100644 perl/configure.ac

diff --git a/.gitignore b/.gitignore
index 951efb4c908..4f7e668e781 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 Makefile.config
+perl/Makefile.config
 
 # /
 /aclocal.m4
diff --git a/Makefile.config.in b/Makefile.config.in
index 15e94380477..e2277c667ec 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -24,9 +24,6 @@ libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
 mandir = @mandir@
-perl = @perl@
-perlbindings = @perlbindings@
-perllibdir = @perllibdir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
 storedir = @storedir@
diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in
index f0f4890a32f..32ce6b399f2 100644
--- a/corepkgs/config.nix.in
+++ b/corepkgs/config.nix.in
@@ -14,6 +14,9 @@ in rec {
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
+  nixLocalstateDir = "@localstatedir@";
+  nixSysconfDir = "@sysconfdir@";
+  nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
 
   # If Nix is installed in the Nix store, then automatically add it as
   # a dependency to the core packages. This ensures that they work
diff --git a/local.mk b/local.mk
index eebd7196119..dc10e6870a8 100644
--- a/local.mk
+++ b/local.mk
@@ -3,7 +3,7 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
 endif
 
-dist-files += configure config.h.in nix.spec
+dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
diff --git a/perl/Makefile b/perl/Makefile
new file mode 100644
index 00000000000..41a32576e9b
--- /dev/null
+++ b/perl/Makefile
@@ -0,0 +1,14 @@
+makefiles = local.mk
+
+GLOBAL_CXXFLAGS += -std=c++11 -g -Wall
+
+-include Makefile.config
+
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CFLAGS += -O3
+  GLOBAL_CXXFLAGS += -O3
+endif
+
+include mk/lib.mk
diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in
new file mode 100644
index 00000000000..901d1283e55
--- /dev/null
+++ b/perl/Makefile.config.in
@@ -0,0 +1,19 @@
+CC = @CC@
+CFLAGS = @CFLAGS@
+CXX = @CXX@
+CXXFLAGS = @CXXFLAGS@
+HAVE_SODIUM = @HAVE_SODIUM@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+SODIUM_LIBS = @SODIUM_LIBS@
+NIX_CFLAGS = @NIX_CFLAGS@
+NIX_LIBS = @NIX_LIBS@
+nixbindir = @nixbindir@
+curl = @curl@
+nixlibexecdir = @nixlibexecdir@
+nixlocalstatedir = @nixlocalstatedir@
+perl = @perl@
+perllibdir = @perllibdir@
+nixstoredir = @nixstoredir@
+nixsysconfdir = @nixsysconfdir@
+perlbindings = @perlbindings@
diff --git a/perl/configure.ac b/perl/configure.ac
new file mode 100644
index 00000000000..dea2b614004
--- /dev/null
+++ b/perl/configure.ac
@@ -0,0 +1,117 @@
+AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"]))
+AC_CONFIG_SRCDIR(MANIFEST)
+AC_CONFIG_AUX_DIR(../config)
+
+CFLAGS=
+CXXFLAGS=
+AC_PROG_CC
+AC_PROG_CXX
+AX_CXX_COMPILE_STDCXX_11
+
+# Use 64-bit file system calls so that we can support files > 2 GiB.
+AC_SYS_LARGEFILE
+
+AC_DEFUN([NEED_PROG],
+[
+AC_PATH_PROG($1, $2)
+if test -z "$$1"; then
+    AC_MSG_ERROR([$2 is required])
+fi
+])
+
+NEED_PROG(perl, perl)
+NEED_PROG(curl, curl)
+NEED_PROG(bzip2, bzip2)
+NEED_PROG(xz, xz)
+
+# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
+AC_MSG_CHECKING([whether Perl is recent enough])
+if ! $perl -e 'open(FOO, "-|", "true"); while (<FOO>) { print; }; close FOO or die;'; then
+    AC_MSG_RESULT(no)
+    AC_MSG_ERROR([Your Perl version is too old.  Nix requires Perl 5.8.0 or newer.])
+fi
+AC_MSG_RESULT(yes)
+
+
+# Figure out where to install Perl modules.
+AC_MSG_CHECKING([for the Perl installation prefix])
+perlversion=$($perl -e 'use Config; print $Config{version};')
+perlarchname=$($perl -e 'use Config; print $Config{archname};')
+AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
+AC_MSG_RESULT($perllibdir)
+
+AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
+  [path of the Nix store (defaults to /nix/store)]),
+  storedir=$withval, storedir='/nix/store')
+AC_SUBST(storedir)
+
+# Look for libsodium, an optional dependency.
+PKG_CHECK_MODULES([SODIUM], [libsodium],
+  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
+   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
+   have_sodium=1], [have_sodium=])
+AC_SUBST(HAVE_SODIUM, [$have_sodium])
+
+# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl).
+perlFlags="-I$perllibdir"
+
+AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
+  [prefix of the Perl DBI library]),
+  perlFlags="$perlFlags -I$withval")
+
+AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
+  [prefix of the Perl DBD::SQLite library]),
+  perlFlags="$perlFlags -I$withval")
+
+AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH],
+  [prefix of the Perl WWW::Curl library]),
+  perlFlags="$perlFlags -I$withval")
+
+AC_MSG_CHECKING([whether DBD::SQLite works])
+if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
+    AC_MSG_RESULT(no)
+    AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.])
+fi
+AC_MSG_RESULT(yes)
+
+AC_MSG_CHECKING([whether WWW::Curl works])
+if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then
+    AC_MSG_RESULT(no)
+    AC_MSG_FAILURE([The Perl module WWW::Curl is missing.])
+fi
+AC_MSG_RESULT(yes)
+
+AC_SUBST(perlFlags)
+
+PKG_CHECK_MODULES([NIX], [nix-store])
+
+NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate])
+
+# Get nix configure values
+nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixBinDir | tr -d \")
+nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLibexecDir | tr -d \")
+nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLocalstateDir | tr -d \")
+nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixSysconfDir | tr -d \")
+nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixStoreDir | tr -d \")
+AC_SUBST(nixbindir)
+AC_SUBST(nixlibexecdir)
+AC_SUBST(nixlocalstatedir)
+AC_SUBST(nixsysconfdir)
+AC_SUBST(nixstoredir)
+
+AC_SUBST(perlbindings, "yes")
+
+# Expand all variables in config.status.
+test "$prefix" = NONE && prefix=$ac_default_prefix
+test "$exec_prefix" = NONE && exec_prefix='${prefix}'
+for name in $ac_subst_vars; do
+    declare $name="$(eval echo "${!name}")"
+    declare $name="$(eval echo "${!name}")"
+    declare $name="$(eval echo "${!name}")"
+done
+
+rm -f Makefile.config
+ln -s ../mk mk
+
+AC_CONFIG_FILES([])
+AC_OUTPUT
diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index 3575d99cb67..4f1dd967491 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -4,18 +4,18 @@ use MIME::Base64;
 
 $version = "@PACKAGE_VERSION@";
 
-$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
-$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
-$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
-$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
-$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
-$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
+$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@";
+$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@";
+$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix";
+$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
+$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
+$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
 
 $bzip2 = "@bzip2@";
 $xz = "@xz@";
 $curl = "@curl@";
 
-$useBindings = "@perlbindings@" eq "yes";
+$useBindings = 1;
 
 %config = ();
 
diff --git a/perl/local.mk b/perl/local.mk
index 5b43c4b717f..1793ececfd6 100644
--- a/perl/local.mk
+++ b/perl/local.mk
@@ -1,10 +1,10 @@
 nix_perl_sources := \
-  $(d)/lib/Nix/Store.pm \
-  $(d)/lib/Nix/Manifest.pm \
-  $(d)/lib/Nix/SSH.pm \
-  $(d)/lib/Nix/CopyClosure.pm \
-  $(d)/lib/Nix/Config.pm.in \
-  $(d)/lib/Nix/Utils.pm
+  lib/Nix/Store.pm \
+  lib/Nix/Manifest.pm \
+  lib/Nix/SSH.pm \
+  lib/Nix/CopyClosure.pm \
+  lib/Nix/Config.pm.in \
+  lib/Nix/Utils.pm
 
 nix_perl_modules := $(nix_perl_sources:.in=)
 
@@ -12,12 +12,12 @@ $(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllib
 
 ifeq ($(perlbindings), yes)
 
-  $(d)/lib/Nix/Store.cc: $(d)/lib/Nix/Store.xs
+  lib/Nix/Store.cc: lib/Nix/Store.xs
 	$(trace-gen) xsubpp $^ -output $@
 
   libraries += Store
 
-  Store_DIR := $(d)/lib/Nix
+  Store_DIR := lib/Nix
 
   Store_SOURCES := $(Store_DIR)/Store.cc
 
@@ -25,11 +25,10 @@ ifeq ($(perlbindings), yes)
     -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \
     -D_FILE_OFFSET_BITS=64 \
     -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \
-    -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion
+    -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion \
+    $(NIX_CFLAGS)
 
-  Store_LIBS = libstore libutil
-
-  Store_LDFLAGS := $(SODIUM_LIBS)
+  Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
   ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
     archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
@@ -45,4 +44,4 @@ ifeq ($(perlbindings), yes)
 
 endif
 
-clean-files += $(d)/lib/Nix/Config.pm $(d)/lib/Nix/Store.cc
+clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config
diff --git a/release.nix b/release.nix
index 7bfde71a644..b93b64ea923 100644
--- a/release.nix
+++ b/release.nix
@@ -41,6 +41,7 @@ let
         '';
 
         preConfigure = ''
+          (cd perl ; autoreconf --install --force --verbose)
           # TeX needs a writable font cache.
           export VARTEXFONTS=$TMPDIR/texfonts
         '';
@@ -96,6 +97,32 @@ let
       });
 
 
+    perl = pkgs.lib.genAttrs systems (system:
+
+      let pkgs = import <nixpkgs> { inherit system; }; in with pkgs;
+
+      releaseTools.nixBuild {
+        name = "nix-perl";
+        src = tarball;
+
+        buildInputs =
+          [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ]
+          ++ lib.optional stdenv.isLinux libsodium;
+
+	configureFlags = ''
+          --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
+          --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
+          --with-www-curl=${perlPackages.WWWCurl}/${pkgs.perl.libPrefix}
+        '';
+
+        enableParallelBuilding = true;
+
+	postUnpack = "sourceRoot=$sourceRoot/perl";
+
+        preBuild = "unset NIX_INDENT_MAKE";
+      });
+
+
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
       # FIXME: temporarily use a different branch for the Darwin build.

From 2b4c24f46a9a5ab5b0eb6d714efa73de201344f2 Mon Sep 17 00:00:00 2001
From: Janus Troelsen <ysangkok@gmail.com>
Date: Mon, 22 Aug 2016 15:01:58 +0200
Subject: [PATCH 0100/2196] Remove Perl dependency listing in doc

---
 doc/manual/installation/prerequisites-source.xml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index cd6d61356ba..7311e4885e7 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -12,8 +12,6 @@
 
   <listitem><para>A version of GCC or Clang that supports C++11.</para></listitem>
 
-  <listitem><para>Perl 5.8 or higher.</para></listitem>
-
   <listitem><para><command>pkg-config</command> to locate
   dependencies.  If your distribution does not provide it, you can get
   it from <link
@@ -34,11 +32,6 @@
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
 
-  <listitem><para>The Perl DBI and DBD::SQLite libraries, which are
-  available from <link
-  xlink:href="http://search.cpan.org/">CPAN</link> if your
-  distribution does not provide them.</para></listitem>
-
   <listitem><para>The <link
   xlink:href="http://www.hboehm.info/gc/">Boehm
   garbage collector</link> to reduce the evaluator’s memory

From 81c53fe8e56f4a4ce10088fe2d7b6a524a6dc126 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Wed, 8 Feb 2017 20:29:37 +0200
Subject: [PATCH 0101/2196] configure.ac: We require C++14 now

At least in the main Makefile we have:

GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index e6b11be2df1..bfe81840c3b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -61,7 +61,7 @@ CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX_11
+AX_CXX_COMPILE_STDCXX_14
 
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.

From 2cd468874fe512387820bd47d23fa6351d069da2 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Wed, 8 Feb 2017 20:35:06 +0200
Subject: [PATCH 0102/2196] Include config.h implicitly with '-include
 config.h' in CFLAGS

Because config.h can #define things like _FILE_OFFSET_BITS=64 and not
every compilation unit includes config.h, we currently compile half of
Nix with _FILE_OFFSET_BITS=64 and other half with _FILE_OFFSET_BITS
unset. This causes major havoc with the Settings class on e.g. 32-bit ARM,
where different compilation units disagree with the struct layout.

E.g.:

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
@@ -166,6 +166,8 @@ void Settings::update()
     _get(useSubstitutes, "build-use-substitutes");
+    fprintf(stderr, "at Settings::update(): &useSubstitutes = %p\n", &nix::settings.useSubstitutes);
     _get(buildUsersGroup, "build-users-group");
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -138,6 +138,8 @@ void RemoteStore::initConnection(Connection & conn)
 void RemoteStore::setOptions(Connection & conn)
 {
+    fprintf(stderr, "at RemoteStore::setOptions(): &useSubstitutes = %p\n", &nix::settings.useSubstitutes);
     conn.to << wopSetOptions

Gave me:

at Settings::update(): &useSubstitutes = 0xb6e5c5cb
at RemoteStore::setOptions(): &useSubstitutes = 0xb6e5c5c7

That was not a fun one to debug!
---
 Makefile                              | 2 +-
 src/libexpr/json-to-value.cc          | 1 -
 src/libexpr/symbol-table.hh           | 2 --
 src/libexpr/value.hh                  | 1 -
 src/libmain/shared.cc                 | 2 --
 src/libmain/stack.cc                  | 2 --
 src/libstore/build.cc                 | 2 --
 src/libstore/globals.cc               | 2 --
 src/libstore/local-store.cc           | 1 -
 src/libstore/optimise-store.cc        | 2 --
 src/libstore/s3-binary-cache-store.cc | 2 --
 src/libutil/archive.cc                | 2 --
 src/libutil/hash.cc                   | 2 --
 src/libutil/types.hh                  | 1 -
 src/libutil/util.cc                   | 2 --
 15 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 62a4850d854..d26cf8d99d2 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ makefiles = \
   doc/manual/local.mk \
   tests/local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
+GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
 
 -include Makefile.config
 
diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index f671802bcc2..c189cdef35e 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -1,4 +1,3 @@
-#include "config.h"
 #include "json-to-value.hh"
 
 #include <cstring>
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index 2fdf820211c..c2ee49dd32f 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -1,7 +1,5 @@
 #pragma once
 
-#include "config.h"
-
 #include <map>
 #include <unordered_set>
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 271e6a1b24a..81f918d48de 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -1,6 +1,5 @@
 #pragma once
 
-#include "config.h"
 #include "symbol-table.hh"
 
 #if HAVE_BOEHMGC
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 52cb2312826..56aa3db0015 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "common-args.hh"
 #include "globals.hh"
 #include "shared.hh"
diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index abf59dc4baa..57b6a197c0f 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "types.hh"
 
 #include <cstring>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5d6fff4e349..1aee150fda3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "references.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 00b46889252..90f83a5bbd9 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "globals.hh"
 #include "util.hh"
 #include "archive.hh"
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 612efde7bb8..4c161cfb341 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1,4 +1,3 @@
-#include "config.h"
 #include "local-store.hh"
 #include "globals.hh"
 #include "archive.hh"
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b71c7e905ff..cf234e35d37 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "util.hh"
 #include "local-store.hh"
 #include "globals.hh"
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ccb71f1eefe..cc1b3310420 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #if ENABLE_S3
 #if __linux__
 
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index fbba7f853f9..e0e6f5dfd73 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include <cerrno>
 #include <algorithm>
 #include <vector>
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index aa50fceb9e3..f447c80c5d8 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include <iostream>
 #include <cstring>
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index b9a93d27d2a..97d79af9b5d 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -1,6 +1,5 @@
 #pragma once
 
-#include "config.h"
 
 #include "ref.hh"
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6c4c5c969d8..33659936800 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"

From e2257d4eeb3f75bac57d9cb77c9ce06b702de050 Mon Sep 17 00:00:00 2001
From: Renzo Carbonara <gnuk0001@gmail.com>
Date: Thu, 9 Feb 2017 18:16:09 +0100
Subject: [PATCH 0103/2196] Documentation.

---
 src/libstore/download.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8f2387b6325..b2adc154818 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -230,8 +230,11 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
+            /* If no file exist in the specified path, curl continues to work
+             * anyway as if netrc support was disabled. */
             Path netrcFile = settings.get("netrc-file",
                (format("%1%/%2%") % settings.nixConfDir % "netrc").str());
+            /* Curl copies the given C string, so the following call is safe. */
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, netrcFile.c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 

From 649a81bcd6445d3b00f400cd6017d184bf0aaa25 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 13 Feb 2017 15:06:46 +0200
Subject: [PATCH 0104/2196] nix-daemon: Don't splice with len=SIZE_MAX

Currently, 'nix-daemon --stdio' is always failing for me, due to the
splice call always failing with (on a 32-bit host):

splice(0, NULL, 3, NULL, 4294967295, SPLICE_F_MOVE) = -1 EINVAL (Invalid argument)

With a bit of ftracing (and luck) the problem seems to be that splice()
always fails with EINVAL if the len cast as ssize_t is negative:
http://lxr.free-electrons.com/source/fs/read_write.c?v=4.4#L384

So use SSIZE_MAX instead of SIZE_MAX.
---
 src/nix-daemon/nix-daemon.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index f3ee0afc11e..3b43ddfa16d 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -23,6 +23,7 @@
 #include <pwd.h>
 #include <grp.h>
 #include <fcntl.h>
+#include <limits.h>
 
 #if __APPLE__ || __FreeBSD__
 #include <sys/ucred.h>
@@ -967,14 +968,14 @@ int main(int argc, char * * argv)
                     if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
                         throw SysError("waiting for data from client or server");
                     if (FD_ISSET(s, &fds)) {
-                        auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SIZE_MAX, SPLICE_F_MOVE);
+                        auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from daemon socket to stdout");
                         else if (res == 0)
                             throw EndOfFile("unexpected EOF from daemon socket");
                     }
                     if (FD_ISSET(STDIN_FILENO, &fds)) {
-                        auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SIZE_MAX, SPLICE_F_MOVE);
+                        auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)

From 9ff9c3f2f80ba4108e9c945bbfda2c64735f987b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Feb 2017 14:20:00 +0100
Subject: [PATCH 0105/2196] Add support for s3:// URIs

This adds support for s3:// URIs in all places where Nix allows URIs,
e.g. in builtins.fetchurl, builtins.fetchTarball, <nix/fetchurl.nix>
and NIX_PATH. It allows fetching resources from private S3 buckets,
using credentials obtained from the standard places (i.e. AWS_*
environment variables, ~/.aws/credentials and the EC2 metadata
server). This may not be super-useful in general, but since we already
depend on aws-sdk-cpp, it's a cheap feature to add.
---
 src/libstore/download.cc              |  28 ++++-
 src/libstore/download.hh              |   2 +-
 src/libstore/s3-binary-cache-store.cc | 141 +++++++++++++++-----------
 src/libstore/s3.hh                    |  33 ++++++
 src/libutil/logging.hh                |   1 +
 5 files changed, 142 insertions(+), 63 deletions(-)
 create mode 100644 src/libstore/s3.hh

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 074e0ca6642..85215439a0f 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -4,6 +4,7 @@
 #include "hash.hh"
 #include "store-api.hh"
 #include "archive.hh"
+#include "s3.hh"
 
 #include <unistd.h>
 #include <fcntl.h>
@@ -480,6 +481,31 @@ struct CurlDownloader : public Downloader
         std::function<void(const DownloadResult &)> success,
         std::function<void(std::exception_ptr exc)> failure) override
     {
+        /* Ugly hack to support s3:// URIs. */
+        if (hasPrefix(request.uri, "s3://")) {
+            // FIXME: do this on a worker thread
+            sync2async<DownloadResult>(success, failure, [&]() {
+#ifdef ENABLE_S3
+                S3Helper s3Helper;
+                auto slash = request.uri.find('/', 5);
+                if (slash == std::string::npos)
+                    throw nix::Error("bad S3 URI ‘%s’", request.uri);
+                std::string bucketName(request.uri, 5, slash - 5);
+                std::string key(request.uri, slash + 1);
+                // FIXME: implement ETag
+                auto s3Res = s3Helper.getObject(bucketName, key);
+                DownloadResult res;
+                if (!s3Res.data)
+                    throw DownloadError(NotFound, fmt("S3 object ‘%s’ does not exist", request.uri));
+                res.data = s3Res.data;
+                return res;
+#else
+                throw nix::Error("cannot download ‘%s’ because Nix is not built with S3 support", request.uri);
+#endif
+            });
+            return;
+        }
+
         auto item = std::make_shared<DownloadItem>(*this, request);
         item->success = success;
         item->failure = failure;
@@ -629,7 +655,7 @@ bool isUri(const string & s)
     size_t pos = s.find("://");
     if (pos == string::npos) return false;
     string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git";
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3";
 }
 
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 82b5d641fde..bdb5011e783 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -23,7 +23,7 @@ struct DownloadRequest
 
 struct DownloadResult
 {
-    bool cached;
+    bool cached = false;
     std::string etag;
     std::string effectiveUrl;
     std::shared_ptr<std::string> data;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index cc1b3310420..ac083410b35 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -1,6 +1,6 @@
 #if ENABLE_S3
-#if __linux__
 
+#include "s3.hh"
 #include "s3-binary-cache-store.hh"
 #include "nar-info.hh"
 #include "nar-info-disk-cache.hh"
@@ -18,15 +18,6 @@
 
 namespace nix {
 
-struct istringstream_nocopy : public std::stringstream
-{
-    istringstream_nocopy(const std::string & s)
-    {
-        rdbuf()->pubsetbuf(
-            (char *) s.data(), s.size());
-    }
-};
-
 struct S3Error : public Error
 {
     Aws::S3::S3Errors err;
@@ -60,21 +51,81 @@ static void initAWS()
     });
 }
 
+S3Helper::S3Helper()
+    : config(makeConfig())
+    , client(make_ref<Aws::S3::S3Client>(*config))
+{
+}
+
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig()
+{
+    initAWS();
+    auto res = make_ref<Aws::Client::ClientConfiguration>();
+    res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
+    res->requestTimeoutMs = 600 * 1000;
+    return res;
+}
+
+S3Helper::DownloadResult S3Helper::getObject(
+    const std::string & bucketName, const std::string & key)
+{
+    debug("fetching ‘s3://%s/%s’...", bucketName, key);
+
+    auto request =
+        Aws::S3::Model::GetObjectRequest()
+        .WithBucket(bucketName)
+        .WithKey(key);
+
+    request.SetResponseStreamFactory([&]() {
+        return Aws::New<std::stringstream>("STRINGSTREAM");
+    });
+
+    DownloadResult res;
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    try {
+
+        auto result = checkAws(fmt("AWS error fetching ‘%s’", key),
+            client->GetObject(request));
+
+        res.data = std::make_shared<std::string>(
+            dynamic_cast<std::stringstream &>(result.GetBody()).str());
+
+    } catch (S3Error & e) {
+        if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+
+    return res;
+}
+
+#if __linux__
+
+struct istringstream_nocopy : public std::stringstream
+{
+    istringstream_nocopy(const std::string & s)
+    {
+        rdbuf()->pubsetbuf(
+            (char *) s.data(), s.size());
+    }
+};
+
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     std::string bucketName;
 
-    ref<Aws::Client::ClientConfiguration> config;
-    ref<Aws::S3::S3Client> client;
-
     Stats stats;
 
+    S3Helper s3Helper;
+
     S3BinaryCacheStoreImpl(
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , config(makeConfig())
-        , client(make_ref<Aws::S3::S3Client>(*config))
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -84,15 +135,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return "s3://" + bucketName;
     }
 
-    ref<Aws::Client::ClientConfiguration> makeConfig()
-    {
-        initAWS();
-        auto res = make_ref<Aws::Client::ClientConfiguration>();
-        res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
-        res->requestTimeoutMs = 600 * 1000;
-        return res;
-    }
-
     void init() override
     {
         if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
@@ -100,7 +142,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             /* Create the bucket if it doesn't already exists. */
             // FIXME: HeadBucket would be more appropriate, but doesn't return
             // an easily parsed 404 message.
-            auto res = client->GetBucketLocation(
+            auto res = s3Helper.client->GetBucketLocation(
                 Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
 
             if (!res.IsSuccess()) {
@@ -108,7 +150,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                     throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
 
                 checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
-                    client->CreateBucket(
+                    s3Helper.client->CreateBucket(
                         Aws::S3::Model::CreateBucketRequest()
                         .WithBucket(bucketName)
                         .WithCreateBucketConfiguration(
@@ -146,7 +188,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     {
         stats.head++;
 
-        auto res = client->HeadObject(
+        auto res = s3Helper.client->HeadObject(
             Aws::S3::Model::HeadObjectRequest()
             .WithBucket(bucketName)
             .WithKey(path));
@@ -179,7 +221,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         auto now1 = std::chrono::steady_clock::now();
 
         auto result = checkAws(format("AWS error uploading ‘%s’") % path,
-            client->PutObject(request));
+            s3Helper.client->PutObject(request));
 
         auto now2 = std::chrono::steady_clock::now();
 
@@ -198,42 +240,18 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
             debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
 
-            auto request =
-                Aws::S3::Model::GetObjectRequest()
-                .WithBucket(bucketName)
-                .WithKey(path);
-
-            request.SetResponseStreamFactory([&]() {
-                return Aws::New<std::stringstream>("STRINGSTREAM");
-            });
-
             stats.get++;
 
-            try {
-
-                auto now1 = std::chrono::steady_clock::now();
-
-                auto result = checkAws(format("AWS error fetching ‘%s’") % path,
-                    client->GetObject(request));
-
-                auto now2 = std::chrono::steady_clock::now();
+            auto res = s3Helper.getObject(bucketName, path);
 
-                auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();
+            stats.getBytes += res.data ? res.data->size() : 0;
+            stats.getTimeMs += res.durationMs;
 
-                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+            if (res.data)
+                printTalkative("downloaded ‘s3://%s/%s’ (%d bytes) in %d ms",
+                    bucketName, path, res.data->size(), res.durationMs);
 
-                printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
-                    % bucketName % path % res.size() % duration);
-
-                stats.getBytes += res.size();
-                stats.getTimeMs += duration;
-
-                return std::make_shared<std::string>(res);
-
-            } catch (S3Error & e) {
-                if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return std::shared_ptr<std::string>();
-                throw;
-            }
+            return res.data;
         });
     }
 
@@ -246,7 +264,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker);
 
             auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName,
-                client->ListObjects(
+                s3Helper.client->ListObjects(
                     Aws::S3::Model::ListObjectsRequest()
                     .WithBucket(bucketName)
                     .WithDelimiter("/")
@@ -281,7 +299,8 @@ static RegisterStoreImplementation regStore([](
     return store;
 });
 
+#endif
+
 }
 
 #endif
-#endif
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
new file mode 100644
index 00000000000..5d5d3475c44
--- /dev/null
+++ b/src/libstore/s3.hh
@@ -0,0 +1,33 @@
+#pragma once
+
+#if ENABLE_S3
+
+#include "ref.hh"
+
+namespace Aws { namespace Client { class ClientConfiguration; } }
+namespace Aws { namespace S3 { class S3Client; } }
+
+namespace nix {
+
+struct S3Helper
+{
+    ref<Aws::Client::ClientConfiguration> config;
+    ref<Aws::S3::S3Client> client;
+
+    S3Helper();
+
+    ref<Aws::Client::ClientConfiguration> makeConfig();
+
+    struct DownloadResult
+    {
+        std::shared_ptr<std::string> data;
+        unsigned int durationMs;
+    };
+
+    DownloadResult getObject(
+        const std::string & bucketName, const std::string & key);
+};
+
+}
+
+#endif
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 3e6c4b54853..3f83664794f 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -78,6 +78,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
 
 #define printError(args...) printMsg(lvlError, args)
 #define printInfo(args...) printMsg(lvlInfo, args)
+#define printTalkative(args...) printMsg(lvlTalkative, args)
 #define debug(args...) printMsg(lvlDebug, args)
 #define vomit(args...) printMsg(lvlVomit, args)
 

From 40f0e3b366458a6f1bc09a366a218abf13a1af1e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Feb 2017 13:55:43 +0100
Subject: [PATCH 0106/2196] Include "curl" in the User-Agent header

Some sites (e.g. BitBucket) give a helpful 401 error when trying to
download a private archive if the User-Agent contains "curl", but give
a redirect to a login page otherwise (so for instance
"nix-prefetch-url" will succeed but produce useless output).
---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index f93fb1e968a..d65ac7b873d 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -201,7 +201,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
             curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
-            curl_easy_setopt(req, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str());
+            curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif

From b63f79175e128f8ae394d2be7d65999263eebe27 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Feb 2017 14:06:47 +0100
Subject: [PATCH 0107/2196] <nix/fetchurl.nix>: Remove unnecessary assertion

---
 corepkgs/fetchurl.nix | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix
index 042705b1abb..62359433971 100644
--- a/corepkgs/fetchurl.nix
+++ b/corepkgs/fetchurl.nix
@@ -1,24 +1,20 @@
 { system ? builtins.currentSystem
 , url
-, outputHash ? ""
-, outputHashAlgo ? ""
 , md5 ? "", sha1 ? "", sha256 ? ""
+, outputHash ?
+    if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+, outputHashAlgo ?
+    if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
 , executable ? false
 , unpack ? false
 , name ? baseNameOf (toString url)
 }:
 
-assert (outputHash != "" && outputHashAlgo != "")
-    || md5 != "" || sha1 != "" || sha256 != "";
-
 derivation {
   builder = "builtin:fetchurl";
 
   # New-style output content requirements.
-  outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else
-      if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5";
-  outputHash = if outputHash != "" then outputHash else
-      if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
+  inherit outputHashAlgo outputHash;
   outputHashMode = if unpack || executable then "recursive" else "flat";
 
   inherit name system url executable unpack;

From bd5388e7b22daa2d22c21578ef5735be8b8353a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Feb 2017 14:23:30 +0100
Subject: [PATCH 0108/2196] Tweak netrc docs

---
 doc/manual/command-ref/conf-file.xml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index a7d60538c21..0f7a2deba04 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -437,10 +437,18 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     trying to download from a remote host through HTTP or HTTPS. Defaults to
     <filename>$NIX_CONF_DIR/netrc</filename>.</para>
 
-    <para>The <filename>netrc</filename> file consists of zero or more lines
-    like: <literal>machine <replaceable>my-machine</replaceable> login
-    <replaceable>my-username</replaceable> password
-    <replaceable>my-password</replaceable></literal>.</para></listitem>
+    <para>The <filename>netrc</filename> file consists of a list of
+    accounts in the following format:
+
+<screen>
+machine <replaceable>my-machine</replaceable>
+login <replaceable>my-username</replaceable>
+password <replaceable>my-password</replaceable>
+</screen>
+
+    For the exact syntax, see <link
+    xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
+    <literal>curl</literal> documentation.</link></para></listitem>
 
   </varlistentry>
 

From cde4b609192d11dc299ea3c27d7f92735f161db1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Feb 2017 14:46:36 +0100
Subject: [PATCH 0109/2196] Move netrcFile to Settings

Also get rid of Settings::processEnvironment(), it appears to be
useless.
---
 perl/lib/Nix/Store.xs    |  1 -
 src/libmain/shared.cc    |  1 -
 src/libstore/download.cc |  7 ++-----
 src/libstore/globals.cc  | 36 +++++++++++++++++-------------------
 src/libstore/globals.hh  |  6 ++++--
 5 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 6b137a13c41..f613e3df329 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -26,7 +26,6 @@ static ref<Store> store()
     if (!_store) {
         try {
             logger = makeDefaultLogger();
-            settings.processEnvironment();
             settings.loadConfFile();
             settings.update();
             settings.lockCPU = false;
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 56aa3db0015..53fa83fe0de 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -112,7 +112,6 @@ void initNix()
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
 
-    settings.processEnvironment();
     settings.loadConfFile();
 
     startSignalHandlerThread();
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d65ac7b873d..d301d4409b0 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -232,11 +232,8 @@ struct CurlDownloader : public Downloader
             }
 
             /* If no file exist in the specified path, curl continues to work
-             * anyway as if netrc support was disabled. */
-            Path netrcFile = settings.get("netrc-file",
-               (format("%1%/%2%") % settings.nixConfDir % "netrc").str());
-            /* Curl copies the given C string, so the following call is safe. */
-            curl_easy_setopt(req, CURLOPT_NETRC_FILE, netrcFile.c_str());
+               anyway as if netrc support was disabled. */
+            curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
             result.data = std::make_shared<std::string>();
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 90f83a5bbd9..474288b7812 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -23,6 +23,21 @@ Settings settings;
 
 Settings::Settings()
 {
+    nixPrefix = NIX_PREFIX;
+    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
+    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
+    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
+    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
+    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
+    nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
+    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+    nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
+
+    // should be set with the other config options, but depends on nixLibexecDir
+#ifdef __APPLE__
+    preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies";
+#endif
+
     keepFailed = false;
     keepGoing = false;
     tryFallback = false;
@@ -57,25 +72,7 @@ Settings::Settings()
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
     showTrace = false;
     enableImportNative = false;
-}
-
-
-void Settings::processEnvironment()
-{
-    nixPrefix = NIX_PREFIX;
-    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
-    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
-    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
-    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
-    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
-    nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
-    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
-    nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
-
-    // should be set with the other config options, but depends on nixLibexecDir
-#ifdef __APPLE__
-    preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies";
-#endif
+    netrcFile = fmt("%s/%s", nixConfDir, "netrc");
 }
 
 
@@ -183,6 +180,7 @@ void Settings::update()
     _get(preBuildHook, "pre-build-hook");
     _get(keepGoing, "keep-going");
     _get(keepFailed, "keep-failed");
+    _get(netrcFile, "netrc-file");
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index a423b4e5c0f..0ff18f8b16e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -16,8 +16,6 @@ struct Settings {
 
     Settings();
 
-    void processEnvironment();
-
     void loadConfFile();
 
     void set(const string & name, const string & value);
@@ -193,6 +191,10 @@ struct Settings {
        build settings */
     Path preBuildHook;
 
+    /* Path to the netrc file used to obtain usernames/passwords for
+       downloads. */
+    Path netrcFile;
+
 private:
     SettingsMap settings, overrides;
 

From 302386f775eea309679654e5ea7c972fb6e7b9af Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Feb 2017 15:42:49 +0100
Subject: [PATCH 0110/2196] Support netrc in <nix/fetchurl.nix>

This allows <nix/fetchurl.nix> to fetch private Git/Mercurial
repositories, e.g.

  import <nix/fetchurl.nix> {
    url = https://edolstra@bitbucket.org/edolstra/my-private-repo/get/80a14018daed.tar.bz2;
    sha256 = "1mgqzn7biqkq3hf2697b0jc4wabkqhmzq2srdymjfa6sb9zb6qs7";
  }

where /etc/nix/netrc contains:

  machine bitbucket.org
  login edolstra
  password blabla...

This works even when sandboxing is enabled.

To do: add unpacking support (i.e. fetchzip functionality).
---
 src/libstore/build.cc    | 14 ++++++++++++--
 src/libstore/builtins.cc | 10 +++++++++-
 src/libstore/builtins.hh |  2 +-
 src/libutil/util.cc      |  4 ++--
 src/libutil/util.hh      |  2 +-
 5 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1aee150fda3..1ce23135fc3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2307,6 +2307,14 @@ void DerivationGoal::runChild()
 
         bool setUser = true;
 
+        /* Make the contents of netrc available to builtin:fetchurl
+           (which may run under a different uid and/or in a sandbox). */
+        std::string netrcData;
+        try {
+            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
+                netrcData = readFile(settings.netrcFile);
+        } catch (SysError &) { }
+
 #if __linux__
         if (useChroot) {
 
@@ -2675,7 +2683,7 @@ void DerivationGoal::runChild()
         if (drv->isBuiltin()) {
             try {
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv);
+                    builtinFetchurl(*drv, netrcData);
                 else
                     throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8));
                 _exit(0);
@@ -3072,7 +3080,9 @@ void DerivationGoal::closeLogFile()
 void DerivationGoal::deleteTmpDir(bool force)
 {
     if (tmpDir != "") {
-        if (settings.keepFailed && !force) {
+        /* Don't keep temporary directories for builtins because they
+           might have privileged stuff (like a copy of netrc). */
+        if (settings.keepFailed && !force && !drv->isBuiltin()) {
             printError(
                 format("note: keeping build directory ‘%2%’")
                 % drvPath % tmpDir);
diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index a30f30906f0..c5dbd57f8bc 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -6,8 +6,16 @@
 
 namespace nix {
 
-void builtinFetchurl(const BasicDerivation & drv)
+void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 {
+    /* Make the host's netrc data available. Too bad curl requires
+       this to be stored in a file. It would be nice if we could just
+       pass a pointer to the data. */
+    if (netrcData != "") {
+        settings.netrcFile = "netrc";
+        writeFile(settings.netrcFile, netrcData, 0600);
+    }
+
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
         if (i == drv.env.end()) throw Error(format("attribute ‘%s’ missing") % name);
diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh
index 4b2431aa08c..0cc6ba31f65 100644
--- a/src/libstore/builtins.hh
+++ b/src/libstore/builtins.hh
@@ -4,6 +4,6 @@
 
 namespace nix {
 
-void builtinFetchurl(const BasicDerivation & drv);
+void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
 
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 33659936800..0a5f796e4ea 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -288,9 +288,9 @@ string readFile(const Path & path, bool drain)
 }
 
 
-void writeFile(const Path & path, const string & s)
+void writeFile(const Path & path, const string & s, mode_t mode)
 {
-    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0666);
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
         throw SysError(format("opening file ‘%1%’") % path);
     writeFull(fd.get(), s);
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index cfaaf1486e9..2950f7daa5e 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -89,7 +89,7 @@ string readFile(int fd);
 string readFile(const Path & path, bool drain = false);
 
 /* Write a string to a file. */
-void writeFile(const Path & path, const string & s);
+void writeFile(const Path & path, const string & s, mode_t mode = 0666);
 
 /* Read a line from a file descriptor. */
 string readLine(int fd);

From b8564987a3d9455ee779ca1dd63b1dc9f8ee72d9 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Sun, 19 Feb 2017 18:54:18 +0000
Subject: [PATCH 0111/2196] Document toString better

---
 doc/manual/expressions/builtins.xml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 063bc04be48..6c38941c01b 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1023,10 +1023,18 @@ in foo</programlisting>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.
-    <replaceable>e</replaceable> can be a string (in which case
-    <function>toString</function> is a no-op), a path (e.g.,
-    <literal>toString /foo/bar</literal> yields
-    <literal>"/foo/bar"</literal> or a set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
+    <replaceable>e</replaceable> can be:
+    <itemizedlist>
+        <listitem>a string (in which case the string is returned unmodified)</listitem>
+        <listitem>a path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal></listitem>
+        <listitem>a set containing <literal>{ __toString = self: ...; }</literal></listitem>
+        <listitem>an integer</listitem>
+        <listitem>a list, in which case the string representations of its elements are joined with spaces</listitem>
+        <listitem>a boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal></listitem>
+        <listitem><literal>null</literal>, which yields the empty string.</listitem>
+    </itemizedlist>
+    </para>
+    </listitem>
 
   </varlistentry>
 

From df66d346dfc3c1d1136256ea58d0419d12599a50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 11:42:38 +0100
Subject: [PATCH 0112/2196] Log AWS retries

---
 src/libstore/s3-binary-cache-store.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ac083410b35..041c68c6816 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -8,6 +8,7 @@
 
 #include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetBucketLocationRequest.h>
@@ -57,12 +58,25 @@ S3Helper::S3Helper()
 {
 }
 
+/* Log AWS retries. */
+class RetryStrategy : public Aws::Client::DefaultRetryStrategy
+{
+    long CalculateDelayBeforeNextRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
+    {
+        auto res = Aws::Client::DefaultRetryStrategy::CalculateDelayBeforeNextRetry(error, attemptedRetries);
+        printError("AWS error '%s' (%s), will retry in %d ms",
+            error.GetExceptionName(), error.GetMessage(), res);
+        return res;
+    }
+};
+
 ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig()
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
     res->requestTimeoutMs = 600 * 1000;
+    res->retryStrategy = std::make_shared<RetryStrategy>();
     return res;
 }
 

From 79f4583f8a3facdd6422ab8c8530a4f5295e1045 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 13:04:31 +0100
Subject: [PATCH 0113/2196] Fix XML validity

---
 doc/manual/expressions/builtins.xml | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 6c38941c01b..e9baff65961 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1023,17 +1023,16 @@ in foo</programlisting>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.
-    <replaceable>e</replaceable> can be:
+    <replaceable>e</replaceable> can be:</para>
     <itemizedlist>
-        <listitem>a string (in which case the string is returned unmodified)</listitem>
-        <listitem>a path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal></listitem>
-        <listitem>a set containing <literal>{ __toString = self: ...; }</literal></listitem>
-        <listitem>an integer</listitem>
-        <listitem>a list, in which case the string representations of its elements are joined with spaces</listitem>
-        <listitem>a boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal></listitem>
-        <listitem><literal>null</literal>, which yields the empty string.</listitem>
+      <listitem><para>A string (in which case the string is returned unmodified).</para></listitem>
+      <listitem><para>A path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal>.</para></listitem>
+      <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
+      <listitem><para>An integer.</para></listitem>
+      <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem>
+      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>.</para></listitem>
+      <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem>
     </itemizedlist>
-    </para>
     </listitem>
 
   </varlistentry>

From 99bbddedb1e70fc885e05cb398d8bfb438b039ed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 13:15:07 +0100
Subject: [PATCH 0114/2196] Fix building without S3 support

http://hydra.nixos.org/build/49031196/nixlog/2/raw
---
 src/libstore/download.cc         | 2 +-
 src/libstore/legacy-ssh-store.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d301d4409b0..0ca41b15bf7 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -489,7 +489,7 @@ struct CurlDownloader : public Downloader
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
             // FIXME: do this on a worker thread
-            sync2async<DownloadResult>(success, failure, [&]() {
+            sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
 #ifdef ENABLE_S3
                 S3Helper s3Helper;
                 auto slash = request.uri.find('/', 5);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5d9e5aad6e0..b20ff185f9b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -225,7 +225,7 @@ struct LegacySSHStore : public Store
     void collectGarbage(const GCOptions & options, GCResults & results) override
     { unsupported(); }
 
-    ref<FSAccessor> getFSAccessor()
+    ref<FSAccessor> getFSAccessor() override
     { unsupported(); }
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override

From bb6656b8a259f2ad0f90536ee32bc3080e466f1a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 14:26:23 +0100
Subject: [PATCH 0115/2196] Build RPMs for Fedora 25

Disabled hardened build because it makes the linker fail with messages like

  relocation R_X86_64_PC32 against undefined symbol `BZ2_bzWriteOpen' can not be used when making a shared object; recompile with -fPIC

See https://fedoraproject.org/wiki/Changes/Harden_All_Packages.
---
 nix.spec.in | 2 ++
 release.nix | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 401a2dc8a1f..2447f13a8f8 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -1,3 +1,5 @@
+%undefine _hardened_build
+
 %global nixbld_user "nix-builder-"
 %global nixbld_group "nixbld"
 
diff --git a/release.nix b/release.nix
index ace0f9cc89a..8d71cc841c7 100644
--- a/release.nix
+++ b/release.nix
@@ -171,8 +171,8 @@ let
       };
 
 
-    rpm_fedora21i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora21i386) [ "libsodium-devel" ];
-    rpm_fedora21x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora21x86_64) [ "libsodium-devel" ];
+    rpm_fedora25i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora25i386) [ "libsodium-devel" ];
+    rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
 
 
     deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
@@ -256,8 +256,8 @@ let
           deb_debian8x86_64
           deb_ubuntu1504i386
           deb_ubuntu1504x86_64
-          rpm_fedora21i386
-          rpm_fedora21x86_64
+          rpm_fedora25i386
+          rpm_fedora25x86_64
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball

From e4dd7dadf4f52a2afef0fcd826d8c85ec13cd904 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 14:52:36 +0100
Subject: [PATCH 0116/2196] RPM build: Use parallel make

---
 nix.spec.in | 2 +-
 release.nix | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 2447f13a8f8..0c9b9ab2013 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -107,7 +107,7 @@ extraFlags=
 %configure --localstatedir=/nix/var \
            --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
            $extraFlags
-make %{?_smp_flags}
+make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
 %{_emacs_bytecompile} misc/emacs/nix-mode.el
 
 
diff --git a/release.nix b/release.nix
index 8d71cc841c7..0b5b5579b26 100644
--- a/release.nix
+++ b/release.nix
@@ -284,9 +284,10 @@ let
         { extraPackages =
             [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
             ++ extraPackages; };
-      memSize = 1024;
+      memSize = 8192;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
+      enableParallelBuilding = true;
     };
 
 

From b95ce3194dc431be2da9d79aecd5a7865fb5d486 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 14:56:50 +0100
Subject: [PATCH 0117/2196] Debian build: Use parallel make and add Ubuntu
 16.10

---
 release.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 0b5b5579b26..91573276129 100644
--- a/release.nix
+++ b/release.nix
@@ -186,6 +186,8 @@ let
     deb_ubuntu1510x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1510x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
     deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
 
 
     # System tests.
@@ -306,7 +308,7 @@ let
         { extraPackages =
             [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
             ++ extraPackages; };
-      memSize = 1024;
+      memSize = 8192;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
@@ -315,6 +317,7 @@ let
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;
+      enableParallelBuilding = true;
     };
 
 

From c0a133876ec58d902cf0dd74e550af50eb5d1832 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 15:01:42 +0100
Subject: [PATCH 0118/2196] Revert "configure.ac: We require C++14 now"

This reverts commit 81c53fe8e56f4a4ce10088fe2d7b6a524a6dc126. This
check appears to be stricter than we need (it broke a bunch of
platforms that previously did build:
http://hydra.nixos.org/eval/1331921#tabs-now-fail).
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index bfe81840c3b..e6b11be2df1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -61,7 +61,7 @@ CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX_14
+AX_CXX_COMPILE_STDCXX_11
 
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.

From 1a57f499b06b6b37cdf98787d38614f1449d1dd7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 15:19:54 +0100
Subject: [PATCH 0119/2196] Drop some Ubuntu releases

---
 release.nix | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/release.nix b/release.nix
index 91573276129..dc49b20ccd6 100644
--- a/release.nix
+++ b/release.nix
@@ -7,7 +7,7 @@ let
 
   pkgs = import <nixpkgs> {};
 
-  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" /* "x86_64-freebsd" "i686-freebsd" */ ];
+  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" ];
 
 
   jobs = rec {
@@ -180,10 +180,6 @@ let
 
     deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [];
     deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [];
-    deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
-    deb_ubuntu1510i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1510i386) [ "libsodium-dev" ] [ "libsodium13"];
-    deb_ubuntu1510x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1510x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
     deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
@@ -244,20 +240,16 @@ let
       meta.description = "Release-critical builds";
       constituents =
         [ tarball
-          #build.i686-freebsd
           build.i686-linux
           build.x86_64-darwin
-          #build.x86_64-freebsd
           build.x86_64-linux
-          #binaryTarball.i686-freebsd
           binaryTarball.i686-linux
           binaryTarball.x86_64-darwin
-          #binaryTarball.x86_64-freebsd
           binaryTarball.x86_64-linux
           deb_debian8i386
           deb_debian8x86_64
-          deb_ubuntu1504i386
-          deb_ubuntu1504x86_64
+          deb_ubuntu1604i386
+          deb_ubuntu1604x86_64
           rpm_fedora25i386
           rpm_fedora25x86_64
           tests.remoteBuilds

From 8d7c6644c592b7d7f31e23d8915f0b634c191dd3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 15:21:56 +0100
Subject: [PATCH 0120/2196] useChroot -> useSandbox

---
 tests/remote-builds.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index d14d6ff7f05..63aaa4d88f5 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -14,7 +14,7 @@ let
     { services.openssh.enable = true;
       virtualisation.writableStore = true;
       nix.package = nix;
-      nix.useChroot = true;
+      nix.useSandbox = true;
     };
 
   # Trivial Nix expression to build remotely.

From 8b1b5f9a12d4b5196be791118c58ae253ba02d96 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Feb 2017 16:04:47 +0100
Subject: [PATCH 0121/2196] Handle CURLE_RECV_ERROR as a transient error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes

  unable to download ‘https://cache.nixos.org/nar/077h8ji74y9b0qx7rjk71xd80vjqp6q5gy137r553jlvdlxdcdlk.nar.xz’: HTTP error 200 (curl error: Failure when receiving data from the peer)
---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 0ca41b15bf7..25ccd7d0b52 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -273,7 +273,7 @@ struct CurlDownloader : public Downloader
                     httpStatus == 403 ? Forbidden :
                     (httpStatus == 408 || httpStatus == 500 || httpStatus == 503
                         || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
-                        || code == CURLE_COULDNT_RESOLVE_HOST) ? Transient :
+                        || code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_RECV_ERROR) ? Transient :
                     Misc;
 
                 attempt++;

From b8ce649a352e915a653850767b519b9ee7a6ebc2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Feb 2017 13:54:11 +0100
Subject: [PATCH 0122/2196] Fix 32-bit RPM/Deb builds

http://hydra.nixos.org/build/49130529
---
 release.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/release.nix b/release.nix
index dc49b20ccd6..854650bb905 100644
--- a/release.nix
+++ b/release.nix
@@ -278,10 +278,10 @@ let
         { extraPackages =
             [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
             ++ extraPackages; };
-      memSize = 8192;
+      #memSize = 8192;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
-      enableParallelBuilding = true;
+      #enableParallelBuilding = true;
     };
 
 
@@ -300,7 +300,7 @@ let
         { extraPackages =
             [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
             ++ extraPackages; };
-      memSize = 8192;
+      #memSize = 8192;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
@@ -309,7 +309,7 @@ let
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;
-      enableParallelBuilding = true;
+      #enableParallelBuilding = true;
     };
 
 

From fe2db1dae59f379eda793da952425a8331139c65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Feb 2017 15:39:17 +0100
Subject: [PATCH 0123/2196] Doh

---
 release.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 854650bb905..78f96196399 100644
--- a/release.nix
+++ b/release.nix
@@ -278,7 +278,7 @@ let
         { extraPackages =
             [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
             ++ extraPackages; };
-      #memSize = 8192;
+      memSize = 1024;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;
@@ -300,7 +300,7 @@ let
         { extraPackages =
             [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
             ++ extraPackages; };
-      #memSize = 8192;
+      memSize = 1024;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";

From f023f64f40809afa8a8e3a5e7081209cbcfd2e7e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Feb 2017 16:31:04 +0100
Subject: [PATCH 0124/2196] RemoteStore::addToStore(): Pass
 content-addressability assertion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

... and use this in Downloader::downloadCached(). This fixes

  $ nix-build https://nixos.org/channels/nixos-16.09-small/nixexprs.tar.xz -A hello
  error: cannot import path ‘/nix/store/csfbp1s60dkgmk9f8g0zk0mwb7hzgabd-nixexprs.tar.xz’ because it lacks a valid signature
---
 src/libstore/download.cc     | 1 +
 src/libstore/local-store.cc  | 2 +-
 src/libstore/remote-store.cc | 2 +-
 src/libstore/store-api.cc    | 6 ++++++
 src/libstore/store-api.hh    | 7 ++++++-
 src/nix-daemon/nix-daemon.cc | 1 +
 6 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 25ccd7d0b52..a56fd6922a5 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -612,6 +612,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                 Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
                 info.path = store->makeFixedOutputPath(false, hash, name);
                 info.narHash = hashString(htSHA256, *sink.s);
+                info.ca = makeFixedOutputCA(false, hash);
                 store->addToStore(info, sink.s, false, true);
                 storePath = info.path;
             }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4c161cfb341..9b775e16a38 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1002,7 +1002,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
             info.narHash = hash.first;
             info.narSize = hash.second;
             info.ultimate = true;
-            info.ca = "fixed:" + (recursive ? (std::string) "r:" : "") + h.to_string();
+            info.ca = makeFixedOutputCA(recursive, h);
             registerValidPath(info);
         }
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 42c09ec7e0b..7f398685a2c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -380,7 +380,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
         conn->to << wopAddToStoreNar
                  << info.path << info.deriver << printHash(info.narHash)
                  << info.references << info.registrationTime << info.narSize
-                 << info.ultimate << info.sigs << *nar << repair << dontCheckSigs;
+                 << info.ultimate << info.sigs << info.ca << *nar << repair << dontCheckSigs;
         // FIXME: don't send nar as a string
         conn->processStderr();
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b5934a0d123..401b001b2d8 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -676,6 +676,12 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
+std::string makeFixedOutputCA(bool recursive, const Hash & hash)
+{
+    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
+}
+
+
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d03e70849f9..c344b9d66ed 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -128,7 +128,7 @@ struct ValidPathInfo
        of an output path of a derivation were actually produced by
        that derivation. In the intensional model, we have to trust
        that a particular output path was produced by a derivation; the
-       path name then implies the contents.)
+       path then implies the contents.)
 
        Ideally, the content-addressability assertion would just be a
        Boolean, and the store path would be computed from
@@ -687,6 +687,11 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
     bool hashGiven = false);
 
 
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+   for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(bool recursive, const Hash & hash);
+
+
 MakeError(SubstError, Error)
 MakeError(BuildError, Error) /* denotes a permanent build failure */
 MakeError(InvalidPath, Error)
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 3b43ddfa16d..9fbc43b82d7 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -582,6 +582,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         info.narSize = readLongLong(from);
         info.ultimate = readLongLong(from);
         info.sigs = readStrings<StringSet>(from);
+        info.ca = readString(from);
         auto nar = make_ref<std::string>(readString(from));
         auto repair = readInt(from) ? true : false;
         auto dontCheckSigs = readInt(from) ? true : false;

From e7cb2847ab1cec48eac6a86c56885b3f0df76275 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Tue, 21 Feb 2017 22:50:18 -0500
Subject: [PATCH 0125/2196] Explicitly model all settings and fail on
 unrecognized ones

Previously, the Settings class allowed other code to query for string
properties, which led to a proliferation of code all over the place making
up new options without any sort of central registry of valid options. This
commit pulls all those options back into the central Settings class and
removes the public get() methods, to discourage future abuses like that.

Furthermore, because we know the full set of options ahead of time, we
now fail loudly if someone enters an unrecognized option, thus preventing
subtle typos. With some template fun, we could probably also dump the full
set of options (with documentation, defaults, etc.) to the command line,
but I'm not doing that yet here.
---
 src/libexpr/eval.cc          |   2 +-
 src/libstore/build.cc        |  43 +++-------
 src/libstore/crypto.cc       |   4 +-
 src/libstore/download.cc     |   6 +-
 src/libstore/globals.cc      | 148 +++++++++++++++++++++++++++--------
 src/libstore/globals.hh      |  86 ++++++++++++++++++--
 src/libstore/local-store.cc  |   4 +-
 src/libstore/store-api.cc    |   6 +-
 src/nix-daemon/nix-daemon.cc |   4 +-
 src/nix-store/nix-store.cc   |   2 +-
 10 files changed, 218 insertions(+), 87 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d418ab4e43a..f6cdedb3797 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -299,7 +299,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 {
     countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
 
-    restricted = settings.get("restrict-eval", false);
+    restricted = settings.restrictEval;
 
     assert(gcInitialised);
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1ce23135fc3..4a7e1a62b50 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -34,13 +34,6 @@
 #include <pwd.h>
 #include <grp.h>
 
-/* chroot-like behavior from Apple's sandbox */
-#if __APPLE__
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
-#else
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
-#endif
-
 /* Includes required for chroot support. */
 #if __linux__
 #include <sys/socket.h>
@@ -1279,7 +1272,7 @@ void DerivationGoal::inputsRealised()
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
-    nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1;
+    nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1;
 
     /* Okay, try to build.  Note that here we don't wait for a build
        slot to become available, since we don't need one if there is a
@@ -1685,9 +1678,7 @@ void DerivationGoal::startBuilder()
 
     /* Are we doing a chroot build? */
     {
-        string x = settings.get("build-use-sandbox",
-            /* deprecated alias */
-            settings.get("build-use-chroot", string("false")));
+        string x = settings.useSandbox;
         if (x != "true" && x != "false" && x != "relaxed")
             throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’");
         if (x == "true") {
@@ -1744,21 +1735,10 @@ void DerivationGoal::startBuilder()
 
     if (useChroot) {
 
-        string defaultChrootDirs;
-#if __linux__
-        if (worker.store.isInStore(BASH_PATH))
-            defaultChrootDirs = "/bin/sh=" BASH_PATH;
-#endif
-
         /* Allow a user-configurable set of directories from the
            host file system. */
-        PathSet dirs = tokenizeString<StringSet>(
-            settings.get("build-sandbox-paths",
-                /* deprecated alias with lower priority */
-                settings.get("build-chroot-dirs", defaultChrootDirs)));
-        PathSet dirs2 = tokenizeString<StringSet>(
-            settings.get("build-extra-chroot-dirs",
-                settings.get("build-extra-sandbox-paths", string(""))));
+        PathSet dirs = settings.sandboxPaths;
+        PathSet dirs2 = settings.extraSandboxPaths;
         dirs.insert(dirs2.begin(), dirs2.end());
 
         dirsInChroot.clear();
@@ -1790,8 +1770,7 @@ void DerivationGoal::startBuilder()
         for (auto & i : closure)
             dirsInChroot[i] = i;
 
-        string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES));
-        PathSet allowedPaths = tokenizeString<StringSet>(allowed);
+        PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
         Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps"));
@@ -1811,7 +1790,7 @@ void DerivationGoal::startBuilder()
                 }
             }
             if (!found)
-                throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed);
+                throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps") % drvPath % i);
 
             dirsInChroot[i] = i;
         }
@@ -2433,7 +2412,7 @@ void DerivationGoal::runChild()
             /* Mount a new tmpfs on /dev/shm to ensure that whatever
                the builder puts in /dev/shm is cleaned up automatically. */
             if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
-                    fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1)
+                    fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
                 throw SysError("mounting /dev/shm");
 
 #if 0
@@ -2596,7 +2575,7 @@ void DerivationGoal::runChild()
             sandboxProfile += "(version 1)\n";
 
             /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-            if (settings.get("darwin-log-sandbox-violations", false)) {
+            if (settings.darwinLogSandboxViolations) {
                 sandboxProfile += "(deny default)\n";
             } else {
                 sandboxProfile += "(deny default (with no-log))\n";
@@ -2743,7 +2722,7 @@ void DerivationGoal::registerOutputs()
     InodesSeen inodesSeen;
 
     Path checkSuffix = ".check";
-    bool runDiffHook = settings.get("run-diff-hook", false);
+    bool runDiffHook = settings.runDiffHook;
     bool keepPreviousRound = settings.keepFailed || runDiffHook;
 
     /* Check whether the output paths were created, and grep each
@@ -2981,7 +2960,7 @@ void DerivationGoal::registerOutputs()
                     ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev)
                     : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath);
 
-                auto diffHook = settings.get("diff-hook", std::string(""));
+                auto diffHook = settings.diffHook;
                 if (prevExists && diffHook != "" && runDiffHook) {
                     try {
                         auto diff = runProgram(diffHook, true, {prev, i->path});
@@ -2992,7 +2971,7 @@ void DerivationGoal::registerOutputs()
                     }
                 }
 
-                if (settings.get("enforce-determinism", true))
+                if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);
 
                 printError(msg);
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 747483afb30..9692dd83b4e 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -105,12 +105,12 @@ PublicKeys getDefaultPublicKeys()
 
     // FIXME: filter duplicates
 
-    for (auto s : settings.get("binary-cache-public-keys", Strings())) {
+    for (auto s : settings.binaryCachePublicKeys) {
         PublicKey key(s);
         publicKeys.emplace(key.name, key);
     }
 
-    for (auto secretKeyFile : settings.get("secret-key-files", Strings())) {
+    for (auto secretKeyFile : settings.secretKeyFiles) {
         try {
             SecretKey secretKey(readFile(secretKeyFile));
             publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 25ccd7d0b52..661ee2ed54b 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -331,9 +331,9 @@ struct CurlDownloader : public Downloader
         curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
         #endif
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            settings.get("binary-caches-parallel-connections", 25));
+            settings.binaryCachesParallelConnections);
 
-        enableHttp2 = settings.get("enable-http2", true);
+        enableHttp2 = settings.enableHttp2;
 
         wakeupPipe.create();
         fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
@@ -573,7 +573,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     string expectedETag;
 
-    int ttl = settings.get("tarball-ttl", 60 * 60);
+    int ttl = settings.tarballTtl;
     bool skip = false;
 
     if (pathExists(fileLink) && pathExists(dataFile)) {
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 474288b7812..62ed0376d71 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -17,12 +17,23 @@ namespace nix {
    must be deleted and recreated on startup.) */
 #define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
+/* chroot-like behavior from Apple's sandbox */
+#if __APPLE__
+    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
+#else
+    #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
+#endif
 
 Settings settings;
 
 
 Settings::Settings()
 {
+    deprecatedOptions = StringSet({
+        "build-use-chroot", "build-chroot-dirs", "build-extra-chroot-dirs",
+        "this-option-never-existed-but-who-will-know"
+    });
+
     nixPrefix = NIX_PREFIX;
     nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
     nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
@@ -73,6 +84,32 @@ Settings::Settings()
     showTrace = false;
     enableImportNative = false;
     netrcFile = fmt("%s/%s", nixConfDir, "netrc");
+    useSandbox = "false"; // TODO: make into an enum
+
+#if __linux__
+    sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH);
+#endif
+
+    restrictEval = false;
+    buildRepeat = 0;
+    allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
+    sandboxShmSize = "50%";
+    darwinLogSandboxViolations = false;
+    runDiffHook = false;
+    diffHook = "";
+    enforceDeterminism = true;
+    binaryCachePublicKeys = Strings();
+    secretKeyFiles = Strings();
+    binaryCachesParallelConnections = 25;
+    enableHttp2 = true;
+    tarballTtl = 60 * 60;
+    signedBinaryCaches = "";
+    substituters = Strings();
+    binaryCaches = Strings();
+    extraBinaryCaches = Strings();
+    trustedUsers = Strings({"root"});
+    allowedUsers = Strings({"*"});
+    printMissing = true;
 }
 
 
@@ -115,39 +152,6 @@ void Settings::set(const string & name, const string & value)
     overrides[name] = value;
 }
 
-
-string Settings::get(const string & name, const string & def)
-{
-    auto i = settings.find(name);
-    if (i == settings.end()) return def;
-    return i->second;
-}
-
-
-Strings Settings::get(const string & name, const Strings & def)
-{
-    auto i = settings.find(name);
-    if (i == settings.end()) return def;
-    return tokenizeString<Strings>(i->second);
-}
-
-
-bool Settings::get(const string & name, bool def)
-{
-    bool res = def;
-    _get(res, name);
-    return res;
-}
-
-
-int Settings::get(const string & name, int def)
-{
-    int res = def;
-    _get(res, name);
-    return res;
-}
-
-
 void Settings::update()
 {
     _get(tryFallback, "build-fallback");
@@ -181,13 +185,71 @@ void Settings::update()
     _get(keepGoing, "keep-going");
     _get(keepFailed, "keep-failed");
     _get(netrcFile, "netrc-file");
+    _get(useSandbox, "build-use-sandbox", "build-use-chroot");
+    _get(sandboxPaths, "build-sandbox-paths", "build-chroot-dirs");
+    _get(extraSandboxPaths, "build-extra-sandbox-paths", "build-extra-chroot-dirs");
+    _get(restrictEval, "restrict-eval");
+    _get(buildRepeat, "build-repeat");
+    _get(allowedImpureHostPrefixes, "allowed-impure-host-deps");
+    _get(sandboxShmSize, "sandbox-dev-shm-size");
+    _get(darwinLogSandboxViolations, "darwin-log-sandbox-violations");
+    _get(runDiffHook, "run-diff-hook");
+    _get(diffHook, "diff-hook");
+    _get(enforceDeterminism, "enforce-determinism");
+    _get(binaryCachePublicKeys, "binary-cache-public-keys");
+    _get(secretKeyFiles, "secret-key-files");
+    _get(binaryCachesParallelConnections, "binary-caches-parallel-connections");
+    _get(enableHttp2, "enable-http2");
+    _get(tarballTtl, "tarball-ttl");
+    _get(signedBinaryCaches, "signed-binary-caches");
+    _get(substituters, "substituters");
+    _get(binaryCaches, "binary-caches");
+    _get(extraBinaryCaches, "extra-binary-caches");
+    _get(trustedUsers, "trusted-users");
+    _get(allowedUsers, "allowed-users");
+    _get(printMissing, "print-missing");
+
+    /* Clear out any deprecated options that might be left, so users know we recognize the option
+       but aren't processing it anymore */
+    for (auto &i : deprecatedOptions) {
+        if (settings.find(i) != settings.end()) {
+            printError(format("warning: deprecated option '%1%' is no longer supported and will be ignored") % i);
+            settings.erase(i);
+        }
+    }
+
+    if (settings.size() != 0) {
+        string bad;
+        for (auto &i : settings)
+            bad += "'" + i.first + "', ";
+        bad.pop_back();
+        bad.pop_back();
+        throw Error(format("unrecognized options: %s") % bad);
+    }
 }
 
+void Settings::checkDeprecated(const string & name)
+{
+    if (deprecatedOptions.find(name) != deprecatedOptions.end())
+        printError(format("warning: deprecated option '%1%' will soon be unsupported") % name);
+}
 
 void Settings::_get(string & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
+    res = i->second;
+}
+
+void Settings::_get(string & res, const string & name1, const string & name2)
+{
+    SettingsMap::iterator i = settings.find(name1);
+    if (i == settings.end()) i = settings.find(name2);
+    if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
     res = i->second;
 }
 
@@ -196,6 +258,8 @@ void Settings::_get(bool & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
     if (i->second == "true") res = true;
     else if (i->second == "false") res = false;
     else throw Error(format("configuration option ‘%1%’ should be either ‘true’ or ‘false’, not ‘%2%’")
@@ -207,6 +271,20 @@ void Settings::_get(StringSet & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
+    res.clear();
+    Strings ss = tokenizeString<Strings>(i->second);
+    res.insert(ss.begin(), ss.end());
+}
+
+void Settings::_get(StringSet & res, const string & name1, const string & name2)
+{
+    SettingsMap::iterator i = settings.find(name1);
+    if (i == settings.end()) i = settings.find(name2);
+    if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
     res.clear();
     Strings ss = tokenizeString<Strings>(i->second);
     res.insert(ss.begin(), ss.end());
@@ -216,6 +294,8 @@ void Settings::_get(Strings & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
     res = tokenizeString<Strings>(i->second);
 }
 
@@ -224,6 +304,8 @@ template<class N> void Settings::_get(N & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;
+    checkDeprecated(i->first);
+    settings.erase(i);
     if (!string2Int(i->second, res))
         throw Error(format("configuration setting ‘%1%’ should have an integer value") % name);
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0ff18f8b16e..d74488a41b3 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -20,14 +20,6 @@ struct Settings {
 
     void set(const string & name, const string & value);
 
-    string get(const string & name, const string & def);
-
-    Strings get(const string & name, const Strings & def);
-
-    bool get(const string & name, bool def);
-
-    int get(const string & name, int def);
-
     void update();
 
     string pack();
@@ -36,6 +28,10 @@ struct Settings {
 
     SettingsMap getOverrides();
 
+    /* TODO: the comments below should be strings and exposed via a nice command-line UI or similar.
+       We should probably replace it with some sort of magic template or macro to minimize the amount
+       of duplication and pain here. */
+
     /* The directory where we store sources and derived files. */
     Path nixStore;
 
@@ -187,6 +183,75 @@ struct Settings {
     /* Whether the importNative primop should be enabled */
     bool enableImportNative;
 
+    /* Whether to enable sandboxed builds (string until we get an enum for true/false/relaxed) */
+    string useSandbox;
+
+    /* The basic set of paths to expose in a sandbox */
+    PathSet sandboxPaths;
+
+    /* Any extra sandbox paths to expose */
+    PathSet extraSandboxPaths;
+
+    /* Whether to allow certain questionable operations (like fetching) during evaluation */
+    bool restrictEval;
+
+    /* The number of times to repeat a build to check for determinism */
+    int buildRepeat;
+
+    /* Which prefixes to allow derivations to ask for access to (primarily for Darwin) */
+    PathSet allowedImpureHostPrefixes;
+
+    /* The size of /dev/shm in the build sandbox (for Linux) */
+    string sandboxShmSize;
+
+    /* Whether to log Darwin sandbox access violations to the system log */
+    bool darwinLogSandboxViolations;
+
+    /* ??? */
+    bool runDiffHook;
+
+    /* ??? */
+    string diffHook;
+
+    /* Whether to fail if repeated builds produce different output */
+    bool enforceDeterminism;
+
+    /* The known public keys for a binary cache */
+    Strings binaryCachePublicKeys;
+
+    /* Secret keys to use for build output signing */
+    Strings secretKeyFiles;
+
+    /* Number of parallel connections to hit a binary cache with when finding out if it contains hashes */
+    int binaryCachesParallelConnections;
+
+    /* Whether to enable HTTP2 */
+    bool enableHttp2;
+
+    /* How soon to expire tarballs like builtins.fetchTarball and (ugh, bad name) builtins.fetchurl */
+    int tarballTtl;
+
+    /* ??? */
+    string signedBinaryCaches;
+
+    /* ??? */
+    Strings substituters;
+
+    /* ??? */
+    Strings binaryCaches;
+
+    /* ??? */
+    Strings extraBinaryCaches;
+
+    /* Who we trust to ask the daemon to do unsafe things */
+    Strings trustedUsers;
+
+    /* ?Who we trust to use the daemon in safe ways */
+    Strings allowedUsers;
+
+    /* ??? */
+    bool printMissing;
+
     /* The hook to run just before a build to set derivation-specific
        build settings */
     Path preBuildHook;
@@ -196,11 +261,16 @@ struct Settings {
     Path netrcFile;
 
 private:
+    StringSet deprecatedOptions;
     SettingsMap settings, overrides;
 
+    void checkDeprecated(const string & name);
+
     void _get(string & res, const string & name);
+    void _get(string & res, const string & name1, const string & name2);
     void _get(bool & res, const string & name);
     void _get(StringSet & res, const string & name);
+    void _get(StringSet & res, const string & name1, const string & name2);
     void _get(Strings & res, const string & name);
     template<class N> void _get(N & res, const string & name);
 };
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4c161cfb341..afcda6e2be7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -44,7 +44,7 @@ LocalStore::LocalStore(const Params & params)
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
     , trashDir(realStoreDir + "/trash")
-    , requireSigs(trim(settings.get("signed-binary-caches", std::string(""))) != "") // FIXME: rename option
+    , requireSigs(trim(settings.signedBinaryCaches) != "") // FIXME: rename option
     , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());
@@ -1330,7 +1330,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 {
     // FIXME: keep secret keys in memory.
 
-    auto secretKeyFiles = settings.get("secret-key-files", Strings());
+    auto secretKeyFiles = settings.secretKeyFiles;
 
     for (auto & secretKeyFile : secretKeyFiles) {
         SecretKey secretKey(readFile(secretKeyFile));
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b5934a0d123..60342427751 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -766,13 +766,13 @@ std::list<ref<Store>> getDefaultSubstituters()
         state->stores.push_back(openStore(uri));
     };
 
-    for (auto uri : settings.get("substituters", Strings()))
+    for (auto uri : settings.substituters)
         addStore(uri);
 
-    for (auto uri : settings.get("binary-caches", Strings()))
+    for (auto uri : settings.binaryCaches)
         addStore(uri);
 
-    for (auto uri : settings.get("extra-binary-caches", Strings()))
+    for (auto uri : settings.extraBinaryCaches)
         addStore(uri);
 
     state->done = true;
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 3b43ddfa16d..2fcb5b565a9 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -865,8 +865,8 @@ static void daemonLoop(char * * argv)
             struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0;
             string group = gr ? gr->gr_name : std::to_string(peer.gid);
 
-            Strings trustedUsers = settings.get("trusted-users", Strings({"root"}));
-            Strings allowedUsers = settings.get("allowed-users", Strings({"*"}));
+            Strings trustedUsers = settings.trustedUsers;
+            Strings allowedUsers = settings.allowedUsers;
 
             if (matchUser(user, group, trustedUsers))
                 trusted = true;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 0aabe66c562..bb3b430c9ec 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -146,7 +146,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
         unknown = PathSet();
     }
 
-    if (settings.get("print-missing", true))
+    if (settings.printMissing)
         printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (dryRun) return;

From 668fef2e4f1c7758e2a55e355b4826014d5b7ba9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Feb 2017 13:31:46 +0100
Subject: [PATCH 0126/2196] nix-shell: Overwrite environment variables

Need to remember that std::map::insert() and emplace() don't overwrite
existing entries...

This fixes a regression relative to 1.11 that in particular triggers
in nested nix-shells.

Before:

  $ nativeBuildInputs=/foo nix-shell -p hello --run 'hello'
  build input /foo does not exist

After:

  $ nativeBuildInputs=/foo nix-shell -p hello --run 'hello'
  Hello, world!
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index ee030c57b6b..f4484ba572d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -408,7 +408,7 @@ int main(int argc, char ** argv)
                 env["NIX_STORE"] = store->storeDir;
 
                 for (auto & var : drv.env)
-                    env.emplace(var);
+                    env[var.first] = var.second;
 
                 restoreAffinity();
 

From 1c718f80d3e91c6dd8f1337f81b37e4837384c75 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Feb 2017 16:38:14 +0100
Subject: [PATCH 0127/2196] Verify content-addressability assertions at
 registration time

---
 src/libstore/local-store.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 9b775e16a38..bc63955af0e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -519,6 +519,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
+    assert(info.ca == "" || info.isContentAddressed(*this));
+
     state.stmtRegisterValidPath.use()
         (info.path)
         ("sha256:" + printHash(info.narHash))

From e76df9bd5285272f025025d97fe8993156726eef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Feb 2017 16:39:53 +0100
Subject: [PATCH 0128/2196] Register content-addressability assertion for fixed
 outputs

---
 src/libstore/build.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1ce23135fc3..eef9549669b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2753,6 +2753,8 @@ void DerivationGoal::registerOutputs()
         Path path = i.second.path;
         if (missingPaths.find(path) == missingPaths.end()) continue;
 
+        ValidPathInfo info;
+
         Path actualPath = path;
         if (useChroot) {
             actualPath = chrootRootDir + path;
@@ -2855,6 +2857,8 @@ void DerivationGoal::registerOutputs()
                         format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected")
                         % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h));
             }
+
+            info.ca = makeFixedOutputCA(recursive, h2);
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -2954,7 +2958,6 @@ void DerivationGoal::registerOutputs()
             worker.markContentsGood(path);
         }
 
-        ValidPathInfo info;
         info.path = path;
         info.narHash = hash.first;
         info.narSize = hash.second;

From d3e1aad421a88a7a075cac25142c49224b3ecca2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Feb 2017 17:25:00 +0100
Subject: [PATCH 0129/2196] nix-shell: Better error message when the shell
 can't be started

---
 src/nix-build/nix-build.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f4484ba572d..b81c98868e6 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -448,15 +448,17 @@ int main(int argc, char ** argv)
 
                 auto envPtrs = stringsToCharPtrs(envStrs);
 
+                auto shell = getEnv("NIX_BUILD_SHELL", "bash");
+
                 environ = envPtrs.data();
 
                 auto argPtrs = stringsToCharPtrs(args);
 
                 restoreSignals();
 
-                execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), argPtrs.data());
+                execvp(shell.c_str(), argPtrs.data());
 
-                throw SysError("executing shell");
+                throw SysError("executing shell ‘%s’", shell);
             }
 
             // Ugly hackery to make "nix-build -A foo.all" produce symlinks

From 89ffe1eff946639aa2752177fcc5159b6926af70 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Feb 2017 17:25:22 +0100
Subject: [PATCH 0130/2196] Fix nix-shell tests

The nix-shell fix in 668fef2e4f1c7758e2a55e355b4826014d5b7ba9 revealed
that we had some --pure tests that incorrectly depended on PATH from
config.nix's mkDerivation being overwritten by the caller's PATH.

http://hydra.nixos.org/build/49242478
---
 tests/nix-shell.sh | 1 +
 tests/shell.nix    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 26cc521bbcb..f0f34a5f870 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -4,6 +4,7 @@ clearStore
 
 # Test nix-shell -A
 export IMPURE_VAR=foo
+export NIX_BUILD_SHELL=$SHELL
 output=$(nix-shell --pure shell.nix -A shellDrv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"')
 
diff --git a/tests/shell.nix b/tests/shell.nix
index ed4d6fbaaa0..1a092913b3b 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -34,6 +34,7 @@ rec {
     mkdir -p $out/bin
     echo 'echo foo' > $out/bin/foo
     chmod a+rx $out/bin/foo
+    ln -s ${shell} $out/bin/bash
   '';
 
   bar = runCommand "bar" {} ''

From f72206b736e4fa1be010a05aa12fa57afea2019c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Mon, 27 Feb 2017 10:09:48 +0100
Subject: [PATCH 0131/2196] unpack-channel.nix: fix 'mv' corner case

unpack-channel.nix fails if the tarball contains a directory named the
same as the channel:

  mv: cannot move 'nixpkgs' to a subdirectory of itself, '.../nixpkgs'

This commit fixes that by not moving the directory if it already has the
correct name.
---
 corepkgs/unpack-channel.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix
index 9445532ded0..a654db40e62 100644
--- a/corepkgs/unpack-channel.nix
+++ b/corepkgs/unpack-channel.nix
@@ -15,7 +15,9 @@ let
       else
         ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
       fi
-      mv * $out/$channelName
+      if [ * != $channelName ]; then
+        mv * $out/$channelName
+      fi
       if [ -n "$binaryCacheURL" ]; then
         mkdir $out/binary-caches
         echo -n "$binaryCacheURL" > $out/binary-caches/$channelName

From 3cb0387d3f0932c9ce6f0aca2663860da7a65fb0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 27 Feb 2017 14:36:09 +0100
Subject: [PATCH 0132/2196] Retry downloads on HTTP/2 stream errors

Issue #1254.
---
 src/libstore/download.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index a56fd6922a5..c0cec658cbb 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -273,7 +273,9 @@ struct CurlDownloader : public Downloader
                     httpStatus == 403 ? Forbidden :
                     (httpStatus == 408 || httpStatus == 500 || httpStatus == 503
                         || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
-                        || code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_RECV_ERROR) ? Transient :
+                        || code == CURLE_COULDNT_RESOLVE_HOST
+                        || code == CURLE_RECV_ERROR
+                        || code == CURLE_HTTP2_STREAM) ? Transient :
                     Misc;
 
                 attempt++;

From 4f3fb34844eef457d910fbbb78fe676f696eeac5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 27 Feb 2017 15:58:24 +0100
Subject: [PATCH 0133/2196] Fix 1.8 release note about build-max-jobs

We set build-cores automatically, not build-max-jobs. (The commit
message for de4cdd0d47adc70a4db12397a42c18ee50b4e662 also got this
wrong.)
---
 doc/manual/release-notes/rl-1.8.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml
index 48caac2c6b6..c854c5c5f85 100644
--- a/doc/manual/release-notes/rl-1.8.xml
+++ b/doc/manual/release-notes/rl-1.8.xml
@@ -83,8 +83,8 @@ $ nix-store -l $(which xterm)
   caches).</para></listitem>
 
   <listitem><para>The configuration option
-  <option>build-max-jobs</option> now defaults to the number of
-  available CPU cores.</para></listitem>
+  <option>build-cores</option> now defaults to the number of available
+  CPU cores.</para></listitem>
 
   <listitem><para>Build users are now used by default when Nix is
   invoked as root. This prevents builds from accidentally running as

From 3fab1f04a7d9a5d8ca0f7faf071d5767f93c7e22 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 27 Feb 2017 16:01:54 +0100
Subject: [PATCH 0134/2196] _SC_NPROCESSORS_ONLN ->
 std::thread::hardware_concurrency()

---
 src/libstore/globals.cc | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 474288b7812..12e2a3cf43d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -2,9 +2,9 @@
 #include "util.hh"
 #include "archive.hh"
 
-#include <map>
 #include <algorithm>
-#include <unistd.h>
+#include <map>
+#include <thread>
 
 
 namespace nix {
@@ -42,11 +42,7 @@ Settings::Settings()
     keepGoing = false;
     tryFallback = false;
     maxBuildJobs = 1;
-    buildCores = 1;
-#ifdef _SC_NPROCESSORS_ONLN
-    long res = sysconf(_SC_NPROCESSORS_ONLN);
-    if (res > 0) buildCores = res;
-#endif
+    buildCores = std::max(1U, std::thread::hardware_concurrency());
     readOnlyMode = false;
     thisSystem = SYSTEM;
     maxSilentTime = 0;

From 7251d048fa812d2551b7003bc9f13a8f5d4c95a5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Feb 2017 12:54:50 +0100
Subject: [PATCH 0135/2196] Support auto-configuration of build-max-jobs

"build-max-jobs" and the "-j" option can now be set to "auto" to use
the number of CPUs in the system. (Unlike build-cores, it doesn't use
0 to imply auto-configuration, because a) magic values are a bad idea
in general; b) 0 is a legitimate value used to disable local
building.)

Fixes #1198.
---
 doc/manual/command-ref/conf-file.xml  | 6 +++---
 doc/manual/command-ref/opt-common.xml | 5 +++--
 src/libmain/shared.cc                 | 5 ++++-
 src/libstore/globals.cc               | 9 ++++++++-
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 0f7a2deba04..96f8a4b608a 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -101,9 +101,9 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
-    <literal>1</literal>.  You should generally set it to the number
-    of CPUs in your system (e.g., <literal>2</literal> on an Athlon 64
-    X2).  It can be overridden using the <option
+    <literal>1</literal>. The special value <literal>auto</literal>
+    causes Nix to use the number of CPUs in your system.  It can be
+    overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>
 
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 2a076877a1b..2aa41c4d438 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -93,8 +93,9 @@
   <term><option>-j</option></term>
 
   <listitem><para>Sets the maximum number of build jobs that Nix will
-  perform in parallel to the specified number.  The default is
-  specified by the <link
+  perform in parallel to the specified number.  Specify
+  <literal>auto</literal> to use the number of CPUs in the system.
+  The default is specified by the <link
   linkend='conf-build-max-jobs'><literal>build-max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 53fa83fe0de..326202d295f 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -167,6 +167,10 @@ struct LegacyArgs : public MixCommonArgs
             settings.set("build-fallback", "true");
         });
 
+        mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
+            settings.set("build-max-jobs", s);
+        });
+
         auto intSettingAlias = [&](char shortName, const std::string & longName,
             const std::string & description, const std::string & dest) {
             mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
@@ -174,7 +178,6 @@ struct LegacyArgs : public MixCommonArgs
             });
         };
 
-        intSettingAlias('j', "max-jobs", "maximum number of parallel builds", "build-max-jobs");
         intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores");
         intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time");
         intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout");
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 12e2a3cf43d..07af629260a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -147,7 +147,14 @@ int Settings::get(const string & name, int def)
 void Settings::update()
 {
     _get(tryFallback, "build-fallback");
-    _get(maxBuildJobs, "build-max-jobs");
+
+    auto s = get("build-max-jobs", std::string("1"));
+    if (s == "auto")
+        maxBuildJobs = std::max(1U, std::thread::hardware_concurrency());
+    else
+        if (!string2Int(s, maxBuildJobs))
+            throw Error("configuration setting ‘build-max-jobs’ should be ‘auto’ or an integer");
+
     _get(buildCores, "build-cores");
     _get(thisSystem, "system");
     _get(maxSilentTime, "build-max-silent-time");

From 80027144ae765544aa96d9c38dc2dd345bcf703d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Feb 2017 13:20:12 +0100
Subject: [PATCH 0136/2196] In SQLite errors, include the database path

This is necessary because we have multiple SQLite databases (e.g. the
binary cache cache).
---
 src/libstore/sqlite.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 0197b091cd1..7d656121249 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -8,13 +8,17 @@ namespace nix {
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f)
 {
     int err = sqlite3_errcode(db);
+
+    auto path = sqlite3_db_filename(db, nullptr);
+    if (!path) path = "(in-memory)";
+
     if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
         if (err == SQLITE_PROTOCOL)
-            printError("warning: SQLite database is busy (SQLITE_PROTOCOL)");
+            printError("warning: SQLite database ‘%s’ is busy (SQLITE_PROTOCOL)", path);
         else {
             static bool warned = false;
             if (!warned) {
-                printError("warning: SQLite database is busy");
+                printError("warning: SQLite database ‘%s’ is busy", path);
                 warned = true;
             }
         }
@@ -29,10 +33,10 @@ namespace nix {
 #else
         sleep(1);
 #endif
-        throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+        throw SQLiteBusy("%s: %s (in ‘%s’)", f.str(), sqlite3_errstr(err), path);
     }
     else
-        throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+        throw SQLiteError("%s: %s (in ‘%s’)", f.str(), sqlite3_errstr(err), path);
 }
 
 SQLite::SQLite(const Path & path)

From 34b12bad597a5d9f67408bebefc1bcb65c27bc4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Feb 2017 13:44:11 +0100
Subject: [PATCH 0137/2196] NarInfoDiskCache: Handle SQLite busy errors

---
 src/libstore/nar-info-disk-cache.cc | 219 +++++++++++++++-------------
 1 file changed, 115 insertions(+), 104 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 13b67b81f35..180a936edb8 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -106,25 +106,27 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
 
         /* Periodically purge expired entries from the database. */
-        auto now = time(0);
-
-        SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
-        auto queryLastPurge_(queryLastPurge.use());
-
-        if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) {
-            SQLiteStmt(state->db,
-                "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
-                .use()
-                (now - ttlNegative)
-                (now - ttlPositive)
-                .exec();
-
-            debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
-
-            SQLiteStmt(state->db,
-                "insert or replace into LastPurge(dummy, value) values ('', ?)")
-                .use()(now).exec();
-        }
+        retrySQLite<void>([&]() {
+            auto now = time(0);
+
+            SQLiteStmt queryLastPurge(state->db, "select value from LastPurge");
+            auto queryLastPurge_(queryLastPurge.use());
+
+            if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) {
+                SQLiteStmt(state->db,
+                    "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
+                    .use()
+                    (now - ttlNegative)
+                    (now - ttlPositive)
+                    .exec();
+
+                debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
+
+                SQLiteStmt(state->db,
+                    "insert or replace into LastPurge(dummy, value) values ('', ?)")
+                    .use()(now).exec();
+            }
+        });
     }
 
     Cache & getCache(State & state, const std::string & uri)
@@ -136,114 +138,123 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
     void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
     {
-        auto state(_state.lock());
+        retrySQLite<void>([&]() {
+            auto state(_state.lock());
 
-        // FIXME: race
+            // FIXME: race
 
-        state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
-        assert(sqlite3_changes(state->db) == 1);
-        state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+            state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
+            assert(sqlite3_changes(state->db) == 1);
+            state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+        });
     }
 
     bool cacheExists(const std::string & uri,
         bool & wantMassQuery, int & priority) override
     {
-        auto state(_state.lock());
+        return retrySQLite<bool>([&]() {
+            auto state(_state.lock());
 
-        auto i = state->caches.find(uri);
-        if (i == state->caches.end()) {
-            auto queryCache(state->queryCache.use()(uri));
-            if (!queryCache.next()) return false;
-            state->caches.emplace(uri,
-                Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
-        }
+            auto i = state->caches.find(uri);
+            if (i == state->caches.end()) {
+                auto queryCache(state->queryCache.use()(uri));
+                if (!queryCache.next()) return false;
+                state->caches.emplace(uri,
+                    Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+            }
 
-        auto & cache(getCache(*state, uri));
+            auto & cache(getCache(*state, uri));
 
-        wantMassQuery = cache.wantMassQuery;
-        priority = cache.priority;
+            wantMassQuery = cache.wantMassQuery;
+            priority = cache.priority;
 
-        return true;
+            return true;
+        });
     }
 
     std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
         const std::string & uri, const std::string & hashPart) override
     {
-        auto state(_state.lock());
+        return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>(
+            [&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> {
+            auto state(_state.lock());
+
+            auto & cache(getCache(*state, uri));
 
-        auto & cache(getCache(*state, uri));
-
-        auto now = time(0);
-
-        auto queryNAR(state->queryNAR.use()
-            (cache.id)
-            (hashPart)
-            (now - ttlNegative)
-            (now - ttlPositive));
-
-        if (!queryNAR.next())
-            return {oUnknown, 0};
-
-        if (!queryNAR.getInt(13))
-            return {oInvalid, 0};
-
-        auto narInfo = make_ref<NarInfo>();
-
-        auto namePart = queryNAR.getStr(2);
-        narInfo->path = cache.storeDir + "/" +
-            hashPart + (namePart.empty() ? "" : "-" + namePart);
-        narInfo->url = queryNAR.getStr(3);
-        narInfo->compression = queryNAR.getStr(4);
-        if (!queryNAR.isNull(5))
-            narInfo->fileHash = parseHash(queryNAR.getStr(5));
-        narInfo->fileSize = queryNAR.getInt(6);
-        narInfo->narHash = parseHash(queryNAR.getStr(7));
-        narInfo->narSize = queryNAR.getInt(8);
-        for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
-            narInfo->references.insert(cache.storeDir + "/" + r);
-        if (!queryNAR.isNull(10))
-            narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
-        for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
-            narInfo->sigs.insert(sig);
-
-        return {oValid, narInfo};
+            auto now = time(0);
+
+            auto queryNAR(state->queryNAR.use()
+                (cache.id)
+                (hashPart)
+                (now - ttlNegative)
+                (now - ttlPositive));
+
+            if (!queryNAR.next())
+                return {oUnknown, 0};
+
+            if (!queryNAR.getInt(13))
+                return {oInvalid, 0};
+
+            auto narInfo = make_ref<NarInfo>();
+
+            auto namePart = queryNAR.getStr(2);
+            narInfo->path = cache.storeDir + "/" +
+                hashPart + (namePart.empty() ? "" : "-" + namePart);
+            narInfo->url = queryNAR.getStr(3);
+            narInfo->compression = queryNAR.getStr(4);
+            if (!queryNAR.isNull(5))
+                narInfo->fileHash = parseHash(queryNAR.getStr(5));
+            narInfo->fileSize = queryNAR.getInt(6);
+            narInfo->narHash = parseHash(queryNAR.getStr(7));
+            narInfo->narSize = queryNAR.getInt(8);
+            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
+                narInfo->references.insert(cache.storeDir + "/" + r);
+            if (!queryNAR.isNull(10))
+                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
+            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
+                narInfo->sigs.insert(sig);
+
+            return {oValid, narInfo};
+        });
     }
 
     void upsertNarInfo(
         const std::string & uri, const std::string & hashPart,
         std::shared_ptr<ValidPathInfo> info) override
     {
-        auto state(_state.lock());
-
-        auto & cache(getCache(*state, uri));
-
-        if (info) {
-
-            auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
-
-            assert(hashPart == storePathToHash(info->path));
-
-            state->insertNAR.use()
-                (cache.id)
-                (hashPart)
-                (storePathToName(info->path))
-                (narInfo ? narInfo->url : "", narInfo != 0)
-                (narInfo ? narInfo->compression : "", narInfo != 0)
-                (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
-                (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                (info->narHash.to_string())
-                (info->narSize)
-                (concatStringsSep(" ", info->shortRefs()))
-                (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
-                (concatStringsSep(" ", info->sigs))
-                (time(0)).exec();
-
-        } else {
-            state->insertMissingNAR.use()
-                (cache.id)
-                (hashPart)
-                (time(0)).exec();
-        }
+        retrySQLite<void>([&]() {
+            auto state(_state.lock());
+
+            auto & cache(getCache(*state, uri));
+
+            if (info) {
+
+                auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
+
+                assert(hashPart == storePathToHash(info->path));
+
+                state->insertNAR.use()
+                    (cache.id)
+                    (hashPart)
+                    (storePathToName(info->path))
+                    (narInfo ? narInfo->url : "", narInfo != 0)
+                    (narInfo ? narInfo->compression : "", narInfo != 0)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
+                    (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
+                    (info->narHash.to_string())
+                    (info->narSize)
+                    (concatStringsSep(" ", info->shortRefs()))
+                    (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
+                    (concatStringsSep(" ", info->sigs))
+                    (time(0)).exec();
+
+            } else {
+                state->insertMissingNAR.use()
+                    (cache.id)
+                    (hashPart)
+                    (time(0)).exec();
+            }
+        });
     }
 };
 

From fd86dd93dd44a826235feb0fd82aabcdaa79a65b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Feb 2017 13:59:11 +0100
Subject: [PATCH 0138/2196] Improve SQLite busy handling

---
 configure.ac           |  2 +-
 src/libstore/sqlite.cc | 66 +++++++++++++++++++++++-------------------
 src/libstore/sqlite.hh |  6 +++-
 3 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/configure.ac b/configure.ac
index e6b11be2df1..46b0ac0651b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -265,7 +265,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 
 
 # Nice to have, but not essential.
-AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep sysconf])
+AC_CHECK_FUNCS([strsignal posix_fallocate sysconf])
 
 
 # This is needed if bzip2 is a static library, and the Nix libraries
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 7d656121249..a81e62dbd6e 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -3,6 +3,8 @@
 
 #include <sqlite3.h>
 
+#include <atomic>
+
 namespace nix {
 
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f)
@@ -13,27 +15,10 @@ namespace nix {
     if (!path) path = "(in-memory)";
 
     if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
-        if (err == SQLITE_PROTOCOL)
-            printError("warning: SQLite database ‘%s’ is busy (SQLITE_PROTOCOL)", path);
-        else {
-            static bool warned = false;
-            if (!warned) {
-                printError("warning: SQLite database ‘%s’ is busy", path);
-                warned = true;
-            }
-        }
-        /* Sleep for a while since retrying the transaction right away
-           is likely to fail again. */
-        checkInterrupt();
-#if HAVE_NANOSLEEP
-        struct timespec t;
-        t.tv_sec = 0;
-        t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
-        nanosleep(&t, 0);
-#else
-        sleep(1);
-#endif
-        throw SQLiteBusy("%s: %s (in ‘%s’)", f.str(), sqlite3_errstr(err), path);
+        throw SQLiteBusy(
+            err == SQLITE_PROTOCOL
+            ? fmt("SQLite database ‘%s’ is busy (SQLITE_PROTOCOL)", path)
+            : fmt("SQLite database ‘%s’ is busy", path));
     }
     else
         throw SQLiteError("%s: %s (in ‘%s’)", f.str(), sqlite3_errstr(err), path);
@@ -58,24 +43,27 @@ SQLite::~SQLite()
 
 void SQLite::exec(const std::string & stmt)
 {
-    if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
-        throwSQLiteError(db, format("executing SQLite statement ‘%s’") % stmt);
+    retrySQLite<void>([&]() {
+        if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(db, format("executing SQLite statement ‘%s’") % stmt);
+    });
 }
 
-void SQLiteStmt::create(sqlite3 * db, const string & s)
+void SQLiteStmt::create(sqlite3 * db, const string & sql)
 {
     checkInterrupt();
     assert(!stmt);
-    if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
-        throwSQLiteError(db, "creating statement");
+    if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
+        throwSQLiteError(db, fmt("creating statement ‘%s’", sql));
     this->db = db;
+    this->sql = sql;
 }
 
 SQLiteStmt::~SQLiteStmt()
 {
     try {
         if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
-            throwSQLiteError(db, "finalizing statement");
+            throwSQLiteError(db, fmt("finalizing statement ‘%s’", sql));
     } catch (...) {
         ignoreException();
     }
@@ -132,14 +120,14 @@ void SQLiteStmt::Use::exec()
     int r = step();
     assert(r != SQLITE_ROW);
     if (r != SQLITE_DONE)
-        throwSQLiteError(stmt.db, "executing SQLite statement");
+        throwSQLiteError(stmt.db, fmt("executing SQLite statement ‘%s’", stmt.sql));
 }
 
 bool SQLiteStmt::Use::next()
 {
     int r = step();
     if (r != SQLITE_DONE && r != SQLITE_ROW)
-        throwSQLiteError(stmt.db, "executing SQLite query");
+        throwSQLiteError(stmt.db, fmt("executing SQLite query ‘%s’", stmt.sql));
     return r == SQLITE_ROW;
 }
 
@@ -186,4 +174,24 @@ SQLiteTxn::~SQLiteTxn()
     }
 }
 
+void handleSQLiteBusy(const SQLiteBusy & e)
+{
+    static std::atomic<time_t> lastWarned{0};
+
+    time_t now = time(0);
+
+    if (now > lastWarned + 10) {
+        lastWarned = now;
+        printError("warning: %s", e.what());
+    }
+
+    /* Sleep for a while since retrying the transaction right away
+       is likely to fail again. */
+    checkInterrupt();
+    struct timespec t;
+    t.tv_sec = 0;
+    t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
+    nanosleep(&t, 0);
+}
+
 }
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 4d347a2e56a..14a7a0dd899 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -30,8 +30,9 @@ struct SQLiteStmt
 {
     sqlite3 * db = 0;
     sqlite3_stmt * stmt = 0;
+    std::string sql;
     SQLiteStmt() { }
-    SQLiteStmt(sqlite3 * db, const std::string & s) { create(db, s); }
+    SQLiteStmt(sqlite3 * db, const std::string & sql) { create(db, sql); }
     void create(sqlite3 * db, const std::string & s);
     ~SQLiteStmt();
     operator sqlite3_stmt * () { return stmt; }
@@ -94,6 +95,8 @@ MakeError(SQLiteBusy, SQLiteError);
 
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f);
 
+void handleSQLiteBusy(const SQLiteBusy & e);
+
 /* Convenience function for retrying a SQLite transaction when the
    database is busy. */
 template<typename T>
@@ -103,6 +106,7 @@ T retrySQLite(std::function<T()> fun)
         try {
             return fun();
         } catch (SQLiteBusy & e) {
+            handleSQLiteBusy(e);
         }
     }
 }

From 07808052461e9534dc42f7f98e83a7b58565fd13 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Feb 2017 14:03:43 +0100
Subject: [PATCH 0139/2196] Fix building against older curl versions

http://hydra.nixos.org/build/49490928
---
 src/libstore/download.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index c0cec658cbb..75c00d85d34 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -275,7 +275,10 @@ struct CurlDownloader : public Downloader
                         || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
                         || code == CURLE_COULDNT_RESOLVE_HOST
                         || code == CURLE_RECV_ERROR
-                        || code == CURLE_HTTP2_STREAM) ? Transient :
+#if LIBCURL_VERSION_NUM >= 0x073200
+                        || code == CURLE_HTTP2_STREAM
+#endif
+                        ) ? Transient :
                     Misc;
 
                 attempt++;

From c4a40949d945b4a3be85ad68b8cfb449843f34a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 13:52:54 +0100
Subject: [PATCH 0140/2196] Handle importing NARs containing files greater than
 4 GiB

Also templatize readInt() to work for various integer types.
---
 src/libstore/derivations.cc   |  8 +++---
 src/libstore/export-import.cc |  2 +-
 src/libstore/remote-store.cc  | 24 +++++++---------
 src/libutil/serialise.cc      | 43 ++--------------------------
 src/libutil/serialise.hh      | 54 ++++++++++++++++++++++++++++++++---
 src/nix-daemon/nix-daemon.cc  | 32 +++++++++------------
 src/nix-store/nix-store.cc    |  2 +-
 7 files changed, 84 insertions(+), 81 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 79526c594f7..ce1ac7d3322 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const
 Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
 {
     drv.outputs.clear();
-    auto nr = readInt(in);
-    for (unsigned int n = 0; n < nr; n++) {
+    auto nr = readNum<size_t>(in);
+    for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
         DerivationOutput o;
         in >> o.path >> o.hashAlgo >> o.hash;
@@ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
-    nr = readInt(in);
-    for (unsigned int n = 0; n < nr; n++) {
+    nr = readNum<size_t>(in);
+    for (size_t n = 0; n < nr; n++) {
         auto key = readString(in);
         auto value = readString(in);
         drv.env[key] = value;
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index c5618c826c5..e584ae538da 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 {
     Paths res;
     while (true) {
-        unsigned long long n = readLongLong(source);
+        auto n = readNum<uint64_t>(source);
         if (n == 0) break;
         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 7f398685a2c..6428259144a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn)
         unsigned int magic = readInt(conn.from);
         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
 
-        conn.daemonVersion = readInt(conn.from);
+        conn.from >> conn.daemonVersion;
         if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
             throw Error("Nix daemon protocol version not supported");
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
@@ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
     auto conn(connections->get());
     conn->to << wopIsValidPath << path;
     conn->processStderr();
-    unsigned int reply = readInt(conn->from);
-    return reply != 0;
+    return readInt(conn->from);
 }
 
 
@@ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
         conn->to << wopQuerySubstitutablePathInfos << paths;
         conn->processStderr();
-        unsigned int count = readInt(conn->from);
-        for (unsigned int n = 0; n < count; n++) {
+        size_t count = readNum<size_t>(conn->from);
+        for (size_t n = 0; n < count; n++) {
             Path path = readStorePath(*this, conn->from);
             SubstitutablePathInfo & info(infos[path]);
             info.deriver = readString(conn->from);
@@ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
             throw;
         }
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-            bool valid = readInt(conn->from) != 0;
+            bool valid; conn->from >> valid;
             if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
         }
         auto info = std::make_shared<ValidPathInfo>();
@@ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
         if (info->deriver != "") assertStorePath(info->deriver);
         info->narHash = parseHash(htSHA256, readString(conn->from));
         info->references = readStorePaths<PathSet>(*this, conn->from);
-        info->registrationTime = readInt(conn->from);
-        info->narSize = readLongLong(conn->from);
+        conn->from >> info->registrationTime >> info->narSize;
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-            info->ultimate = readInt(conn->from) != 0;
+            conn->from >> info->ultimate;
             info->sigs = readStrings<StringSet>(conn->from);
-            info->ca = readString(conn->from);
+            conn->from >> info->ca;
         }
         return info;
     });
@@ -515,7 +513,7 @@ Roots RemoteStore::findRoots()
     auto conn(connections->get());
     conn->to << wopFindRoots;
     conn->processStderr();
-    unsigned int count = readInt(conn->from);
+    size_t count = readNum<size_t>(conn->from);
     Roots result;
     while (count--) {
         Path link = readString(conn->from);
@@ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair)
     auto conn(connections->get());
     conn->to << wopVerifyStore << checkContents << repair;
     conn->processStderr();
-    return readInt(conn->from) != 0;
+    return readInt(conn->from);
 }
 
 
@@ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         }
         else if (msg == STDERR_READ) {
             if (!source) throw Error("no source");
-            size_t len = readInt(from);
+            size_t len = readNum<size_t>(from);
             auto buf = std::make_unique<unsigned char[]>(len);
             writeString(buf.get(), source->read(buf.get(), len), to);
             to.flush();
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index a68f7a0fa8e..6064e15f5e6 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source)
 }
 
 
-unsigned int readInt(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    if (buf[4] || buf[5] || buf[6] || buf[7])
-        throw SerialisationError("implementation cannot deal with > 32-bit integers");
-    return
-        buf[0] |
-        (buf[1] << 8) |
-        (buf[2] << 16) |
-        (buf[3] << 24);
-}
-
-
-unsigned long long readLongLong(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    return
-        ((unsigned long long) buf[0]) |
-        ((unsigned long long) buf[1] << 8) |
-        ((unsigned long long) buf[2] << 16) |
-        ((unsigned long long) buf[3] << 24) |
-        ((unsigned long long) buf[4] << 32) |
-        ((unsigned long long) buf[5] << 40) |
-        ((unsigned long long) buf[6] << 48) |
-        ((unsigned long long) buf[7] << 56);
-}
-
-
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
-    size_t len = readInt(source);
+    auto len = readNum<size_t>(source);
     if (len > max) throw Error("string is too long");
     source(buf, len);
     readPadding(len, source);
@@ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
 
 string readString(Source & source)
 {
-    size_t len = readInt(source);
+    auto len = readNum<size_t>(source);
     auto buf = std::make_unique<unsigned char[]>(len);
     source(buf.get(), len);
     readPadding(len, source);
@@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s)
 }
 
 
-Source & operator >> (Source & in, unsigned int & n)
-{
-    n = readInt(in);
-    return in;
-}
-
-
 template<class T> T readStrings(Source & source)
 {
-    unsigned int count = readInt(source);
+    auto count = readNum<size_t>(source);
     T ss;
     while (count--)
         ss.insert(ss.end(), readString(source));
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5646d08c131..3072f422ea9 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s);
 Sink & operator << (Sink & sink, const StringSet & s);
 
 
+MakeError(SerialisationError, Error)
+
+
+template<typename T>
+T readNum(Source & source)
+{
+    unsigned char buf[8];
+    source(buf, sizeof(buf));
+
+    uint64_t n =
+        ((unsigned long long) buf[0]) |
+        ((unsigned long long) buf[1] << 8) |
+        ((unsigned long long) buf[2] << 16) |
+        ((unsigned long long) buf[3] << 24) |
+        ((unsigned long long) buf[4] << 32) |
+        ((unsigned long long) buf[5] << 40) |
+        ((unsigned long long) buf[6] << 48) |
+        ((unsigned long long) buf[7] << 56);
+
+    if (n > std::numeric_limits<T>::max())
+        throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name());
+
+    return n;
+}
+
+
+inline unsigned int readInt(Source & source)
+{
+    return readNum<unsigned int>(source);
+}
+
+
+inline uint64_t readLongLong(Source & source)
+{
+    return readNum<uint64_t>(source);
+}
+
+
 void readPadding(size_t len, Source & source);
-unsigned int readInt(Source & source);
-unsigned long long readLongLong(Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
 string readString(Source & source);
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);
-Source & operator >> (Source & in, unsigned int & n);
 
+template<typename T>
+Source & operator >> (Source & in, T & n)
+{
+    n = readNum<T>(in);
+    return in;
+}
 
-MakeError(SerialisationError, Error)
+template<typename T>
+Source & operator >> (Source & in, bool & b)
+{
+    b = readNum<uint64_t>(in);
+    return in;
+}
 
 
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 9fbc43b82d7..a1b3f2f6fbc 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -273,10 +273,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopAddToStore: {
-        string baseName = readString(from);
-        bool fixed = readInt(from) == 1; /* obsolete */
-        bool recursive = readInt(from) == 1;
-        string s = readString(from);
+        bool fixed, recursive;
+        std::string s, baseName;
+        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
         /* Compatibility hack. */
         if (!fixed) {
             s = "sha256";
@@ -340,7 +339,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         PathSet drvs = readStorePaths<PathSet>(*store, from);
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
-            mode = (BuildMode)readInt(from);
+            mode = (BuildMode) readInt(from);
 
             /* Repairing is not atomic, so disallowed for "untrusted"
                clients.  */
@@ -417,8 +416,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
         options.pathsToDelete = readStorePaths<PathSet>(*store, from);
-        options.ignoreLiveness = readInt(from);
-        options.maxFreed = readLongLong(from);
+        from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
         readInt(from);
@@ -438,8 +436,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopSetOptions: {
-        settings.keepFailed = readInt(from) != 0;
-        settings.keepGoing = readInt(from) != 0;
+        from >> settings.keepFailed;
+        from >> settings.keepGoing;
         settings.set("build-fallback", readInt(from) ? "true" : "false");
         verbosity = (Verbosity) readInt(from);
         settings.set("build-max-jobs", std::to_string(readInt(from)));
@@ -539,8 +537,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         break;
 
     case wopVerifyStore: {
-        bool checkContents = readInt(from) != 0;
-        bool repair = readInt(from) != 0;
+        bool checkContents, repair;
+        from >> checkContents >> repair;
         startWork();
         if (repair && !trusted)
             throw Error("you are not privileged to repair paths");
@@ -573,19 +571,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopAddToStoreNar: {
         ValidPathInfo info;
         info.path = readStorePath(*store, from);
-        info.deriver = readString(from);
+        from >> info.deriver;
         if (!info.deriver.empty())
             store->assertStorePath(info.deriver);
         info.narHash = parseHash(htSHA256, readString(from));
         info.references = readStorePaths<PathSet>(*store, from);
-        info.registrationTime = readInt(from);
-        info.narSize = readLongLong(from);
-        info.ultimate = readLongLong(from);
+        from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        info.ca = readString(from);
+        from >> info.ca;
         auto nar = make_ref<std::string>(readString(from));
-        auto repair = readInt(from) ? true : false;
-        auto dontCheckSigs = readInt(from) ? true : false;
+        bool repair, dontCheckSigs;
+        from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         startWork();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 0aabe66c562..bd889c8515d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs)
         settings.maxSilentTime = readInt(in);
         settings.buildTimeout = readInt(in);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            settings.maxLogSize = readInt(in);
+            in >> settings.maxLogSize;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
             settings.set("build-repeat", std::to_string(readInt(in)));
             settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");

From e321551d540df340fadaa1b088d9e5225193c87a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 14:41:54 +0100
Subject: [PATCH 0141/2196] Fix assertion failure in nix-store --export

Fixes #1173.
---
 src/nix-store/nix-store.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index bd889c8515d..868ec2f5a2d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -708,6 +708,9 @@ static void opExport(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         throw UsageError(format("unknown flag ‘%1%’") % i);
 
+    for (auto & i : opArgs)
+        i = store->followLinksToStorePath(i);
+
     FdSink sink(STDOUT_FILENO);
     store->exportPaths(opArgs, sink);
 }

From 56e19d970db33de6b7aef088cde57817b5a61d61 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 14:47:52 +0100
Subject: [PATCH 0142/2196] nix-store --import: Fix importing unsigned paths

---
 src/nix-store/nix-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 868ec2f5a2d..950c2a7c977 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -724,7 +724,7 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    Paths paths = store->importPaths(source, 0);
+    Paths paths = store->importPaths(source, nullptr, true);
 
     for (auto & i : paths)
         cout << format("%1%\n") % i << std::flush;

From 07a0b8ca671669794f51f9d886397bf038a8881e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 14:52:16 +0100
Subject: [PATCH 0143/2196] Tweak message

---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index bc63955af0e..63f069c2ff1 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -920,7 +920,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
             info.path % info.narHash.to_string() % h.to_string());
 
     if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys))
-        throw Error(format("cannot import path ‘%s’ because it lacks a valid signature") % info.path);
+        throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path);
 
     addTempRoot(info.path);
 

From 374908726b87f6cd137ea7d097fdcda57003594e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 14:54:11 +0100
Subject: [PATCH 0144/2196] readString(): Read directly into std::string

When reading a huge string, this halves memory consumption.

(Strictly speaking, this appears only valid in C++17, but who cares...)
---
 src/libutil/serialise.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 6064e15f5e6..950e6362a24 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -207,10 +207,10 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
 string readString(Source & source)
 {
     auto len = readNum<size_t>(source);
-    auto buf = std::make_unique<unsigned char[]>(len);
-    source(buf.get(), len);
+    std::string res(len, 0);
+    source((unsigned char*) res.data(), len);
     readPadding(len, source);
-    return string((char *) buf.get(), len);
+    return res;
 }
 
 Source & operator >> (Source & in, string & s)

From f61f67ddee12a976a0a6a20652e7c545b49fa46c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 16:07:15 +0100
Subject: [PATCH 0145/2196] RemoteStore::addToStore(): Send NAR rather than
 string containing NAR

This allows the NAR to be streamed in the future (though we're not
doing that yet).
---
 src/libstore/export-import.cc    | 23 +----------------------
 src/libstore/legacy-ssh-store.cc |  4 ++--
 src/libstore/remote-store.cc     |  5 +++--
 src/libutil/serialise.hh         |  9 +++++----
 src/nix-daemon/nix-daemon.cc     | 17 ++++++++++-------
 5 files changed, 21 insertions(+), 37 deletions(-)

diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index e584ae538da..531f010d93a 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -61,27 +61,6 @@ void Store::exportPath(const Path & path, Sink & sink)
     hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
 }
 
-struct TeeSource : Source
-{
-    Source & readSource;
-    ref<std::string> data;
-    TeeSource(Source & readSource)
-        : readSource(readSource)
-        , data(make_ref<std::string>())
-    {
-    }
-    size_t read(unsigned char * data, size_t len)
-    {
-        size_t n = readSource.read(data, len);
-        this->data->append((char *) data, n);
-        return n;
-    }
-};
-
-struct NopSink : ParseSink
-{
-};
-
 Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, bool dontCheckSigs)
 {
     Paths res;
@@ -92,7 +71,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 
         /* Extract the NAR from the source. */
         TeeSource tee(source);
-        NopSink sink;
+        ParseSink sink;
         parseDump(sink, tee);
 
         uint32_t magic = readInt(source);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index b20ff185f9b..031fcac95e5 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -169,9 +169,9 @@ struct LegacySSHStore : public Store
 
         /* FIXME: inefficient. */
         ParseSink parseSink; /* null sink; just parse the NAR */
-        SavingSourceAdapter savedNAR(conn->from);
+        TeeSource savedNAR(conn->from);
         parseDump(parseSink, savedNAR);
-        sink(savedNAR.s);
+        sink(*savedNAR.data);
     }
 
     /* Unsupported methods. */
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 6428259144a..47413d573b7 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -378,8 +378,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
         conn->to << wopAddToStoreNar
                  << info.path << info.deriver << printHash(info.narHash)
                  << info.references << info.registrationTime << info.narSize
-                 << info.ultimate << info.sigs << info.ca << *nar << repair << dontCheckSigs;
-        // FIXME: don't send nar as a string
+                 << info.ultimate << info.sigs << info.ca
+                 << repair << dontCheckSigs;
+        conn->to(*nar);
         conn->processStderr();
     }
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 3072f422ea9..2bdee70807b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -140,15 +140,16 @@ struct StringSource : Source
 
 
 /* Adapter class of a Source that saves all data read to `s'. */
-struct SavingSourceAdapter : Source
+struct TeeSource : Source
 {
     Source & orig;
-    string s;
-    SavingSourceAdapter(Source & orig) : orig(orig) { }
+    ref<std::string> data;
+    TeeSource(Source & orig)
+        : orig(orig), data(make_ref<std::string>()) { }
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        s.append((const char *) data, n);
+        this->data->append((const char *) data, n);
         return n;
     }
 };
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index a1b3f2f6fbc..17482188444 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -283,7 +283,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         }
         HashType hashAlgo = parseHashType(s);
 
-        SavingSourceAdapter savedNAR(from);
+        TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
         if (recursive) {
@@ -297,7 +297,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
 
         startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
-        Path path = store->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo);
+        Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         stopWork();
 
         to << path;
@@ -569,6 +569,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopAddToStoreNar: {
+        bool repair, dontCheckSigs;
         ValidPathInfo info;
         info.path = readStorePath(*store, from);
         from >> info.deriver;
@@ -578,14 +579,16 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         info.references = readStorePaths<PathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        from >> info.ca;
-        auto nar = make_ref<std::string>(readString(from));
-        bool repair, dontCheckSigs;
-        from >> repair >> dontCheckSigs;
+        from >> info.ca >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
+
+        TeeSource tee(from);
+        ParseSink sink;
+        parseDump(sink, tee);
+
         startWork();
-        store->addToStore(info, nar, repair, dontCheckSigs, nullptr);
+        store->addToStore(info, tee.data, repair, dontCheckSigs, nullptr);
         stopWork();
         break;
     }

From fa125b9b28bea25a4eeb4d39a71a481563127cb9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Mar 2017 16:16:04 +0100
Subject: [PATCH 0146/2196] TeeSink: Pre-reserve string space

When receiving a very large file, this can prevent the string from
having tobe copied, which temporarily doubles memory consumption.
---
 src/libstore/export-import.cc | 11 +++++------
 src/libutil/archive.hh        | 13 +++++++++++++
 src/nix-daemon/nix-daemon.cc  |  7 +++----
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 531f010d93a..2b8ab063e18 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -70,9 +70,8 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
 
         /* Extract the NAR from the source. */
-        TeeSource tee(source);
-        ParseSink sink;
-        parseDump(sink, tee);
+        TeeSink tee(source);
+        parseDump(tee, tee.source);
 
         uint32_t magic = readInt(source);
         if (magic != exportMagic)
@@ -89,14 +88,14 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         info.deriver = readString(source);
         if (info.deriver != "") assertStorePath(info.deriver);
 
-        info.narHash = hashString(htSHA256, *tee.data);
-        info.narSize = tee.data->size();
+        info.narHash = hashString(htSHA256, *tee.source.data);
+        info.narSize = tee.source.data->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, tee.data, false, dontCheckSigs, accessor);
+        addToStore(info, tee.source.data, false, dontCheckSigs, accessor);
 
         res.push_back(info.path);
     }
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index d58b91df046..c067cd2ad24 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -70,6 +70,19 @@ struct ParseSink
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
+struct TeeSink : ParseSink
+{
+    TeeSource source;
+
+    TeeSink(Source & source) : source(source) { }
+
+    void preallocateContents(unsigned long long size) override
+    {
+        source.data->reserve(source.data->size() + size + 1024);
+    };
+
+};
+
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 17482188444..ab5826b0d1a 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -583,12 +583,11 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
 
-        TeeSource tee(from);
-        ParseSink sink;
-        parseDump(sink, tee);
+        TeeSink tee(from);
+        parseDump(tee, tee.source);
 
         startWork();
-        store->addToStore(info, tee.data, repair, dontCheckSigs, nullptr);
+        store->addToStore(info, tee.source.data, repair, dontCheckSigs, nullptr);
         stopWork();
         break;
     }

From fbf17f1ad7259f08ab53dc8319ae0f452321f081 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Mar 2017 11:40:11 +0100
Subject: [PATCH 0147/2196] builtins.fetchgit: Fix bad format string

---
 src/libexpr/primops/fetchgit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index bd440c8c62a..6a5b5ac36e6 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -63,7 +63,7 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (name == "rev")
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError(format("unsupported argument ‘%1%’ to ‘fetchgit’, at %3%") % attr.name % attr.pos);
+                throw EvalError("unsupported argument ‘%s’ to ‘fetchgit’, at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())

From ffcf9d24a68fd166f5c60ebae721a9badc469b0e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Mar 2017 11:46:28 +0100
Subject: [PATCH 0148/2196] builtins.fetchgit: Support paths

---
 src/libexpr/primops/fetchgit.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 6a5b5ac36e6..09e2c077bab 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -58,9 +58,11 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
 
         for (auto & attr : *args[0]->attrs) {
             string name(attr.name);
-            if (name == "url")
-                url = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (name == "rev")
+            if (name == "url") {
+                PathSet context;
+                url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+                if (hasPrefix(url, "/")) url = "file://" + url;
+            } else if (name == "rev")
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument ‘%s’ to ‘fetchgit’, at %s", attr.name, *attr.pos);

From 2f992692e2badef2d9084335bd7290940a031671 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 15:40:06 +0100
Subject: [PATCH 0149/2196] Fix fatal "broken pipe" error when $NIX_BUILD_HOOK
 is missing

---
 src/libstore/build.cc | 66 +++++++++++++++++++++++++------------------
 1 file changed, 39 insertions(+), 27 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index eef9549669b..fd1f5dc3a4d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1580,36 +1580,48 @@ HookReply DerivationGoal::tryBuildHook()
     if (!worker.hook)
         worker.hook = std::make_unique<HookInstance>();
 
-    /* Tell the hook about system features (beyond the system type)
-       required from the build machine.  (The hook could parse the
-       drv file itself, but this is easier.) */
-    Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
-    for (auto & i : features) checkStoreName(i); /* !!! abuse */
-
-    /* Send the request to the hook. */
-    writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%")
-        % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
-        % drv->platform % drvPath % concatStringsSep(",", features)).str());
-
-    /* Read the first line of input, which should be a word indicating
-       whether the hook wishes to perform the build. */
-    string reply;
-    while (true) {
-        string s = readLine(worker.hook->fromHook.readSide.get());
-        if (string(s, 0, 2) == "# ") {
-            reply = string(s, 2);
-            break;
+    try {
+
+        /* Tell the hook about system features (beyond the system type)
+           required from the build machine.  (The hook could parse the
+           drv file itself, but this is easier.) */
+        Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
+        for (auto & i : features) checkStoreName(i); /* !!! abuse */
+
+        /* Send the request to the hook. */
+        writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%")
+                % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
+                % drv->platform % drvPath % concatStringsSep(",", features)).str());
+
+        /* Read the first line of input, which should be a word indicating
+           whether the hook wishes to perform the build. */
+        string reply;
+        while (true) {
+            string s = readLine(worker.hook->fromHook.readSide.get());
+            if (string(s, 0, 2) == "# ") {
+                reply = string(s, 2);
+                break;
+            }
+            s += "\n";
+            writeToStderr(s);
         }
-        s += "\n";
-        writeToStderr(s);
-    }
 
-    debug(format("hook reply is ‘%1%’") % reply);
+        debug(format("hook reply is ‘%1%’") % reply);
+
+        if (reply == "decline" || reply == "postpone")
+            return reply == "decline" ? rpDecline : rpPostpone;
+        else if (reply != "accept")
+            throw Error(format("bad hook reply ‘%1%’") % reply);
 
-    if (reply == "decline" || reply == "postpone")
-        return reply == "decline" ? rpDecline : rpPostpone;
-    else if (reply != "accept")
-        throw Error(format("bad hook reply ‘%1%’") % reply);
+    } catch (SysError & e) {
+        if (e.errNo == EPIPE) {
+            printError("build hook died unexpectedly: %s",
+                chomp(drainFD(worker.hook->fromHook.readSide.get())));
+            worker.hook = 0;
+            return rpDecline;
+        } else
+            throw;
+    }
 
     printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths));
 

From 5a1fb03b8fc32fd93708f5030537dd580910a35a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 16:18:49 +0100
Subject: [PATCH 0150/2196] build-remote: Misc cleanup

---
 src/build-remote/build-remote.cc | 89 +++++++++++++++-----------------
 1 file changed, 41 insertions(+), 48 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 2ce20882da1..fe68b4cd8c4 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,13 +17,12 @@
 #include "derivations.hh"
 
 using namespace nix;
-using std::cerr;
 using std::cin;
 
-static void handle_alarm(int sig) {
+static void handleAlarm(int sig) {
 }
 
-class machine {
+class Machine {
     const std::set<string> supportedFeatures;
     const std::set<string> mandatoryFeatures;
 
@@ -31,8 +30,8 @@ class machine {
     const string hostName;
     const std::vector<string> systemTypes;
     const string sshKey;
-    const unsigned long long maxJobs;
-    const unsigned long long speedFactor;
+    const unsigned int maxJobs;
+    const unsigned int speedFactor;
     bool enabled;
 
     bool allSupported(const std::set<string> & features) const {
@@ -50,28 +49,29 @@ class machine {
             });
     }
 
-    machine(decltype(hostName) hostName,
+    Machine(decltype(hostName) hostName,
         decltype(systemTypes) systemTypes,
         decltype(sshKey) sshKey,
         decltype(maxJobs) maxJobs,
         decltype(speedFactor) speedFactor,
         decltype(supportedFeatures) supportedFeatures,
         decltype(mandatoryFeatures) mandatoryFeatures) :
-        supportedFeatures{std::move(supportedFeatures)},
-        mandatoryFeatures{std::move(mandatoryFeatures)},
-        hostName{std::move(hostName)},
-        systemTypes{std::move(systemTypes)},
-        sshKey{std::move(sshKey)},
-        maxJobs{std::move(maxJobs)},
-        speedFactor{speedFactor == 0 ? 1 : std::move(speedFactor)},
-        enabled{true} {};
+        supportedFeatures(supportedFeatures),
+        mandatoryFeatures(mandatoryFeatures),
+        hostName(hostName),
+        systemTypes(systemTypes),
+        sshKey(sshKey),
+        maxJobs(maxJobs),
+        speedFactor(std::max(1U, speedFactor)),
+        enabled(true)
+    {};
 };;
 
-static std::vector<machine> read_conf()
+static std::vector<Machine> readConf()
 {
     auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines");
 
-    auto machines = std::vector<machine>{};
+    auto machines = std::vector<Machine>{};
     auto lines = std::vector<string>{};
     try {
         lines = tokenizeString<std::vector<string>>(readFile(conf), "\n");
@@ -87,10 +87,8 @@ static std::vector<machine> read_conf()
         }
         auto tokens = tokenizeString<std::vector<string>>(line);
         auto sz = tokens.size();
-        if (sz < 4) {
-            throw new FormatError(format("Bad machines.conf file %1%")
-                % conf);
-        }
+        if (sz < 4)
+            throw FormatError("bad machines.conf file ‘%1%’", conf);
         machines.emplace_back(tokens[0],
             tokenizeString<std::vector<string>>(tokens[1], ","),
             tokens[2],
@@ -108,7 +106,7 @@ static std::vector<machine> read_conf()
 
 static string currentLoad;
 
-static AutoCloseFD openSlotLock(const machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
 {
     std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out);
     fn_stream << "/";
@@ -126,15 +124,14 @@ int main (int argc, char * * argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+
         /* Ensure we don't get any SSH passphrase or host key popups. */
         if (putenv(display_env) == -1 ||
-            putenv(ssh_env) == -1) {
-            throw SysError("Setting SSH env vars");
-        }
+            putenv(ssh_env) == -1)
+            throw SysError("setting SSH env vars");
 
-        if (argc != 4) {
+        if (argc != 4)
             throw UsageError("called without required arguments");
-        }
 
         auto store = openStore();
 
@@ -147,15 +144,14 @@ int main (int argc, char * * argv)
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
 
-        auto machines = read_conf();
+        auto machines = readConf();
         string drvPath;
         string hostName;
         for (string line; getline(cin, line);) {
             auto tokens = tokenizeString<std::vector<string>>(line);
             auto sz = tokens.size();
-            if (sz != 3 && sz != 4) {
-                throw Error(format("invalid build hook line %1%") % line);
-            }
+            if (sz != 3 && sz != 4)
+                throw Error("invalid build hook line ‘%1%’", line);
             auto amWilling = tokens[0] == "1";
             auto neededSystem = tokens[1];
             drvPath = tokens[2];
@@ -174,7 +170,7 @@ int main (int argc, char * * argv)
 
                 bool rightType = false;
 
-                machine * bestMachine = nullptr;
+                Machine * bestMachine = nullptr;
                 unsigned long long bestLoad = 0;
                 for (auto & m : machines) {
                     if (m.enabled && std::find(m.systemTypes.begin(),
@@ -221,11 +217,10 @@ int main (int argc, char * * argv)
                 }
 
                 if (!bestSlotLock) {
-                    if (rightType && !canBuildLocally) {
-                        cerr << "# postpone\n";
-                    } else {
-                        cerr << "# decline\n";
-                    }
+                    if (rightType && !canBuildLocally)
+                        std::cerr << "# postpone\n";
+                    else
+                        std::cerr << "# decline\n";
                     break;
                 }
 
@@ -241,37 +236,35 @@ int main (int argc, char * * argv)
                     sshStore = openStore("ssh://" + bestMachine->hostName + "?key=" + bestMachine->sshKey);
                     hostName = bestMachine->hostName;
                 } catch (std::exception & e) {
-                    cerr << e.what() << '\n';
-                    cerr << "unable to open SSH connection to ‘" << bestMachine->hostName << "’, trying other available machines...\n";
+                    printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...",
+                        bestMachine->hostName, e.what());
                     bestMachine->enabled = false;
                     continue;
                 }
                 goto connected;
             }
         }
+
 connected:
-        cerr << "# accept\n";
+        std::cerr << "# accept\n";
         string line;
-        if (!getline(cin, line)) {
+        if (!getline(cin, line))
             throw Error("hook caller didn't send inputs");
-        }
         auto inputs = tokenizeString<std::list<string>>(line);
-        if (!getline(cin, line)) {
+        if (!getline(cin, line))
             throw Error("hook caller didn't send outputs");
-        }
         auto outputs = tokenizeString<Strings>(line);
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true);
-        auto old = signal(SIGALRM, handle_alarm);
+        auto old = signal(SIGALRM, handleAlarm);
         alarm(15 * 60);
-        if (!lockFile(uploadLock.get(), ltWrite, true)) {
-            cerr << "somebody is hogging the upload lock for " << hostName << ", continuing...\n";
-        }
+        if (!lockFile(uploadLock.get(), ltWrite, true))
+            printError("somebody is hogging the upload lock for ‘%s’, continuing...");
         alarm(0);
         signal(SIGALRM, old);
         copyPaths(store, ref<Store>(sshStore), inputs);
         uploadLock = -1;
 
-        cerr << "building ‘" << drvPath << "’ on ‘" << hostName << "’\n";
+        printError("building ‘%s’ on ‘%s’", drvPath, hostName);
         sshStore->buildDerivation(drvPath, readDerivation(drvPath));
 
         std::remove_if(outputs.begin(), outputs.end(), [=](const Path & path) { return store->isValidPath(path); });

From 7f62be1bcd2a228076a6c39eb435ad1931bb66e4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 16:33:18 +0100
Subject: [PATCH 0151/2196] build-remote: Fix passing SSH key

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index fe68b4cd8c4..fd710c2b6b7 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -233,7 +233,7 @@ int main (int argc, char * * argv)
                 lock = -1;
 
                 try {
-                    sshStore = openStore("ssh://" + bestMachine->hostName + "?key=" + bestMachine->sshKey);
+                    sshStore = openStore("ssh://" + bestMachine->hostName + "?ssh-key=" + bestMachine->sshKey);
                     hostName = bestMachine->hostName;
                 } catch (std::exception & e) {
                     printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...",

From 577ebeaefb71020f0d6b79488602fd56ba2c1863 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 19:05:50 +0100
Subject: [PATCH 0152/2196] Improve SSH handling

* Unify SSH code in SSHStore and LegacySSHStore.

* Fix a race starting the SSH master. We now wait synchronously for
  the SSH master to finish starting. This prevents the SSH clients
  from starting their own connections.

* Don't use a master if max-connections == 1.

* Add a "max-connections" store parameter.

* Add a "compress" store parameter.
---
 src/libstore/legacy-ssh-store.cc | 60 ++++++---------------
 src/libstore/remote-store.cc     | 10 ++--
 src/libstore/remote-store.hh     |  4 +-
 src/libstore/ssh-store.cc        | 83 ++++++++--------------------
 src/libstore/ssh.cc              | 93 ++++++++++++++++++++++++++++++++
 src/libstore/ssh.hh              | 41 ++++++++++++++
 src/libutil/pool.hh              |  7 ++-
 7 files changed, 185 insertions(+), 113 deletions(-)
 create mode 100644 src/libstore/ssh.cc
 create mode 100644 src/libstore/ssh.hh

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 031fcac95e5..4a2ac42f31c 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -4,6 +4,7 @@
 #include "serve-protocol.hh"
 #include "store-api.hh"
 #include "worker-protocol.hh"
+#include "ssh.hh"
 
 namespace nix {
 
@@ -11,73 +12,42 @@ static std::string uriScheme = "legacy-ssh://";
 
 struct LegacySSHStore : public Store
 {
-    string host;
-
     struct Connection
     {
-        Pid sshPid;
-        AutoCloseFD out;
-        AutoCloseFD in;
+        std::unique_ptr<SSHMaster::Connection> sshConn;
         FdSink to;
         FdSource from;
     };
 
-    AutoDelete tmpDir;
-
-    Path socketPath;
-
-    Pid sshMaster;
+    std::string host;
 
     ref<Pool<Connection>> connections;
 
-    Path key;
+    SSHMaster master;
 
-    LegacySSHStore(const string & host, const Params & params,
-        size_t maxConnections = std::numeric_limits<size_t>::max())
+    LegacySSHStore(const string & host, const Params & params)
         : Store(params)
         , host(host)
-        , tmpDir(createTempDir("", "nix", true, true, 0700))
-        , socketPath((Path) tmpDir + "/ssh.sock")
         , connections(make_ref<Pool<Connection>>(
-            maxConnections,
+            std::max(1, std::stoi(get(params, "max-connections", "1"))),
             [this]() { return openConnection(); },
             [](const ref<Connection> & r) { return true; }
             ))
-        , key(get(params, "ssh-key", ""))
+        , master(
+            host,
+            get(params, "ssh-key", ""),
+            // Use SSH master only if using more than 1 connection.
+            connections->capacity() > 1,
+            get(params, "compress", "") == "true")
     {
     }
 
     ref<Connection> openConnection()
     {
-        if ((pid_t) sshMaster == -1) {
-            sshMaster = startProcess([&]() {
-                restoreSignals();
-                Strings args{ "ssh", "-M", "-S", socketPath, "-N", "-x", "-a", host };
-                if (!key.empty())
-                    args.insert(args.end(), {"-i", key});
-                execvp("ssh", stringsToCharPtrs(args).data());
-                throw SysError("starting SSH master connection to host ‘%s’", host);
-            });
-        }
-
         auto conn = make_ref<Connection>();
-        Pipe in, out;
-        in.create();
-        out.create();
-        conn->sshPid = startProcess([&]() {
-            if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
-                throw SysError("duping over STDIN");
-            if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
-                throw SysError("duping over STDOUT");
-            execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-store", "--serve", "--write", nullptr);
-            throw SysError("executing ‘nix-store --serve’ on remote host ‘%s’", host);
-        });
-        in.readSide = -1;
-        out.writeSide = -1;
-        conn->out = std::move(out.readSide);
-        conn->in = std::move(in.writeSide);
-        conn->to = FdSink(conn->in.get());
-        conn->from = FdSource(conn->out.get());
+        conn->sshConn = master.startCommand("nix-store --serve");
+        conn->to = FdSink(conn->sshConn->in.get());
+        conn->from = FdSource(conn->sshConn->out.get());
 
         int remoteVersion;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 47413d573b7..5e62bd3d5ac 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -40,10 +40,10 @@ template PathSet readStorePaths(Store & store, Source & from);
 template Paths readStorePaths(Store & store, Source & from);
 
 /* TODO: Separate these store impls into different files, give them better names */
-RemoteStore::RemoteStore(const Params & params, size_t maxConnections)
+RemoteStore::RemoteStore(const Params & params)
     : Store(params)
     , connections(make_ref<Pool<Connection>>(
-            maxConnections,
+            std::max(1, std::stoi(get(params, "max-connections", "1"))),
             [this]() { return openConnection(); },
             [](const ref<Connection> & r) { return r->to.good() && r->from.good(); }
             ))
@@ -51,10 +51,10 @@ RemoteStore::RemoteStore(const Params & params, size_t maxConnections)
 }
 
 
-UDSRemoteStore::UDSRemoteStore(const Params & params, size_t maxConnections)
+UDSRemoteStore::UDSRemoteStore(const Params & params)
     : Store(params)
     , LocalFSStore(params)
-    , RemoteStore(params, maxConnections)
+    , RemoteStore(params)
 {
 }
 
@@ -129,7 +129,7 @@ void RemoteStore::initConnection(Connection & conn)
         conn.processStderr();
     }
     catch (Error & e) {
-        throw Error(format("cannot start daemon worker: %1%") % e.msg());
+        throw Error("cannot open connection to remote store ‘%s’: %s", getUri(), e.what());
     }
 
     setOptions(conn);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 40f17da300d..ed7b27c8886 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -22,7 +22,7 @@ class RemoteStore : public virtual Store
 {
 public:
 
-    RemoteStore(const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+    RemoteStore(const Params & params);
 
     /* Implementations of abstract store API methods. */
 
@@ -113,7 +113,7 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore
 {
 public:
 
-    UDSRemoteStore(const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+    UDSRemoteStore(const Params & params);
 
     std::string getUri() override;
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 6f1862afa89..20f020bdada 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -4,6 +4,7 @@
 #include "archive.hh"
 #include "worker-protocol.hh"
 #include "pool.hh"
+#include "ssh.hh"
 
 namespace nix {
 
@@ -13,9 +14,23 @@ class SSHStore : public RemoteStore
 {
 public:
 
-    SSHStore(string host, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+    SSHStore(const std::string & host, const Params & params)
+        : Store(params)
+        , RemoteStore(params)
+        , host(host)
+        , master(
+            host,
+            get(params, "ssh-key", ""),
+            // Use SSH master only if using more than 1 connection.
+            connections->capacity() > 1,
+            get(params, "compress", "") == "true")
+    {
+    }
 
-    std::string getUri() override;
+    std::string getUri() override
+    {
+        return uriScheme + host;
+    }
 
     void narFromPath(const Path & path, Sink & sink) override;
 
@@ -25,43 +40,16 @@ class SSHStore : public RemoteStore
 
     struct Connection : RemoteStore::Connection
     {
-        Pid sshPid;
-        AutoCloseFD out;
-        AutoCloseFD in;
+        std::unique_ptr<SSHMaster::Connection> sshConn;
     };
 
     ref<RemoteStore::Connection> openConnection() override;
 
-    AutoDelete tmpDir;
-
-    Path socketPath;
-
-    Pid sshMaster;
-
-    string host;
-
-    Path key;
+    std::string host;
 
-    bool compress;
+    SSHMaster master;
 };
 
-SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
-    : Store(params)
-    , RemoteStore(params, maxConnections)
-    , tmpDir(createTempDir("", "nix", true, true, 0700))
-    , socketPath((Path) tmpDir + "/ssh.sock")
-    , host(std::move(host))
-    , key(get(params, "ssh-key", ""))
-    , compress(get(params, "compress", "") == "true")
-{
-    /* open a connection and perform the handshake to verify all is well */
-    connections->get();
-}
-
-string SSHStore::getUri()
-{
-    return uriScheme + host;
-}
 
 class ForwardSource : public Source
 {
@@ -94,35 +82,10 @@ ref<FSAccessor> SSHStore::getFSAccessor()
 
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
-    if ((pid_t) sshMaster == -1) {
-        sshMaster = startProcess([&]() {
-            restoreSignals();
-            if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), host.c_str(), NULL);
-            else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), host.c_str(), NULL);
-            throw SysError("starting ssh master");
-        });
-    }
-
     auto conn = make_ref<Connection>();
-    Pipe in, out;
-    in.create();
-    out.create();
-    conn->sshPid = startProcess([&]() {
-        if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
-            throw SysError("duping over STDIN");
-        if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
-            throw SysError("duping over STDOUT");
-        execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-daemon", "--stdio", NULL);
-        throw SysError("executing nix-daemon --stdio over ssh");
-    });
-    in.readSide = -1;
-    out.writeSide = -1;
-    conn->out = std::move(out.readSide);
-    conn->in = std::move(in.writeSide);
-    conn->to = FdSink(conn->in.get());
-    conn->from = FdSource(conn->out.get());
+    conn->sshConn = master.startCommand("nix-daemon --stdio");
+    conn->to = FdSink(conn->sshConn->in.get());
+    conn->from = FdSource(conn->sshConn->out.get());
     initConnection(*conn);
     return conn;
 }
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
new file mode 100644
index 00000000000..7c3de4a4827
--- /dev/null
+++ b/src/libstore/ssh.cc
@@ -0,0 +1,93 @@
+#include "ssh.hh"
+
+namespace nix {
+
+std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
+{
+    startMaster();
+
+    Pipe in, out;
+    in.create();
+    out.create();
+
+    auto conn = std::make_unique<Connection>();
+    conn->sshPid = startProcess([&]() {
+        restoreSignals();
+
+        close(in.writeSide.get());
+        close(out.readSide.get());
+
+        if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("duping over stdin");
+        if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("duping over stdout");
+
+        Strings args = { "ssh", host.c_str(), "-x", "-a" };
+        if (!keyFile.empty())
+            args.insert(args.end(), {"-i", keyFile});
+        if (compress)
+            args.push_back("-C");
+        if (useMaster)
+            args.insert(args.end(), {"-S", socketPath});
+        args.push_back(command);
+        execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("executing ‘%s’ on ‘%s’", command, host);
+    });
+
+
+    in.readSide = -1;
+    out.writeSide = -1;
+
+    conn->out = std::move(out.readSide);
+    conn->in = std::move(in.writeSide);
+
+    return conn;
+}
+
+void SSHMaster::startMaster()
+{
+    if (!useMaster || sshMaster != -1) return;
+
+    tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
+
+    socketPath = (Path) *tmpDir + "/ssh.sock";
+
+    Pipe out;
+    out.create();
+
+    sshMaster = startProcess([&]() {
+        restoreSignals();
+
+        close(out.readSide.get());
+
+        if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("duping over stdout");
+
+        Strings args =
+            { "ssh", host.c_str(), "-M", "-N", "-S", socketPath
+            , "-o", "LocalCommand=echo started"
+            , "-o", "PermitLocalCommand=yes"
+            };
+        if (!keyFile.empty())
+            args.insert(args.end(), {"-i", keyFile});
+        if (compress)
+            args.push_back("-C");
+
+        execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("starting SSH master");
+    });
+
+    out.writeSide = -1;
+
+    std::string reply;
+    try {
+        reply = readLine(out.readSide.get());
+    } catch (EndOfFile & e) { }
+
+    if (reply != "started")
+        throw Error("failed to start SSH master connection to ‘%s’", host);
+}
+
+}
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
new file mode 100644
index 00000000000..2d2b9837039
--- /dev/null
+++ b/src/libstore/ssh.hh
@@ -0,0 +1,41 @@
+#pragma once
+
+#include "util.hh"
+
+namespace nix {
+
+class SSHMaster
+{
+private:
+
+    std::string host;
+    std::string keyFile;
+    bool useMaster;
+    bool compress;
+    Pid sshMaster;
+    std::unique_ptr<AutoDelete> tmpDir;
+    Path socketPath;
+
+public:
+
+    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress)
+        : host(host)
+        , keyFile(keyFile)
+        , useMaster(useMaster)
+        , compress(compress)
+    {
+    }
+
+    struct Connection
+    {
+        Pid sshPid;
+        AutoCloseFD out, in;
+    };
+
+    std::unique_ptr<Connection> startCommand(const std::string & command);
+
+    void startMaster();
+
+};
+
+}
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index f291cd57838..3c3dd4b074f 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -141,11 +141,16 @@ public:
         }
     }
 
-    unsigned int count()
+    size_t count()
     {
         auto state_(state.lock());
         return state_->idle.size() + state_->inUse;
     }
+
+    size_t capacity()
+    {
+        return state.lock()->max;
+    }
 };
 
 }

From d140c75530d385431850bed2f2c250f889c43c2f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 19:21:43 +0100
Subject: [PATCH 0153/2196] Pool: Don't hang if creating a connection fails

---
 src/libutil/pool.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index 3c3dd4b074f..20df2194884 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -137,6 +137,7 @@ public:
         } catch (...) {
             auto state_(state.lock());
             state_->inUse--;
+            wakeup.notify_one();
             throw;
         }
     }

From d3eb1cf3bbf57a33ac2e71a19a150c077011ecd9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 19:23:20 +0100
Subject: [PATCH 0154/2196] build-remote: Don't use a SSH master

This is unnecessary because we make only one connection.
---
 src/build-remote/build-remote.cc | 4 +++-
 src/libstore/store-api.cc        | 4 ++++
 src/libstore/store-api.hh        | 2 ++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index fd710c2b6b7..3908dfac487 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -233,7 +233,9 @@ int main (int argc, char * * argv)
                 lock = -1;
 
                 try {
-                    sshStore = openStore("ssh://" + bestMachine->hostName + "?ssh-key=" + bestMachine->sshKey);
+                    sshStore = openStore("ssh://" + bestMachine->hostName,
+                        { {"ssh-key", bestMachine->sshKey },
+                          {"max-connections", "1" } });
                     hostName = bestMachine->hostName;
                 } catch (std::exception & e) {
                     printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...",
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 401b001b2d8..9c755965e45 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -708,7 +708,11 @@ ref<Store> openStore(const std::string & uri_)
         }
         uri = uri_.substr(0, q);
     }
+    return openStore(uri, params);
+}
 
+ref<Store> openStore(const std::string & uri, const Store::Params & params)
+{
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) return ref<Store>(store);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c344b9d66ed..481d0b79906 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -642,6 +642,8 @@ void removeTempRoots();
    set to true *unless* you're going to collect garbage. */
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
+ref<Store> openStore(const std::string & uri, const Store::Params & params);
+
 
 void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 

From 8490ee37a6dbfb66e1b3dbaf88918bea044b143a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 19:28:27 +0100
Subject: [PATCH 0155/2196] SSHMaster: Make thread-safe

---
 src/libstore/ssh.cc | 22 ++++++++++++++--------
 src/libstore/ssh.hh | 24 +++++++++++++++---------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 7c3de4a4827..4f88fa64dbd 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -4,7 +4,7 @@ namespace nix {
 
 std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
 {
-    startMaster();
+    Path socketPath = startMaster();
 
     Pipe in, out;
     in.create();
@@ -27,7 +27,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             args.insert(args.end(), {"-i", keyFile});
         if (compress)
             args.push_back("-C");
-        if (useMaster)
+        if (socketPath != "")
             args.insert(args.end(), {"-S", socketPath});
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
@@ -45,18 +45,22 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
     return conn;
 }
 
-void SSHMaster::startMaster()
+Path SSHMaster::startMaster()
 {
-    if (!useMaster || sshMaster != -1) return;
+    if (!useMaster) return "";
 
-    tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
+    auto state(state_.lock());
 
-    socketPath = (Path) *tmpDir + "/ssh.sock";
+    if (state->sshMaster != -1) return state->socketPath;
+
+    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
+
+    state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
 
     Pipe out;
     out.create();
 
-    sshMaster = startProcess([&]() {
+    state->sshMaster = startProcess([&]() {
         restoreSignals();
 
         close(out.readSide.get());
@@ -65,7 +69,7 @@ void SSHMaster::startMaster()
             throw SysError("duping over stdout");
 
         Strings args =
-            { "ssh", host.c_str(), "-M", "-N", "-S", socketPath
+            { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath
             , "-o", "LocalCommand=echo started"
             , "-o", "PermitLocalCommand=yes"
             };
@@ -88,6 +92,8 @@ void SSHMaster::startMaster()
 
     if (reply != "started")
         throw Error("failed to start SSH master connection to ‘%s’", host);
+
+    return state->socketPath;
 }
 
 }
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 2d2b9837039..72238dad79a 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "util.hh"
+#include "sync.hh"
 
 namespace nix {
 
@@ -8,13 +9,19 @@ class SSHMaster
 {
 private:
 
-    std::string host;
-    std::string keyFile;
-    bool useMaster;
-    bool compress;
-    Pid sshMaster;
-    std::unique_ptr<AutoDelete> tmpDir;
-    Path socketPath;
+    const std::string host;
+    const std::string keyFile;
+    const bool useMaster;
+    const bool compress;
+
+    struct State
+    {
+        Pid sshMaster;
+        std::unique_ptr<AutoDelete> tmpDir;
+        Path socketPath;
+    };
+
+    Sync<State> state_;
 
 public:
 
@@ -34,8 +41,7 @@ public:
 
     std::unique_ptr<Connection> startCommand(const std::string & command);
 
-    void startMaster();
-
+    Path startMaster();
 };
 
 }

From d1158bb8168804b27508972988d4b85ba9d5e49d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Mar 2017 19:35:34 +0100
Subject: [PATCH 0156/2196] Cache connection failures

---
 src/libstore/remote-store.cc | 15 ++++++++++++++-
 src/libstore/remote-store.hh |  4 ++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5e62bd3d5ac..1ac2d7b6e78 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -44,13 +44,26 @@ RemoteStore::RemoteStore(const Params & params)
     : Store(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, std::stoi(get(params, "max-connections", "1"))),
-            [this]() { return openConnection(); },
+            [this]() { return openConnectionWrapper(); },
             [](const ref<Connection> & r) { return r->to.good() && r->from.good(); }
             ))
 {
 }
 
 
+ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
+{
+    if (failed)
+        throw Error("opening a connection to remote store ‘%s’ previously failed", getUri());
+    try {
+        return openConnection();
+    } catch (...) {
+        failed = true;
+        throw;
+    }
+}
+
+
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : Store(params)
     , LocalFSStore(params)
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ed7b27c8886..66540a2a2ec 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -98,6 +98,8 @@ protected:
         void processStderr(Sink * sink = 0, Source * source = 0);
     };
 
+    ref<Connection> openConnectionWrapper();
+
     virtual ref<Connection> openConnection() = 0;
 
     void initConnection(Connection & conn);
@@ -106,6 +108,8 @@ protected:
 
 private:
 
+    std::atomic_bool failed{false};
+
     void setOptions(Connection & conn);
 };
 

From 5789eaa3f45cadec719b4f642de2169b8b0a56c4 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Fri, 3 Mar 2017 16:12:17 -0500
Subject: [PATCH 0157/2196] Add aws-region param to S3 store URLs

---
 src/libstore/download.cc              | 5 ++++-
 src/libstore/s3-binary-cache-store.cc | 9 +++++----
 src/libstore/s3.hh                    | 4 ++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 75c00d85d34..11374b1da52 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -5,6 +5,9 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "s3.hh"
+#ifdef ENABLE_S3
+#include <aws/core/client/ClientConfiguration.h>
+#endif
 
 #include <unistd.h>
 #include <fcntl.h>
@@ -496,7 +499,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
 #ifdef ENABLE_S3
-                S3Helper s3Helper;
+                S3Helper s3Helper(Aws::Region::US_EAST_1); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI ‘%s’", request.uri);
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 041c68c6816..a110f5ade48 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -52,8 +52,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper()
-    : config(makeConfig())
+S3Helper::S3Helper(const string & region)
+    : config(makeConfig(region))
     , client(make_ref<Aws::S3::S3Client>(*config))
 {
 }
@@ -70,11 +70,11 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig()
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
-    res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
+    res->region = region;
     res->requestTimeoutMs = 600 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     return res;
@@ -140,6 +140,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
+        , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 5d5d3475c44..08a7fbf96e9 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper();
+    S3Helper(const std::string & region);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig();
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
 
     struct DownloadResult
     {

From b667abc699c7d63e0a6e2beba6daae57f05ddab4 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 5 Mar 2017 07:39:10 -0500
Subject: [PATCH 0158/2196] Add signing and s3 support on darwin

---
 release.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 78f96196399..e61e81bdf37 100644
--- a/release.nix
+++ b/release.nix
@@ -74,8 +74,8 @@ let
 
         buildInputs =
           [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ]
-          ++ lib.optional stdenv.isLinux libsodium
-          ++ lib.optional stdenv.isLinux
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {
               apis = ["s3"];
               customMemoryManagement = false;

From 689b8256274f669c9a18d8af221e12ff5abbad68 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 5 Mar 2017 18:15:44 -0500
Subject: [PATCH 0159/2196] nix-daemon.plist: Set XDG_CACHE_HOME

---
 misc/launchd/org.nixos.nix-daemon.plist.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index c5ef97ee9a3..5d57a5ec8ff 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -16,6 +16,8 @@
     <dict>
       <key>NIX_SSL_CERT_FILE</key>
       <string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
+      <key>XDG_CACHE_HOME</key>
+      <string>/root/.cache</string>
     </dict>
   </dict>
 </plist>

From 1cf480110879ffc8aee94b4b75999da405b71d7c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Mar 2017 13:56:20 +0100
Subject: [PATCH 0160/2196] Revert fa125b9b28bea25a4eeb4d39a71a481563127cb9

This causes quadratic performance.
---
 src/libutil/archive.hh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index c067cd2ad24..607ebf8b28f 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -75,12 +75,6 @@ struct TeeSink : ParseSink
     TeeSource source;
 
     TeeSink(Source & source) : source(source) { }
-
-    void preallocateContents(unsigned long long size) override
-    {
-        source.data->reserve(source.data->size() + size + 1024);
-    };
-
 };
 
 void parseDump(ParseSink & sink, Source & source);

From 4fc30922cf00d79bd603ac46255fa73a3c2ee565 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 6 Mar 2017 13:03:02 -0500
Subject: [PATCH 0161/2196] istringstream_nocopy: Implement in a
 standards-compliant way.

Fixes the problem mentioned in e6a61b8da788efbbbb0eb690c49434b6b5fc9741

See #1135
---
 src/libstore/derivations.cc           |  2 +-
 src/libstore/s3-binary-cache-store.cc | 13 ----
 src/libutil/hash.cc                   |  2 +-
 src/libutil/util.hh                   | 87 +++++++++++++++++++++++++++
 4 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index ce1ac7d3322..38a87240c3c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -152,7 +152,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 static Derivation parseDerivation(const string & s)
 {
     Derivation drv;
-    std::istringstream str(s);
+    istringstream_nocopy str(s);
     expect(str, "Derive([");
 
     /* Parse the list of outputs. */
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 041c68c6816..1121b4d4cd0 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -117,17 +117,6 @@ S3Helper::DownloadResult S3Helper::getObject(
     return res;
 }
 
-#if __linux__
-
-struct istringstream_nocopy : public std::stringstream
-{
-    istringstream_nocopy(const std::string & s)
-    {
-        rdbuf()->pubsetbuf(
-            (char *) s.data(), s.size());
-    }
-};
-
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     std::string bucketName;
@@ -313,8 +302,6 @@ static RegisterStoreImplementation regStore([](
     return store;
 });
 
-#endif
-
 }
 
 #endif
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index f447c80c5d8..a8bbcf8c175 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -104,7 +104,7 @@ Hash parseHash(HashType ht, const string & s)
         string s2(s, i * 2, 2);
         if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
             throw BadHash(format("invalid hash ‘%1%’") % s);
-        std::istringstream str(s2);
+        istringstream_nocopy str(s2);
         int n;
         str >> std::hex >> n;
         hash.hash[i] = n;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 2950f7daa5e..7cb3e68b9ef 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -449,4 +449,91 @@ struct ReceiveInterrupts
 };
 
 
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
+{
+public:
+    typedef std::basic_string<CharT, Traits, Allocator> string_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
+
+private:
+    const string_type & s;
+
+    off_type off;
+
+public:
+    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
+    {
+    }
+
+private:
+    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
+    {
+        if (which & std::ios_base::in) {
+            this->off = dir == std::ios_base::beg
+                ? off
+                : (dir == std::ios_base::end
+                    ? s.size() + off
+                    : this->off + off);
+        }
+        return pos_type(this->off);
+    }
+
+    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
+    {
+        return seekoff(pos, std::ios_base::beg, which);
+    }
+
+    std::streamsize showmanyc()
+    {
+        return s.size() - off;
+    }
+
+    int_type underflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off]);
+    }
+
+    int_type uflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off++]);
+    }
+
+    int_type pbackfail(int_type ch)
+    {
+        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
+            return traits_type::eof();
+
+        return traits_type::to_int_type(s[--off]);
+    }
+
+};
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
+{
+    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
+    buf_type buf;
+public:
+    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
+        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
+};
+
+/* A variant of std::istringstream that doesn't its string
+   argument. This is useful for large strings. The caller must ensure
+   that the string object is not destroyed while it's referenced by
+   this object. */
+typedef basic_istringstream_nocopy<char> istringstream_nocopy;
+
 }

From 3cc18d3753af79946ba7c21ccdc49a6f58642108 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 6 Mar 2017 14:30:35 -0500
Subject: [PATCH 0162/2196] Properly set the caFile for aws-sdk-cpp s3

---
 src/libstore/download.cc              | 3 +--
 src/libstore/globals.cc               | 1 +
 src/libstore/globals.hh               | 3 +++
 src/libstore/s3-binary-cache-store.cc | 1 +
 4 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 75c00d85d34..ebea3800ac3 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -224,8 +224,7 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
             if (request.verifyTLS)
-                curl_easy_setopt(req, CURLOPT_CAINFO,
-                    getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")).c_str());
+                curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
             else {
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0);
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 07af629260a..fcd6347294c 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -69,6 +69,7 @@ Settings::Settings()
     showTrace = false;
     enableImportNative = false;
     netrcFile = fmt("%s/%s", nixConfDir, "netrc");
+    caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0ff18f8b16e..1e6b7c083a6 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -195,6 +195,9 @@ struct Settings {
        downloads. */
     Path netrcFile;
 
+    /* Path to the SSL CA file used */
+    Path caFile;
+
 private:
     SettingsMap settings, overrides;
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 041c68c6816..800380c62d4 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -77,6 +77,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig()
     res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
     res->requestTimeoutMs = 600 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
+    res->caFile = settings.caFile;
     return res;
 }
 

From 93f863be9626e5455458abf9e449586270e98163 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 8 Mar 2017 08:46:12 -0500
Subject: [PATCH 0163/2196] Add option to disable import-from-derivation
 completely, even if the drv is already realized

---
 src/libexpr/primops.cc  | 2 ++
 src/libstore/globals.cc | 2 ++
 src/libstore/globals.hh | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5a570cefb2f..93097f3d1bf 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -59,6 +59,8 @@ void EvalState::realiseContext(const PathSet & context)
             drvs.insert(decoded.first + "!" + decoded.second);
     }
     if (!drvs.empty()) {
+        if (!settings.enableImportFromDerivation)
+            throw EvalError(format("attempted to realize ‘%1%’ during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
         /* For performance, prefetch all substitute info. */
         PathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index fcd6347294c..df537a51255 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -70,6 +70,7 @@ Settings::Settings()
     enableImportNative = false;
     netrcFile = fmt("%s/%s", nixConfDir, "netrc");
     caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
+    enableImportFromDerivation = true;
 }
 
 
@@ -185,6 +186,7 @@ void Settings::update()
     _get(keepGoing, "keep-going");
     _get(keepFailed, "keep-failed");
     _get(netrcFile, "netrc-file");
+    _get(enableImportFromDerivation, "allow-import-from-derivation");
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 1e6b7c083a6..7a9a9f6c0ca 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -198,6 +198,9 @@ struct Settings {
     /* Path to the SSL CA file used */
     Path caFile;
 
+    /* Whether we allow import-from-derivation */
+    bool enableImportFromDerivation;
+
 private:
     SettingsMap settings, overrides;
 

From 5f831c10574ae7ec44d0d357a6a371ac55d00593 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 8 Mar 2017 09:12:03 -0500
Subject: [PATCH 0164/2196] Add docs for allow-import-from-derivation

---
 doc/manual/command-ref/conf-file.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 96f8a4b608a..b1b604100f9 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -644,6 +644,16 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
+
+    <listitem><para>By default, nix allows you to <function>import</function> from a derivation,
+    allowing building at evaluation time. With this option set to false, nix will throw an error
+    when evaluating an expression that uses this feature, allowing users to ensure their evaluation
+    will not require any builds to take place.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>

From d853877ce92e0c202d70645bc716f2796a907cdc Mon Sep 17 00:00:00 2001
From: Adrien Devresse <adrien.devresse@epfl.ch>
Date: Wed, 8 Mar 2017 22:24:10 +0100
Subject: [PATCH 0165/2196] Add missing header <sstream>

---
 src/libutil/util.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 2950f7daa5e..b74c1d41739 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -13,6 +13,7 @@
 #include <limits>
 #include <cstdio>
 #include <map>
+#include <sstream>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0

From 19643a781ea3baaeb19d321093d99cb74b3ffc7e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 9 Mar 2017 13:36:56 +0100
Subject: [PATCH 0166/2196] nix -> Nix

---
 doc/manual/command-ref/conf-file.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index b1b604100f9..36b70f0c48f 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -646,8 +646,8 @@ password <replaceable>my-password</replaceable>
 
   <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
 
-    <listitem><para>By default, nix allows you to <function>import</function> from a derivation,
-    allowing building at evaluation time. With this option set to false, nix will throw an error
+    <listitem><para>By default, Nix allows you to <function>import</function> from a derivation,
+    allowing building at evaluation time. With this option set to false, Nix will throw an error
     when evaluating an expression that uses this feature, allowing users to ensure their evaluation
     will not require any builds to take place.</para></listitem>
 

From ae568847f5ee2dfd4226f30fa64fdc122623229b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Sat, 11 Mar 2017 19:39:13 +0100
Subject: [PATCH 0167/2196] Dockerfile: 1.11.2 -> 1.11.7

---
 misc/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 7b2865c946d..85bd32e199a 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM alpine
 
-RUN wget -O- http://nixos.org/releases/nix/nix-1.11.2/nix-1.11.2-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
+RUN wget -O- http://nixos.org/releases/nix/nix-1.11.7/nix-1.11.7-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
     && echo "nixbld:x:30000:nixbld1,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld30" >> /etc/group \
     && for i in $(seq 1 30); do echo "nixbld$i:x:$((30000 + $i)):30000:::" >> /etc/passwd; done \
     && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \

From f628ca2a1f6a4d2bbaa0d24204fad8fe530e375a Mon Sep 17 00:00:00 2001
From: Stefan Junker <steveeJ@users.noreply.github.com>
Date: Sun, 12 Mar 2017 01:04:21 +0100
Subject: [PATCH 0168/2196] nix-shell/pure: keep environment variable SHLVL

---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b81c98868e6..b4206033cf5 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -394,7 +394,7 @@ int main(int argc, char ** argv)
                 auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
 
                 if (pure) {
-                    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL"};
+                    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
                     decltype(env) newEnv;
                     for (auto & i : env)
                         if (keepVars.count(i.first))

From 0afeb7f51e3465c7c27bc5a83017e9ffde8c6725 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 12:07:50 +0100
Subject: [PATCH 0169/2196] Store: Add a method for getting build logs

This allows various Store implementations to provide different ways to
get build logs. For example, BinaryCacheStore can get the build logs
from the binary cache.

Also, remove the log-servers option since we can use substituters for
this.
---
 doc/manual/command-ref/conf-file.xml | 14 -------
 doc/manual/command-ref/nix-store.xml |  7 +---
 src/libstore/build.cc                |  5 +--
 src/libstore/globals.cc              |  1 -
 src/libstore/globals.hh              |  3 --
 src/libstore/local-fs-store.cc       | 35 +++++++++++++++++
 src/libstore/local-store.hh          |  3 --
 src/libstore/store-api.hh            |  8 ++++
 src/nix-store/nix-store.cc           | 57 +++-------------------------
 9 files changed, 50 insertions(+), 83 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 36b70f0c48f..3bd133918f4 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -512,20 +512,6 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-log-servers"><term><literal>log-servers</literal></term>
-
-    <listitem>
-
-      <para>A list of URL prefixes (such as
-      <literal>http://hydra.nixos.org/log</literal>) from which
-      <command>nix-store -l</command> will try to fetch build logs if
-      they’re not available locally.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
 
     <listitem>
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 0f6172defb3..fb017b741da 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -1236,12 +1236,7 @@ the store path is used.</para>
 <filename>/nix/var/log/nix/drvs</filename>.  However, there is no
 guarantee that a build log is available for any particular store path.
 For instance, if the path was downloaded as a pre-built binary through
-a substitute, then the log is unavailable. If the log is not available
-locally, then <command>nix-store</command> will try to download the
-log from the servers specified in the Nix option
-<option>log-servers</option>. For example, if it’s set to
-<literal>http://hydra.nixos.org/log</literal>, then Nix will check
-<literal>http://hydra.nixos.org/log/<replaceable>base-name</replaceable></literal>.</para>
+a substitute, then the log is unavailable.</para>
 
 </refsection>
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index fd1f5dc3a4d..2b0f8e592d5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3048,9 +3048,6 @@ void DerivationGoal::registerOutputs()
 }
 
 
-string drvsLogDir = "drvs";
-
-
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
@@ -3060,7 +3057,7 @@ Path DerivationGoal::openLogFile()
     string baseName = baseNameOf(drvPath);
 
     /* Create a log file. */
-    Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % drvsLogDir % string(baseName, 0, 2)).str();
+    Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str();
     createDirs(dir);
 
     Path logFileName = (format("%1%/%2%%3%")
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index df537a51255..012b3d5b8b9 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -179,7 +179,6 @@ void Settings::update()
     _get(envKeepDerivations, "env-keep-derivations");
     _get(sshSubstituterHosts, "ssh-substituter-hosts");
     _get(useSshSubstituter, "use-ssh-substituter");
-    _get(logServers, "log-servers");
     _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
     _get(useCaseHack, "use-case-hack");
     _get(preBuildHook, "pre-build-hook");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 7a9a9f6c0ca..46272168191 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -181,9 +181,6 @@ struct Settings {
     /* Whether to show a stack trace if Nix evaluation fails. */
     bool showTrace;
 
-    /* A list of URL prefixes that can return Nix build logs. */
-    Strings logServers;
-
     /* Whether the importNative primop should be enabled */
     bool enableImportNative;
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 4571a2211cd..c5da73dba36 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -2,6 +2,8 @@
 #include "fs-accessor.hh"
 #include "store-api.hh"
 #include "globals.hh"
+#include "compression.hh"
+#include "derivations.hh"
 
 namespace nix {
 
@@ -84,4 +86,37 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink)
     dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
 }
 
+const string LocalFSStore::drvsLogDir = "drvs";
+
+std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
+{
+    auto path(path_);
+
+    assertStorePath(path);
+
+    if (!isDerivation(path)) {
+        path = queryPathInfo(path)->deriver;
+        if (path == "") return nullptr;
+    }
+
+    string baseName = baseNameOf(path);
+
+    for (int j = 0; j < 2; j++) {
+
+        Path logPath =
+            j == 0
+            ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
+            : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str();
+        Path logBz2Path = logPath + ".bz2";
+
+        if (pathExists(logPath))
+            return std::make_shared<std::string>(readFile(logPath));
+
+        else if (pathExists(logBz2Path))
+            return decompress("bzip2", readFile(logBz2Path));
+    }
+
+    return nullptr;
+}
+
 }
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 511209d8404..49a0d7e0d63 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -21,9 +21,6 @@ namespace nix {
 const int nixSchemaVersion = 10;
 
 
-extern string drvsLogDir;
-
-
 struct Derivation;
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 481d0b79906..3aea30c286a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -566,6 +566,11 @@ public:
        if they lack a signature. */
     virtual bool isTrusted() { return false; }
 
+    /* Return the build log of the specified store path, if available,
+       or null otherwise. */
+    virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
+    { return nullptr; }
+
 protected:
 
     Stats stats;
@@ -579,6 +584,7 @@ public:
     const Path rootDir;
     const Path stateDir;
     const Path logDir;
+    const static string drvsLogDir;
 
     LocalFSStore(const Params & params);
 
@@ -595,6 +601,8 @@ public:
     {
         return getRealStoreDir() + "/" + baseNameOf(storePath);
     }
+
+    std::shared_ptr<std::string> getBuildLog(const Path & path) override;
 };
 
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 950c2a7c977..024fa4168eb 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,7 +9,6 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "xmlgraph.hh"
-#include "compression.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -482,58 +481,12 @@ static void opReadLog(Strings opFlags, Strings opArgs)
 
     RunPager pager;
 
-    // FIXME: move getting logs into Store.
-    auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
-    if (!store2) throw Error(format("store ‘%s’ does not support reading logs") % store->getUri());
-
     for (auto & i : opArgs) {
-        Path path = useDeriver(store->followLinksToStorePath(i));
-
-        string baseName = baseNameOf(path);
-        bool found = false;
-
-        for (int j = 0; j < 2; j++) {
-
-            Path logPath =
-                j == 0
-                ? (format("%1%/%2%/%3%/%4%") % store2->logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
-                : (format("%1%/%2%/%3%") % store2->logDir % drvsLogDir % baseName).str();
-            Path logBz2Path = logPath + ".bz2";
-
-            if (pathExists(logPath)) {
-                /* !!! Make this run in O(1) memory. */
-                string log = readFile(logPath);
-                writeFull(STDOUT_FILENO, log);
-                found = true;
-                break;
-            }
-
-            else if (pathExists(logBz2Path)) {
-                std::cout << *decompress("bzip2", readFile(logBz2Path));
-                found = true;
-                break;
-            }
-        }
-
-        if (!found) {
-            for (auto & i : settings.logServers) {
-                string prefix = i;
-                if (!prefix.empty() && prefix.back() != '/') prefix += '/';
-                string url = prefix + baseName;
-                try {
-                    string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url});
-                    std::cout << "(using build log from " << url << ")" << std::endl;
-                    std::cout << log;
-                    found = true;
-                    break;
-                } catch (ExecError & e) {
-                    /* Ignore errors from curl. FIXME: actually, might be
-                       nice to print a warning on HTTP status != 404. */
-                }
-            }
-        }
-
-        if (!found) throw Error(format("build log of derivation ‘%1%’ is not available") % path);
+        auto path = store->followLinksToStorePath(i);
+        auto log = store->getBuildLog(path);
+        if (!log)
+            throw Error("build log of derivation ‘%s’ is not available", path);
+        std::cout << *log;
     }
 }
 

From 5b86451f0279c384fb379fed9d2979d8025aa269 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 13:38:29 +0100
Subject: [PATCH 0170/2196] Add a "nix log" command

This replaces "nix-store --read-log". It checks the local store and
any configured substituters for the requested logs.
---
 src/nix/log.cc | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 src/nix/log.cc

diff --git a/src/nix/log.cc b/src/nix/log.cc
new file mode 100644
index 00000000000..d8a3830e91c
--- /dev/null
+++ b/src/nix/log.cc
@@ -0,0 +1,57 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "installables.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdLog : StoreCommand, MixInstallables
+{
+    CmdLog()
+    {
+    }
+
+    std::string name() override
+    {
+        return "log";
+    }
+
+    std::string description() override
+    {
+        return "show the build log of the specified packages or paths";
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto elems = evalInstallables(store);
+
+        PathSet paths;
+
+        for (auto & elem : elems) {
+            if (elem.isDrv)
+                paths.insert(elem.drvPath);
+            else
+                paths.insert(elem.outPaths.begin(), elem.outPaths.end());
+        }
+
+        auto subs = getDefaultSubstituters();
+
+        subs.push_front(store);
+
+        for (auto & path : paths) {
+            bool found = false;
+            for (auto & sub : subs) {
+                auto log = sub->getBuildLog(path);
+                if (!log) continue;
+                std::cout << *log;
+                found = true;
+                break;
+            }
+            if (!found)
+                throw Error("build log of path ‘%s’ is not available", path);
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdLog>());

From 532d73d5d86c3d25d08a8d771a189708dac323e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 14:07:58 +0100
Subject: [PATCH 0171/2196] BinaryCacheStore: Implement getBuildLog()

We assume that build logs are stored under log/<drv>, e.g.

  /nix/store/q7ab198v13p0f8x8wgnd75dva7d5mip6-friday-devil-0.1.1.1.drv

maps to

  https://cache.nixos.org/log/q7ab198v13p0f8x8wgnd75dva7d5mip6-friday-devil-0.1.1.1.drv
---
 src/libstore/binary-cache-store.cc | 24 ++++++++++++++++++++++++
 src/libstore/binary-cache-store.hh |  2 ++
 src/libstore/local-fs-store.cc     |  6 +++++-
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 3e07a2aa2b6..120345b2670 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -382,4 +382,28 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
 }
 
+std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
+{
+    Path drvPath;
+
+    if (isDerivation(path))
+        drvPath = path;
+    else {
+        try {
+            auto info = queryPathInfo(path);
+            // FIXME: add a "Log" field to .narinfo
+            if (info->deriver == "") return nullptr;
+            drvPath = info->deriver;
+        } catch (InvalidPath &) {
+            return nullptr;
+        }
+    }
+
+    auto logPath = "log/" + baseNameOf(drvPath);
+
+    debug("fetching build log from binary cache ‘%s/%s’", getUri(), logPath);
+
+    return getFile(logPath);
+}
+
 }
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index a70d50d4949..1c287056ce5 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -122,6 +122,8 @@ public:
     void addSignatures(const Path & storePath, const StringSet & sigs) override
     { notImpl(); }
 
+    std::shared_ptr<std::string> getBuildLog(const Path & path) override;
+
 };
 
 }
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index c5da73dba36..002ee4a65ce 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -95,7 +95,11 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
     assertStorePath(path);
 
     if (!isDerivation(path)) {
-        path = queryPathInfo(path)->deriver;
+        try {
+            path = queryPathInfo(path)->deriver;
+        } catch (InvalidPath &) {
+            return nullptr;
+        }
         if (path == "") return nullptr;
     }
 

From 73d7a51ee6942f681db468dc8e3c631b4d3daa4e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 14:39:18 +0100
Subject: [PATCH 0172/2196] Remove dependency on "curl" binary

---
 Makefile.config.in     | 1 -
 configure.ac           | 1 -
 src/nix-store/local.mk | 2 --
 3 files changed, 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 15e94380477..d4953b521c4 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -14,7 +14,6 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
-curl = @curl@
 datadir = @datadir@
 datarootdir = @datarootdir@
 docdir = @docdir@
diff --git a/configure.ac b/configure.ac
index 46b0ac0651b..34dcd6b2a7d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -114,7 +114,6 @@ if test -z "$$1"; then
 fi
 ])
 
-NEED_PROG(curl, curl)
 NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk
index 84ff15b241f..ade0b233adf 100644
--- a/src/nix-store/local.mk
+++ b/src/nix-store/local.mk
@@ -7,5 +7,3 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc)
 nix-store_LIBS = libmain libstore libutil libformat
 
 nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS)
-
-nix-store_CXXFLAGS = -DCURL=\"$(curl)\"

From e8186085e07104d4b844208613c2d704b5b57dec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 14:40:15 +0100
Subject: [PATCH 0173/2196] Add support for brotli compression

Build logs on cache.nixos.org are compressed using Brotli (since this
allows them to be decompressed automatically by Chrome and Firefox),
so it's handy if "nix log" can decompress them.
---
 Makefile.config.in         |  1 +
 configure.ac               |  1 +
 release.nix                |  8 ++++++--
 shell.nix                  |  3 ++-
 src/libstore/download.cc   | 21 +++++++++++++++++++--
 src/libutil/compression.cc |  7 +++++++
 src/libutil/local.mk       |  2 ++
 7 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index d4953b521c4..fccf63b3627 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
+bro = @bro@
 datadir = @datadir@
 datarootdir = @datarootdir@
 docdir = @docdir@
diff --git a/configure.ac b/configure.ac
index 34dcd6b2a7d..21ca78af0fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
+NEED_PROG(bro, bro)
 
 
 # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
diff --git a/release.nix b/release.nix
index e61e81bdf37..a266af7c2e1 100644
--- a/release.nix
+++ b/release.nix
@@ -24,7 +24,8 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison flex perl libxml2 libxslt bzip2 xz
+          [ curl bison flex perl libxml2 libxslt
+            bzip2 xz brotli
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
@@ -73,7 +74,10 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ]
+          [ curl perl
+            bzip2 xz brotli
+            openssl pkgconfig sqlite boehmgc
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {
diff --git a/shell.nix b/shell.nix
index 4c1608230ce..df0ad01df58 100644
--- a/shell.nix
+++ b/shell.nix
@@ -6,7 +6,8 @@ with import <nixpkgs> {};
   name = "nix";
 
   buildInputs =
-    [ curl bison flex perl libxml2 libxslt bzip2 xz
+    [ curl bison flex perl libxml2 libxslt
+      bzip2 xz brotli
       pkgconfig sqlite libsodium boehmgc
       docbook5 docbook5_xsl
       autoconf-archive
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 6567a4dc475..d9b8fbc0808 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -5,6 +5,8 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "s3.hh"
+#include "compression.hh"
+
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
 #endif
@@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader
 
         struct curl_slist * requestHeaders = 0;
 
+        std::string encoding;
+
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader), request(request)
         {
@@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader
                 auto ss = tokenizeString<vector<string>>(line, " ");
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
+                encoding = "";
             } else {
                 auto i = line.find(':');
                 if (i != string::npos) {
@@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader
                             debug(format("shutting down on 200 HTTP response with expected ETag"));
                             return 0;
                         }
-                    }
+                    } else if (name == "content-encoding")
+                        encoding = trim(string(line, i + 1));;
                 }
             }
             return realSize;
@@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader
             {
                 result.cached = httpStatus == 304;
                 done = true;
-                callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+
+                /* Ad hoc support for brotli, since curl doesn't do
+                   this yet. */
+                try {
+                    if (encoding == "br")
+                        result.data = decompress("br", *result.data);
+
+                    callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+                } catch (...) {
+                    done = true;
+                    callFailure(failure, std::current_exception());
+                }
             } else {
                 Error err =
                     (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound :
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a3bbb5170d9..723b072af96 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -89,6 +89,11 @@ static ref<std::string> decompressBzip2(const std::string & in)
     }
 }
 
+static ref<std::string> decompressBrotli(const std::string & in)
+{
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
+}
+
 ref<std::string> compress(const std::string & method, const std::string & in)
 {
     StringSink ssink;
@@ -106,6 +111,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
         return decompressXZ(in);
     else if (method == "bzip2")
         return decompressBzip2(in);
+    else if (method == "br")
+        return decompressBrotli(in);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index cac5c8795db..0721b21c208 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
 
 libutil_LIBS = libformat
+
+libutil_CXXFLAGS = -DBRO=\"$(bro)\"

From fbbc4d8dda4d347351c0667d1d9a571b201256cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Mar 2017 14:56:33 +0100
Subject: [PATCH 0174/2196] Fix deadlock in runProgram() when input is larger
 than the pipe buffer size

---
 src/libutil/util.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 0a5f796e4ea..bc66b0c5322 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -870,11 +870,14 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 
     out.writeSide = -1;
 
-    /* FIXME: This can deadlock if the input is too long. */
+    std::thread writerThread;
+
     if (!input.empty()) {
         in.readSide = -1;
-        writeFull(in.writeSide.get(), input);
-        in.writeSide = -1;
+        writerThread = std::thread([&]() {
+            writeFull(in.writeSide.get(), input);
+            in.writeSide = -1;
+        });
     }
 
     string result = drainFD(out.readSide.get());
@@ -885,6 +888,9 @@ string runProgram(Path program, bool searchPath, const Strings & args,
         throw ExecError(status, format("program ‘%1%’ %2%")
             % program % statusToString(status));
 
+    if (!input.empty())
+        writerThread.join();
+
     return result;
 }
 

From 2691498b5c68a9c8908da296bf867bfc7f9a068f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Mar 2017 13:13:29 +0100
Subject: [PATCH 0175/2196] Fix assertion failure on SIGINT

  nix: src/libutil/compression.cc:142: virtual nix::XzSink::~XzSink(): Assertion `finished' failed.
---
 src/libutil/compression.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 723b072af96..8cb1dde662f 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -146,7 +146,6 @@ struct XzSink : CompressionSink
 
     ~XzSink()
     {
-        assert(finished);
         lzma_end(&strm);
     }
 
@@ -217,7 +216,6 @@ struct BzipSink : CompressionSink
 
     ~BzipSink()
     {
-        assert(finished);
         BZ2_bzCompressEnd(&strm);
     }
 

From 8b1d65bebe5af8960ba813e1233f2596a3ffebb7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Mar 2017 15:03:53 +0100
Subject: [PATCH 0176/2196] S3BinaryCacheStore: Support compression of narinfo
 and log files

You can now set the store parameter "text-compression=br" to compress
textual files in the binary cache (i.e. narinfo and logs) using
Brotli. This sets the Content-Encoding header; the extension of
compressed files is unchanged.

You can separately specify the compression of log files using
"log-compression=br". This is useful when you don't want to compress
narinfo files for backward compatibility.
---
 src/libstore/binary-cache-store.cc    |  1 +
 src/libstore/download.cc              | 16 +++++++++-----
 src/libstore/download.hh              |  3 +++
 src/libstore/s3-binary-cache-store.cc | 28 +++++++++++++++++++++---
 src/libutil/compression.cc            | 31 +++++++++++++++++++++++++++
 5 files changed, 71 insertions(+), 8 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 120345b2670..804e3f6aa1e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -250,6 +250,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
            "");
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d9b8fbc0808..da29b2fc6fc 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -39,6 +39,16 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
+ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data)
+{
+    if (encoding == "")
+        return data;
+    else if (encoding == "br")
+        return decompress(encoding, *data);
+    else
+        throw Error("unsupported Content-Encoding ‘%s’", encoding);
+}
+
 struct CurlDownloader : public Downloader
 {
     CURLM * curlm = 0;
@@ -275,12 +285,8 @@ struct CurlDownloader : public Downloader
                 result.cached = httpStatus == 304;
                 done = true;
 
-                /* Ad hoc support for brotli, since curl doesn't do
-                   this yet. */
                 try {
-                    if (encoding == "br")
-                        result.data = decompress("br", *result.data);
-
+                    result.data = decodeContent(encoding, ref<std::string>(result.data));
                     callSuccess(success, failure, const_cast<const DownloadResult &>(result));
                 } catch (...) {
                     done = true;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index bdb5011e783..e2e16b36103 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -73,4 +73,7 @@ public:
 
 bool isUri(const string & s);
 
+/* Decode data according to the Content-Encoding header. */
+ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data);
+
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 5134dd17526..1d44e68321f 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -5,6 +5,8 @@
 #include "nar-info.hh"
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
+#include "compression.hh"
+#include "download.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
@@ -104,8 +106,10 @@ S3Helper::DownloadResult S3Helper::getObject(
         auto result = checkAws(fmt("AWS error fetching ‘%s’", key),
             client->GetObject(request));
 
-        res.data = std::make_shared<std::string>(
-            dynamic_cast<std::stringstream &>(result.GetBody()).str());
+        res.data = decodeContent(
+            result.GetContentEncoding(),
+            make_ref<std::string>(
+                dynamic_cast<std::stringstream &>(result.GetBody()).str()));
 
     } catch (S3Error & e) {
         if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
@@ -137,11 +141,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     S3Helper s3Helper;
 
+    std::string textCompression, logCompression;
+
     S3BinaryCacheStoreImpl(
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
+        , textCompression(get(params, "text-compression", "gzip"))
+        , logCompression(get(params, "log-compression", textCompression))
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -220,13 +228,17 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return true;
     }
 
-    void upsertFile(const std::string & path, const std::string & data) override
+    void uploadFile(const std::string & path, const std::string & data,
+        const std::string & contentEncoding)
     {
         auto request =
             Aws::S3::Model::PutObjectRequest()
             .WithBucket(bucketName)
             .WithKey(path);
 
+        if (contentEncoding != "")
+            request.SetContentEncoding(contentEncoding);
+
         auto stream = std::make_shared<istringstream_nocopy>(data);
 
         request.SetBody(stream);
@@ -249,6 +261,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         stats.putTimeMs += duration;
     }
 
+    void upsertFile(const std::string & path, const std::string & data) override
+    {
+        if (path.find(".narinfo") != std::string::npos)
+            uploadFile(path, *compress(textCompression, data), textCompression);
+        else if (path.find("/log") != std::string::npos)
+            uploadFile(path, *compress(logCompression, data), logCompression);
+        else
+            uploadFile(path, data, "");
+    }
+
     void getFile(const std::string & path,
         std::function<void(std::shared_ptr<std::string>)> success,
         std::function<void(std::exception_ptr exc)> failure) override
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 8cb1dde662f..5df97e73923 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -91,6 +91,7 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
 static ref<std::string> decompressBrotli(const std::string & in)
 {
+    // FIXME: use libbrotli
     return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
 }
 
@@ -266,6 +267,34 @@ struct BzipSink : CompressionSink
     }
 };
 
+struct BrotliSink : CompressionSink
+{
+    Sink & nextSink;
+    std::string data;
+
+    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    {
+    }
+
+    ~BrotliSink()
+    {
+    }
+
+    // FIXME: use libbrotli
+
+    void finish() override
+    {
+        flush();
+        nextSink(runProgram(BRO, true, {}, data));
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        checkInterrupt();
+        this->data.append((const char *) data, len);
+    }
+};
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none")
@@ -274,6 +303,8 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
         return make_ref<XzSink>(nextSink);
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
+    else if (method == "br")
+        return make_ref<BrotliSink>(nextSink);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }

From 45c70382ac152107d40956c6a3ab8c329086733f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Mar 2017 15:26:01 +0100
Subject: [PATCH 0177/2196] S3BinaryCacheStore: Set Content-Type

This is necessary for serving log files to browsers.
---
 src/libstore/binary-cache-store.cc       |  8 ++++----
 src/libstore/binary-cache-store.hh       |  4 +++-
 src/libstore/http-binary-cache-store.cc  |  4 +++-
 src/libstore/local-binary-cache-store.cc |  8 ++++++--
 src/libstore/s3-binary-cache-store.cc    | 14 +++++++++-----
 5 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 804e3f6aa1e..d8e68fd5892 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -97,7 +97,7 @@ void BinaryCacheStore::init()
 
     auto cacheInfo = getFile(cacheInfoFile);
     if (!cacheInfo) {
-        upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n");
+        upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
             size_t colon = line.find(':');
@@ -224,7 +224,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             }
         }
 
-        upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
+        upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()), "application/x-nix-nar-listing");
     }
 
     else {
@@ -254,7 +254,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
            "");
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, *narCompressed);
+        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
@@ -265,7 +265,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo->sign(*secretKey);
 
-    upsertFile(narInfoFile, narInfo->to_string());
+    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
 
     auto hashPart = storePathToHash(narInfo->path);
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 1c287056ce5..d42b1abd245 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -31,7 +31,9 @@ public:
 
     virtual bool fileExists(const std::string & path) = 0;
 
-    virtual void upsertFile(const std::string & path, const std::string & data) = 0;
+    virtual void upsertFile(const std::string & path,
+        const std::string & data,
+        const std::string & mimeType) = 0;
 
     /* Return the contents of the specified file, or null if it
        doesn't exist. */
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 9d31f77c921..37a7d6ace14 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -64,7 +64,9 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         }
     }
 
-    void upsertFile(const std::string & path, const std::string & data) override
+    void upsertFile(const std::string & path,
+        const std::string & data,
+        const std::string & mimeType) override
     {
         throw UploadToHTTP("uploading to an HTTP binary cache is not supported");
     }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 0f377989bd8..aff22f9fcc2 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -30,7 +30,9 @@ class LocalBinaryCacheStore : public BinaryCacheStore
 
     bool fileExists(const std::string & path) override;
 
-    void upsertFile(const std::string & path, const std::string & data) override;
+    void upsertFile(const std::string & path,
+        const std::string & data,
+        const std::string & mimeType) override;
 
     void getFile(const std::string & path,
         std::function<void(std::shared_ptr<std::string>)> success,
@@ -83,7 +85,9 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
+void LocalBinaryCacheStore::upsertFile(const std::string & path,
+    const std::string & data,
+    const std::string & mimeType)
 {
     atomicWrite(binaryCacheDir + "/" + path, data);
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 1d44e68321f..5ecf3996d7e 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -148,7 +148,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
-        , textCompression(get(params, "text-compression", "gzip"))
+        , textCompression(get(params, "text-compression", ""))
         , logCompression(get(params, "log-compression", textCompression))
     {
         diskCache = getNarInfoDiskCache();
@@ -229,6 +229,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     }
 
     void uploadFile(const std::string & path, const std::string & data,
+        const std::string & mimeType,
         const std::string & contentEncoding)
     {
         auto request =
@@ -236,6 +237,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             .WithBucket(bucketName)
             .WithKey(path);
 
+        request.SetContentType(mimeType);
+
         if (contentEncoding != "")
             request.SetContentEncoding(contentEncoding);
 
@@ -261,14 +264,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         stats.putTimeMs += duration;
     }
 
-    void upsertFile(const std::string & path, const std::string & data) override
+    void upsertFile(const std::string & path, const std::string & data,
+        const std::string & mimeType) override
     {
         if (path.find(".narinfo") != std::string::npos)
-            uploadFile(path, *compress(textCompression, data), textCompression);
+            uploadFile(path, *compress(textCompression, data), mimeType, textCompression);
         else if (path.find("/log") != std::string::npos)
-            uploadFile(path, *compress(logCompression, data), logCompression);
+            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
         else
-            uploadFile(path, data, "");
+            uploadFile(path, data, mimeType, "");
     }
 
     void getFile(const std::string & path,

From 042975ea8e2e081c0d44190c8b41104131f8c6d4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Mar 2017 15:55:02 +0100
Subject: [PATCH 0178/2196] Compress NAR listings using the "text-compression"
 method

So if "text-compression=br", the .ls file in S3 will get a
Content-Encoding of "br". Brotli appears to compress better than xz
for this kind of file and is natively supported by browsers.
---
 src/libstore/binary-cache-store.cc    | 2 +-
 src/libstore/s3-binary-cache-store.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index d8e68fd5892..25ad0d75b70 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -224,7 +224,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             }
         }
 
-        upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()), "application/x-nix-nar-listing");
+        upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
     }
 
     else {
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 5ecf3996d7e..3804e0b0fb8 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -267,9 +267,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     void upsertFile(const std::string & path, const std::string & data,
         const std::string & mimeType) override
     {
-        if (path.find(".narinfo") != std::string::npos)
+        if (textCompression != "" && (hasSuffix(path, ".narinfo") || hasSuffix(path, ".ls")))
             uploadFile(path, *compress(textCompression, data), mimeType, textCompression);
-        else if (path.find("/log") != std::string::npos)
+        else if (logCompression != "" && hasPrefix(path, "log/"))
             uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
         else
             uploadFile(path, data, mimeType, "");

From 25dff2b7dbab54bd97b5ec22a05e594504555a12 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Mar 2017 14:40:47 +0100
Subject: [PATCH 0179/2196] runProgram(): Distinguish between empty input and
 no input

For example, if we call brotli with an empty input, it shouldn't read
from the caller's stdin.
---
 src/libstore/download.cc                 |  2 +-
 src/libutil/compression.cc               |  2 +-
 src/libutil/util.cc                      | 33 +++++++++++++++++-------
 src/libutil/util.hh                      |  4 ++-
 src/nix-prefetch-url/nix-prefetch-url.cc |  4 +--
 5 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index da29b2fc6fc..22bde086e6a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -671,7 +671,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
+            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"});
             unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5df97e73923..8ffd55efb23 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -92,7 +92,7 @@ static ref<std::string> decompressBzip2(const std::string & in)
 static ref<std::string> decompressBrotli(const std::string & in)
 {
     // FIXME: use libbrotli
-    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in}));
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index bc66b0c5322..d2d32782d02 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,6 +1,7 @@
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"
+#include "finally.hh"
 
 #include <cctype>
 #include <cerrno>
@@ -10,6 +11,7 @@
 #include <iostream>
 #include <sstream>
 #include <thread>
+#include <future>
 
 #include <sys/wait.h>
 #include <unistd.h>
@@ -837,23 +839,21 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 
 
 string runProgram(Path program, bool searchPath, const Strings & args,
-    const string & input)
+    const std::experimental::optional<std::string> & input)
 {
     checkInterrupt();
 
     /* Create a pipe. */
     Pipe out, in;
     out.create();
-    if (!input.empty()) in.create();
+    if (input) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
-        if (!input.empty()) {
-            if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
-                throw SysError("dupping stdin");
-        }
+        if (input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("dupping stdin");
 
         Strings args_(args);
         args_.push_front(program);
@@ -872,10 +872,23 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 
     std::thread writerThread;
 
-    if (!input.empty()) {
+    std::promise<void> promise;
+
+    Finally doJoin([&]() {
+        if (writerThread.joinable())
+            writerThread.join();
+    });
+
+
+    if (input) {
         in.readSide = -1;
         writerThread = std::thread([&]() {
-            writeFull(in.writeSide.get(), input);
+            try {
+                writeFull(in.writeSide.get(), *input);
+                promise.set_value();
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
             in.writeSide = -1;
         });
     }
@@ -888,8 +901,8 @@ string runProgram(Path program, bool searchPath, const Strings & args,
         throw ExecError(status, format("program ‘%1%’ %2%")
             % program % statusToString(status));
 
-    if (!input.empty())
-        writerThread.join();
+    /* Wait for the writer thread to finish. */
+    if (input) promise.get_future().get();
 
     return result;
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b74c1d41739..4e3a011b349 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -14,6 +14,7 @@
 #include <cstdio>
 #include <map>
 #include <sstream>
+#include <experimental/optional>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -232,7 +233,8 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
 /* Run a program and return its stdout in a string (i.e., like the
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
-    const Strings & args = Strings(), const string & input = "");
+    const Strings & args = Strings(),
+    const std::experimental::optional<std::string> & input = {});
 
 class ExecError : public Error
 {
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index acf60302569..b3b2fcac713 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -170,10 +170,10 @@ int main(int argc, char * * argv)
                 Path unpacked = (Path) tmpDir + "/unpacked";
                 createDirs(unpacked);
                 if (hasSuffix(baseNameOf(uri), ".zip"))
-                    runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}, "");
+                    runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
                 else
                     // FIXME: this requires GNU tar for decompression.
-                    runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}, "");
+                    runProgram("tar", true, {"xf", tmpFile, "-C", unpacked});
 
                 /* If the archive unpacks to a single file/directory, then use
                    that as the top-level. */

From 3f35612c041ecc4e6a4f29f01d5a87fba983cf0f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Mar 2017 17:20:19 +0100
Subject: [PATCH 0180/2196] More precise compression settings

---
 src/libstore/s3-binary-cache-store.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 3804e0b0fb8..571bf7dfd80 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -141,15 +141,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     S3Helper s3Helper;
 
-    std::string textCompression, logCompression;
+    std::string narinfoCompression, lsCompression, logCompression;
 
     S3BinaryCacheStoreImpl(
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
-        , textCompression(get(params, "text-compression", ""))
-        , logCompression(get(params, "log-compression", textCompression))
+        , narinfoCompression(get(params, "narinfo-compression", ""))
+        , lsCompression(get(params, "ls-compression", ""))
+        , logCompression(get(params, "log-compression", ""))
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -267,8 +268,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     void upsertFile(const std::string & path, const std::string & data,
         const std::string & mimeType) override
     {
-        if (textCompression != "" && (hasSuffix(path, ".narinfo") || hasSuffix(path, ".ls")))
-            uploadFile(path, *compress(textCompression, data), mimeType, textCompression);
+        if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
+            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
+        else if (lsCompression != "" && hasSuffix(path, ".ls"))
+            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
         else if (logCompression != "" && hasPrefix(path, "log/"))
             uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
         else

From 43f158bb0885c2eb9180ef7ab24034b1b9353e8b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 10:45:45 +0100
Subject: [PATCH 0181/2196] nix-copy-closure: Fix assertion failure

  $ ./inst/bin/nix-copy-closure --to bla $(type -p firefox)
  nix-copy-closure: src/libstore/store-api.cc:80: std::__cxx11::string nix::storePathToHash(const Path&): Assertion `base.size() >= storePathHashLen' failed.
---
 src/nix-copy-closure/nix-copy-closure.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 4340443b5cc..05a4deea3d5 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -51,8 +51,12 @@ int main(int argc, char ** argv)
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 
+        PathSet storePaths2;
+        for (auto & path : storePaths)
+            storePaths2.insert(from->followLinksToStorePath(path));
+
         PathSet closure;
-        from->computeFSClosure(storePaths, closure, false, includeOutputs);
+        from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
         copyPaths(from, to, Paths(closure.begin(), closure.end()), useSubstitutes);
     });

From 0ec7f47b00659cf043eb7fbe72385a08f7e9f8e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 10:52:28 +0100
Subject: [PATCH 0182/2196] Remove "killing process <pid>" messages

They convey no useful information.
---
 src/libmain/shared.cc | 6 +-----
 src/libstore/build.cc | 4 ++--
 src/libutil/util.cc   | 5 ++---
 src/libutil/util.hh   | 2 +-
 4 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 326202d295f..a720afd6cdd 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -332,11 +332,7 @@ RunPager::~RunPager()
             pid.wait();
         }
     } catch (...) {
-        try {
-            pid.kill(true);
-        } catch (...) {
-            ignoreException();
-        }
+        ignoreException();
     }
 }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2b0f8e592d5..fc840df81a5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -642,7 +642,7 @@ HookInstance::~HookInstance()
 {
     try {
         toHook.writeSide = -1;
-        if (pid != -1) pid.kill(true);
+        if (pid != -1) pid.kill();
     } catch (...) {
         ignoreException();
     }
@@ -1437,7 +1437,7 @@ void DerivationGoal::buildDone()
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe, so just to be sure,
        kill it. */
-    int status = hook ? hook->pid.kill(true) : pid.kill(true);
+    int status = hook ? hook->pid.kill() : pid.kill();
 
     debug(format("builder process for ‘%1%’ finished") % drvPath);
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index d2d32782d02..99a91c8cc64 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -678,12 +678,11 @@ Pid::operator pid_t()
 }
 
 
-int Pid::kill(bool quiet)
+int Pid::kill()
 {
     assert(pid != -1);
 
-    if (!quiet)
-        printError(format("killing process %1%") % pid);
+    debug(format("killing process %1%") % pid);
 
     /* Send the requested signal to the child.  If it has its own
        process group, send the signal to every process in the child
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 4e3a011b349..f94c0ff1c5e 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -203,7 +203,7 @@ public:
     ~Pid();
     void operator =(pid_t pid);
     operator pid_t();
-    int kill(bool quiet = false);
+    int kill();
     int wait();
 
     void setSeparatePG(bool separatePG);

From 7a716ef2a5759506e602f3ceaaadc42edbf6d12f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 10:58:48 +0100
Subject: [PATCH 0183/2196] Fix nix-copy-closure --to

---
 src/libstore/legacy-ssh-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 4a2ac42f31c..d804b27c1c8 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -45,7 +45,7 @@ struct LegacySSHStore : public Store
     ref<Connection> openConnection()
     {
         auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand("nix-store --serve");
+        conn->sshConn = master.startCommand("nix-store --serve --write");
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
 

From ea7fa88131eb486ffe54f32601d36a64374f7b7c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 11:44:01 +0100
Subject: [PATCH 0184/2196] LegacySSHStore: Provide a faster implementation of
 computeFSClosure()

This avoids the latency of the standard implementation, which can make
a huge difference (e.g. 16.5s -> 0.5s on a NixOS system closure).
---
 src/libstore/legacy-ssh-store.cc | 22 ++++++++++++++++++++++
 src/libstore/store-api.hh        |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index d804b27c1c8..1a933259b52 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -204,6 +204,28 @@ struct LegacySSHStore : public Store
     bool isTrusted() override
     { return true; }
 
+    void computeFSClosure(const PathSet & paths,
+        PathSet & out, bool flipDirection = false,
+        bool includeOutputs = false, bool includeDerivers = false) override
+    {
+        if (flipDirection || includeDerivers) {
+            Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
+            return;
+        }
+
+        auto conn(connections->get());
+
+        conn->to
+            << cmdQueryClosure
+            << includeOutputs
+            << paths;
+        conn->to.flush();
+
+        auto res = readStorePaths<PathSet>(*this, conn->from);
+
+        out.insert(res.begin(), res.end());
+    }
+
 };
 
 static RegisterStoreImplementation regStore([](
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3aea30c286a..92aa8862f65 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -511,7 +511,7 @@ public:
        `storePath' is returned; that is, the closures under the
        `referrers' relation instead of the `references' relation is
        returned. */
-    void computeFSClosure(const PathSet & paths,
+    virtual void computeFSClosure(const PathSet & paths,
         PathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false);
 

From 91d67692cfb05e4ece744fb9d144921ae920f2de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 12:05:51 +0100
Subject: [PATCH 0185/2196] copyPaths(): Don't query path info for a path the
 target already has

For example, this cuts "nix-copy-closure --from" on a NixOS system
closure from 15.9s to 0.5s.
---
 src/libstore/store-api.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9c755965e45..9ab3f36329b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -822,6 +822,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool su
         PathSet(storePaths.begin(), storePaths.end()),
 
         [&](const Path & storePath) {
+            if (to->isValidPath(storePath)) return PathSet();
             return from->queryPathInfo(storePath)->references;
         },
 

From c5b83d8913d73ea58ff9437c41bf6bd0c6839ad0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 13:50:01 +0100
Subject: [PATCH 0186/2196] copyPaths(): Use queryValidPaths() to reduce SSH
 latency

---
 src/build-remote/build-remote.cc         | 16 +++++++----
 src/libstore/legacy-ssh-store.cc         | 13 +++++++++
 src/libstore/local-store.cc              |  2 +-
 src/libstore/local-store.hh              |  2 +-
 src/libstore/remote-store.cc             |  2 +-
 src/libstore/remote-store.hh             |  2 +-
 src/libstore/store-api.cc                | 35 ++++++++----------------
 src/libstore/store-api.hh                |  8 ++++--
 src/nix-copy-closure/nix-copy-closure.cc |  2 +-
 src/nix/copy.cc                          |  2 +-
 10 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 3908dfac487..6b142db9826 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -252,10 +252,10 @@ int main (int argc, char * * argv)
         string line;
         if (!getline(cin, line))
             throw Error("hook caller didn't send inputs");
-        auto inputs = tokenizeString<std::list<string>>(line);
+        auto inputs = tokenizeString<PathSet>(line);
         if (!getline(cin, line))
             throw Error("hook caller didn't send outputs");
-        auto outputs = tokenizeString<Strings>(line);
+        auto outputs = tokenizeString<PathSet>(line);
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true);
         auto old = signal(SIGALRM, handleAlarm);
         alarm(15 * 60);
@@ -269,11 +269,15 @@ int main (int argc, char * * argv)
         printError("building ‘%s’ on ‘%s’", drvPath, hostName);
         sshStore->buildDerivation(drvPath, readDerivation(drvPath));
 
-        std::remove_if(outputs.begin(), outputs.end(), [=](const Path & path) { return store->isValidPath(path); });
-        if (!outputs.empty()) {
-            setenv("NIX_HELD_LOCKS", concatStringsSep(" ", outputs).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, outputs);
+        PathSet missing;
+        for (auto & path : outputs)
+            if (!store->isValidPath(path)) missing.insert(path);
+
+        if (!missing.empty()) {
+            setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
+            copyPaths(ref<Store>(sshStore), store, missing);
         }
+
         return;
     });
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 1a933259b52..84bf0f727f4 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -226,6 +226,19 @@ struct LegacySSHStore : public Store
         out.insert(res.begin(), res.end());
     }
 
+    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override
+    {
+        auto conn(connections->get());
+
+        conn->to
+            << cmdQueryValidPaths
+            << false // lock
+            << maybeSubstitute
+            << paths;
+        conn->to.flush();
+
+        return readStorePaths<PathSet>(*this, conn->from);
+    }
 };
 
 static RegisterStoreImplementation regStore([](
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 63f069c2ff1..dcfa000c432 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -669,7 +669,7 @@ bool LocalStore::isValidPathUncached(const Path & path)
 }
 
 
-PathSet LocalStore::queryValidPaths(const PathSet & paths)
+PathSet LocalStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
 {
     PathSet res;
     for (auto & i : paths)
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 49a0d7e0d63..28e9a31c9fe 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -99,7 +99,7 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths) override;
+    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override;
 
     PathSet queryAllValidPaths() override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1ac2d7b6e78..a1f2db5b0ec 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -187,7 +187,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
 }
 
 
-PathSet RemoteStore::queryValidPaths(const PathSet & paths)
+PathSet RemoteStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
 {
     auto conn(connections->get());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 66540a2a2ec..a08bd305639 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -28,7 +28,7 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths) override;
+    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override;
 
     PathSet queryAllValidPaths() override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9ab3f36329b..b1bf961e1bf 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -377,7 +377,7 @@ void Store::queryPathInfo(const Path & storePath,
 }
 
 
-PathSet Store::queryValidPaths(const PathSet & paths)
+PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
 {
     struct State
     {
@@ -550,6 +550,8 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
     for (auto & path : storePaths)
         srcStore->computeFSClosure(path, closure);
 
+    // FIXME: use copyStorePaths()
+
     PathSet valid = dstStore->queryValidPaths(closure);
 
     if (valid.size() == closure.size()) return;
@@ -791,35 +793,22 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
-{
-    if (substitute) {
-        /* Filter out .drv files (we don't want to build anything). */
-        PathSet paths2;
-        for (auto & path : storePaths)
-            if (!isDerivation(path)) paths2.insert(path);
-        unsigned long long downloadSize, narSize;
-        PathSet willBuild, willSubstitute, unknown;
-        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
-            willBuild, willSubstitute, unknown, downloadSize, narSize);
-        /* FIXME: should use ensurePath(), but it only
-           does one path at a time. */
-        if (!willSubstitute.empty())
-            try {
-                to->buildPaths(willSubstitute);
-            } catch (Error & e) {
-                printMsg(lvlError, format("warning: %1%") % e.msg());
-            }
-    }
+void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute)
+{
+    PathSet valid = to->queryValidPaths(storePaths, substitute);
+
+    PathSet missing;
+    for (auto & path : storePaths)
+        if (!valid.count(path)) missing.insert(path);
 
     std::string copiedLabel = "copied";
 
-    logger->setExpected(copiedLabel, storePaths.size());
+    logger->setExpected(copiedLabel, missing.size());
 
     ThreadPool pool;
 
     processGraph<Path>(pool,
-        PathSet(storePaths.begin(), storePaths.end()),
+        PathSet(missing.begin(), missing.end()),
 
         [&](const Path & storePath) {
             if (to->isValidPath(storePath)) return PathSet();
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 92aa8862f65..98f2803f813 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -324,8 +324,10 @@ protected:
 
 public:
 
-    /* Query which of the given paths is valid. */
-    virtual PathSet queryValidPaths(const PathSet & paths);
+    /* Query which of the given paths is valid. Optionally, try to
+       substitute missing paths. */
+    virtual PathSet queryValidPaths(const PathSet & paths,
+        bool maybeSubstitute = false);
 
     /* Query the set of all valid paths. Note that for some store
        backends, the name part of store paths may be omitted
@@ -653,7 +655,7 @@ ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 ref<Store> openStore(const std::string & uri, const Store::Params & params);
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
+void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 05a4deea3d5..ab80d96b5ef 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -58,6 +58,6 @@ int main(int argc, char ** argv)
         PathSet closure;
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
-        copyPaths(from, to, Paths(closure.begin(), closure.end()), useSubstitutes);
+        copyPaths(from, to, closure, useSubstitutes);
     });
 }
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 976b0d3e0b8..083dc350686 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -46,7 +46,7 @@ struct CmdCopy : StorePathsCommand
         ref<Store> srcStore = srcUri.empty() ? store : openStore(srcUri);
         ref<Store> dstStore = dstUri.empty() ? store : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, storePaths);
+        copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()));
     }
 };
 

From 287084d688c3316d5840a9a7b5b2dff29b3dda94 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 14:19:32 +0100
Subject: [PATCH 0187/2196] ssh:// -> ssh-ng://, legacy-ssh:// -> ssh://

---
 src/build-remote/build-remote.cc         | 2 +-
 src/libstore/legacy-ssh-store.cc         | 2 +-
 src/libstore/ssh-store.cc                | 2 +-
 src/nix-copy-closure/nix-copy-closure.cc | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 6b142db9826..d7aee288670 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -233,7 +233,7 @@ int main (int argc, char * * argv)
                 lock = -1;
 
                 try {
-                    sshStore = openStore("ssh://" + bestMachine->hostName,
+                    sshStore = openStore("ssh-ng://" + bestMachine->hostName,
                         { {"ssh-key", bestMachine->sshKey },
                           {"max-connections", "1" } });
                     hostName = bestMachine->hostName;
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 84bf0f727f4..0e838846c79 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-static std::string uriScheme = "legacy-ssh://";
+static std::string uriScheme = "ssh://";
 
 struct LegacySSHStore : public Store
 {
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 20f020bdada..2a81a8b1ebe 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-static std::string uriScheme = "ssh://";
+static std::string uriScheme = "ssh-ng://";
 
 class SSHStore : public RemoteStore
 {
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index ab80d96b5ef..ed43bffbc8c 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -47,7 +47,7 @@ int main(int argc, char ** argv)
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
-        auto remoteUri = "legacy-ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 

From 558eda01154d47b3c88983576eedb582185b2201 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Mar 2017 14:25:54 +0100
Subject: [PATCH 0188/2196] nix copy: Make -r option use the "from" store

Previously, we tried to compute the closure in the local store, which
obviously doesn't work.
---
 src/nix/command.cc |  7 ++++++-
 src/nix/command.hh |  1 +
 src/nix/copy.cc    | 10 +++++++---
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 5a8288da912..a1b2c120a5d 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -79,9 +79,14 @@ StoreCommand::StoreCommand()
     mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri);
 }
 
+ref<Store> StoreCommand::createStore()
+{
+    return openStore(storeUri);
+}
+
 void StoreCommand::run()
 {
-    run(openStore(storeUri));
+    run(createStore());
 }
 
 StorePathsCommand::StorePathsCommand()
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a29cdcf7f50..fa6c21abf8a 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -33,6 +33,7 @@ struct StoreCommand : virtual Command
     std::string storeUri;
     StoreCommand();
     void run() override;
+    virtual ref<Store> createStore();
     virtual void run(ref<Store>) = 0;
 };
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 083dc350686..b2165cb8f85 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -38,13 +38,17 @@ struct CmdCopy : StorePathsCommand
         };
     }
 
-    void run(ref<Store> store, Paths storePaths) override
+    ref<Store> createStore() override
+    {
+        return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+    }
+
+    void run(ref<Store> srcStore, Paths storePaths) override
     {
         if (srcUri.empty() && dstUri.empty())
             throw UsageError("you must pass ‘--from’ and/or ‘--to’");
 
-        ref<Store> srcStore = srcUri.empty() ? store : openStore(srcUri);
-        ref<Store> dstStore = dstUri.empty() ? store : openStore(dstUri);
+        ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
         copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()));
     }

From 3229f85585136e5c1d53a2ef2a434fdec75d912e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 14:35:50 +0100
Subject: [PATCH 0189/2196] Honor $NIX_SSHOPTS again

NixOps needs this.
---
 src/libstore/ssh.cc | 21 ++++++++++++---------
 src/libstore/ssh.hh |  2 ++
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 4f88fa64dbd..e54f3f4ba28 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -2,6 +2,16 @@
 
 namespace nix {
 
+void SSHMaster::addCommonSSHOpts(Strings & args)
+{
+    for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
+        args.push_back(i);
+    if (!keyFile.empty())
+        args.insert(args.end(), {"-i", keyFile});
+    if (compress)
+        args.push_back("-C");
+}
+
 std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
 {
     Path socketPath = startMaster();
@@ -23,10 +33,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             throw SysError("duping over stdout");
 
         Strings args = { "ssh", host.c_str(), "-x", "-a" };
-        if (!keyFile.empty())
-            args.insert(args.end(), {"-i", keyFile});
-        if (compress)
-            args.push_back("-C");
+        addCommonSSHOpts(args);
         if (socketPath != "")
             args.insert(args.end(), {"-S", socketPath});
         args.push_back(command);
@@ -73,11 +80,7 @@ Path SSHMaster::startMaster()
             , "-o", "LocalCommand=echo started"
             , "-o", "PermitLocalCommand=yes"
             };
-        if (!keyFile.empty())
-            args.insert(args.end(), {"-i", keyFile});
-        if (compress)
-            args.push_back("-C");
-
+        addCommonSSHOpts(args);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("starting SSH master");
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 72238dad79a..b4396467e54 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -23,6 +23,8 @@ private:
 
     Sync<State> state_;
 
+    void addCommonSSHOpts(Strings & args);
+
 public:
 
     SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress)

From f8b84a3b8c5eff26114008ababc333c7d14c92de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 14:43:03 +0100
Subject: [PATCH 0190/2196] Move istringstream_nocopy to a separate file

---
 src/libstore/derivations.cc           |  2 +-
 src/libstore/s3-binary-cache-store.cc |  1 +
 src/libutil/hash.cc                   |  2 +-
 src/libutil/istringstream_nocopy.hh   | 92 +++++++++++++++++++++++++++
 src/libutil/util.hh                   | 88 -------------------------
 5 files changed, 95 insertions(+), 90 deletions(-)
 create mode 100644 src/libutil/istringstream_nocopy.hh

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 38a87240c3c..0c6ceb9f674 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -4,7 +4,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
-
+#include "istringstream_nocopy.hh"
 
 namespace nix {
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 5a8acfb506e..3053f908c4e 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -7,6 +7,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "download.hh"
+#include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index a8bbcf8c175..9f4afd93c2f 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -7,12 +7,12 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
+#include "istringstream_nocopy.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 
-
 namespace nix {
 
 
diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh
new file mode 100644
index 00000000000..f7beac578e3
--- /dev/null
+++ b/src/libutil/istringstream_nocopy.hh
@@ -0,0 +1,92 @@
+/* This file provides a variant of std::istringstream that doesn't
+   copy its string argument. This is useful for large strings. The
+   caller must ensure that the string object is not destroyed while
+   it's referenced by this object. */
+
+#pragma once
+
+#include <string>
+#include <iostream>
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
+{
+public:
+    typedef std::basic_string<CharT, Traits, Allocator> string_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
+
+    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
+
+private:
+    const string_type & s;
+
+    off_type off;
+
+public:
+    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
+    {
+    }
+
+private:
+    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
+    {
+        if (which & std::ios_base::in) {
+            this->off = dir == std::ios_base::beg
+                ? off
+                : (dir == std::ios_base::end
+                    ? s.size() + off
+                    : this->off + off);
+        }
+        return pos_type(this->off);
+    }
+
+    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
+    {
+        return seekoff(pos, std::ios_base::beg, which);
+    }
+
+    std::streamsize showmanyc()
+    {
+        return s.size() - off;
+    }
+
+    int_type underflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off]);
+    }
+
+    int_type uflow()
+    {
+        if (typename string_type::size_type(off) == s.size())
+            return traits_type::eof();
+        return traits_type::to_int_type(s[off++]);
+    }
+
+    int_type pbackfail(int_type ch)
+    {
+        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
+            return traits_type::eof();
+
+        return traits_type::to_int_type(s[--off]);
+    }
+
+};
+
+template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
+class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
+{
+    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
+    buf_type buf;
+public:
+    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
+        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
+};
+
+typedef basic_istringstream_nocopy<char> istringstream_nocopy;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 645289f67c9..0e6941e4a8d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -451,92 +451,4 @@ struct ReceiveInterrupts
     { }
 };
 
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
-{
-public:
-    typedef std::basic_string<CharT, Traits, Allocator> string_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
-
-private:
-    const string_type & s;
-
-    off_type off;
-
-public:
-    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
-    {
-    }
-
-private:
-    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
-    {
-        if (which & std::ios_base::in) {
-            this->off = dir == std::ios_base::beg
-                ? off
-                : (dir == std::ios_base::end
-                    ? s.size() + off
-                    : this->off + off);
-        }
-        return pos_type(this->off);
-    }
-
-    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
-    {
-        return seekoff(pos, std::ios_base::beg, which);
-    }
-
-    std::streamsize showmanyc()
-    {
-        return s.size() - off;
-    }
-
-    int_type underflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off]);
-    }
-
-    int_type uflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off++]);
-    }
-
-    int_type pbackfail(int_type ch)
-    {
-        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
-            return traits_type::eof();
-
-        return traits_type::to_int_type(s[--off]);
-    }
-
-};
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
-{
-    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
-    buf_type buf;
-public:
-    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
-        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
-};
-
-/* A variant of std::istringstream that doesn't its string
-   argument. This is useful for large strings. The caller must ensure
-   that the string object is not destroyed while it's referenced by
-   this object. */
-typedef basic_istringstream_nocopy<char> istringstream_nocopy;
-
 }

From 7347daba8cf3a12706d3714bdaf9b7c104accde6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 14:47:35 +0100
Subject: [PATCH 0191/2196] Don't make brotli a hard dependency

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 21ca78af0fd..f18ba799e74 100644
--- a/configure.ac
+++ b/configure.ac
@@ -128,7 +128,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
-NEED_PROG(bro, bro)
+AC_PATH_PROG(bro, bro, bro)
 
 
 # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).

From 4bb38591e591e3ef59f7881575ca1f28015a8c47 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 15:06:46 +0100
Subject: [PATCH 0192/2196] Restore cache.nixos.org as the default substituter

Fixes #1283.
---
 perl/lib/Nix/Config.pm.in | 10 ----------
 src/libstore/crypto.cc    |  4 +++-
 src/libstore/store-api.cc |  5 +----
 3 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index 3575d99cb67..3613926f573 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -19,10 +19,6 @@ $useBindings = "@perlbindings@" eq "yes";
 
 %config = ();
 
-%binaryCachePublicKeys = ();
-
-$defaultPublicKeys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
-
 sub readConfig {
     if (defined $ENV{'_NIX_OPTIONS'}) {
         foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
@@ -40,12 +36,6 @@ sub readConfig {
         }
         close CONFIG;
     }
-
-    foreach my $s (split(/ /, $config{"binary-cache-public-keys"} // $defaultPublicKeys)) {
-        my ($keyName, $publicKey) = split ":", $s;
-        next unless defined $keyName && defined $publicKey;
-        $binaryCachePublicKeys{$keyName} = decode_base64($publicKey);
-    }
 }
 
 return 1;
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 747483afb30..0fc86a1fe92 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -105,7 +105,9 @@ PublicKeys getDefaultPublicKeys()
 
     // FIXME: filter duplicates
 
-    for (auto s : settings.get("binary-cache-public-keys", Strings())) {
+    for (auto s : settings.get("binary-cache-public-keys",
+            Strings{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}))
+    {
         PublicKey key(s);
         publicKeys.emplace(key.name, key);
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b1bf961e1bf..8c3422e0b43 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -778,10 +778,7 @@ std::list<ref<Store>> getDefaultSubstituters()
         state->stores.push_back(openStore(uri));
     };
 
-    for (auto uri : settings.get("substituters", Strings()))
-        addStore(uri);
-
-    for (auto uri : settings.get("binary-caches", Strings()))
+    for (auto uri : settings.get("substituters", settings.get("binary-caches", Strings{"https://cache.nixos.org/"})))
         addStore(uri);
 
     for (auto uri : settings.get("extra-binary-caches", Strings()))

From ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 15:10:35 +0100
Subject: [PATCH 0193/2196] Require signatures by default

This corresponds to the NixOS default.
---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index dcfa000c432..8610841d722 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -44,7 +44,7 @@ LocalStore::LocalStore(const Params & params)
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
     , trashDir(realStoreDir + "/trash")
-    , requireSigs(trim(settings.get("signed-binary-caches", std::string(""))) != "") // FIXME: rename option
+    , requireSigs(trim(settings.get("signed-binary-caches", std::string("*"))) != "") // FIXME: rename option
     , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());

From e1e49c58e1218fb6240dbd9780a8c67462e95d2d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 17:59:18 +0100
Subject: [PATCH 0194/2196] Only use cache.nixos.org when the store is
 /nix/store

This is consistent with the behaviour of the old
download-from-binary-cache substituter.
---
 src/libstore/store-api.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8c3422e0b43..441166d04d8 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -778,7 +778,11 @@ std::list<ref<Store>> getDefaultSubstituters()
         state->stores.push_back(openStore(uri));
     };
 
-    for (auto uri : settings.get("substituters", settings.get("binary-caches", Strings{"https://cache.nixos.org/"})))
+    Strings defaultSubstituters;
+    if (settings.nixStore == "/nix/store")
+        defaultSubstituters.push_back("https://cache.nixos.org/");
+
+    for (auto uri : settings.get("substituters", settings.get("binary-caches", defaultSubstituters)))
         addStore(uri);
 
     for (auto uri : settings.get("extra-binary-caches", Strings()))

From aa23bba27f402913a7cfa7854b5d3b6ddf321e74 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 18:06:13 +0100
Subject: [PATCH 0195/2196] Fix tests to reflect the signed-binary-caches
 default change

---
 doc/manual/command-ref/conf-file.xml |  7 ++++---
 tests/binary-cache.sh                | 10 +++++-----
 tests/repair.sh                      |  4 ++--
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 3bd133918f4..6952829e8f7 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -394,9 +394,10 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   <varlistentry><term><literal>signed-binary-caches</literal></term>
 
-    <listitem><para>If set to <literal>*</literal>, Nix will only
-    download binaries if they are signed using one of the keys listed
-    in <option>binary-cache-public-keys</option>.</para></listitem>
+    <listitem><para>If set to <literal>*</literal> (the default), Nix
+    will only download binaries if they are signed using one of the
+    keys listed in <option>binary-cache-public-keys</option>. Set to
+    the empty string to disable signature checking.</para></listitem>
 
   </varlistentry>
 
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 4ce428f643e..532099d0214 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -18,7 +18,7 @@ basicTests() {
 
     nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
 
-    nix-store --option binary-caches "file://$cacheDir" -r $outPath
+    nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '' -r $outPath
 
     [ -x $outPath/program ]
 
@@ -34,7 +34,7 @@ basicTests() {
     x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
     [ -z "$x" ]
 
-    nix-store --option binary-caches "file://$cacheDir" -r $outPath
+    nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '' -r $outPath
 
     nix-store --check-validity $outPath
     nix-store -qR $outPath | grep input-2
@@ -63,7 +63,7 @@ mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar
 
-nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "hash mismatch" $TEST_ROOT/log
 
 mv $nar.good $nar
@@ -73,7 +73,7 @@ mv $nar.good $nar
 clearStore
 clearCacheCache
 
-if nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' -r $outPath; then
+if nix-store --option binary-caches "file://$cacheDir" -r $outPath; then
     echo "unsigned binary cache incorrectly accepted"
     exit 1
 fi
@@ -99,7 +99,7 @@ clearStore
 
 rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
-nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "fetching path" $TEST_ROOT/log
 
 
diff --git a/tests/repair.sh b/tests/repair.sh
index 782838704da..57152d450a1 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -51,7 +51,7 @@ nix copy --recursive --to file://$cacheDir $path
 chmod u+w $path2
 rm -rf $path2
 
-nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir"
+nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir" --option signed-binary-caches ''
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2
@@ -69,7 +69,7 @@ if nix-store --verify-path $path2; then
     exit 1
 fi
 
-nix-store --repair-path $path2 --option binary-caches "file://$cacheDir"
+nix-store --repair-path $path2 --option binary-caches "file://$cacheDir" --option signed-binary-caches ''
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2

From ed5c0f69f28732879a7aac2d67367446f6d3152d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 19:20:21 +0100
Subject: [PATCH 0196/2196] Don't hang in decompression if bzip2 data ends
 prematurely

---
 src/libutil/compression.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 8ffd55efb23..11eec7a7bc2 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -49,6 +49,9 @@ static ref<std::string> decompressXZ(const std::string & in)
 
         if (ret != LZMA_OK)
             throw Error("error while decompressing xz file");
+
+        if (strm.avail_in == 0)
+            throw Error("xz data ends prematurely");
     }
 }
 
@@ -86,6 +89,9 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
         if (ret != BZ_OK)
             throw Error("error while decompressing bzip2 file");
+
+        if (strm.avail_in == 0)
+            throw Error("bzip2 data ends prematurely");
     }
 }
 

From 895a74a814cd67cd2e13d0621603583a2d15b159 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Mar 2017 19:23:07 +0100
Subject: [PATCH 0197/2196] LocalFSStore::getBuildLog(): Handle corrupted logs

---
 src/libstore/local-fs-store.cc |  9 +++++++--
 src/libutil/compression.cc     | 24 ++++++++++++------------
 src/libutil/compression.hh     |  2 ++
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 002ee4a65ce..57e1b8a09fe 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -94,6 +94,7 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
 
     assertStorePath(path);
 
+
     if (!isDerivation(path)) {
         try {
             path = queryPathInfo(path)->deriver;
@@ -116,8 +117,12 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
         if (pathExists(logPath))
             return std::make_shared<std::string>(readFile(logPath));
 
-        else if (pathExists(logBz2Path))
-            return decompress("bzip2", readFile(logBz2Path));
+        else if (pathExists(logBz2Path)) {
+            try {
+                return decompress("bzip2", readFile(logBz2Path));
+            } catch (Error &) { }
+        }
+
     }
 
     return nullptr;
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 11eec7a7bc2..f913d0f5bad 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -18,7 +18,7 @@ static ref<std::string> decompressXZ(const std::string & in)
     lzma_ret ret = lzma_stream_decoder(
         &strm, UINT64_MAX, LZMA_CONCATENATED);
     if (ret != LZMA_OK)
-        throw Error("unable to initialise lzma decoder");
+        throw CompressionError("unable to initialise lzma decoder");
 
     Finally free([&]() { lzma_end(&strm); });
 
@@ -48,10 +48,10 @@ static ref<std::string> decompressXZ(const std::string & in)
             return res;
 
         if (ret != LZMA_OK)
-            throw Error("error while decompressing xz file");
+            throw CompressionError("error while decompressing xz file");
 
         if (strm.avail_in == 0)
-            throw Error("xz data ends prematurely");
+            throw CompressionError("xz data ends prematurely");
     }
 }
 
@@ -62,7 +62,7 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
     int ret = BZ2_bzDecompressInit(&strm, 0, 0);
     if (ret != BZ_OK)
-        throw Error("unable to initialise bzip2 decoder");
+        throw CompressionError("unable to initialise bzip2 decoder");
 
     Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
 
@@ -88,10 +88,10 @@ static ref<std::string> decompressBzip2(const std::string & in)
             return res;
 
         if (ret != BZ_OK)
-            throw Error("error while decompressing bzip2 file");
+            throw CompressionError("error while decompressing bzip2 file");
 
         if (strm.avail_in == 0)
-            throw Error("bzip2 data ends prematurely");
+            throw CompressionError("bzip2 data ends prematurely");
     }
 }
 
@@ -144,7 +144,7 @@ struct XzSink : CompressionSink
         lzma_ret ret = lzma_easy_encoder(
             &strm, 6, LZMA_CHECK_CRC64);
         if (ret != LZMA_OK)
-            throw Error("unable to initialise lzma encoder");
+            throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
 
         strm.next_out = outbuf;
@@ -168,7 +168,7 @@ struct XzSink : CompressionSink
 
             lzma_ret ret = lzma_code(&strm, LZMA_FINISH);
             if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw Error("error while flushing xz file");
+                throw CompressionError("error while flushing xz file");
 
             if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
                 nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
@@ -192,7 +192,7 @@ struct XzSink : CompressionSink
 
             lzma_ret ret = lzma_code(&strm, LZMA_RUN);
             if (ret != LZMA_OK)
-                throw Error("error while compressing xz file");
+                throw CompressionError("error while compressing xz file");
 
             if (strm.avail_out == 0) {
                 nextSink(outbuf, sizeof(outbuf));
@@ -215,7 +215,7 @@ struct BzipSink : CompressionSink
         memset(&strm, 0, sizeof(strm));
         int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
         if (ret != BZ_OK)
-            throw Error("unable to initialise bzip2 encoder");
+            throw CompressionError("unable to initialise bzip2 encoder");
 
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
@@ -238,7 +238,7 @@ struct BzipSink : CompressionSink
 
             int ret = BZ2_bzCompress(&strm, BZ_FINISH);
             if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw Error("error while flushing bzip2 file");
+                throw CompressionError("error while flushing bzip2 file");
 
             if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
                 nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out);
@@ -262,7 +262,7 @@ struct BzipSink : CompressionSink
 
             int ret = BZ2_bzCompress(&strm, BZ_RUN);
             if (ret != BZ_OK)
-                Error("error while compressing bzip2 file");
+                CompressionError("error while compressing bzip2 file");
 
             if (strm.avail_out == 0) {
                 nextSink((unsigned char *) outbuf, sizeof(outbuf));
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index eacf559d65e..e3e6f5a9930 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -21,4 +21,6 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
 
 MakeError(UnknownCompressionMethod, Error);
 
+MakeError(CompressionError, Error);
+
 }

From 0a7ca24c26d57d532df614bdfe78b8b5c228e90d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Mar 2017 11:53:33 +0100
Subject: [PATCH 0198/2196] Fix xz decompression

Fixes #1285.
---
 src/libutil/compression.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index f913d0f5bad..b0b1d709fa4 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -48,10 +48,7 @@ static ref<std::string> decompressXZ(const std::string & in)
             return res;
 
         if (ret != LZMA_OK)
-            throw CompressionError("error while decompressing xz file");
-
-        if (strm.avail_in == 0)
-            throw CompressionError("xz data ends prematurely");
+            throw CompressionError("error %d while decompressing xz file", ret);
     }
 }
 

From 8edf1071773244095aacd059ebe5b03098d255e3 Mon Sep 17 00:00:00 2001
From: Matt Audesse <matt@mattaudesse.com>
Date: Wed, 22 Mar 2017 10:11:23 -0400
Subject: [PATCH 0199/2196] Fix minor grammatical nitpick ("it's" vs. "its") in
 `README.md`. See: http://data.grammarbook.com/blog/pronouns/1-grammar-error/

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1eb73b256f5..3173c6c448a 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 Nix, the purely functional package manager
 ------------------------------------------
 
-Nix is a new take on package management that is fairly unique. Because of it's
+Nix is a new take on package management that is fairly unique. Because of its
 purity aspects, a lot of issues found in traditional package managers don't
 appear with Nix.
 

From 023217f07c1acd75403bfea88bc38eb48905dc4e Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 24 Mar 2017 23:05:49 +0100
Subject: [PATCH 0200/2196] use std::tuple for ValueMap allocator

---
 src/libexpr/value.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 81f918d48de..802e8ed2ee7 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -256,7 +256,7 @@ size_t valueSize(Value & v);
 
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<Value *> > ValueMap;
+typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;

From c60715e937e3773bbb8a114fc9b9c6577f8c5cb5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Mar 2017 13:08:13 +0200
Subject: [PATCH 0201/2196] Ignore broken "Deriver: unknown-deriver" fields in
 .narinfo

These were generated by a legacy tool.
---
 src/libstore/nar-info.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 201cac671a5..d1042c6de25 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -59,9 +59,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
             }
         }
         else if (name == "Deriver") {
-            auto p = store.storeDir + "/" + value;
-            if (!store.isStorePath(p)) corrupt();
-            deriver = p;
+            if (value != "unknown-deriver") {
+                auto p = store.storeDir + "/" + value;
+                if (!store.isStorePath(p)) corrupt();
+                deriver = p;
+            }
         }
         else if (name == "System")
             system = value;

From 0bb8db257d98a32abde759f4d07d28b5178bd3bf Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 30 Mar 2017 08:04:21 -0400
Subject: [PATCH 0202/2196] Add exec primop behind
 allow-unsafe-native-code-during-evaluation.

Execute a given program with the (optional) given arguments as the
user running the evaluation, parsing stdout as an expression to be
evaluated.

There are many use cases for nix that would benefit from being able to
run arbitrary code during evaluation, including but not limited to:

* Automatic git fetching to get a sha256 from a git revision
* git rev-parse HEAD
* Automatic extraction of information from build specifications from
  other tools, particularly language-specific package managers like
  cabal or npm
* Secrets decryption (e.g. with nixops)
* Private repository fetching

Ideally, we would add this functionality in a more principled way to
nix, but in the mean time 'builtins.exec' can be used to get these
tasks done.

The primop is only available when the
'allow-unsafe-native-code-during-evaluation' nix option is true. That
flag also enables the 'importNative' primop, which is strictly more
powerful but less convenient (since it requires compiling a plugin
against the running version of nix).
---
 src/libexpr/primops.cc  | 56 ++++++++++++++++++++++++++++++++++++++++-
 src/libstore/globals.cc |  4 +--
 src/libstore/globals.hh |  4 +--
 3 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 93097f3d1bf..a98da737e4a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -178,6 +178,58 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/* Execute a program and parse its output */
+static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+    auto sProgram = state.symbols.create("program");
+    auto sArguments = state.symbols.create("arguments");
+    PathSet context;
+    string program;
+    bool programSet = false;
+    Strings commandArgs;
+    for (auto & attr : *args[0]->attrs) {
+        if (attr.name == sProgram) {
+            program = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+            programSet = true;
+        } else if (attr.name == sArguments) {
+            state.forceList(*attr.value, *attr.pos);
+            auto elems = attr.value->listElems();
+            for (unsigned int i = 0; i < attr.value->listSize(); ++i) {
+                commandArgs.emplace_back(state.coerceToString(*attr.pos, *elems[i], context, false, false));
+            }
+        } else {
+            throw EvalError(format("unexpected attribute ‘%1%’ in argument to builtins.exec, at %2%")
+                % attr.name % pos);
+        }
+    }
+    if (!programSet) {
+        throw EvalError(format("attribute ‘programSet’ required, at %1%") % pos);
+    }
+    try {
+        state.realiseContext(context);
+    } catch (InvalidPathError & e) {
+        throw EvalError(format("cannot execute ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+            % program % e.path % pos);
+    }
+
+    auto output = runProgram(program, true, commandArgs);
+    Expr * parsed;
+    try {
+        parsed = state.parseExprFromString(output, pos.file);
+    } catch (Error & e) {
+        e.addPrefix(format("While parsing the output from ‘%1%’, at %2%\n") % program % pos);
+        throw;
+    }
+    try {
+        state.eval(parsed, v);
+    } catch (Error & e) {
+        e.addPrefix(format("While evaluating the output from ‘%1%’, at %2%\n") % program % pos);
+        throw;
+    }
+}
+
+
 /* Return a string representing the type of the expression. */
 static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -1903,8 +1955,10 @@ void EvalState::createBaseEnv()
     mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2);
     forceValue(v);
     addConstant("import", v);
-    if (settings.enableImportNative)
+    if (settings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
+        addPrimOp("__exec", 1, prim_exec);
+    }
     addPrimOp("__typeOf", 1, prim_typeOf);
     addPrimOp("isNull", 1, prim_isNull);
     addPrimOp("__isFunction", 1, prim_isFunction);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 012b3d5b8b9..8c900be77b8 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -67,7 +67,7 @@ Settings::Settings()
     envKeepDerivations = false;
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
     showTrace = false;
-    enableImportNative = false;
+    enableNativeCode = false;
     netrcFile = fmt("%s/%s", nixConfDir, "netrc");
     caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
     enableImportFromDerivation = true;
@@ -179,7 +179,7 @@ void Settings::update()
     _get(envKeepDerivations, "env-keep-derivations");
     _get(sshSubstituterHosts, "ssh-substituter-hosts");
     _get(useSshSubstituter, "use-ssh-substituter");
-    _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
+    _get(enableNativeCode, "allow-unsafe-native-code-during-evaluation");
     _get(useCaseHack, "use-case-hack");
     _get(preBuildHook, "pre-build-hook");
     _get(keepGoing, "keep-going");
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 46272168191..ccec300f776 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -181,8 +181,8 @@ struct Settings {
     /* Whether to show a stack trace if Nix evaluation fails. */
     bool showTrace;
 
-    /* Whether the importNative primop should be enabled */
-    bool enableImportNative;
+    /* Whether native-code enabling primops should be enabled */
+    bool enableNativeCode;
 
     /* The hook to run just before a build to set derivation-specific
        build settings */

From a75475ca611fbc9074792a30740d19fd3a3a6cf7 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 30 Mar 2017 16:51:50 -0400
Subject: [PATCH 0203/2196] Remove tabs

---
 release.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release.nix b/release.nix
index b93b64ea923..c0e1385e13f 100644
--- a/release.nix
+++ b/release.nix
@@ -31,7 +31,7 @@ let
             git
           ];
 
-	configureFlags = "--enable-gc";
+        configureFlags = "--enable-gc";
 
         postUnpack = ''
           # Clean up when building from a working tree.
@@ -109,7 +109,7 @@ let
           [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ]
           ++ lib.optional stdenv.isLinux libsodium;
 
-	configureFlags = ''
+        configureFlags = ''
           --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
           --with-www-curl=${perlPackages.WWWCurl}/${pkgs.perl.libPrefix}
@@ -117,7 +117,7 @@ let
 
         enableParallelBuilding = true;
 
-	postUnpack = "sourceRoot=$sourceRoot/perl";
+        postUnpack = "sourceRoot=$sourceRoot/perl";
 
         preBuild = "unset NIX_INDENT_MAKE";
       });

From d0877003475ecb3ea18a256239a9948fe0f28035 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Mar 2017 15:31:34 +0200
Subject: [PATCH 0204/2196] Fix perl build

---
 perl/Makefile           |  2 +-
 perl/Makefile.config.in |  1 -
 perl/configure.ac       |  2 --
 perl/local.mk           | 42 +++++++++++++++++++----------------------
 4 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/perl/Makefile b/perl/Makefile
index 41a32576e9b..cf655ae3d65 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -std=c++11 -g -Wall
+GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include nix/config.h
 
 -include Makefile.config
 
diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in
index 901d1283e55..c87d4817e17 100644
--- a/perl/Makefile.config.in
+++ b/perl/Makefile.config.in
@@ -16,4 +16,3 @@ perl = @perl@
 perllibdir = @perllibdir@
 nixstoredir = @nixstoredir@
 nixsysconfdir = @nixsysconfdir@
-perlbindings = @perlbindings@
diff --git a/perl/configure.ac b/perl/configure.ac
index dea2b614004..d617c78535f 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -99,8 +99,6 @@ AC_SUBST(nixlocalstatedir)
 AC_SUBST(nixsysconfdir)
 AC_SUBST(nixstoredir)
 
-AC_SUBST(perlbindings, "yes")
-
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
diff --git a/perl/local.mk b/perl/local.mk
index 1793ececfd6..35113bd960d 100644
--- a/perl/local.mk
+++ b/perl/local.mk
@@ -10,38 +10,34 @@ nix_perl_modules := $(nix_perl_sources:.in=)
 
 $(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix)))
 
-ifeq ($(perlbindings), yes)
-
-  lib/Nix/Store.cc: lib/Nix/Store.xs
+lib/Nix/Store.cc: lib/Nix/Store.xs
 	$(trace-gen) xsubpp $^ -output $@
 
-  libraries += Store
-
-  Store_DIR := lib/Nix
+libraries += Store
 
-  Store_SOURCES := $(Store_DIR)/Store.cc
+Store_DIR := lib/Nix
 
-  Store_CXXFLAGS = \
-    -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \
-    -D_FILE_OFFSET_BITS=64 \
-    -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \
-    -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion \
-    $(NIX_CFLAGS)
+Store_SOURCES := $(Store_DIR)/Store.cc
 
-  Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
+Store_CXXFLAGS = \
+  -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \
+  -D_FILE_OFFSET_BITS=64 \
+  -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \
+  -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion \
+  $(NIX_CFLAGS)
 
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
-    archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
-    libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
-    Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})
-  endif
+Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
-  Store_ALLOW_UNDEFINED = 1
+ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
+  libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
+  Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})
+endif
 
-  Store_FORCE_INSTALL = 1
+Store_ALLOW_UNDEFINED = 1
 
-  Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store
+Store_FORCE_INSTALL = 1
 
-endif
+Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store
 
 clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config

From e1509adbbb12851c30ba230650005bd141958249 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Mar 2017 15:50:11 +0200
Subject: [PATCH 0205/2196] Retry curl error 16

---
 src/libstore/download.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 22bde086e6a..78fcdc62189 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -301,6 +301,7 @@ struct CurlDownloader : public Downloader
                         || code == CURLE_COULDNT_RESOLVE_HOST
                         || code == CURLE_RECV_ERROR
 #if LIBCURL_VERSION_NUM >= 0x073200
+                        || code == CURLE_HTTP2
                         || code == CURLE_HTTP2_STREAM
 #endif
                         ) ? Transient :

From b9b8b8a63ba49dc027b08950bd3cf30cc8f09ec5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Mar 2017 15:54:15 +0200
Subject: [PATCH 0206/2196] Fix evaluation error

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 1a8d0927c10..8727c2520b1 100644
--- a/release.nix
+++ b/release.nix
@@ -101,7 +101,7 @@ let
       });
 
 
-    perl = pkgs.lib.genAttrs systems (system:
+    perlBindings = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import <nixpkgs> { inherit system; }; in with pkgs;
 

From 3ecb09a40a8500d1052b087295b589ca4856fd7a Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Fri, 31 Mar 2017 11:58:41 -0400
Subject: [PATCH 0207/2196] builtins.exec: Make the argument just a list

---
 src/libexpr/primops.cc | 31 +++++++++----------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a98da737e4a..615cc813843 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -181,30 +181,17 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
 /* Execute a program and parse its output */
 static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceAttrs(*args[0], pos);
-    auto sProgram = state.symbols.create("program");
-    auto sArguments = state.symbols.create("arguments");
+    state.forceList(*args[0], pos);
+    auto elems = args[0]->listElems();
+    auto count = args[0]->listSize();
+    if (count == 0) {
+        throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos);
+    }
     PathSet context;
-    string program;
-    bool programSet = false;
+    auto program = state.coerceToString(pos, *elems[0], context, false, false);
     Strings commandArgs;
-    for (auto & attr : *args[0]->attrs) {
-        if (attr.name == sProgram) {
-            program = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            programSet = true;
-        } else if (attr.name == sArguments) {
-            state.forceList(*attr.value, *attr.pos);
-            auto elems = attr.value->listElems();
-            for (unsigned int i = 0; i < attr.value->listSize(); ++i) {
-                commandArgs.emplace_back(state.coerceToString(*attr.pos, *elems[i], context, false, false));
-            }
-        } else {
-            throw EvalError(format("unexpected attribute ‘%1%’ in argument to builtins.exec, at %2%")
-                % attr.name % pos);
-        }
-    }
-    if (!programSet) {
-        throw EvalError(format("attribute ‘programSet’ required, at %1%") % pos);
+    for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
+        commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false));
     }
     try {
         state.realiseContext(context);

From 29d35805c63f316aa19b33a481f953ca332d9b65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Mar 2017 18:12:01 +0200
Subject: [PATCH 0208/2196] Sandbox: Fix /dev/ptmx on recent kernels

This fixes "No such file or directory" when opening /dev/ptmx
(e.g. http://hydra.nixos.org/build/51094249).

The reason appears to be some changes to /dev/ptmx / /dev/pts handling
between Linux 4.4 and 4.9. See
https://patchwork.kernel.org/patch/7832531/.

The fix is to go back to mounting a proper /dev/pts instance inside
the sandbox. Happily, this now works inside user namespaces, even for
unprivileged users. So

  NIX_REMOTE=local?root=/tmp/nix nix-build \
    '<nixpkgs/nixos/tests/misc.nix>' -A test

works for non-root users.

The downside is that the fix breaks sandbox builds on older kernels
(probably pre-4.6), since mounting a devpts fails inside user
namespaces for some reason I've never been able to figure out. Builds
on those systems will fail with

  error: while setting up the build environment: mounting /dev/pts: Invalid argument

Ah well.
---
 src/libstore/build.cc | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index fc840df81a5..43a8dadf822 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2391,8 +2391,6 @@ void DerivationGoal::runChild()
                 ss.push_back("/dev/tty");
                 ss.push_back("/dev/urandom");
                 ss.push_back("/dev/zero");
-                ss.push_back("/dev/ptmx");
-                ss.push_back("/dev/pts");
                 createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
                 createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
                 createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
@@ -2448,17 +2446,13 @@ void DerivationGoal::runChild()
                     fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1)
                 throw SysError("mounting /dev/shm");
 
-#if 0
-            // FIXME: can't figure out how to do this in a user
-            // namespace.
-
             /* Mount a new devpts on /dev/pts.  Note that this
                requires the kernel to be compiled with
                CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
                if /dev/ptx/ptmx exists). */
             if (pathExists("/dev/pts/ptmx") &&
                 !pathExists(chrootRootDir + "/dev/ptmx")
-                && dirsInChroot.find("/dev/pts") == dirsInChroot.end())
+                && !dirsInChroot.count("/dev/pts"))
             {
                 if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1)
                     throw SysError("mounting /dev/pts");
@@ -2468,7 +2462,6 @@ void DerivationGoal::runChild()
                    Linux versions, it is created with permissions 0.  */
                 chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
             }
-#endif
 
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)

From 3b8946e09a55a27c01da863d8c6e008e3fe4076e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Mar 2017 18:30:50 +0200
Subject: [PATCH 0209/2196] Maybe fix the RPM build

http://hydra.nixos.org/build/51095532
---
 nix.spec.in | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 0c9b9ab2013..e879352fe82 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -16,12 +16,7 @@ Source0: %{name}-%{version}.tar.bz2
 %if 0%{?el5}
 BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 %endif
-BuildRequires: perl(DBD::SQLite)
-BuildRequires: perl(DBI)
-BuildRequires: perl(ExtUtils::ParseXS)
-Requires: /usr/bin/perl
 Requires: curl
-Requires: perl-DBD-SQLite
 Requires: bzip2
 Requires: gzip
 Requires: xz

From dfcd78d851c14355a09f5b8845042ade098e739a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 3 Apr 2017 16:25:20 +0200
Subject: [PATCH 0210/2196] Really fix the RPM build

---
 nix.spec.in | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index e879352fe82..390893d64dc 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -87,11 +87,6 @@ the emacs-%{name} package to edit Nix expressions with GNU Emacs.
 
 %prep
 %setup -q
-# Install Perl modules to vendor_perl
-# configure.ac need to be changed to make this global; however, this will
-# also affect NixOS. Use discretion.
-%{__sed} -i 's|perl5/site_perl/$perlversion/$perlarchname|perl5/vendor_perl|' \
-  configure
 
 
 %build
@@ -164,8 +159,6 @@ systemctl start  nix-daemon.socket
 %files
 %{_bindir}/nix*
 %{_libdir}/*.so
-%{perl_vendorarch}/*
-%exclude %dir %{perl_vendorarch}/auto/
 %{_prefix}/libexec/*
 %if ! 0%{?rhel} || 0%{?rhel} >= 7
 %{_prefix}/lib/systemd/system/nix-daemon.socket

From 488792a87d3dea18109fee7fa987f7edf6ba1149 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Apr 2017 17:40:50 +0200
Subject: [PATCH 0211/2196] Make /var/run/nscd/socket optional

Not every distribution uses nscd.
---
 src/libstore/build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 43a8dadf822..14982424d5e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2405,7 +2405,8 @@ void DerivationGoal::runChild()
                 ss.push_back("/etc/nsswitch.conf");
                 ss.push_back("/etc/services");
                 ss.push_back("/etc/hosts");
-                ss.push_back("/var/run/nscd/socket");
+                if (pathExists("/var/run/nscd/socket"))
+                    ss.push_back("/var/run/nscd/socket");
             }
 
             for (auto & i : ss) dirsInChroot[i] = i;

From 8decb07c31581febab664bedde12c8bf1367279e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Apr 2017 17:54:16 +0200
Subject: [PATCH 0212/2196] Allow default sandbox paths to be overriden

E.g. you can now redirect /etc/resolv.conf to a different file.
---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 14982424d5e..968e291129f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2409,7 +2409,7 @@ void DerivationGoal::runChild()
                     ss.push_back("/var/run/nscd/socket");
             }
 
-            for (auto & i : ss) dirsInChroot[i] = i;
+            for (auto & i : ss) dirsInChroot.emplace(i, i);
 
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot

From 256940fc48a6db950136fb0bc43590b701a3e857 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Apr 2017 14:30:31 +0200
Subject: [PATCH 0213/2196] nix-daemon: Disable path info cache

This is useless because the client also caches path info, and can
cause problems for long-running clients like hydra-queue-runner
(i.e. it may return cached info about paths that have been
garbage-collected).
---
 src/libstore/store-api.cc    | 1 +
 src/libstore/store-api.hh    | 7 ++++++-
 src/libutil/lru-cache.hh     | 8 +++++---
 src/libutil/sync.hh          | 1 +
 src/nix-daemon/nix-daemon.cc | 5 ++++-
 5 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 441166d04d8..59348c5d0b5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -242,6 +242,7 @@ Path Store::computeStorePathForText(const string & name, const string & s,
 
 Store::Store(const Params & params)
     : storeDir(get(params, "store", settings.nixStore))
+    , state({std::stoi(get(params, "path-info-cache-size", "65536"))})
 {
 }
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 98f2803f813..f58dbde350c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -241,7 +241,7 @@ protected:
 
     struct State
     {
-        LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
+        LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
     };
 
     Sync<State> state;
@@ -252,6 +252,11 @@ protected:
 
 public:
 
+    size_t getCacheSize()
+    {
+        return state.lock()->pathInfoCache.size();
+    }
+
     virtual ~Store() { }
 
     virtual std::string getUri() = 0;
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 35983aa2c91..3cb5d50889d 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -11,7 +11,7 @@ class LRUCache
 {
 private:
 
-    size_t maxSize;
+    size_t capacity;
 
     // Stupid wrapper to get around circular dependency between Data
     // and LRU.
@@ -27,14 +27,16 @@ private:
 
 public:
 
-    LRUCache(size_t maxSize) : maxSize(maxSize) { }
+    LRUCache(size_t capacity) : capacity(capacity) { }
 
     /* Insert or upsert an item in the cache. */
     void upsert(const Key & key, const Value & value)
     {
+        if (capacity == 0) return;
+
         erase(key);
 
-        if (data.size() >= maxSize) {
+        if (data.size() >= capacity) {
             /* Retire the oldest item. */
             auto oldest = lru.begin();
             data.erase(*oldest);
diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
index 2aa074299b2..611c900e0a3 100644
--- a/src/libutil/sync.hh
+++ b/src/libutil/sync.hh
@@ -33,6 +33,7 @@ public:
 
     Sync() { }
     Sync(const T & data) : data(data) { }
+    Sync(T && data) noexcept : data(std::move(data)) { }
 
     class Lock
     {
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index ab5826b0d1a..b6a46642c7c 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -637,7 +637,10 @@ static void processConnection(bool trusted)
 #endif
 
         /* Open the store. */
-        auto store = make_ref<LocalStore>(Store::Params()); // FIXME: get params from somewhere
+        Store::Params params; // FIXME: get params from somewhere
+        // Disable caching since the client already does that.
+        params["path-info-cache-size"] = "0";
+        auto store = make_ref<LocalStore>(params);
 
         stopWork();
         to.flush();

From 6b5e2711632c345f2b6a371ef8a859106436980a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Apr 2017 15:22:37 +0200
Subject: [PATCH 0214/2196] Add a method to allow hydra-queue-runner to flush
 the path info cache

---
 src/libstore/store-api.hh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f58dbde350c..e07dec49547 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -252,11 +252,6 @@ protected:
 
 public:
 
-    size_t getCacheSize()
-    {
-        return state.lock()->pathInfoCache.size();
-    }
-
     virtual ~Store() { }
 
     virtual std::string getUri() = 0;
@@ -578,6 +573,13 @@ public:
     virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
     { return nullptr; }
 
+    /* Hack to allow long-running processes like hydra-queue-runner to
+       occasionally flush their path info cache. */
+    void clearPathInfoCache()
+    {
+        state.lock()->pathInfoCache.clear();
+    }
+
 protected:
 
     Stats stats;

From 963f2bf12b4098a3b46059a26d21cdfe27662cdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Apr 2017 17:18:56 +0200
Subject: [PATCH 0215/2196] Fix bogus "unexpected Nix daemon error: interrupted
 by the user"

---
 src/libutil/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 99a91c8cc64..a640a64c724 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -487,6 +487,7 @@ void readFull(int fd, unsigned char * buf, size_t count)
 void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts)
 {
     while (count) {
+        if (allowInterrupts) checkInterrupt();
         ssize_t res = write(fd, (char *) buf, count);
         if (res == -1 && errno != EINTR)
             throw SysError("writing to file");
@@ -494,7 +495,6 @@ void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterr
             count -= res;
             buf += res;
         }
-        if (allowInterrupts) checkInterrupt();
     }
 }
 
@@ -1212,7 +1212,7 @@ static void signalHandlerThread(sigset_t set)
 
 void triggerInterrupt()
 {
-    _isInterrupted = 1;
+    _isInterrupted = true;
 
     {
         auto interruptCallbacks(_interruptCallbacks.lock());

From ba20730b3f7ad6b09aa86b66748df575e56d442e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Apr 2017 18:40:19 +0200
Subject: [PATCH 0216/2196] Implement RemoteStore::queryMissing()

This provides a significant speedup, e.g. 64 s -> 12 s for

  nix-build --dry-run -I nixpkgs=channel:nixos-16.03 '<nixpkgs/nixos/tests/misc.nix>' -A test

on a cold local and CloudFront cache.

The alternative is to use lots of concurrent daemon connections but
that seems wasteful.
---
 src/libstore/remote-store.cc    | 25 +++++++++++++++++++++++++
 src/libstore/remote-store.hh    |  4 ++++
 src/libstore/store-api.hh       |  2 +-
 src/libstore/worker-protocol.hh |  5 +++--
 src/nix-daemon/nix-daemon.cc    | 11 +++++++++++
 5 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a1f2db5b0ec..c9c59078745 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -588,6 +588,31 @@ void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
 }
 
 
+void RemoteStore::queryMissing(const PathSet & targets,
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize, unsigned long long & narSize)
+{
+    {
+        auto conn(connections->get());
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
+            // Don't hold the connection handle in the fallback case
+            // to prevent a deadlock.
+            goto fallback;
+        conn->to << wopQueryMissing << targets;
+        conn->processStderr();
+        willBuild = readStorePaths<PathSet>(*this, conn->from);
+        willSubstitute = readStorePaths<PathSet>(*this, conn->from);
+        unknown = readStorePaths<PathSet>(*this, conn->from);
+        conn->from >> downloadSize >> narSize;
+        return;
+    }
+
+ fallback:
+    return Store::queryMissing(targets, willBuild, willSubstitute,
+        unknown, downloadSize, narSize);
+}
+
+
 RemoteStore::Connection::~Connection()
 {
     try {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index a08bd305639..db8da7eaa8c 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -85,6 +85,10 @@ public:
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
+    void queryMissing(const PathSet & targets,
+        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+        unsigned long long & downloadSize, unsigned long long & narSize) override;
+
 protected:
 
     struct Connection
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e07dec49547..68c59a9f292 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -524,7 +524,7 @@ public:
     /* Given a set of paths that are to be built, return the set of
        derivations that will be built, and the set of output paths
        that will be substituted. */
-    void queryMissing(const PathSet & targets,
+    virtual void queryMissing(const PathSet & targets,
         PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
         unsigned long long & downloadSize, unsigned long long & narSize);
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6a4ed47cc9f..6c6766b3612 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x112
+#define PROTOCOL_VERSION 0x113
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -47,7 +47,8 @@ typedef enum {
     wopBuildDerivation = 36,
     wopAddSignatures = 37,
     wopNarFromPath = 38,
-    wopAddToStoreNar = 39
+    wopAddToStoreNar = 39,
+    wopQueryMissing = 40,
 } WorkerOp;
 
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index b6a46642c7c..8786e2561b9 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -592,6 +592,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         break;
     }
 
+    case wopQueryMissing: {
+        PathSet targets = readStorePaths<PathSet>(*store, from);
+        startWork();
+        PathSet willBuild, willSubstitute, unknown;
+        unsigned long long downloadSize, narSize;
+        store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
+        stopWork();
+        to << willBuild << willSubstitute << unknown << downloadSize << narSize;
+        break;
+    }
+
     default:
         throw Error(format("invalid operation %1%") % op);
     }

From 98283915f54df275170cc1c8bdaa0b00d343cd69 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Thu, 6 Apr 2017 18:18:43 +0000
Subject: [PATCH 0217/2196] Retry downloads on transient SSL errors too

---
 src/libstore/download.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 78fcdc62189..d1f760fdc30 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -300,6 +300,8 @@ struct CurlDownloader : public Downloader
                         || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
                         || code == CURLE_COULDNT_RESOLVE_HOST
                         || code == CURLE_RECV_ERROR
+                        // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable
+                        || code == CURLE_SSL_CONNECT_ERROR
 #if LIBCURL_VERSION_NUM >= 0x073200
                         || code == CURLE_HTTP2
                         || code == CURLE_HTTP2_STREAM

From 30f89e0d6564394747e20e04d486b81aeec96752 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 5 Apr 2017 07:34:10 -0500
Subject: [PATCH 0218/2196] Process nix.conf options in "new" nix commands, add
 test.

Without this (minor) change, the options set using "--option"
or read from nix.conf were parsed but not used.
---
 src/nix/main.cc  | 1 +
 tests/timeout.sh | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 440ced97dfc..fdb8f6e3a19 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -42,6 +42,7 @@ void mainWrapped(int argc, char * * argv)
     NixArgs args;
 
     args.parseCmdline(argvToStrings(argc, argv));
+    settings.update();
 
     assert(args.command);
 
diff --git a/tests/timeout.sh b/tests/timeout.sh
index ce1ae7d674a..77b227e89ba 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -29,3 +29,8 @@ if nix-build timeout.nix -A closeLog; then
     echo "build should have failed"
     exit 1
 fi
+
+if nix build -f timeout.nix silent --option build-max-silent-time 2; then
+    echo "build should have failed"
+    exit 1
+fi

From a0c56197fcb6d13e77ccbdd28322a540352b880c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 5 Apr 2017 08:13:07 -0500
Subject: [PATCH 0219/2196] tests/timeout: create output so tests don't
 trivially pass

Timeout tests rely on failed build to determine success,
so make sure these derivations (silent in particular)
don't fail regardless of timeout behavior.
---
 tests/timeout.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/timeout.nix b/tests/timeout.nix
index 540fba934ff..e18d717eff1 100644
--- a/tests/timeout.nix
+++ b/tests/timeout.nix
@@ -5,6 +5,7 @@ with import ./config.nix;
   infiniteLoop = mkDerivation {
     name = "timeout";
     buildCommand = ''
+      touch $out
       echo "‘timeout’ builder entering an infinite loop"
       while true ; do echo -n .; done
     '';
@@ -13,6 +14,7 @@ with import ./config.nix;
   silent = mkDerivation {
     name = "silent";
     buildCommand = ''
+      touch $out
       sleep 60
     '';
   };
@@ -20,6 +22,7 @@ with import ./config.nix;
   closeLog = mkDerivation {
     name = "silent";
     buildCommand = ''
+      touch $out
       exec > /dev/null 2>&1
       sleep 1000000000
     '';

From f12a048a0595ea75453c6b67a3b72da8106fc6ad Mon Sep 17 00:00:00 2001
From: Neil Mayhew <neil@neil.mayhew.name>
Date: Sat, 8 Apr 2017 18:31:11 -0600
Subject: [PATCH 0220/2196] Propagate NIX_BUILD_CORES to nix-shell environments

---
 src/nix-build/nix-build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b4206033cf5..065447684aa 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -406,6 +406,7 @@ int main(int argc, char ** argv)
 
                 env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
                 env["NIX_STORE"] = store->storeDir;
+                env["NIX_BUILD_CORES"] = settings.buildCores;
 
                 for (auto & var : drv.env)
                     env[var.first] = var.second;

From 95295482ea629fd21992f564b8e4f1710a7a61e6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 6 Apr 2017 19:01:05 +0200
Subject: [PATCH 0221/2196] Allow "auto" as a store URI

Using the empty string is likely to be ambiguous in some contexts.
---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 59348c5d0b5..53c802044ea 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -731,7 +731,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
         return tDaemon;
     } else if (uri == "local") {
         return tLocal;
-    } else if (uri == "") {
+    } else if (uri == "" || uri == "auto") {
         if (access(stateDir.c_str(), R_OK | W_OK) == 0)
             return tLocal;
         else if (pathExists(settings.nixDaemonSocketFile))

From 105f8ffc98d25e23c9a7eb5a5327e711b1e21e50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 10 Apr 2017 11:12:45 +0200
Subject: [PATCH 0222/2196] Minor cleanup

Also, possible fix for #1310 on 32-bit systems.
---
 src/libstore/gc.cc          |  2 +-
 src/libstore/local-store.hh | 11 +++--------
 src/libstore/store-api.hh   |  7 +------
 3 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 8e90913cc3f..0b03d61a789 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -679,7 +679,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (unlink(path.c_str()) == -1)
             throw SysError(format("deleting ‘%1%’") % path);
 
-        state.results.bytesFreed += st.st_blocks * 512;
+        state.results.bytesFreed += st.st_blocks * 512ULL;
     }
 
     struct stat st;
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 28e9a31c9fe..750da0c142d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -26,14 +26,9 @@ struct Derivation;
 
 struct OptimiseStats
 {
-    unsigned long filesLinked;
-    unsigned long long bytesFreed;
-    unsigned long long blocksFreed;
-    OptimiseStats()
-    {
-        filesLinked = 0;
-        bytesFreed = blocksFreed = 0;
-    }
+    unsigned long filesLinked = 0;
+    unsigned long long bytesFreed = 0;
+    unsigned long long blocksFreed = 0;
 };
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 68c59a9f292..c0a52145af5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -81,12 +81,7 @@ struct GCResults
 
     /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
        number of bytes that would be or was freed. */
-    unsigned long long bytesFreed;
-
-    GCResults()
-    {
-        bytesFreed = 0;
-    }
+    unsigned long long bytesFreed = 0;
 };
 
 

From 53edb55588e362ee6aa8f6b436c6db6b0409b615 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 10 Apr 2017 11:50:01 +0200
Subject: [PATCH 0223/2196] shell.nix: Remove obsolete flags

---
 shell.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/shell.nix b/shell.nix
index df0ad01df58..17cfbfbe6f9 100644
--- a/shell.nix
+++ b/shell.nix
@@ -22,8 +22,6 @@ with import <nixpkgs> {};
   configureFlags =
     [ "--disable-init-state"
       "--enable-gc"
-      "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}"
-      "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}"
     ];
 
   enableParallelBuilding = true;

From e43e8be8e76b4457c1dfe7578870b2c995dc6a43 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Mon, 10 Apr 2017 09:22:24 -0400
Subject: [PATCH 0224/2196] Default to 5 download retries

This should help certain downloaders that don't request anything special
for the number of retries, like nix-channel.
---
 src/libstore/download.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index e2e16b36103..62f3860b9da 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -15,7 +15,7 @@ struct DownloadRequest
     bool verifyTLS = true;
     enum { yes, no, automatic } showProgress = yes;
     bool head = false;
-    size_t tries = 1;
+    size_t tries = 5;
     unsigned int baseRetryTimeMs = 250;
 
     DownloadRequest(const std::string & uri) : uri(uri) { }

From d1fdade75562d711ec62e9cdf953f2a6b0e891ce Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Mon, 10 Apr 2017 09:28:44 -0400
Subject: [PATCH 0225/2196] Add CURLE_WRITE_ERROR as a transient error
 condition

We've observed it failing downloads in the wild and retrying the same URL
a few moments later seemed to fix it.
---
 src/libstore/download.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d1f760fdc30..f8f57869503 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -300,6 +300,11 @@ struct CurlDownloader : public Downloader
                         || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
                         || code == CURLE_COULDNT_RESOLVE_HOST
                         || code == CURLE_RECV_ERROR
+
+                        // this seems to occur occasionally for retriable reasons, and shows up in an error like this:
+                        //   curl: (23) Failed writing body (315 != 16366)
+                        || code == CURLE_WRITE_ERROR
+
                         // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable
                         || code == CURLE_SSL_CONNECT_ERROR
 #if LIBCURL_VERSION_NUM >= 0x073200

From 915f62fa19790d8f826aeb4dd3d2bb5bde2f67e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 10 Apr 2017 17:22:52 +0200
Subject: [PATCH 0226/2196] shell.nix: Remove more dependencies

Thanks @copumpkin.
---
 shell.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/shell.nix b/shell.nix
index 17cfbfbe6f9..425eb0a191f 100644
--- a/shell.nix
+++ b/shell.nix
@@ -6,7 +6,7 @@ with import <nixpkgs> {};
   name = "nix";
 
   buildInputs =
-    [ curl bison flex perl libxml2 libxslt
+    [ curl bison flex libxml2 libxslt
       bzip2 xz brotli
       pkgconfig sqlite libsodium boehmgc
       docbook5 docbook5_xsl
@@ -16,7 +16,6 @@ with import <nixpkgs> {};
         customMemoryManagement = false;
       })
       autoreconfHook
-      perlPackages.DBDSQLite
     ];
 
   configureFlags =

From 503cc4431b8fa008caca7d06026dcfcab8626884 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 10 Apr 2017 18:16:36 -0400
Subject: [PATCH 0227/2196] nix-channel: error out if direct tarball unpack
 fails.

It's very unlikely a path ending in .tar.gz is a directory

Fixes #1318
---
 src/nix-channel/nix-channel.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 36162782312..0f50f6242c4 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -103,12 +103,9 @@ static void update(const StringSet & channelNames)
 
         auto unpacked = false;
         if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
-            try {
-                runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
-                            "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
-                unpacked = true;
-            } catch (ExecError & e) {
-            }
+            runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
+                        "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
+            unpacked = true;
         }
 
         if (!unpacked) {

From b134c2d05280be9615c81f018a5309fc8dcbca0e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 11 Apr 2017 15:41:21 +0200
Subject: [PATCH 0228/2196] Drop WWW::Curl dependency

Somehow this came back after d1da6967b8891763ce04d668027cf300c9bbf0b2.
---
 perl/configure.ac | 13 +------------
 release.nix       |  1 -
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/perl/configure.ac b/perl/configure.ac
index d617c78535f..7a6b28be23e 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -52,7 +52,7 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl).
+# Check for the required Perl dependencies (DBI and DBD::SQLite).
 perlFlags="-I$perllibdir"
 
 AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
@@ -63,10 +63,6 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
   [prefix of the Perl DBD::SQLite library]),
   perlFlags="$perlFlags -I$withval")
 
-AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH],
-  [prefix of the Perl WWW::Curl library]),
-  perlFlags="$perlFlags -I$withval")
-
 AC_MSG_CHECKING([whether DBD::SQLite works])
 if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
     AC_MSG_RESULT(no)
@@ -74,13 +70,6 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
 fi
 AC_MSG_RESULT(yes)
 
-AC_MSG_CHECKING([whether WWW::Curl works])
-if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then
-    AC_MSG_RESULT(no)
-    AC_MSG_FAILURE([The Perl module WWW::Curl is missing.])
-fi
-AC_MSG_RESULT(yes)
-
 AC_SUBST(perlFlags)
 
 PKG_CHECK_MODULES([NIX], [nix-store])
diff --git a/release.nix b/release.nix
index 8727c2520b1..6136f650d95 100644
--- a/release.nix
+++ b/release.nix
@@ -116,7 +116,6 @@ let
         configureFlags = ''
           --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
-          --with-www-curl=${perlPackages.WWWCurl}/${pkgs.perl.libPrefix}
         '';
 
         enableParallelBuilding = true;

From d267db0d758c7a926168cacdbaa7a9800564af11 Mon Sep 17 00:00:00 2001
From: Benjamin Staffin <benley@gmail.com>
Date: Wed, 12 Apr 2017 18:01:43 -0400
Subject: [PATCH 0229/2196] Move note about float support out of the wrong
 release notes

Looks like this snuck into the 1.11 release notes post-release, but
float support isn't actually present until 1.12.
---
 doc/manual/release-notes/rl-1.11.xml | 7 -------
 doc/manual/release-notes/rl-1.12.xml | 7 +++++++
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml
index efb03d61393..fe422dd1f89 100644
--- a/doc/manual/release-notes/rl-1.11.xml
+++ b/doc/manual/release-notes/rl-1.11.xml
@@ -121,13 +121,6 @@ $ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zl
     also improves performance.</para>
   </listitem>
 
-  <listitem>
-    <para>The Nix language now supports floating point numbers. They are
-    based on regular C++ <literal>float</literal> and compatible with
-    existing integers and number-related operations. Export and import to and
-    from JSON and XML works, too.
-  </para>
-  </listitem>
   <listitem>
     <para>All "chroot"-containing strings got renamed to "sandbox".
       In particular, some Nix options got renamed, but the old names
diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index d6864b3f55d..b7f45fc44a3 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -17,6 +17,13 @@
     have write access to the Nix database.</para>
   </listitem>
 
+  <listitem>
+    <para>The Nix language now supports floating point numbers. They are
+    based on regular C++ <literal>float</literal> and compatible with
+    existing integers and number-related operations. Export and import to and
+    from JSON and XML works, too.
+  </para>
+  </listitem>
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>

From 31cc9366fc81c3f478b31a206daa6b38801cd3b1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 12 Apr 2017 14:52:49 +0200
Subject: [PATCH 0230/2196] Initialise logger

---
 perl/lib/Nix/Store.xs  | 1 -
 src/libmain/shared.cc  | 2 --
 src/libutil/logging.cc | 2 +-
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index f613e3df329..46b41f92392 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -25,7 +25,6 @@ static ref<Store> store()
     static std::shared_ptr<Store> _store;
     if (!_store) {
         try {
-            logger = makeDefaultLogger();
             settings.loadConfFile();
             settings.update();
             settings.lockCPU = false;
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index a720afd6cdd..c1828aa7db8 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -106,8 +106,6 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
-    logger = makeDefaultLogger();
-
     /* Initialise OpenSSL locking. */
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index d9e8d22d768..53f6260b788 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -3,7 +3,7 @@
 
 namespace nix {
 
-Logger * logger = 0;
+Logger * logger = makeDefaultLogger();
 
 class SimpleLogger : public Logger
 {

From 6d97d816565505606792050131b5d4d7fca33245 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 12 Apr 2017 14:53:10 +0200
Subject: [PATCH 0231/2196] Add warn function

---
 src/libutil/logging.cc |  7 ++++++-
 src/libutil/logging.hh | 10 ++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 53f6260b788..afcc2ec5854 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -5,6 +5,11 @@ namespace nix {
 
 Logger * logger = makeDefaultLogger();
 
+void Logger::warn(const std::string & msg)
+{
+    log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
+}
+
 class SimpleLogger : public Logger
 {
 public:
@@ -52,7 +57,7 @@ Verbosity verbosity = lvlInfo;
 void warnOnce(bool & haveWarned, const FormatOrString & fs)
 {
     if (!haveWarned) {
-        printError(format("warning: %1%") % fs.s);
+        warn(fs.s);
         haveWarned = true;
     }
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 3f83664794f..81aebccdca4 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -30,6 +30,8 @@ public:
         log(lvlInfo, fs);
     }
 
+    virtual void warn(const std::string & msg);
+
     virtual void setExpected(const std::string & label, uint64_t value = 1) { }
     virtual void setProgress(const std::string & label, uint64_t value = 1) { }
     virtual void incExpected(const std::string & label, uint64_t value = 1) { }
@@ -82,6 +84,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
 #define debug(args...) printMsg(lvlDebug, args)
 #define vomit(args...) printMsg(lvlVomit, args)
 
+template<typename... Args>
+inline void warn(const std::string & fs, Args... args)
+{
+    boost::format f(fs);
+    formatHelper(f, args...);
+    logger->warn(f.str());
+}
+
 void warnOnce(bool & haveWarned, const FormatOrString & fs);
 
 void writeToStderr(const string & s);

From 568a099c889e7ccc5a49b15575078e99acf8bc2f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 15:32:43 +0200
Subject: [PATCH 0232/2196] canonPath(): Check against empty paths

---
 src/libutil/util.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a640a64c724..0bd51afd1a9 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -96,6 +96,8 @@ Path absPath(Path path, Path dir)
 
 Path canonPath(const Path & path, bool resolveSymlinks)
 {
+    assert(path != "");
+
     string s;
 
     if (path[0] != '/')

From 2040240e238a41c2eb799bf4dbf394fec297ac16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 15:55:38 +0200
Subject: [PATCH 0233/2196] Add a Config class to simplify adding configuration
 settings

The typical use is to inherit Config and add Setting<T> members:

  class MyClass : private Config
  {
    Setting<int> foo{this, 123, "foo", "the number of foos to use"};
    Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"};

    MyClass() : Config(readConfigFile("/etc/my-app.conf"))
    {
      std::cout << foo << "\n"; // will print 123 unless overriden
    }
  };

Currently, this is used by Store and its subclasses for store
parameters. You now get a warning if you specify a non-existant store
parameter in a store URI.
---
 src/libstore/binary-cache-store.cc    |   3 -
 src/libstore/binary-cache-store.hh    |  10 +-
 src/libstore/build.cc                 |   8 +-
 src/libstore/legacy-ssh-store.cc      |  10 +-
 src/libstore/local-fs-store.cc        |   9 +-
 src/libstore/local-store.cc           |   5 +-
 src/libstore/local-store.hh           |   6 +-
 src/libstore/remote-store.cc          |   2 +-
 src/libstore/remote-store.hh          |   3 +
 src/libstore/s3-binary-cache-store.cc |  12 +-
 src/libstore/ssh-store.cc             |   7 +-
 src/libstore/store-api.cc             |   9 +-
 src/libstore/store-api.hh             |  25 ++++-
 src/libutil/config.cc                 | 112 +++++++++++++++++++
 src/libutil/config.hh                 | 151 ++++++++++++++++++++++++++
 src/libutil/types.hh                  |   2 +
 16 files changed, 334 insertions(+), 40 deletions(-)
 create mode 100644 src/libutil/config.cc
 create mode 100644 src/libutil/config.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 25ad0d75b70..b536c6c0004 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -79,10 +79,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor
 
 BinaryCacheStore::BinaryCacheStore(const Params & params)
     : Store(params)
-    , compression(get(params, "compression", "xz"))
-    , writeNARListing(get(params, "write-nar-listing", "0") == "1")
 {
-    auto secretKeyFile = get(params, "secret-key", "");
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index d42b1abd245..5c2d0acfdbb 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -13,13 +13,15 @@ struct NarInfo;
 
 class BinaryCacheStore : public Store
 {
-private:
+public:
 
-    std::unique_ptr<SecretKey> secretKey;
+    const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
 
-    std::string compression;
+private:
 
-    bool writeNARListing;
+    std::unique_ptr<SecretKey> secretKey;
 
 protected:
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 968e291129f..d9c299d099b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3051,13 +3051,11 @@ Path DerivationGoal::openLogFile()
     string baseName = baseNameOf(drvPath);
 
     /* Create a log file. */
-    Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str();
+    Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
     createDirs(dir);
 
-    Path logFileName = (format("%1%/%2%%3%")
-        % dir
-        % string(baseName, 2)
-        % (settings.compressLog ? ".bz2" : "")).str();
+    Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
+        settings.compressLog ? ".bz2" : "");
 
     fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
     if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 0e838846c79..befc560bfce 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -12,6 +12,10 @@ static std::string uriScheme = "ssh://";
 
 struct LegacySSHStore : public Store
 {
+    const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
+    const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
+
     struct Connection
     {
         std::unique_ptr<SSHMaster::Connection> sshConn;
@@ -29,16 +33,16 @@ struct LegacySSHStore : public Store
         : Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
-            std::max(1, std::stoi(get(params, "max-connections", "1"))),
+            std::max(1, (int) maxConnections),
             [this]() { return openConnection(); },
             [](const ref<Connection> & r) { return true; }
             ))
         , master(
             host,
-            get(params, "ssh-key", ""),
+            sshKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
-            get(params, "compress", "") == "true")
+            compress)
     {
     }
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 57e1b8a09fe..bf247903c9d 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -9,9 +9,6 @@ namespace nix {
 
 LocalFSStore::LocalFSStore(const Params & params)
     : Store(params)
-    , rootDir(get(params, "root"))
-    , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir)))
-    , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir)))
 {
 }
 
@@ -88,6 +85,8 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink)
 
 const string LocalFSStore::drvsLogDir = "drvs";
 
+
+
 std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
 {
     auto path(path_);
@@ -110,8 +109,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
 
         Path logPath =
             j == 0
-            ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
-            : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str();
+            ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
+            : fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
         Path logBz2Path = logPath + ".bz2";
 
         if (pathExists(logPath))
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 8610841d722..0ea897526bb 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -38,13 +38,14 @@ namespace nix {
 LocalStore::LocalStore(const Params & params)
     : Store(params)
     , LocalFSStore(params)
-    , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir))
+    , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
+        "physical path to the Nix store"}
+    , realStoreDir(realStoreDir_)
     , dbDir(stateDir + "/db")
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
     , trashDir(realStoreDir + "/trash")
-    , requireSigs(trim(settings.get("signed-binary-caches", std::string("*"))) != "") // FIXME: rename option
     , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 750da0c142d..fec67ee7d9e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -67,6 +67,8 @@ private:
 
 public:
 
+    PathSetting realStoreDir_;
+
     const Path realStoreDir;
     const Path dbDir;
     const Path linksDir;
@@ -76,7 +78,9 @@ public:
 
 private:
 
-    bool requireSigs;
+    Setting<bool> requireSigs{(Store*) this,
+        trim(settings.get("signed-binary-caches", std::string("*"))) != "",
+        "require-sigs", "whether store paths should have a trusted signature on import"};
 
     PublicKeys publicKeys;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index c9c59078745..e1df137e4db 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -43,7 +43,7 @@ template Paths readStorePaths(Store & store, Source & from);
 RemoteStore::RemoteStore(const Params & params)
     : Store(params)
     , connections(make_ref<Pool<Connection>>(
-            std::max(1, std::stoi(get(params, "max-connections", "1"))),
+            std::max(1, (int) maxConnections),
             [this]() { return openConnectionWrapper(); },
             [](const ref<Connection> & r) { return r->to.good() && r->from.good(); }
             ))
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index db8da7eaa8c..479cf3a7909 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -22,6 +22,9 @@ class RemoteStore : public virtual Store
 {
 public:
 
+    const Setting<int> maxConnections{(Store*) this, 1,
+            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+
     RemoteStore(const Params & params);
 
     /* Implementations of abstract store API methods. */
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 3053f908c4e..24545529601 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -125,22 +125,22 @@ S3Helper::DownloadResult S3Helper::getObject(
 
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
+    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
+    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
+    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+
     std::string bucketName;
 
     Stats stats;
 
     S3Helper s3Helper;
 
-    std::string narinfoCompression, lsCompression, logCompression;
-
     S3BinaryCacheStoreImpl(
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
-        , narinfoCompression(get(params, "narinfo-compression", ""))
-        , lsCompression(get(params, "ls-compression", ""))
-        , logCompression(get(params, "log-compression", ""))
+        , s3Helper(region)
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 2a81a8b1ebe..bb536fadfd5 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -14,16 +14,19 @@ class SSHStore : public RemoteStore
 {
 public:
 
+    const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
+
     SSHStore(const std::string & host, const Params & params)
         : Store(params)
         , RemoteStore(params)
         , host(host)
         , master(
             host,
-            get(params, "ssh-key", ""),
+            sshKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
-            get(params, "compress", "") == "true")
+            compress)
     {
     }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 53c802044ea..cb62bdc0b62 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -241,8 +241,8 @@ Path Store::computeStorePathForText(const string & name, const string & s,
 
 
 Store::Store(const Params & params)
-    : storeDir(get(params, "store", settings.nixStore))
-    , state({std::stoi(get(params, "path-info-cache-size", "65536"))})
+    : Config(params)
+    , state({(size_t) pathInfoCacheSize})
 {
 }
 
@@ -718,7 +718,10 @@ ref<Store> openStore(const std::string & uri, const Store::Params & params)
 {
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
-        if (store) return ref<Store>(store);
+        if (store) {
+            store->warnUnused();
+            return ref<Store>(store);
+        }
     }
 
     throw Error(format("don't know how to open Nix store ‘%s’") % uri);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c0a52145af5..067309c9e95 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -6,6 +6,7 @@
 #include "lru-cache.hh"
 #include "sync.hh"
 #include "globals.hh"
+#include "config.hh"
 
 #include <atomic>
 #include <limits>
@@ -224,13 +225,17 @@ struct BuildResult
 };
 
 
-class Store : public std::enable_shared_from_this<Store>
+class Store : public std::enable_shared_from_this<Store>, public Config
 {
 public:
 
     typedef std::map<std::string, std::string> Params;
 
-    const Path storeDir;
+    const PathSetting storeDir_{this, false, settings.nixStore,
+        "store", "path to the Nix store"};
+    const Path storeDir = storeDir_;
+
+    const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
 
 protected:
 
@@ -585,9 +590,19 @@ protected:
 class LocalFSStore : public virtual Store
 {
 public:
-    const Path rootDir;
-    const Path stateDir;
-    const Path logDir;
+
+    // FIXME: the (Store*) cast works around a bug in gcc that causes
+    // it to emit the call to the Option constructor. Clang works fine
+    // either way.
+    const PathSetting rootDir{(Store*) this, true, "",
+        "root", "directory prefixed to all other paths"};
+    const PathSetting stateDir{(Store*) this, false,
+        rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
+        "state", "directory where Nix will store state"};
+    const PathSetting logDir{(Store*) this, false,
+        rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
+        "log", "directory where Nix will store state"};
+
     const static string drvsLogDir;
 
     LocalFSStore(const Params & params);
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
new file mode 100644
index 00000000000..2f9f988607e
--- /dev/null
+++ b/src/libutil/config.cc
@@ -0,0 +1,112 @@
+#include "config.hh"
+#include "args.hh"
+
+namespace nix {
+
+void Config::set(const std::string & name, const std::string & value)
+{
+    auto i = _settings.find(name);
+    if (i == _settings.end())
+        throw UsageError("unknown setting '%s'", name);
+    i->second.setting->set(value);
+}
+
+void Config::add(AbstractSetting * setting)
+{
+    _settings.emplace(setting->name, Config::SettingData{false, setting});
+    for (auto & alias : setting->aliases)
+        _settings.emplace(alias, Config::SettingData{true, setting});
+
+    bool set = false;
+
+    auto i = initials.find(setting->name);
+    if (i != initials.end()) {
+        setting->set(i->second);
+        initials.erase(i);
+        set = true;
+    }
+
+    for (auto & alias : setting->aliases) {
+        auto i = initials.find(alias);
+        if (i != initials.end()) {
+            if (set)
+                warn("setting '%s' is set, but it's an alias of '%s' which is also set",
+                    alias, setting->name);
+            else {
+                setting->set(i->second);
+                initials.erase(i);
+                set = true;
+            }
+        }
+    }
+}
+
+void Config::warnUnused()
+{
+    for (auto & i : initials)
+        warn("unknown setting '%s'", i.first);
+}
+
+std::string Config::dump()
+{
+    std::string res;
+    for (auto & opt : _settings)
+        if (!opt.second.isAlias)
+            res += opt.first + " = " + opt.second.setting->to_string() + "\n";
+    return res;
+}
+
+AbstractSetting::AbstractSetting(
+    const std::string & name,
+    const std::string & description,
+    const std::set<std::string> & aliases)
+    : name(name), description(description), aliases(aliases)
+{
+}
+
+template<> void Setting<std::string>::set(const std::string & str)
+{
+    value = str;
+}
+
+template<> std::string Setting<std::string>::to_string()
+{
+    return value;
+}
+
+template<> void Setting<int>::set(const std::string & str)
+{
+    try {
+        value = std::stoi(str);
+    } catch (...) {
+        throw UsageError("setting '%s' has invalid value '%s'", name, str);
+    }
+}
+
+template<> std::string Setting<int>::to_string()
+{
+    return std::to_string(value);
+}
+
+template<> void Setting<bool>::set(const std::string & str)
+{
+    value = str == "true" || str == "1";
+}
+
+template<> std::string Setting<bool>::to_string()
+{
+    return value ? "true" : "false";
+}
+
+void PathSetting::set(const std::string & str)
+{
+    if (str == "") {
+        if (allowEmpty)
+            value = "";
+        else
+            throw UsageError("setting '%s' cannot be empty", name);
+    } else
+        value = canonPath(str);
+}
+
+}
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
new file mode 100644
index 00000000000..fb2d48e9c83
--- /dev/null
+++ b/src/libutil/config.hh
@@ -0,0 +1,151 @@
+#include <map>
+#include <set>
+
+#include "types.hh"
+
+#pragma once
+
+namespace nix {
+
+class Args;
+class AbstractSetting;
+
+/* A class to simplify providing configuration settings. The typical
+   use is to inherit Config and add Setting<T> members:
+
+   class MyClass : private Config
+   {
+     Setting<int> foo{this, 123, "foo", "the number of foos to use"};
+     Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"};
+
+     MyClass() : Config(readConfigFile("/etc/my-app.conf"))
+     {
+       std::cout << foo << "\n"; // will print 123 unless overriden
+     }
+   };
+*/
+
+class Config
+{
+    friend class AbstractSetting;
+
+    struct SettingData
+    {
+        bool isAlias = false;
+        AbstractSetting * setting;
+    };
+
+    std::map<std::string, SettingData> _settings;
+
+    StringMap initials;
+
+public:
+
+    Config(const StringMap & initials)
+        : initials(initials)
+    { }
+
+    void set(const std::string & name, const std::string & value);
+
+    void add(AbstractSetting * setting);
+
+    void warnUnused();
+
+    std::string dump();
+};
+
+class AbstractSetting
+{
+    friend class Config;
+
+public:
+
+    const std::string name;
+    const std::string description;
+    const std::set<std::string> aliases;
+
+    int created = 123;
+
+protected:
+
+    AbstractSetting(
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases);
+
+    virtual ~AbstractSetting()
+    {
+        // Check against a gcc miscompilation causing our constructor
+        // not to run.
+        assert(created == 123);
+    }
+
+    virtual void set(const std::string & value) = 0;
+
+    virtual std::string to_string() = 0;
+};
+
+/* A setting of type T. */
+template<typename T>
+class Setting : public AbstractSetting
+{
+protected:
+
+    T value;
+
+public:
+
+    Setting(Config * options,
+        const T & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : AbstractSetting(name, description, aliases)
+        , value(def)
+    {
+        options->add(this);
+    }
+
+    operator const T &() const { return value; }
+    bool operator ==(const T & v2) const { return value == v2; }
+    bool operator !=(const T & v2) const { return value != v2; }
+    void operator =(const T & v) { value = v; }
+
+    void set(const std::string & str) override;
+
+    std::string to_string() override;
+};
+
+template<typename T>
+std::ostream & operator <<(std::ostream & str, const Setting<T> & opt)
+{
+    str << (const T &) opt;
+    return str;
+}
+
+/* A special setting for Paths. These are automatically canonicalised
+   (e.g. "/foo//bar/" becomes "/foo/bar"). */
+class PathSetting : public Setting<Path>
+{
+    bool allowEmpty;
+
+public:
+
+    PathSetting(Config * options,
+        bool allowEmpty,
+        const Path & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : Setting<Path>(options, def, name, description, aliases)
+        , allowEmpty(allowEmpty)
+    {
+        set(value);
+    }
+
+    void set(const std::string & str) override;
+
+    Path operator +(const char * p) const { return value + p; }
+};
+
+}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 97d79af9b5d..1429c238513 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -7,6 +7,7 @@
 #include <list>
 #include <set>
 #include <memory>
+#include <map>
 
 #include <boost/format.hpp>
 
@@ -141,6 +142,7 @@ private:
 
 typedef list<string> Strings;
 typedef set<string> StringSet;
+typedef std::map<std::string, std::string> StringMap;
 
 
 /* Paths are just strings. */

From 0bf34de43b2fc4c9c3104b986eaea5c5cc856b83 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 16:31:28 +0200
Subject: [PATCH 0234/2196] Validate Boolean settings better

---
 src/libutil/config.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 2f9f988607e..893cdccce34 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -90,7 +90,12 @@ template<> std::string Setting<int>::to_string()
 
 template<> void Setting<bool>::set(const std::string & str)
 {
-    value = str == "true" || str == "1";
+    if (str == "true" || str == "yes" || str == "1")
+        value = true;
+    else if (str == "false" || str == "no" || str == "0")
+        value = false;
+    else
+        throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
 }
 
 template<> std::string Setting<bool>::to_string()

From 6bd9576aeb55927cb551736a47b4e8e3fd1063bb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 17:54:05 +0200
Subject: [PATCH 0235/2196] Support arbitrary numeric types for settings

---
 src/libutil/config.cc | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 893cdccce34..c05a3253bce 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -74,17 +74,25 @@ template<> std::string Setting<std::string>::to_string()
     return value;
 }
 
-template<> void Setting<int>::set(const std::string & str)
+template<typename T>
+void Setting<T>::set(const std::string & str)
 {
+    static_assert(std::is_integral<T>::value, "Integer required.");
     try {
-        value = std::stoi(str);
-    } catch (...) {
+        auto i = std::stoll(str);
+        if (i < std::numeric_limits<T>::min() ||
+            i > std::numeric_limits<T>::max())
+            throw UsageError("setting '%s' has out-of-range value %d", name, i);
+        value = i;
+    } catch (std::logic_error&) {
         throw UsageError("setting '%s' has invalid value '%s'", name, str);
     }
 }
 
-template<> std::string Setting<int>::to_string()
+template<typename T>
+std::string Setting<T>::to_string()
 {
+    static_assert(std::is_integral<T>::value, "Integer required.");
     return std::to_string(value);
 }
 
@@ -103,6 +111,11 @@ template<> std::string Setting<bool>::to_string()
     return value ? "true" : "false";
 }
 
+template class Setting<int>;
+template class Setting<unsigned int>;
+template class Setting<long>;
+template class Setting<size_t>;
+
 void PathSetting::set(const std::string & str)
 {
     if (str == "") {

From ba9ad29fdbfda3836bb06b35817f08fd10beaa22 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 20:53:23 +0200
Subject: [PATCH 0236/2196] Convert Settings to the new config system

This makes all config options self-documenting.

Unknown or unparseable config settings and --option flags now cause a
warning.
---
 doc/manual/command-ref/conf-file.xml |  10 -
 perl/lib/Nix/Config.pm.in            |  23 +-
 src/libmain/common-args.cc           |   6 +-
 src/libmain/shared.cc                |  10 +-
 src/libstore/build.cc                |  11 +-
 src/libstore/crypto.cc               |   4 +-
 src/libstore/download.cc             |   2 +-
 src/libstore/globals.cc              | 317 ++++-----------------------
 src/libstore/globals.hh              | 312 +++++++++++++-------------
 src/libstore/local-store.cc          |   4 +-
 src/libstore/remote-store.cc         |   4 +-
 src/libstore/store-api.cc            |   9 +-
 src/libutil/config.cc                | 104 +++++++--
 src/libutil/config.hh                |  22 +-
 src/nix-daemon/nix-daemon.cc         |  13 +-
 src/nix-store/nix-store.cc           |   8 +-
 src/nix/main.cc                      |   1 -
 tests/shell.shebang.sh               |   2 +-
 18 files changed, 332 insertions(+), 530 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 6952829e8f7..3de9647aa4e 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -334,16 +334,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>use-binary-caches</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    Nix will check the binary caches specified by
-    <option>binary-caches</option> and related options to obtain
-    binary substitutes.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry><term><literal>binary-caches</literal></term>
 
     <listitem><para>A list of URLs of binary caches, separated by
diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index f494e34a5e7..4bdee7fd89f 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -20,22 +20,15 @@ $useBindings = 1;
 %config = ();
 
 sub readConfig {
-    if (defined $ENV{'_NIX_OPTIONS'}) {
-        foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
-            my ($n, $v) = split '=', $s, 2;
-            $config{$n} = $v;
-        }
-    } else {
-        my $config = "$confDir/nix.conf";
-        return unless -f $config;
-
-        open CONFIG, "<$config" or die "cannot open ‘$config’";
-        while (<CONFIG>) {
-            /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next;
-            $config{$1} = $2;
-        }
-        close CONFIG;
+    my $config = "$confDir/nix.conf";
+    return unless -f $config;
+
+    open CONFIG, "<$config" or die "cannot open ‘$config’";
+    while (<CONFIG>) {
+        /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next;
+        $config{$1} = $2;
     }
+    close CONFIG;
 }
 
 return 1;
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 98693d78a7f..9a7a893138d 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -22,7 +22,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         [](Strings ss) {
             auto name = ss.front(); ss.pop_front();
             auto value = ss.front();
-            settings.set(name, value);
+            try {
+                settings.set(name, value);
+            } catch (UsageError & e) {
+                warn(e.what());
+            }
         });
 }
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index c1828aa7db8..9d506d01923 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -138,9 +138,6 @@ void initNix()
     struct timeval tv;
     gettimeofday(&tv, 0);
     srandom(tv.tv_usec);
-
-    if (char *pack = getenv("_NIX_OPTIONS"))
-        settings.unpack(pack);
 }
 
 
@@ -156,10 +153,10 @@ struct LegacyArgs : public MixCommonArgs
             &settings.verboseBuild, false);
 
         mkFlag('K', "keep-failed", "keep temporary directories of failed builds",
-            &settings.keepFailed);
+            &(bool&) settings.keepFailed);
 
         mkFlag('k', "keep-going", "keep going after a build fails",
-            &settings.keepGoing);
+            &(bool&) settings.keepGoing);
 
         mkFlag(0, "fallback", "build from source if substitution fails", []() {
             settings.set("build-fallback", "true");
@@ -184,7 +181,7 @@ struct LegacyArgs : public MixCommonArgs
             &settings.readOnlyMode);
 
         mkFlag(0, "no-build-hook", "disable use of the build hook mechanism",
-            &settings.useBuildHook, false);
+            &(bool&) settings.useBuildHook, false);
 
         mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
             &settings.showTrace);
@@ -218,7 +215,6 @@ void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
 {
     LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv));
-    settings.update();
 }
 
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b23447fa073..33c9e37047f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -462,7 +462,7 @@ UserLock::UserLock()
     assert(settings.buildUsersGroup != "");
 
     /* Get the members of the build-users-group. */
-    struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
+    struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
     if (!gr)
         throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist")
             % settings.buildUsersGroup);
@@ -1690,10 +1690,7 @@ void DerivationGoal::startBuilder()
 
     /* Are we doing a chroot build? */
     {
-        string x = settings.useSandbox;
-        if (x != "true" && x != "false" && x != "relaxed")
-            throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’");
-        if (x == "true") {
+        if (settings.sandboxMode == smEnabled) {
             if (get(drv->env, "__noChroot") == "1")
                 throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, "
                     "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath);
@@ -1704,9 +1701,9 @@ void DerivationGoal::startBuilder()
 #endif
             useChroot = true;
         }
-        else if (x == "false")
+        else if (settings.sandboxMode == smDisabled)
             useChroot = false;
-        else if (x == "relaxed")
+        else if (settings.sandboxMode == smRelaxed)
             useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
     }
 
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 9692dd83b4e..f56a6adab9c 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -105,12 +105,12 @@ PublicKeys getDefaultPublicKeys()
 
     // FIXME: filter duplicates
 
-    for (auto s : settings.binaryCachePublicKeys) {
+    for (auto s : settings.binaryCachePublicKeys.get()) {
         PublicKey key(s);
         publicKeys.emplace(key.name, key);
     }
 
-    for (auto secretKeyFile : settings.secretKeyFiles) {
+    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
         try {
             SecretKey secretKey(readFile(secretKeyFile));
             publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 95e6f7bace0..d073e870b4e 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -251,7 +251,7 @@ struct CurlDownloader : public Downloader
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
-            curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.c_str());
+            curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
             result.data = std::make_shared<std::string>();
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index b9f4fada59f..bb61daa5164 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -1,6 +1,7 @@
 #include "globals.hh"
 #include "util.hh"
 #include "archive.hh"
+#include "args.hh"
 
 #include <algorithm>
 #include <map>
@@ -26,329 +27,89 @@ namespace nix {
 
 Settings settings;
 
-
 Settings::Settings()
+    : Config({})
+    , nixPrefix(NIX_PREFIX)
+    , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
+    , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
+    , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
+    , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)))
+    , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
+    , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
+    , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
+    , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
-    deprecatedOptions = StringSet({
-        "build-use-chroot", "build-chroot-dirs", "build-extra-chroot-dirs",
-        "this-option-never-existed-but-who-will-know"
-    });
-
-    nixPrefix = NIX_PREFIX;
-    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
-    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
-    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
-    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
-    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
-    nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
-    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
-    nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
-
-    // should be set with the other config options, but depends on nixLibexecDir
-#ifdef __APPLE__
-    preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies";
-#endif
-
-    keepFailed = false;
-    keepGoing = false;
-    tryFallback = false;
-    maxBuildJobs = 1;
-    buildCores = std::max(1U, std::thread::hardware_concurrency());
-    readOnlyMode = false;
-    thisSystem = SYSTEM;
-    maxSilentTime = 0;
-    buildTimeout = 0;
-    useBuildHook = true;
-    reservedSize = 8 * 1024 * 1024;
-    fsyncMetadata = true;
-    useSQLiteWAL = true;
-    syncBeforeRegistering = false;
-    useSubstitutes = true;
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
-    useSshSubstituter = true;
-    impersonateLinux26 = false;
-    keepLog = true;
-    compressLog = true;
-    maxLogSize = 0;
-    pollInterval = 5;
-    checkRootReachability = false;
-    gcKeepOutputs = false;
-    gcKeepDerivations = true;
-    autoOptimiseStore = false;
-    envKeepDerivations = false;
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
-    showTrace = false;
-    enableNativeCode = false;
-    netrcFile = fmt("%s/%s", nixConfDir, "netrc");
     caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
-    enableImportFromDerivation = true;
-    useSandbox = "false"; // TODO: make into an enum
 
 #if __linux__
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH);
 #endif
 
-    restrictEval = false;
-    buildRepeat = 0;
     allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
-    sandboxShmSize = "50%";
-    darwinLogSandboxViolations = false;
-    runDiffHook = false;
-    diffHook = "";
-    enforceDeterminism = true;
-    binaryCachePublicKeys = Strings{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="};
-    secretKeyFiles = Strings();
-    binaryCachesParallelConnections = 25;
-    enableHttp2 = true;
-    tarballTtl = 60 * 60;
-    signedBinaryCaches = "";
-    substituters = Strings();
-    binaryCaches = nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings();
-    extraBinaryCaches = Strings();
-    trustedUsers = Strings({"root"});
-    allowedUsers = Strings({"*"});
-    printMissing = true;
 }
 
-
 void Settings::loadConfFile()
 {
-    Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
-    if (!pathExists(settingsFile)) return;
-    string contents = readFile(settingsFile);
-
-    unsigned int pos = 0;
-
-    while (pos < contents.size()) {
-        string line;
-        while (pos < contents.size() && contents[pos] != '\n')
-            line += contents[pos++];
-        pos++;
-
-        string::size_type hash = line.find('#');
-        if (hash != string::npos)
-            line = string(line, 0, hash);
-
-        vector<string> tokens = tokenizeString<vector<string> >(line);
-        if (tokens.empty()) continue;
-
-        if (tokens.size() < 2 || tokens[1] != "=")
-            throw Error(format("illegal configuration line ‘%1%’ in ‘%2%’") % line % settingsFile);
-
-        string name = tokens[0];
-
-        vector<string>::iterator i = tokens.begin();
-        advance(i, 2);
-        settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
-    };
+    applyConfigFile(nixConfDir + "/nix.conf");
 }
 
-
 void Settings::set(const string & name, const string & value)
 {
-    settings[name] = value;
     overrides[name] = value;
+    Config::set(name, value);
 }
 
-void Settings::update()
-{
-    _get(tryFallback, "build-fallback");
-
-    std::string s = "1";
-    _get(s, "build-max-jobs");
-    if (s == "auto")
-        maxBuildJobs = std::max(1U, std::thread::hardware_concurrency());
-    else
-        if (!string2Int(s, maxBuildJobs))
-            throw Error("configuration setting ‘build-max-jobs’ should be ‘auto’ or an integer");
-
-    _get(buildCores, "build-cores");
-    _get(thisSystem, "system");
-    _get(maxSilentTime, "build-max-silent-time");
-    _get(buildTimeout, "build-timeout");
-    _get(reservedSize, "gc-reserved-space");
-    _get(fsyncMetadata, "fsync-metadata");
-    _get(useSQLiteWAL, "use-sqlite-wal");
-    _get(syncBeforeRegistering, "sync-before-registering");
-    _get(useSubstitutes, "build-use-substitutes");
-    _get(buildUsersGroup, "build-users-group");
-    _get(impersonateLinux26, "build-impersonate-linux-26");
-    _get(keepLog, "build-keep-log");
-    _get(compressLog, "build-compress-log");
-    _get(maxLogSize, "build-max-log-size");
-    _get(pollInterval, "build-poll-interval");
-    _get(checkRootReachability, "gc-check-reachability");
-    _get(gcKeepOutputs, "gc-keep-outputs");
-    _get(gcKeepDerivations, "gc-keep-derivations");
-    _get(autoOptimiseStore, "auto-optimise-store");
-    _get(envKeepDerivations, "env-keep-derivations");
-    _get(sshSubstituterHosts, "ssh-substituter-hosts");
-    _get(useSshSubstituter, "use-ssh-substituter");
-    _get(enableNativeCode, "allow-unsafe-native-code-during-evaluation");
-    _get(useCaseHack, "use-case-hack");
-    _get(preBuildHook, "pre-build-hook");
-    _get(keepGoing, "keep-going");
-    _get(keepFailed, "keep-failed");
-    _get(netrcFile, "netrc-file");
-    _get(enableImportFromDerivation, "allow-import-from-derivation");
-    _get(useSandbox, "build-use-sandbox", "build-use-chroot");
-    _get(sandboxPaths, "build-sandbox-paths", "build-chroot-dirs");
-    _get(extraSandboxPaths, "build-extra-sandbox-paths", "build-extra-chroot-dirs");
-    _get(restrictEval, "restrict-eval");
-    _get(buildRepeat, "build-repeat");
-    _get(allowedImpureHostPrefixes, "allowed-impure-host-deps");
-    _get(sandboxShmSize, "sandbox-dev-shm-size");
-    _get(darwinLogSandboxViolations, "darwin-log-sandbox-violations");
-    _get(runDiffHook, "run-diff-hook");
-    _get(diffHook, "diff-hook");
-    _get(enforceDeterminism, "enforce-determinism");
-    _get(binaryCachePublicKeys, "binary-cache-public-keys");
-    _get(secretKeyFiles, "secret-key-files");
-    _get(binaryCachesParallelConnections, "binary-caches-parallel-connections");
-    _get(enableHttp2, "enable-http2");
-    _get(tarballTtl, "tarball-ttl");
-    _get(signedBinaryCaches, "signed-binary-caches");
-    _get(substituters, "substituters");
-    _get(binaryCaches, "binary-caches");
-    _get(extraBinaryCaches, "extra-binary-caches");
-    _get(trustedUsers, "trusted-users");
-    _get(allowedUsers, "allowed-users");
-    _get(printMissing, "print-missing");
-
-    /* Clear out any deprecated options that might be left, so users know we recognize the option
-       but aren't processing it anymore */
-    for (auto &i : deprecatedOptions) {
-        if (settings.find(i) != settings.end()) {
-            printError(format("warning: deprecated option '%1%' is no longer supported and will be ignored") % i);
-            settings.erase(i);
-        }
-    }
-
-    if (settings.size() != 0) {
-        string bad;
-        for (auto &i : settings)
-            bad += "'" + i.first + "', ";
-        bad.pop_back();
-        bad.pop_back();
-        throw Error(format("unrecognized options: %s") % bad);
-    }
-}
-
-void Settings::checkDeprecated(const string & name)
+StringMap Settings::getOverrides()
 {
-    if (deprecatedOptions.find(name) != deprecatedOptions.end())
-        printError(format("warning: deprecated option '%1%' will soon be unsupported") % name);
-}
-
-void Settings::_get(string & res, const string & name)
-{
-    SettingsMap::iterator i = settings.find(name);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    res = i->second;
-}
-
-void Settings::_get(string & res, const string & name1, const string & name2)
-{
-    SettingsMap::iterator i = settings.find(name1);
-    if (i == settings.end()) i = settings.find(name2);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    res = i->second;
+    return overrides;
 }
 
-
-void Settings::_get(bool & res, const string & name)
+unsigned int Settings::getDefaultCores()
 {
-    SettingsMap::iterator i = settings.find(name);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    if (i->second == "true") res = true;
-    else if (i->second == "false") res = false;
-    else throw Error(format("configuration option ‘%1%’ should be either ‘true’ or ‘false’, not ‘%2%’")
-        % name % i->second);
+    return std::max(1U, std::thread::hardware_concurrency());
 }
 
+const string nixVersion = PACKAGE_VERSION;
 
-void Settings::_get(StringSet & res, const string & name)
+template<> void Setting<SandboxMode>::set(const std::string & str)
 {
-    SettingsMap::iterator i = settings.find(name);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    res.clear();
-    Strings ss = tokenizeString<Strings>(i->second);
-    res.insert(ss.begin(), ss.end());
+    if (str == "true") value = smEnabled;
+    else if (str == "relaxed") value = smRelaxed;
+    else if (str == "false") value = smDisabled;
+    else throw UsageError("option '%s' has invalid value '%s'", name, str);
 }
 
-void Settings::_get(StringSet & res, const string & name1, const string & name2)
+template<> std::string Setting<SandboxMode>::to_string()
 {
-    SettingsMap::iterator i = settings.find(name1);
-    if (i == settings.end()) i = settings.find(name2);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    res.clear();
-    Strings ss = tokenizeString<Strings>(i->second);
-    res.insert(ss.begin(), ss.end());
+    if (value == smEnabled) return "true";
+    else if (value == smRelaxed) return "relaxed";
+    else if (value == smDisabled) return "false";
+    else abort();
 }
 
-void Settings::_get(Strings & res, const string & name)
+template<> void Setting<unsigned int, Settings::MaxBuildJobsTag>::set(const std::string & str)
 {
-    SettingsMap::iterator i = settings.find(name);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    res = tokenizeString<Strings>(i->second);
+    if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
+    else if (!string2Int(str, value))
+        throw UsageError("configuration setting ‘%s’ should be ‘auto’ or an integer", name);
 }
 
-
-template<class N> void Settings::_get(N & res, const string & name)
+template<> std::string Setting<unsigned int, Settings::MaxBuildJobsTag>::to_string()
 {
-    SettingsMap::iterator i = settings.find(name);
-    if (i == settings.end()) return;
-    checkDeprecated(i->first);
-    settings.erase(i);
-    if (!string2Int(i->second, res))
-        throw Error(format("configuration setting ‘%1%’ should have an integer value") % name);
+    return std::to_string(value);
 }
 
-
-string Settings::pack()
+template<> void Setting<bool, Settings::CaseHackTag>::set(const std::string & str)
 {
-    string s;
-    for (auto & i : settings) {
-        if (i.first.find('\n') != string::npos ||
-            i.first.find('=') != string::npos ||
-            i.second.find('\n') != string::npos)
-            throw Error("illegal option name/value");
-        s += i.first; s += '='; s += i.second; s += '\n';
-    }
-    return s;
+    value = parseBool(str);
+    nix::useCaseHack = true;
 }
 
-
-void Settings::unpack(const string & pack) {
-    Strings lines = tokenizeString<Strings>(pack, "\n");
-    for (auto & i : lines) {
-        string::size_type eq = i.find('=');
-        if (eq == string::npos)
-            throw Error("illegal option name/value");
-        set(i.substr(0, eq), i.substr(eq + 1));
-    }
-}
-
-
-Settings::SettingsMap Settings::getOverrides()
+template<> std::string Setting<bool, Settings::CaseHackTag>::to_string()
 {
-    return overrides;
+    return printBool(value);
 }
 
-
-const string nixVersion = PACKAGE_VERSION;
-
-
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d47fdb7c9de..95c8859cfa9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "types.hh"
-#include "logging.hh"
+#include "config.hh"
 
 #include <map>
 #include <sys/types.h>
@@ -9,36 +9,33 @@
 
 namespace nix {
 
+typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
 
-struct Settings {
+extern bool useCaseHack; // FIXME
 
-    typedef std::map<string, string> SettingsMap;
+class Settings : public Config {
 
-    Settings();
+    StringMap overrides;
 
-    void loadConfFile();
+    unsigned int getDefaultCores();
 
-    void set(const string & name, const string & value);
+public:
 
-    void update();
+    Settings();
 
-    string pack();
+    void loadConfFile();
 
-    void unpack(const string & pack);
+    void set(const string & name, const string & value);
 
-    SettingsMap getOverrides();
+    StringMap getOverrides();
 
-    /* TODO: the comments below should be strings and exposed via a nice command-line UI or similar.
-       We should probably replace it with some sort of magic template or macro to minimize the amount
-       of duplication and pain here. */
+    Path nixPrefix;
 
     /* The directory where we store sources and derived files. */
     Path nixStore;
 
     Path nixDataDir; /* !!! fix */
 
-    Path nixPrefix;
-
     /* The directory where we log various operations. */
     Path nixLogDir;
 
@@ -57,17 +54,14 @@ struct Settings {
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
-    /* Whether to keep temporary directories of failed builds. */
-    bool keepFailed;
+    Setting<bool> keepFailed{this, false, "keep-failed",
+        "Whether to keep temporary directories of failed builds."};
 
-    /* Whether to keep building subgoals when a sibling (another
-       subgoal of the same goal) fails. */
-    bool keepGoing;
+    Setting<bool> keepGoing{this, false, "keep-going",
+        "Whether to keep building derivations when another build fails."};
 
-    /* Whether, if we cannot realise the known closure corresponding
-       to a derivation, we should try to normalise the derivation
-       instead. */
-    bool tryFallback;
+    Setting<bool> tryFallback{this, tryFallback, "build-fallback",
+        "Whether to fall back to building when substitution fails."};
 
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
@@ -76,206 +70,206 @@ struct Settings {
        the log to show if a build fails. */
     size_t logLines = 10;
 
-    /* Maximum number of parallel build jobs.  0 means unlimited. */
-    unsigned int maxBuildJobs;
+    struct MaxBuildJobsTag { };
+    Setting<unsigned int, MaxBuildJobsTag> maxBuildJobs{this, 1, "build-max-jobs",
+        "Maximum number of parallel build jobs. \"auto\" means use number of cores."};
 
-    /* Number of CPU cores to utilize in parallel within a build,
-       i.e. by passing this number to Make via '-j'. 0 means that the
-       number of actual CPU cores on the local host ought to be
-       auto-detected. */
-    unsigned int buildCores;
+    Setting<unsigned int> buildCores{this, getDefaultCores(), "build-cores",
+        "Number of CPU cores to utilize in parallel within a build, "
+        "i.e. by passing this number to Make via '-j'. 0 means that the "
+        "number of actual CPU cores on the local host ought to be "
+        "auto-detected."};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
-    bool readOnlyMode;
-
-    /* The canonical system name, as returned by config.guess. */
-    string thisSystem;
-
-    /* The maximum time in seconds that a builer can go without
-       producing any output on stdout/stderr before it is killed.  0
-       means infinity. */
-    time_t maxSilentTime;
+    bool readOnlyMode = false;
 
-    /* The maximum duration in seconds that a builder can run.  0
-       means infinity.  */
-    time_t buildTimeout;
+    Setting<std::string> thisSystem{this, SYSTEM, "system",
+        "The canonical Nix system name."};
 
-    /* Whether to use build hooks (for distributed builds).  Sometimes
-       users want to disable this from the command-line. */
-    bool useBuildHook;
+    Setting<time_t> maxSilentTime{this, 0, "build-max-silent-time",
+        "The maximum time in seconds that a builer can go without "
+        "producing any output on stdout/stderr before it is killed. "
+        "0 means infinity."};
 
-    /* Amount of reserved space for the garbage collector
-       (/nix/var/nix/db/reserved). */
-    off_t reservedSize;
+    Setting<time_t> buildTimeout{this, 0, "build-timeout",
+        "The maximum duration in seconds that a builder can run. "
+        "0 means infinity."};
 
-    /* Whether SQLite should use fsync. */
-    bool fsyncMetadata;
+    Setting<bool> useBuildHook{this, true, "remote-builds",
+        "Whether to use build hooks (for distributed builds)."};
 
-    /* Whether SQLite should use WAL mode. */
-    bool useSQLiteWAL;
+    Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
+        "Amount of reserved disk space for the garbage collector."};
 
-    /* Whether to call sync() before registering a path as valid. */
-    bool syncBeforeRegistering;
+    Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
+        "Whether SQLite should use fsync()."};
 
-    /* Whether to use substitutes. */
-    bool useSubstitutes;
+    Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
+        "Whether SQLite should use WAL mode."};
 
-    /* The Unix group that contains the build users. */
-    string buildUsersGroup;
+    Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
+        "Whether to call sync() before registering a path as valid."};
 
-    /* Set of ssh connection strings for the ssh substituter */
-    Strings sshSubstituterHosts;
+    Setting<bool> useSubstitutes{this, true, "build-use-substitutes",
+        "Whether to use substitutes."};
 
-    /* Whether to use the ssh substituter at all */
-    bool useSshSubstituter;
+    Setting<std::string> buildUsersGroup{this, "", "build-users-group",
+        "The Unix group that contains the build users."};
 
-    /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
-    bool impersonateLinux26;
+    Setting<bool> impersonateLinux26{this, false, "build-impersonate-linux-26",
+        "Whether to impersonate a Linux 2.6 machine on newer kernels."};
 
-    /* Whether to store build logs. */
-    bool keepLog;
+    Setting<bool> keepLog{this, true, "build-keep-log",
+        "Whether to store build logs."};
 
-    /* Whether to compress logs. */
-    bool compressLog;
+    Setting<bool> compressLog{this, true, "build-compress-log",
+        "Whether to compress logs."};
 
-    /* Maximum number of bytes a builder can write to stdout/stderr
-       before being killed (0 means no limit). */
-    unsigned long maxLogSize;
+    Setting<unsigned long> maxLogSize{this, 0, "build-max-log-size",
+        "Maximum number of bytes a builder can write to stdout/stderr "
+        "before being killed (0 means no limit)."};
 
     /* When build-repeat > 0 and verboseBuild == true, whether to
        print repeated builds (i.e. builds other than the first one) to
        stderr. Hack to prevent Hydra logs from being polluted. */
     bool printRepeatedBuilds = true;
 
-    /* How often (in seconds) to poll for locks. */
-    unsigned int pollInterval;
+    Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
+        "How often (in seconds) to poll for locks."};
 
-    /* Whether to check if new GC roots can in fact be found by the
-       garbage collector. */
-    bool checkRootReachability;
+    Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
+        "Whether to check if new GC roots can in fact be found by the "
+        "garbage collector."};
 
-    /* Whether the garbage collector should keep outputs of live
-       derivations. */
-    bool gcKeepOutputs;
+    Setting<bool> gcKeepOutputs{this, false, "gc-keep-outputs",
+        "Whether the garbage collector should keep outputs of live derivations."};
 
-    /* Whether the garbage collector should keep derivers of live
-       paths. */
-    bool gcKeepDerivations;
+    Setting<bool> gcKeepDerivations{this, true, "gc-keep-derivations",
+        "Whether the garbage collector should keep derivers of live paths."};
 
-    /* Whether to automatically replace files with identical contents
-       with hard links. */
-    bool autoOptimiseStore;
+    Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
+        "Whether to automatically replace files with identical contents with hard links."};
 
-    /* Whether to add derivations as a dependency of user environments
-       (to prevent them from being GCed). */
-    bool envKeepDerivations;
+    Setting<bool> envKeepDerivations{this, false, "env-keep-derivations",
+        "Whether to add derivations as a dependency of user environments "
+        "(to prevent them from being GCed)."};
 
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
     /* Whether to show a stack trace if Nix evaluation fails. */
-    bool showTrace;
+    bool showTrace = false;
 
-    /* Whether native-code enabling primops should be enabled */
-    bool enableNativeCode;
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
 
-    /* Whether to enable sandboxed builds (string until we get an enum for true/false/relaxed) */
-    string useSandbox;
+    Setting<SandboxMode> sandboxMode{this, smDisabled, "build-use-sandbox",
+        "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
+        {"build-use-chroot"}};
 
-    /* The basic set of paths to expose in a sandbox */
-    PathSet sandboxPaths;
+    Setting<PathSet> sandboxPaths{this, {}, "build-sandbox-paths",
+        "The paths to make available inside the build sandbox.",
+        {"build-chroot-dirs"}};
 
-    /* Any extra sandbox paths to expose */
-    PathSet extraSandboxPaths;
+    Setting<PathSet> extraSandboxPaths{this, {}, "build-extra-sandbox-paths",
+        "Additional paths to make available inside the build sandbox.",
+        {"build-extra-chroot-dirs"}};
 
-    /* Whether to allow certain questionable operations (like fetching) during evaluation */
-    bool restrictEval;
+    Setting<bool> restrictEval{this, false, "restrict-eval",
+        "Whether to restrict file system access to paths in $NIX_PATH, "
+        "and to disallow fetching files from the network."};
 
-    /* The number of times to repeat a build to check for determinism */
-    int buildRepeat;
+    Setting<size_t> buildRepeat{this, 0, "build-repeat",
+        "The number of times to repeat a build in order to verify determinism."};
 
-    /* Which prefixes to allow derivations to ask for access to (primarily for Darwin) */
-    PathSet allowedImpureHostPrefixes;
+#if __linux__
+    Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
+        "The size of /dev/shm in the build sandbox."};
+#endif
 
-    /* The size of /dev/shm in the build sandbox (for Linux) */
-    string sandboxShmSize;
+    Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
+        "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
 
-    /* Whether to log Darwin sandbox access violations to the system log */
-    bool darwinLogSandboxViolations;
+#if __APPLE__
+    Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations",
+        "Whether to log Darwin sandbox access violations to the system log."};
+#endif
 
-    /* ??? */
-    bool runDiffHook;
+    Setting<bool> runDiffHook{this, false, "run-diff-hook",
+        "Whether to run the program specified by the diff-hook setting "
+        "repeated builds produce a different result. Typically used to "
+        "plug in diffoscope."};
 
-    /* ??? */
-    string diffHook;
+    PathSetting diffHook{this, true, "", "diff-hook",
+        "A program that prints out the differences between the two paths "
+        "specified on its command line."};
 
-    /* Whether to fail if repeated builds produce different output */
-    bool enforceDeterminism;
+    Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
+        "Whether to fail if repeated builds produce different output."};
 
-    /* The known public keys for a binary cache */
-    Strings binaryCachePublicKeys;
+    Setting<Strings> binaryCachePublicKeys{this,
+        {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
+        "binary-cache-public-keys",
+        "Trusted public keys for secure substitution."};
 
-    /* Secret keys to use for build output signing */
-    Strings secretKeyFiles;
+    Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
+        "Secret keys with which to sign local builds."};
 
-    /* Number of parallel connections to hit a binary cache with when finding out if it contains hashes */
-    int binaryCachesParallelConnections;
+    Setting<size_t> binaryCachesParallelConnections{this, 25, "binary-caches-parallel-connections",
+        "Number of parallel connections to binary caches."};
 
-    /* Whether to enable HTTP2 */
-    bool enableHttp2;
+    Setting<bool> enableHttp2{this, true, "enable-http2",
+        "Whether to enable HTTP/2 support."};
 
-    /* How soon to expire tarballs like builtins.fetchTarball and (ugh, bad name) builtins.fetchurl */
-    int tarballTtl;
+    Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
+        "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
-    /* ??? */
-    string signedBinaryCaches;
+    Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches",
+        "Obsolete."};
 
-    /* ??? */
-    Strings substituters;
+    Setting<Strings> substituters{this,
+        nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
+        "substituters",
+        "The URIs of substituters (such as https://cache.nixos.org/).",
+        {"binary-caches"}};
 
-    /* ??? */
-    Strings binaryCaches;
+    // FIXME: provide a way to add to option values.
+    Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
+        "Additional URIs of substituters.",
+        {"extra-binary-caches"}};
 
-    /* ??? */
-    Strings extraBinaryCaches;
-
-    /* Who we trust to ask the daemon to do unsafe things */
-    Strings trustedUsers;
+    Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
+        "Which users or groups are trusted to ask the daemon to do unsafe things."};
 
     /* ?Who we trust to use the daemon in safe ways */
-    Strings allowedUsers;
+    Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
+        "Which users or groups are allowed to connect to the daemon."};
 
-    /* ??? */
-    bool printMissing;
+    Setting<bool> printMissing{this, true, "print-missing",
+        "Whether to print what paths need to be built or downloaded."};
 
-    /* The hook to run just before a build to set derivation-specific
-       build settings */
-    Path preBuildHook;
+    Setting<std::string> preBuildHook{this,
+#if __APPLE__
+        nixLibexecDir + "/nix/resolve-system-dependencies",
+#else
+        "",
+#endif
+        "pre-build-hook",
+        "A program to run just before a build to set derivation-specific build settings."};
 
-    /* Path to the netrc file used to obtain usernames/passwords for
-       downloads. */
-    Path netrcFile;
+    Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
+        "Path to the netrc file used to obtain usernames/passwords for downloads."};
 
     /* Path to the SSL CA file used */
     Path caFile;
 
-    /* Whether we allow import-from-derivation */
-    bool enableImportFromDerivation;
-
-private:
-    StringSet deprecatedOptions;
-    SettingsMap settings, overrides;
-
-    void checkDeprecated(const string & name);
+    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
+        "Whether the evaluator allows importing the result of a derivation."};
 
-    void _get(string & res, const string & name);
-    void _get(string & res, const string & name1, const string & name2);
-    void _get(bool & res, const string & name);
-    void _get(StringSet & res, const string & name);
-    void _get(StringSet & res, const string & name1, const string & name2);
-    void _get(Strings & res, const string & name);
-    template<class N> void _get(N & res, const string & name);
+    struct CaseHackTag { };
+    Setting<bool, CaseHackTag> useCaseHack{this, nix::useCaseHack, "use-case-hack",
+        "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 9111a45f886..5a98454ab38 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -75,7 +75,7 @@ LocalStore::LocalStore(const Params & params)
 
         mode_t perm = 01775;
 
-        struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
+        struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
             printError(format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist")
                 % settings.buildUsersGroup);
@@ -1335,7 +1335,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 
     auto secretKeyFiles = settings.secretKeyFiles;
 
-    for (auto & secretKeyFile : secretKeyFiles) {
+    for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
         info.sign(secretKey);
     }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e1df137e4db..da3c8eb8d89 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -166,9 +166,7 @@ void RemoteStore::setOptions(Connection & conn)
        << settings.useSubstitutes;
 
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
-        Settings::SettingsMap overrides = settings.getOverrides();
-        if (overrides["ssh-auth-sock"] == "")
-            overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK");
+        StringMap overrides = settings.getOverrides();
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 514d1c2ff8b..835bbb90e0b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -719,7 +719,7 @@ ref<Store> openStore(const std::string & uri, const Store::Params & params)
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->warnUnused();
+            store->warnUnknownSettings();
             return ref<Store>(store);
         }
     }
@@ -782,13 +782,10 @@ std::list<ref<Store>> getDefaultSubstituters()
         state->stores.push_back(openStore(uri));
     };
 
-    for (auto uri : settings.substituters)
+    for (auto uri : settings.substituters.get())
         addStore(uri);
 
-    for (auto uri : settings.binaryCaches)
-        addStore(uri);
-
-    for (auto uri : settings.extraBinaryCaches)
+    for (auto uri : settings.extraSubstituters.get())
         addStore(uri);
 
     state->done = true;
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index c05a3253bce..85e5ce330be 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -11,7 +11,7 @@ void Config::set(const std::string & name, const std::string & value)
     i->second.setting->set(value);
 }
 
-void Config::add(AbstractSetting * setting)
+void Config::addSetting(AbstractSetting * setting)
 {
     _settings.emplace(setting->name, Config::SettingData{false, setting});
     for (auto & alias : setting->aliases)
@@ -41,21 +41,59 @@ void Config::add(AbstractSetting * setting)
     }
 }
 
-void Config::warnUnused()
+void Config::warnUnknownSettings()
 {
     for (auto & i : initials)
         warn("unknown setting '%s'", i.first);
 }
 
-std::string Config::dump()
+StringMap Config::getSettings()
 {
-    std::string res;
+    StringMap res;
     for (auto & opt : _settings)
         if (!opt.second.isAlias)
-            res += opt.first + " = " + opt.second.setting->to_string() + "\n";
+            res.emplace(opt.first, opt.second.setting->to_string());
     return res;
 }
 
+void Config::applyConfigFile(const Path & path, bool fatal)
+{
+    try {
+        string contents = readFile(path);
+
+        unsigned int pos = 0;
+
+        while (pos < contents.size()) {
+            string line;
+            while (pos < contents.size() && contents[pos] != '\n')
+                line += contents[pos++];
+            pos++;
+
+            string::size_type hash = line.find('#');
+            if (hash != string::npos)
+                line = string(line, 0, hash);
+
+            vector<string> tokens = tokenizeString<vector<string> >(line);
+            if (tokens.empty()) continue;
+
+            if (tokens.size() < 2 || tokens[1] != "=")
+                throw UsageError("illegal configuration line ‘%1%’ in ‘%2%’", line, path);
+
+            string name = tokens[0];
+
+            vector<string>::iterator i = tokens.begin();
+            advance(i, 2);
+
+            try {
+                set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
+            } catch (UsageError & e) {
+                if (fatal) throw;
+                warn("in configuration file '%s': %s", path, e.what());
+            }
+        };
+    } catch (SysError &) { }
+}
+
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
@@ -74,41 +112,65 @@ template<> std::string Setting<std::string>::to_string()
     return value;
 }
 
-template<typename T>
-void Setting<T>::set(const std::string & str)
+template<typename T, typename Tag>
+void Setting<T, Tag>::set(const std::string & str)
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
-    try {
-        auto i = std::stoll(str);
-        if (i < std::numeric_limits<T>::min() ||
-            i > std::numeric_limits<T>::max())
-            throw UsageError("setting '%s' has out-of-range value %d", name, i);
-        value = i;
-    } catch (std::logic_error&) {
+    if (!string2Int(str, value))
         throw UsageError("setting '%s' has invalid value '%s'", name, str);
-    }
 }
 
-template<typename T>
-std::string Setting<T>::to_string()
+template<typename T, typename Tag>
+std::string Setting<T, Tag>::to_string()
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     return std::to_string(value);
 }
 
-template<> void Setting<bool>::set(const std::string & str)
+bool AbstractSetting::parseBool(const std::string & str)
 {
     if (str == "true" || str == "yes" || str == "1")
-        value = true;
+        return true;
     else if (str == "false" || str == "no" || str == "0")
-        value = false;
+        return false;
     else
         throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
 }
 
+template<> void Setting<bool>::set(const std::string & str)
+{
+    value = parseBool(str);
+}
+
+std::string AbstractSetting::printBool(bool b)
+{
+    return b ? "true" : "false";
+}
+
+
 template<> std::string Setting<bool>::to_string()
 {
-    return value ? "true" : "false";
+    return printBool(value);
+}
+
+template<> void Setting<Strings>::set(const std::string & str)
+{
+    value = tokenizeString<Strings>(str);
+}
+
+template<> std::string Setting<Strings>::to_string()
+{
+    return concatStringsSep(" ", value);
+}
+
+template<> void Setting<StringSet>::set(const std::string & str)
+{
+    value = tokenizeString<StringSet>(str);
+}
+
+template<> std::string Setting<StringSet>::to_string()
+{
+    return concatStringsSep(" ", value);
 }
 
 template class Setting<int>;
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index fb2d48e9c83..6c8612f675c 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -47,11 +47,13 @@ public:
 
     void set(const std::string & name, const std::string & value);
 
-    void add(AbstractSetting * setting);
+    void addSetting(AbstractSetting * setting);
 
-    void warnUnused();
+    void warnUnknownSettings();
 
-    std::string dump();
+    StringMap getSettings();
+
+    void applyConfigFile(const Path & path, bool fatal = false);
 };
 
 class AbstractSetting
@@ -83,10 +85,15 @@ protected:
     virtual void set(const std::string & value) = 0;
 
     virtual std::string to_string() = 0;
+
+    bool parseBool(const std::string & str);
+    std::string printBool(bool b);
 };
 
+struct DefaultSettingTag { };
+
 /* A setting of type T. */
-template<typename T>
+template<typename T, typename Tag = DefaultSettingTag>
 class Setting : public AbstractSetting
 {
 protected:
@@ -103,10 +110,12 @@ public:
         : AbstractSetting(name, description, aliases)
         , value(def)
     {
-        options->add(this);
+        options->addSetting(this);
     }
 
     operator const T &() const { return value; }
+    operator T &() { return value; }
+    const T & get() const { return value; }
     bool operator ==(const T & v2) const { return value == v2; }
     bool operator !=(const T & v2) const { return value != v2; }
     void operator =(const T & v) { value = v; }
@@ -123,6 +132,9 @@ std::ostream & operator <<(std::ostream & str, const Setting<T> & opt)
     return str;
 }
 
+template<typename T>
+bool operator ==(const T & v1, const Setting<T> & v2) { return v1 == (const T &) v2; }
+
 /* A special setting for Paths. These are automatically canonicalised
    (e.g. "/foo//bar/" becomes "/foo/bar"). */
 class PathSetting : public Setting<Path>
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index f4285693f7f..9fcb81dd5aa 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -436,30 +436,29 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopSetOptions: {
-        from >> settings.keepFailed;
-        from >> settings.keepGoing;
-        settings.set("build-fallback", readInt(from) ? "true" : "false");
+        settings.keepFailed = readInt(from);
+        settings.keepGoing = readInt(from);
+        settings.tryFallback = readInt(from);
         verbosity = (Verbosity) readInt(from);
         settings.set("build-max-jobs", std::to_string(readInt(from)));
-        settings.set("build-max-silent-time", std::to_string(readInt(from)));
+        settings.maxSilentTime = readInt(from);
         settings.useBuildHook = readInt(from) != 0;
         settings.verboseBuild = lvlError == (Verbosity) readInt(from);
         readInt(from); // obsolete logType
         readInt(from); // obsolete printBuildTrace
         settings.set("build-cores", std::to_string(readInt(from)));
-        settings.set("build-use-substitutes", readInt(from) ? "true" : "false");
+        settings.useSubstitutes  = readInt(from);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
             unsigned int n = readInt(from);
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                if (name == "build-timeout" || name == "use-ssh-substituter")
+                if (name == "build-timeout")
                     settings.set(name, value);
                 else
                     settings.set(trusted ? name : "untrusted-" + name, value);
             }
         }
-        settings.update();
         startWork();
         stopWork();
         break;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a40cca9824e..9131b74dfb4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -795,11 +795,11 @@ static void opServe(Strings opFlags, Strings opArgs)
         settings.maxSilentTime = readInt(in);
         settings.buildTimeout = readInt(in);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            in >> settings.maxLogSize;
+            settings.maxLogSize = readNum<unsigned long>(in);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
-            settings.set("build-repeat", std::to_string(readInt(in)));
-            settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");
-            settings.set("run-diff-hook", "true");
+            settings.buildRepeat = readInt(in);
+            settings.enforceDeterminism = readInt(in);
+            settings.runDiffHook = readInt(in);
         }
         settings.printRepeatedBuilds = false;
     };
diff --git a/src/nix/main.cc b/src/nix/main.cc
index fdb8f6e3a19..440ced97dfc 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -42,7 +42,6 @@ void mainWrapped(int argc, char * * argv)
     NixArgs args;
 
     args.parseCmdline(argvToStrings(argc, argv));
-    settings.update();
 
     assert(args.command);
 
diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh
index 3dadd591572..c9a83aaf83d 100755
--- a/tests/shell.shebang.sh
+++ b/tests/shell.shebang.sh
@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --option use-binary-caches false
+#! nix-shell -I nixpkgs=shell.nix --option build-use-substitutes false
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"

From 872ba75d8b6212c3a5e399ec62cbda86e58d3680 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Apr 2017 20:59:38 +0200
Subject: [PATCH 0237/2196] Add "nix show-config" command

This dumps the entire Nix configuration, including all options that
have default values.
---
 src/nix/show-config.cc | 43 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 src/nix/show-config.cc

diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
new file mode 100644
index 00000000000..ba39e2bb29b
--- /dev/null
+++ b/src/nix/show-config.cc
@@ -0,0 +1,43 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "installables.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "json.hh"
+
+using namespace nix;
+
+struct CmdShowConfig : Command
+{
+    bool json = false;
+
+    CmdShowConfig()
+    {
+        mkFlag(0, "json", "produce JSON output", &json);
+    }
+
+    std::string name() override
+    {
+        return "show-config";
+    }
+
+    std::string description() override
+    {
+        return "show the Nix configuration";
+    }
+
+    void run() override
+    {
+        if (json) {
+            // FIXME: use appropriate JSON types (bool, ints, etc).
+            JSONObject jsonObj(std::cout, true);
+            for (auto & s : settings.getSettings())
+                jsonObj.attr(s.first, s.second);
+        } else {
+            for (auto & s : settings.getSettings())
+                std::cout << s.first + " = " + s.second + "\n";
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdShowConfig>());

From 1673c373c9ddd1fc7ce1e12c79621dafe13fe930 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 11:57:02 +0200
Subject: [PATCH 0238/2196] nix-daemon: Don't die if the user sends an unknown
 setting

---
 src/nix-daemon/nix-daemon.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 9fcb81dd5aa..b4e753f5521 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -453,10 +453,14 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                if (name == "build-timeout")
-                    settings.set(name, value);
-                else
-                    settings.set(trusted ? name : "untrusted-" + name, value);
+                try {
+                    if (name == "build-timeout")
+                        settings.set(name, value);
+                    else
+                        settings.set(trusted ? name : "untrusted-" + name, value);
+                } catch (UsageError & e) {
+                    warn(e.what());
+                }
             }
         }
         startWork();

From 01dcdfcf336a127438b35b81751a783cde063ab8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 11:59:51 +0200
Subject: [PATCH 0239/2196] nix-daemon: Don't set untrusted-* settings

These are no longer used anywhere.
---
 src/nix-daemon/nix-daemon.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index b4e753f5521..90bfc67d877 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -454,10 +454,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
                 string name = readString(from);
                 string value = readString(from);
                 try {
-                    if (name == "build-timeout")
+                    if (trusted || name == "build-timeout")
                         settings.set(name, value);
-                    else
-                        settings.set(trusted ? name : "untrusted-" + name, value);
                 } catch (UsageError & e) {
                     warn(e.what());
                 }

From 3872371f25acb9250d34554192196caee855fcb4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 13:42:22 +0200
Subject: [PATCH 0240/2196] Minor cleanup

---
 src/libmain/shared.cc        | 2 +-
 src/nix-daemon/nix-daemon.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 9d506d01923..4747b9bf9b4 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -159,7 +159,7 @@ struct LegacyArgs : public MixCommonArgs
             &(bool&) settings.keepGoing);
 
         mkFlag(0, "fallback", "build from source if substitution fails", []() {
-            settings.set("build-fallback", "true");
+            settings.tryFallback = true;
         });
 
         mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 90bfc67d877..1389353bb5d 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -440,13 +440,13 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         settings.keepGoing = readInt(from);
         settings.tryFallback = readInt(from);
         verbosity = (Verbosity) readInt(from);
-        settings.set("build-max-jobs", std::to_string(readInt(from)));
+        settings.maxBuildJobs = readInt(from);
         settings.maxSilentTime = readInt(from);
         settings.useBuildHook = readInt(from) != 0;
         settings.verboseBuild = lvlError == (Verbosity) readInt(from);
         readInt(from); // obsolete logType
         readInt(from); // obsolete printBuildTrace
-        settings.set("build-cores", std::to_string(readInt(from)));
+        settings.buildCores = readInt(from);
         settings.useSubstitutes  = readInt(from);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
             unsigned int n = readInt(from);

From 3b36c64b15ac31f56c13591092c8a24aff4ee6c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 13:47:10 +0200
Subject: [PATCH 0241/2196] Fix Perl bindings

---
 perl/lib/Nix/Store.xs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 46b41f92392..1920942a4c0 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -26,7 +26,6 @@ static ref<Store> store()
     if (!_store) {
         try {
             settings.loadConfFile();
-            settings.update();
             settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {

From 6520b757c58df15a2d5d5daf234ff7a7740049cd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 13:59:39 +0200
Subject: [PATCH 0242/2196] Fix 32-bit build

http://hydra.nixos.org/build/51569816
---
 src/libutil/config.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 85e5ce330be..e7a810cec4d 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -176,7 +176,9 @@ template<> std::string Setting<StringSet>::to_string()
 template class Setting<int>;
 template class Setting<unsigned int>;
 template class Setting<long>;
-template class Setting<size_t>;
+template class Setting<unsigned long>;
+template class Setting<long long>;
+template class Setting<unsigned long long>;
 
 void PathSetting::set(const std::string & str)
 {

From da76c72bc9b247092e68411f0bd91ca37c176d0a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 14:02:43 +0200
Subject: [PATCH 0243/2196] Build on aarch64-linux

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 6136f650d95..294af54cd61 100644
--- a/release.nix
+++ b/release.nix
@@ -7,7 +7,7 @@ let
 
   pkgs = import <nixpkgs> {};
 
-  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" ];
+  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
 
 
   jobs = rec {

From dd3714f6eff44e0777c5e8afcc7a29e283986629 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 14:42:08 +0200
Subject: [PATCH 0244/2196] Doh

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 95c8859cfa9..f3a6038cafa 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -60,7 +60,7 @@ public:
     Setting<bool> keepGoing{this, false, "keep-going",
         "Whether to keep building derivations when another build fails."};
 
-    Setting<bool> tryFallback{this, tryFallback, "build-fallback",
+    Setting<bool> tryFallback{this, false, "build-fallback",
         "Whether to fall back to building when substitution fails."};
 
     /* Whether to show build log output in real time. */

From f8a2e8a55203f2cc16d70ad43afcc186adaab6b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Apr 2017 14:42:20 +0200
Subject: [PATCH 0245/2196] Shut up some warnings

---
 local.mk                       |  3 +--
 src/boost/format/feed_args.hpp |  7 +++++++
 src/libexpr/eval.cc            |  2 +-
 src/libexpr/local.mk           |  2 --
 src/libstore/build.cc          | 14 +++++++-------
 src/libstore/download.cc       |  2 +-
 6 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/local.mk b/local.mk
index dc10e6870a8..0a225423741 100644
--- a/local.mk
+++ b/local.mk
@@ -7,8 +7,7 @@ dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr \
-  -Wno-unneeded-internal-declaration
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp
index 3d0b47b4a12..cdd57fdf2bf 100644
--- a/src/boost/format/feed_args.hpp
+++ b/src/boost/format/feed_args.hpp
@@ -37,6 +37,13 @@ namespace  {
     os.str(emptyStr); 
   }
 
+  void do_pad( std::string & s, 
+                std::streamsize w, 
+                const char c, 
+                std::ios::fmtflags f, 
+                bool center)
+    __attribute__ ((unused));
+
   void do_pad( std::string & s, 
                 std::streamsize w, 
                 const char c, 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f6cdedb3797..5e1ae63c482 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -202,7 +202,7 @@ void initGC()
 
     GC_INIT();
 
-    GC_oom_fn = oomHandler;
+    GC_set_oom_fn(oomHandler);
 
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 620050a13b0..daa3258f0d3 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,8 +6,6 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_CXXFLAGS := -Wno-deprecated-register
-
 libexpr_LIBS = libutil libstore libformat
 
 libexpr_LDFLAGS =
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 33c9e37047f..9bf1ab5aa58 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3376,10 +3376,10 @@ void SubstitutionGoal::tryToRun()
     trace("trying to run");
 
     /* Make sure that we are allowed to start a build.  Note that even
-       is maxBuildJobs == 0 (no local builds allowed), we still allow
+       if maxBuildJobs == 0 (no local builds allowed), we still allow
        a substituter to run.  This is because substitutions cannot be
        distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) {
+    if (worker.getNrLocalBuilds() >= std::min(1U, (unsigned int) settings.maxBuildJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         return;
     }
@@ -3660,7 +3660,7 @@ void Worker::run(const Goals & _topGoals)
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
         else {
-            if (awake.empty() && settings.maxBuildJobs == 0) throw Error(
+            if (awake.empty() && 0 == settings.maxBuildJobs) throw Error(
                 "unable to start any build; either increase ‘--max-jobs’ "
                 "or enable distributed builds");
             assert(!awake.empty());
@@ -3697,9 +3697,9 @@ void Worker::waitForInput()
     auto nearest = steady_time_point::max(); // nearest deadline
     for (auto & i : children) {
         if (!i.respectTimeouts) continue;
-        if (settings.maxSilentTime != 0)
+        if (0 != settings.maxSilentTime)
             nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
-        if (settings.buildTimeout != 0)
+        if (0 != settings.buildTimeout)
             nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
     }
     if (nearest != steady_time_point::max()) {
@@ -3777,7 +3777,7 @@ void Worker::waitForInput()
         }
 
         if (goal->getExitCode() == Goal::ecBusy &&
-            settings.maxSilentTime != 0 &&
+            0 != settings.maxSilentTime &&
             j->respectTimeouts &&
             after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
@@ -3788,7 +3788,7 @@ void Worker::waitForInput()
         }
 
         else if (goal->getExitCode() == Goal::ecBusy &&
-            settings.buildTimeout != 0 &&
+            0 != settings.buildTimeout &&
             j->respectTimeouts &&
             after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d073e870b4e..42fd05bd80d 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -369,7 +369,7 @@ struct CurlDownloader : public Downloader
         curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
         #endif
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            settings.binaryCachesParallelConnections);
+            settings.binaryCachesParallelConnections.get());
 
         enableHttp2 = settings.enableHttp2;
 

From 9622d00afa6e94ef3ca026f9e15a7d122566e7d0 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Mon, 17 Apr 2017 10:33:53 +0100
Subject: [PATCH 0246/2196] Manual: document tryEval

---
 doc/manual/expressions/builtins.xml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index e9baff65961..8a18b71008c 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1163,6 +1163,19 @@ stdenv.mkDerivation (rec {
 
   </varlistentry>
 
+  <varlistentry><term><function>builtins.tryEval</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Try to evaluate <replaceable>e</replaceable>.
+    Return a set containing the attributes <literal>success</literal>
+    (<literal>true</literal> if <replaceable>e</replaceable> evaluated
+    successfully, <literal>false</literal> if an error was thrown) and
+    <literal>value</literal>, equalling <replaceable>e</replaceable>
+    if successful and <literal>false</literal> otherwise.
+    </para></listitem>
+
+  </varlistentry>
+
 
   <varlistentry><term><function>builtins.typeOf</function>
   <replaceable>e</replaceable></term>

From b0cb11722626e906a73f10dd9a0c9eea29faf43a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Apr 2017 14:18:26 +0200
Subject: [PATCH 0247/2196] getDerivations(): Filter out packages with bad
 derivation names

In particular, this disallows attribute names containing dots or
starting with dots. Hydra already disallowed these. This affects the
following packages in Nixpkgs master:

  2048-in-terminal
  2bwm
  389-ds-base
  90secondportraits
  lispPackages.3bmd
  lispPackages.hu.dwim.asdf
  lispPackages.hu.dwim.def

Closes #1342.
---
 src/libexpr/get-drvs.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 5342739c53c..ae9fb0e5ec3 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -3,6 +3,7 @@
 #include "eval-inline.hh"
 
 #include <cstring>
+#include <regex>
 
 
 namespace nix {
@@ -262,6 +263,9 @@ static string addToPath(const string & s1, const string & s2)
 }
 
 
+static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
+
+
 static void getDerivations(EvalState & state, Value & vIn,
     const string & pathPrefix, Bindings & autoArgs,
     DrvInfos & drvs, Done & done,
@@ -286,6 +290,8 @@ static void getDerivations(EvalState & state, Value & vIn,
            precedence). */
         for (auto & i : v.attrs->lexicographicOrder()) {
             Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name);
+            if (!std::regex_match(std::string(i->name), attrRegex))
+                continue;
             string pathPrefix2 = addToPath(pathPrefix, i->name);
             if (combineChannels)
                 getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);

From 9cc8047f44b3d333c2c55c140165bfd507f4d41e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Apr 2017 14:54:52 +0200
Subject: [PATCH 0248/2196] Reimplement connect-timeout

Fixes #1339.
---
 src/libstore/download.cc | 2 ++
 src/libstore/globals.hh  | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 42fd05bd80d..4d502219ed8 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -249,6 +249,8 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get());
+
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index f3a6038cafa..72863920de9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -270,6 +270,9 @@ public:
     struct CaseHackTag { };
     Setting<bool, CaseHackTag> useCaseHack{this, nix::useCaseHack, "use-case-hack",
         "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
+
+    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
+        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
 };
 
 

From 00b6c6d0c3047e6e0b2ab6ea8fef9bef94586ce4 Mon Sep 17 00:00:00 2001
From: kballou <kballou@devnulllabs.io>
Date: Wed, 19 Apr 2017 09:27:28 -0600
Subject: [PATCH 0249/2196] Fix small grammar issue about page

Fix subject-verb agreement issue in introduction/about.
---
 doc/manual/introduction/about-nix.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index 0c58984ac48..e6dfb7a5a56 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -93,7 +93,7 @@ time window in which the package has some files from the old version
 and some files from the new version — which would be bad because a
 program might well crash if it’s started during that period.</para>
 
-<para>And since package aren’t overwritten, the old versions are still
+<para>And since packages aren’t overwritten, the old versions are still
 there after an upgrade.  This means that you can <emphasis>roll
 back</emphasis> to the old version:</para>
 

From 1559c596f6301628bbf04468cb897ea0a0010c1e Mon Sep 17 00:00:00 2001
From: Armijn Hemel <armijn@tjaldur.nl>
Date: Wed, 19 Apr 2017 19:10:12 +0200
Subject: [PATCH 0250/2196] document option

---
 doc/manual/command-ref/nix-env.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 85f10e0760b..8462cf8a027 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1136,7 +1136,7 @@ user environment elements, etc. -->
 
     <listitem><para>Print all of the meta-attributes of the
     derivation.  This option is only available with
-    <option>--xml</option>.</para></listitem>
+    <option>--xml</option> or <option>--json</option>.</para></listitem>
 
   </varlistentry>
 

From 76cb3c702cc1769438940b65b83971f483cca062 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 13:20:49 +0200
Subject: [PATCH 0251/2196] Reimplement trusted-substituters (aka
 trusted-binary-caches)

---
 src/libstore/globals.hh      |  4 +++
 src/nix-daemon/nix-daemon.cc | 48 +++++++++++++++++++++++++++++++-----
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 72863920de9..a5d5a3f5057 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -239,6 +239,10 @@ public:
         "Additional URIs of substituters.",
         {"extra-binary-caches"}};
 
+    Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
+        "Disabled substituters that may be enabled via the substituters option by untrusted users.",
+        {"trusted-binary-caches"}};
+
     Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
         "Which users or groups are trusted to ask the daemon to do unsafe things."};
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 1389353bb5d..5c2641eac6f 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -448,20 +448,56 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         readInt(from); // obsolete printBuildTrace
         settings.buildCores = readInt(from);
         settings.useSubstitutes  = readInt(from);
+
+        StringMap overrides;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
             unsigned int n = readInt(from);
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                try {
-                    if (trusted || name == "build-timeout")
-                        settings.set(name, value);
-                } catch (UsageError & e) {
-                    warn(e.what());
-                }
+                overrides.emplace(name, value);
             }
         }
+
         startWork();
+
+        for (auto & i : overrides) {
+            auto & name(i.first);
+            auto & value(i.second);
+
+            auto setSubstituters = [&](Setting<Strings> & res) {
+                if (name != res.name && res.aliases.count(name) == 0)
+                    return false;
+                StringSet trusted = settings.trustedSubstituters;
+                for (auto & s : settings.substituters.get())
+                    trusted.insert(s);
+                Strings subs;
+                auto ss = tokenizeString<Strings>(value);
+                for (auto & s : ss)
+                    if (trusted.count(s))
+                        subs.push_back(s);
+                    else
+                        warn("ignoring untrusted substituter '%s'", s);
+                res = subs;
+                return true;
+            };
+
+            try {
+                if (trusted
+                    || name == settings.buildTimeout.name
+                    || name == settings.connectTimeout.name)
+                    settings.set(name, value);
+                else if (setSubstituters(settings.substituters))
+                    ;
+                else if (setSubstituters(settings.extraSubstituters))
+                    ;
+                else
+                    debug("ignoring untrusted setting '%s'", name);
+            } catch (UsageError & e) {
+                warn(e.what());
+            }
+        }
+
         stopWork();
         break;
     }

From 4222402219ca3a356d23e05e0474f8648b111d91 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 13:39:01 +0200
Subject: [PATCH 0252/2196] nix.conf man page: binary-caches -> substituters

---
 doc/manual/command-ref/conf-file.xml | 34 +++++++++++-----------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 3de9647aa4e..73e0017ccef 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -334,15 +334,16 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>binary-caches</literal></term>
+  <varlistentry><term><literal>substituters</literal></term>
 
-    <listitem><para>A list of URLs of binary caches, separated by
+    <listitem><para>A list of URLs of substituters, separated by
     whitespace.  The default is
     <literal>https://cache.nixos.org</literal>.</para></listitem>
 
   </varlistentry>
 
 
+  <!--
   <varlistentry><term><literal>binary-caches-files</literal></term>
 
     <listitem><para>A list of names of files that will be read to
@@ -355,28 +356,28 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     option to read files created by untrusted users!</para></listitem>
 
   </varlistentry>
+  -->
 
 
-  <varlistentry><term><literal>trusted-binary-caches</literal></term>
+  <varlistentry><term><literal>trusted-substituters</literal></term>
 
-    <listitem><para>A list of URLs of binary caches, separated by
+    <listitem><para>A list of URLs of substituters, separated by
     whitespace.  These are not used by default, but can be enabled by
     users of the Nix daemon by specifying <literal>--option
-    binary-caches <replaceable>urls</replaceable></literal> on the
+    substituters <replaceable>urls</replaceable></literal> on the
     command line.  Unprivileged users are only allowed to pass a
-    subset of the URLs listed in <literal>binary-caches</literal> and
-    <literal>trusted-binary-caches</literal>.</para></listitem>
+    subset of the URLs listed in <literal>substituters</literal> and
+    <literal>trusted-substituters</literal>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>extra-binary-caches</literal></term>
+  <varlistentry><term><literal>extra-substituters</literal></term>
 
     <listitem><para>Additional binary caches appended to those
-    specified in <option>binary-caches</option> and
-    <option>binary-caches-files</option>.  When used by unprivileged
-    users, untrusted binary caches (i.e. those not listed in
-    <option>trusted-binary-caches</option>) are silently
+    specified in <option>substituters</option>.  When used by
+    unprivileged users, untrusted substituters (i.e. those not listed
+    in <option>trusted-substituters</option>) are silently
     ignored.</para></listitem>
 
   </varlistentry>
@@ -412,15 +413,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>verify-https-binary-caches</literal></term>
-
-    <listitem><para>Whether HTTPS binary caches are required to have a
-    certificate that can be verified. Defaults to
-    <literal>true</literal>.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry><term><literal>netrc-file</literal></term>
 
     <listitem><para>If set to an absolute path to a <filename>netrc</filename>

From 562585e901d9e5f2ef62be11c2e74badcacb1f50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 14:04:00 +0200
Subject: [PATCH 0253/2196] binary-caches-parallel-connections ->
 http-connections

---
 doc/manual/command-ref/conf-file.xml | 2 +-
 src/libstore/globals.hh              | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 73e0017ccef..79e18de9b17 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -404,7 +404,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
+  <varlistentry><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections
     used to fetch files from binary caches and by other downloads. It
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index a5d5a3f5057..de64e982659 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -216,8 +216,9 @@ public:
     Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
         "Secret keys with which to sign local builds."};
 
-    Setting<size_t> binaryCachesParallelConnections{this, 25, "binary-caches-parallel-connections",
-        "Number of parallel connections to binary caches."};
+    Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections",
+        "Number of parallel HTTP connections.",
+        {"binary-caches-parallel-connections"}};
 
     Setting<bool> enableHttp2{this, true, "enable-http2",
         "Whether to enable HTTP/2 support."};

From f05d5f89ff4ec52ed2f6d576b2b2323b5292f815 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 14:58:16 +0200
Subject: [PATCH 0254/2196] Read per-user settings from ~/.config/nix/nix.conf

---
 doc/manual/command-ref/conf-file.xml | 36 +++++++++++++++++++++-------
 src/libstore/globals.cc              | 12 +++++-----
 src/libstore/globals.hh              |  4 ----
 src/libstore/remote-store.cc         |  2 +-
 src/libutil/config.cc                | 13 ++++++++--
 src/libutil/config.hh                | 10 ++++++--
 src/libutil/util.cc                  | 12 ++++++++++
 src/libutil/util.hh                  |  5 +++-
 8 files changed, 70 insertions(+), 24 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 79e18de9b17..616983bc7f0 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -17,13 +17,32 @@
 
 <refsection><title>Description</title>
 
-<para>A number of persistent settings of Nix are stored in the file
-<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> or
-<filename>$NIX_CONF_DIR/nix.conf</filename> if <envar>NIX_CONF_DIR</envar> is set.
-This file is a list of <literal><replaceable>name</replaceable> =
+<para>Nix reads settings from two configuration files:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The system-wide configuration file
+    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+    <filename>$NIX_CONF_DIR/nix.conf</filename> if
+    <envar>NIX_CONF_DIR</envar> is set.</para>
+  </listitem>
+
+  <listitem>
+    <para>The user configuration file
+    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
+    <filename>~/.config/nix/nix.conf</filename> if
+    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>The configuration files consist of
+<literal><replaceable>name</replaceable> =
 <replaceable>value</replaceable></literal> pairs, one per line.
-Comments start with a <literal>#</literal> character.  Here is an example
-configuration file:</para>
+Comments start with a <literal>#</literal> character.  Here is an
+example configuration file:</para>
 
 <programlisting>
 gc-keep-outputs = true       # Nice for developers
@@ -31,8 +50,9 @@ gc-keep-derivations = true   # Idem
 env-keep-derivations = false
 </programlisting>
 
-<para>You can override settings using the <option>--option</option>
-flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
+<para>You can override settings on the command line using the
+<option>--option</option> flag, e.g. <literal>--option gc-keep-outputs
+false</literal>.</para>
 
 <para>The following settings are currently available:
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index bb61daa5164..6b9d077469c 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -53,19 +53,19 @@ Settings::Settings()
 void Settings::loadConfFile()
 {
     applyConfigFile(nixConfDir + "/nix.conf");
+
+    /* We only want to send overrides to the daemon, i.e. stuff from
+       ~/.nix/nix.conf or the command line. */
+    resetOverriden();
+
+    applyConfigFile(getConfigDir() + "/nix/nix.conf");
 }
 
 void Settings::set(const string & name, const string & value)
 {
-    overrides[name] = value;
     Config::set(name, value);
 }
 
-StringMap Settings::getOverrides()
-{
-    return overrides;
-}
-
 unsigned int Settings::getDefaultCores()
 {
     return std::max(1U, std::thread::hardware_concurrency());
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index de64e982659..d3ecaadb639 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -15,8 +15,6 @@ extern bool useCaseHack; // FIXME
 
 class Settings : public Config {
 
-    StringMap overrides;
-
     unsigned int getDefaultCores();
 
 public:
@@ -27,8 +25,6 @@ public:
 
     void set(const string & name, const string & value);
 
-    StringMap getOverrides();
-
     Path nixPrefix;
 
     /* The directory where we store sources and derived files. */
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index da3c8eb8d89..bc9ef3d47c5 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -166,7 +166,7 @@ void RemoteStore::setOptions(Connection & conn)
        << settings.useSubstitutes;
 
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
-        StringMap overrides = settings.getOverrides();
+        auto overrides = settings.getSettings(true);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second;
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index e7a810cec4d..bf137299718 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -9,6 +9,7 @@ void Config::set(const std::string & name, const std::string & value)
     if (i == _settings.end())
         throw UsageError("unknown setting '%s'", name);
     i->second.setting->set(value);
+    i->second.setting->overriden = true;
 }
 
 void Config::addSetting(AbstractSetting * setting)
@@ -22,6 +23,7 @@ void Config::addSetting(AbstractSetting * setting)
     auto i = initials.find(setting->name);
     if (i != initials.end()) {
         setting->set(i->second);
+        setting->overriden = true;
         initials.erase(i);
         set = true;
     }
@@ -34,6 +36,7 @@ void Config::addSetting(AbstractSetting * setting)
                     alias, setting->name);
             else {
                 setting->set(i->second);
+                setting->overriden = true;
                 initials.erase(i);
                 set = true;
             }
@@ -47,11 +50,11 @@ void Config::warnUnknownSettings()
         warn("unknown setting '%s'", i.first);
 }
 
-StringMap Config::getSettings()
+StringMap Config::getSettings(bool overridenOnly)
 {
     StringMap res;
     for (auto & opt : _settings)
-        if (!opt.second.isAlias)
+        if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
             res.emplace(opt.first, opt.second.setting->to_string());
     return res;
 }
@@ -94,6 +97,12 @@ void Config::applyConfigFile(const Path & path, bool fatal)
     } catch (SysError &) { }
 }
 
+void Config::resetOverriden()
+{
+    for (auto & s : _settings)
+        s.second.setting->overriden = false;
+}
+
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 6c8612f675c..952bf04b8a2 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -51,9 +51,11 @@ public:
 
     void warnUnknownSettings();
 
-    StringMap getSettings();
+    StringMap getSettings(bool overridenOnly = false);
 
     void applyConfigFile(const Path & path, bool fatal = false);
+
+    void resetOverriden();
 };
 
 class AbstractSetting
@@ -68,6 +70,8 @@ public:
 
     int created = 123;
 
+    bool overriden = false;
+
 protected:
 
     AbstractSetting(
@@ -78,7 +82,7 @@ protected:
     virtual ~AbstractSetting()
     {
         // Check against a gcc miscompilation causing our constructor
-        // not to run.
+        // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431).
         assert(created == 123);
     }
 
@@ -88,6 +92,8 @@ protected:
 
     bool parseBool(const std::string & str);
     std::string printBool(bool b);
+
+    bool isOverriden() { return overriden; }
 };
 
 struct DefaultSettingTag { };
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 0bd51afd1a9..37dfccd0669 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -429,6 +429,18 @@ Path getCacheDir()
 }
 
 
+Path getConfigDir()
+{
+    Path configDir = getEnv("XDG_CONFIG_HOME");
+    if (configDir.empty()) {
+        Path homeDir = getEnv("HOME");
+        if (homeDir.empty()) throw Error("$XDG_CONFIG_HOME and $HOME are not set");
+        configDir = homeDir + "/.config";
+    }
+    return configDir;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 0e6941e4a8d..121423cdaa3 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -110,9 +110,12 @@ void deletePath(const Path & path, unsigned long long & bytesFreed);
 Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
     bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
 
-/* Return the path to $XDG_CACHE_HOME/.cache. */
+/* Return $XDG_CACHE_HOME or $HOME/.cache. */
 Path getCacheDir();
 
+/* Return $XDG_CONFIG_HOME or $HOME/.config. */
+Path getConfigDir();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);

From 4410e9d995bcd53a7a4cff0bbee3917375adcba3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 16:52:53 +0200
Subject: [PATCH 0255/2196] Setting: Remove "Tag" template argument

---
 src/libstore/globals.cc      | 22 ++-------------
 src/libstore/globals.hh      | 39 +++++++++++++++++++++++---
 src/libutil/config.cc        | 53 ++++++++++++++----------------------
 src/libutil/config.hh        | 48 ++++++++++++++++++++------------
 src/nix-daemon/nix-daemon.cc |  2 +-
 5 files changed, 90 insertions(+), 74 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 6b9d077469c..3242ef9d635 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -73,7 +73,7 @@ unsigned int Settings::getDefaultCores()
 
 const string nixVersion = PACKAGE_VERSION;
 
-template<> void Setting<SandboxMode>::set(const std::string & str)
+template<> void BaseSetting<SandboxMode>::set(const std::string & str)
 {
     if (str == "true") value = smEnabled;
     else if (str == "relaxed") value = smRelaxed;
@@ -81,7 +81,7 @@ template<> void Setting<SandboxMode>::set(const std::string & str)
     else throw UsageError("option '%s' has invalid value '%s'", name, str);
 }
 
-template<> std::string Setting<SandboxMode>::to_string()
+template<> std::string BaseSetting<SandboxMode>::to_string()
 {
     if (value == smEnabled) return "true";
     else if (value == smRelaxed) return "relaxed";
@@ -89,27 +89,11 @@ template<> std::string Setting<SandboxMode>::to_string()
     else abort();
 }
 
-template<> void Setting<unsigned int, Settings::MaxBuildJobsTag>::set(const std::string & str)
+void MaxBuildJobsSetting::set(const std::string & str)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
     else if (!string2Int(str, value))
         throw UsageError("configuration setting ‘%s’ should be ‘auto’ or an integer", name);
 }
 
-template<> std::string Setting<unsigned int, Settings::MaxBuildJobsTag>::to_string()
-{
-    return std::to_string(value);
-}
-
-template<> void Setting<bool, Settings::CaseHackTag>::set(const std::string & str)
-{
-    value = parseBool(str);
-    nix::useCaseHack = true;
-}
-
-template<> std::string Setting<bool, Settings::CaseHackTag>::to_string()
-{
-    return printBool(value);
-}
-
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d3ecaadb639..b4f44de2e65 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -13,6 +13,39 @@ typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
 
 extern bool useCaseHack; // FIXME
 
+struct CaseHackSetting : public BaseSetting<bool>
+{
+    CaseHackSetting(Config * options,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<bool>(useCaseHack, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    void set(const std::string & str) override
+    {
+        BaseSetting<bool>::set(str);
+        nix::useCaseHack = true;
+    }
+};
+
+struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
+{
+    MaxBuildJobsSetting(Config * options,
+        unsigned int def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<unsigned int>(def, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    void set(const std::string & str) override;
+};
+
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -66,8 +99,7 @@ public:
        the log to show if a build fails. */
     size_t logLines = 10;
 
-    struct MaxBuildJobsTag { };
-    Setting<unsigned int, MaxBuildJobsTag> maxBuildJobs{this, 1, "build-max-jobs",
+    MaxBuildJobsSetting maxBuildJobs{this, 1, "build-max-jobs",
         "Maximum number of parallel build jobs. \"auto\" means use number of cores."};
 
     Setting<unsigned int> buildCores{this, getDefaultCores(), "build-cores",
@@ -268,8 +300,7 @@ public:
     Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
         "Whether the evaluator allows importing the result of a derivation."};
 
-    struct CaseHackTag { };
-    Setting<bool, CaseHackTag> useCaseHack{this, nix::useCaseHack, "use-case-hack",
+    CaseHackSetting useCaseHack{this, "use-case-hack",
         "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
 
     Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index bf137299718..72b6cf80684 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -111,83 +111,72 @@ AbstractSetting::AbstractSetting(
 {
 }
 
-template<> void Setting<std::string>::set(const std::string & str)
+template<> void BaseSetting<std::string>::set(const std::string & str)
 {
     value = str;
 }
 
-template<> std::string Setting<std::string>::to_string()
+template<> std::string BaseSetting<std::string>::to_string()
 {
     return value;
 }
 
-template<typename T, typename Tag>
-void Setting<T, Tag>::set(const std::string & str)
+template<typename T>
+void BaseSetting<T>::set(const std::string & str)
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     if (!string2Int(str, value))
         throw UsageError("setting '%s' has invalid value '%s'", name, str);
 }
 
-template<typename T, typename Tag>
-std::string Setting<T, Tag>::to_string()
+template<typename T>
+std::string BaseSetting<T>::to_string()
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     return std::to_string(value);
 }
 
-bool AbstractSetting::parseBool(const std::string & str)
+template<> void BaseSetting<bool>::set(const std::string & str)
 {
     if (str == "true" || str == "yes" || str == "1")
-        return true;
+        value = true;
     else if (str == "false" || str == "no" || str == "0")
-        return false;
+        value = false;
     else
         throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
 }
 
-template<> void Setting<bool>::set(const std::string & str)
+template<> std::string BaseSetting<bool>::to_string()
 {
-    value = parseBool(str);
+    return value ? "true" : "false";
 }
 
-std::string AbstractSetting::printBool(bool b)
-{
-    return b ? "true" : "false";
-}
-
-
-template<> std::string Setting<bool>::to_string()
-{
-    return printBool(value);
-}
-
-template<> void Setting<Strings>::set(const std::string & str)
+template<> void BaseSetting<Strings>::set(const std::string & str)
 {
     value = tokenizeString<Strings>(str);
 }
 
-template<> std::string Setting<Strings>::to_string()
+template<> std::string BaseSetting<Strings>::to_string()
 {
     return concatStringsSep(" ", value);
 }
 
-template<> void Setting<StringSet>::set(const std::string & str)
+template<> void BaseSetting<StringSet>::set(const std::string & str)
 {
     value = tokenizeString<StringSet>(str);
 }
 
-template<> std::string Setting<StringSet>::to_string()
+template<> std::string BaseSetting<StringSet>::to_string()
 {
     return concatStringsSep(" ", value);
 }
 
-template class Setting<int>;
-template class Setting<unsigned int>;
-template class Setting<long>;
-template class Setting<unsigned long>;
-template class Setting<long long>;
-template class Setting<unsigned long long>;
+template class BaseSetting<int>;
+template class BaseSetting<unsigned int>;
+template class BaseSetting<long>;
+template class BaseSetting<unsigned long>;
+template class BaseSetting<long long>;
+template class BaseSetting<unsigned long long>;
 
 void PathSetting::set(const std::string & str)
 {
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 952bf04b8a2..130f59e2bd8 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -90,17 +90,12 @@ protected:
 
     virtual std::string to_string() = 0;
 
-    bool parseBool(const std::string & str);
-    std::string printBool(bool b);
-
     bool isOverriden() { return overriden; }
 };
 
-struct DefaultSettingTag { };
-
 /* A setting of type T. */
-template<typename T, typename Tag = DefaultSettingTag>
-class Setting : public AbstractSetting
+template<typename T>
+class BaseSetting : public AbstractSetting
 {
 protected:
 
@@ -108,23 +103,21 @@ protected:
 
 public:
 
-    Setting(Config * options,
-        const T & def,
+    BaseSetting(const T & def,
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
         : AbstractSetting(name, description, aliases)
         , value(def)
-    {
-        options->addSetting(this);
-    }
+    { }
 
     operator const T &() const { return value; }
     operator T &() { return value; }
     const T & get() const { return value; }
     bool operator ==(const T & v2) const { return value == v2; }
     bool operator !=(const T & v2) const { return value != v2; }
-    void operator =(const T & v) { value = v; }
+    void operator =(const T & v) { assign(v); }
+    virtual void assign(const T & v) { value = v; }
 
     void set(const std::string & str) override;
 
@@ -132,18 +125,35 @@ public:
 };
 
 template<typename T>
-std::ostream & operator <<(std::ostream & str, const Setting<T> & opt)
+std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt)
 {
     str << (const T &) opt;
     return str;
 }
 
 template<typename T>
-bool operator ==(const T & v1, const Setting<T> & v2) { return v1 == (const T &) v2; }
+bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; }
+
+template<typename T>
+class Setting : public BaseSetting<T>
+{
+public:
+    Setting(Config * options,
+        const T & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {})
+        : BaseSetting<T>(def, name, description, aliases)
+    {
+        options->addSetting(this);
+    }
+
+    void operator =(const T & v) { this->assign(v); }
+};
 
 /* A special setting for Paths. These are automatically canonicalised
    (e.g. "/foo//bar/" becomes "/foo/bar"). */
-class PathSetting : public Setting<Path>
+class PathSetting : public BaseSetting<Path>
 {
     bool allowEmpty;
 
@@ -155,15 +165,17 @@ public:
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : Setting<Path>(options, def, name, description, aliases)
+        : BaseSetting<Path>(def, name, description, aliases)
         , allowEmpty(allowEmpty)
     {
-        set(value);
+        options->addSetting(this);
     }
 
     void set(const std::string & str) override;
 
     Path operator +(const char * p) const { return value + p; }
+
+    void operator =(const Path & v) { this->assign(v); }
 };
 
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 5c2641eac6f..07ad0b45b3e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -440,7 +440,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         settings.keepGoing = readInt(from);
         settings.tryFallback = readInt(from);
         verbosity = (Verbosity) readInt(from);
-        settings.maxBuildJobs = readInt(from);
+        settings.maxBuildJobs.assign(readInt(from));
         settings.maxSilentTime = readInt(from);
         settings.useBuildHook = readInt(from) != 0;
         settings.verboseBuild = lvlError == (Verbosity) readInt(from);

From efa4bdbfcd1489527bcf6f20a49c9a3bca8bbf6b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 17:34:47 +0200
Subject: [PATCH 0256/2196] Improve nix show-config --json

In particular, show descriptions. This could be used for manpage
generation etc.
---
 src/libstore/globals.cc |  5 +++++
 src/libutil/config.cc   | 38 ++++++++++++++++++++++++++++++++++++
 src/libutil/config.hh   |  8 ++++++++
 src/libutil/json.cc     | 43 +++++++++++++----------------------------
 src/libutil/json.hh     | 11 +++--------
 src/nix/show-config.cc  |  3 +--
 6 files changed, 68 insertions(+), 40 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 3242ef9d635..953bf6aaaa0 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -89,6 +89,11 @@ template<> std::string BaseSetting<SandboxMode>::to_string()
     else abort();
 }
 
+template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
+{
+    AbstractSetting::toJSON(out);
+}
+
 void MaxBuildJobsSetting::set(const std::string & str)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 72b6cf80684..62c6433c741 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -1,5 +1,6 @@
 #include "config.hh"
 #include "args.hh"
+#include "json.hh"
 
 namespace nix {
 
@@ -103,6 +104,17 @@ void Config::resetOverriden()
         s.second.setting->overriden = false;
 }
 
+void Config::toJSON(JSONObject & out)
+{
+    for (auto & s : _settings)
+        if (!s.second.isAlias) {
+            JSONObject out2(out.object(s.first));
+            out2.attr("description", s.second.setting->description);
+            JSONPlaceholder out3(out2.placeholder("value"));
+            s.second.setting->toJSON(out3);
+        }
+}
+
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
@@ -111,6 +123,17 @@ AbstractSetting::AbstractSetting(
 {
 }
 
+void AbstractSetting::toJSON(JSONPlaceholder & out)
+{
+    out.write(to_string());
+}
+
+template<typename T>
+void BaseSetting<T>::toJSON(JSONPlaceholder & out)
+{
+    out.write(value);
+}
+
 template<> void BaseSetting<std::string>::set(const std::string & str)
 {
     value = str;
@@ -161,6 +184,13 @@ template<> std::string BaseSetting<Strings>::to_string()
     return concatStringsSep(" ", value);
 }
 
+template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out)
+{
+    JSONList list(out.list());
+    for (auto & s : value)
+        list.elem(s);
+}
+
 template<> void BaseSetting<StringSet>::set(const std::string & str)
 {
     value = tokenizeString<StringSet>(str);
@@ -171,12 +201,20 @@ template<> std::string BaseSetting<StringSet>::to_string()
     return concatStringsSep(" ", value);
 }
 
+template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out)
+{
+    JSONList list(out.list());
+    for (auto & s : value)
+        list.elem(s);
+}
+
 template class BaseSetting<int>;
 template class BaseSetting<unsigned int>;
 template class BaseSetting<long>;
 template class BaseSetting<unsigned long>;
 template class BaseSetting<long long>;
 template class BaseSetting<unsigned long long>;
+template class BaseSetting<bool>;
 
 void PathSetting::set(const std::string & str)
 {
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 130f59e2bd8..91962109100 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -9,6 +9,8 @@ namespace nix {
 
 class Args;
 class AbstractSetting;
+class JSONPlaceholder;
+class JSONObject;
 
 /* A class to simplify providing configuration settings. The typical
    use is to inherit Config and add Setting<T> members:
@@ -56,6 +58,8 @@ public:
     void applyConfigFile(const Path & path, bool fatal = false);
 
     void resetOverriden();
+
+    void toJSON(JSONObject & out);
 };
 
 class AbstractSetting
@@ -90,6 +94,8 @@ protected:
 
     virtual std::string to_string() = 0;
 
+    virtual void toJSON(JSONPlaceholder & out);
+
     bool isOverriden() { return overriden; }
 };
 
@@ -122,6 +128,8 @@ public:
     void set(const std::string & str) override;
 
     std::string to_string() override;
+
+    void toJSON(JSONPlaceholder & out) override;
 };
 
 template<typename T>
diff --git a/src/libutil/json.cc b/src/libutil/json.cc
index 6023d1d4fb8..b8b8ef9c8cc 100644
--- a/src/libutil/json.cc
+++ b/src/libutil/json.cc
@@ -19,49 +19,32 @@ void toJSON(std::ostream & str, const char * start, const char * end)
     str << '"';
 }
 
-void toJSON(std::ostream & str, const std::string & s)
-{
-    toJSON(str, s.c_str(), s.c_str() + s.size());
-}
-
 void toJSON(std::ostream & str, const char * s)
 {
     if (!s) str << "null"; else toJSON(str, s, s + strlen(s));
 }
 
-void toJSON(std::ostream & str, unsigned long long n)
-{
-    str << n;
-}
-
-void toJSON(std::ostream & str, unsigned long n)
-{
-    str << n;
-}
-
-void toJSON(std::ostream & str, long n)
-{
-    str << n;
-}
+template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
+template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; }
+template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; }
+template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; }
+template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
+template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
+template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
 
-void toJSON(std::ostream & str, unsigned int n)
+template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
 {
-    str << n;
-}
-
-void toJSON(std::ostream & str, int n)
-{
-    str << n;
+    toJSON(str, s.c_str(), s.c_str() + s.size());
 }
 
-void toJSON(std::ostream & str, double f)
+template<> void toJSON<bool>(std::ostream & str, const bool & b)
 {
-    str << f;
+    str << (b ? "true" : "false");
 }
 
-void toJSON(std::ostream & str, bool b)
+template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b)
 {
-    str << (b ? "true" : "false");
+    str << "null";
 }
 
 JSONWriter::JSONWriter(std::ostream & str, bool indent)
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
index 03eecb73258..595e9bbe349 100644
--- a/src/libutil/json.hh
+++ b/src/libutil/json.hh
@@ -7,15 +7,10 @@
 namespace nix {
 
 void toJSON(std::ostream & str, const char * start, const char * end);
-void toJSON(std::ostream & str, const std::string & s);
 void toJSON(std::ostream & str, const char * s);
-void toJSON(std::ostream & str, unsigned long long n);
-void toJSON(std::ostream & str, unsigned long n);
-void toJSON(std::ostream & str, long n);
-void toJSON(std::ostream & str, unsigned int n);
-void toJSON(std::ostream & str, int n);
-void toJSON(std::ostream & str, double f);
-void toJSON(std::ostream & str, bool b);
+
+template<typename T>
+void toJSON(std::ostream & str, const T & n);
 
 class JSONWriter
 {
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index ba39e2bb29b..aade2adeace 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -31,8 +31,7 @@ struct CmdShowConfig : Command
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
             JSONObject jsonObj(std::cout, true);
-            for (auto & s : settings.getSettings())
-                jsonObj.attr(s.first, s.second);
+            settings.toJSON(jsonObj);
         } else {
             for (auto & s : settings.getSettings())
                 std::cout << s.first + " = " + s.second + "\n";

From 749696e71c9d637a36ab09a1368dff01b2b1bda8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Apr 2017 19:11:45 +0200
Subject: [PATCH 0257/2196] Detect lsof

Also, don't use lsof on Linux since it's not needed.

Fixes #1328.
---
 Makefile.config.in    |  1 +
 configure.ac          |  1 +
 src/libstore/gc.cc    | 11 ++++++++---
 src/libstore/local.mk |  3 ++-
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 53dca1fcf10..6948dad5a60 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -15,6 +15,7 @@ SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
 bro = @bro@
+lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
 docdir = @docdir@
diff --git a/configure.ac b/configure.ac
index 3e6a894e3b1..c7026cf954d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -128,6 +128,7 @@ NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
 AC_PATH_PROG(bro, bro, bro)
+AC_PATH_PROG(lsof, lsof, lsof)
 
 
 NEED_PROG(cat, cat)
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0b03d61a789..b6d462d2bff 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -426,22 +426,27 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
             throw SysError("iterating /proc");
     }
 
+#if !defined(__linux__)
     try {
-        auto lsofRegex = std::regex(R"(^n(/.*)$)");
+        printError("RUN LSOF %s", LSOF);
+        std::regex lsofRegex(R"(^n(/.*)$)");
         auto lsofLines =
-            tokenizeString<std::vector<string>>(runProgram("lsof", true, { "-n", "-w", "-F", "n" }), "\n");
+            tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
         for (const auto & line : lsofLines) {
-            auto match = std::smatch{};
+            std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
                 paths.emplace(match[1]);
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
     }
+#endif
 
+#if defined(__linux__)
     readFileRoots("/proc/sys/kernel/modprobe", paths);
     readFileRoots("/proc/sys/kernel/fbsplash", paths);
     readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
+#endif
 
     for (auto & i : paths)
         if (isInStore(i)) {
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 9d5c04dca0c..4da20330cf3 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -27,7 +27,8 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
- -DBASH_PATH="\"$(bash)\""
+ -DBASH_PATH="\"$(bash)\"" \
+ -DLSOF=\"$(lsof)\"
 
 $(d)/local-store.cc: $(d)/schema.sql.hh
 

From 111d347237d7f313ab9acfafc0de859715c31564 Mon Sep 17 00:00:00 2001
From: James Earl Douglas <james@earldouglas.com>
Date: Thu, 20 Apr 2017 18:07:23 -0600
Subject: [PATCH 0258/2196] Drop misleading ellipses

This portion of the quick start guide may lead to confusion for
newcomers to Nix.  This change clarifies the example to one that can be
copied in its entirety.
---
 doc/manual/introduction/quick-start.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml
index 5ae9f6ad543..aa239b7538b 100644
--- a/doc/manual/introduction/quick-start.xml
+++ b/doc/manual/introduction/quick-start.xml
@@ -40,7 +40,7 @@ libxslt-1.1.28
 <step><para>Install some packages from the channel:
 
 <screen>
-$ nix-env -i hello <replaceable>...</replaceable> </screen>
+$ nix-env -i hello</screen>
 
 This should download pre-built packages; it should not build them
 locally (if it does, something went wrong).</para></step>

From d35231ec6017d72492a23b882b9275a60e1737a6 Mon Sep 17 00:00:00 2001
From: David McFarland <davidm@hb-studios.com>
Date: Fri, 21 Apr 2017 11:27:27 -0300
Subject: [PATCH 0259/2196] set _GNU_SOURCE on cygwin

this is needed for pipe2()
---
 mk/lib.mk | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mk/lib.mk b/mk/lib.mk
index bb82801d3b4..1da51d87973 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -53,8 +53,8 @@ BUILD_SHARED_LIBS ?= 1
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
   ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
-    GLOBAL_CFLAGS += -U__STRICT_ANSI__
-    GLOBAL_CXXFLAGS += -U__STRICT_ANSI__
+    GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
+    GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
   else
     GLOBAL_CFLAGS += -fPIC
     GLOBAL_CXXFLAGS += -fPIC

From 804ac524891ead6aa70e27c093686b71391ea232 Mon Sep 17 00:00:00 2001
From: David McFarland <davidm@hb-studios.com>
Date: Fri, 21 Apr 2017 11:28:10 -0300
Subject: [PATCH 0260/2196] add helper function to set 'interruptThrown'

this fixes a linker failure on cygwin 64 due to some bad
interaction between tls and shared libraries.

see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64697
---
 src/libmain/shared.cc | 2 +-
 src/libutil/util.cc   | 7 ++++++-
 src/libutil/util.hh   | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 4747b9bf9b4..d6c1c0c9cb4 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -259,7 +259,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
                condition is discharged before we reach printMsg()
                below, since otherwise it will throw an (uncaught)
                exception. */
-            interruptThrown = true;
+            setInterruptThrown();
             throw;
         }
     } catch (Exit & e) {
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 37dfccd0669..88a2f752c02 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -946,7 +946,12 @@ void closeOnExec(int fd)
 
 bool _isInterrupted = false;
 
-thread_local bool interruptThrown = false;
+static thread_local bool interruptThrown = false;
+
+void setInterruptThrown()
+{
+    interruptThrown = true;
+}
 
 void _interrupted()
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 121423cdaa3..ae40dcd4cd2 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -267,7 +267,7 @@ void closeOnExec(int fd);
 
 extern bool _isInterrupted;
 
-extern thread_local bool interruptThrown;
+void setInterruptThrown();
 
 void _interrupted();
 

From 9b63bb88c8873d192b8b01608e5d230817dd3375 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Apr 2017 12:04:01 +0200
Subject: [PATCH 0261/2196] nix-shell -p: Use runCommandCC

This restores pre-17.03 behaviour by making gcc available.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b4206033cf5..bb031d51513 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -325,7 +325,7 @@ int main(int argc, char ** argv)
         if (packages) {
             instArgs.push_back("--expr");
             std::ostringstream joined;
-            joined << "with import <nixpkgs> { }; runCommand \"shell\" { buildInputs = [ ";
+            joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ ";
             for (const auto & i : exprs)
                 joined << '(' << i << ") ";
             joined << "]; } \"\"";

From 66577a1c64ac5d9f07aa2c207c96e13077576a4e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Apr 2017 14:21:36 +0200
Subject: [PATCH 0262/2196] Factor out --json

---
 src/libmain/common-args.hh | 12 +++++++++++-
 src/nix/path-info.cc       |  5 ++---
 src/nix/show-config.cc     |  5 +----
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh
index 2c0d71edd81..a4de3dccf0a 100644
--- a/src/libmain/common-args.hh
+++ b/src/libmain/common-args.hh
@@ -12,7 +12,7 @@ struct MixCommonArgs : virtual Args
 
 struct MixDryRun : virtual Args
 {
-    bool dryRun;
+    bool dryRun = false;
 
     MixDryRun()
     {
@@ -20,4 +20,14 @@ struct MixDryRun : virtual Args
     }
 };
 
+struct MixJSON : virtual Args
+{
+    bool json = false;
+
+    MixJSON()
+    {
+        mkFlag(0, "json", "produce JSON output", &json);
+    }
+};
+
 }
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 0f9a1125f2e..30b193798f6 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -2,25 +2,24 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "json.hh"
+#include "common-args.hh"
 
 #include <iomanip>
 #include <algorithm>
 
 using namespace nix;
 
-struct CmdPathInfo : StorePathsCommand
+struct CmdPathInfo : StorePathsCommand, MixJSON
 {
     bool showSize = false;
     bool showClosureSize = false;
     bool showSigs = false;
-    bool json = false;
 
     CmdPathInfo()
     {
         mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
         mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
         mkFlag(0, "sigs", "show signatures", &showSigs);
-        mkFlag(0, "json", "produce JSON output", &json);
     }
 
     std::string name() override
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index aade2adeace..e354891a82e 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -7,13 +7,10 @@
 
 using namespace nix;
 
-struct CmdShowConfig : Command
+struct CmdShowConfig : Command, MixJSON
 {
-    bool json = false;
-
     CmdShowConfig()
     {
-        mkFlag(0, "json", "produce JSON output", &json);
     }
 
     std::string name() override

From 1bb87c0487ba2a10f20c07dfd828b5d043249e31 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Apr 2017 15:01:28 +0200
Subject: [PATCH 0263/2196] Remove debug statement

---
 src/libstore/gc.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index b6d462d2bff..3e7e42cbc96 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -428,7 +428,6 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
 
 #if !defined(__linux__)
     try {
-        printError("RUN LSOF %s", LSOF);
         std::regex lsofRegex(R"(^n(/.*)$)");
         auto lsofLines =
             tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");

From bcecc990071fd36bb88c8fd29cb009ed4c04d6a2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 11:20:37 +0200
Subject: [PATCH 0264/2196] Restructure installables handling in the "nix"
 command

---
 src/libexpr/value.hh    |   8 ++
 src/nix/build.cc        |  19 +---
 src/nix/command.cc      |   7 ++
 src/nix/command.hh      |   4 +
 src/nix/installables.cc | 233 +++++++++++++++++++++++++++++++---------
 src/nix/installables.hh |  55 ++++++----
 src/nix/log.cc          |  35 +++---
 src/nix/path-info.cc    |   1 -
 src/nix/run.cc          |  19 +---
 9 files changed, 257 insertions(+), 124 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 802e8ed2ee7..9df516f062e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -220,6 +220,14 @@ static inline void mkApp(Value & v, Value & left, Value & right)
 }
 
 
+static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
+{
+    v.type = tPrimOpApp;
+    v.app.left = &left;
+    v.app.right = &right;
+}
+
+
 static inline void mkStringNoCopy(Value & v, const char * s)
 {
     v.type = tString;
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 812464d7582..0a34c68f881 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -6,7 +6,7 @@
 
 using namespace nix;
 
-struct CmdBuild : StoreCommand, MixDryRun, MixInstallables
+struct CmdBuild : MixDryRun, MixInstallables
 {
     CmdBuild()
     {
@@ -24,22 +24,9 @@ struct CmdBuild : StoreCommand, MixDryRun, MixInstallables
 
     void run(ref<Store> store) override
     {
-        auto elems = evalInstallables(store);
+        auto paths = buildInstallables(store, dryRun);
 
-        PathSet pathsToBuild;
-
-        for (auto & elem : elems) {
-            if (elem.isDrv)
-                pathsToBuild.insert(elem.drvPath);
-            else
-                pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end());
-        }
-
-        printMissing(store, pathsToBuild);
-
-        if (dryRun) return;
-
-        store->buildPaths(pathsToBuild);
+        printInfo("build result: %s", showPaths(paths));
     }
 };
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index a1b2c120a5d..4034de96c16 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -79,6 +79,13 @@ StoreCommand::StoreCommand()
     mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri);
 }
 
+ref<Store> StoreCommand::getStore()
+{
+    if (!_store)
+        _store = createStore();
+    return ref<Store>(_store);
+}
+
 ref<Store> StoreCommand::createStore()
 {
     return openStore(storeUri);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index fa6c21abf8a..bb667ee325c 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -33,8 +33,12 @@ struct StoreCommand : virtual Command
     std::string storeUri;
     StoreCommand();
     void run() override;
+    ref<Store> getStore();
     virtual ref<Store> createStore();
     virtual void run(ref<Store>) = 0;
+
+private:
+    std::shared_ptr<Store> _store;
 };
 
 /* A command that operates on zero or more store paths. */
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 8341bbc5a3a..70007d62a29 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -6,16 +6,21 @@
 #include "get-drvs.hh"
 #include "installables.hh"
 #include "store-api.hh"
+#include "shared.hh"
+
+#include <regex>
 
 namespace nix {
 
-Value * MixInstallables::buildSourceExpr(EvalState & state)
+Value * MixInstallables::getSourceExpr(EvalState & state)
 {
-    Value * vRoot = state.allocValue();
+    if (vSourceExpr) return vSourceExpr;
+
+    vSourceExpr = state.allocValue();
 
     if (file != "") {
         Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file)));
-        state.eval(e, *vRoot);
+        state.eval(e, *vSourceExpr);
     }
 
     else {
@@ -24,7 +29,7 @@ Value * MixInstallables::buildSourceExpr(EvalState & state)
 
         auto searchPath = state.getSearchPath();
 
-        state.mkAttrs(*vRoot, searchPath.size());
+        state.mkAttrs(*vSourceExpr, searchPath.size());
 
         std::unordered_set<std::string> seen;
 
@@ -32,76 +37,208 @@ Value * MixInstallables::buildSourceExpr(EvalState & state)
             if (i.first == "") continue;
             if (seen.count(i.first)) continue;
             seen.insert(i.first);
-            if (!pathExists(i.second)) continue;
-            mkApp(*state.allocAttr(*vRoot, state.symbols.create(i.first)),
+#if 0
+            auto res = state.resolveSearchPathElem(i);
+            if (!res.first) continue;
+            if (!pathExists(res.second)) continue;
+            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)),
                 state.getBuiltin("import"),
-                mkString(*state.allocValue(), i.second));
+                mkString(*state.allocValue(), res.second));
+#endif
+            Value * v1 = state.allocValue();
+            mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath"));
+            Value * v2 = state.allocValue();
+            mkApp(*v2, *v1, mkString(*state.allocValue(), i.first));
+            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)),
+                state.getBuiltin("import"), *v2);
         }
 
-        vRoot->attrs->sort();
+        vSourceExpr->attrs->sort();
     }
 
-    return vRoot;
+    return vSourceExpr;
 }
 
-UserEnvElems MixInstallables::evalInstallables(ref<Store> store)
+struct InstallableStoreDrv : Installable
+{
+    Path storePath;
+
+    InstallableStoreDrv(const Path & storePath) : storePath(storePath) { }
+
+    std::string what() override { return storePath; }
+
+    PathSet toBuildable() override
+    {
+        return {storePath};
+    }
+};
+
+struct InstallableStorePath : Installable
 {
-    UserEnvElems res;
+    Path storePath;
+
+    InstallableStorePath(const Path & storePath) : storePath(storePath) { }
+
+    std::string what() override { return storePath; }
+
+    PathSet toBuildable() override
+    {
+        return {storePath};
+    }
+};
+
+struct InstallableExpr : Installable
+{
+    MixInstallables & installables;
+    std::string text;
+
+    InstallableExpr(MixInstallables & installables, const std::string & text)
+         : installables(installables), text(text) { }
+
+    std::string what() override { return text; }
+
+    PathSet toBuildable() override
+    {
+        auto state = installables.getEvalState();
+
+        auto v = toValue(*state);
+
+        // FIXME
+        std::map<string, string> autoArgs_;
+        Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_));
+
+        DrvInfos drvs;
+        getDerivations(*state, *v, "", autoArgs, drvs, false);
+
+        PathSet res;
+
+        for (auto & i : drvs)
+            res.insert(i.queryDrvPath());
+
+        return res;
+    }
+
+    Value * toValue(EvalState & state) override
+    {
+        auto v = state.allocValue();
+        state.eval(state.parseExprFromString(text, absPath(".")), *v);
+        return v;
+    }
+};
+
+struct InstallableAttrPath : Installable
+{
+    MixInstallables & installables;
+    std::string attrPath;
+
+    InstallableAttrPath(MixInstallables & installables, const std::string & attrPath)
+        : installables(installables), attrPath(attrPath)
+    { }
+
+    std::string what() override { return attrPath; }
+
+    PathSet toBuildable() override
+    {
+        auto state = installables.getEvalState();
+
+        auto v = toValue(*state);
+
+        // FIXME
+        std::map<string, string> autoArgs_;
+        Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_));
+
+        DrvInfos drvs;
+        getDerivations(*state, *v, "", autoArgs, drvs, false);
+
+        PathSet res;
+
+        for (auto & i : drvs)
+            res.insert(i.queryDrvPath());
+
+        return res;
+    }
+
+    Value * toValue(EvalState & state) override
+    {
+        auto source = installables.getSourceExpr(state);
+
+        // FIXME
+        std::map<string, string> autoArgs_;
+        Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+
+        Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source);
+        state.forceValue(*v);
+
+        return v;
+    }
+};
+
+// FIXME: extend
+std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
+static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
+
+std::vector<std::shared_ptr<Installable>> MixInstallables::parseInstallables(ref<Store> store, Strings installables)
+{
+    std::vector<std::shared_ptr<Installable>> result;
 
     for (auto & installable : installables) {
 
         if (std::string(installable, 0, 1) == "/") {
 
             if (store->isStorePath(installable)) {
+                if (isDerivation(installable))
+                    result.push_back(std::make_shared<InstallableStoreDrv>(installable));
+                else
+                    result.push_back(std::make_shared<InstallableStorePath>(installable));
+            }
 
-                if (isDerivation(installable)) {
-                    UserEnvElem elem;
-                    // FIXME: handle empty case, drop version
-                    elem.attrPath = {storePathToName(installable)};
-                    elem.isDrv = true;
-                    elem.drvPath = installable;
-                    res.push_back(elem);
-                }
-
-                else {
-                    UserEnvElem elem;
-                    // FIXME: handle empty case, drop version
-                    elem.attrPath = {storePathToName(installable)};
-                    elem.isDrv = false;
-                    elem.outPaths = {installable};
-                    res.push_back(elem);
-                }
+            else {
+                result.push_back(std::make_shared<InstallableStorePath>(
+                        store->toStorePath(store->followLinksToStore(installable))));
             }
 
-            else
-                throw UsageError(format("don't know what to do with ‘%1%’") % installable);
         }
 
-        else {
-
-            EvalState state({}, store);
+        else if (installable.compare(0, 1, "(") == 0)
+            result.push_back(std::make_shared<InstallableExpr>(*this, installable));
 
-            auto vRoot = buildSourceExpr(state);
+        else if (std::regex_match(installable, attrPathRegex))
+            result.push_back(std::make_shared<InstallableAttrPath>(*this, installable));
 
-            std::map<string, string> autoArgs_;
-            Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+        else
+            throw UsageError("don't know what to do with argument ‘%s’", installable);
+    }
 
-            Value & v(*findAlongAttrPath(state, installable, autoArgs, *vRoot));
-            state.forceValue(v);
+    return result;
+}
 
-            DrvInfos drvs;
-            getDerivations(state, v, "", autoArgs, drvs, false);
+PathSet MixInstallables::buildInstallables(ref<Store> store, bool dryRun)
+{
+    PathSet buildables;
 
-            for (auto & i : drvs) {
-                UserEnvElem elem;
-                elem.isDrv = true;
-                elem.drvPath = i.queryDrvPath();
-                res.push_back(elem);
-            }
-        }
+    for (auto & i : installables) {
+        auto b = i->toBuildable();
+        buildables.insert(b.begin(), b.end());
     }
 
-    return res;
+    printMissing(store, buildables);
+
+    if (!dryRun)
+        store->buildPaths(buildables);
+
+    return buildables;
+}
+
+ref<EvalState> MixInstallables::getEvalState()
+{
+    if (!evalState)
+        evalState = std::make_shared<EvalState>(Strings{}, getStore());
+    return ref<EvalState>(evalState);
+}
+
+void MixInstallables::prepare()
+{
+    installables = parseInstallables(getStore(), _installables);
 }
 
 }
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index a58f7dc59bb..5f0b0a29224 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -1,48 +1,61 @@
 #pragma once
 
 #include "args.hh"
+#include "command.hh"
 
 namespace nix {
 
-struct UserEnvElem
-{
-    Strings attrPath;
+struct Value;
+class EvalState;
+class Expr;
 
-    // FIXME: should use boost::variant or so.
-    bool isDrv;
+struct Installable
+{
+    virtual std::string what() = 0;
 
-    // Derivation case:
-    Path drvPath;
-    StringSet outputNames;
+    virtual PathSet toBuildable()
+    {
+        throw Error("argument ‘%s’ cannot be built", what());
+    }
 
-    // Non-derivation case:
-    PathSet outPaths;
+    virtual Value * toValue(EvalState & state)
+    {
+        throw Error("argument ‘%s’ cannot be evaluated", what());
+    }
 };
 
-typedef std::vector<UserEnvElem> UserEnvElems;
-
-struct Value;
-class EvalState;
-
-struct MixInstallables : virtual Args
+struct MixInstallables : virtual Args, StoreCommand
 {
-    Strings installables;
+    std::vector<std::shared_ptr<Installable>> installables;
     Path file;
 
     MixInstallables()
     {
         mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file);
-        expectArgs("installables", &installables);
+        expectArgs("installables", &_installables);
     }
 
-    UserEnvElems evalInstallables(ref<Store> store);
-
     /* Return a value representing the Nix expression from which we
        are installing. This is either the file specified by ‘--file’,
        or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs
        = import ...; bla = import ...; }’. */
-    Value * buildSourceExpr(EvalState & state);
+    Value * getSourceExpr(EvalState & state);
+
+    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables);
+
+    PathSet buildInstallables(ref<Store> store, bool dryRun);
+
+    ref<EvalState> getEvalState();
+
+    void prepare() override;
+
+private:
+
+    Strings _installables;
+
+    std::shared_ptr<EvalState> evalState;
 
+    Value * vSourceExpr = 0;
 };
 
 }
diff --git a/src/nix/log.cc b/src/nix/log.cc
index d8a3830e91c..75f3c1ab0d6 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -6,7 +6,7 @@
 
 using namespace nix;
 
-struct CmdLog : StoreCommand, MixInstallables
+struct CmdLog : MixInstallables
 {
     CmdLog()
     {
@@ -24,32 +24,23 @@ struct CmdLog : StoreCommand, MixInstallables
 
     void run(ref<Store> store) override
     {
-        auto elems = evalInstallables(store);
-
-        PathSet paths;
-
-        for (auto & elem : elems) {
-            if (elem.isDrv)
-                paths.insert(elem.drvPath);
-            else
-                paths.insert(elem.outPaths.begin(), elem.outPaths.end());
-        }
-
         auto subs = getDefaultSubstituters();
 
         subs.push_front(store);
 
-        for (auto & path : paths) {
-            bool found = false;
-            for (auto & sub : subs) {
-                auto log = sub->getBuildLog(path);
-                if (!log) continue;
-                std::cout << *log;
-                found = true;
-                break;
+        for (auto & inst : installables) {
+            for (auto & path : inst->toBuildable()) {
+                bool found = false;
+                for (auto & sub : subs) {
+                    auto log = sub->getBuildLog(path);
+                    if (!log) continue;
+                    std::cout << *log;
+                    found = true;
+                    break;
+                }
+                if (!found)
+                    throw Error("build log of path ‘%s’ is not available", path);
             }
-            if (!found)
-                throw Error("build log of path ‘%s’ is not available", path);
         }
     }
 };
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 30b193798f6..f1620923861 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -99,7 +99,6 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             }
 
         }
-
     }
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index a30031ad07b..f3333b77780 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -13,7 +13,7 @@
 
 using namespace nix;
 
-struct CmdRun : StoreCommand, MixInstallables
+struct CmdRun : MixInstallables
 {
     CmdRun()
     {
@@ -31,20 +31,7 @@ struct CmdRun : StoreCommand, MixInstallables
 
     void run(ref<Store> store) override
     {
-        auto elems = evalInstallables(store);
-
-        PathSet pathsToBuild;
-
-        for (auto & elem : elems) {
-            if (elem.isDrv)
-                pathsToBuild.insert(elem.drvPath);
-            else
-                pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end());
-        }
-
-        printMissing(store, pathsToBuild);
-
-        store->buildPaths(pathsToBuild);
+        auto paths = buildInstallables(store, false);
 
         auto store2 = store.dynamic_pointer_cast<LocalStore>();
 
@@ -104,7 +91,7 @@ struct CmdRun : StoreCommand, MixInstallables
         }
 
         PathSet outPaths;
-        for (auto & path : pathsToBuild)
+        for (auto & path : paths)
             if (isDerivation(path)) {
                 Derivation drv = store->derivationFromPath(path);
                 for (auto & output : drv.outputs)

From 6267d748891b3c6e6a41b5bd1f6684ae8b88f31c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 11:23:47 +0200
Subject: [PATCH 0265/2196] Add "nix eval" command

This replaces "nix-instantiate --eval". The result is evaluated
strictly since this seems more useful.
---
 src/nix/eval.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 src/nix/eval.cc

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
new file mode 100644
index 00000000000..7a6bf07c353
--- /dev/null
+++ b/src/nix/eval.cc
@@ -0,0 +1,44 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "installables.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "eval.hh"
+#include "json.hh"
+#include "value-to-json.hh"
+
+using namespace nix;
+
+struct CmdEval : MixJSON, MixInstallables
+{
+    std::string name() override
+    {
+        return "eval";
+    }
+
+    std::string description() override
+    {
+        return "evaluate a Nix expression";
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto state = getEvalState();
+
+        auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr;
+
+        for (auto & i : installables) {
+            auto v = i->toValue(*state);
+            if (json) {
+                PathSet context;
+                auto jsonElem = jsonOut->placeholder();
+                printValueAsJSON(*state, true, *v, jsonElem, context);
+            } else {
+                state->forceValueDeep(*v);
+                std::cout << *v << "\n";
+            }
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdEval>());

From c769841bc4ecb9dd3d8456931fec78e102c3832f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 12:06:32 +0200
Subject: [PATCH 0266/2196] Move code around

---
 src/nix/build.cc        |  3 +-
 src/nix/command.hh      | 54 ++++++++++++++++++++++++++++++++++++
 src/nix/eval.cc         |  3 +-
 src/nix/installables.cc | 20 +++++++-------
 src/nix/installables.hh | 61 -----------------------------------------
 src/nix/log.cc          |  3 +-
 src/nix/run.cc          |  3 +-
 src/nix/show-config.cc  |  1 -
 8 files changed, 68 insertions(+), 80 deletions(-)
 delete mode 100644 src/nix/installables.hh

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0a34c68f881..00bda1fd104 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -1,12 +1,11 @@
 #include "command.hh"
 #include "common-args.hh"
-#include "installables.hh"
 #include "shared.hh"
 #include "store-api.hh"
 
 using namespace nix;
 
-struct CmdBuild : MixDryRun, MixInstallables
+struct CmdBuild : MixDryRun, InstallablesCommand
 {
     CmdBuild()
     {
diff --git a/src/nix/command.hh b/src/nix/command.hh
index bb667ee325c..ee9485e5dd1 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -4,6 +4,9 @@
 
 namespace nix {
 
+struct Value;
+class EvalState;
+
 /* A command is an argument parser that can be executed by calling its
    run() method. */
 struct Command : virtual Args
@@ -61,6 +64,57 @@ public:
     void run(ref<Store> store) override;
 };
 
+struct Installable
+{
+    virtual std::string what() = 0;
+
+    virtual PathSet toBuildable()
+    {
+        throw Error("argument ‘%s’ cannot be built", what());
+    }
+
+    virtual Value * toValue(EvalState & state)
+    {
+        throw Error("argument ‘%s’ cannot be evaluated", what());
+    }
+};
+
+/* A command that operates on a list of "installables", which can be
+   store paths, attribute paths, Nix expressions, etc. */
+struct InstallablesCommand : virtual Args, StoreCommand
+{
+    std::vector<std::shared_ptr<Installable>> installables;
+    Path file;
+
+    InstallablesCommand()
+    {
+        mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file);
+        expectArgs("installables", &_installables);
+    }
+
+    /* Return a value representing the Nix expression from which we
+       are installing. This is either the file specified by ‘--file’,
+       or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs
+       = import ...; bla = import ...; }’. */
+    Value * getSourceExpr(EvalState & state);
+
+    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables);
+
+    PathSet buildInstallables(ref<Store> store, bool dryRun);
+
+    ref<EvalState> getEvalState();
+
+    void prepare() override;
+
+private:
+
+    Strings _installables;
+
+    std::shared_ptr<EvalState> evalState;
+
+    Value * vSourceExpr = 0;
+};
+
 typedef std::map<std::string, ref<Command>> Commands;
 
 /* An argument parser that supports multiple subcommands,
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 7a6bf07c353..eb2b13a2dcd 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -1,6 +1,5 @@
 #include "command.hh"
 #include "common-args.hh"
-#include "installables.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "eval.hh"
@@ -9,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdEval : MixJSON, MixInstallables
+struct CmdEval : MixJSON, InstallablesCommand
 {
     std::string name() override
     {
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 70007d62a29..3cf4a6f8d32 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -1,10 +1,10 @@
+#include "command.hh"
 #include "attr-path.hh"
 #include "common-opts.hh"
 #include "derivations.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "get-drvs.hh"
-#include "installables.hh"
 #include "store-api.hh"
 #include "shared.hh"
 
@@ -12,7 +12,7 @@
 
 namespace nix {
 
-Value * MixInstallables::getSourceExpr(EvalState & state)
+Value * InstallablesCommand::getSourceExpr(EvalState & state)
 {
     if (vSourceExpr) return vSourceExpr;
 
@@ -89,10 +89,10 @@ struct InstallableStorePath : Installable
 
 struct InstallableExpr : Installable
 {
-    MixInstallables & installables;
+    InstallablesCommand & installables;
     std::string text;
 
-    InstallableExpr(MixInstallables & installables, const std::string & text)
+    InstallableExpr(InstallablesCommand & installables, const std::string & text)
          : installables(installables), text(text) { }
 
     std::string what() override { return text; }
@@ -128,10 +128,10 @@ struct InstallableExpr : Installable
 
 struct InstallableAttrPath : Installable
 {
-    MixInstallables & installables;
+    InstallablesCommand & installables;
     std::string attrPath;
 
-    InstallableAttrPath(MixInstallables & installables, const std::string & attrPath)
+    InstallableAttrPath(InstallablesCommand & installables, const std::string & attrPath)
         : installables(installables), attrPath(attrPath)
     { }
 
@@ -177,7 +177,7 @@ struct InstallableAttrPath : Installable
 std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
 static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
 
-std::vector<std::shared_ptr<Installable>> MixInstallables::parseInstallables(ref<Store> store, Strings installables)
+std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings installables)
 {
     std::vector<std::shared_ptr<Installable>> result;
 
@@ -212,7 +212,7 @@ std::vector<std::shared_ptr<Installable>> MixInstallables::parseInstallables(ref
     return result;
 }
 
-PathSet MixInstallables::buildInstallables(ref<Store> store, bool dryRun)
+PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
 {
     PathSet buildables;
 
@@ -229,14 +229,14 @@ PathSet MixInstallables::buildInstallables(ref<Store> store, bool dryRun)
     return buildables;
 }
 
-ref<EvalState> MixInstallables::getEvalState()
+ref<EvalState> InstallablesCommand::getEvalState()
 {
     if (!evalState)
         evalState = std::make_shared<EvalState>(Strings{}, getStore());
     return ref<EvalState>(evalState);
 }
 
-void MixInstallables::prepare()
+void InstallablesCommand::prepare()
 {
     installables = parseInstallables(getStore(), _installables);
 }
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
deleted file mode 100644
index 5f0b0a29224..00000000000
--- a/src/nix/installables.hh
+++ /dev/null
@@ -1,61 +0,0 @@
-#pragma once
-
-#include "args.hh"
-#include "command.hh"
-
-namespace nix {
-
-struct Value;
-class EvalState;
-class Expr;
-
-struct Installable
-{
-    virtual std::string what() = 0;
-
-    virtual PathSet toBuildable()
-    {
-        throw Error("argument ‘%s’ cannot be built", what());
-    }
-
-    virtual Value * toValue(EvalState & state)
-    {
-        throw Error("argument ‘%s’ cannot be evaluated", what());
-    }
-};
-
-struct MixInstallables : virtual Args, StoreCommand
-{
-    std::vector<std::shared_ptr<Installable>> installables;
-    Path file;
-
-    MixInstallables()
-    {
-        mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file);
-        expectArgs("installables", &_installables);
-    }
-
-    /* Return a value representing the Nix expression from which we
-       are installing. This is either the file specified by ‘--file’,
-       or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs
-       = import ...; bla = import ...; }’. */
-    Value * getSourceExpr(EvalState & state);
-
-    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables);
-
-    PathSet buildInstallables(ref<Store> store, bool dryRun);
-
-    ref<EvalState> getEvalState();
-
-    void prepare() override;
-
-private:
-
-    Strings _installables;
-
-    std::shared_ptr<EvalState> evalState;
-
-    Value * vSourceExpr = 0;
-};
-
-}
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 75f3c1ab0d6..ed610261d1c 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -1,12 +1,11 @@
 #include "command.hh"
 #include "common-args.hh"
-#include "installables.hh"
 #include "shared.hh"
 #include "store-api.hh"
 
 using namespace nix;
 
-struct CmdLog : MixInstallables
+struct CmdLog : InstallablesCommand
 {
     CmdLog()
     {
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f3333b77780..a0ce56134b0 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -1,6 +1,5 @@
 #include "command.hh"
 #include "common-args.hh"
-#include "installables.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "derivations.hh"
@@ -13,7 +12,7 @@
 
 using namespace nix;
 
-struct CmdRun : MixInstallables
+struct CmdRun : InstallablesCommand
 {
     CmdRun()
     {
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index e354891a82e..c628c2898d7 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -1,6 +1,5 @@
 #include "command.hh"
 #include "common-args.hh"
-#include "installables.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "json.hh"

From 7ee81f3887353cae2feecb98b482badf57a1fd5b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 13:20:26 +0200
Subject: [PATCH 0267/2196] Make StorePathsCommand a subclass of
 InstallablesCommand

This allows commands like 'nix path-info', 'nix copy', 'nix verify'
etc. to work on arbitrary installables. E.g. to copy geeqie to a
binary cache:

  $ nix copy -r --to file:///tmp/binary-cache nixpkgs.geeqie

Or to get the closure size of thunderbird:

  $ nix path-info -S nixpkgs.thunderbird
---
 src/nix/command.cc | 18 ++++++++++++++----
 src/nix/command.hh | 39 +++++++++++++++++++--------------------
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 4034de96c16..2809a9b4f28 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -1,5 +1,6 @@
 #include "command.hh"
 #include "store-api.hh"
+#include "derivations.hh"
 
 namespace nix {
 
@@ -98,23 +99,32 @@ void StoreCommand::run()
 
 StorePathsCommand::StorePathsCommand()
 {
-    expectArgs("paths", &storePaths);
     mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive);
     mkFlag(0, "all", "apply operation to the entire store", &all);
 }
 
 void StorePathsCommand::run(ref<Store> store)
 {
+    Paths storePaths;
+
     if (all) {
-        if (storePaths.size())
+        if (installables.size())
             throw UsageError("‘--all’ does not expect arguments");
         for (auto & p : store->queryAllValidPaths())
             storePaths.push_back(p);
     }
 
     else {
-        for (auto & storePath : storePaths)
-            storePath = store->followLinksToStorePath(storePath);
+        for (auto & i : installables) {
+            for (auto & path : i->toBuildable()) {
+                if (isDerivation(path)) {
+                    Derivation drv = store->derivationFromPath(path);
+                    for (auto & output : drv.outputs)
+                        storePaths.push_back(output.second.path);
+                } else
+                    storePaths.push_back(path);
+            }
+        }
 
         if (recursive) {
             PathSet closure;
diff --git a/src/nix/command.hh b/src/nix/command.hh
index ee9485e5dd1..dc7b2637d66 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -44,26 +44,6 @@ private:
     std::shared_ptr<Store> _store;
 };
 
-/* A command that operates on zero or more store paths. */
-struct StorePathsCommand : public StoreCommand
-{
-private:
-
-    Paths storePaths;
-    bool recursive = false;
-    bool all = false;
-
-public:
-
-    StorePathsCommand();
-
-    using StoreCommand::run;
-
-    virtual void run(ref<Store> store, Paths storePaths) = 0;
-
-    void run(ref<Store> store) override;
-};
-
 struct Installable
 {
     virtual std::string what() = 0;
@@ -115,6 +95,25 @@ private:
     Value * vSourceExpr = 0;
 };
 
+/* A command that operates on zero or more store paths. */
+struct StorePathsCommand : public InstallablesCommand
+{
+private:
+
+    bool recursive = false;
+    bool all = false;
+
+public:
+
+    StorePathsCommand();
+
+    using StoreCommand::run;
+
+    virtual void run(ref<Store> store, Paths storePaths) = 0;
+
+    void run(ref<Store> store) override;
+};
+
 typedef std::map<std::string, ref<Command>> Commands;
 
 /* An argument parser that supports multiple subcommands,

From 0b6220fbd6869ec9f48094b85350ef2ea0429b14 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 14:09:01 +0200
Subject: [PATCH 0268/2196] Interpret any installable containing a slash as a
 path

So "nix path-info ./result" now works.
---
 src/nix/installables.cc | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 3cf4a6f8d32..45517443ca3 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -183,20 +183,16 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
 
     for (auto & installable : installables) {
 
-        if (std::string(installable, 0, 1) == "/") {
+        if (installable.find("/") != std::string::npos) {
 
-            if (store->isStorePath(installable)) {
-                if (isDerivation(installable))
-                    result.push_back(std::make_shared<InstallableStoreDrv>(installable));
-                else
-                    result.push_back(std::make_shared<InstallableStorePath>(installable));
-            }
+            auto path = store->toStorePath(store->followLinksToStore(installable));
 
-            else {
-                result.push_back(std::make_shared<InstallableStorePath>(
-                        store->toStorePath(store->followLinksToStore(installable))));
+            if (store->isStorePath(path)) {
+                if (isDerivation(path))
+                    result.push_back(std::make_shared<InstallableStoreDrv>(path));
+                else
+                    result.push_back(std::make_shared<InstallableStorePath>(path));
             }
-
         }
 
         else if (installable.compare(0, 1, "(") == 0)

From d48c973ece20875391bebde3c167d6e0cc1e666e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 15:18:05 +0200
Subject: [PATCH 0269/2196] Set default installable

Thus

  $ nix build -f foo.nix

will build foo.nix.

And

  $ nix build

will build default.nix. However, this may not be a good idea because
it's kind of inconsistent, given that "nix build foo" will build the
"foo" attribute from the default installation source (i.e. the
synthesis of $NIX_PATH), rather than ./default.nix. So I may revert
this.
---
 src/nix/installables.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 45517443ca3..ff345c45d8d 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -181,6 +181,12 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
 {
     std::vector<std::shared_ptr<Installable>> result;
 
+    if (installables.empty()) {
+        if (file == "")
+            file = ".";
+        installables = Strings{""};
+    }
+
     for (auto & installable : installables) {
 
         if (installable.find("/") != std::string::npos) {
@@ -198,7 +204,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
         else if (installable.compare(0, 1, "(") == 0)
             result.push_back(std::make_shared<InstallableExpr>(*this, installable));
 
-        else if (std::regex_match(installable, attrPathRegex))
+        else if (installable == "" || std::regex_match(installable, attrPathRegex))
             result.push_back(std::make_shared<InstallableAttrPath>(*this, installable));
 
         else

From c30330df6f67c81986dfb124631bc756c8e58c0d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 16:19:22 +0200
Subject: [PATCH 0270/2196] StorePathCommands: Build installables

So for instance "nix copy --to ... nixpkgs.hello" will build
nixpkgs.hello first. It's debatable whether this is a good idea. It
seems desirable for commands like "nix copy" but maybe not for
commands like "nix path-info".
---
 src/nix/command.cc      | 12 ++----------
 src/nix/installables.cc | 11 ++++++++++-
 src/nix/run.cc          | 10 +---------
 3 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 2809a9b4f28..a45f2888bfb 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -115,16 +115,8 @@ void StorePathsCommand::run(ref<Store> store)
     }
 
     else {
-        for (auto & i : installables) {
-            for (auto & path : i->toBuildable()) {
-                if (isDerivation(path)) {
-                    Derivation drv = store->derivationFromPath(path);
-                    for (auto & output : drv.outputs)
-                        storePaths.push_back(output.second.path);
-                } else
-                    storePaths.push_back(path);
-            }
-        }
+        for (auto & p : buildInstallables(store, false))
+            storePaths.push_back(p);
 
         if (recursive) {
             PathSet closure;
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index ff345c45d8d..57580049f25 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -228,7 +228,16 @@ PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
     if (!dryRun)
         store->buildPaths(buildables);
 
-    return buildables;
+    PathSet outPaths;
+    for (auto & path : buildables)
+        if (isDerivation(path)) {
+            Derivation drv = store->derivationFromPath(path);
+            for (auto & output : drv.outputs)
+                outPaths.insert(output.second.path);
+        } else
+            outPaths.insert(path);
+
+    return outPaths;
 }
 
 ref<EvalState> InstallablesCommand::getEvalState()
diff --git a/src/nix/run.cc b/src/nix/run.cc
index a0ce56134b0..bcfa74eb5f5 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -30,7 +30,7 @@ struct CmdRun : InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto paths = buildInstallables(store, false);
+        auto outPaths = buildInstallables(store, false);
 
         auto store2 = store.dynamic_pointer_cast<LocalStore>();
 
@@ -89,14 +89,6 @@ struct CmdRun : InstallablesCommand
 #endif
         }
 
-        PathSet outPaths;
-        for (auto & path : paths)
-            if (isDerivation(path)) {
-                Derivation drv = store->derivationFromPath(path);
-                for (auto & output : drv.outputs)
-                    outPaths.insert(output.second.path);
-            } else
-                outPaths.insert(path);
 
         auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
         for (auto & path : outPaths)

From 2dff9556a4131af8a50647f23fe03bfc3c295e12 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 16:55:03 +0200
Subject: [PATCH 0271/2196] Fix build

---
 default.nix | 8 ++++----
 nix-repl.cc | 4 +++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/default.nix b/default.nix
index 8690325e528..919082981e1 100644
--- a/default.nix
+++ b/default.nix
@@ -4,13 +4,13 @@ with import nixpkgs { inherit system; };
 
 let nix = nixUnstable; in
 
-runCommand "nix-repl"
-  { buildInputs = [ readline nix boehmgc ]; }
+runCommandCC "nix-repl"
+  { buildInputs = [ pkgconfig readline nix boehmgc ]; }
   ''
     mkdir -p $out/bin
-    g++ -O3 -Wall -std=c++0x \
+    g++ -O3 -Wall -std=c++14 \
       -o $out/bin/nix-repl ${./nix-repl.cc} \
-      -I${nix}/include/nix \
+      $(pkg-config --cflags nix-main) \
       -lnixformat -lnixutil -lnixstore -lnixexpr -lnixmain -lreadline -lgc \
       -DNIX_VERSION=\"${(builtins.parseDrvName nix.name).version}\"
   ''
diff --git a/nix-repl.cc b/nix-repl.cc
index 0e8c67cf7f2..0c50f468330 100644
--- a/nix-repl.cc
+++ b/nix-repl.cc
@@ -1,3 +1,5 @@
+#include <nix/config.h>
+
 #include <iostream>
 #include <cstdlib>
 
@@ -291,7 +293,7 @@ static int runProgram(const string & program, const Strings & args)
         _exit(1);
     }
 
-    return pid.wait(true);
+    return pid.wait();
 }
 
 

From 40daf0d800d6a248a57bae0fff8c3989d4814840 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 18:13:23 +0200
Subject: [PATCH 0272/2196] Cleanup in preparation of merging nix-repl repo
 into nix repo

---
 COPYING                        | 674 ---------------------------------
 README.md                      | 108 ------
 default.nix                    |  16 -
 nix-repl.cc => src/nix/repl.cc |   0
 4 files changed, 798 deletions(-)
 delete mode 100644 COPYING
 delete mode 100644 README.md
 delete mode 100644 default.nix
 rename nix-repl.cc => src/nix/repl.cc (100%)

diff --git a/COPYING b/COPYING
deleted file mode 100644
index 94a9ed024d3..00000000000
--- a/COPYING
+++ /dev/null
@@ -1,674 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
-
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/README.md b/README.md
deleted file mode 100644
index ac5ad98cf53..00000000000
--- a/README.md
+++ /dev/null
@@ -1,108 +0,0 @@
-nix-repl
-========
-
-`nix-repl` is a simple read–eval–print loop (REPL) for the Nix package
-manager.
-
-Installation
-------------
-
-Assuming you have Nix installed, just do
-
-    $ git clone https://github.com/edolstra/nix-repl.git
-    $ cd nix-repl
-    $ nix-env -f . -i nix-repl
-
-Example
--------
-
-Here is a typical `nix-repl` session:
-
-    $ nix-repl
-    Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
-
-    nix-repl> 3 * 4
-    12
-
-    nix-repl> :l <nixpkgs>
-    Added 3337 variables.
-
-    nix-repl> lib.range 1 5
-    [ 1 2 3 4 5 ]
-
-    nix-repl> :a lib
-    Added 299 variables.
-
-    nix-repl> range 1 5
-    [ 1 2 3 4 5 ]
-
-    nix-repl> xs = range 1 5
-
-    nix-repl> map (x: x * x) xs
-    [ 1 4 9 16 25 ]
-
-    nix-repl> :l <nixpkgs/nixos>
-    Added 7 variables.
-
-    nix-repl> config.services.dhcpd
-    { configFile = null; enable = false; extraConfig = ""; interfaces = [ ... ]; machines = [ ... ]; }
-
-    nix-repl> :p config.services.dhcpd
-    { configFile = null; enable = false; extraConfig = ""; interfaces = [ "eth0" ]; machines = [ ]; }
-
-    nix-repl> config.fileSystems
-    { "/" = { ... }; "/boot" = { ... }; }
-
-    nix-repl> mapAttrsToList (n: v: v.device) config.fileSystems
-    [ "/dev/disk/by-label/nixos" "/dev/disk/by-label/boot" ]
-
-    nix-repl> :b libjson
-    these derivations will be built:
-      /nix/store/h910xqb36pysxcxkayb1zkr1zcvvk1zy-libjson_7.6.1.zip.drv
-      /nix/store/iv0rdx08di0fg704zyxklkvdz6i96lm8-libjson-7.6.1.drv
-    ...
-    this derivation produced the following outputs:
-      out -> /nix/store/ys6bvgfia81rjwqxjlgkwnx6jhsml8h9-libjson-7.6.1
-
-    nix-repl> :t makeFontsConf
-    a function
-
-    nix-repl> :b makeFontsConf { fontDirectories = [ "${freefont_ttf}/share/fonts/truetype" ]; }
-    ...
-    this derivation produced the following outputs:
-      out -> /nix/store/jkw848xj0gkbhmvxi0hwpnhzn2716v3c-fonts.conf
-
-    nix-repl> :s pan
-    # Builds dependencies of the ‘pan’ derivation, then starts a shell
-    # in which the environment variables of the derivation are set
-
-    [nix-shell:/tmp/nix-repl]$ echo $src
-    /nix/store/0ibx15r02nnkwiclmfbpzrzjm2y204fh-pan-0.139.tar.bz2
-
-    [nix-shell:/tmp/nix-repl]$ exit
-
-    nix-repl>
-
-Tab completion works on variables in scope and on attribute sets.  For
-example:
-
-    $ nix-repl  '<nixpkgs>' '<nixpkgs/nixos>'
-    Welcome to Nix version 1.6pre3215_2c1ecf8. Type :? for help.
-
-    nix-repl> thunder<TAB> => thunderbird
-
-    nix-repl> <TAB>
-    Display all 3634 possibilities? (y or n)
-
-    nix-repl> lib<TAB>
-    Display all 291 possibilities? (y or n)
-
-    nix-repl> xorg.libX<TAB>
-    xorg.libXdamage  xorg.libXdmcp
-
-    nix-repl> config.networking.use<TAB>
-    config.networking.useDHCP   config.networking.usePredictableInterfaceNames
-
-Input history is preserved by readline in ~/.nix-repl-history
-The readline "application name" is nix-repl. This allows for nix-repl specific
-settings in ~/.inputrc
diff --git a/default.nix b/default.nix
deleted file mode 100644
index 919082981e1..00000000000
--- a/default.nix
+++ /dev/null
@@ -1,16 +0,0 @@
-{ nixpkgs ? <nixpkgs>, system ? builtins.currentSystem }:
-
-with import nixpkgs { inherit system; };
-
-let nix = nixUnstable; in
-
-runCommandCC "nix-repl"
-  { buildInputs = [ pkgconfig readline nix boehmgc ]; }
-  ''
-    mkdir -p $out/bin
-    g++ -O3 -Wall -std=c++14 \
-      -o $out/bin/nix-repl ${./nix-repl.cc} \
-      $(pkg-config --cflags nix-main) \
-      -lnixformat -lnixutil -lnixstore -lnixexpr -lnixmain -lreadline -lgc \
-      -DNIX_VERSION=\"${(builtins.parseDrvName nix.name).version}\"
-  ''
diff --git a/nix-repl.cc b/src/nix/repl.cc
similarity index 100%
rename from nix-repl.cc
rename to src/nix/repl.cc

From 921a2aeb0537f34bc2b41e98e67a1c829321ee81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 18:48:40 +0200
Subject: [PATCH 0273/2196] Make "nix repl" build

---
 release.nix      |  3 ++-
 shell.nix        |  1 +
 src/nix/local.mk |  2 ++
 src/nix/repl.cc  | 66 +++++++++++++++++++++++++-----------------------
 4 files changed, 39 insertions(+), 33 deletions(-)

diff --git a/release.nix b/release.nix
index 294af54cd61..534c218c112 100644
--- a/release.nix
+++ b/release.nix
@@ -73,7 +73,8 @@ let
         buildInputs =
           [ curl
             bzip2 xz brotli
-            openssl pkgconfig sqlite boehmgc
+            openssl pkgconfig sqlite boehmgc readline
+
           ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
diff --git a/shell.nix b/shell.nix
index 425eb0a191f..3c57826d1a1 100644
--- a/shell.nix
+++ b/shell.nix
@@ -16,6 +16,7 @@ with import <nixpkgs> {};
         customMemoryManagement = false;
       })
       autoreconfHook
+      readline
     ];
 
   configureFlags =
diff --git a/src/nix/local.mk b/src/nix/local.mk
index f6e7073b6e7..21f190e476f 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
+nix_LDFLAGS = -lreadline
+
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 71790eb481a..54e48e405b8 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -1,5 +1,3 @@
-#include <nix/config.h>
-
 #include <iostream>
 #include <cstdlib>
 
@@ -17,9 +15,11 @@
 #include "derivations.hh"
 #include "affinity.hh"
 #include "globals.hh"
+#include "command.hh"
+
+namespace nix {
 
 using namespace std;
-using namespace nix;
 
 #define ESC_RED "\033[31m"
 #define ESC_GRE "\033[32m"
@@ -49,6 +49,7 @@ struct NixRepl
     StringSet::iterator curCompletion;
 
     NixRepl(const Strings & searchPath, nix::ref<Store> store);
+    ~NixRepl();
     void mainLoop(const Strings & files);
     void completePrefix(string prefix);
     bool getLine(string & input, const char * prompt);
@@ -119,10 +120,16 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
 }
 
 
+NixRepl::~NixRepl()
+{
+    write_history(historyFile.c_str());
+}
+
+
 void NixRepl::mainLoop(const Strings & files)
 {
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
-    std::cout << "Welcome to Nix version " << NIX_VERSION << ". Type :? for help." << std::endl << std::endl;
+    std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl;
 
     for (auto & i : files)
         loadedFiles.push_back(i);
@@ -685,35 +692,30 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     return str;
 }
 
-
-int main(int argc, char * * argv)
+struct CmdRepl : StoreCommand
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
-        Strings files, searchPath;
-
-        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
-            if (*arg == "--version")
-                printVersion("nix-repl");
-            else if (*arg == "--help") {
-                printHelp();
-                // exit with 0 since user asked for help
-                _exit(0);
-            }
-            else if (parseSearchPathArg(arg, end, searchPath))
-                ;
-            else if (*arg != "" && arg->at(0) == '-')
-                return false;
-            else
-                files.push_back(*arg);
-            return true;
-        });
-
-        NixRepl repl(searchPath, openStore());
+    Strings files;
+
+    CmdRepl()
+    {
+        expectArgs("files", &files);
+    }
+
+    std::string name() override { return "repl"; }
+
+    std::string description() override
+    {
+        return "start an interactive environment for evaluating Nix expressions";
+    }
+
+    void run(ref<Store> store) override
+    {
+        // FIXME: pass searchPath
+        NixRepl repl({}, openStore());
         repl.mainLoop(files);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdRepl>());
 
-        write_history(historyFile.c_str());
-    });
 }

From 5bd8795e1fa9255836fbbeab0a22f5038d92d53c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 18:56:29 +0200
Subject: [PATCH 0274/2196] nix repl: Use $XDG_DATA_HOME for the readline
 history

---
 src/libutil/util.cc | 12 ++++++++++++
 src/libutil/util.hh |  3 +++
 src/nix/repl.cc     |  7 ++++---
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 88a2f752c02..026e493514e 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -441,6 +441,18 @@ Path getConfigDir()
 }
 
 
+Path getDataDir()
+{
+    Path dataDir = getEnv("XDG_DATA_HOME");
+    if (dataDir.empty()) {
+        Path homeDir = getEnv("HOME");
+        if (homeDir.empty()) throw Error("$XDG_DATA_HOME and $HOME are not set");
+        dataDir = homeDir + "/.local/share";
+    }
+    return dataDir;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ae40dcd4cd2..a9950f830c5 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -116,6 +116,9 @@ Path getCacheDir();
 /* Return $XDG_CONFIG_HOME or $HOME/.config. */
 Path getConfigDir();
 
+/* Return $XDG_DATA_HOME or $HOME/.local/share. */
+Path getDataDir();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 54e48e405b8..78d973ba832 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -29,9 +29,6 @@ using namespace std;
 #define ESC_CYA "\033[36m"
 #define ESC_END "\033[0m"
 
-string programId = "nix-repl";
-const string historyFile = string(getenv("HOME")) + "/.nix-repl-history";
-
 struct NixRepl
 {
     string curDir;
@@ -45,6 +42,8 @@ struct NixRepl
     int displ;
     StringSet varNames;
 
+    const Path historyFile;
+
     StringSet completions;
     StringSet::iterator curCompletion;
 
@@ -115,6 +114,7 @@ string removeWhitespace(string s)
 NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
     : state(searchPath, store)
     , staticEnv(false, &state.staticBaseEnv)
+    , historyFile(getDataDir() + "/nix/repl-history")
 {
     curDir = absPath(".");
 }
@@ -140,6 +140,7 @@ void NixRepl::mainLoop(const Strings & files)
     // Allow nix-repl specific settings in .inputrc
     rl_readline_name = "nix-repl";
     using_history();
+    createDirs(dirOf(historyFile));
     read_history(historyFile.c_str());
 
     string input;

From 536f06176519072011aebe0d9b4b5b75e1691f67 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 18:58:02 +0200
Subject: [PATCH 0275/2196] "using namespace std" considered harmful

---
 src/nix/repl.cc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 78d973ba832..964bb85eb6f 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -19,8 +19,6 @@
 
 namespace nix {
 
-using namespace std;
-
 #define ESC_RED "\033[31m"
 #define ESC_GRE "\033[32m"
 #define ESC_YEL "\033[33m"
@@ -70,7 +68,8 @@ struct NixRepl
 
 void printHelp()
 {
-    cout << "Usage: nix-repl [--help] [--version] [-I path] paths...\n"
+    std::cout
+         << "Usage: nix-repl [--help] [--version] [-I path] paths...\n"
          << "\n"
          << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n"
          << "\n"
@@ -97,8 +96,7 @@ void printHelp()
          << "        If an element of paths starts with http:// or https://, it is interpreted\n"
          << "        as the URL of a tarball that will be downloaded and unpacked to a temporary\n"
          << "        location. The tarball must include a single top-level directory containing\n"
-         << "        at least a file named default.nix.\n"
-         << flush;
+         << "        at least a file named default.nix.\n";
 }
 
 
@@ -351,7 +349,8 @@ bool NixRepl::processLine(string line)
     }
 
     if (command == ":?" || command == ":help") {
-        cout << "The following commands are available:\n"
+        std::cout
+             << "The following commands are available:\n"
              << "\n"
              << "  <expr>        Evaluate and print expression\n"
              << "  <x> = <expr>  Bind expression to variable\n"

From 4c95ef3768f8415579d9dd8bda69407021b72017 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 18:59:18 +0200
Subject: [PATCH 0276/2196] Fix nix-shell test

---
 tests/shell.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/shell.nix b/tests/shell.nix
index 1a092913b3b..5845d36fc16 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -2,7 +2,7 @@
 
 with import ./config.nix;
 
-rec {
+let pkgs = rec {
   setupSh = builtins.toFile "setup" ''
     export VAR_FROM_STDENV_SETUP=foo
     for pkg in $buildInputs; do
@@ -44,4 +44,6 @@ rec {
   '';
 
   bash = shell;
-}
+
+  inherit pkgs;
+}; in pkgs

From 23aa1619daace5db30233a53183911adb42322d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 19:10:47 +0200
Subject: [PATCH 0277/2196] Minor cleanup

---
 src/nix/repl.cc | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 964bb85eb6f..ae30502641a 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -289,18 +289,15 @@ void NixRepl::completePrefix(string prefix)
 
 static int runProgram(const string & program, const Strings & args)
 {
-    std::vector<const char *> cargs; /* careful with c_str()! */
-    cargs.push_back(program.c_str());
-    for (Strings::const_iterator i = args.begin(); i != args.end(); ++i)
-        cargs.push_back(i->c_str());
-    cargs.push_back(0);
+    Strings args2(args);
+    args2.push_front(program);
 
     Pid pid;
     pid = fork();
     if (pid == -1) throw SysError("forking");
     if (pid == 0) {
         restoreAffinity();
-        execvp(program.c_str(), (char * *) &cargs[0]);
+        execvp(program.c_str(), stringsToCharPtrs(args2).data());
         _exit(1);
     }
 
@@ -394,7 +391,7 @@ bool NixRepl::processLine(string line)
         state.callFunction(f, v, result, Pos());
 
         Path drvPath = getDerivationPath(result);
-        runProgram("nix-shell", Strings{drvPath});
+        runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
@@ -406,16 +403,16 @@ bool NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram("nix-store", Strings{"-r", drvPath}) == 0) {
+            if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) {
                 Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)
                     std::cout << format("  %1% -> %2%") % i.first % i.second.path << std::endl;
             }
         } else if (command == ":i") {
-            runProgram("nix-env", Strings{"-i", drvPath});
+            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
         } else {
-            runProgram("nix-shell", Strings{drvPath});
+            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
         }
     }
 

From 6734c18c99f8fa33a50a3045a8dd915bbf084255 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Apr 2017 19:19:15 +0200
Subject: [PATCH 0278/2196] nix repl: Fix Ctrl-C

---
 src/nix/repl.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index ae30502641a..17203d3c299 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -217,6 +217,13 @@ bool NixRepl::getLine(string & input, const char * prompt)
     if (sigaction(SIGINT, &act, &old))
         throw SysError("installing handler for SIGINT");
 
+    static sigset_t savedSignalMask, set;
+    sigemptyset(&set);
+    sigaddset(&set, SIGINT);
+
+    if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+        throw SysError("unblocking SIGINT");
+
     if (sigsetjmp(sigintJmpBuf, 1)) {
         input.clear();
     } else {
@@ -236,6 +243,9 @@ bool NixRepl::getLine(string & input, const char * prompt)
 
     _isInterrupted = 0;
 
+    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+        throw SysError("restoring signals");
+
     if (sigaction(SIGINT, &old, 0))
         throw SysError("restoring handler for SIGINT");
 

From 98a2adb1359ec35ac8da7a52754d2290531f8cef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Apr 2017 17:04:45 +0200
Subject: [PATCH 0279/2196] Simplify building nix-perl in nix-shell

---
 perl/Makefile         | 2 +-
 perl/configure.ac     | 2 +-
 perl/lib/Nix/Store.xs | 2 ++
 perl/local.mk         | 6 +++---
 shell.nix             | 9 ++++++++-
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/perl/Makefile b/perl/Makefile
index cf655ae3d65..684a37e8121 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include nix/config.h
+GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
 
 -include Makefile.config
 
diff --git a/perl/configure.ac b/perl/configure.ac
index 7a6b28be23e..80bcdb8ff5b 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -98,7 +98,7 @@ for name in $ac_subst_vars; do
 done
 
 rm -f Makefile.config
-ln -s ../mk mk
+ln -sfn ../mk mk
 
 AC_CONFIG_FILES([])
 AC_OUTPUT
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 1920942a4c0..aa14bfa6270 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -1,3 +1,5 @@
+#include "config.h"
+
 #include "EXTERN.h"
 #include "perl.h"
 #include "XSUB.h"
diff --git a/perl/local.mk b/perl/local.mk
index 35113bd960d..b13d4c0d639 100644
--- a/perl/local.mk
+++ b/perl/local.mk
@@ -20,11 +20,11 @@ Store_DIR := lib/Nix
 Store_SOURCES := $(Store_DIR)/Store.cc
 
 Store_CXXFLAGS = \
-  -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \
+  $(NIX_CFLAGS) \
+  -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \
   -D_FILE_OFFSET_BITS=64 \
   -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \
-  -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion \
-  $(NIX_CFLAGS)
+  -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion
 
 Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
diff --git a/shell.nix b/shell.nix
index 3c57826d1a1..37a936fd2ef 100644
--- a/shell.nix
+++ b/shell.nix
@@ -17,6 +17,10 @@ with import <nixpkgs> {};
       })
       autoreconfHook
       readline
+
+      # For nix-perl
+      perl
+      perlPackages.DBDSQLite
     ];
 
   configureFlags =
@@ -30,6 +34,9 @@ with import <nixpkgs> {};
 
   shellHook =
     ''
-      configureFlags+=" --prefix=$(pwd)/inst"
+      export prefix=$(pwd)/inst
+      configureFlags+=" --prefix=prefix"
+      PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
+      PATH=$prefix/bin:$PATH
     '';
 }

From 45ce2c7413bf8c87aa7d5bc70a5a1c920c40a3b8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Apr 2017 17:58:09 +0200
Subject: [PATCH 0280/2196] Doh

---
 src/nix-store/nix-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 9131b74dfb4..950222812e2 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -799,7 +799,7 @@ static void opServe(Strings opFlags, Strings opArgs)
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
             settings.buildRepeat = readInt(in);
             settings.enforceDeterminism = readInt(in);
-            settings.runDiffHook = readInt(in);
+            settings.runDiffHook = true;
         }
         settings.printRepeatedBuilds = false;
     };

From 4bc00760f9bc36d5b4e8bba7de9bd71a30d7f31a Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 26 Apr 2017 12:38:16 -0400
Subject: [PATCH 0281/2196] Add Store nesting to fix import-from-derivation
 within filterSource

---
 src/libstore/remote-store.cc |  2 ++
 src/libutil/pool.hh          | 16 ++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index bc9ef3d47c5..af59d51106f 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -414,7 +414,9 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
     try {
         conn->to.written = 0;
         conn->to.warn = true;
+        connections->incCapacity();
         dumpPath(srcPath, conn->to, filter);
+        connections->decCapacity();
         conn->to.warn = false;
         conn->processStderr();
     } catch (SysError & e) {
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index 20df2194884..7033090020e 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -68,6 +68,22 @@ public:
         state_->max = max;
     }
 
+    void incCapacity()
+    {
+        auto state_(state.lock());
+        state_->max++;
+        /* we could wakeup here, but this is only used when we're
+         * about to nest Pool usages, and we want to save the slot for
+         * the nested use if we can
+         */
+    }
+
+    void decCapacity()
+    {
+        auto state_(state.lock());
+        state_->max--;
+    }
+
     ~Pool()
     {
         auto state_(state.lock());

From 41c4558afe04d1cad0d0ef3b18a1a8155b40b06e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Apr 2017 15:10:29 +0200
Subject: [PATCH 0282/2196] Fix hash computation when importing NARs greater
 than 4 GiB

This caused "nix-store --import" to compute an incorrect hash on NARs
that don't fit in an unsigned int. The import would succeed, but
"nix-store --verify-path" or subsequent exports would detect an
incorrect hash.

A deeper issue is that the export/import format does not contain a
hash, so we can't detect such issues early.

Also, I learned that -Wall does not warn about this.
---
 src/libutil/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 9f4afd93c2f..fa1bb5d9718 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -224,7 +224,7 @@ static void start(HashType ht, Ctx & ctx)
 
 
 static void update(HashType ht, Ctx & ctx,
-    const unsigned char * bytes, unsigned int len)
+    const unsigned char * bytes, size_t len)
 {
     if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len);
     else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len);

From a1a5e63e1456b5905b73065f635a324f3c309a5d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Apr 2017 16:20:46 +0200
Subject: [PATCH 0283/2196] Fix brainfart

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9bf1ab5aa58..01a3203dd90 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3379,7 +3379,7 @@ void SubstitutionGoal::tryToRun()
        if maxBuildJobs == 0 (no local builds allowed), we still allow
        a substituter to run.  This is because substitutions cannot be
        distributed to another machine via the build hook. */
-    if (worker.getNrLocalBuilds() >= std::min(1U, (unsigned int) settings.maxBuildJobs)) {
+    if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         return;
     }

From 73bba12d8ba1bff2b2c0266a7fdbe1ee8ab0ad6a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Apr 2017 16:53:56 +0200
Subject: [PATCH 0284/2196] Check for libreadline

---
 Makefile.config.in | 1 +
 configure.ac       | 8 ++++++++
 nix.spec.in        | 2 ++
 release.nix        | 6 +++---
 shell.nix          | 2 +-
 src/nix/local.mk   | 4 +++-
 src/nix/repl.cc    | 4 ++++
 7 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 6948dad5a60..3cae30d487d 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -5,6 +5,7 @@ CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
+HAVE_READLINE = @HAVE_READLINE@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
diff --git a/configure.ac b/configure.ac
index c7026cf954d..ac37456ae5f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -196,6 +196,14 @@ if test "$gc" = yes; then
 fi
 
 
+# Check for readline, needed by "nix repl".
+AX_LIB_READLINE
+if test "$ax_cv_lib_readline" != "no"; then
+  have_readline=1
+fi
+AC_SUBST(HAVE_READLINE, [$have_readline])
+
+
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
   [do not initialise DB etc. in `make install']),
   init_state=$enableval, init_state=yes)
diff --git a/nix.spec.in b/nix.spec.in
index 390893d64dc..3ba2dfc94b4 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -20,9 +20,11 @@ Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
+Requires: readline
 BuildRequires: bzip2-devel
 BuildRequires: sqlite-devel
 BuildRequires: libcurl-devel
+BuildRequires: readline-devel
 
 # Hack to make that shitty RPM scanning hack shut up.
 Provides: perl(Nix::SSH)
diff --git a/release.nix b/release.nix
index 534c218c112..7adc87386f9 100644
--- a/release.nix
+++ b/release.nix
@@ -299,7 +299,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" "readline-devel" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
@@ -321,14 +321,14 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libreadline-dev" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libreadline6" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;
diff --git a/shell.nix b/shell.nix
index 37a936fd2ef..bbce68564b9 100644
--- a/shell.nix
+++ b/shell.nix
@@ -35,7 +35,7 @@ with import <nixpkgs> {};
   shellHook =
     ''
       export prefix=$(pwd)/inst
-      configureFlags+=" --prefix=prefix"
+      configureFlags+=" --prefix=$prefix"
       PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
       PATH=$prefix/bin:$PATH
     '';
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 21f190e476f..e71cf16fabf 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -6,6 +6,8 @@ nix_SOURCES := $(wildcard $(d)/*.cc)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
-nix_LDFLAGS = -lreadline
+ifeq ($(HAVE_READLINE), 1)
+  nix_LDFLAGS += -lreadline
+endif
 
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 17203d3c299..13488bf1dbd 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -1,3 +1,5 @@
+#if HAVE_LIBREADLINE
+
 #include <iostream>
 #include <cstdlib>
 
@@ -726,3 +728,5 @@ struct CmdRepl : StoreCommand
 static RegisterCommand r1(make_ref<CmdRepl>());
 
 }
+
+#endif

From 895f00c37267086cc6fcd8326c8ecad2818223e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Apr 2017 16:55:52 +0200
Subject: [PATCH 0285/2196] Suppress warning about ssh-auth-sock

---
 src/nix-daemon/nix-daemon.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 07ad0b45b3e..1b90fad165a 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -483,7 +483,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
             };
 
             try {
-                if (trusted
+                if (name == "ssh-auth-sock") // obsolete
+                    ;
+                else if (trusted
                     || name == settings.buildTimeout.name
                     || name == settings.connectTimeout.name)
                     settings.set(name, value);

From 2f21d522c28b1e902bd7f0b5b9e7523975102d81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Apr 2017 17:13:55 +0200
Subject: [PATCH 0286/2196] Hopefully fix the Darwin build

http://hydra.nixos.org/build/52080911
---
 src/libutil/config.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 62c6433c741..497afaa1fed 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -215,6 +215,7 @@ template class BaseSetting<unsigned long>;
 template class BaseSetting<long long>;
 template class BaseSetting<unsigned long long>;
 template class BaseSetting<bool>;
+template class BaseSetting<std::string>;
 
 void PathSetting::set(const std::string & str)
 {

From b3f55fdf621b6433967fd501423bedcb6ab2e4ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 29 Apr 2017 11:33:55 +0200
Subject: [PATCH 0287/2196] nix-daemon.service: set XDG_CONFIG_HOME
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Otherwise starting nix-daemon fails

● nix-daemon.service - Nix Daemon
   Loaded: loaded
(/nix/store/mnf00a6gc55xl47smk0b32gmi7xpvlfp-nix-1.12pre5308_2f21d522/lib/systemd/system/nix-daemon.service;
enabled; vendor preset: enabled)
  Drop-In:
/nix/store/m2rgjp71n4kyp8j5fxgbrlv13scd5vvv-system-units/nix-daemon.service.d
           └─overrides.conf
   Active: failed (Result: exit-code) since Sat 2017-04-29 11:29:21
CEST; 9s ago
  Process: 7299 ExecStart=nix-daemon --daemon (code=exited, status=1/FAILURE)
 Main PID: 7299 (code=exited, status=1/FAILURE)
      CPU: 19ms

... systemd[1]: Started Nix Daemon.
... nix-daemon[7299]: error: $XDG_CONFIG_HOME and $HOME are not set
... systemd[1]: nix-daemon.service: Main process exited, code=exited, status=1/FAILURE
... systemd[1]: nix-daemon.service: Unit entered failed state.
... systemd[1]: nix-daemon.service: Failed with result 'exit-code'.
... systemd[1]: nix-daemon.service: Start request repeated too quickly.
... systemd[1]: Failed to start Nix Daemon.
... systemd[1]: nix-daemon.service: Failed with result 'exit-code'.
---
 misc/systemd/nix-daemon.service.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in
index fcd799e177d..9bfb00e306b 100644
--- a/misc/systemd/nix-daemon.service.in
+++ b/misc/systemd/nix-daemon.service.in
@@ -8,3 +8,4 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
 Environment=XDG_CACHE_HOME=/root/.cache
+Environment=XDG_CONFIG_HOME=/root/.config

From a4744254250e170f1b858e46a8b1c7904a030a2b Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Mon, 1 May 2017 01:05:41 +0200
Subject: [PATCH 0288/2196] Fix lexer to support `$'` in multiline strings.

---
 src/libexpr/lexer.l                 |  3 ++-
 tests/lang/eval-okay-ind-string.exp |  2 +-
 tests/lang/eval-okay-ind-string.nix | 10 +++++++++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 5b1ff0350cd..d4fae2d7da1 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -160,7 +160,8 @@ or          { return OR_KW; }
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
                  }
-<IND_STRING>\'\'\$ {
+<IND_STRING>\'\'\$ |
+<IND_STRING>\$   {
                    yylval->e = new ExprIndStr("$");
                    return IND_STR;
                  }
diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/lang/eval-okay-ind-string.exp
index 886219dcf65..9cf4bd2ee78 100644
--- a/tests/lang/eval-okay-ind-string.exp
+++ b/tests/lang/eval-okay-ind-string.exp
@@ -1 +1 @@
-"This is an indented multi-line string\nliteral.  An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed.  Thus,\nin this case four spaces will be\nstripped from each line, even though\n  THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n  followed by a newline, it's stripped, but\n  that's not the case here. Two spaces are\n  stripped because of the \"  \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n  The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n   Tabs are not interpreted as whitespace (since we can't guess\n   what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n   space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored.  But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n  Similarly you can force an indentation level,\n  in this case to 2 spaces.  This works because the anti-quote\n  is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n  rm -f /var/run/opengl-driver\n  ln -sf 123 /var/run/opengl-driver\n\n  rm -f /var/log/slim.log\n   \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf  \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin         \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/          # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\n"
+"This is an indented multi-line string\nliteral.  An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed.  Thus,\nin this case four spaces will be\nstripped from each line, even though\n  THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n  followed by a newline, it's stripped, but\n  that's not the case here. Two spaces are\n  stripped because of the \"  \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n  The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n   Tabs are not interpreted as whitespace (since we can't guess\n   what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n   space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored.  But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n  Similarly you can force an indentation level,\n  in this case to 2 spaces.  This works because the anti-quote\n  is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n  rm -f /var/run/opengl-driver\n  ln -sf 123 /var/run/opengl-driver\n\n  rm -f /var/log/slim.log\n   \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf  \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin         \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/          # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n"
diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/lang/eval-okay-ind-string.nix
index 1556aae9f54..1669dc0648e 100644
--- a/tests/lang/eval-okay-ind-string.nix
+++ b/tests/lang/eval-okay-ind-string.nix
@@ -117,4 +117,12 @@ let
     bar
   '';
 
-in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15
+  # Regression test: accept $'.
+  s16 = ''
+    cut -d $'\t' -f 1
+  '';
+
+  # Accept dollars at end of strings 
+  s17 = ''ending dollar $'' + ''$'' + "\n";
+
+in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17

From a143014d738758b5558efb73fee9f351cd00cbda Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Mon, 1 May 2017 01:07:33 +0200
Subject: [PATCH 0289/2196] lexer: remove catch-all rules hiding real errors

With catch-all rules, we hide potential errors.
It turns out that a4744254 made one cath-all useless. Flex detected that
is was impossible to reach.
The other is more subtle, as it can only trigger on unfinished escapes
in unfinished strings, which only occurs at EOF.
---
 src/libexpr/lexer.l | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index d4fae2d7da1..40ca7725803 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -142,18 +142,26 @@ or          { return OR_KW; }
 \{                           { return '{'; }
 <INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\"          { PUSH_STATE(STRING); return '"'; }
+<INITIAL,INSIDE_DOLLAR_CURLY>\" {
+                PUSH_STATE(STRING); return '"';
+              }
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" |
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ {
-              /* It is impossible to match strings ending with '$' with one
-                 regex because trailing contexts are only valid at the end
-                 of a rule. (A sane but undocumented limitation.) */
-              yylval->e = unescapeStr(data->symbols, yytext);
-              return STR;
-            }
+                /* It is impossible to match strings ending with '$' with one
+                   regex because trailing contexts are only valid at the end
+                   of a rule. (A sane but undocumented limitation.) */
+                yylval->e = unescapeStr(data->symbols, yytext);
+                return STR;
+              }
 <STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
-<STRING>\"  { POP_STATE(); return '"'; }
-<STRING>.   return yytext[0]; /* just in case: shouldn't be reached */
+<STRING>\"    { POP_STATE(); return '"'; }
+<STRING>\$|\\|\$\\ {
+                /* This can only occur when we reach EOF, otherwise the above
+                   (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
+                   This is technically invalid, but we leave the problem to the
+                   parser who fails with exact location. */
+                return STR;
+              }
 
 <INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
@@ -179,7 +187,6 @@ or          { return OR_KW; }
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
-<IND_STRING>.    return yytext[0]; /* just in case: shouldn't be reached */
 
 <INITIAL,INSIDE_DOLLAR_CURLY>{
 

From 0dddcf867a65723ddf96343dd694f970f2f9538a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 13:43:34 +0200
Subject: [PATCH 0290/2196] Add a dummy Store::buildPaths() method

This default implementation of buildPaths() does nothing if all
requested paths are already valid, and throws an "unsupported
operation" error otherwise. This fixes a regression introduced by
c30330df6f67c81986dfb124631bc756c8e58c0d in binary cache and legacy
SSH stores.
---
 src/libstore/binary-cache-store.cc |  5 -----
 src/libstore/binary-cache-store.hh | 29 ++++++++++++-----------------
 src/libstore/legacy-ssh-store.cc   |  9 ---------
 src/libstore/store-api.cc          | 11 +++++++++++
 src/libstore/store-api.hh          | 20 +++++++++++++-------
 5 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index b536c6c0004..46c5aa21b2e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -114,11 +114,6 @@ void BinaryCacheStore::init()
     }
 }
 
-void BinaryCacheStore::notImpl()
-{
-    throw Error("operation not implemented for binary cache stores");
-}
-
 std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
     std::promise<std::shared_ptr<std::string>> promise;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 5c2d0acfdbb..87d4aa43838 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -27,8 +27,6 @@ protected:
 
     BinaryCacheStore(const Params & params);
 
-    [[noreturn]] void notImpl();
-
 public:
 
     virtual bool fileExists(const std::string & path) = 0;
@@ -65,7 +63,7 @@ public:
     bool isValidPathUncached(const Path & path) override;
 
     PathSet queryAllValidPaths() override
-    { notImpl(); }
+    { unsupported(); }
 
     void queryPathInfoUncached(const Path & path,
         std::function<void(std::shared_ptr<ValidPathInfo>)> success,
@@ -73,16 +71,16 @@ public:
 
     void queryReferrers(const Path & path,
         PathSet & referrers) override
-    { notImpl(); }
+    { unsupported(); }
 
     PathSet queryDerivationOutputs(const Path & path) override
-    { notImpl(); }
+    { unsupported(); }
 
     StringSet queryDerivationOutputNames(const Path & path) override
-    { notImpl(); }
+    { unsupported(); }
 
     Path queryPathFromHashPart(const string & hashPart) override
-    { notImpl(); }
+    { unsupported(); }
 
     bool wantMassQuery() override { return wantMassQuery_; }
 
@@ -99,32 +97,29 @@ public:
 
     void narFromPath(const Path & path, Sink & sink) override;
 
-    void buildPaths(const PathSet & paths, BuildMode buildMode) override
-    { notImpl(); }
-
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
-    { notImpl(); }
+    { unsupported(); }
 
     void ensurePath(const Path & path) override
-    { notImpl(); }
+    { unsupported(); }
 
     void addTempRoot(const Path & path) override
-    { notImpl(); }
+    { unsupported(); }
 
     void addIndirectRoot(const Path & path) override
-    { notImpl(); }
+    { unsupported(); }
 
     Roots findRoots() override
-    { notImpl(); }
+    { unsupported(); }
 
     void collectGarbage(const GCOptions & options, GCResults & results) override
-    { notImpl(); }
+    { unsupported(); }
 
     ref<FSAccessor> getFSAccessor() override;
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { notImpl(); }
+    { unsupported(); }
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index befc560bfce..de0562aef49 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -148,12 +148,6 @@ struct LegacySSHStore : public Store
         sink(*savedNAR.data);
     }
 
-    /* Unsupported methods. */
-    [[noreturn]] void unsupported()
-    {
-        throw Error("operation not supported on SSH stores");
-    }
-
     PathSet queryAllValidPaths() override { unsupported(); }
 
     void queryReferrers(const Path & path, PathSet & referrers) override
@@ -177,9 +171,6 @@ struct LegacySSHStore : public Store
         const PathSet & references, bool repair) override
     { unsupported(); }
 
-    void buildPaths(const PathSet & paths, BuildMode buildMode) override
-    { unsupported(); }
-
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     { unsupported(); }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 835bbb90e0b..850ea211dd5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -523,6 +523,17 @@ const Store::Stats & Store::getStats()
 }
 
 
+void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
+{
+    for (auto & path : paths)
+        if (isDerivation(path))
+            unsupported();
+
+    if (queryValidPaths(paths).size() != paths.size())
+        unsupported();
+}
+
+
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, bool repair, bool dontCheckSigs)
 {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 067309c9e95..b763849ade9 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -18,6 +18,12 @@
 namespace nix {
 
 
+MakeError(SubstError, Error)
+MakeError(BuildError, Error) /* denotes a permanent build failure */
+MakeError(InvalidPath, Error)
+MakeError(Unsupported, Error)
+
+
 struct BasicDerivation;
 struct Derivation;
 class FSAccessor;
@@ -414,7 +420,7 @@ public:
        output paths can be created by running the builder, after
        recursively building any sub-derivations. For inputs that are
        not derivations, substitute them. */
-    virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0;
+    virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal);
 
     /* Build a single non-materialized derivation (i.e. not from an
        on-disk .drv file). Note that ‘drvPath’ is only used for
@@ -584,6 +590,12 @@ protected:
 
     Stats stats;
 
+    /* Unsupported methods. */
+    [[noreturn]] void unsupported()
+    {
+        throw Unsupported("requested operation is not supported by store ‘%s’", getUri());
+    }
+
 };
 
 
@@ -720,10 +732,4 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
    for paths created by makeFixedOutputPath() / addToStore(). */
 std::string makeFixedOutputCA(bool recursive, const Hash & hash);
 
-
-MakeError(SubstError, Error)
-MakeError(BuildError, Error) /* denotes a permanent build failure */
-MakeError(InvalidPath, Error)
-
-
 }

From 227a48f86f8c26c495783c5946bb75c2819bb7ac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 14:36:56 +0200
Subject: [PATCH 0291/2196] Reduce severity of EMLINK warnings

Fixes #1357.
---
 src/libstore/optimise-store.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index cf234e35d37..d354812e3da 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -220,8 +220,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
                rather than on the original link.  (Probably it
                temporarily increases the st_nlink field before
                decreasing it again.) */
-            if (st.st_size)
-                printInfo(format("‘%1%’ has maximum number of links") % linkPath);
+            debug("‘%s’ has reached maximum number of links", linkPath);
             return;
         }
         throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path);

From b986c7f8b14c1270e012f22183737ebbaa33173d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 14:43:14 +0200
Subject: [PATCH 0292/2196] Pass verbosity level to build hook

---
 src/build-remote/build-remote.cc | 9 ++++++---
 src/libstore/build.cc            | 7 ++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index d7aee288670..f3195f6317d 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -130,14 +130,15 @@ int main (int argc, char * * argv)
             putenv(ssh_env) == -1)
             throw SysError("setting SSH env vars");
 
-        if (argc != 4)
+        if (argc != 5)
             throw UsageError("called without required arguments");
 
         auto store = openStore();
 
         auto localSystem = argv[1];
-        settings.maxSilentTime = stoull(string(argv[2]));
-        settings.buildTimeout = stoull(string(argv[3]));
+        settings.maxSilentTime = std::stoll(argv[2]);
+        settings.buildTimeout = std::stoll(argv[3]);
+        verbosity = (Verbosity) std::stoll(argv[4]);
 
         currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load");
 
@@ -145,6 +146,8 @@ int main (int argc, char * * argv)
         AutoCloseFD bestSlotLock;
 
         auto machines = readConf();
+        debug("got %d remote builders", machines.size());
+
         string drvPath;
         string hostName;
         for (string line; getline(cin, line);) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 01a3203dd90..8b869063d5e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -614,10 +614,11 @@ HookInstance::HookInstance()
             throw SysError("dupping builder's stdout/stderr");
 
         Strings args = {
-            baseNameOf(buildHook),
+            baseNameOf(settings.buildHook),
             settings.thisSystem,
-            (format("%1%") % settings.maxSilentTime).str(),
-            (format("%1%") % settings.buildTimeout).str()
+            std::to_string(settings.maxSilentTime),
+            std::to_string(settings.buildTimeout),
+            std::to_string(verbosity)
         };
 
         execv(buildHook.c_str(), stringsToCharPtrs(args).data());

From ca9f589a93309ca548d772f1634169007568d6a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 15:00:39 +0200
Subject: [PATCH 0293/2196] build-remote: Don't copy the .drv closure

Since build-remote uses buildDerivation() now, we don't need to copy
the .drv file anymore. This greatly reduces the set of input paths
copied to the remote side (e.g. from 392 to 51 store paths for GNU
hello on x86_64-darwin).
---
 src/build-remote/build-remote.cc |  7 +++++--
 src/libstore/build.cc            | 17 +++--------------
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index f3195f6317d..1ee8a625b6b 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -269,8 +269,11 @@ int main (int argc, char * * argv)
         copyPaths(store, ref<Store>(sshStore), inputs);
         uploadLock = -1;
 
-        printError("building ‘%s’ on ‘%s’", drvPath, hostName);
-        sshStore->buildDerivation(drvPath, readDerivation(drvPath));
+        BasicDerivation drv(readDerivation(drvPath));
+        drv.inputSrcs = inputs;
+
+        printError("building ‘%s’ on ‘%s’", drvPath, storeUri);
+        sshStore->buildDerivation(drvPath, drv);
 
         PathSet missing;
         for (auto & path : outputs)
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8b869063d5e..a0efd880400 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1622,23 +1622,12 @@ HookReply DerivationGoal::tryBuildHook()
     hook = std::move(worker.hook);
 
     /* Tell the hook all the inputs that have to be copied to the
-       remote system.  This unfortunately has to contain the entire
-       derivation closure to ensure that the validity invariant holds
-       on the remote system.  (I.e., it's unfortunate that we have to
-       list it since the remote system *probably* already has it.) */
-    PathSet allInputs;
-    allInputs.insert(inputPaths.begin(), inputPaths.end());
-    worker.store.computeFSClosure(drvPath, allInputs);
-
-    string s;
-    for (auto & i : allInputs) { s += i; s += ' '; }
-    writeLine(hook->toHook.writeSide.get(), s);
+       remote system. */
+    writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths));
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    s = "";
-    for (auto & i : missingPaths) { s += i; s += ' '; }
-    writeLine(hook->toHook.writeSide.get(), s);
+    writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths));
 
     hook->toHook.writeSide = -1;
 

From d7653dfc6dea076ecbe00520c6137977e0fced35 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 15:46:47 +0200
Subject: [PATCH 0294/2196] Remove $NIX_BUILD_HOOK and $NIX_CURRENT_LOAD

This is to simplify remote build configuration. These environment
variables predate nix.conf.

The build hook now has a sensible default (namely build-remote).

The current load is kept in the Nix state directory now.
---
 .../advanced-topics/distributed-builds.xml    |  15 +-
 doc/manual/command-ref/env-common.xml         | 133 ------------------
 src/build-remote/build-remote.cc              |   4 +-
 src/libstore/build.cc                         |  12 +-
 src/libstore/globals.hh                       |   3 +
 tests/remote-builds.nix                       |   1 -
 6 files changed, 12 insertions(+), 156 deletions(-)

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index d5bc1c59255..1957e1105e6 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -22,10 +22,7 @@ will call whenever it wants to build a derivation.  The build hook
 will perform it in the usual way if possible, or it can accept it, in
 which case it is responsible for somehow getting the inputs of the
 build to another machine, doing the build there, and getting the
-results back.  The details of the build hook protocol are described in
-the documentation of the <link
-linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
-variable</link>.</para>
+results back.</para>
 
 <example xml:id='ex-remote-systems'><title>Remote machine configuration:
 <filename>remote-systems.conf</filename></title>
@@ -103,14 +100,6 @@ requiredSystemFeatures = [ "kvm" ];
 
 </orderedlist>
 
-You should also set up the environment variable
-<envar>NIX_CURRENT_LOAD</envar> to point at a directory (e.g.,
-<filename>/var/run/nix/current-load</filename>) that
-<filename>build-remote</filename> uses to remember how many builds
-it is currently executing remotely.  It doesn't look at the actual
-load on the remote machine, so if you have multiple instances of Nix
-running, they should use the same <envar>NIX_CURRENT_LOAD</envar>
-file.  Maybe in the future <filename>build-remote</filename> will
-look at the actual remote load.</para>
+</para>
 
 </chapter>
diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index c757cb17bd1..a83aeaf2e57 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -148,139 +148,6 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
-<varlistentry xml:id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
-
-  <listitem>
-
-  <para>Specifies the location of the <emphasis>build hook</emphasis>,
-  which is a program (typically some script) that Nix will call
-  whenever it wants to build a derivation.  This is used to implement
-  distributed builds<phrase condition="manual"> (see <xref
-  linkend="chap-distributed-builds" />)</phrase>.</para>
-
-  <!--
-  The protocol by
-  which the calling Nix process and the build hook communicate is as
-  follows.
-
-  <para>The build hook is called with the following command-line
-  arguments:
-
-  <orderedlist>
-
-    <listitem><para>A boolean value <literal>0</literal> or
-    <literal>1</literal> specifying whether Nix can locally execute
-    more builds, as per the <link
-    linkend="opt-max-jobs"><option>- -max-jobs</option> option</link>.
-    The purpose of this argument is to allow the hook to not have to
-    maintain bookkeeping for the local machine.</para></listitem>
-
-    <listitem><para>The Nix platform identifier for the local machine
-    (e.g., <literal>i686-linux</literal>).</para></listitem>
-
-    <listitem><para>The Nix platform identifier for the derivation,
-    i.e., its <link linkend="attr-system"><varname>system</varname>
-    attribute</link>.</para></listitem>
-
-    <listitem><para>The store path of the derivation.</para></listitem>
-
-  </orderedlist>
-
-  </para>
-
-  <para>On the basis of this information, and whatever persistent
-  state the build hook keeps about other machines and their current
-  load, it has to decide what to do with the build.  It should print
-  out on standard error one of the following responses (terminated by
-  a newline, <literal>"\n"</literal>):
-
-  <variablelist>
-
-    <varlistentry><term><literal># decline</literal></term>
-
-      <listitem><para>The build hook is not willing or able to perform
-      the build; the calling Nix process should do the build itself,
-      if possible.</para></listitem>
-
-    </varlistentry>
-
-    <varlistentry><term><literal># postpone</literal></term>
-
-      <listitem><para>The build hook cannot perform the build now, but
-      can do so in the future (e.g., because all available build slots
-      on remote machines are in use).  The calling Nix process should
-      postpone this build until at least one currently running build
-      has terminated.</para></listitem>
-
-    </varlistentry>
-
-    <varlistentry><term><literal># accept</literal></term>
-
-      <listitem><para>The build hook has accepted the
-      build.</para></listitem>
-
-    </varlistentry>
-
-  </variablelist>
-
-  </para>
-
-  <para>After sending <literal># accept</literal>, the hook should
-  read one line from standard input, which will be the string
-  <literal>okay</literal>.  It can then proceed with the build.
-  Before sending <literal>okay</literal>, Nix will store in the hook’s
-  current directory a number of text files that contain information
-  about the derivation:
-
-  <variablelist>
-
-    <varlistentry><term><filename>inputs</filename></term>
-
-      <listitem><para>The set of store paths that are inputs to the
-      build process (one per line).  These have to be copied
-      <emphasis>to</emphasis> the remote machine (in addition to the
-      store derivation itself).</para></listitem>
-
-    </varlistentry>
-
-    <varlistentry><term><filename>outputs</filename></term>
-
-      <listitem><para>The set of store paths that are outputs of the
-      derivation (one per line).  These have to be copied
-      <emphasis>from</emphasis> the remote machine if the build
-      succeeds.</para></listitem>
-
-    </varlistentry>
-
-    <varlistentry><term><filename>references</filename></term>
-
-      <listitem><para>The reference graph of the inputs, in the format
-      accepted by the command <command>nix-store
-      - -register-validity</command>.  It is necessary to run this
-      command on the remote machine after copying the inputs to inform
-      Nix on the remote machine that the inputs are valid
-      paths.</para></listitem>
-
-    </varlistentry>
-
-  </variablelist>
-
-  </para>
-
-  <para>The hook should copy the inputs to the remote machine,
-  register the validity of the inputs, perform the remote build, and
-  copy the outputs back to the local machine.  An exit code other than
-  <literal>0</literal> indicates that the hook has failed.  An exit
-  code equal to 100 means that the remote build failed (as opposed to,
-  e.g., a network error).</para>
-  -->
-
-  </listitem>
-
-
-</varlistentry>
-
-
 <varlistentry xml:id="envar-remote"><term><envar>NIX_REMOTE</envar></term>
 
   <listitem><para>This variable should be set to
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 1ee8a625b6b..c41383bcf27 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -140,7 +140,9 @@ int main (int argc, char * * argv)
         settings.buildTimeout = std::stoll(argv[3]);
         verbosity = (Verbosity) std::stoll(argv[4]);
 
-        currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load");
+        /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
+           that gets cleared on reboot, but it wouldn't work on OS X. */
+        currentLoad = settings.nixStateDir + "/current-load";
 
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a0efd880400..9b3f799b9f2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -583,11 +583,7 @@ struct HookInstance
 
 HookInstance::HookInstance()
 {
-    debug("starting build hook");
-
-    Path buildHook = getEnv("NIX_BUILD_HOOK");
-    if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook;
-    buildHook = canonPath(buildHook);
+    debug("starting build hook ‘%s’", settings.buildHook);
 
     /* Create a pipe to get the output of the child. */
     fromHook.create();
@@ -621,9 +617,9 @@ HookInstance::HookInstance()
             std::to_string(verbosity)
         };
 
-        execv(buildHook.c_str(), stringsToCharPtrs(args).data());
+        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError(format("executing ‘%1%’") % buildHook);
+        throw SysError("executing ‘%s’", settings.buildHook);
     });
 
     pid.setSeparatePG(true);
@@ -1569,7 +1565,7 @@ void DerivationGoal::buildDone()
 
 HookReply DerivationGoal::tryBuildHook()
 {
-    if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline;
+    if (!settings.useBuildHook || !useDerivation) return rpDecline;
 
     if (!worker.hook)
         worker.hook = std::make_unique<HookInstance>();
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b4f44de2e65..25cc3e068ee 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -127,6 +127,9 @@ public:
     Setting<bool> useBuildHook{this, true, "remote-builds",
         "Whether to use build hooks (for distributed builds)."};
 
+    PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
+        "The path of the helper program that executes builds to remote machines."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 63aaa4d88f5..39bd090e43e 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -43,7 +43,6 @@ in
         { config, pkgs, ... }:
         { nix.maxJobs = 0; # force remote building
           nix.distributedBuilds = true;
-          nix.envVars = pkgs.lib.mkAfter { NIX_BUILD_HOOK = "${nix}/libexec/nix/build-remote"; };
           nix.buildMachines =
             [ { hostName = "slave1";
                 sshUser = "root";

From 3f5b98e65a86abd31f97bd763ae5cb41ff4aeda8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 15:59:43 +0200
Subject: [PATCH 0295/2196] Chomp log output from the build hook

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9b3f799b9f2..ec368463252 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3091,7 +3091,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     }
 
     if (hook && fd == hook->fromHook.readSide.get())
-        printError(data); // FIXME?
+        printError(chomp(data));
 }
 
 

From deac171925bf2e3960d2f837d95b71c0427d26dd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 16:08:13 +0200
Subject: [PATCH 0296/2196] Implement LegacySSHStore::buildDerivation()

This makes LegacySSHStore usable by build-remote and
hydra-queue-runner.
---
 src/libstore/legacy-ssh-store.cc | 37 +++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index de0562aef49..d6b70b99297 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -5,6 +5,7 @@
 #include "store-api.hh"
 #include "worker-protocol.hh"
 #include "ssh.hh"
+#include "derivations.hh"
 
 namespace nix {
 
@@ -21,6 +22,7 @@ struct LegacySSHStore : public Store
         std::unique_ptr<SSHMaster::Connection> sshConn;
         FdSink to;
         FdSource from;
+        int remoteVersion;
     };
 
     std::string host;
@@ -53,8 +55,6 @@ struct LegacySSHStore : public Store
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
 
-        int remoteVersion;
-
         try {
             conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
             conn->to.flush();
@@ -62,8 +62,8 @@ struct LegacySSHStore : public Store
             unsigned int magic = readInt(conn->from);
             if (magic != SERVE_MAGIC_2)
                 throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host);
-            remoteVersion = readInt(conn->from);
-            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
+            conn->remoteVersion = readInt(conn->from);
+            if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
                 throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host);
 
         } catch (EndOfFile & e) {
@@ -173,7 +173,34 @@ struct LegacySSHStore : public Store
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
-    { unsupported(); }
+    {
+        auto conn(connections->get());
+
+        conn->to
+            << cmdBuildDerivation
+            << drvPath
+            << drv
+            << settings.maxSilentTime
+            << settings.buildTimeout;
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
+            conn->to
+                << settings.maxLogSize;
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
+            conn->to
+                << settings.buildRepeat
+                << settings.enforceDeterminism;
+
+        conn->to.flush();
+
+        BuildResult status;
+        status.status = (BuildResult::Status) readInt(conn->from);
+        conn->from >> status.errorMsg;
+
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
+            conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
+
+        return status;
+    }
 
     void ensurePath(const Path & path) override
     { unsupported(); }

From 3e4bdfedee747868a32b8a9c7a89a6f860415be8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 17:28:19 +0200
Subject: [PATCH 0297/2196] Minor cleanup

---
 src/libutil/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 026e493514e..98c0aff1e72 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1078,9 +1078,9 @@ bool statusOk(int status)
 }
 
 
-bool hasPrefix(const string & s, const string & suffix)
+bool hasPrefix(const string & s, const string & prefix)
 {
-    return s.compare(0, suffix.size(), suffix) == 0;
+    return s.compare(0, prefix.size(), prefix) == 0;
 }
 
 

From 031d70e5009fcce58afabc9113d5a5de4a16b19a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 17:35:30 +0200
Subject: [PATCH 0298/2196] Support arbitrary store URIs in nix.machines

For backwards compatibility, if the URI is just a hostname, ssh://
(i.e. LegacySSHStore) is prepended automatically.

Also, all fields except the URI are now optional. For example, this is
a valid nix.machines file:

  local?root=/tmp/nix

This is useful for testing the remote build machinery since you don't
have to mess around with ssh.
---
 src/build-remote/build-remote.cc | 77 ++++++++++++++++++--------------
 src/libstore/build.cc            |  1 +
 src/libstore/store-api.cc        | 11 ++---
 src/libstore/store-api.hh        | 23 +++++++---
 4 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index c41383bcf27..a19dac241d0 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -27,12 +27,12 @@ class Machine {
     const std::set<string> mandatoryFeatures;
 
 public:
-    const string hostName;
+    const string storeUri;
     const std::vector<string> systemTypes;
     const string sshKey;
     const unsigned int maxJobs;
     const unsigned int speedFactor;
-    bool enabled;
+    bool enabled = true;
 
     bool allSupported(const std::set<string> & features) const {
         return std::all_of(features.begin(), features.end(),
@@ -49,7 +49,7 @@ class Machine {
             });
     }
 
-    Machine(decltype(hostName) hostName,
+    Machine(decltype(storeUri) storeUri,
         decltype(systemTypes) systemTypes,
         decltype(sshKey) sshKey,
         decltype(maxJobs) maxJobs,
@@ -58,14 +58,18 @@ class Machine {
         decltype(mandatoryFeatures) mandatoryFeatures) :
         supportedFeatures(supportedFeatures),
         mandatoryFeatures(mandatoryFeatures),
-        hostName(hostName),
+        storeUri(
+            // Backwards compatibility: if the URI is a hostname,
+            // prepend ssh://.
+            storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto")
+            ? storeUri
+            : "ssh://" + storeUri),
         systemTypes(systemTypes),
         sshKey(sshKey),
         maxJobs(maxJobs),
-        speedFactor(std::max(1U, speedFactor)),
-        enabled(true)
-    {};
-};;
+        speedFactor(std::max(1U, speedFactor))
+    {}
+};
 
 static std::vector<Machine> readConf()
 {
@@ -87,13 +91,13 @@ static std::vector<Machine> readConf()
         }
         auto tokens = tokenizeString<std::vector<string>>(line);
         auto sz = tokens.size();
-        if (sz < 4)
+        if (sz < 1)
             throw FormatError("bad machines.conf file ‘%1%’", conf);
         machines.emplace_back(tokens[0],
-            tokenizeString<std::vector<string>>(tokens[1], ","),
-            tokens[2],
-            stoull(tokens[3]),
-            sz >= 5 ? stoull(tokens[4]) : 1LL,
+            sz >= 2 ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
+            sz >= 3 ? tokens[2] : "",
+            sz >= 4 ? std::stoull(tokens[3]) : 1LL,
+            sz >= 5 ? std::stoull(tokens[4]) : 1LL,
             sz >= 6 ?
             tokenizeString<std::set<string>>(tokens[5], ",") :
             std::set<string>{},
@@ -104,31 +108,27 @@ static std::vector<Machine> readConf()
     return machines;
 }
 
+std::string escapeUri(std::string uri)
+{
+    std::replace(uri.begin(), uri.end(), '/', '_');
+    return uri;
+}
+
 static string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
 {
-    std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out);
-    fn_stream << "/";
-    for (auto t : m.systemTypes) {
-        fn_stream << t << "-";
-    }
-    fn_stream << m.hostName << "-" << slot;
-    return openLockFile(fn_stream.str(), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-static char display_env[] = "DISPLAY=";
-static char ssh_env[] = "SSH_ASKPASS=";
-
 int main (int argc, char * * argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
-        if (putenv(display_env) == -1 ||
-            putenv(ssh_env) == -1)
-            throw SysError("setting SSH env vars");
+        unsetenv("DISPLAY");
+        unsetenv("SSH_ASKPASS");
 
         if (argc != 5)
             throw UsageError("called without required arguments");
@@ -151,7 +151,7 @@ int main (int argc, char * * argv)
         debug("got %d remote builders", machines.size());
 
         string drvPath;
-        string hostName;
+        string storeUri;
         for (string line; getline(cin, line);) {
             auto tokens = tokenizeString<std::vector<string>>(line);
             auto sz = tokens.size();
@@ -178,6 +178,8 @@ int main (int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 unsigned long long bestLoad = 0;
                 for (auto & m : machines) {
+                    debug("considering building on ‘%s’", m.storeUri);
+
                     if (m.enabled && std::find(m.systemTypes.begin(),
                             m.systemTypes.end(),
                             neededSystem) != m.systemTypes.end() &&
@@ -238,16 +240,21 @@ int main (int argc, char * * argv)
                 lock = -1;
 
                 try {
-                    sshStore = openStore("ssh-ng://" + bestMachine->hostName,
-                        { {"ssh-key", bestMachine->sshKey },
-                          {"max-connections", "1" } });
-                    hostName = bestMachine->hostName;
+
+                    Store::Params storeParams{{"max-connections", "1"}};
+                    if (bestMachine->sshKey != "")
+                        storeParams["ssh-key"] = bestMachine->sshKey;
+
+                    sshStore = openStore(bestMachine->storeUri, storeParams);
+                    storeUri = bestMachine->storeUri;
+
                 } catch (std::exception & e) {
                     printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...",
-                        bestMachine->hostName, e.what());
+                        bestMachine->storeUri, e.what());
                     bestMachine->enabled = false;
                     continue;
                 }
+
                 goto connected;
             }
         }
@@ -257,11 +264,15 @@ int main (int argc, char * * argv)
         string line;
         if (!getline(cin, line))
             throw Error("hook caller didn't send inputs");
+
         auto inputs = tokenizeString<PathSet>(line);
         if (!getline(cin, line))
             throw Error("hook caller didn't send outputs");
+
         auto outputs = tokenizeString<PathSet>(line);
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true);
+
+        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+
         auto old = signal(SIGALRM, handleAlarm);
         alarm(15 * 60);
         if (!lockFile(uploadLock.get(), ltWrite, true))
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ec368463252..a9649ea378c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1862,6 +1862,7 @@ void DerivationGoal::startBuilder()
                 dirsInChroot[i] = r;
             else {
                 Path p = chrootRootDir + i;
+                debug("linking ‘%1%’ to ‘%2%’", p, r);
                 if (link(r.c_str(), p.c_str()) == -1) {
                     /* Hard-linking fails if we exceed the maximum
                        link count on a file (e.g. 32000 of ext3),
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 850ea211dd5..75de4c93323 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -709,10 +709,11 @@ namespace nix {
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
 
-ref<Store> openStore(const std::string & uri_)
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
 {
     auto uri(uri_);
-    Store::Params params;
+    Store::Params params(extraParams);
     auto q = uri.find('?');
     if (q != std::string::npos) {
         for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@@ -722,11 +723,7 @@ ref<Store> openStore(const std::string & uri_)
         }
         uri = uri_.substr(0, q);
     }
-    return openStore(uri, params);
-}
 
-ref<Store> openStore(const std::string & uri, const Store::Params & params)
-{
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
@@ -735,7 +732,7 @@ ref<Store> openStore(const std::string & uri, const Store::Params & params)
         }
     }
 
-    throw Error(format("don't know how to open Nix store ‘%s’") % uri);
+    throw Error("don't know how to open Nix store ‘%s’", uri);
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b763849ade9..2388558b362 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -668,20 +668,31 @@ void removeTempRoots();
 /* Return a Store object to access the Nix store denoted by
    ‘uri’ (slight misnomer...). Supported values are:
 
-   * ‘direct’: The Nix store in /nix/store and database in
+   * ‘local’: The Nix store in /nix/store and database in
      /nix/var/nix/db, accessed directly.
 
    * ‘daemon’: The Nix store accessed via a Unix domain socket
      connection to nix-daemon.
 
+   * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
+     whether the user has write access to the local Nix
+     store/database.
+
    * ‘file://<path>’: A binary cache stored in <path>.
 
-   If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on
-   whether the user has write access to the local Nix store/database.
-   set to true *unless* you're going to collect garbage. */
-ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
+   * ‘https://<path>’: A binary cache accessed via HTTP.
+
+   * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
+     Storage Service.
+
+   * ‘ssh://[user@]<host>’: A remote Nix store accessed by running
+     ‘nix-store --serve’ via SSH.
 
-ref<Store> openStore(const std::string & uri, const Store::Params & params);
+   You can pass parameters to the store implementation by appending
+   ‘?key=value&key=value&...’ to the URI.
+*/
+ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"),
+    const Store::Params & extraParams = Store::Params());
 
 
 void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false);

From 3a5f04f48cc39eec5cc454e387aa290e08295aff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 May 2017 20:03:25 +0200
Subject: [PATCH 0299/2196] build-remote: Don't require signatures

This restores the old behaviour.
---
 src/build-remote/build-remote.cc         |  4 ++--
 src/libstore/legacy-ssh-store.cc         |  3 ---
 src/libstore/local-store.cc              |  2 ++
 src/libstore/store-api.cc                | 18 +++++++++++++-----
 src/libstore/store-api.hh                |  7 ++-----
 src/nix-copy-closure/nix-copy-closure.cc |  2 +-
 6 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index a19dac241d0..ba909ec44d6 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -279,7 +279,7 @@ int main (int argc, char * * argv)
             printError("somebody is hogging the upload lock for ‘%s’, continuing...");
         alarm(0);
         signal(SIGALRM, old);
-        copyPaths(store, ref<Store>(sshStore), inputs);
+        copyPaths(store, ref<Store>(sshStore), inputs, false, true);
         uploadLock = -1;
 
         BasicDerivation drv(readDerivation(drvPath));
@@ -294,7 +294,7 @@ int main (int argc, char * * argv)
 
         if (!missing.empty()) {
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing);
+            copyPaths(ref<Store>(sshStore), store, missing, false, true);
         }
 
         return;
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index d6b70b99297..1f6ea4dc1cf 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -223,9 +223,6 @@ struct LegacySSHStore : public Store
     void addSignatures(const Path & storePath, const StringSet & sigs) override
     { unsupported(); }
 
-    bool isTrusted() override
-    { return true; }
-
     void computeFSClosure(const PathSet & paths,
         PathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false) override
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5a98454ab38..c8e61126c1b 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -915,6 +915,8 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
 {
+    assert(info.narHash);
+
     Hash h = hashString(htSHA256, *nar);
     if (h != info.narHash)
         throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") %
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 75de4c93323..b5a91e53672 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -542,15 +542,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
-    if (srcStore->isTrusted())
-        dontCheckSigs = true;
-
     if (!info->narHash && dontCheckSigs) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         info = info2;
     }
 
+    assert(info->narHash);
+
+    if (info->ultimate) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->ultimate = false;
+        info = info2;
+    }
+
+    assert(info->narHash);
+
     dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
 }
 
@@ -802,7 +809,8 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute)
+void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
+    bool substitute, bool dontCheckSigs)
 {
     PathSet valid = to->queryValidPaths(storePaths, substitute);
 
@@ -830,7 +838,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool
             if (!to->isValidPath(storePath)) {
                 Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
 
-                copyStorePath(from, to, storePath);
+                copyStorePath(from, to, storePath, false, dontCheckSigs);
 
                 logger->incProgress(copiedLabel);
             } else
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 2388558b362..8ca3f4b27d0 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -570,10 +570,6 @@ public:
 
     const Stats & getStats();
 
-    /* Whether this store paths from this store can be imported even
-       if they lack a signature. */
-    virtual bool isTrusted() { return false; }
-
     /* Return the build log of the specified store path, if available,
        or null otherwise. */
     virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
@@ -695,7 +691,8 @@ ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"),
     const Store::Params & extraParams = Store::Params());
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false);
+void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
+    bool substitute = false, bool dontCheckSigs = false);
 
 enum StoreType {
     tDaemon,
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index ed43bffbc8c..dc324abcb3b 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -58,6 +58,6 @@ int main(int argc, char ** argv)
         PathSet closure;
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
-        copyPaths(from, to, closure, useSubstitutes);
+        copyPaths(from, to, closure, useSubstitutes, true);
     });
 }

From feefcb3a982d3e3b8e89798d72d8afa996169569 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 12:01:46 +0200
Subject: [PATCH 0300/2196] build-remote: Ugly hackery to get build logs to
 work

The build hook mechanism expects build log output to go to file
descriptor 4, so do that.
---
 src/build-remote/build-remote.cc | 2 +-
 src/libstore/legacy-ssh-store.cc | 6 +++++-
 src/libstore/ssh.cc              | 2 ++
 src/libstore/ssh.hh              | 4 +++-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index ba909ec44d6..acf571ff156 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -241,7 +241,7 @@ int main (int argc, char * * argv)
 
                 try {
 
-                    Store::Params storeParams{{"max-connections", "1"}};
+                    Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}};
                     if (bestMachine->sshKey != "")
                         storeParams["ssh-key"] = bestMachine->sshKey;
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 1f6ea4dc1cf..a6479a450e3 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -17,6 +17,9 @@ struct LegacySSHStore : public Store
     const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
 
+    // Hack for getting remote build log output.
+    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+
     struct Connection
     {
         std::unique_ptr<SSHMaster::Connection> sshConn;
@@ -44,7 +47,8 @@ struct LegacySSHStore : public Store
             sshKey,
             // Use SSH master only if using more than 1 connection.
             connections->capacity() > 1,
-            compress)
+            compress,
+            logFD)
     {
     }
 
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index e54f3f4ba28..6edabaa3a1d 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -31,6 +31,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             throw SysError("duping over stdin");
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("duping over stdout");
+        if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1)
+            throw SysError("duping over stderr");
 
         Strings args = { "ssh", host.c_str(), "-x", "-a" };
         addCommonSSHOpts(args);
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index b4396467e54..18dea227ad1 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -13,6 +13,7 @@ private:
     const std::string keyFile;
     const bool useMaster;
     const bool compress;
+    const int logFD;
 
     struct State
     {
@@ -27,11 +28,12 @@ private:
 
 public:
 
-    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress)
+    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1)
         : host(host)
         , keyFile(keyFile)
         , useMaster(useMaster)
         , compress(compress)
+        , logFD(logFD)
     {
     }
 

From 70581b63633016329789872c73dc48b1d498c729 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 12:07:56 +0200
Subject: [PATCH 0301/2196] Fix build hook test

---
 tests/build-hook.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/build-hook.sh b/tests/build-hook.sh
index ef77a3ae528..2005c7cebdc 100644
--- a/tests/build-hook.sh
+++ b/tests/build-hook.sh
@@ -1,8 +1,8 @@
 source common.sh
 
-export NIX_BUILD_HOOK="$(pwd)/build-hook.hook.sh"
+clearStore
 
-outPath=$(nix-build build-hook.nix --no-out-link)
+outPath=$(nix-build build-hook.nix --no-out-link --option build-hook $(pwd)/build-hook.hook.sh)
 
 echo "output path is $outPath"
 

From 174b68a2a2e9e58fa1a1a0036858a566c51684dc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 12:16:29 +0200
Subject: [PATCH 0302/2196] build-hook: If there are no machines defined, quit
 permanently

---
 src/build-remote/build-remote.cc |  5 +++++
 src/libstore/build.cc            | 11 +++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index acf571ff156..388f1e04686 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -150,6 +150,11 @@ int main (int argc, char * * argv)
         auto machines = readConf();
         debug("got %d remote builders", machines.size());
 
+        if (machines.empty()) {
+            std::cerr << "# decline-permanently\n";
+            return;
+        }
+
         string drvPath;
         string storeUri;
         for (string line; getline(cin, line);) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a9649ea378c..e756d3377c3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1598,8 +1598,15 @@ HookReply DerivationGoal::tryBuildHook()
 
         debug(format("hook reply is ‘%1%’") % reply);
 
-        if (reply == "decline" || reply == "postpone")
-            return reply == "decline" ? rpDecline : rpPostpone;
+        if (reply == "decline")
+            return rpDecline;
+        else if (reply == "decline-permanently") {
+            settings.useBuildHook = false;
+            worker.hook = 0;
+            return rpDecline;
+        }
+        else if (reply == "postpone")
+            return rpPostpone;
         else if (reply != "accept")
             throw Error(format("bad hook reply ‘%1%’") % reply);
 

From ebc9f36a8111ddecc8e265e8a6a70048218f244d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 13:17:37 +0200
Subject: [PATCH 0303/2196] Factor out machines.conf parsing

This allows hydra-queue-runner to use it.
---
 src/build-remote/build-remote.cc | 95 +++-----------------------------
 src/libstore/machines.cc         | 65 ++++++++++++++++++++++
 src/libstore/machines.hh         | 35 ++++++++++++
 3 files changed, 108 insertions(+), 87 deletions(-)
 create mode 100644 src/libstore/machines.cc
 create mode 100644 src/libstore/machines.hh

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 388f1e04686..8a9d4571fef 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -9,6 +9,7 @@
 #include <sys/time.h>
 #endif
 
+#include "machines.hh"
 #include "shared.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
@@ -22,92 +23,6 @@ using std::cin;
 static void handleAlarm(int sig) {
 }
 
-class Machine {
-    const std::set<string> supportedFeatures;
-    const std::set<string> mandatoryFeatures;
-
-public:
-    const string storeUri;
-    const std::vector<string> systemTypes;
-    const string sshKey;
-    const unsigned int maxJobs;
-    const unsigned int speedFactor;
-    bool enabled = true;
-
-    bool allSupported(const std::set<string> & features) const {
-        return std::all_of(features.begin(), features.end(),
-            [&](const string & feature) {
-                return supportedFeatures.count(feature) ||
-                    mandatoryFeatures.count(feature);
-            });
-    }
-
-    bool mandatoryMet(const std::set<string> & features) const {
-        return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
-            [&](const string & feature) {
-                return features.count(feature);
-            });
-    }
-
-    Machine(decltype(storeUri) storeUri,
-        decltype(systemTypes) systemTypes,
-        decltype(sshKey) sshKey,
-        decltype(maxJobs) maxJobs,
-        decltype(speedFactor) speedFactor,
-        decltype(supportedFeatures) supportedFeatures,
-        decltype(mandatoryFeatures) mandatoryFeatures) :
-        supportedFeatures(supportedFeatures),
-        mandatoryFeatures(mandatoryFeatures),
-        storeUri(
-            // Backwards compatibility: if the URI is a hostname,
-            // prepend ssh://.
-            storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto")
-            ? storeUri
-            : "ssh://" + storeUri),
-        systemTypes(systemTypes),
-        sshKey(sshKey),
-        maxJobs(maxJobs),
-        speedFactor(std::max(1U, speedFactor))
-    {}
-};
-
-static std::vector<Machine> readConf()
-{
-    auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines");
-
-    auto machines = std::vector<Machine>{};
-    auto lines = std::vector<string>{};
-    try {
-        lines = tokenizeString<std::vector<string>>(readFile(conf), "\n");
-    } catch (const SysError & e) {
-        if (e.errNo != ENOENT)
-            throw;
-    }
-    for (auto line : lines) {
-        chomp(line);
-        line.erase(std::find(line.begin(), line.end(), '#'), line.end());
-        if (line.empty()) {
-            continue;
-        }
-        auto tokens = tokenizeString<std::vector<string>>(line);
-        auto sz = tokens.size();
-        if (sz < 1)
-            throw FormatError("bad machines.conf file ‘%1%’", conf);
-        machines.emplace_back(tokens[0],
-            sz >= 2 ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
-            sz >= 3 ? tokens[2] : "",
-            sz >= 4 ? std::stoull(tokens[3]) : 1LL,
-            sz >= 5 ? std::stoull(tokens[4]) : 1LL,
-            sz >= 6 ?
-            tokenizeString<std::set<string>>(tokens[5], ",") :
-            std::set<string>{},
-            sz >= 7 ?
-            tokenizeString<std::set<string>>(tokens[6], ",") :
-            std::set<string>{});
-    }
-    return machines;
-}
-
 std::string escapeUri(std::string uri)
 {
     std::replace(uri.begin(), uri.end(), '/', '_');
@@ -147,7 +62,13 @@ int main (int argc, char * * argv)
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
 
-        auto machines = readConf();
+        Machines machines;
+        try {
+            parseMachines(readFile(getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines")), machines);
+        } catch (const SysError & e) {
+            if (e.errNo != ENOENT)
+                throw;
+        }
         debug("got %d remote builders", machines.size());
 
         if (machines.empty()) {
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
new file mode 100644
index 00000000000..471ce8efb9a
--- /dev/null
+++ b/src/libstore/machines.cc
@@ -0,0 +1,65 @@
+#include "machines.hh"
+#include "util.hh"
+#include "globals.hh"
+
+#include <algorithm>
+
+namespace nix {
+
+Machine::Machine(decltype(storeUri) storeUri,
+    decltype(systemTypes) systemTypes,
+    decltype(sshKey) sshKey,
+    decltype(maxJobs) maxJobs,
+    decltype(speedFactor) speedFactor,
+    decltype(supportedFeatures) supportedFeatures,
+    decltype(mandatoryFeatures) mandatoryFeatures) :
+    storeUri(
+        // Backwards compatibility: if the URI is a hostname,
+        // prepend ssh://.
+        storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto")
+        ? storeUri
+        : "ssh://" + storeUri),
+    systemTypes(systemTypes),
+    sshKey(sshKey),
+    maxJobs(maxJobs),
+    speedFactor(std::max(1U, speedFactor)),
+    supportedFeatures(supportedFeatures),
+    mandatoryFeatures(mandatoryFeatures)
+{}
+
+bool Machine::allSupported(const std::set<string> & features) const {
+    return std::all_of(features.begin(), features.end(),
+        [&](const string & feature) {
+            return supportedFeatures.count(feature) ||
+                mandatoryFeatures.count(feature);
+        });
+}
+
+bool Machine::mandatoryMet(const std::set<string> & features) const {
+    return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
+        [&](const string & feature) {
+            return features.count(feature);
+        });
+}
+
+void parseMachines(const std::string & s, Machines & machines)
+{
+    for (auto line : tokenizeString<std::vector<string>>(s, "\n")) {
+        chomp(line);
+        line.erase(std::find(line.begin(), line.end(), '#'), line.end());
+        if (line.empty()) continue;
+        auto tokens = tokenizeString<std::vector<string>>(line);
+        auto sz = tokens.size();
+        if (sz < 1)
+            throw FormatError("bad machine specification ‘%s’", line);
+        machines.emplace_back(tokens[0],
+            sz >= 2 ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
+            sz >= 3 ? tokens[2] : "",
+            sz >= 4 ? std::stoull(tokens[3]) : 1LL,
+            sz >= 5 ? std::stoull(tokens[4]) : 1LL,
+            sz >= 6 ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
+            sz >= 7 ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{});
+    }
+}
+
+}
diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh
new file mode 100644
index 00000000000..96c4bd81a46
--- /dev/null
+++ b/src/libstore/machines.hh
@@ -0,0 +1,35 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+struct Machine {
+
+    const string storeUri;
+    const std::vector<string> systemTypes;
+    const string sshKey;
+    const unsigned int maxJobs;
+    const unsigned int speedFactor;
+    const std::set<string> supportedFeatures;
+    const std::set<string> mandatoryFeatures;
+    bool enabled = true;
+
+    bool allSupported(const std::set<string> & features) const;
+
+    bool mandatoryMet(const std::set<string> & features) const;
+
+    Machine(decltype(storeUri) storeUri,
+        decltype(systemTypes) systemTypes,
+        decltype(sshKey) sshKey,
+        decltype(maxJobs) maxJobs,
+        decltype(speedFactor) speedFactor,
+        decltype(supportedFeatures) supportedFeatures,
+        decltype(mandatoryFeatures) mandatoryFeatures);
+};
+
+typedef std::vector<Machine> Machines;
+
+void parseMachines(const std::string & s, Machines & machines);
+
+}

From 1a68710d4dff609bbaf61db3e17a2573f0aadf17 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 13:44:10 +0200
Subject: [PATCH 0304/2196] Add an option for specifying remote builders

This is useful for one-off situations where you want to specify a
builder on the command line instead of having to mess with
nix.machines. E.g.

  $ nix-build -A hello --argstr system x86_64-darwin \
    --option builders 'root@macstadium1 x86_64-darwin'

will perform the specified build on "macstadium1".

It also removes the need for a separate nix.machines file since you
can specify builders in nix.conf directly. (In fact nix.machines is
yet another hack that predates the general nix.conf configuration
file, IIRC.)

Note: this option is supported by the daemon for trusted users. The
fact that this allows trusted users to specify paths to SSH keys to
which they don't normally have access is maybe a bit too much trust...
---
 src/build-remote/build-remote.cc | 11 +++--------
 src/build-remote/local.mk        |  2 --
 src/libstore/build.cc            |  3 ++-
 src/libstore/globals.hh          |  3 +++
 src/libstore/machines.cc         | 18 +++++++++++++++++-
 src/libstore/machines.hh         |  2 ++
 6 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8a9d4571fef..5cd3c518b78 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -45,7 +45,7 @@ int main (int argc, char * * argv)
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
 
-        if (argc != 5)
+        if (argc != 6)
             throw UsageError("called without required arguments");
 
         auto store = openStore();
@@ -54,6 +54,7 @@ int main (int argc, char * * argv)
         settings.maxSilentTime = std::stoll(argv[2]);
         settings.buildTimeout = std::stoll(argv[3]);
         verbosity = (Verbosity) std::stoll(argv[4]);
+        settings.builders = argv[5];
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on OS X. */
@@ -62,13 +63,7 @@ int main (int argc, char * * argv)
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;
 
-        Machines machines;
-        try {
-            parseMachines(readFile(getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines")), machines);
-        } catch (const SysError & e) {
-            if (e.errNo != ENOENT)
-                throw;
-        }
+        auto machines = getMachines();
         debug("got %d remote builders", machines.size());
 
         if (machines.empty()) {
diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
index 62d5a010c24..64368a43ff7 100644
--- a/src/build-remote/local.mk
+++ b/src/build-remote/local.mk
@@ -7,5 +7,3 @@ build-remote_INSTALL_DIR := $(libexecdir)/nix
 build-remote_LIBS = libmain libutil libformat libstore
 
 build-remote_SOURCES := $(d)/build-remote.cc
-
-build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e756d3377c3..8c2602a701b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -614,7 +614,8 @@ HookInstance::HookInstance()
             settings.thisSystem,
             std::to_string(settings.maxSilentTime),
             std::to_string(settings.buildTimeout),
-            std::to_string(verbosity)
+            std::to_string(verbosity),
+            settings.builders
         };
 
         execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 25cc3e068ee..d7a0b86a088 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -130,6 +130,9 @@ public:
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
+    Setting<std::string> builders{this, "", "builders",
+        "A semicolon-separated list of build machines, in the format of nix.machines."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 471ce8efb9a..479ed1432fb 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -44,7 +44,7 @@ bool Machine::mandatoryMet(const std::set<string> & features) const {
 
 void parseMachines(const std::string & s, Machines & machines)
 {
-    for (auto line : tokenizeString<std::vector<string>>(s, "\n")) {
+    for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
         chomp(line);
         line.erase(std::find(line.begin(), line.end(), '#'), line.end());
         if (line.empty()) continue;
@@ -62,4 +62,20 @@ void parseMachines(const std::string & s, Machines & machines)
     }
 }
 
+Machines getMachines()
+{
+    Machines machines;
+
+    try {
+        parseMachines(readFile(getEnv("NIX_REMOTE_SYSTEMS", settings.nixConfDir + "/machines")), machines);
+    } catch (const SysError & e) {
+        if (e.errNo != ENOENT)
+            throw;
+    }
+
+    parseMachines(settings.builders, machines);
+
+    return machines;
+}
+
 }
diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh
index 96c4bd81a46..e0455742844 100644
--- a/src/libstore/machines.hh
+++ b/src/libstore/machines.hh
@@ -32,4 +32,6 @@ typedef std::vector<Machine> Machines;
 
 void parseMachines(const std::string & s, Machines & machines);
 
+Machines getMachines();
+
 }

From cd4d2705ec6e641ffa3b11dc1aabad22fc38251a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 14:18:46 +0200
Subject: [PATCH 0305/2196] build-remote: Fix fallback to other machines when
 connecting fails

Opening an SSHStore or LegacySSHStore does not actually establish a
connection, so the try/catch block here did nothing. Added a
Store::connect() method to test whether a connection can be
established.
---
 src/build-remote/build-remote.cc | 1 +
 src/libstore/legacy-ssh-store.cc | 5 +++++
 src/libstore/remote-store.cc     | 8 +++++++-
 src/libstore/remote-store.hh     | 2 ++
 src/libstore/store-api.hh        | 4 ++++
 5 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 5cd3c518b78..8876da6c063 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -167,6 +167,7 @@ int main (int argc, char * * argv)
                         storeParams["ssh-key"] = bestMachine->sshKey;
 
                     sshStore = openStore(bestMachine->storeUri, storeParams);
+                    sshStore->connect();
                     storeUri = bestMachine->storeUri;
 
                 } catch (std::exception & e) {
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index a6479a450e3..e09932e3d18 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -262,6 +262,11 @@ struct LegacySSHStore : public Store
 
         return readStorePaths<PathSet>(*this, conn->from);
     }
+
+    void connect() override
+    {
+        auto conn(connections->get());
+    }
 };
 
 static RegisterStoreImplementation regStore([](
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index af59d51106f..be8819bbc00 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -100,7 +100,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
         throw Error(format("socket path ‘%1%’ is too long") % socketPath);
     strcpy(addr.sun_path, socketPath.c_str());
 
-    if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
+    if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
         throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath);
 
     conn->from.fd = conn->fd.get();
@@ -613,6 +613,12 @@ void RemoteStore::queryMissing(const PathSet & targets,
 }
 
 
+void RemoteStore::connect()
+{
+    auto conn(connections->get());
+}
+
+
 RemoteStore::Connection::~Connection()
 {
     try {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 479cf3a7909..ed430e4cabb 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -92,6 +92,8 @@ public:
         PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
         unsigned long long & downloadSize, unsigned long long & narSize) override;
 
+    void connect() override;
+
 protected:
 
     struct Connection
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8ca3f4b27d0..b06f5d86a93 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -582,6 +582,10 @@ public:
         state.lock()->pathInfoCache.clear();
     }
 
+    /* Establish a connection to the store, for store types that have
+       a notion of connection. Otherwise this is a no-op. */
+    virtual void connect() { };
+
 protected:
 
     Stats stats;

From 7f6837a0f6e7702a9e8c6da622873b955aa414cd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 14:36:59 +0200
Subject: [PATCH 0306/2196] Replace $NIX_REMOTE_SYSTEMS with an option
 "builder-files"

Also, to unify with hydra-queue-runner, allow it to be a list of
files.
---
 src/libstore/globals.cc  |  4 ++++
 src/libstore/globals.hh  |  4 ++++
 src/libstore/machines.cc | 36 +++++++++++++++++++++++-------------
 src/libstore/machines.hh |  4 +++-
 4 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 953bf6aaaa0..4bdbde989ab 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -43,6 +43,10 @@ Settings::Settings()
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
     caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
 
+    /* Backwards compatibility. */
+    auto s = getEnv("NIX_REMOTE_SYSTEMS");
+    if (s != "") builderFiles = tokenizeString<Strings>(s, ":");
+
 #if __linux__
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH);
 #endif
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d7a0b86a088..ac6f6a2cfa3 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -133,6 +133,10 @@ public:
     Setting<std::string> builders{this, "", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
+    Setting<Strings> builderFiles{this,
+        {nixConfDir + "/machines"}, "builder-files",
+        "A list of files specifying build machines."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 479ed1432fb..c1d9047537d 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -12,7 +12,8 @@ Machine::Machine(decltype(storeUri) storeUri,
     decltype(maxJobs) maxJobs,
     decltype(speedFactor) speedFactor,
     decltype(supportedFeatures) supportedFeatures,
-    decltype(mandatoryFeatures) mandatoryFeatures) :
+    decltype(mandatoryFeatures) mandatoryFeatures,
+    decltype(sshPublicHostKey) sshPublicHostKey) :
     storeUri(
         // Backwards compatibility: if the URI is a hostname,
         // prepend ssh://.
@@ -24,7 +25,8 @@ Machine::Machine(decltype(storeUri) storeUri,
     maxJobs(maxJobs),
     speedFactor(std::max(1U, speedFactor)),
     supportedFeatures(supportedFeatures),
-    mandatoryFeatures(mandatoryFeatures)
+    mandatoryFeatures(mandatoryFeatures),
+    sshPublicHostKey(sshPublicHostKey)
 {}
 
 bool Machine::allSupported(const std::set<string> & features) const {
@@ -52,13 +54,19 @@ void parseMachines(const std::string & s, Machines & machines)
         auto sz = tokens.size();
         if (sz < 1)
             throw FormatError("bad machine specification ‘%s’", line);
+
+        auto isSet = [&](int n) {
+            return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
+        };
+
         machines.emplace_back(tokens[0],
-            sz >= 2 ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
-            sz >= 3 ? tokens[2] : "",
-            sz >= 4 ? std::stoull(tokens[3]) : 1LL,
-            sz >= 5 ? std::stoull(tokens[4]) : 1LL,
-            sz >= 6 ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
-            sz >= 7 ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{});
+            isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
+            isSet(2) ? tokens[2] : "",
+            isSet(3) ? std::stoull(tokens[3]) : 1LL,
+            isSet(4) ? std::stoull(tokens[4]) : 1LL,
+            isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
+            isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
+            isSet(7) ? tokens[7] : "");
     }
 }
 
@@ -66,11 +74,13 @@ Machines getMachines()
 {
     Machines machines;
 
-    try {
-        parseMachines(readFile(getEnv("NIX_REMOTE_SYSTEMS", settings.nixConfDir + "/machines")), machines);
-    } catch (const SysError & e) {
-        if (e.errNo != ENOENT)
-            throw;
+    for (auto & file : settings.builderFiles.get()) {
+        try {
+            parseMachines(readFile(file), machines);
+        } catch (const SysError & e) {
+            if (e.errNo != ENOENT)
+                throw;
+        }
     }
 
     parseMachines(settings.builders, machines);
diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh
index e0455742844..de92eb924e4 100644
--- a/src/libstore/machines.hh
+++ b/src/libstore/machines.hh
@@ -13,6 +13,7 @@ struct Machine {
     const unsigned int speedFactor;
     const std::set<string> supportedFeatures;
     const std::set<string> mandatoryFeatures;
+    const std::string sshPublicHostKey;
     bool enabled = true;
 
     bool allSupported(const std::set<string> & features) const;
@@ -25,7 +26,8 @@ struct Machine {
         decltype(maxJobs) maxJobs,
         decltype(speedFactor) speedFactor,
         decltype(supportedFeatures) supportedFeatures,
-        decltype(mandatoryFeatures) mandatoryFeatures);
+        decltype(mandatoryFeatures) mandatoryFeatures,
+        decltype(sshPublicHostKey) sshPublicHostKey);
 };
 
 typedef std::vector<Machine> Machines;

From c5bea16611ae1aa99680bbd5d94bc69811869885 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 15:07:11 +0200
Subject: [PATCH 0307/2196] LocalStoreAccessor: Fix handling of diverted stores

---
 src/libstore/local-fs-store.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index bf247903c9d..bf28a1c70c6 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -31,7 +31,7 @@ struct LocalStoreAccessor : public FSAccessor
         auto realPath = toRealPath(path);
 
         struct stat st;
-        if (lstat(path.c_str(), &st)) {
+        if (lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
             throw SysError(format("getting status of ‘%1%’") % path);
         }
@@ -51,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor
     {
         auto realPath = toRealPath(path);
 
-        auto entries = nix::readDirectory(path);
+        auto entries = nix::readDirectory(realPath);
 
         StringSet res;
         for (auto & entry : entries)
@@ -73,7 +73,8 @@ struct LocalStoreAccessor : public FSAccessor
 
 ref<FSAccessor> LocalFSStore::getFSAccessor()
 {
-    return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
+    return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(
+            std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
 }
 
 void LocalFSStore::narFromPath(const Path & path, Sink & sink)

From 7dedd3fa2455f1e219bc671d04d1dd1eaec54dfa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 15:09:35 +0200
Subject: [PATCH 0308/2196] Add a test for diverted stores

---
 tests/linux-sandbox.sh | 27 +++++++++++++++++++++++++++
 tests/local.mk         |  3 ++-
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/linux-sandbox.sh

diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
new file mode 100644
index 00000000000..740b2c35709
--- /dev/null
+++ b/tests/linux-sandbox.sh
@@ -0,0 +1,27 @@
+source common.sh
+
+clearStore
+
+if [[ $(uname) != Linux ]]; then exit; fi
+
+# Note: we need to bind-mount $SHELL into the chroot. Currently we
+# only support the case where $SHELL is in the Nix store, because
+# otherwise things get complicated (e.g. if it's in /bin, do we need
+# /lib as well?).
+if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
+
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
+
+export NIX_STORE_DIR=/my/store
+export NIX_REMOTE="local?root=$TEST_ROOT/store0"
+
+outPath=$( nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store)
+
+[[ $outPath =~ /my/store/.*-dependencies ]]
+
+nix path-info -r $outPath | grep input-2
+
+nix ls-store -R -l $outPath | grep foobar
+
+nix cat-store $outPath/foobar | grep FOOBAR
diff --git a/tests/local.mk b/tests/local.mk
index b3ce39cda80..06532dc58d2 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -11,7 +11,8 @@ nix_tests = \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
-  placeholders.sh nix-shell.sh
+  placeholders.sh nix-shell.sh \
+  linux-sandbox.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From cef8c169b1d100685d7c7b7bfb921eaa43b5521b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 15:28:35 +0200
Subject: [PATCH 0309/2196] Fix "nix ... --all"

When "--all" is used, we should not fill in a default installable.
---
 src/nix/command.hh      |  6 +++++-
 src/nix/installables.cc | 22 +++++++++++-----------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/nix/command.hh b/src/nix/command.hh
index dc7b2637d66..cf0097d7892 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -78,7 +78,7 @@ struct InstallablesCommand : virtual Args, StoreCommand
        = import ...; bla = import ...; }’. */
     Value * getSourceExpr(EvalState & state);
 
-    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables);
+    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss);
 
     PathSet buildInstallables(ref<Store> store, bool dryRun);
 
@@ -86,6 +86,8 @@ struct InstallablesCommand : virtual Args, StoreCommand
 
     void prepare() override;
 
+    virtual bool useDefaultInstallables() { return true; }
+
 private:
 
     Strings _installables;
@@ -112,6 +114,8 @@ public:
     virtual void run(ref<Store> store, Paths storePaths) = 0;
 
     void run(ref<Store> store) override;
+
+    bool useDefaultInstallables() override { return !all; }
 };
 
 typedef std::map<std::string, ref<Command>> Commands;
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 57580049f25..4756fc44bba 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -177,21 +177,21 @@ struct InstallableAttrPath : Installable
 std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
 static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
 
-std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings installables)
+std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss)
 {
     std::vector<std::shared_ptr<Installable>> result;
 
-    if (installables.empty()) {
+    if (ss.empty() && useDefaultInstallables()) {
         if (file == "")
             file = ".";
-        installables = Strings{""};
+        ss = Strings{""};
     }
 
-    for (auto & installable : installables) {
+    for (auto & s : ss) {
 
-        if (installable.find("/") != std::string::npos) {
+        if (s.find("/") != std::string::npos) {
 
-            auto path = store->toStorePath(store->followLinksToStore(installable));
+            auto path = store->toStorePath(store->followLinksToStore(s));
 
             if (store->isStorePath(path)) {
                 if (isDerivation(path))
@@ -201,14 +201,14 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
             }
         }
 
-        else if (installable.compare(0, 1, "(") == 0)
-            result.push_back(std::make_shared<InstallableExpr>(*this, installable));
+        else if (s.compare(0, 1, "(") == 0)
+            result.push_back(std::make_shared<InstallableExpr>(*this, s));
 
-        else if (installable == "" || std::regex_match(installable, attrPathRegex))
-            result.push_back(std::make_shared<InstallableAttrPath>(*this, installable));
+        else if (s == "" || std::regex_match(s, attrPathRegex))
+            result.push_back(std::make_shared<InstallableAttrPath>(*this, s));
 
         else
-            throw UsageError("don't know what to do with argument ‘%s’", installable);
+            throw UsageError("don't know what to do with argument ‘%s’", s);
     }
 
     return result;

From 16535552ad69ca5397967b84fd70cc0d27a38ac1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 May 2017 15:35:35 +0200
Subject: [PATCH 0310/2196] build-remote: Add a basic test

This only runs on Linux because it requires a diverted store (which
uses mount/user namespaces).
---
 tests/build-hook.nix  |  1 +
 tests/build-remote.sh | 24 ++++++++++++++++++++++++
 tests/local.mk        |  3 ++-
 3 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 tests/build-remote.sh

diff --git a/tests/build-hook.nix b/tests/build-hook.nix
index 666cc6ef804..8bff0fe7903 100644
--- a/tests/build-hook.nix
+++ b/tests/build-hook.nix
@@ -5,6 +5,7 @@ let
   input1 = mkDerivation {
     name = "build-hook-input-1";
     builder = ./dependencies.builder1.sh;
+    requiredSystemFeatures = ["foo"];
   };
 
   input2 = mkDerivation {
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
new file mode 100644
index 00000000000..071011dcb71
--- /dev/null
+++ b/tests/build-remote.sh
@@ -0,0 +1,24 @@
+source common.sh
+
+clearStore
+
+if [[ $(uname) != Linux ]]; then exit; fi
+if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
+
+chmod -R u+w $TEST_ROOT/store0 || true
+chmod -R u+w $TEST_ROOT/store1 || true
+rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
+
+# FIXME: --option is not passed to build-remote, so have to create a config file.
+export NIX_CONF_DIR=$TEST_ROOT/etc2
+mkdir -p $NIX_CONF_DIR
+echo "build-sandbox-paths = /nix/store" > $NIX_CONF_DIR/nix.conf
+
+outPath=$(nix-build build-hook.nix --no-out-link -j0 --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo" --option build-sandbox-paths /nix/store)
+
+cat $outPath/foobar | grep FOOBAR
+
+# Ensure that input1 was built on store1 due to the required feature.
+p=$(readlink -f $outPath/input-2)
+(! nix path-info --store local?root=$TEST_ROOT/store0 --all | grep dependencies.builder1.sh)
+nix path-info --store local?root=$TEST_ROOT/store1 --all | grep dependencies.builder1.sh
diff --git a/tests/local.mk b/tests/local.mk
index 06532dc58d2..108e3febdb0 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -12,7 +12,8 @@ nix_tests = \
   binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
   placeholders.sh nix-shell.sh \
-  linux-sandbox.sh
+  linux-sandbox.sh \
+  build-remote.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From a786d26dc27798a0d00b5433442c69815c8ba763 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Wed, 3 May 2017 10:49:34 +0200
Subject: [PATCH 0311/2196] doc: fix the description of --xml and --json

Those options seem to only apply with --eval and not with --parse.
---
 doc/manual/command-ref/nix-instantiate.xml | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 1e556c7ed7c..3d03358bea3 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -117,21 +117,19 @@ input.</para>
 
   <varlistentry><term><option>--xml</option></term>
 
-    <listitem><para>When used with <option>--parse</option> and
-    <option>--eval</option>, print the resulting expression as an
-    XML representation of the abstract syntax tree rather than as an
-    ATerm.  The schema is the same as that used by the <link
-    linkend="builtin-toXML"><function>toXML</function>
-    built-in</link>.</para></listitem>
+    <listitem><para>When used with <option>--eval</option>, print the resulting
+    value as an XML representation of the abstract syntax tree rather than as
+    an ATerm. The schema is the same as that used by the <link
+    linkend="builtin-toXML"><function>toXML</function> built-in</link>.
+    </para></listitem>
 
   </varlistentry>
 
   <varlistentry><term><option>--json</option></term>
 
-    <listitem><para>When used with <option>--parse</option> and
-    <option>--eval</option>, print the resulting expression as an
-    JSON representation of the abstract syntax tree rather than as an
-    ATerm.</para></listitem>
+    <listitem><para>When used with <option>--eval</option>, print the resulting
+    value as an JSON representation of the abstract syntax tree rather
+    than as an ATerm.</para></listitem>
 
   </varlistentry>
 

From d3dcdfa00691cfe6f6a939fde218f1980d3cf73c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 11:30:02 +0200
Subject: [PATCH 0312/2196] Fix perlBindings.x86_64-darwin

http://hydra.nixos.org/build/52401151
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 7adc87386f9..f21f5867cc8 100644
--- a/release.nix
+++ b/release.nix
@@ -112,7 +112,7 @@ let
 
         buildInputs =
           [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ]
-          ++ lib.optional stdenv.isLinux libsodium;
+          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
           --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}

From 782c0bff45593e7116d9b17b7de71b7ee636a807 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 14:08:18 +0200
Subject: [PATCH 0313/2196] nix eval: Add a --raw flag

Similar to "jq -r", this prints the evaluation result (which must be a
string value) unquoted.
---
 src/nix/eval.cc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index eb2b13a2dcd..981cefa800c 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -10,6 +10,13 @@ using namespace nix;
 
 struct CmdEval : MixJSON, InstallablesCommand
 {
+    bool raw = false;
+
+    CmdEval()
+    {
+        mkFlag(0, "raw", "print strings unquoted", &raw);
+    }
+
     std::string name() override
     {
         return "eval";
@@ -22,13 +29,18 @@ struct CmdEval : MixJSON, InstallablesCommand
 
     void run(ref<Store> store) override
     {
+        if (raw && json)
+            throw UsageError("--raw and --json are mutually exclusive");
+
         auto state = getEvalState();
 
         auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr;
 
         for (auto & i : installables) {
             auto v = i->toValue(*state);
-            if (json) {
+            if (raw) {
+                std::cout << state->forceString(*v);
+            } else if (json) {
                 PathSet context;
                 auto jsonElem = jsonOut->placeholder();
                 printValueAsJSON(*state, true, *v, jsonElem, context);

From 08355643ab2811256b8d78265757d9aab216b38e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 15:01:15 +0200
Subject: [PATCH 0314/2196] nix-shell: Implement passAsFile

---
 src/nix-build/nix-build.cc | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index bb031d51513..cd4dee32674 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -15,6 +15,7 @@
 #include "shared.hh"
 
 using namespace nix;
+using namespace std::string_literals;
 
 extern char * * environ;
 
@@ -407,8 +408,20 @@ int main(int argc, char ** argv)
                 env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
                 env["NIX_STORE"] = store->storeDir;
 
+                auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", ""));
+
+                bool keepTmp = false;
+                int fileNr = 0;
+
                 for (auto & var : drv.env)
-                    env[var.first] = var.second;
+                    if (passAsFile.count(var.first)) {
+                        keepTmp = true;
+                        string fn = ".attr-" + std::to_string(fileNr++);
+                        Path p = (Path) tmpDir + "/" + fn;
+                        writeFile(p, var.second);
+                        env[var.first + "Path"] = p;
+                    } else
+                        env[var.first] = var.second;
 
                 restoreAffinity();
 
@@ -418,7 +431,7 @@ int main(int argc, char ** argv)
                 // the current $PATH directories.
                 auto rcfile = (Path) tmpDir + "/rc";
                 writeFile(rcfile, fmt(
-                        "rm -rf '%1%'; "
+                        (keepTmp ? "" : "rm -rf '%1%'; "s) +
                         "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; "
                         "%2%"
                         "dontAddDisableDepTrack=1; "

From 72fb2a7edc169fa480f3a3d8da5a9fd263868491 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 16:08:48 +0200
Subject: [PATCH 0315/2196] Fix build on gcc 4.9

http://hydra.nixos.org/build/52408843
---
 src/libutil/config.cc | 4 ++--
 src/libutil/config.hh | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 497afaa1fed..f7a46bfee63 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -15,9 +15,9 @@ void Config::set(const std::string & name, const std::string & value)
 
 void Config::addSetting(AbstractSetting * setting)
 {
-    _settings.emplace(setting->name, Config::SettingData{false, setting});
+    _settings.emplace(setting->name, Config::SettingData(false, setting));
     for (auto & alias : setting->aliases)
-        _settings.emplace(alias, Config::SettingData{true, setting});
+        _settings.emplace(alias, Config::SettingData(true, setting));
 
     bool set = false;
 
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 91962109100..77620d47d37 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -33,8 +33,11 @@ class Config
 
     struct SettingData
     {
-        bool isAlias = false;
+        bool isAlias;
         AbstractSetting * setting;
+        SettingData(bool isAlias, AbstractSetting * setting)
+            : isAlias(isAlias), setting(setting)
+        { }
     };
 
     std::map<std::string, SettingData> _settings;

From 493d4bd9490bd4a4b4cd5ad9dd62e9a01f94e84c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 17:43:52 +0200
Subject: [PATCH 0316/2196] perl-bindings: Remove unused --with-store-dir flag

---
 perl/configure.ac | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/perl/configure.ac b/perl/configure.ac
index 80bcdb8ff5b..9f49db4d281 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -40,11 +40,6 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};')
 AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
-AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
-  [path of the Nix store (defaults to /nix/store)]),
-  storedir=$withval, storedir='/nix/store')
-AC_SUBST(storedir)
-
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
   [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])

From 44309c506767fcfb8aae15761b329a87b0dd4b8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 May 2017 18:30:47 +0200
Subject: [PATCH 0317/2196] Fix Ubuntu 16.10 build

http://hydra.nixos.org/build/52420073
---
 release.nix | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/release.nix b/release.nix
index f21f5867cc8..fa2fde4f609 100644
--- a/release.nix
+++ b/release.nix
@@ -198,15 +198,15 @@ let
     rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
 
 
-    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
+    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" "libreadline6" ];
+    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" "libreadline6" ];
 
-    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [];
-    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [];
-    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [ "libreadline6" ];
+    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [ "libreadline6" ];
+    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" "libreadline6" ];
+    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" "libreadline6" ];
+    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" "libreadline7" ];
+    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" "libreadline7" ];
 
 
     # System tests.
@@ -328,7 +328,7 @@ let
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libreadline6" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;

From 2da6a424486e16b4b30e448a15a9b4a608df602d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 May 2017 14:16:26 +0200
Subject: [PATCH 0318/2196] nix dump-path: Add

This is primarily useful for extracting NARs from other stores (like
binary caches), which "nix-store --dump" cannot do.
---
 src/nix/command.cc   | 10 ++++++++++
 src/nix/command.hh   | 10 ++++++++++
 src/nix/dump-path.cc | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+)
 create mode 100644 src/nix/dump-path.cc

diff --git a/src/nix/command.cc b/src/nix/command.cc
index a45f2888bfb..3c82e0df57f 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -129,4 +129,14 @@ void StorePathsCommand::run(ref<Store> store)
     run(store, storePaths);
 }
 
+void StorePathCommand::run(ref<Store> store)
+{
+    auto storePaths = buildInstallables(store, false);
+
+    if (storePaths.size() != 1)
+        throw UsageError("this command requires exactly one store path");
+
+    run(store, *storePaths.begin());
+}
+
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index cf0097d7892..4800b5c912e 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -118,6 +118,16 @@ public:
     bool useDefaultInstallables() override { return !all; }
 };
 
+/* A command that operates on exactly one store path. */
+struct StorePathCommand : public InstallablesCommand
+{
+    using StoreCommand::run;
+
+    virtual void run(ref<Store> store, const Path & storePath) = 0;
+
+    void run(ref<Store> store) override;
+};
+
 typedef std::map<std::string, ref<Command>> Commands;
 
 /* An argument parser that supports multiple subcommands,
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
new file mode 100644
index 00000000000..1a1866437b0
--- /dev/null
+++ b/src/nix/dump-path.cc
@@ -0,0 +1,35 @@
+#include "command.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdDumpPath : StorePathCommand
+{
+    std::string name() override
+    {
+        return "dump-path";
+    }
+
+    std::string description() override
+    {
+        return "dump a store path to stdout (in NAR format)";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To get a NAR from the binary cache https://cache.nixos.org/:",
+                "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"
+            },
+        };
+    }
+
+    void run(ref<Store> store, const Path & storePath) override
+    {
+        FdSink sink(STDOUT_FILENO);
+        store->narFromPath(storePath, sink);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdDumpPath>());

From eba840c8a13b465ace90172ff76a0db2899ab11b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 May 2017 16:57:03 +0200
Subject: [PATCH 0319/2196] Linux sandbox: Use /build instead of /tmp as
 $TMPDIR

There is a security issue when a build accidentally stores its $TMPDIR
in some critical place, such as an RPATH. If
TMPDIR=/tmp/nix-build-..., then any user on the system can recreate
that directory and inject libraries into the RPATH of programs
executed by other users. Since /build probably doesn't exist (or isn't
world-writable), this mitigates the issue.
---
 src/libstore/build.cc | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8c2602a701b..6d20512e1f8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1661,6 +1661,9 @@ int childEntry(void * arg)
 }
 
 
+const std::string buildDir = "/build";
+
+
 void DerivationGoal::startBuilder()
 {
     auto f = format(
@@ -1721,7 +1724,14 @@ void DerivationGoal::startBuilder()
 
     /* In a sandbox, for determinism, always use the same temporary
        directory. */
+#if __linux__
+    tmpDirInSandbox = useChroot ? buildDir : tmpDir;
+#elif __APPLE__
+    // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp.
     tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
+#else
+    tmpDirInSandbox = tmpDir;
+#endif
     chownToBuilder(tmpDir);
 
     /* Substitute output placeholders with the actual output paths. */
@@ -1829,11 +1839,11 @@ void DerivationGoal::startBuilder()
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
 
-        writeFile(chrootRootDir + "/etc/passwd",
-            (format(
-                "root:x:0:0:Nix build user:/:/noshell\n"
-                "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n"
-                "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str());
+        writeFile(chrootRootDir + "/etc/passwd", fmt(
+                "root:x:0:0:Nix build user:%3%:/noshell\n"
+                "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
+                "nobody:x:65534:65534:Nobody:/:/noshell\n",
+                sandboxUid, sandboxGid, buildDir));
 
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */

From 465cb6824401541d82489e11b5223dbfd50bb132 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 May 2017 16:40:12 +0200
Subject: [PATCH 0320/2196] Figure out the user's home directory if $HOME is
 not set

---
 misc/launchd/org.nixos.nix-daemon.plist.in |  2 -
 misc/systemd/nix-daemon.service.in         |  2 -
 src/libexpr/parser.y                       |  2 +-
 src/libutil/lazy.hh                        | 48 ++++++++++++++++++++++
 src/libutil/util.cc                        | 45 ++++++++++++--------
 src/libutil/util.hh                        |  3 ++
 src/nix-channel/nix-channel.cc             |  4 +-
 src/nix-env/nix-env.cc                     | 14 ++-----
 8 files changed, 84 insertions(+), 36 deletions(-)
 create mode 100644 src/libutil/lazy.hh

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 5d57a5ec8ff..c5ef97ee9a3 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -16,8 +16,6 @@
     <dict>
       <key>NIX_SSL_CERT_FILE</key>
       <string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
-      <key>XDG_CACHE_HOME</key>
-      <string>/root/.cache</string>
     </dict>
   </dict>
 </plist>
diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in
index 9bfb00e306b..5fc04a3f571 100644
--- a/misc/systemd/nix-daemon.service.in
+++ b/misc/systemd/nix-daemon.service.in
@@ -7,5 +7,3 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
-Environment=XDG_CACHE_HOME=/root/.cache
-Environment=XDG_CONFIG_HOME=/root/.config
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index d07eedddaf6..62982650a22 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -376,7 +376,7 @@ expr_simple
       $$ = stripIndentation(CUR_POS, data->symbols, *$2);
   }
   | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
-  | HPATH { $$ = new ExprPath(getEnv("HOME", "") + string{$1 + 1}); }
+  | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); }
   | SPATH {
       string path($1 + 1, strlen($1) - 2);
       $$ = new ExprApp(CUR_POS,
diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh
new file mode 100644
index 00000000000..d073e486c2e
--- /dev/null
+++ b/src/libutil/lazy.hh
@@ -0,0 +1,48 @@
+#include <exception>
+#include <functional>
+#include <mutex>
+
+namespace nix {
+
+/* A helper class for lazily-initialized variables.
+
+     Lazy<T> var([]() { return value; });
+
+   declares a variable of type T that is initialized to 'value' (in a
+   thread-safe way) on first use, that is, when var() is first
+   called. If the initialiser code throws an exception, then all
+   subsequent calls to var() will rethrow that exception. */
+template<typename T>
+class Lazy
+{
+
+    typedef std::function<T()> Init;
+
+    Init init;
+
+    std::once_flag done;
+
+    T value;
+
+    std::exception_ptr ex;
+
+public:
+
+    Lazy(Init init) : init(init)
+    { }
+
+    const T & operator () ()
+    {
+        std::call_once(done, [&]() {
+            try {
+                value = init();
+            } catch (...) {
+                ex = std::current_exception();
+            }
+        });
+        if (ex) std::rethrow_exception(ex);
+        return value;
+    }
+};
+
+}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 98c0aff1e72..1d1f68fc845 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1,3 +1,4 @@
+#include "lazy.hh"
 #include "util.hh"
 #include "affinity.hh"
 #include "sync.hh"
@@ -13,10 +14,12 @@
 #include <thread>
 #include <future>
 
-#include <sys/wait.h>
-#include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
+#include <pwd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
 
 #ifdef __APPLE__
 #include <sys/syscall.h>
@@ -417,14 +420,28 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 }
 
 
+static Lazy<Path> getHome2([]() {
+    Path homeDir = getEnv("HOME");
+    if (homeDir.empty()) {
+        char buf[16384];
+        struct passwd pwbuf;
+        struct passwd * pw;
+        if (getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pw) != 0
+            || !pw || !pw->pw_dir || !pw->pw_dir[0])
+            throw Error("cannot determine user's home directory");
+        homeDir = pw->pw_dir;
+    }
+    return homeDir;
+});
+
+Path getHome() { return getHome2(); }
+
+
 Path getCacheDir()
 {
     Path cacheDir = getEnv("XDG_CACHE_HOME");
-    if (cacheDir.empty()) {
-        Path homeDir = getEnv("HOME");
-        if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set");
-        cacheDir = homeDir + "/.cache";
-    }
+    if (cacheDir.empty())
+        cacheDir = getHome() + "/.cache";
     return cacheDir;
 }
 
@@ -432,11 +449,8 @@ Path getCacheDir()
 Path getConfigDir()
 {
     Path configDir = getEnv("XDG_CONFIG_HOME");
-    if (configDir.empty()) {
-        Path homeDir = getEnv("HOME");
-        if (homeDir.empty()) throw Error("$XDG_CONFIG_HOME and $HOME are not set");
-        configDir = homeDir + "/.config";
-    }
+    if (configDir.empty())
+        configDir = getHome() + "/.config";
     return configDir;
 }
 
@@ -444,11 +458,8 @@ Path getConfigDir()
 Path getDataDir()
 {
     Path dataDir = getEnv("XDG_DATA_HOME");
-    if (dataDir.empty()) {
-        Path homeDir = getEnv("HOME");
-        if (homeDir.empty()) throw Error("$XDG_DATA_HOME and $HOME are not set");
-        dataDir = homeDir + "/.local/share";
-    }
+    if (dataDir.empty())
+        dataDir = getHome() + "/.local/share";
     return dataDir;
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a9950f830c5..5a9c9513fd5 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -110,6 +110,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed);
 Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
     bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
 
+/* Return $HOME or the user's home directory from /etc/passwd. */
+Path getHome();
+
 /* Return $XDG_CACHE_HOME or $HOME/.cache. */
 Path getCacheDir();
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 0f50f6242c4..2aaae2f471b 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -169,9 +169,7 @@ int main(int argc, char ** argv)
         setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1);
 
         // Figure out the name of the `.nix-channels' file to use
-        auto home = getEnv("HOME");
-        if (home.empty())
-            throw Error("$HOME not set");
+        auto home = getHome();
         channelsList = home + "/.nix-channels";
         nixDefExpr = home + "/.nix-defexpr";
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 908c09bc8c8..da39bf36ab6 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -192,17 +192,9 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
 }
 
 
-static Path getHomeDir()
-{
-    Path homeDir(getEnv("HOME", ""));
-    if (homeDir == "") throw Error("HOME environment variable not set");
-    return homeDir;
-}
-
-
 static Path getDefNixExprPath()
 {
-    return getHomeDir() + "/.nix-defexpr";
+    return getHome() + "/.nix-defexpr";
 }
 
 
@@ -1188,7 +1180,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs)
         throw UsageError(format("exactly one argument expected"));
 
     Path profile = absPath(opArgs.front());
-    Path profileLink = getHomeDir() + "/.nix-profile";
+    Path profileLink = getHome() + "/.nix-profile";
 
     switchLink(profileLink, profile);
 }
@@ -1413,7 +1405,7 @@ int main(int argc, char * * argv)
             globals.profile = getEnv("NIX_PROFILE", "");
 
         if (globals.profile == "") {
-            Path profileLink = getHomeDir() + "/.nix-profile";
+            Path profileLink = getHome() + "/.nix-profile";
             globals.profile = pathExists(profileLink)
                 ? absPath(readLink(profileLink), dirOf(profileLink))
                 : canonPath(settings.nixStateDir + "/profiles/default");

From bb50c8931934d04dbf61bd245b4583f8c1ac4fd9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 May 2017 17:45:22 +0200
Subject: [PATCH 0321/2196] Make the location of the build directory in the
 sandbox configurable

This is mostly for use in the sandbox tests, since if the Nix store is
under /build, then we can't use /build as the build directory.
---
 src/libstore/build.cc    | 7 ++-----
 src/libstore/globals.hh  | 3 +++
 src/libstore/machines.cc | 2 +-
 tests/build-remote.sh    | 8 ++++++--
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6d20512e1f8..70ecf4bad99 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1661,9 +1661,6 @@ int childEntry(void * arg)
 }
 
 
-const std::string buildDir = "/build";
-
-
 void DerivationGoal::startBuilder()
 {
     auto f = format(
@@ -1725,7 +1722,7 @@ void DerivationGoal::startBuilder()
     /* In a sandbox, for determinism, always use the same temporary
        directory. */
 #if __linux__
-    tmpDirInSandbox = useChroot ? buildDir : tmpDir;
+    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
 #elif __APPLE__
     // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp.
     tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
@@ -1843,7 +1840,7 @@ void DerivationGoal::startBuilder()
                 "root:x:0:0:Nix build user:%3%:/noshell\n"
                 "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
                 "nobody:x:65534:65534:Nobody:/:/noshell\n",
-                sandboxUid, sandboxGid, buildDir));
+                sandboxUid, sandboxGid, settings.sandboxBuildDir));
 
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index ac6f6a2cfa3..7295b0d30af 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -224,6 +224,9 @@ public:
 #if __linux__
     Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
         "The size of /dev/shm in the build sandbox."};
+
+    Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
+        "The build directory inside the sandbox."};
 #endif
 
     Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index c1d9047537d..7491037b2d7 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -55,7 +55,7 @@ void parseMachines(const std::string & s, Machines & machines)
         if (sz < 1)
             throw FormatError("bad machine specification ‘%s’", line);
 
-        auto isSet = [&](int n) {
+        auto isSet = [&](size_t n) {
             return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
         };
 
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 071011dcb71..927a217f376 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -12,9 +12,13 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 # FIXME: --option is not passed to build-remote, so have to create a config file.
 export NIX_CONF_DIR=$TEST_ROOT/etc2
 mkdir -p $NIX_CONF_DIR
-echo "build-sandbox-paths = /nix/store" > $NIX_CONF_DIR/nix.conf
+echo "
+build-sandbox-paths = /nix/store
+sandbox-build-dir = /build-tmp
+" > $NIX_CONF_DIR/nix.conf
 
-outPath=$(nix-build build-hook.nix --no-out-link -j0 --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo" --option build-sandbox-paths /nix/store)
+outPath=$(nix-build build-hook.nix --no-out-link -j0 \
+  --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo")
 
 cat $outPath/foobar | grep FOOBAR
 

From 542fe0d8f3f25025a2642583d8593580a5d10e3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <Mic92@users.noreply.github.com>
Date: Sun, 7 May 2017 07:41:19 +0100
Subject: [PATCH 0322/2196] nix-profile.sh: remove sbin from PATH

sbin is a symlink to bin.
profiles only contains packages, which have this symlink.
It is a subset of bin.

related to https://github.com/NixOS/nixpkgs/pull/25550
---
 scripts/nix-profile.sh.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 3cdf431041c..ab95c09c830 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -85,6 +85,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 
-    export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath"
+    export PATH="$NIX_LINK/bin:$__savedpath"
     unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES
 fi

From 0a97eb6bd7d459de80432a5dbf39daf16647bb8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 May 2017 11:25:23 +0200
Subject: [PATCH 0323/2196] Remove superfluous #ifdef

---
 src/libstore/build.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 70ecf4bad99..dc9b9e023cb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2360,10 +2360,8 @@ void DerivationGoal::runChild()
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
                 ss.push_back("/dev/full");
-#ifdef __linux__
                 if (pathExists("/dev/kvm"))
                     ss.push_back("/dev/kvm");
-#endif
                 ss.push_back("/dev/null");
                 ss.push_back("/dev/random");
                 ss.push_back("/dev/tty");

From ebfceeb333120411af46d0af4805f3e7d557159f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 May 2017 14:27:12 +0200
Subject: [PATCH 0324/2196] build-remote: Check remote build status

---
 src/build-remote/build-remote.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8876da6c063..7ffbdca7c0f 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -208,7 +208,10 @@ int main (int argc, char * * argv)
         drv.inputSrcs = inputs;
 
         printError("building ‘%s’ on ‘%s’", drvPath, storeUri);
-        sshStore->buildDerivation(drvPath, drv);
+        auto result = sshStore->buildDerivation(drvPath, drv);
+
+        if (!result.success())
+            throw Error("build of ‘%s’ on ‘%s’ failed: %s", drvPath, storeUri, result.errorMsg);
 
         PathSet missing;
         for (auto & path : outputs)

From 00b286275c1a77458e45bd73528c9ca729cca7f6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 May 2017 15:42:59 +0200
Subject: [PATCH 0325/2196] Linux sandbox: Fix compatibility with older kernels

---
 src/libstore/build.cc  | 38 +++++++++++++++++++++++---------------
 tests/linux-sandbox.sh |  2 +-
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dc9b9e023cb..270500d81c9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2390,17 +2390,14 @@ void DerivationGoal::runChild()
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot
                environment. */
-            for (auto & i : dirsInChroot) {
-                struct stat st;
-                Path source = i.second.source;
-                Path target = chrootRootDir + i.first;
-                if (source == "/proc") continue; // backwards compatibility
+            auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
                 debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target);
+                struct stat st;
                 if (stat(source.c_str(), &st) == -1) {
-                    if (i.second.optional && errno == ENOENT)
-                        continue;
+                    if (optional && errno == ENOENT)
+                        return;
                     else
-                        throw SysError(format("getting attributes of path ‘%1%’") % source);
+                        throw SysError("getting attributes of path ‘%1%’", source);
                 }
                 if (S_ISDIR(st.st_mode))
                     createDirs(target);
@@ -2409,7 +2406,12 @@ void DerivationGoal::runChild()
                     writeFile(target, "");
                 }
                 if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target);
+                    throw SysError("bind mount from ‘%1%’ to ‘%2%’ failed", source, target);
+            };
+
+            for (auto & i : dirsInChroot) {
+                if (i.second.source == "/proc") continue; // backwards compatibility
+                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
             }
 
             /* Bind a new instance of procfs on /proc. */
@@ -2431,13 +2433,19 @@ void DerivationGoal::runChild()
                 !pathExists(chrootRootDir + "/dev/ptmx")
                 && !dirsInChroot.count("/dev/pts"))
             {
-                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1)
-                    throw SysError("mounting /dev/pts");
-                createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
+                if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
+                {
+                    createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
 
-                /* Make sure /dev/pts/ptmx is world-writable.  With some
-                   Linux versions, it is created with permissions 0.  */
-                chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+                    /* Make sure /dev/pts/ptmx is world-writable.  With some
+                       Linux versions, it is created with permissions 0.  */
+                    chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+                } else {
+                    if (errno != EINVAL)
+                        throw SysError("mounting /dev/pts");
+                    doBind("/dev/pts", "/dev/pts");
+                    doBind("/dev/ptmx", "/dev/ptmx");
+                }
             }
 
             /* Do the chroot(). */
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 740b2c35709..54cdef8178e 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -16,7 +16,7 @@ rm -rf $TEST_ROOT/store0
 export NIX_STORE_DIR=/my/store
 export NIX_REMOTE="local?root=$TEST_ROOT/store0"
 
-outPath=$( nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store)
+outPath=$(nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store)
 
 [[ $outPath =~ /my/store/.*-dependencies ]]
 

From 7689181e4f5921d3356736996079ec0310e834c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 May 2017 13:36:23 +0200
Subject: [PATCH 0326/2196] Minor cleanup

---
 src/libstore/store-api.cc | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b5a91e53672..8405c66dd47 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -482,21 +482,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
         if (showClosureSize)
             jsonPath.attr("closureSize", getClosureSize(storePath));
 
-        if (!includeImpureInfo) continue;
+        if (includeImpureInfo) {
 
-        if (info->deriver != "")
-            jsonPath.attr("deriver", info->deriver);
+            if (info->deriver != "")
+                jsonPath.attr("deriver", info->deriver);
 
-        if (info->registrationTime)
-            jsonPath.attr("registrationTime", info->registrationTime);
+            if (info->registrationTime)
+                jsonPath.attr("registrationTime", info->registrationTime);
 
-        if (info->ultimate)
-            jsonPath.attr("ultimate", info->ultimate);
+            if (info->ultimate)
+                jsonPath.attr("ultimate", info->ultimate);
+
+            if (!info->sigs.empty()) {
+                auto jsonSigs = jsonPath.list("signatures");
+                for (auto & sig : info->sigs)
+                    jsonSigs.elem(sig);
+            }
 
-        if (!info->sigs.empty()) {
-            auto jsonSigs = jsonPath.list("signatures");
-            for (auto & sig : info->sigs)
-                jsonSigs.elem(sig);
         }
     }
 }

From 03ae5e64592d6d62f30158ccfa804f1b4135a596 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 May 2017 18:39:33 +0200
Subject: [PATCH 0327/2196] Add "nix edit" command

This is a little convenience command that opens the Nix expression of
the specified package. For example,

  nix edit nixpkgs.perlPackages.Moose

opens <nixpkgs/pkgs/top-level/perl-packages.nix> in $EDITOR (at the
right line number for some editors).

This requires the package to have a meta.position attribute.
---
 src/nix/edit.cc | 75 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 src/nix/edit.cc

diff --git a/src/nix/edit.cc b/src/nix/edit.cc
new file mode 100644
index 00000000000..632b555771c
--- /dev/null
+++ b/src/nix/edit.cc
@@ -0,0 +1,75 @@
+#include "command.hh"
+#include "shared.hh"
+#include "eval.hh"
+#include "attr-path.hh"
+
+#include <unistd.h>
+
+using namespace nix;
+
+struct CmdEdit : InstallablesCommand
+{
+    std::string name() override
+    {
+        return "edit";
+    }
+
+    std::string description() override
+    {
+        return "open the Nix expression of a Nix package in $EDITOR";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To open the Nix expression of the GNU Hello package:",
+                "nix edit nixpkgs.hello"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto state = getEvalState();
+
+        for (auto & i : installables) {
+            auto v = i->toValue(*state);
+
+            Value * v2;
+            try {
+                auto dummyArgs = state->allocBindings(0);
+                v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v);
+            } catch (Error &) {
+                throw Error("package ‘%s’ has no source location information", i->what());
+            }
+
+            auto pos = state->forceString(*v2);
+            debug("position is %s", pos);
+
+            auto colon = pos.rfind(':');
+            if (colon == std::string::npos)
+                throw Error("cannot parse meta.position attribute ‘%s’", pos);
+
+            std::string filename(pos, 0, colon);
+            int lineno = std::stoi(std::string(pos, colon + 1));
+
+            auto editor = getEnv("EDITOR", "cat");
+
+            Strings args{editor};
+
+            if (editor.find("emacs") != std::string::npos ||
+                editor.find("nano") != std::string::npos ||
+                editor.find("vim") != std::string::npos)
+                args.push_back(fmt("+%d", lineno));
+
+            args.push_back(filename);
+
+            execvp(editor.c_str(), stringsToCharPtrs(args).data());
+
+            throw SysError("cannot run editor ‘%s’", editor);
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdEdit>());

From 82a9c93c7f090d5a4eebe84894669aa13d31ed61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 10 May 2017 11:23:04 +0200
Subject: [PATCH 0328/2196] doc: builtins.attrNames returns alphabetically
 sorted list

---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 8a18b71008c..163153380ce 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -65,7 +65,7 @@ available as <function>builtins.derivation</function>.</para>
   <replaceable>set</replaceable></term>
 
     <listitem><para>Return the names of the attributes in the set
-    <replaceable>set</replaceable> in a sorted list.  For instance,
+    <replaceable>set</replaceable> in alphabetically sorted list.  For instance,
     <literal>builtins.attrNames { y = 1; x = "foo"; }</literal>
     evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem>
 

From d48edcc3a5f7b20e116b3a1148af565d7f27f3cd Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Sat, 22 Apr 2017 09:17:08 +0100
Subject: [PATCH 0329/2196] nix-shell: use appropriate prompt terminator

If running nix-shell as root, the terminator should be # and not $.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index cd4dee32674..b7cf36d1607 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -438,7 +438,7 @@ int main(int argc, char ** argv)
                         "[ -e $stdenv/setup ] && source $stdenv/setup; "
                         "%3%"
                         "set +e; "
-                        "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]$\\[\\033[0m\\] \"; "
+                        "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]\\$\\[\\033[0m\\] \"; "
                         "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                         "unset NIX_ENFORCE_PURITY; "
                         "unset NIX_INDENT_MAKE; "

From c5f23f10a84f568874321c04984b1a14d2dce978 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 May 2017 18:34:18 +0200
Subject: [PATCH 0330/2196] Replace readline by linenoise

Using linenoise avoids a license compatibility issue (#1356), is a lot
smaller and doesn't pull in ncurses.
---
 configure.ac                          |    8 -
 doc/manual/introduction/about-nix.xml |    6 +
 nix.spec.in                           |    2 -
 release.nix                           |   22 +-
 shell.nix                             |    1 -
 src/linenoise/LICENSE                 |   25 +
 src/linenoise/linenoise.c             | 1199 +++++++++++++++++++++++++
 src/linenoise/linenoise.h             |   73 ++
 src/nix/local.mk                      |    6 +-
 src/nix/repl.cc                       |  167 ++--
 10 files changed, 1377 insertions(+), 132 deletions(-)
 create mode 100644 src/linenoise/LICENSE
 create mode 100644 src/linenoise/linenoise.c
 create mode 100644 src/linenoise/linenoise.h

diff --git a/configure.ac b/configure.ac
index ac37456ae5f..c7026cf954d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -196,14 +196,6 @@ if test "$gc" = yes; then
 fi
 
 
-# Check for readline, needed by "nix repl".
-AX_LIB_READLINE
-if test "$ax_cv_lib_readline" != "no"; then
-  have_readline=1
-fi
-AC_SUBST(HAVE_READLINE, [$have_readline])
-
-
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
   [do not initialise DB etc. in `make install']),
   init_state=$enableval, init_state=yes)
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index e6dfb7a5a56..be065da3eb2 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -261,6 +261,12 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
+<para>Nix uses the <link
+xlink:href="https://github.com/antirez/linenoise">linenoise
+library</link>, which has the following license:</para>
+
+<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting>
+
 </simplesect>
 
 
diff --git a/nix.spec.in b/nix.spec.in
index 3ba2dfc94b4..390893d64dc 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -20,11 +20,9 @@ Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
-Requires: readline
 BuildRequires: bzip2-devel
 BuildRequires: sqlite-devel
 BuildRequires: libcurl-devel
-BuildRequires: readline-devel
 
 # Hack to make that shitty RPM scanning hack shut up.
 Provides: perl(Nix::SSH)
diff --git a/release.nix b/release.nix
index fa2fde4f609..54d20c868da 100644
--- a/release.nix
+++ b/release.nix
@@ -73,7 +73,7 @@ let
         buildInputs =
           [ curl
             bzip2 xz brotli
-            openssl pkgconfig sqlite boehmgc readline
+            openssl pkgconfig sqlite boehmgc
 
           ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
@@ -198,15 +198,15 @@ let
     rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
 
 
-    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" "libreadline6" ];
-    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" "libreadline6" ];
+    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
+    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
 
-    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [ "libreadline6" ];
-    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [ "libreadline6" ];
-    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" "libreadline6" ];
-    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" "libreadline6" ];
-    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" "libreadline7" ];
-    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" "libreadline7" ];
+    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [];
+    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [];
+    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
 
 
     # System tests.
@@ -299,7 +299,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" "readline-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
@@ -321,7 +321,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libreadline-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
diff --git a/shell.nix b/shell.nix
index bbce68564b9..8645d36020e 100644
--- a/shell.nix
+++ b/shell.nix
@@ -16,7 +16,6 @@ with import <nixpkgs> {};
         customMemoryManagement = false;
       })
       autoreconfHook
-      readline
 
       # For nix-perl
       perl
diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE
new file mode 100644
index 00000000000..18e814865a5
--- /dev/null
+++ b/src/linenoise/LICENSE
@@ -0,0 +1,25 @@
+Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/linenoise/linenoise.c b/src/linenoise/linenoise.c
new file mode 100644
index 00000000000..fce14a7c53a
--- /dev/null
+++ b/src/linenoise/linenoise.c
@@ -0,0 +1,1199 @@
+/* linenoise.c -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * You can find the latest source code at:
+ *
+ *   http://github.com/antirez/linenoise
+ *
+ * Does a number of crazy assumptions that happen to be true in 99.9999% of
+ * the 2010 UNIX computers around.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  *  Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *
+ *  *  Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * References:
+ * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+ * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
+ *
+ * Todo list:
+ * - Filter bogus Ctrl+<char> combinations.
+ * - Win32 support
+ *
+ * Bloat:
+ * - History search like Ctrl+r in readline?
+ *
+ * List of escape sequences used by this program, we do everything just
+ * with three sequences. In order to be so cheap we may have some
+ * flickering effect with some slow terminal, but the lesser sequences
+ * the more compatible.
+ *
+ * EL (Erase Line)
+ *    Sequence: ESC [ n K
+ *    Effect: if n is 0 or missing, clear from cursor to end of line
+ *    Effect: if n is 1, clear from beginning of line to cursor
+ *    Effect: if n is 2, clear entire line
+ *
+ * CUF (CUrsor Forward)
+ *    Sequence: ESC [ n C
+ *    Effect: moves cursor forward n chars
+ *
+ * CUB (CUrsor Backward)
+ *    Sequence: ESC [ n D
+ *    Effect: moves cursor backward n chars
+ *
+ * The following is used to get the terminal width if getting
+ * the width with the TIOCGWINSZ ioctl fails
+ *
+ * DSR (Device Status Report)
+ *    Sequence: ESC [ 6 n
+ *    Effect: reports the current cusor position as ESC [ n ; m R
+ *            where n is the row and m is the column
+ *
+ * When multi line mode is enabled, we also use an additional escape
+ * sequence. However multi line editing is disabled by default.
+ *
+ * CUU (Cursor Up)
+ *    Sequence: ESC [ n A
+ *    Effect: moves cursor up of n chars.
+ *
+ * CUD (Cursor Down)
+ *    Sequence: ESC [ n B
+ *    Effect: moves cursor down of n chars.
+ *
+ * When linenoiseClearScreen() is called, two additional escape sequences
+ * are used in order to clear the screen and position the cursor at home
+ * position.
+ *
+ * CUP (Cursor position)
+ *    Sequence: ESC [ H
+ *    Effect: moves the cursor to upper left corner
+ *
+ * ED (Erase display)
+ *    Sequence: ESC [ 2 J
+ *    Effect: clear the whole screen
+ *
+ */
+
+#include <termios.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#include "linenoise.h"
+
+#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
+#define LINENOISE_MAX_LINE 4096
+static char *unsupported_term[] = {"dumb","cons25","emacs",NULL};
+static linenoiseCompletionCallback *completionCallback = NULL;
+static linenoiseHintsCallback *hintsCallback = NULL;
+static linenoiseFreeHintsCallback *freeHintsCallback = NULL;
+
+static struct termios orig_termios; /* In order to restore at exit.*/
+static int rawmode = 0; /* For atexit() function to check if restore is needed*/
+static int mlmode = 0;  /* Multi line mode. Default is single line. */
+static int atexit_registered = 0; /* Register atexit just 1 time. */
+static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
+static int history_len = 0;
+static char **history = NULL;
+
+/* The linenoiseState structure represents the state during line editing.
+ * We pass this state to functions implementing specific editing
+ * functionalities. */
+struct linenoiseState {
+    int ifd;            /* Terminal stdin file descriptor. */
+    int ofd;            /* Terminal stdout file descriptor. */
+    char *buf;          /* Edited line buffer. */
+    size_t buflen;      /* Edited line buffer size. */
+    const char *prompt; /* Prompt to display. */
+    size_t plen;        /* Prompt length. */
+    size_t pos;         /* Current cursor position. */
+    size_t oldpos;      /* Previous refresh cursor position. */
+    size_t len;         /* Current edited line length. */
+    size_t cols;        /* Number of columns in terminal. */
+    size_t maxrows;     /* Maximum num of rows used so far (multiline mode) */
+    int history_index;  /* The history index we are currently editing. */
+};
+
+enum KEY_ACTION{
+	KEY_NULL = 0,	    /* NULL */
+	CTRL_A = 1,         /* Ctrl+a */
+	CTRL_B = 2,         /* Ctrl-b */
+	CTRL_C = 3,         /* Ctrl-c */
+	CTRL_D = 4,         /* Ctrl-d */
+	CTRL_E = 5,         /* Ctrl-e */
+	CTRL_F = 6,         /* Ctrl-f */
+	CTRL_H = 8,         /* Ctrl-h */
+	TAB = 9,            /* Tab */
+	CTRL_K = 11,        /* Ctrl+k */
+	CTRL_L = 12,        /* Ctrl+l */
+	ENTER = 13,         /* Enter */
+	CTRL_N = 14,        /* Ctrl-n */
+	CTRL_P = 16,        /* Ctrl-p */
+	CTRL_T = 20,        /* Ctrl-t */
+	CTRL_U = 21,        /* Ctrl+u */
+	CTRL_W = 23,        /* Ctrl+w */
+	ESC = 27,           /* Escape */
+	BACKSPACE =  127    /* Backspace */
+};
+
+static void linenoiseAtExit(void);
+int linenoiseHistoryAdd(const char *line);
+static void refreshLine(struct linenoiseState *l);
+
+/* Debugging macro. */
+#if 0
+FILE *lndebug_fp = NULL;
+#define lndebug(...) \
+    do { \
+        if (lndebug_fp == NULL) { \
+            lndebug_fp = fopen("/tmp/lndebug.txt","a"); \
+            fprintf(lndebug_fp, \
+            "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \
+            (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \
+            (int)l->maxrows,old_rows); \
+        } \
+        fprintf(lndebug_fp, ", " __VA_ARGS__); \
+        fflush(lndebug_fp); \
+    } while (0)
+#else
+#define lndebug(fmt, ...)
+#endif
+
+/* ======================= Low level terminal handling ====================== */
+
+/* Set if to use or not the multi line mode. */
+void linenoiseSetMultiLine(int ml) {
+    mlmode = ml;
+}
+
+/* Return true if the terminal name is in the list of terminals we know are
+ * not able to understand basic escape sequences. */
+static int isUnsupportedTerm(void) {
+    char *term = getenv("TERM");
+    int j;
+
+    if (term == NULL) return 0;
+    for (j = 0; unsupported_term[j]; j++)
+        if (!strcasecmp(term,unsupported_term[j])) return 1;
+    return 0;
+}
+
+/* Raw mode: 1960 magic shit. */
+static int enableRawMode(int fd) {
+    struct termios raw;
+
+    if (!isatty(STDIN_FILENO)) goto fatal;
+    if (!atexit_registered) {
+        atexit(linenoiseAtExit);
+        atexit_registered = 1;
+    }
+    if (tcgetattr(fd,&orig_termios) == -1) goto fatal;
+
+    raw = orig_termios;  /* modify the original mode */
+    /* input modes: no break, no CR to NL, no parity check, no strip char,
+     * no start/stop output control. */
+    raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+    /* output modes - disable post processing */
+    raw.c_oflag &= ~(OPOST);
+    /* control modes - set 8 bit chars */
+    raw.c_cflag |= (CS8);
+    /* local modes - choing off, canonical off, no extended functions,
+     * no signal chars (^Z,^C) */
+    raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+    /* control chars - set return condition: min number of bytes and timer.
+     * We want read to return every single byte, without timeout. */
+    raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+    /* put terminal in raw mode after flushing */
+    if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal;
+    rawmode = 1;
+    return 0;
+
+fatal:
+    errno = ENOTTY;
+    return -1;
+}
+
+static void disableRawMode(int fd) {
+    /* Don't even check the return value as it's too late. */
+    if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1)
+        rawmode = 0;
+}
+
+/* Use the ESC [6n escape sequence to query the horizontal cursor position
+ * and return it. On error -1 is returned, on success the position of the
+ * cursor. */
+static int getCursorPosition(int ifd, int ofd) {
+    char buf[32];
+    int cols, rows;
+    unsigned int i = 0;
+
+    /* Report cursor location */
+    if (write(ofd, "\x1b[6n", 4) != 4) return -1;
+
+    /* Read the response: ESC [ rows ; cols R */
+    while (i < sizeof(buf)-1) {
+        if (read(ifd,buf+i,1) != 1) break;
+        if (buf[i] == 'R') break;
+        i++;
+    }
+    buf[i] = '\0';
+
+    /* Parse it. */
+    if (buf[0] != ESC || buf[1] != '[') return -1;
+    if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1;
+    return cols;
+}
+
+/* Try to get the number of columns in the current terminal, or assume 80
+ * if it fails. */
+static int getColumns(int ifd, int ofd) {
+    struct winsize ws;
+
+    if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) {
+        /* ioctl() failed. Try to query the terminal itself. */
+        int start, cols;
+
+        /* Get the initial position so we can restore it later. */
+        start = getCursorPosition(ifd,ofd);
+        if (start == -1) goto failed;
+
+        /* Go to right margin and get position. */
+        if (write(ofd,"\x1b[999C",6) != 6) goto failed;
+        cols = getCursorPosition(ifd,ofd);
+        if (cols == -1) goto failed;
+
+        /* Restore position. */
+        if (cols > start) {
+            char seq[32];
+            snprintf(seq,32,"\x1b[%dD",cols-start);
+            if (write(ofd,seq,strlen(seq)) == -1) {
+                /* Can't recover... */
+            }
+        }
+        return cols;
+    } else {
+        return ws.ws_col;
+    }
+
+failed:
+    return 80;
+}
+
+/* Clear the screen. Used to handle ctrl+l */
+void linenoiseClearScreen(void) {
+    if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) {
+        /* nothing to do, just to avoid warning. */
+    }
+}
+
+/* Beep, used for completion when there is nothing to complete or when all
+ * the choices were already shown. */
+static void linenoiseBeep(void) {
+    fprintf(stderr, "\x7");
+    fflush(stderr);
+}
+
+/* ============================== Completion ================================ */
+
+/* Free a list of completion option populated by linenoiseAddCompletion(). */
+static void freeCompletions(linenoiseCompletions *lc) {
+    size_t i;
+    for (i = 0; i < lc->len; i++)
+        free(lc->cvec[i]);
+    if (lc->cvec != NULL)
+        free(lc->cvec);
+}
+
+/* This is an helper function for linenoiseEdit() and is called when the
+ * user types the <tab> key in order to complete the string currently in the
+ * input.
+ *
+ * The state of the editing is encapsulated into the pointed linenoiseState
+ * structure as described in the structure definition. */
+static int completeLine(struct linenoiseState *ls) {
+    linenoiseCompletions lc = { 0, NULL };
+    int nread, nwritten;
+    char c = 0;
+
+    completionCallback(ls->buf,&lc);
+    if (lc.len == 0) {
+        linenoiseBeep();
+    } else {
+        size_t stop = 0, i = 0;
+
+        while(!stop) {
+            /* Show completion or original buffer */
+            if (i < lc.len) {
+                struct linenoiseState saved = *ls;
+
+                ls->len = ls->pos = strlen(lc.cvec[i]);
+                ls->buf = lc.cvec[i];
+                refreshLine(ls);
+                ls->len = saved.len;
+                ls->pos = saved.pos;
+                ls->buf = saved.buf;
+            } else {
+                refreshLine(ls);
+            }
+
+            nread = read(ls->ifd,&c,1);
+            if (nread <= 0) {
+                freeCompletions(&lc);
+                return -1;
+            }
+
+            switch(c) {
+                case 9: /* tab */
+                    i = (i+1) % (lc.len+1);
+                    if (i == lc.len) linenoiseBeep();
+                    break;
+                case 27: /* escape */
+                    /* Re-show original buffer */
+                    if (i < lc.len) refreshLine(ls);
+                    stop = 1;
+                    break;
+                default:
+                    /* Update buffer and return */
+                    if (i < lc.len) {
+                        nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]);
+                        ls->len = ls->pos = nwritten;
+                    }
+                    stop = 1;
+                    break;
+            }
+        }
+    }
+
+    freeCompletions(&lc);
+    return c; /* Return last read character */
+}
+
+/* Register a callback function to be called for tab-completion. */
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) {
+    completionCallback = fn;
+}
+
+/* Register a hits function to be called to show hits to the user at the
+ * right of the prompt. */
+void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) {
+    hintsCallback = fn;
+}
+
+/* Register a function to free the hints returned by the hints callback
+ * registered with linenoiseSetHintsCallback(). */
+void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) {
+    freeHintsCallback = fn;
+}
+
+/* This function is used by the callback function registered by the user
+ * in order to add completion options given the input string when the
+ * user typed <tab>. See the example.c source code for a very easy to
+ * understand example. */
+void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
+    size_t len = strlen(str);
+    char *copy, **cvec;
+
+    copy = malloc(len+1);
+    if (copy == NULL) return;
+    memcpy(copy,str,len+1);
+    cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1));
+    if (cvec == NULL) {
+        free(copy);
+        return;
+    }
+    lc->cvec = cvec;
+    lc->cvec[lc->len++] = copy;
+}
+
+/* =========================== Line editing ================================= */
+
+/* We define a very simple "append buffer" structure, that is an heap
+ * allocated string where we can append to. This is useful in order to
+ * write all the escape sequences in a buffer and flush them to the standard
+ * output in a single call, to avoid flickering effects. */
+struct abuf {
+    char *b;
+    int len;
+};
+
+static void abInit(struct abuf *ab) {
+    ab->b = NULL;
+    ab->len = 0;
+}
+
+static void abAppend(struct abuf *ab, const char *s, int len) {
+    char *new = realloc(ab->b,ab->len+len);
+
+    if (new == NULL) return;
+    memcpy(new+ab->len,s,len);
+    ab->b = new;
+    ab->len += len;
+}
+
+static void abFree(struct abuf *ab) {
+    free(ab->b);
+}
+
+/* Helper of refreshSingleLine() and refreshMultiLine() to show hints
+ * to the right of the prompt. */
+void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
+    char seq[64];
+    if (hintsCallback && plen+l->len < l->cols) {
+        int color = -1, bold = 0;
+        char *hint = hintsCallback(l->buf,&color,&bold);
+        if (hint) {
+            int hintlen = strlen(hint);
+            int hintmaxlen = l->cols-(plen+l->len);
+            if (hintlen > hintmaxlen) hintlen = hintmaxlen;
+            if (bold == 1 && color == -1) color = 37;
+            if (color != -1 || bold != 0)
+                snprintf(seq,64,"\033[%d;%d;49m",bold,color);
+            abAppend(ab,seq,strlen(seq));
+            abAppend(ab,hint,hintlen);
+            if (color != -1 || bold != 0)
+                abAppend(ab,"\033[0m",4);
+            /* Call the function to free the hint returned. */
+            if (freeHintsCallback) freeHintsCallback(hint);
+        }
+    }
+}
+
+/* Single line low level line refresh.
+ *
+ * Rewrite the currently edited line accordingly to the buffer content,
+ * cursor position, and number of columns of the terminal. */
+static void refreshSingleLine(struct linenoiseState *l) {
+    char seq[64];
+    size_t plen = strlen(l->prompt);
+    int fd = l->ofd;
+    char *buf = l->buf;
+    size_t len = l->len;
+    size_t pos = l->pos;
+    struct abuf ab;
+
+    while((plen+pos) >= l->cols) {
+        buf++;
+        len--;
+        pos--;
+    }
+    while (plen+len > l->cols) {
+        len--;
+    }
+
+    abInit(&ab);
+    /* Cursor to left edge */
+    snprintf(seq,64,"\r");
+    abAppend(&ab,seq,strlen(seq));
+    /* Write the prompt and the current buffer content */
+    abAppend(&ab,l->prompt,strlen(l->prompt));
+    abAppend(&ab,buf,len);
+    /* Show hits if any. */
+    refreshShowHints(&ab,l,plen);
+    /* Erase to right */
+    snprintf(seq,64,"\x1b[0K");
+    abAppend(&ab,seq,strlen(seq));
+    /* Move cursor to original position. */
+    snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen));
+    abAppend(&ab,seq,strlen(seq));
+    if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
+    abFree(&ab);
+}
+
+/* Multi line low level line refresh.
+ *
+ * Rewrite the currently edited line accordingly to the buffer content,
+ * cursor position, and number of columns of the terminal. */
+static void refreshMultiLine(struct linenoiseState *l) {
+    char seq[64];
+    int plen = strlen(l->prompt);
+    int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */
+    int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */
+    int rpos2; /* rpos after refresh. */
+    int col; /* colum position, zero-based. */
+    int old_rows = l->maxrows;
+    int fd = l->ofd, j;
+    struct abuf ab;
+
+    /* Update maxrows if needed. */
+    if (rows > (int)l->maxrows) l->maxrows = rows;
+
+    /* First step: clear all the lines used before. To do so start by
+     * going to the last row. */
+    abInit(&ab);
+    if (old_rows-rpos > 0) {
+        lndebug("go down %d", old_rows-rpos);
+        snprintf(seq,64,"\x1b[%dB", old_rows-rpos);
+        abAppend(&ab,seq,strlen(seq));
+    }
+
+    /* Now for every row clear it, go up. */
+    for (j = 0; j < old_rows-1; j++) {
+        lndebug("clear+up");
+        snprintf(seq,64,"\r\x1b[0K\x1b[1A");
+        abAppend(&ab,seq,strlen(seq));
+    }
+
+    /* Clean the top line. */
+    lndebug("clear");
+    snprintf(seq,64,"\r\x1b[0K");
+    abAppend(&ab,seq,strlen(seq));
+
+    /* Write the prompt and the current buffer content */
+    abAppend(&ab,l->prompt,strlen(l->prompt));
+    abAppend(&ab,l->buf,l->len);
+
+    /* Show hits if any. */
+    refreshShowHints(&ab,l,plen);
+
+    /* If we are at the very end of the screen with our prompt, we need to
+     * emit a newline and move the prompt to the first column. */
+    if (l->pos &&
+        l->pos == l->len &&
+        (l->pos+plen) % l->cols == 0)
+    {
+        lndebug("<newline>");
+        abAppend(&ab,"\n",1);
+        snprintf(seq,64,"\r");
+        abAppend(&ab,seq,strlen(seq));
+        rows++;
+        if (rows > (int)l->maxrows) l->maxrows = rows;
+    }
+
+    /* Move cursor to right position. */
+    rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */
+    lndebug("rpos2 %d", rpos2);
+
+    /* Go up till we reach the expected positon. */
+    if (rows-rpos2 > 0) {
+        lndebug("go-up %d", rows-rpos2);
+        snprintf(seq,64,"\x1b[%dA", rows-rpos2);
+        abAppend(&ab,seq,strlen(seq));
+    }
+
+    /* Set column. */
+    col = (plen+(int)l->pos) % (int)l->cols;
+    lndebug("set col %d", 1+col);
+    if (col)
+        snprintf(seq,64,"\r\x1b[%dC", col);
+    else
+        snprintf(seq,64,"\r");
+    abAppend(&ab,seq,strlen(seq));
+
+    lndebug("\n");
+    l->oldpos = l->pos;
+
+    if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
+    abFree(&ab);
+}
+
+/* Calls the two low level functions refreshSingleLine() or
+ * refreshMultiLine() according to the selected mode. */
+static void refreshLine(struct linenoiseState *l) {
+    if (mlmode)
+        refreshMultiLine(l);
+    else
+        refreshSingleLine(l);
+}
+
+/* Insert the character 'c' at cursor current position.
+ *
+ * On error writing to the terminal -1 is returned, otherwise 0. */
+int linenoiseEditInsert(struct linenoiseState *l, char c) {
+    if (l->len < l->buflen) {
+        if (l->len == l->pos) {
+            l->buf[l->pos] = c;
+            l->pos++;
+            l->len++;
+            l->buf[l->len] = '\0';
+            if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) {
+                /* Avoid a full update of the line in the
+                 * trivial case. */
+                if (write(l->ofd,&c,1) == -1) return -1;
+            } else {
+                refreshLine(l);
+            }
+        } else {
+            memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos);
+            l->buf[l->pos] = c;
+            l->len++;
+            l->pos++;
+            l->buf[l->len] = '\0';
+            refreshLine(l);
+        }
+    }
+    return 0;
+}
+
+/* Move cursor on the left. */
+void linenoiseEditMoveLeft(struct linenoiseState *l) {
+    if (l->pos > 0) {
+        l->pos--;
+        refreshLine(l);
+    }
+}
+
+/* Move cursor on the right. */
+void linenoiseEditMoveRight(struct linenoiseState *l) {
+    if (l->pos != l->len) {
+        l->pos++;
+        refreshLine(l);
+    }
+}
+
+/* Move cursor to the start of the line. */
+void linenoiseEditMoveHome(struct linenoiseState *l) {
+    if (l->pos != 0) {
+        l->pos = 0;
+        refreshLine(l);
+    }
+}
+
+/* Move cursor to the end of the line. */
+void linenoiseEditMoveEnd(struct linenoiseState *l) {
+    if (l->pos != l->len) {
+        l->pos = l->len;
+        refreshLine(l);
+    }
+}
+
+/* Substitute the currently edited line with the next or previous history
+ * entry as specified by 'dir'. */
+#define LINENOISE_HISTORY_NEXT 0
+#define LINENOISE_HISTORY_PREV 1
+void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) {
+    if (history_len > 1) {
+        /* Update the current history entry before to
+         * overwrite it with the next one. */
+        free(history[history_len - 1 - l->history_index]);
+        history[history_len - 1 - l->history_index] = strdup(l->buf);
+        /* Show the new entry */
+        l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1;
+        if (l->history_index < 0) {
+            l->history_index = 0;
+            return;
+        } else if (l->history_index >= history_len) {
+            l->history_index = history_len-1;
+            return;
+        }
+        strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen);
+        l->buf[l->buflen-1] = '\0';
+        l->len = l->pos = strlen(l->buf);
+        refreshLine(l);
+    }
+}
+
+/* Delete the character at the right of the cursor without altering the cursor
+ * position. Basically this is what happens with the "Delete" keyboard key. */
+void linenoiseEditDelete(struct linenoiseState *l) {
+    if (l->len > 0 && l->pos < l->len) {
+        memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1);
+        l->len--;
+        l->buf[l->len] = '\0';
+        refreshLine(l);
+    }
+}
+
+/* Backspace implementation. */
+void linenoiseEditBackspace(struct linenoiseState *l) {
+    if (l->pos > 0 && l->len > 0) {
+        memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos);
+        l->pos--;
+        l->len--;
+        l->buf[l->len] = '\0';
+        refreshLine(l);
+    }
+}
+
+/* Delete the previosu word, maintaining the cursor at the start of the
+ * current word. */
+void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
+    size_t old_pos = l->pos;
+    size_t diff;
+
+    while (l->pos > 0 && l->buf[l->pos-1] == ' ')
+        l->pos--;
+    while (l->pos > 0 && l->buf[l->pos-1] != ' ')
+        l->pos--;
+    diff = old_pos - l->pos;
+    memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1);
+    l->len -= diff;
+    refreshLine(l);
+}
+
+/* This function is the core of the line editing capability of linenoise.
+ * It expects 'fd' to be already in "raw mode" so that every key pressed
+ * will be returned ASAP to read().
+ *
+ * The resulting string is put into 'buf' when the user type enter, or
+ * when ctrl+d is typed.
+ *
+ * The function returns the length of the current buffer. */
+static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt)
+{
+    struct linenoiseState l;
+
+    /* Populate the linenoise state that we pass to functions implementing
+     * specific editing functionalities. */
+    l.ifd = stdin_fd;
+    l.ofd = stdout_fd;
+    l.buf = buf;
+    l.buflen = buflen;
+    l.prompt = prompt;
+    l.plen = strlen(prompt);
+    l.oldpos = l.pos = 0;
+    l.len = 0;
+    l.cols = getColumns(stdin_fd, stdout_fd);
+    l.maxrows = 0;
+    l.history_index = 0;
+
+    /* Buffer starts empty. */
+    l.buf[0] = '\0';
+    l.buflen--; /* Make sure there is always space for the nulterm */
+
+    /* The latest history entry is always our current buffer, that
+     * initially is just an empty string. */
+    linenoiseHistoryAdd("");
+
+    if (write(l.ofd,prompt,l.plen) == -1) return -1;
+    while(1) {
+        char c;
+        int nread;
+        char seq[3];
+
+        nread = read(l.ifd,&c,1);
+        if (nread <= 0) return l.len;
+
+        /* Only autocomplete when the callback is set. It returns < 0 when
+         * there was an error reading from fd. Otherwise it will return the
+         * character that should be handled next. */
+        if (c == 9 && completionCallback != NULL) {
+            c = completeLine(&l);
+            /* Return on errors */
+            if (c < 0) return l.len;
+            /* Read next character when 0 */
+            if (c == 0) continue;
+        }
+
+        switch(c) {
+        case ENTER:    /* enter */
+            history_len--;
+            free(history[history_len]);
+            if (mlmode) linenoiseEditMoveEnd(&l);
+            if (hintsCallback) {
+                /* Force a refresh without hints to leave the previous
+                 * line as the user typed it after a newline. */
+                linenoiseHintsCallback *hc = hintsCallback;
+                hintsCallback = NULL;
+                refreshLine(&l);
+                hintsCallback = hc;
+            }
+            return (int)l.len;
+        case CTRL_C:     /* ctrl-c */
+            errno = EAGAIN;
+            return -1;
+        case BACKSPACE:   /* backspace */
+        case 8:     /* ctrl-h */
+            linenoiseEditBackspace(&l);
+            break;
+        case CTRL_D:     /* ctrl-d, remove char at right of cursor, or if the
+                            line is empty, act as end-of-file. */
+            if (l.len > 0) {
+                linenoiseEditDelete(&l);
+            } else {
+                history_len--;
+                free(history[history_len]);
+                return -1;
+            }
+            break;
+        case CTRL_T:    /* ctrl-t, swaps current character with previous. */
+            if (l.pos > 0 && l.pos < l.len) {
+                int aux = buf[l.pos-1];
+                buf[l.pos-1] = buf[l.pos];
+                buf[l.pos] = aux;
+                if (l.pos != l.len-1) l.pos++;
+                refreshLine(&l);
+            }
+            break;
+        case CTRL_B:     /* ctrl-b */
+            linenoiseEditMoveLeft(&l);
+            break;
+        case CTRL_F:     /* ctrl-f */
+            linenoiseEditMoveRight(&l);
+            break;
+        case CTRL_P:    /* ctrl-p */
+            linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV);
+            break;
+        case CTRL_N:    /* ctrl-n */
+            linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT);
+            break;
+        case ESC:    /* escape sequence */
+            /* Read the next two bytes representing the escape sequence.
+             * Use two calls to handle slow terminals returning the two
+             * chars at different times. */
+            if (read(l.ifd,seq,1) == -1) break;
+            if (read(l.ifd,seq+1,1) == -1) break;
+
+            /* ESC [ sequences. */
+            if (seq[0] == '[') {
+                if (seq[1] >= '0' && seq[1] <= '9') {
+                    /* Extended escape, read additional byte. */
+                    if (read(l.ifd,seq+2,1) == -1) break;
+                    if (seq[2] == '~') {
+                        switch(seq[1]) {
+                        case '3': /* Delete key. */
+                            linenoiseEditDelete(&l);
+                            break;
+                        }
+                    }
+                } else {
+                    switch(seq[1]) {
+                    case 'A': /* Up */
+                        linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV);
+                        break;
+                    case 'B': /* Down */
+                        linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT);
+                        break;
+                    case 'C': /* Right */
+                        linenoiseEditMoveRight(&l);
+                        break;
+                    case 'D': /* Left */
+                        linenoiseEditMoveLeft(&l);
+                        break;
+                    case 'H': /* Home */
+                        linenoiseEditMoveHome(&l);
+                        break;
+                    case 'F': /* End*/
+                        linenoiseEditMoveEnd(&l);
+                        break;
+                    }
+                }
+            }
+
+            /* ESC O sequences. */
+            else if (seq[0] == 'O') {
+                switch(seq[1]) {
+                case 'H': /* Home */
+                    linenoiseEditMoveHome(&l);
+                    break;
+                case 'F': /* End*/
+                    linenoiseEditMoveEnd(&l);
+                    break;
+                }
+            }
+            break;
+        default:
+            if (linenoiseEditInsert(&l,c)) return -1;
+            break;
+        case CTRL_U: /* Ctrl+u, delete the whole line. */
+            buf[0] = '\0';
+            l.pos = l.len = 0;
+            refreshLine(&l);
+            break;
+        case CTRL_K: /* Ctrl+k, delete from current to end of line. */
+            buf[l.pos] = '\0';
+            l.len = l.pos;
+            refreshLine(&l);
+            break;
+        case CTRL_A: /* Ctrl+a, go to the start of the line */
+            linenoiseEditMoveHome(&l);
+            break;
+        case CTRL_E: /* ctrl+e, go to the end of the line */
+            linenoiseEditMoveEnd(&l);
+            break;
+        case CTRL_L: /* ctrl+l, clear screen */
+            linenoiseClearScreen();
+            refreshLine(&l);
+            break;
+        case CTRL_W: /* ctrl+w, delete previous word */
+            linenoiseEditDeletePrevWord(&l);
+            break;
+        }
+    }
+    return l.len;
+}
+
+/* This special mode is used by linenoise in order to print scan codes
+ * on screen for debugging / development purposes. It is implemented
+ * by the linenoise_example program using the --keycodes option. */
+void linenoisePrintKeyCodes(void) {
+    char quit[4];
+
+    printf("Linenoise key codes debugging mode.\n"
+            "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
+    if (enableRawMode(STDIN_FILENO) == -1) return;
+    memset(quit,' ',4);
+    while(1) {
+        char c;
+        int nread;
+
+        nread = read(STDIN_FILENO,&c,1);
+        if (nread <= 0) continue;
+        memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */
+        quit[sizeof(quit)-1] = c; /* Insert current char on the right. */
+        if (memcmp(quit,"quit",sizeof(quit)) == 0) break;
+
+        printf("'%c' %02x (%d) (type quit to exit)\n",
+            isprint(c) ? c : '?', (int)c, (int)c);
+        printf("\r"); /* Go left edge manually, we are in raw mode. */
+        fflush(stdout);
+    }
+    disableRawMode(STDIN_FILENO);
+}
+
+/* This function calls the line editing function linenoiseEdit() using
+ * the STDIN file descriptor set in raw mode. */
+static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
+    int count;
+
+    if (buflen == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (enableRawMode(STDIN_FILENO) == -1) return -1;
+    count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
+    disableRawMode(STDIN_FILENO);
+    printf("\n");
+    return count;
+}
+
+/* This function is called when linenoise() is called with the standard
+ * input file descriptor not attached to a TTY. So for example when the
+ * program using linenoise is called in pipe or with a file redirected
+ * to its standard input. In this case, we want to be able to return the
+ * line regardless of its length (by default we are limited to 4k). */
+static char *linenoiseNoTTY(void) {
+    char *line = NULL;
+    size_t len = 0, maxlen = 0;
+
+    while(1) {
+        if (len == maxlen) {
+            if (maxlen == 0) maxlen = 16;
+            maxlen *= 2;
+            char *oldval = line;
+            line = realloc(line,maxlen);
+            if (line == NULL) {
+                if (oldval) free(oldval);
+                return NULL;
+            }
+        }
+        int c = fgetc(stdin);
+        if (c == EOF || c == '\n') {
+            if (c == EOF && len == 0) {
+                free(line);
+                return NULL;
+            } else {
+                line[len] = '\0';
+                return line;
+            }
+        } else {
+            line[len] = c;
+            len++;
+        }
+    }
+}
+
+/* The high level function that is the main API of the linenoise library.
+ * This function checks if the terminal has basic capabilities, just checking
+ * for a blacklist of stupid terminals, and later either calls the line
+ * editing function or uses dummy fgets() so that you will be able to type
+ * something even in the most desperate of the conditions. */
+char *linenoise(const char *prompt) {
+    char buf[LINENOISE_MAX_LINE];
+    int count;
+
+    if (!isatty(STDIN_FILENO)) {
+        /* Not a tty: read from file / pipe. In this mode we don't want any
+         * limit to the line size, so we call a function to handle that. */
+        return linenoiseNoTTY();
+    } else if (isUnsupportedTerm()) {
+        size_t len;
+
+        printf("%s",prompt);
+        fflush(stdout);
+        if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL;
+        len = strlen(buf);
+        while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) {
+            len--;
+            buf[len] = '\0';
+        }
+        return strdup(buf);
+    } else {
+        count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt);
+        if (count == -1) return NULL;
+        return strdup(buf);
+    }
+}
+
+/* This is just a wrapper the user may want to call in order to make sure
+ * the linenoise returned buffer is freed with the same allocator it was
+ * created with. Useful when the main program is using an alternative
+ * allocator. */
+void linenoiseFree(void *ptr) {
+    free(ptr);
+}
+
+/* ================================ History ================================= */
+
+/* Free the history, but does not reset it. Only used when we have to
+ * exit() to avoid memory leaks are reported by valgrind & co. */
+static void freeHistory(void) {
+    if (history) {
+        int j;
+
+        for (j = 0; j < history_len; j++)
+            free(history[j]);
+        free(history);
+    }
+}
+
+/* At exit we'll try to fix the terminal to the initial conditions. */
+static void linenoiseAtExit(void) {
+    disableRawMode(STDIN_FILENO);
+    freeHistory();
+}
+
+/* This is the API call to add a new entry in the linenoise history.
+ * It uses a fixed array of char pointers that are shifted (memmoved)
+ * when the history max length is reached in order to remove the older
+ * entry and make room for the new one, so it is not exactly suitable for huge
+ * histories, but will work well for a few hundred of entries.
+ *
+ * Using a circular buffer is smarter, but a bit more complex to handle. */
+int linenoiseHistoryAdd(const char *line) {
+    char *linecopy;
+
+    if (history_max_len == 0) return 0;
+
+    /* Initialization on first call. */
+    if (history == NULL) {
+        history = malloc(sizeof(char*)*history_max_len);
+        if (history == NULL) return 0;
+        memset(history,0,(sizeof(char*)*history_max_len));
+    }
+
+    /* Don't add duplicated lines. */
+    if (history_len && !strcmp(history[history_len-1], line)) return 0;
+
+    /* Add an heap allocated copy of the line in the history.
+     * If we reached the max length, remove the older line. */
+    linecopy = strdup(line);
+    if (!linecopy) return 0;
+    if (history_len == history_max_len) {
+        free(history[0]);
+        memmove(history,history+1,sizeof(char*)*(history_max_len-1));
+        history_len--;
+    }
+    history[history_len] = linecopy;
+    history_len++;
+    return 1;
+}
+
+/* Set the maximum length for the history. This function can be called even
+ * if there is already some history, the function will make sure to retain
+ * just the latest 'len' elements if the new history length value is smaller
+ * than the amount of items already inside the history. */
+int linenoiseHistorySetMaxLen(int len) {
+    char **new;
+
+    if (len < 1) return 0;
+    if (history) {
+        int tocopy = history_len;
+
+        new = malloc(sizeof(char*)*len);
+        if (new == NULL) return 0;
+
+        /* If we can't copy everything, free the elements we'll not use. */
+        if (len < tocopy) {
+            int j;
+
+            for (j = 0; j < tocopy-len; j++) free(history[j]);
+            tocopy = len;
+        }
+        memset(new,0,sizeof(char*)*len);
+        memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy);
+        free(history);
+        history = new;
+    }
+    history_max_len = len;
+    if (history_len > history_max_len)
+        history_len = history_max_len;
+    return 1;
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int linenoiseHistorySave(const char *filename) {
+    mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO);
+    FILE *fp;
+    int j;
+
+    fp = fopen(filename,"w");
+    umask(old_umask);
+    if (fp == NULL) return -1;
+    chmod(filename,S_IRUSR|S_IWUSR);
+    for (j = 0; j < history_len; j++)
+        fprintf(fp,"%s\n",history[j]);
+    fclose(fp);
+    return 0;
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int linenoiseHistoryLoad(const char *filename) {
+    FILE *fp = fopen(filename,"r");
+    char buf[LINENOISE_MAX_LINE];
+
+    if (fp == NULL) return -1;
+
+    while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) {
+        char *p;
+
+        p = strchr(buf,'\r');
+        if (!p) p = strchr(buf,'\n');
+        if (p) *p = '\0';
+        linenoiseHistoryAdd(buf);
+    }
+    fclose(fp);
+    return 0;
+}
diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h
new file mode 100644
index 00000000000..ed20232c576
--- /dev/null
+++ b/src/linenoise/linenoise.h
@@ -0,0 +1,73 @@
+/* linenoise.h -- VERSION 1.0
+ *
+ * Guerrilla line editing library against the idea that a line editing lib
+ * needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * ------------------------------------------------------------------------
+ *
+ * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *  *  Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *
+ *  *  Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINENOISE_H
+#define __LINENOISE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct linenoiseCompletions {
+  size_t len;
+  char **cvec;
+} linenoiseCompletions;
+
+typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
+typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold);
+typedef void(linenoiseFreeHintsCallback)(void *);
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
+void linenoiseSetHintsCallback(linenoiseHintsCallback *);
+void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
+void linenoiseAddCompletion(linenoiseCompletions *, const char *);
+
+char *linenoise(const char *prompt);
+void linenoiseFree(void *ptr);
+int linenoiseHistoryAdd(const char *line);
+int linenoiseHistorySetMaxLen(int len);
+int linenoiseHistorySave(const char *filename);
+int linenoiseHistoryLoad(const char *filename);
+void linenoiseClearScreen(void);
+void linenoiseSetMultiLine(int ml);
+void linenoisePrintKeyCodes(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LINENOISE_H */
diff --git a/src/nix/local.mk b/src/nix/local.mk
index e71cf16fabf..c7d2d328aab 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -2,12 +2,8 @@ programs += nix
 
 nix_DIR := $(d)
 
-nix_SOURCES := $(wildcard $(d)/*.cc)
+nix_SOURCES := $(wildcard $(d)/*.cc) src/linenoise/linenoise.c
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
-ifeq ($(HAVE_READLINE), 1)
-  nix_LDFLAGS += -lreadline
-endif
-
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 13488bf1dbd..437c7903ed4 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -1,13 +1,8 @@
-#if HAVE_LIBREADLINE
-
 #include <iostream>
 #include <cstdlib>
 
 #include <setjmp.h>
 
-#include <readline/readline.h>
-#include <readline/history.h>
-
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
@@ -18,6 +13,9 @@
 #include "affinity.hh"
 #include "globals.hh"
 #include "command.hh"
+#include "finally.hh"
+
+#include "src/linenoise/linenoise.h"
 
 namespace nix {
 
@@ -44,14 +42,11 @@ struct NixRepl
 
     const Path historyFile;
 
-    StringSet completions;
-    StringSet::iterator curCompletion;
-
     NixRepl(const Strings & searchPath, nix::ref<Store> store);
     ~NixRepl();
     void mainLoop(const Strings & files);
-    void completePrefix(string prefix);
-    bool getLine(string & input, const char * prompt);
+    StringSet completePrefix(string prefix);
+    bool getLine(string & input, const std::string &prompt);
     Path getDerivationPath(Value & v);
     bool processLine(string line);
     void loadFile(const Path & path);
@@ -122,7 +117,17 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
 
 NixRepl::~NixRepl()
 {
-    write_history(historyFile.c_str());
+    linenoiseHistorySave(historyFile.c_str());
+}
+
+
+static NixRepl * curRepl; // ugly
+
+static void completionCallback(const char * s, linenoiseCompletions *lc)
+{
+    /* Otherwise, return all symbols that start with the prefix. */
+    for (auto & c : curRepl->completePrefix(s))
+        linenoiseAddCompletion(lc, c.c_str());
 }
 
 
@@ -137,22 +142,20 @@ void NixRepl::mainLoop(const Strings & files)
     reloadFiles();
     if (!loadedFiles.empty()) std::cout << std::endl;
 
-    // Allow nix-repl specific settings in .inputrc
-    rl_readline_name = "nix-repl";
-    using_history();
     createDirs(dirOf(historyFile));
-    read_history(historyFile.c_str());
+    linenoiseHistorySetMaxLen(1000);
+    linenoiseHistoryLoad(historyFile.c_str());
 
-    string input;
+    curRepl = this;
+    linenoiseSetCompletionCallback(completionCallback);
+
+    std::string input;
 
     while (true) {
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
-        const char * prompt = input.empty() ? "nix-repl> " : "          ";
-        if (!getLine(input, prompt)) {
-            std::cout << std::endl;
+        if (!getLine(input, input.empty() ? "nix-repl> " : "          "))
             break;
-        }
 
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
@@ -170,103 +173,57 @@ void NixRepl::mainLoop(const Strings & files)
             printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         }
 
-        // We handled the current input fully, so we should clear it and read brand new input.
+        // We handled the current input fully, so we should clear it
+        // and read brand new input.
+        linenoiseHistoryAdd(input.c_str());
         input.clear();
         std::cout << std::endl;
     }
 }
 
 
-/* Apparently, the only way to get readline() to return on Ctrl-C
-   (SIGINT) is to use siglongjmp().  That's fucked up... */
-static sigjmp_buf sigintJmpBuf;
-
-
-static void sigintHandler(int signo)
+bool NixRepl::getLine(string & input, const std::string &prompt)
 {
-    siglongjmp(sigintJmpBuf, 1);
-}
-
-
-/* Oh, if only g++ had nested functions... */
-NixRepl * curRepl;
-
-char * completerThunk(const char * s, int state)
-{
-    string prefix(s);
-
-    /* If the prefix has a slash in it, use readline's builtin filename
-       completer. */
-    if (prefix.find('/') != string::npos)
-        return rl_filename_completion_function(s, state);
-
-    /* Otherwise, return all symbols that start with the prefix. */
-    if (state == 0) {
-        curRepl->completePrefix(s);
-        curRepl->curCompletion = curRepl->completions.begin();
-    }
-    if (curRepl->curCompletion == curRepl->completions.end()) return 0;
-    return strdup((curRepl->curCompletion++)->c_str());
+    char * s = linenoise(prompt.c_str());
+    Finally doFree([&]() { linenoiseFree(s); });
+    if (!s) return false;
+    input += s;
+    return true;
 }
 
 
-bool NixRepl::getLine(string & input, const char * prompt)
+StringSet NixRepl::completePrefix(string prefix)
 {
-    struct sigaction act, old;
-    act.sa_handler = sigintHandler;
-    sigfillset(&act.sa_mask);
-    act.sa_flags = 0;
-    if (sigaction(SIGINT, &act, &old))
-        throw SysError("installing handler for SIGINT");
-
-    static sigset_t savedSignalMask, set;
-    sigemptyset(&set);
-    sigaddset(&set, SIGINT);
-
-    if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
-        throw SysError("unblocking SIGINT");
+    StringSet completions;
 
-    if (sigsetjmp(sigintJmpBuf, 1)) {
-        input.clear();
+    size_t start = prefix.find_last_of(" \n\r\t(){}[]");
+    std::string prev, cur;
+    if (start == std::string::npos) {
+        prev = "";
+        cur = prefix;
     } else {
-        curRepl = this;
-        rl_completion_entry_function = completerThunk;
-
-        char * s = readline(prompt);
-        if (!s) return false;
-        input.append(s);
-        input.push_back('\n');
-        if (!removeWhitespace(s).empty()) {
-            add_history(s);
-            append_history(1, 0);
-        }
-        free(s);
+        prev = std::string(prefix, 0, start + 1);
+        cur = std::string(prefix, start + 1);
     }
 
-    _isInterrupted = 0;
-
-    if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
-        throw SysError("restoring signals");
-
-    if (sigaction(SIGINT, &old, 0))
-        throw SysError("restoring handler for SIGINT");
-
-    return true;
-}
-
-
-void NixRepl::completePrefix(string prefix)
-{
-    completions.clear();
-
-    size_t dot = prefix.rfind('.');
+    size_t slash, dot;
 
-    if (dot == string::npos) {
+    if ((slash = cur.rfind('/')) != string::npos) {
+        try {
+            auto dir = std::string(cur, 0, slash);
+            auto prefix2 = std::string(cur, slash + 1);
+            for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
+                if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2))
+                    completions.insert(prev + dir + "/" + entry.name);
+            }
+        } catch (Error &) {
+        }
+    } else if ((dot = cur.rfind('.')) == string::npos) {
         /* This is a variable name; look it up in the current scope. */
-        StringSet::iterator i = varNames.lower_bound(prefix);
+        StringSet::iterator i = varNames.lower_bound(cur);
         while (i != varNames.end()) {
-            if (string(*i, 0, prefix.size()) != prefix) break;
-            completions.insert(*i);
+            if (string(*i, 0, cur.size()) != cur) break;
+            completions.insert(prev + *i);
             i++;
         }
     } else {
@@ -274,8 +231,8 @@ void NixRepl::completePrefix(string prefix)
             /* This is an expression that should evaluate to an
                attribute set.  Evaluate it to get the names of the
                attributes. */
-            string expr(prefix, 0, dot);
-            string prefix2 = string(prefix, dot + 1);
+            string expr(cur, 0, dot);
+            string cur2 = string(cur, dot + 1);
 
             Expr * e = parseString(expr);
             Value v;
@@ -284,8 +241,8 @@ void NixRepl::completePrefix(string prefix)
 
             for (auto & i : *v.attrs) {
                 string name = i.name;
-                if (string(name, 0, prefix2.size()) != prefix2) continue;
-                completions.insert(expr + "." + name);
+                if (string(name, 0, cur2.size()) != cur2) continue;
+                completions.insert(prev + expr + "." + name);
             }
 
         } catch (ParseError & e) {
@@ -296,6 +253,8 @@ void NixRepl::completePrefix(string prefix)
             // Quietly ignore undefined variable errors.
         }
     }
+
+    return completions;
 }
 
 
@@ -728,5 +687,3 @@ struct CmdRepl : StoreCommand
 static RegisterCommand r1(make_ref<CmdRepl>());
 
 }
-
-#endif

From 1fd59447d56a88add8874f9a8b0885a1acd13606 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 May 2017 18:38:17 +0200
Subject: [PATCH 0331/2196] Typo

---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 163153380ce..63d13e184a1 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -65,7 +65,7 @@ available as <function>builtins.derivation</function>.</para>
   <replaceable>set</replaceable></term>
 
     <listitem><para>Return the names of the attributes in the set
-    <replaceable>set</replaceable> in alphabetically sorted list.  For instance,
+    <replaceable>set</replaceable> in an alphabetically sorted list.  For instance,
     <literal>builtins.attrNames { y = 1; x = "foo"; }</literal>
     evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem>
 

From 45d7b1a9e9018a2be5add1dc6d983e6b7b339c61 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 13:26:03 +0200
Subject: [PATCH 0332/2196] LocalStore::addToStore(): Check info.narSize

It allowed the client to specify bogus narSize values. In particular,
Downloader::downloadCached wasn't setting narSize at all.
---
 src/libstore/download.cc    | 1 +
 src/libstore/local-store.cc | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4d502219ed8..93519ec663a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -652,6 +652,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                 Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
                 info.path = store->makeFixedOutputPath(false, hash, name);
                 info.narHash = hashString(htSHA256, *sink.s);
+                info.narSize = sink.s->size();
                 info.ca = makeFixedOutputCA(false, hash);
                 store->addToStore(info, sink.s, false, true);
                 storePath = info.path;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c8e61126c1b..ed03c3f425b 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -919,8 +919,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
 
     Hash h = hashString(htSHA256, *nar);
     if (h != info.narHash)
-        throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") %
-            info.path % info.narHash.to_string() % h.to_string());
+        throw Error("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’",
+            info.path, info.narHash.to_string(), h.to_string());
+
+    if (nar->size() != info.narSize)
+        throw Error("szie mismatch importing path ‘%s’; expected %s, got %s",
+            info.path, info.narSize, nar->size());
 
     if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys))
         throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path);

From 6f245bf24a1154142acf2b8dfa620b891f461d55 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 13:31:23 +0200
Subject: [PATCH 0333/2196] Change the meaning of info.ultimate

It now means "paths that were built locally". It no longer includes
paths that were added locally. For those we don't need info.ultimate,
since we have the content-addressability assertion (info.ca).
---
 src/libstore/local-store.cc | 2 --
 src/libstore/store-api.hh   | 5 ++---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ed03c3f425b..207e8a40b6d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1010,7 +1010,6 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
             info.path = dstPath;
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ultimate = true;
             info.ca = makeFixedOutputCA(recursive, h);
             registerValidPath(info);
         }
@@ -1073,7 +1072,6 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
             info.narHash = narHash;
             info.narSize = sink.s->size();
             info.references = references;
-            info.ultimate = true;
             info.ca = "text:" + hash.to_string();
             registerValidPath(info);
         }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b06f5d86a93..929c95a0f2f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -113,9 +113,8 @@ struct ValidPathInfo
     uint64_t narSize = 0; // 0 = unknown
     uint64_t id; // internal use only
 
-    /* Whether the path is ultimately trusted, that is, it was built
-       locally or is content-addressable (e.g. added via addToStore()
-       or the result of a fixed-output derivation). */
+    /* Whether the path is ultimately trusted, that is, it's a
+       derivation output that was built locally. */
     bool ultimate = false;
 
     StringSet sigs; // note: not necessarily verified

From 2b2de5ef6aeb023f22e551e74e454acc73106cc8 Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <freddyrietdijk@fridh.nl>
Date: Thu, 11 May 2017 13:38:13 +0200
Subject: [PATCH 0334/2196] Document fetchTarball can take a sha256

Note that I refer to `nix-prefetch-url`.
---
 doc/manual/expressions/builtins.xml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 63d13e184a1..e9c641ab305 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -292,6 +292,24 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo
 stdenv.mkDerivation { … }
 </programlisting>
 
+    Note that when obtaining the hash with <varname>nix-prefetch-url
+    </varname> the option <varname>--unpack</varname> is required.
+    </para> 
+      
+    <para>This function can also verify the contents against a hash. 
+    In that case, the function takes a set instead of a URL. The set 
+    requires the attribute <varname>url</varname> and the attribute
+    <varname>sha256</varname>, e.g.
+      
+<programlisting>
+with import (fetchTarball {
+  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
+  sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+}) {};
+
+stdenv.mkDerivation { … }
+</programlisting>
+      
     </para>
 
     <para>This function is not available if <link

From 1a8e15053a6611963042306d4d3b0d5b89629eb1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 13:58:09 +0200
Subject: [PATCH 0335/2196] Don't allow untrusted users to set info.ultimate

Note that a trusted signature was still required in this case so it
was not a huge deal.
---
 src/nix-daemon/nix-daemon.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 1b90fad165a..d2bb7b8c88b 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -621,6 +621,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         from >> info.ca >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
+        if (!trusted)
+            info.ultimate = false;
 
         TeeSink tee(from);
         parseDump(tee, tee.source);

From ea65ae0f9c3461f9b098c8757e01f5c3add8b40c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 13:59:47 +0200
Subject: [PATCH 0336/2196] Tweak error message

---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 93519ec663a..5390bdbf5ed 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -690,7 +690,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     }
 
     if (expectedStorePath != "" && storePath != expectedStorePath)
-        throw nix::Error(format("hash mismatch in file downloaded from ‘%s’") % url);
+        throw nix::Error("store path mismatch in file downloaded from ‘%s’", url);
 
     return storePath;
 }

From 62d476c7ee5dbb79fb435895e0cda3fac8f53ba3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 14:02:03 +0200
Subject: [PATCH 0337/2196] Fix typo

---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8405c66dd47..96799adb1e6 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -167,7 +167,7 @@ void checkStoreName(const string & name)
    collisions (for security).  For instance, it shouldn't be feasible
    to come up with a derivation whose output path collides with the
    path for a copied source.  The former would have a <s> starting with
-   "output:out:", while the latter would have a <2> starting with
+   "output:out:", while the latter would have a <s> starting with
    "source:".
 */
 

From 510bc1735b3507b0f434303fdec5e824c879c838 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 15:09:09 +0200
Subject: [PATCH 0338/2196] Add an option for extending the user agent header

This is useful e.g. for distinguishing traffic to a binary cache
(e.g. certain machines can use a different tag in the user agent).
---
 src/libstore/download.cc | 4 +++-
 src/libstore/globals.hh  | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 5390bdbf5ed..dc1a3d94f74 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -220,7 +220,9 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
             curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
-            curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion).c_str());
+            curl_easy_setopt(req, CURLOPT_USERAGENT,
+                ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
+                    (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 7295b0d30af..af37ec61d7a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -318,6 +318,9 @@ public:
 
     Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
         "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+
+    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
+        "String appended to the user agent in HTTP requests."};
 };
 
 

From 75a1d9849d7355c227ce76be17809a71852956b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= <benno.fuenfstueck@gmail.com>
Date: Mon, 15 May 2017 10:17:53 +0200
Subject: [PATCH 0339/2196] nar-accessor: use tree, fixes readDirectory missing
 children

Previously, if a directory `foo` existed and a file `foo-` (where `-` is any character that is sorted before `/`), then  `readDirectory` would return an empty list.

To fix this, we now use a tree where we can just access the children of the node, and do not need to rely on sorting behavior to list the contents of a directory.
---
 src/libstore/nar-accessor.cc | 109 ++++++++++++++++++++++++-----------
 1 file changed, 76 insertions(+), 33 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 4cb5de7449e..ee1cf385c08 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -2,6 +2,8 @@
 #include "archive.hh"
 
 #include <map>
+#include <stack>
+#include <algorithm>
 
 namespace nix {
 
@@ -16,16 +18,36 @@ struct NarMember
     size_t start, size;
 
     std::string target;
+
+    /* If this is a directory, all the children of the directory. */
+    std::map<std::string, NarMember> children;
+
+    NarMember* find(const Path & path)
+    {
+        if(path == "") return this;
+
+        if(type != FSAccessor::Type::tDirectory) {
+            return nullptr;
+        }
+
+        auto split = std::find(path.begin() + 1, path.end(), '/');
+        std::string child_name(path.begin() + 1, split);
+        std::string remaining(split, path.end());
+
+        auto child = children.find(child_name);
+        if(child == children.end()) return nullptr;
+
+        return child->second.find(remaining);
+    }
+
 };
 
 struct NarIndexer : ParseSink, StringSource
 {
-    // FIXME: should store this as a tree. Now we're vulnerable to
-    // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
-    typedef std::map<Path, NarMember> Members;
-    Members members;
+    NarMember root;
+    std::stack<NarMember*> parents;
 
-    Path currentPath;
+    std::string currentName;
     std::string currentStart;
     bool isExec = false;
 
@@ -33,28 +55,45 @@ struct NarIndexer : ParseSink, StringSource
     {
     }
 
+    void createMember(const Path & path, NarMember member) {
+        size_t level = std::count(path.begin(), path.end(), '/');
+        while(parents.size() > level) {
+            parents.pop();
+        }
+
+        if(parents.empty()) {
+            root = std::move(member);
+            parents.push(&root);
+        } else {
+            if(parents.top()->type != FSAccessor::Type::tDirectory) {
+                throw Error(format("NAR file missing parent directory of path ‘%1%’") % path);
+            }
+            auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
+            parents.push(&result.first->second);
+        }
+    }
+
     void createDirectory(const Path & path) override
     {
-        members.emplace(path,
-            NarMember{FSAccessor::Type::tDirectory, false, 0, 0});
+        createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 }); 
     }
 
     void createRegularFile(const Path & path) override
     {
-        currentPath = path;
+        createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 });
     }
 
     void isExecutable() override
     {
-        isExec = true;
+        parents.top()->isExecutable = true;
     }
 
     void preallocateContents(unsigned long long size) override
     {
         currentStart = string(s, pos, 16);
         assert(size <= std::numeric_limits<size_t>::max());
-        members.emplace(currentPath,
-            NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size});
+        parents.top()->size = (size_t)size;
+        parents.top()->start = pos;
     }
 
     void receiveContents(unsigned char * data, unsigned int len) override
@@ -68,16 +107,23 @@ struct NarIndexer : ParseSink, StringSource
 
     void createSymlink(const Path & path, const string & target) override
     {
-        members.emplace(path,
+        createMember(path,
             NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
     }
 
-    Members::iterator find(const Path & path)
+    NarMember* find(const Path & path)
     {
-        auto i = members.find(path);
-        if (i == members.end())
+        Path canon = path == "" ? "" : canonPath(path);
+        NarMember* result = root.find(canon);
+        return result;
+    }
+
+    NarMember& at(const Path & path) {
+        auto result = find(path);
+        if(result == nullptr) {
             throw Error(format("NAR file does not contain path ‘%1%’") % path);
-        return i;
+        }
+        return *result;
     }
 };
 
@@ -93,44 +139,41 @@ struct NarAccessor : public FSAccessor
 
     Stat stat(const Path & path) override
     {
-        auto i = indexer.members.find(path);
-        if (i == indexer.members.end())
+        auto i = indexer.find(path);
+        if (i == nullptr)
             return {FSAccessor::Type::tMissing, 0, false};
-        return {i->second.type, i->second.size, i->second.isExecutable};
+        return {i->type, i->size, i->isExecutable};
     }
 
     StringSet readDirectory(const Path & path) override
     {
-        auto i = indexer.find(path);
+        auto i = indexer.at(path);
 
-        if (i->second.type != FSAccessor::Type::tDirectory)
+        if (i.type != FSAccessor::Type::tDirectory)
             throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
 
-        ++i;
         StringSet res;
-        while (i != indexer.members.end() && isInDir(i->first, path)) {
-            // FIXME: really bad performance.
-            if (i->first.find('/', path.size() + 1) == std::string::npos)
-                res.insert(std::string(i->first, path.size() + 1));
-            ++i;
+        for(auto&& child : i.children) {
+            res.insert(child.first);
+
         }
         return res;
     }
 
     std::string readFile(const Path & path) override
     {
-        auto i = indexer.find(path);
-        if (i->second.type != FSAccessor::Type::tRegular)
+        auto i = indexer.at(path);
+        if (i.type != FSAccessor::Type::tRegular)
             throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
-        return std::string(*nar, i->second.start, i->second.size);
+        return std::string(*nar, i.start, i.size);
     }
 
     std::string readLink(const Path & path) override
     {
-        auto i = indexer.find(path);
-        if (i->second.type != FSAccessor::Type::tSymlink)
+        auto i = indexer.at(path);
+        if (i.type != FSAccessor::Type::tSymlink)
             throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
-        return i->second.target;
+        return i.target;
     }
 };
 

From 06880d7ed8b8ef8658eb965e6614136c67988970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= <benno.fuenfstueck@gmail.com>
Date: Mon, 15 May 2017 10:18:27 +0200
Subject: [PATCH 0340/2196] nix ls: support '/' for the root directory

---
 src/nix/ls.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 3476dfb0528..417b7b421b1 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -61,6 +61,10 @@ struct MixLs : virtual Args
                 showFile(curPath, relPath);
         };
 
+        if (path == "/") {
+            path = "";
+        }
+
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
             throw Error(format("path ‘%1%’ does not exist") % path);

From 4412f7c08367b17b3be723ee42df159100d93922 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= <benno.fuenfstueck@gmail.com>
Date: Mon, 15 May 2017 12:23:21 +0200
Subject: [PATCH 0341/2196] nar-archive.cc: add tests for the nar index

---
 tests/local.mk      |  3 ++-
 tests/nar-index.nix | 23 +++++++++++++++++++++++
 tests/nar-index.sh  | 23 +++++++++++++++++++++++
 3 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 tests/nar-index.nix
 create mode 100644 tests/nar-index.sh

diff --git a/tests/local.mk b/tests/local.mk
index 108e3febdb0..7d99a0fc767 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -13,7 +13,8 @@ nix_tests = \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
-  build-remote.sh
+  build-remote.sh \
+  nar-index.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/nar-index.nix b/tests/nar-index.nix
new file mode 100644
index 00000000000..0e2a7f72113
--- /dev/null
+++ b/tests/nar-index.nix
@@ -0,0 +1,23 @@
+with import ./config.nix;
+
+rec {
+    a = mkDerivation {
+        name = "nar-index-a";
+        builder = builtins.toFile "builder.sh"
+      ''
+        mkdir $out
+        mkdir $out/foo
+        touch $out/foo-x
+        touch $out/foo/bar
+        touch $out/foo/baz
+        touch $out/qux
+        mkdir $out/zyx
+
+        cat >$out/foo/data <<EOF
+        lasjdöaxnasd
+asdom 12398
+ä"§Æẞ¢«»”alsd
+EOF
+      '';
+    };
+}
\ No newline at end of file
diff --git a/tests/nar-index.sh b/tests/nar-index.sh
new file mode 100644
index 00000000000..51369346c88
--- /dev/null
+++ b/tests/nar-index.sh
@@ -0,0 +1,23 @@
+source common.sh
+
+echo "building test path"
+storePath="$(nix-build nar-index.nix -A a --no-out-link)"
+
+cd "$TEST_ROOT"
+
+echo "dumping path to nar"
+narFile="$TEST_ROOT/path.nar"
+nix-store --dump $storePath > $narFile
+
+echo "check that find and ls-nar match"
+( cd $storePath; find . | sort ) > files.find
+nix ls-nar -R -d $narFile "" | sort > files.ls-nar
+diff -u files.find files.ls-nar
+
+echo "check that file contents of data match"
+nix cat-nar $narFile /foo/data > data.cat-nar
+diff -u data.cat-nar $storePath/foo/data
+
+echo "check that file contents of baz match"
+nix cat-nar $narFile /foo/baz > baz.cat-nar
+diff -u baz.cat-nar $storePath/foo/baz
\ No newline at end of file

From 2b761d5f50b7dc17dc55c31980c2253c37b3c920 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 May 2017 17:06:07 +0200
Subject: [PATCH 0342/2196] Make fmt() non-recursive

---
 src/libutil/logging.hh |  2 +-
 src/libutil/types.hh   | 17 ++++++-----------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 81aebccdca4..a8c69dbd956 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -88,7 +88,7 @@ template<typename... Args>
 inline void warn(const std::string & fs, Args... args)
 {
     boost::format f(fs);
-    formatHelper(f, args...);
+    nop{boost::io::detail::feed(f, args)...};
     logger->warn(f.str());
 }
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 1429c238513..9f32d31addb 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -32,6 +32,11 @@ using std::vector;
 using boost::format;
 
 
+/* A variadic template that does nothing. Useful to call a function
+   for all variadic arguments but ignoring the result. */
+struct nop { template<typename... T> nop(T...) {} };
+
+
 struct FormatOrString
 {
     string s;
@@ -46,16 +51,6 @@ struct FormatOrString
    ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
    takes place). */
 
-inline void formatHelper(boost::format & f)
-{
-}
-
-template<typename T, typename... Args>
-inline void formatHelper(boost::format & f, T x, Args... args)
-{
-    formatHelper(f % x, args...);
-}
-
 inline std::string fmt(const std::string & s)
 {
     return s;
@@ -75,7 +70,7 @@ template<typename... Args>
 inline std::string fmt(const std::string & fs, Args... args)
 {
     boost::format f(fs);
-    formatHelper(f, args...);
+    nop{boost::io::detail::feed(f, args)...};
     return f.str();
 }
 

From b30f5784d0184688de964f6239e373b62101ebc4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 May 2017 17:26:20 +0200
Subject: [PATCH 0343/2196] Linux sandbox: Don't barf on invalid paths

This is useful when we're using a diverted store (e.g. "--store
local?root=/tmp/nix") in conjunction with a statically-linked sh from
the host store (e.g. "sandbox-paths =/bin/sh=/nix/store/.../bin/busybox").
---
 src/libstore/build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 270500d81c9..5ec4cbf6654 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1774,6 +1774,7 @@ void DerivationGoal::startBuilder()
             try {
                 if (worker.store.isInStore(i.second.source))
                     worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure);
+            } catch (InvalidPath & e) {
             } catch (Error & e) {
                 throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what());
             }

From a2d92bb20e82a0957067ede60e91fab256948b41 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 May 2017 17:30:33 +0200
Subject: [PATCH 0344/2196] Add --with-sandbox-shell configure flag

And add a 116 KiB ash shell from busybox to the release build. This
helps to make sandbox builds work out of the box on non-NixOS systems
and with diverted stores.
---
 Makefile.config.in      |  1 +
 configure.ac            |  6 ++++++
 release-common.nix      | 21 +++++++++++++++++++++
 release.nix             |  9 ++++-----
 shell.nix               |  7 +++----
 src/libstore/globals.cc |  4 ++--
 src/libstore/local.mk   |  2 +-
 7 files changed, 38 insertions(+), 12 deletions(-)
 create mode 100644 release-common.nix

diff --git a/Makefile.config.in b/Makefile.config.in
index 3cae30d487d..45a70cd6dd1 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -28,6 +28,7 @@ localstatedir = @localstatedir@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
+sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 doc_generate = @doc_generate@
diff --git a/configure.ac b/configure.ac
index c7026cf954d..24a95ce56f3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -240,6 +240,12 @@ fi
 AC_SUBST(tarFlags)
 
 
+AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
+  [path of a statically-linked shell to use as /bin/sh in sandboxes]),
+  sandbox_shell=$withval)
+AC_SUBST(sandbox_shell)
+
+
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
diff --git a/release-common.nix b/release-common.nix
new file mode 100644
index 00000000000..8047c75bdb7
--- /dev/null
+++ b/release-common.nix
@@ -0,0 +1,21 @@
+{ pkgs }:
+
+rec {
+  sh = pkgs.busybox.override {
+    useMusl = true;
+    enableStatic = true;
+    enableMinimal = true;
+    extraConfig = ''
+      CONFIG_ASH y
+      CONFIG_ASH_BUILTIN_ECHO y
+      CONFIG_ASH_BUILTIN_TEST y
+      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+    '';
+  };
+
+  configureFlags =
+    [ "--disable-init-state"
+      "--enable-gc"
+      "--with-sandbox-shell=${sh}/bin/busybox"
+    ];
+}
diff --git a/release.nix b/release.nix
index 54d20c868da..f1a553d01cc 100644
--- a/release.nix
+++ b/release.nix
@@ -66,6 +66,8 @@ let
 
       with import <nixpkgs> { inherit system; };
 
+      with import ./release-common.nix { inherit pkgs; };
+
       releaseTools.nixBuild {
         name = "nix";
         src = tarball;
@@ -83,11 +85,8 @@ let
               customMemoryManagement = false;
             });
 
-        configureFlags = ''
-          --disable-init-state
-          --enable-gc
-          --sysconfdir=/etc
-        '';
+        configureFlags = configureFlags ++
+          [ "--sysconfdir=/etc" ];
 
         enableParallelBuilding = true;
 
diff --git a/shell.nix b/shell.nix
index 8645d36020e..c4e2a20f8fa 100644
--- a/shell.nix
+++ b/shell.nix
@@ -2,6 +2,8 @@
 
 with import <nixpkgs> {};
 
+with import ./release-common.nix { inherit pkgs; };
+
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
@@ -22,10 +24,7 @@ with import <nixpkgs> {};
       perlPackages.DBDSQLite
     ];
 
-  configureFlags =
-    [ "--disable-init-state"
-      "--enable-gc"
-    ];
+  inherit configureFlags;
 
   enableParallelBuilding = true;
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4bdbde989ab..3dd2508a26d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -47,8 +47,8 @@ Settings::Settings()
     auto s = getEnv("NIX_REMOTE_SYSTEMS");
     if (s != "") builderFiles = tokenizeString<Strings>(s, ":");
 
-#if __linux__
-    sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH);
+#if defined(__linux__) && defined(SANDBOX_SHELL)
+    sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
 #endif
 
     allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 4da20330cf3..e06002587f9 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -27,7 +27,7 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
- -DBASH_PATH="\"$(bash)\"" \
+ -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 
 $(d)/local-store.cc: $(d)/schema.sql.hh

From c05d9ae7a5f72a6575130cf92fb54e3f90d1927d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 May 2017 18:44:58 +0200
Subject: [PATCH 0345/2196] Disallow outputHash being null or an empty string

Fixes #1384.
---
 src/libexpr/primops.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 615cc813843..cc23827a17a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -534,7 +534,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    string outputHash, outputHashAlgo;
+    std::experimental::optional<std::string> outputHash;
+    std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
     StringSet outputs;
@@ -703,7 +704,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         throw EvalError(format("derivation names are not allowed to end in ‘%1%’, at %2%")
             % drvExtension % posDrvName);
 
-    if (outputHash != "") {
+    if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
@@ -711,13 +712,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = parseHashType(outputHashAlgo);
         if (ht == htUnknown)
             throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName);
-        Hash h = parseHash16or32(ht, outputHash);
+        Hash h = parseHash16or32(ht, *outputHash);
         outputHash = printHash(h);
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash);
+        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
     }
 
     else {

From e80257f12209c8fbb709b901039ef5199111276e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 May 2017 18:50:54 +0200
Subject: [PATCH 0346/2196] Simplify fixed-output check

---
 src/libstore/build.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5ec4cbf6654..91156d313e2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1066,12 +1066,8 @@ void DerivationGoal::haveDerivation()
 
     /* Reject doing a hash build of anything other than a fixed-output
        derivation. */
-    if (buildMode == bmHash) {
-        if (drv->outputs.size() != 1 ||
-            drv->outputs.find("out") == drv->outputs.end() ||
-            drv->outputs["out"].hashAlgo == "")
-            throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath);
-    }
+    if (buildMode == bmHash && !drv->isFixedOutput())
+        throw Error("cannot do a hash build of non-fixed-output derivation ‘%1%’", drvPath);
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build

From 5ee06e612a93a30bfa3b2129a3951e0c36f95602 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= <benno.fuenfstueck@gmail.com>
Date: Mon, 15 May 2017 19:32:51 +0200
Subject: [PATCH 0347/2196] nar-accessor: non-recursive NarMember::find

This avoids a possible stack overflow if directories are very deeply nested.
---
 src/libstore/nar-accessor.cc | 42 ++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index ee1cf385c08..c84bb1dea32 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -21,25 +21,6 @@ struct NarMember
 
     /* If this is a directory, all the children of the directory. */
     std::map<std::string, NarMember> children;
-
-    NarMember* find(const Path & path)
-    {
-        if(path == "") return this;
-
-        if(type != FSAccessor::Type::tDirectory) {
-            return nullptr;
-        }
-
-        auto split = std::find(path.begin() + 1, path.end(), '/');
-        std::string child_name(path.begin() + 1, split);
-        std::string remaining(split, path.end());
-
-        auto child = children.find(child_name);
-        if(child == children.end()) return nullptr;
-
-        return child->second.find(remaining);
-    }
-
 };
 
 struct NarIndexer : ParseSink, StringSource
@@ -114,8 +95,27 @@ struct NarIndexer : ParseSink, StringSource
     NarMember* find(const Path & path)
     {
         Path canon = path == "" ? "" : canonPath(path);
-        NarMember* result = root.find(canon);
-        return result;
+        NarMember* current = &root;
+        auto end = path.end();
+        for(auto it = path.begin(); it != end; ) {
+            // because it != end, the remaining component is non-empty so we need
+            // a directory
+            if(current->type != FSAccessor::Type::tDirectory) return nullptr;
+
+            // skip slash (canonPath above ensures that this is always a slash)
+            assert(*it == '/');
+            it += 1;
+
+            // lookup current component
+            auto next = std::find(it, end, '/');
+            auto child = current->children.find(std::string(it, next));
+            if(child == current->children.end()) return nullptr;
+            current = &child->second;
+
+            it = next;
+        }
+
+        return current;
     }
 
     NarMember& at(const Path & path) {

From a1f428b13bd003caaf3a1d1da6e934d52b6ea6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= <benno.fuenfstueck@gmail.com>
Date: Mon, 15 May 2017 19:35:36 +0200
Subject: [PATCH 0348/2196] nar-accessor.cc: remove unused member
 NarIndexer::currentName

---
 src/libstore/nar-accessor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index c84bb1dea32..82595e76a9b 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -28,7 +28,6 @@ struct NarIndexer : ParseSink, StringSource
     NarMember root;
     std::stack<NarMember*> parents;
 
-    std::string currentName;
     std::string currentStart;
     bool isExec = false;
 
@@ -56,7 +55,7 @@ struct NarIndexer : ParseSink, StringSource
 
     void createDirectory(const Path & path) override
     {
-        createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 }); 
+        createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 });
     }
 
     void createRegularFile(const Path & path) override

From b01d62285cdcd376a8db1863049c68d8c7238837 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 May 2017 16:09:57 +0200
Subject: [PATCH 0349/2196] Improve progress indicator

---
 src/libexpr/eval.cc                     |   2 +-
 src/libexpr/get-drvs.cc                 |   3 +-
 src/libexpr/primops.cc                  |   8 +-
 src/libexpr/primops/fetchgit.cc         |   2 +-
 src/libstore/build.cc                   |  31 ++-
 src/libstore/builtins.cc                |   3 -
 src/libstore/download.cc                |  38 +---
 src/libstore/download.hh                |   1 -
 src/libstore/export-import.cc           |   8 +-
 src/libstore/gc.cc                      |   2 +-
 src/libstore/http-binary-cache-store.cc |   2 -
 src/libstore/optimise-store.cc          |   2 +-
 src/libstore/store-api.cc               |   9 +-
 src/libutil/logging.cc                  |  13 +-
 src/libutil/logging.hh                  |  91 ++++++---
 src/libutil/util.cc                     |   2 +-
 src/libutil/util.hh                     |   2 +
 src/nix-channel/nix-channel.cc          |   1 -
 src/nix-daemon/nix-daemon.cc            |   7 +-
 src/nix-env/nix-env.cc                  |   2 +-
 src/nix-instantiate/nix-instantiate.cc  |   2 +-
 src/nix/installables.cc                 |   6 +-
 src/nix/main.cc                         |   1 +
 src/nix/progress-bar.cc                 | 251 +++++++++++++++++++-----
 src/nix/sigs.cc                         |   6 +-
 src/nix/verify.cc                       |  12 +-
 26 files changed, 339 insertions(+), 168 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5e1ae63c482..0cdce602d7b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -642,7 +642,7 @@ void EvalState::evalFile(const Path & path, Value & v)
         return;
     }
 
-    Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2);
+    printTalkative("evaluating file ‘%1%’", path2);
     Expr * e = parseExprFromFile(checkSourcePath(path2));
     try {
         eval(e, v);
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index ae9fb0e5ec3..4200e8fd675 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -289,7 +289,7 @@ static void getDerivations(EvalState & state, Value & vIn,
            bound to the attribute with the "lower" name should take
            precedence). */
         for (auto & i : v.attrs->lexicographicOrder()) {
-            Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name);
+            debug("evaluating attribute ‘%1%’", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
             string pathPrefix2 = addToPath(pathPrefix, i->name);
@@ -310,7 +310,6 @@ static void getDerivations(EvalState & state, Value & vIn,
 
     else if (v.isList()) {
         for (unsigned int n = 0; n < v.listSize(); ++n) {
-            Activity act(*logger, lvlDebug, "evaluating list element");
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
             if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
                 getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cc23827a17a..93b66269dd0 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -127,7 +127,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
                 env->values[displ++] = attr.value;
             }
 
-            Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path);
+            printTalkative("evaluating file ‘%1%’", path);
             Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
 
             e->eval(state, *env, v);
@@ -326,8 +326,6 @@ typedef list<Value *> ValueList;
 
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    Activity act(*logger, lvlDebug, "finding dependencies");
-
     state.forceAttrs(*args[0], pos);
 
     /* Get the start set. */
@@ -499,8 +497,6 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value &
    derivation. */
 static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    Activity act(*logger, lvlVomit, "evaluating derivation");
-
     state.forceAttrs(*args[0], pos);
 
     /* Figure out the name first (for stack backtraces). */
@@ -544,7 +540,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     for (auto & i : args[0]->attrs->lexicographicOrder()) {
         if (i->name == state.sIgnoreNulls) continue;
         string key = i->name;
-        Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key);
+        vomit("processing attribute ‘%1%’", key);
 
         auto handleHashMode = [&](const std::string & s) {
             if (s == "recursive") outputHashRecursive = true;
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 09e2c077bab..3e4ece2cffd 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -17,7 +17,7 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri);
+    //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri);
 
     std::string localRef = "pid-" + std::to_string(getpid());
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 91156d313e2..44cae3431fc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -120,6 +120,8 @@ class Goal : public std::enable_shared_from_this<Goal>
     /* Whether the goal is finished. */
     ExitCode exitCode;
 
+    Activity act;
+
     Goal(Worker & worker) : worker(worker)
     {
         nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
@@ -168,7 +170,8 @@ class Goal : public std::enable_shared_from_this<Goal>
     virtual string key() = 0;
 
 protected:
-    void amDone(ExitCode result);
+
+    virtual void amDone(ExitCode result);
 };
 
 
@@ -902,6 +905,12 @@ class DerivationGoal : public Goal
 
     void repairClosure();
 
+    void amDone(ExitCode result)
+    {
+        logger->event(evBuildFinished, act, result == ecSuccess);
+        Goal::amDone(result);
+    }
+
     void done(BuildResult::Status status, const string & msg = "");
 };
 
@@ -920,6 +929,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     state = &DerivationGoal::getDerivation;
     name = (format("building of ‘%1%’") % drvPath).str();
     trace("created");
+
+    logger->event(evBuildCreated, act, drvPath);
 }
 
 
@@ -935,6 +946,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv
     name = (format("building of %1%") % showPaths(drv.outputPaths())).str();
     trace("created");
 
+    logger->event(evBuildCreated, act, drvPath);
+
     /* Prevent the .chroot directory from being
        garbage-collected. (See isActiveTempFile() in gc.cc.) */
     worker.store.addTempRoot(drvPath);
@@ -2112,6 +2125,8 @@ void DerivationGoal::startBuilder()
         }
         debug(msg);
     }
+
+    logger->event(evBuildStarted, act);
 }
 
 
@@ -2857,7 +2872,7 @@ void DerivationGoal::registerOutputs()
            contained in it.  Compute the SHA-256 NAR hash at the same
            time.  The hash is stored in the database so that we can
            verify later on whether nobody has messed with the store. */
-        Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path);
+        debug("scanning for references inside ‘%1%’", path);
         HashResult hash;
         PathSet references = scanForReferences(actualPath, allPaths, hash);
 
@@ -3130,6 +3145,7 @@ void DerivationGoal::flushLine()
         logTail.push_back(currentLogLine);
         if (logTail.size() > settings.logLines) logTail.pop_front();
     }
+    logger->event(evBuildOutput, act, currentLogLine);
     currentLogLine = "";
     currentLogLinePos = 0;
 }
@@ -3244,6 +3260,12 @@ class SubstitutionGoal : public Goal
     void handleEOF(int fd);
 
     Path getStorePath() { return storePath; }
+
+    void amDone(ExitCode result)
+    {
+        logger->event(evSubstitutionFinished, act, result == ecSuccess);
+        Goal::amDone(result);
+    }
 };
 
 
@@ -3256,6 +3278,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool
     state = &SubstitutionGoal::init;
     name = (format("substitution of ‘%1%’") % storePath).str();
     trace("created");
+    logger->event(evSubstitutionCreated, act, storePath);
 }
 
 
@@ -3391,6 +3414,8 @@ void SubstitutionGoal::tryToRun()
 
     printInfo(format("fetching path ‘%1%’...") % storePath);
 
+    logger->event(evSubstitutionStarted, act);
+
     outPipe.create();
 
     promise = std::promise<void>();
@@ -3637,7 +3662,7 @@ void Worker::run(const Goals & _topGoals)
 {
     for (auto & i : _topGoals) topGoals.insert(i);
 
-    Activity act(*logger, lvlDebug, "entered goal loop");
+    debug("entered goal loop");
 
     while (1) {
 
diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index c5dbd57f8bc..8a5cf3327d4 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -28,9 +28,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         DownloadRequest request(url);
         request.verifyTLS = false;
 
-        /* Show a progress indicator, even though stderr is not a tty. */
-        request.showProgress = DownloadRequest::yes;
-
         /* Note: have to use a fresh downloader here because we're in
            a forked process. */
         auto data = makeDownloader()->download(request);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index dc1a3d94f74..63e498f0603 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -63,6 +63,7 @@ struct CurlDownloader : public Downloader
         CurlDownloader & downloader;
         DownloadRequest request;
         DownloadResult result;
+        Activity act;
         bool done = false; // whether either the success or failure function has been called
         std::function<void(const DownloadResult &)> success;
         std::function<void(std::exception_ptr exc)> failure;
@@ -70,10 +71,6 @@ struct CurlDownloader : public Downloader
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
 
-        bool showProgress = false;
-        double prevProgressTime{0}, startTime{0};
-        unsigned int moveBack{1};
-
         unsigned int attempt = 0;
 
         /* Don't start this download until the specified time point
@@ -87,12 +84,10 @@ struct CurlDownloader : public Downloader
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader), request(request)
         {
-            showProgress =
-                request.showProgress == DownloadRequest::yes ||
-                (request.showProgress == DownloadRequest::automatic && isatty(STDERR_FILENO));
-
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+
+            logger->event(evDownloadCreated, act, request.uri);
         }
 
         ~DownloadItem()
@@ -109,6 +104,7 @@ struct CurlDownloader : public Downloader
             } catch (...) {
                 ignoreException();
             }
+            logger->event(evDownloadDestroyed, act);
         }
 
         template<class T>
@@ -171,19 +167,7 @@ struct CurlDownloader : public Downloader
 
         int progressCallback(double dltotal, double dlnow)
         {
-            if (showProgress) {
-                double now = getTime();
-                if (prevProgressTime <= now - 1) {
-                    string s = (format(" [%1$.0f/%2$.0f KiB, %3$.1f KiB/s]")
-                        % (dlnow / 1024.0)
-                        % (dltotal / 1024.0)
-                        % (now == startTime ? 0 : dlnow / 1024.0 / (now - startTime))).str();
-                    std::cerr << "\e[" << moveBack << "D" << s;
-                    moveBack = s.size();
-                    std::cerr.flush();
-                    prevProgressTime = now;
-                }
-            }
+            logger->event(evDownloadProgress, act, dltotal, dlnow);
             return _isInterrupted;
         }
 
@@ -201,13 +185,6 @@ struct CurlDownloader : public Downloader
 
         void init()
         {
-            // FIXME: handle parallel downloads.
-            if (showProgress) {
-                std::cerr << (format("downloading ‘%1%’... ") % request.uri);
-                std::cerr.flush();
-                startTime = getTime();
-            }
-
             if (!req) req = curl_easy_init();
 
             curl_easy_reset(req);
@@ -263,10 +240,6 @@ struct CurlDownloader : public Downloader
 
         void finish(CURLcode code)
         {
-            if (showProgress)
-                //std::cerr << "\e[" << moveBack << "D\e[K\n";
-                std::cerr << "\n";
-
             long httpStatus = 0;
             curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
 
@@ -292,6 +265,7 @@ struct CurlDownloader : public Downloader
                 try {
                     result.data = decodeContent(encoding, ref<std::string>(result.data));
                     callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+                    logger->event(evDownloadSucceeded, act, result.data->size());
                 } catch (...) {
                     done = true;
                     callFailure(failure, std::current_exception());
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 62f3860b9da..7d8982d64c4 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -13,7 +13,6 @@ struct DownloadRequest
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
-    enum { yes, no, automatic } showProgress = yes;
     bool head = false;
     size_t tries = 5;
     unsigned int baseRetryTimeMs = 250;
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 2b8ab063e18..6e8bc692cdf 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -30,13 +30,13 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
     std::reverse(sorted.begin(), sorted.end());
 
     std::string doneLabel("paths exported");
-    logger->incExpected(doneLabel, sorted.size());
+    //logger->incExpected(doneLabel, sorted.size());
 
     for (auto & path : sorted) {
-        Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path);
+        //Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path);
         sink << 1;
         exportPath(path, sink);
-        logger->incProgress(doneLabel);
+        //logger->incProgress(doneLabel);
     }
 
     sink << 0;
@@ -81,7 +81,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 
         info.path = readStorePath(*this, source);
 
-        Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path);
+        //Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path);
 
         info.references = readStorePaths<PathSet>(*this, source);
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 3e7e42cbc96..3cdbb114a79 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -615,7 +615,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
     auto realPath = realStoreDir + "/" + baseNameOf(path);
     if (realPath == linksDir || realPath == trashDir) return;
 
-    Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
+    //Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
 
     if (!isStorePath(path) || !isValidPath(path)) {
         /* A lock file belonging to a path that we're building right
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 37a7d6ace14..cead81514ab 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -50,7 +50,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     {
         try {
             DownloadRequest request(cacheUri + "/" + path);
-            request.showProgress = DownloadRequest::no;
             request.head = true;
             request.tries = 5;
             getDownloader()->download(request);
@@ -76,7 +75,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         std::function<void(std::exception_ptr exc)> failure) override
     {
         DownloadRequest request(cacheUri + "/" + path);
-        request.showProgress = DownloadRequest::no;
         request.tries = 8;
 
         getDownloader()->enqueueDownload(request,
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index d354812e3da..56167c4dfae 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -240,7 +240,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
     for (auto & i : paths) {
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
-        Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
+        //Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
         optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
     }
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 96799adb1e6..e6cbd53dc80 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -822,7 +822,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
 
     std::string copiedLabel = "copied";
 
-    logger->setExpected(copiedLabel, missing.size());
+    //logger->setExpected(copiedLabel, missing.size());
 
     ThreadPool pool;
 
@@ -838,13 +838,14 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
             checkInterrupt();
 
             if (!to->isValidPath(storePath)) {
-                Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
+                //Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
 
                 copyStorePath(from, to, storePath, false, dontCheckSigs);
 
-                logger->incProgress(copiedLabel);
+                //logger->incProgress(copiedLabel);
             } else
-                logger->incExpected(copiedLabel, -1);
+                ;
+                //logger->incExpected(copiedLabel, -1);
         });
 
     pool.process();
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index afcc2ec5854..2d0acca2421 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -1,6 +1,8 @@
 #include "logging.hh"
 #include "util.hh"
 
+#include <atomic>
+
 namespace nix {
 
 Logger * logger = makeDefaultLogger();
@@ -42,12 +44,7 @@ class SimpleLogger : public Logger
         writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
     }
 
-    void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override
-    {
-        log(lvl, fs);
-    }
-
-    void stopActivity(Activity & activity) override
+    void event(const Event & ev) override
     {
     }
 };
@@ -79,4 +76,8 @@ Logger * makeDefaultLogger()
     return new SimpleLogger();
 }
 
+std::atomic<uint64_t> Activity::nextId{(uint64_t) getpid() << 32};
+
+Activity::Activity() : id(nextId++) { };
+
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index a8c69dbd956..ddfc336fee0 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -13,7 +13,64 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
-class Activity;
+class Activity
+{
+    static std::atomic<uint64_t> nextId;
+public:
+    typedef uint64_t t;
+    const t id;
+    Activity();
+    Activity(const Activity & act) : id(act.id) { };
+    Activity(uint64_t id) : id(id) { };
+};
+
+typedef enum {
+    evBuildCreated = 0,
+    evBuildStarted = 1,
+    evBuildOutput = 2,
+    evBuildFinished = 3,
+    evDownloadCreated = 4,
+    evDownloadDestroyed = 5,
+    evDownloadProgress = 6,
+    evDownloadSucceeded = 7,
+    evSubstitutionCreated = 8,
+    evSubstitutionStarted = 9,
+    evSubstitutionFinished = 10,
+} EventType;
+
+struct Event
+{
+    struct Field
+    {
+        // FIXME: use std::variant.
+        enum { tInt, tString } type;
+        uint64_t i = 0;
+        std::string s;
+        Field(const std::string & s) : type(tString), s(s) { }
+        Field(const char * s) : type(tString), s(s) { }
+        Field(const uint64_t & i) : type(tInt), i(i) { }
+        Field(const Activity & act) : type(tInt), i(act.id) { }
+    };
+
+    typedef std::vector<Field> Fields;
+
+    EventType type;
+    Fields fields;
+
+    std::string getS(size_t n) const
+    {
+        assert(n < fields.size());
+        assert(fields[n].type == Field::tString);
+        return fields[n].s;
+    }
+
+    uint64_t getI(size_t n) const
+    {
+        assert(n < fields.size());
+        assert(fields[n].type == Field::tInt);
+        return fields[n].i;
+    }
+};
 
 class Logger
 {
@@ -32,34 +89,16 @@ public:
 
     virtual void warn(const std::string & msg);
 
-    virtual void setExpected(const std::string & label, uint64_t value = 1) { }
-    virtual void setProgress(const std::string & label, uint64_t value = 1) { }
-    virtual void incExpected(const std::string & label, uint64_t value = 1) { }
-    virtual void incProgress(const std::string & label, uint64_t value = 1) { }
-
-private:
-
-    virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0;
-
-    virtual void stopActivity(Activity & activity) = 0;
-
-};
-
-class Activity
-{
-public:
-    Logger & logger;
-
-    Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs)
-        : logger(logger)
+    template<typename... Args>
+    void event(EventType type, const Args & ... args)
     {
-        logger.startActivity(*this, lvl, fs);
+        Event ev;
+        ev.type = type;
+        nop{(ev.fields.emplace_back(Event::Field(args)), 1)...};
+        event(ev);
     }
 
-    ~Activity()
-    {
-        logger.stopActivity(*this);
-    }
+    virtual void event(const Event & ev) = 0;
 };
 
 extern Logger * logger;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1d1f68fc845..16f4b232e6c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -372,7 +372,7 @@ void deletePath(const Path & path)
 
 void deletePath(const Path & path, unsigned long long & bytesFreed)
 {
-    Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path);
+    //Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path);
     bytesFreed = 0;
     _deletePath(path, bytesFreed);
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 5a9c9513fd5..7ea32e8d9f1 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -364,6 +364,8 @@ void ignoreException();
 #define ANSI_NORMAL "\e[0m"
 #define ANSI_BOLD "\e[1m"
 #define ANSI_RED "\e[31;1m"
+#define ANSI_GREEN "\e[32;1m"
+#define ANSI_BLUE "\e[34;1m"
 
 
 /* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 2aaae2f471b..f2742bc3bbd 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -112,7 +112,6 @@ static void update(const StringSet & channelNames)
             // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
             // Check if the channel advertises a binary cache.
             DownloadRequest request(url + "/binary-cache-url");
-            request.showProgress = DownloadRequest::no;
             try {
                 auto dlRes = dl->download(request);
                 extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";";
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index d2bb7b8c88b..44127635ded 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -82,12 +82,7 @@ class TunnelLogger : public Logger
             defaultLogger->log(lvl, fs);
     }
 
-    void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override
-    {
-        log(lvl, fs);
-    }
-
-    void stopActivity(Activity & activity) override
+    void event(const Event & ev) override
     {
     }
 };
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index da39bf36ab6..464bcee4a84 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -989,7 +989,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         try {
             if (i.hasFailed()) continue;
 
-            Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath);
+            //Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath);
 
             if (globals.prebuiltOnly &&
                 validPaths.find(i.queryOutPath()) == validPaths.end() &&
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index c1b0b0ea092..25f0b1bd692 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -19,7 +19,7 @@ using namespace nix;
 
 static Expr * parseStdin(EvalState & state)
 {
-    Activity act(*logger, lvlTalkative, format("parsing standard input"));
+    //Activity act(*logger, lvlTalkative, format("parsing standard input"));
     return state.parseExprFromString(drainFD(0), absPath("."));
 }
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 4756fc44bba..f23308b9bc3 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -223,9 +223,9 @@ PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
         buildables.insert(b.begin(), b.end());
     }
 
-    printMissing(store, buildables);
-
-    if (!dryRun)
+    if (dryRun)
+        printMissing(store, buildables);
+    else
         store->buildPaths(buildables);
 
     PathSet outPaths;
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 440ced97dfc..216f0bccef1 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -27,6 +27,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
 void mainWrapped(int argc, char * * argv)
 {
+    verbosity = lvlError;
     settings.verboseBuild = false;
 
     initNix();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 69811b28280..24e435f81e8 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -1,8 +1,12 @@
 #include "progress-bar.hh"
 #include "util.hh"
 #include "sync.hh"
+#include "store-api.hh"
 
 #include <map>
+#include <atomic>
+
+#include <sys/ioctl.h>
 
 namespace nix {
 
@@ -12,31 +16,47 @@ class ProgressBar : public Logger
 
     struct ActInfo
     {
-        Activity * activity;
-        Verbosity lvl;
-        std::string s;
+        std::string s, s2;
     };
 
-    struct Progress
+    struct DownloadInfo
     {
-        uint64_t expected = 0, progress = 0;
+        std::string uri;
+        uint64_t current = 0;
+        uint64_t expected = 0;
     };
 
     struct State
     {
+        std::map<Activity::t, Path> builds;
+        std::set<Activity::t> runningBuilds;
+        uint64_t succeededBuilds = 0;
+        uint64_t failedBuilds = 0;
+        std::map<Activity::t, Path> substitutions;
+        std::set<Activity::t> runningSubstitutions;
+        uint64_t succeededSubstitutions = 0;
+        uint64_t downloadedBytes = 0; // finished downloads
+        std::map<Activity::t, DownloadInfo> downloads;
         std::list<ActInfo> activities;
-        std::map<Activity *, std::list<ActInfo>::iterator> its;
-        std::map<std::string, Progress> progress;
+        std::map<Activity::t, std::list<ActInfo>::iterator> its;
     };
 
     Sync<State> state_;
 
+    int width = 0;
+
 public:
 
+    ProgressBar()
+    {
+        struct winsize ws;
+        if (ioctl(1, TIOCGWINSZ, &ws) == 0)
+            width = ws.ws_col;
+    }
+
     ~ProgressBar()
     {
         auto state(state_.lock());
-        assert(state->activities.empty());
         writeToStderr("\r\e[K");
     }
 
@@ -52,52 +72,36 @@ class ProgressBar : public Logger
         update(state);
     }
 
-    void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override
-    {
-        if (lvl > verbosity) return;
-        auto state(state_.lock());
-        state->activities.emplace_back(ActInfo{&activity, lvl, fs.s});
-        state->its.emplace(&activity, std::prev(state->activities.end()));
-        update(*state);
-    }
-
-    void stopActivity(Activity & activity) override
-    {
-        auto state(state_.lock());
-        auto i = state->its.find(&activity);
-        if (i == state->its.end()) return;
-        state->activities.erase(i->second);
-        state->its.erase(i);
-        update(*state);
-    }
-
-    void setExpected(const std::string & label, uint64_t value) override
+    void createActivity(State & state, Activity::t activity, const std::string & s)
     {
-        auto state(state_.lock());
-        state->progress[label].expected = value;
-    }
-
-    void setProgress(const std::string & label, uint64_t value) override
-    {
-        auto state(state_.lock());
-        state->progress[label].progress = value;
+        state.activities.emplace_back(ActInfo{s});
+        state.its.emplace(activity, std::prev(state.activities.end()));
     }
 
-    void incExpected(const std::string & label, uint64_t value) override
+    void deleteActivity(State & state, Activity::t activity)
     {
-        auto state(state_.lock());
-        state->progress[label].expected += value;
+        auto i = state.its.find(activity);
+        if (i != state.its.end()) {
+            state.activities.erase(i->second);
+            state.its.erase(i);
+        }
     }
 
-    void incProgress(const std::string & label, uint64_t value) override
+    void updateActivity(State & state, Activity::t activity, const std::string & s2)
     {
-        auto state(state_.lock());
-        state->progress[label].progress += value;
+        auto i = state.its.find(activity);
+        assert(i != state.its.end());
+        ActInfo info = *i->second;
+        state.activities.erase(i->second);
+        info.s2 = s2;
+        state.activities.emplace_back(info);
+        i->second = std::prev(state.activities.end());
     }
 
     void update()
     {
         auto state(state_.lock());
+        update(*state);
     }
 
     void update(State & state)
@@ -113,28 +117,169 @@ class ProgressBar : public Logger
 
         if (!state.activities.empty()) {
             if (!status.empty()) line += " ";
-            line += state.activities.rbegin()->s;
+            auto i = state.activities.rbegin();
+            line += i->s;
+            if (!i->s2.empty()) {
+                line += ": ";
+                line += i->s2;
+            }
         }
 
         line += "\e[K";
-        writeToStderr(line);
+        writeToStderr(std::string(line, 0, width - 1));
     }
 
     std::string getStatus(State & state)
     {
         std::string res;
-        for (auto & p : state.progress)
-            if (p.second.expected || p.second.progress) {
-                if (!res.empty()) res += ", ";
-                res += std::to_string(p.second.progress);
-                if (p.second.expected) {
-                    res += "/";
-                    res += std::to_string(p.second.expected);
-                }
-                res += " "; res += p.first;
+
+        if (state.failedBuilds) {
+            if (!res.empty()) res += ", ";
+            res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds);
+        }
+
+        if (!state.builds.empty() || state.succeededBuilds)
+        {
+            if (!res.empty()) res += ", ";
+            if (!state.runningBuilds.empty())
+                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size());
+            res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL,
+                state.succeededBuilds, state.succeededBuilds + state.builds.size());
+        }
+
+        if (!state.substitutions.empty() || state.succeededSubstitutions) {
+            if (!res.empty()) res += ", ";
+            if (!state.runningSubstitutions.empty())
+                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size());
+            res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL,
+                state.succeededSubstitutions,
+                state.succeededSubstitutions + state.substitutions.size());
+        }
+
+        if (!state.downloads.empty() || state.downloadedBytes) {
+            if (!res.empty()) res += ", ";
+            uint64_t expected = state.downloadedBytes, current = state.downloadedBytes;
+            for (auto & i : state.downloads) {
+                expected += i.second.expected;
+                current += i.second.current;
             }
+            res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0);
+        }
+
         return res;
     }
+
+    void event(const Event & ev) override
+    {
+        if (ev.type == evBuildCreated) {
+            auto state(state_.lock());
+            state->builds[ev.getI(0)] = ev.getS(1);
+            update(*state);
+        }
+
+        if (ev.type == evBuildStarted) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            state->runningBuilds.insert(act);
+            auto name = storePathToName(state->builds[act]);
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name));
+            update(*state);
+        }
+
+        if (ev.type == evBuildFinished) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            if (ev.getI(1)) {
+                if (state->runningBuilds.count(act))
+                    state->succeededBuilds++;
+            } else
+                state->failedBuilds++;
+            state->runningBuilds.erase(act);
+            state->builds.erase(act);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evBuildOutput) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            assert(state->runningBuilds.count(act));
+            updateActivity(*state, act, ev.getS(1));
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionCreated) {
+            auto state(state_.lock());
+            state->substitutions[ev.getI(0)] = ev.getS(1);
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionStarted) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            state->runningSubstitutions.insert(act);
+            auto name = storePathToName(state->substitutions[act]);
+            createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name));
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionFinished) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            if (ev.getI(1)) {
+                if (state->runningSubstitutions.count(act))
+                    state->succeededSubstitutions++;
+            }
+            state->runningSubstitutions.erase(act);
+            state->substitutions.erase(act);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadCreated) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            std::string uri = ev.getS(1);
+            state->downloads.emplace(act, DownloadInfo{uri});
+            if (state->runningSubstitutions.empty()) // FIXME: hack
+                createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri));
+            update(*state);
+        }
+
+        if (ev.type == evDownloadProgress) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            assert(i != state->downloads.end());
+            i->second.expected = ev.getI(1);
+            i->second.current = ev.getI(2);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadSucceeded) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            assert(i != state->downloads.end());
+            state->downloadedBytes += ev.getI(1);
+            state->downloads.erase(i);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadDestroyed) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            if (i != state->downloads.end()) {
+                state->downloads.erase(i);
+                deleteActivity(*state, act);
+                update(*state);
+            }
+        }
+    }
 };
 
 StartProgressBar::StartProgressBar()
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index d8d8c0f53df..3dd03771619 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -42,10 +42,10 @@ struct CmdCopySigs : StorePathsCommand
         std::string doneLabel = "done";
         std::atomic<size_t> added{0};
 
-        logger->setExpected(doneLabel, storePaths.size());
+        //logger->setExpected(doneLabel, storePaths.size());
 
         auto doPath = [&](const Path & storePath) {
-            Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
+            //Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
 
             checkInterrupt();
 
@@ -76,7 +76,7 @@ struct CmdCopySigs : StorePathsCommand
                 added += newSigs.size();
             }
 
-            logger->incProgress(doneLabel);
+            //logger->incProgress(doneLabel);
         };
 
         for (auto & storePath : storePaths)
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 2f8d02fa060..8facb4bef8a 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand
         std::string untrustedLabel("untrusted");
         std::string corruptedLabel("corrupted");
         std::string failedLabel("failed");
-        logger->setExpected(doneLabel, storePaths.size());
+        //logger->setExpected(doneLabel, storePaths.size());
 
         ThreadPool pool;
 
@@ -73,7 +73,7 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
+                //Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
 
                 auto info = store->queryPathInfo(storePath);
 
@@ -85,7 +85,7 @@ struct CmdVerify : StorePathsCommand
                     auto hash = sink.finish();
 
                     if (hash.first != info->narHash) {
-                        logger->incProgress(corruptedLabel);
+                        //logger->incProgress(corruptedLabel);
                         corrupted = 1;
                         printError(
                             format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
@@ -137,19 +137,19 @@ struct CmdVerify : StorePathsCommand
                     }
 
                     if (!good) {
-                        logger->incProgress(untrustedLabel);
+                        //logger->incProgress(untrustedLabel);
                         untrusted++;
                         printError(format("path ‘%s’ is untrusted") % info->path);
                     }
 
                 }
 
-                logger->incProgress(doneLabel);
+                //logger->incProgress(doneLabel);
                 done++;
 
             } catch (Error & e) {
                 printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
-                logger->incProgress(failedLabel);
+                //logger->incProgress(failedLabel);
                 failed++;
             }
         };

From e46090edb101acac20ab1e6260a0ba98c177206a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 May 2017 11:58:01 +0200
Subject: [PATCH 0350/2196] builtins.match: Improve error message for bad
 regular expression

Issue #1331.
---
 src/libexpr/primops.cc | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 93b66269dd0..99ffddaeb80 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1709,26 +1709,33 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
    ‘null’ or a list containing substring matches. */
 static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    std::regex regex(state.forceStringNoCtx(*args[0], pos), std::regex::extended);
+    auto re = state.forceStringNoCtx(*args[0], pos);
 
-    PathSet context;
-    const std::string str = state.forceString(*args[1], context, pos);
+    try {
 
+        std::regex regex(re, std::regex::extended);
 
-    std::smatch match;
-    if (!std::regex_match(str, match, regex)) {
-        mkNull(v);
-        return;
-    }
+        PathSet context;
+        const std::string str = state.forceString(*args[1], context, pos);
 
-    // the first match is the whole string
-    const size_t len = match.size() - 1;
-    state.mkList(v, len);
-    for (size_t i = 0; i < len; ++i) {
-        if (!match[i+1].matched)
-            mkNull(*(v.listElems()[i] = state.allocValue()));
-        else
-            mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str());
+        std::smatch match;
+        if (!std::regex_match(str, match, regex)) {
+            mkNull(v);
+            return;
+        }
+
+        // the first match is the whole string
+        const size_t len = match.size() - 1;
+        state.mkList(v, len);
+        for (size_t i = 0; i < len; ++i) {
+            if (!match[i+1].matched)
+                mkNull(*(v.listElems()[i] = state.allocValue()));
+            else
+                mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str());
+        }
+
+    } catch (std::regex_error &) {
+        throw EvalError("invalid regular expression ‘%s’, at %s", re, pos);
     }
 }
 

From f134fc4cbecc258771272c2418af3b92ade88f80 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 May 2017 12:18:18 +0200
Subject: [PATCH 0351/2196] Document that builtins.match takes a POSIX extended
 RE

---
 doc/manual/expressions/builtins.xml  | 14 ++++++++++----
 tests/lang/eval-okay-regex-match.nix |  3 +++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 63d13e184a1..8a32661066f 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -213,10 +213,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   <varlistentry><term><function>builtins.match</function>
   <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
 
-  <listitem><para>Returns a list if
-    <replaceable>regex</replaceable> matches
-    <replaceable>str</replaceable> precisely, otherwise returns <literal>null</literal>.
-    Each item in the list is a regex group.
+  <listitem><para>Returns a list if the <link
+  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+  POSIX regular expression</link> <replaceable>regex</replaceable>
+  matches <replaceable>str</replaceable> precisely, otherwise returns
+  <literal>null</literal>.  Each item in the list is a regex group.
 
 <programlisting>
 builtins.match "ab" "abc"
@@ -236,6 +237,11 @@ builtins.match "a(b)(c)" "abc"
 
 Evaluates to <literal>[ "b" "c" ]</literal>.
 
+<programlisting>
+builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+</programlisting>
+
+Evaluates to <literal>[ "foo" ]</literal>.
 
   </para></listitem>
   </varlistentry>
diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/lang/eval-okay-regex-match.nix
index ae6501532d1..273e2590713 100644
--- a/tests/lang/eval-okay-regex-match.nix
+++ b/tests/lang/eval-okay-regex-match.nix
@@ -17,8 +17,11 @@ assert  matches "fo+" "foo";
 assert  matches "fo{1,2}" "foo";
 assert !matches "fo{1,2}" "fooo";
 assert !matches "fo*" "foobar";
+assert  matches "[[:space:]]+([^[:space:]]+)[[:space:]]+" "  foo   ";
+assert !matches "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  foo   ";
 
 assert match "(.*)\\.nix" "foobar.nix" == [ "foobar" ];
+assert match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   " == [ "FOO" ];
 
 assert splitFN "/path/to/foobar.nix" == [ "/path/to/" "/path/to" "foobar" "nix" ];
 assert splitFN "foobar.cc" == [ null null "foobar" "cc" ];

From 9711524188494ff5bcbba8c6945add89fbb29713 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 24 May 2017 11:21:38 +0200
Subject: [PATCH 0352/2196] Fix #1380

It lacked a backslash. Use a raw string and single quotes around PS1
to simplify this.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b7cf36d1607..d5f23a37fe6 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -438,7 +438,7 @@ int main(int argc, char ** argv)
                         "[ -e $stdenv/setup ] && source $stdenv/setup; "
                         "%3%"
                         "set +e; "
-                        "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]\\$\\[\\033[0m\\] \"; "
+                        R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                         "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                         "unset NIX_ENFORCE_PURITY; "
                         "unset NIX_INDENT_MAKE; "

From a7e55151a8d45d987ca42ba318c44ed3ccdeecca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 24 May 2017 11:33:42 +0200
Subject: [PATCH 0353/2196] Fix #1314

Also, make nix-shell respect --option. (Previously it only passed it
along to nix-instantiate and nix-build.)
---
 src/nix-build/nix-build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 7167e96f11d..42d68fdfdd7 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -196,6 +196,7 @@ int main(int argc, char ** argv)
                 buildArgs.push_back(arg);
                 buildArgs.push_back(args[n + 1]);
                 buildArgs.push_back(args[n + 2]);
+                settings.set(args[n + 1], args[n + 2]);
                 n += 2;
             }
 
@@ -407,7 +408,7 @@ int main(int argc, char ** argv)
 
                 env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
                 env["NIX_STORE"] = store->storeDir;
-                env["NIX_BUILD_CORES"] = settings.buildCores;
+                env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 
                 auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", ""));
 

From 86ea7d15665b7bf2bc018c3bfd91d0a948c8f3be Mon Sep 17 00:00:00 2001
From: Pyry Jahkola <pyry.jahkola@iki.fi>
Date: Sun, 28 May 2017 15:48:57 +0300
Subject: [PATCH 0354/2196] Fix variable name typo in derivations doc

---
 doc/manual/expressions/derivations.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml
index 5efe2213e37..6f6297565ca 100644
--- a/doc/manual/expressions/derivations.xml
+++ b/doc/manual/expressions/derivations.xml
@@ -100,7 +100,7 @@ outputs = [ "lib" "headers" "doc" ];
 <programlisting>
 buildInputs = [ pkg.lib pkg.headers ];
 </programlisting>
-  The first element of <varname>output</varname> determines the
+  The first element of <varname>outputs</varname> determines the
   <emphasis>default output</emphasis>.  Thus, you could also write
 <programlisting>
 buildInputs = [ pkg pkg.headers ];

From 370428f86d03488756cfb27cb9126bf29767e848 Mon Sep 17 00:00:00 2001
From: Pyry Jahkola <pyry.jahkola@iki.fi>
Date: Sun, 28 May 2017 20:47:35 +0300
Subject: [PATCH 0355/2196] Remove stray `>` in builtins doc

---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index f46a93ae0d5..125cbd78255 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -440,7 +440,7 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
 
     <listitem><para>Generate list of size
     <replaceable>length</replaceable>, with each element
-    <replaceable>i></replaceable> equal to the value returned by
+    <replaceable>i</replaceable> equal to the value returned by
     <replaceable>generator</replaceable> <literal>i</literal>. For
     example,
 

From 63145be2a5ac46a283f85c835fa84bf54db59bbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 15:50:25 +0200
Subject: [PATCH 0356/2196] Fix typo

---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 207e8a40b6d..3ac23ec268f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -923,7 +923,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
             info.path, info.narHash.to_string(), h.to_string());
 
     if (nar->size() != info.narSize)
-        throw Error("szie mismatch importing path ‘%s’; expected %s, got %s",
+        throw Error("size mismatch importing path ‘%s’; expected %s, got %s",
             info.path, info.narSize, nar->size());
 
     if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys))

From 588dad4084711e71a4dcb0758b2daf2fe03e2c8b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 15:53:16 +0200
Subject: [PATCH 0357/2196] Fix build failure on Debian/Ubuntu

http://hydra.nixos.org/build/53537463
---
 src/libutil/logging.cc  | 2 +-
 src/libutil/logging.hh  | 1 -
 src/nix/progress-bar.cc | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 2d0acca2421..43245f61c60 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -76,7 +76,7 @@ Logger * makeDefaultLogger()
     return new SimpleLogger();
 }
 
-std::atomic<uint64_t> Activity::nextId{(uint64_t) getpid() << 32};
+std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
 Activity::Activity() : id(nextId++) { };
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index ddfc336fee0..9ef6e3ee30e 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -15,7 +15,6 @@ typedef enum {
 
 class Activity
 {
-    static std::atomic<uint64_t> nextId;
 public:
     typedef uint64_t t;
     const t id;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 24e435f81e8..2ecbea8eeea 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -24,6 +24,7 @@ class ProgressBar : public Logger
         std::string uri;
         uint64_t current = 0;
         uint64_t expected = 0;
+        DownloadInfo(const std::string & uri) : uri(uri) { }
     };
 
     struct State

From 6e01ecd112dce8d8bbe46c839f982892a3ffb589 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 16:08:56 +0200
Subject: [PATCH 0358/2196] Fix nix-copy-closure test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes

  client# error: size mismatch importing path ‘/nix/store/ywf5fihjlxwijm6ygh6s0a353b5yvq4d-libidn2-0.16’; expected 0, got 120264

This is mostly an artifact of the NixOS VM test environment, where the
Nix database doesn't contain hashes/sizes.

http://hydra.nixos.org/build/53537471
---
 src/libstore/store-api.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e6cbd53dc80..76ed9942256 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -547,6 +547,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     if (!info->narHash && dontCheckSigs) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
+        if (!info->narSize) info2->narSize = sink.s->size();
         info = info2;
     }
 

From 6cc6c15a2d50d0021d7242e9806ed6d54538de17 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 11:34:24 +0200
Subject: [PATCH 0359/2196] Add a seccomp filter to prevent creating
 setuid/setgid binaries

This prevents builders from setting the S_ISUID or S_ISGID bits,
preventing users from using a nixbld* user to create a setuid/setgid
binary to interfere with subsequent builds under the same nixbld* uid.

This is based on aszlig's seccomp code
(47f587700d646f5b03a42f2fa57c28875a31efbe).

Reported by Linus Heckemann.
---
 configure.ac          |  7 +++++++
 release.nix           |  2 ++
 shell.nix             |  3 ++-
 src/libstore/build.cc | 39 +++++++++++++++++++++++++++++++++++++++
 src/libstore/local.mk |  4 ++++
 5 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 24a95ce56f3..9d8a81d0427 100644
--- a/configure.ac
+++ b/configure.ac
@@ -176,6 +176,13 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 
 
+# Look for libseccomp, required for Linux sandboxing.
+if test "$sys_name" = linux; then
+  PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
+                    [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+fi
+
+
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
diff --git a/release.nix b/release.nix
index f1a553d01cc..1e854a075f6 100644
--- a/release.nix
+++ b/release.nix
@@ -30,6 +30,7 @@ let
             docbook5 docbook5_xsl
             autoconf-archive
             git
+            libseccomp
           ];
 
         configureFlags = "--enable-gc";
@@ -78,6 +79,7 @@ let
             openssl pkgconfig sqlite boehmgc
 
           ]
+          ++ lib.optional stdenv.isLinux libseccomp
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {
diff --git a/shell.nix b/shell.nix
index c4e2a20f8fa..435399f0b25 100644
--- a/shell.nix
+++ b/shell.nix
@@ -22,7 +22,8 @@ with import ./release-common.nix { inherit pkgs; };
       # For nix-perl
       perl
       perlPackages.DBDSQLite
-    ];
+    ]
+    ++ lib.optional stdenv.isLinux libseccomp;
 
   inherit configureFlags;
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 44cae3431fc..06d25960666 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -46,6 +46,7 @@
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/syscall.h>
+#include <seccomp.h>
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
@@ -2298,6 +2299,42 @@ void DerivationGoal::doExportReferencesGraph()
 }
 
 
+void setupSeccomp()
+{
+#if __linux__
+    scmp_filter_ctx ctx;
+
+    if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
+        throw SysError("unable to initialize seccomp mode 2");
+
+    Finally cleanup([&]() {
+        seccomp_release(ctx);
+    });
+
+    if (seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
+        throw SysError("unable to add 32-bit seccomp architecture");
+
+    for (int perm : { S_ISUID, S_ISGID }) {
+        // TODO: test chmod and fchmod.
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
+                SCMP_A1(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
+                SCMP_A1(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
+                SCMP_A2(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+            throw SysError("unable to add seccomp rule");
+    }
+
+    if (seccomp_load(ctx) != 0)
+        throw SysError("unable to load seccomp BPF program");
+#endif
+}
+
+
 void DerivationGoal::runChild()
 {
     /* Warning: in the child we should absolutely not make any SQLite
@@ -2307,6 +2344,8 @@ void DerivationGoal::runChild()
 
         commonChildInit(builderOut);
 
+        setupSeccomp();
+
         bool setUser = true;
 
         /* Make the contents of netrc available to builtin:fetchurl
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index e06002587f9..ffdb55abc65 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -18,6 +18,10 @@ ifeq ($(OS), SunOS)
 	libstore_LDFLAGS += -lsocket
 endif
 
+ifeq ($(OS), Linux)
+	libstore_LDFLAGS += -lseccomp
+endif
+
 libstore_CXXFLAGS = \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \

From cf93397d3f1d2a8165a100482d07b7f4b7e5bf7f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 14:18:36 +0200
Subject: [PATCH 0360/2196] Fix seccomp initialisation on i686-linux

---
 src/libstore/build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 06d25960666..b6293c15ebc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2311,7 +2311,8 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
-    if (seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
+    if (settings.thisSystem == "x86_64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
     for (int perm : { S_ISUID, S_ISGID }) {

From 1d9ab273bad34b004dfcfd486273d0df5fed1eca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 May 2017 14:19:11 +0200
Subject: [PATCH 0361/2196] Add test for setuid seccomp filter

---
 release.nix      |   5 +++
 tests/setuid.nix | 108 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 tests/setuid.nix

diff --git a/release.nix b/release.nix
index 1e854a075f6..ec6bda99577 100644
--- a/release.nix
+++ b/release.nix
@@ -219,6 +219,11 @@ let
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
+    tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
+      import ./tests/setuid.nix rec {
+        nix = build.${system}; inherit system;
+      });
+
     tests.binaryTarball =
       with import <nixpkgs> { system = "x86_64-linux"; };
       vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
diff --git a/tests/setuid.nix b/tests/setuid.nix
new file mode 100644
index 00000000000..2508549c546
--- /dev/null
+++ b/tests/setuid.nix
@@ -0,0 +1,108 @@
+# Verify that Linux builds cannot create setuid or setgid binaries.
+
+{ system, nix }:
+
+with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+
+makeTest {
+
+  machine =
+    { config, lib, pkgs, ... }:
+    { virtualisation.writableStore = true;
+      nix.package = nix;
+      nix.binaryCaches = [ ];
+      nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ];
+      virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ];
+    };
+
+  testScript = { nodes }:
+    ''
+      startAll;
+
+      # Copying to /tmp should succeed.
+      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      # Creating a setuid binary should fail.
+      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        chmod 4755 /tmp/id
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      # Creating a setgid binary should fail.
+      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        chmod 2755 /tmp/id
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      # The checks should also work on 32-bit binaries.
+      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        chmod 2755 /tmp/id
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      # The tests above use fchmodat(). Test chmod() as well.
+      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        perl -e \"chmod 0666, qw(/tmp/id) or die\"
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        perl -e \"chmod 04755, qw(/tmp/id) or die\"
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      # And test fchmod().
+      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\"
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]');
+
+      $machine->succeed("rm /tmp/id");
+
+      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+        mkdir -p $out
+        cp ${pkgs.coreutils}/bin/id /tmp/id
+        perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\"
+      ")\' ');
+
+      $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]');
+
+      $machine->succeed("rm /tmp/id");
+    '';
+
+}

From ff6becafa8efc2f7e6f2b9b889ba4adf20b8d524 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 12:37:04 +0200
Subject: [PATCH 0362/2196] Require seccomp only in multi-user setups

---
 src/libstore/build.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b6293c15ebc..09cc2709ab7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2345,7 +2345,11 @@ void DerivationGoal::runChild()
 
         commonChildInit(builderOut);
 
-        setupSeccomp();
+        try {
+            setupSeccomp();
+        } catch (...) {
+            if (buildUser) throw;
+        }
 
         bool setUser = true;
 

From d798349ede3d6eb6e92a2e4f95f6b2179407ceb9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 13:43:51 +0200
Subject: [PATCH 0363/2196] canonicalisePathMetaData(): Remove extended
 attributes / ACLs

EAs/ACLs are not part of the NAR canonicalisation. Worse, setting an
ACL allows a builder to create writable files in the Nix store. So get
rid of them.

Closes #185.
---
 src/libstore/local-store.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3ac23ec268f..5b03e86f3ea 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -27,6 +27,7 @@
 #include <sys/statvfs.h>
 #include <sys/mount.h>
 #include <sys/ioctl.h>
+#include <sys/xattr.h>
 #endif
 
 #include <sqlite3.h>
@@ -407,6 +408,27 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error(format("file ‘%1%’ has an unsupported type") % path);
 
+#if __linux__
+    /* Remove extended attributes / ACLs. */
+    ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
+
+    if (eaSize < 0) {
+        if (errno != ENOTSUP)
+            throw SysError("querying extended attributes of ‘%s’", path);
+    } else if (eaSize > 0) {
+        std::vector<char> eaBuf(eaSize);
+
+        if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
+            throw SysError("querying extended attributes of ‘%s’", path);
+
+        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1)))
+            if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
+                throw SysError("removing extended attribute ‘%s’ from ‘%s’", eaName, path);
+
+        assert(llistxattr(path.c_str(), nullptr, 0) == 0);
+    }
+#endif
+
     /* Fail if the file is not owned by the build user.  This prevents
        us from messing up the ownership/permissions of files
        hard-linked into the output (e.g. "ln /etc/shadow $out/foo").

From 2ac99a32dab0d2ea59cb9e926f6d6d5b7ef638c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 13:55:17 +0200
Subject: [PATCH 0364/2196] Add a seccomp rule to disallow setxattr()

---
 src/libstore/build.cc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 09cc2709ab7..0a10efaed1d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2315,8 +2315,8 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
+    /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
-        // TODO: test chmod and fchmod.
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
                 SCMP_A1(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
             throw SysError("unable to add seccomp rule");
@@ -2330,6 +2330,14 @@ void setupSeccomp()
             throw SysError("unable to add seccomp rule");
     }
 
+    /* Prevent builders from creating EAs or ACLs. Not all filesystems
+       support these, and they're not allowed in the Nix store because
+       they're not representable in the NAR serialisation. */
+    if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
+        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
+        seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
+        throw SysError("unable to add seccomp rule");
+
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
 #endif

From d552d387585840f68fcc7507fca83feb6f937a10 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 14:35:50 +0200
Subject: [PATCH 0365/2196] Shut up some clang warnings

---
 src/libstore/build.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0a10efaed1d..0a874bbf1cd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -906,7 +906,7 @@ class DerivationGoal : public Goal
 
     void repairClosure();
 
-    void amDone(ExitCode result)
+    void amDone(ExitCode result) override
     {
         logger->event(evBuildFinished, act, result == ecSuccess);
         Goal::amDone(result);
@@ -3288,16 +3288,16 @@ class SubstitutionGoal : public Goal
     SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
     ~SubstitutionGoal();
 
-    void timedOut() { abort(); };
+    void timedOut() override { abort(); };
 
-    string key()
+    string key() override
     {
         /* "a$" ensures substitution goals happen before derivation
            goals. */
         return "a$" + storePathToName(storePath) + "$" + storePath;
     }
 
-    void work();
+    void work() override;
 
     /* The states. */
     void init();
@@ -3308,12 +3308,12 @@ class SubstitutionGoal : public Goal
     void finished();
 
     /* Callback used by the worker to write to the log. */
-    void handleChildOutput(int fd, const string & data);
-    void handleEOF(int fd);
+    void handleChildOutput(int fd, const string & data) override;
+    void handleEOF(int fd) override;
 
     Path getStorePath() { return storePath; }
 
-    void amDone(ExitCode result)
+    void amDone(ExitCode result) override
     {
         logger->event(evSubstitutionFinished, act, result == ecSuccess);
         Goal::amDone(result);

From fe08d17934e6abe3e8566706f53063166b881f8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 14:37:24 +0200
Subject: [PATCH 0366/2196] Fix seccomp build failure on clang

Fixes

  src/libstore/build.cc:2321:45: error: non-constant-expression cannot be narrowed from type 'int' to 'scmp_datum_t' (aka 'unsigned long') in initializer list [-Wc++11-narrowing]
---
 src/libstore/build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0a874bbf1cd..86cab9f35b6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2318,15 +2318,15 @@ void setupSeccomp()
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
             throw SysError("unable to add seccomp rule");
 
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
-                SCMP_A1(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+                SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
             throw SysError("unable to add seccomp rule");
 
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
-                SCMP_A2(SCMP_CMP_MASKED_EQ, perm, perm)) != 0)
+                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
             throw SysError("unable to add seccomp rule");
     }
 

From 9bdb88ea6e833ad61d6833aa09560d32c9bc39df Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 15:56:15 +0200
Subject: [PATCH 0367/2196] Only pass --with-sandbox-shell on Linux

---
 release-common.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release-common.nix b/release-common.nix
index 8047c75bdb7..c64fc619df6 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -16,6 +16,7 @@ rec {
   configureFlags =
     [ "--disable-init-state"
       "--enable-gc"
+    ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 }

From 83eec5a997cd121158b7adb32a688dc5a63d6c9c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 15:56:40 +0200
Subject: [PATCH 0368/2196] resolve-system-dependencies: Several fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes

  error: getting attributes of path ‘Versions/Current/CoreFoundation’: No such file or directory

when /System/Library/Frameworks/CoreFoundation.framework/CoreFoundation is a symlink.

Also fixes a segfault when encounting a file that is not a MACH binary (such
as /dev/null, which is included in __impureHostDeps in Nixpkgs).

Possibly fixes #786.
---
 .../resolve-system-dependencies.cc            | 118 ++++++++++--------
 1 file changed, 65 insertions(+), 53 deletions(-)

diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index ae8ca36ba9d..61504298608 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -17,59 +17,75 @@ using namespace nix;
 
 static auto cacheDir = Path{};
 
-Path resolveCacheFile(Path lib) {
+Path resolveCacheFile(Path lib)
+{
     std::replace(lib.begin(), lib.end(), '/', '%');
     return cacheDir + "/" + lib;
 }
 
-std::set<string> readCacheFile(const Path & file) {
+std::set<string> readCacheFile(const Path & file)
+{
     return tokenizeString<set<string>>(readFile(file), "\n");
 }
 
-void writeCacheFile(const Path & file, std::set<string> & deps) {
+void writeCacheFile(const Path & file, std::set<string> & deps)
+{
     std::ofstream fp;
     fp.open(file);
-    for (auto & d : deps) {
+    for (auto & d : deps)
         fp << d << "\n";
-    }
     fp.close();
 }
 
-std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start) {
+std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start)
+{
     struct dylib_command *dylc = (struct dylib_command*)dylib_command_start;
     return std::string((char*)(dylib_command_start + DO_SWAP(should_swap, dylc->dylib.name.offset)));
 }
 
-std::set<std::string> runResolver(const Path & filename) {
-    int fd = open(filename.c_str(), O_RDONLY);
-    struct stat s;
-    fstat(fd, &s);
-    void *obj = mmap(NULL, s.st_size, PROT_READ, MAP_SHARED, fd, 0);
+std::set<std::string> runResolver(const Path & filename)
+{
+    AutoCloseFD fd = open(filename.c_str(), O_RDONLY);
+    if (!fd)
+        throw SysError("opening ‘%s’", filename);
+
+    struct stat st;
+    if (fstat(fd.get(), &st))
+        throw SysError("statting ‘%s’", filename);
+
+    if (st.st_size < sizeof(mach_header_64)) {
+        printError("file ‘%s’ is too short for a MACH binary", filename);
+        return {};
+    }
+
+    void *obj = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0);
+    if (!obj)
+        throw SysError("mmapping ‘%s’", filename);
 
     ptrdiff_t mach64_offset = 0;
 
-    uint32_t magic = ((struct mach_header_64*)obj)->magic;
-    if(magic == FAT_CIGAM || magic == FAT_MAGIC) {
+    uint32_t magic = ((struct mach_header_64*) obj)->magic;
+    if (magic == FAT_CIGAM || magic == FAT_MAGIC) {
         bool should_swap = magic == FAT_CIGAM;
         uint32_t narches = DO_SWAP(should_swap, ((struct fat_header*)obj)->nfat_arch);
 
-        for(uint32_t iter = 0; iter < narches; iter++) {
+        for (uint32_t iter = 0; iter < narches; iter++) {
             ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header) * (iter + 1);
             struct fat_arch* arch = (struct fat_arch*)header_offset;
-            if(DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) {
+            if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) {
                 mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset);
                 break;
             }
         }
         if (mach64_offset == 0) {
             printError(format("Could not find any mach64 blobs in file ‘%1%’, continuing...") % filename);
-            return std::set<string>();
+            return {};
         }
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
         printError(format("Object file has unknown magic number ‘%1%’, skipping it...") % magic);
-        return std::set<string>();
+        return {};
     }
 
     ptrdiff_t mach_header_offset = (ptrdiff_t)obj + mach64_offset;
@@ -94,30 +110,28 @@ std::set<std::string> runResolver(const Path & filename) {
     return libs;
 }
 
-bool isSymlink(const Path & path) {
+bool isSymlink(const Path & path)
+{
     struct stat st;
-    if(lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError("getting attributes of path ‘%1%’", path);
 
     return S_ISLNK(st.st_mode);
 }
 
-Path resolveSymlink(const Path & path) {
-    char buf[PATH_MAX];
-    ssize_t len = readlink(path.c_str(), buf, sizeof(buf) - 1);
-    if(len != -1) {
-        buf[len] = 0;
-        return Path(buf);
-    } else {
-        throw SysError(format("readlink('%1%')") % path);
-    }
+Path resolveSymlink(const Path & path)
+{
+    auto target = readLink(path);
+    return hasPrefix(target, "/")
+        ? target
+        : dirOf(path) + "/" + target;
 }
 
-std::set<string> resolveTree(const Path & path, PathSet & deps) {
+std::set<string> resolveTree(const Path & path, PathSet & deps)
+{
     std::set<string> results;
-    if(deps.find(path) != deps.end()) {
-        return std::set<string>();
-    }
+    if (deps.count(path))
+        return {};
     deps.insert(path);
     for (auto & lib : runResolver(path)) {
         results.insert(lib);
@@ -128,32 +142,33 @@ std::set<string> resolveTree(const Path & path, PathSet & deps) {
     return results;
 }
 
-std::set<string> getPath(const Path & path) {
+std::set<string> getPath(const Path & path)
+{
+    if (hasPrefix(path, "/dev")) return {};
+
     Path cacheFile = resolveCacheFile(path);
-    if(pathExists(cacheFile)) {
+    if (pathExists(cacheFile))
         return readCacheFile(cacheFile);
-    }
 
-    std::set<string> deps;
-    std::set<string> paths;
+    std::set<string> deps, paths;
     paths.insert(path);
 
-    Path next_path = Path(path);
-    while(isSymlink(next_path)) {
-        next_path = resolveSymlink(next_path);
-        paths.insert(next_path);
+    Path nextPath(path);
+    while (isSymlink(nextPath)) {
+        nextPath = resolveSymlink(nextPath);
+        paths.insert(nextPath);
     }
 
-    for(auto & t : resolveTree(next_path, deps)) {
+    for (auto & t : resolveTree(nextPath, deps))
         paths.insert(t);
-    }
 
     writeCacheFile(cacheFile, paths);
 
     return paths;
 }
 
-int main(int argc, char ** argv) {
+int main(int argc, char ** argv)
+{
     return handleExceptions(argv[0], [&]() {
         initNix();
 
@@ -177,18 +192,15 @@ int main(int argc, char ** argv) {
         auto drv = store->derivationFromPath(Path(argv[1]));
         Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps"));
 
-        std::set<string> all_paths;
+        std::set<string> allPaths;
 
-        for (auto & path : impurePaths) {
-            for(auto & p : getPath(path)) {
-                all_paths.insert(p);
-            }
-        }
+        for (auto & path : impurePaths)
+            for (auto & p : getPath(path))
+                allPaths.insert(p);
 
         std::cout << "extra-chroot-dirs" << std::endl;
-        for(auto & path : all_paths) {
+        for (auto & path : allPaths)
             std::cout << path << std::endl;
-        }
         std::cout << std::endl;
     });
 }

From 53a16441878ebd4a5ac6ef8a95a649bfd521da3d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 17:16:49 +0200
Subject: [PATCH 0369/2196] Darwin sandbox: Disallow creating setuid/setgid
 binaries

Suggested by Daiderd Jordan.
---
 src/libstore/build.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 86cab9f35b6..46ce562f798 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2656,6 +2656,10 @@ void DerivationGoal::runChild()
                 sandboxProfile += "(deny default (with no-log))\n";
             }
 
+            /* Disallow creating setuid/setgid binaries, since that
+               would allow breaking build user isolation. */
+            sandboxProfile += "(deny file-write-setugid)\n";
+
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
             Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);

From acc889c82179e96537ebe1494ec13b9536d579ca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 17:40:12 +0200
Subject: [PATCH 0370/2196] Darwin sandbox: Use sandbox-defaults.sb

Issue #759.

Also, remove nix.conf from the sandbox since I don't really see a
legitimate reason for builders to access the Nix configuration.
---
 .gitignore                                         |  4 ++--
 src/libstore/build.cc                              |  6 +++---
 src/libstore/local-store.cc                        |  2 +-
 src/libstore/local.mk                              | 14 +++++++++-----
 ...{sandbox-defaults.sb.in => sandbox-defaults.sb} |  9 ++++-----
 5 files changed, 19 insertions(+), 16 deletions(-)
 rename src/libstore/{sandbox-defaults.sb.in => sandbox-defaults.sb} (90%)

diff --git a/.gitignore b/.gitignore
index 4f7e668e781..60bd7d06499 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,8 +48,8 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/schema.sql.hh
-/src/libstore/sandbox-defaults.sb
+/src/libstore/schema.sql.gen.hh
+/src/libstore/sandbox-defaults.sb.gen.hh
 
 /src/nix/nix
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 46ce562f798..92471b228d0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2656,9 +2656,9 @@ void DerivationGoal::runChild()
                 sandboxProfile += "(deny default (with no-log))\n";
             }
 
-            /* Disallow creating setuid/setgid binaries, since that
-               would allow breaking build user isolation. */
-            sandboxProfile += "(deny file-write-setugid)\n";
+            sandboxProfile +=
+#include "sandbox-defaults.sb.gen.hh"
+                ;
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5b03e86f3ea..a226e0110af 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -320,7 +320,7 @@ void LocalStore::openDB(State & state, bool create)
     /* Initialise the database schema, if necessary. */
     if (create) {
         const char * schema =
-#include "schema.sql.hh"
+#include "schema.sql.gen.hh"
             ;
         db.exec(schema);
     }
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index ffdb55abc65..7bc69f65d80 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -34,12 +34,16 @@ libstore_CXXFLAGS = \
  -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 
-$(d)/local-store.cc: $(d)/schema.sql.hh
+$(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
-%.sql.hh: %.sql
-	$(trace-gen) sed -e 's/"/\\"/g' -e 's/\(.*\)/"\1\\n"/' < $< > $@ || (rm $@ && exit 1)
+$(d)/build.cc: $(d)/sandbox-defaults.sb.gen.hh
 
-clean-files += $(d)/schema.sql.hh
+%.gen.hh: %
+	echo 'R"foo(' >> $@.tmp
+	cat $< >> $@.tmp
+	echo ')foo"' >> $@.tmp
+	mv $@.tmp $@
+
+clean-files += $(d)/schema.sql.gen.hh $(d)/sandbox-defaults.sb.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
-$(eval $(call install-file-in, $(d)/sandbox-defaults.sb, $(datadir)/nix, 0644))
diff --git a/src/libstore/sandbox-defaults.sb.in b/src/libstore/sandbox-defaults.sb
similarity index 90%
rename from src/libstore/sandbox-defaults.sb.in
rename to src/libstore/sandbox-defaults.sb
index b5e80085fbe..6bd15603e79 100644
--- a/src/libstore/sandbox-defaults.sb.in
+++ b/src/libstore/sandbox-defaults.sb
@@ -28,15 +28,10 @@
 (allow file-read-metadata
        (literal "/var")
        (literal "/tmp")
-       ; symlinks
-       (literal "@sysconfdir@")
-       (literal "@sysconfdir@/nix")
-       (literal "@sysconfdir@/nix/nix.conf")
        (literal "/etc/resolv.conf")
        (literal "/private/etc/resolv.conf"))
 
 (allow file-read*
-       (literal "/private@sysconfdir@/nix/nix.conf")
        (literal "/private/var/run/resolv.conf"))
 
 ; some builders use filehandles other than stdin/stdout
@@ -61,3 +56,7 @@
 
 ; allow local networking
 (allow network* (local ip) (remote unix-socket))
+
+; Disallow creating setuid/setgid binaries, since that
+; would allow breaking build user isolation.
+(deny file-write-setugid)

From 683a499ebbb3d5e8803feeab9097930a9ce91d3f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 May 2017 20:39:40 +0200
Subject: [PATCH 0371/2196] resolve-system-dependencies: Fix another segfault

runResolver() was barfing on directories like
/System/Library/Frameworks/Security.framework/Versions/Current/PlugIns. It
should probably do something sophisticated for frameworks, but let's
ignore them for now.
---
 .../resolve-system-dependencies.cc                           | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 61504298608..b04595732b9 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -53,6 +53,11 @@ std::set<std::string> runResolver(const Path & filename)
     if (fstat(fd.get(), &st))
         throw SysError("statting ‘%s’", filename);
 
+    if (!S_ISREG(st.st_mode)) {
+        printError("file ‘%s’ is not a regular file", filename);
+        return {};
+    }
+
     if (st.st_size < sizeof(mach_header_64)) {
         printError("file ‘%s’ is too short for a MACH binary", filename);
         return {};

From c740c3ce500af2b7eb34651b5eeec01288d79dca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 13:39:27 +0200
Subject: [PATCH 0372/2196] OS X sandbox: Store .sb file in $TMPDIR rather than
 the Nix store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The filename used was not unique and owned by the build user, so
builds could fail with

error: while setting up the build environment: cannot unlink ‘/nix/store/99i210ihnsjacajaw8r33fmgjvzpg6nr-bison-3.0.4.drv.sb’: Permission denied
---
 src/libstore/build.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 92471b228d0..8695850b344 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -778,7 +778,6 @@ class DerivationGoal : public Goal
 #if __APPLE__
     typedef string SandboxProfile;
     SandboxProfile additionalSandboxProfile;
-    AutoDelete autoDelSandbox;
 #endif
 
     /* Hash rewriting. */
@@ -2711,9 +2710,7 @@ void DerivationGoal::runChild()
             debug("Generated sandbox profile:");
             debug(sandboxProfile);
 
-            Path sandboxFile = drvPath + ".sb";
-            deletePath(sandboxFile);
-            autoDelSandbox.reset(sandboxFile, false);
+            Path sandboxFile = tmpDir + "/.sandbox.sb";
 
             writeFile(sandboxFile, sandboxProfile);
 

From 44f3f8048fdc2564f25389ec2c613880763bbd03 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 14:00:06 +0200
Subject: [PATCH 0373/2196] OS X sandbox: Don't use a deterministic $TMPDIR

This doesn't work because the OS X sandbox cannot bind-mount
path to a different location.
---
 src/libstore/build.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8695850b344..7f7d2fdb8bd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1732,9 +1732,6 @@ void DerivationGoal::startBuilder()
        directory. */
 #if __linux__
     tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#elif __APPLE__
-    // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp.
-    tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir;
 #else
     tmpDirInSandbox = tmpDir;
 #endif

From c368e079ca27195aa7dbed1e834479ab17ccae73 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 15:34:03 +0200
Subject: [PATCH 0374/2196] resolve-system-dependencies: Simplify

---
 .../resolve-system-dependencies.cc                    | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index b04595732b9..aadfdc9476b 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -28,15 +28,6 @@ std::set<string> readCacheFile(const Path & file)
     return tokenizeString<set<string>>(readFile(file), "\n");
 }
 
-void writeCacheFile(const Path & file, std::set<string> & deps)
-{
-    std::ofstream fp;
-    fp.open(file);
-    for (auto & d : deps)
-        fp << d << "\n";
-    fp.close();
-}
-
 std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start)
 {
     struct dylib_command *dylc = (struct dylib_command*)dylib_command_start;
@@ -167,7 +158,7 @@ std::set<string> getPath(const Path & path)
     for (auto & t : resolveTree(nextPath, deps))
         paths.insert(t);
 
-    writeCacheFile(cacheFile, paths);
+    writeFile(cacheFile, concatStringsSep("\n", paths));
 
     return paths;
 }

From 5ea8161b552ad79b7caf9b68b3c7d6daab203266 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 16:10:10 +0200
Subject: [PATCH 0375/2196] resolve-system-dependencies: Misc fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes

  Could not find any mach64 blobs in file ‘/usr/lib/libSystem.B.dylib’, continuing...
---
 .../resolve-system-dependencies.cc            | 42 +++++++++----------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index aadfdc9476b..63e557ec5f1 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -28,12 +28,6 @@ std::set<string> readCacheFile(const Path & file)
     return tokenizeString<set<string>>(readFile(file), "\n");
 }
 
-std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start)
-{
-    struct dylib_command *dylc = (struct dylib_command*)dylib_command_start;
-    return std::string((char*)(dylib_command_start + DO_SWAP(should_swap, dylc->dylib.name.offset)));
-}
-
 std::set<std::string> runResolver(const Path & filename)
 {
     AutoCloseFD fd = open(filename.c_str(), O_RDONLY);
@@ -54,22 +48,20 @@ std::set<std::string> runResolver(const Path & filename)
         return {};
     }
 
-    void *obj = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0);
+    char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0);
     if (!obj)
         throw SysError("mmapping ‘%s’", filename);
 
     ptrdiff_t mach64_offset = 0;
 
-    uint32_t magic = ((struct mach_header_64*) obj)->magic;
+    uint32_t magic = ((mach_header_64*) obj)->magic;
     if (magic == FAT_CIGAM || magic == FAT_MAGIC) {
         bool should_swap = magic == FAT_CIGAM;
-        uint32_t narches = DO_SWAP(should_swap, ((struct fat_header*)obj)->nfat_arch);
-
-        for (uint32_t iter = 0; iter < narches; iter++) {
-            ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header) * (iter + 1);
-            struct fat_arch* arch = (struct fat_arch*)header_offset;
+        uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch);
+        for (uint32_t i = 0; i < narches; i++) {
+            fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i);
             if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) {
-                mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset);
+                mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset);
                 break;
             }
         }
@@ -84,20 +76,19 @@ std::set<std::string> runResolver(const Path & filename)
         return {};
     }
 
-    ptrdiff_t mach_header_offset = (ptrdiff_t)obj + mach64_offset;
-    struct mach_header_64 *m_header = (struct mach_header_64 *)mach_header_offset;
+    mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset);
 
     bool should_swap = magic == MH_CIGAM_64;
-    ptrdiff_t cmd_offset = mach_header_offset + sizeof(struct mach_header_64);
+    ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64);
 
     std::set<string> libs;
-    for(uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) {
-        struct load_command *cmd = (struct load_command*)cmd_offset;
+    for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) {
+        load_command * cmd = (load_command *) (obj + cmd_offset);
         switch(DO_SWAP(should_swap, cmd->cmd)) {
             case LC_LOAD_UPWARD_DYLIB:
             case LC_LOAD_DYLIB:
             case LC_REEXPORT_DYLIB:
-                libs.insert(findDylibName(should_swap, cmd_offset));
+                libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset));
                 break;
         }
         cmd_offset += DO_SWAP(should_swap, cmd->cmdsize);
@@ -185,8 +176,15 @@ int main(int argc, char ** argv)
 
         auto store = openStore();
 
-        auto drv = store->derivationFromPath(Path(argv[1]));
-        Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps"));
+        StringSet impurePaths;
+
+        if (std::string(argv[1]) == "--test")
+            impurePaths.insert(argv[2]);
+        else {
+            auto drv = store->derivationFromPath(Path(argv[1]));
+            impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps"));
+            impurePaths.insert("/usr/lib/libSystem.dylib");
+        }
 
         std::set<string> allPaths;
 

From c96e8cd097ce0d181467fddd92acad4341ca566a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 17:23:27 +0200
Subject: [PATCH 0376/2196] OS X sandbox: Improve builtin sandbox profile

Also, add rules to allow fixed-output derivations to access the
network.

These rules are sufficient to build stdenvDarwin without any
__sandboxProfile magic.
---
 .gitignore                       |  1 +
 src/libstore/build.cc            | 13 ++++-
 src/libstore/local.mk            | 12 ++--
 src/libstore/sandbox-defaults.sb | 94 +++++++++++++++-----------------
 src/libstore/sandbox-network.sb  | 16 ++++++
 5 files changed, 77 insertions(+), 59 deletions(-)
 create mode 100644 src/libstore/sandbox-network.sb

diff --git a/.gitignore b/.gitignore
index 60bd7d06499..1cf7a3c3515 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,7 @@ perl/Makefile.config
 # /src/libstore/
 /src/libstore/schema.sql.gen.hh
 /src/libstore/sandbox-defaults.sb.gen.hh
+/src/libstore/sandbox-network.sb.gen.hh
 
 /src/nix/nix
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 7f7d2fdb8bd..55c8ac58837 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2614,8 +2614,9 @@ void DerivationGoal::runChild()
         string sandboxProfile;
         if (drv->isBuiltin()) {
             ;
+        }
 #if __APPLE__
-        } else if (useChroot) {
+        else if (useChroot) {
             /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
             PathSet ancestry;
 
@@ -2653,9 +2654,14 @@ void DerivationGoal::runChild()
             }
 
             sandboxProfile +=
-#include "sandbox-defaults.sb.gen.hh"
+                #include "sandbox-defaults.sb.gen.hh"
                 ;
 
+            if (fixedOutput)
+                sandboxProfile +=
+                    #include "sandbox-network.sb.gen.hh"
+                    ;
+
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
             Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
@@ -2718,8 +2724,9 @@ void DerivationGoal::runChild()
             args.push_back("-D");
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
             args.push_back(drv->builder);
+        }
 #endif
-        } else {
+        else {
             builder = drv->builder.c_str();
             string builderBasename = baseNameOf(drv->builder);
             args.push_back(builderBasename);
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 7bc69f65d80..c0cc91c2658 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -36,14 +36,14 @@ libstore_CXXFLAGS = \
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
-$(d)/build.cc: $(d)/sandbox-defaults.sb.gen.hh
+$(d)/build.cc: $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh
 
 %.gen.hh: %
-	echo 'R"foo(' >> $@.tmp
-	cat $< >> $@.tmp
-	echo ')foo"' >> $@.tmp
-	mv $@.tmp $@
+	@echo 'R"foo(' >> $@.tmp
+	$(trace-gen) cat $< >> $@.tmp
+	@echo ')foo"' >> $@.tmp
+	@mv $@.tmp $@
 
-clean-files += $(d)/schema.sql.gen.hh $(d)/sandbox-defaults.sb.gen.hh
+clean-files += $(d)/schema.sql.gen.hh $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index 6bd15603e79..0292f5ee882 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -1,62 +1,56 @@
-(allow file-read* file-write-data (literal "/dev/null"))
-(allow ipc-posix*)
-(allow mach-lookup (global-name "com.apple.SecurityServer"))
+(define TMPDIR (param "_GLOBAL_TMP_DIR"))
 
-(allow file-read*
-       (literal "/dev/dtracehelper")
-       (literal "/dev/tty")
-       (literal "/dev/autofs_nowait")
-       (literal "/System/Library/CoreServices/SystemVersion.plist")
-       (literal "/private/var/run/systemkeychaincheck.done")
-       (literal "/private/etc/protocols")
-       (literal "/private/var/tmp")
-       (literal "/private/var/db")
-       (subpath "/private/var/db/mds"))
+; Disallow creating setuid/setgid binaries, since that
+; would allow breaking build user isolation.
+(deny file-write-setugid)
 
-(allow file-read*
-       (subpath "/usr/share/icu")
-       (subpath "/usr/share/locale")
-       (subpath "/usr/share/zoneinfo"))
+; Allow forking.
+(allow process-fork)
 
-(allow file-write*
-       (literal "/dev/tty")
-       (literal "/dev/dtracehelper")
-       (literal "/mds"))
+; Allow reading system information like #CPUs, etc.
+(allow sysctl-read)
 
-(allow file-ioctl (literal "/dev/dtracehelper"))
+; Allow POSIX semaphores and shared memory.
+(allow ipc-posix*)
 
-(allow file-read-metadata
-       (literal "/var")
-       (literal "/tmp")
-       (literal "/etc/resolv.conf")
-       (literal "/private/etc/resolv.conf"))
+; Allow socket creation.
+(allow system-socket)
 
-(allow file-read*
-       (literal "/private/var/run/resolv.conf"))
+; Allow sending signals within the sandbox.
+(allow signal (target same-sandbox))
 
-; some builders use filehandles other than stdin/stdout
-(allow file*
-        (subpath "/dev/fd")
-        (literal "/dev/ptmx")
-        (regex #"^/dev/[pt]ty.*$"))
+; Access to /tmp.
+(allow file* process-exec (literal "/tmp") (subpath TMPDIR))
 
-; allow everything inside TMP
-(allow file* process-exec
-       (subpath (param "_GLOBAL_TMP_DIR"))
-       (subpath "/private/tmp"))
+; Some packages like to read the system version.
+(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist"))
 
-(allow process-fork)
-(allow sysctl-read)
-(allow signal (target same-sandbox))
+; Without this line clang cannot write to /dev/null, breaking some configure tests.
+(allow file-read-metadata (literal "/dev"))
 
-; allow getpwuid (for git and other packages)
-(allow mach-lookup
-       (global-name "com.apple.system.notification_center")
-       (global-name "com.apple.system.opendirectoryd.libinfo"))
+; Standard devices.
+(allow file*
+       (literal "/dev/null")
+       (literal "/dev/random")
+       (literal "/dev/stdin")
+       (literal "/dev/stdout")
+       (literal "/dev/tty")
+       (literal "/dev/urandom")
+       (literal "/dev/zero")
+       (subpath "/dev/fd"))
 
-; allow local networking
-(allow network* (local ip) (remote unix-socket))
+; Does nothing, but reduces build noise.
+(allow file* (literal "/dev/dtracehelper"))
 
-; Disallow creating setuid/setgid binaries, since that
-; would allow breaking build user isolation.
-(deny file-write-setugid)
+; Allow access to zoneinfo since libSystem needs it.
+(allow file-read* (subpath "/usr/share/zoneinfo"))
+
+(allow file-read* (subpath "/usr/share/locale"))
+
+; This is mostly to get more specific log messages when builds try to
+; access something in /etc or /var.
+(allow file-read-metadata
+       (literal "/etc")
+       (literal "/var")
+       (literal "/private/var/tmp")
+       )
diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb
new file mode 100644
index 00000000000..56beec761fa
--- /dev/null
+++ b/src/libstore/sandbox-network.sb
@@ -0,0 +1,16 @@
+; Allow local and remote network traffic.
+(allow network* (local ip) (remote ip))
+
+; Allow access to /etc/resolv.conf (which is a symlink to
+; /private/var/run/resolv.conf).
+(allow file-read-metadata
+       (literal "/var")
+       (literal "/etc")
+       (literal "/etc/resolv.conf")
+       (literal "/private/etc/resolv.conf"))
+
+(allow file-read*
+       (literal "/private/var/run/resolv.conf"))
+
+; Allow DNS lookups.
+(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))

From 52fec8dde862264874a4f19be329124ac46adb81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 May 2017 20:43:47 +0200
Subject: [PATCH 0377/2196] Remove listxattr assertion

It appears that sometimes, listxattr() returns a different value for
the query case (i.e. when the buffer size is 0).
---
 src/libstore/local-store.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a226e0110af..ee36428af03 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -424,8 +424,6 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1)))
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
                 throw SysError("removing extended attribute ‘%s’ from ‘%s’", eaName, path);
-
-        assert(llistxattr(path.c_str(), nullptr, 0) == 0);
     }
 #endif
 

From ab5834f7a1c2cae9b7071d5a6944ff8b1eeb6e38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Jun 2017 14:28:21 +0200
Subject: [PATCH 0378/2196] RPM, Deb: Add dependency on libseccomp

---
 nix.spec.in | 2 ++
 release.nix | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 390893d64dc..88f0d04d390 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -20,9 +20,11 @@ Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
+Requires: libseccomp
 BuildRequires: bzip2-devel
 BuildRequires: sqlite-devel
 BuildRequires: libcurl-devel
+BuildRequires: libseccomp-devel
 
 # Hack to make that shitty RPM scanning hack shut up.
 Provides: perl(Nix::SSH)
diff --git a/release.nix b/release.nix
index ec6bda99577..fbd644cd62b 100644
--- a/release.nix
+++ b/release.nix
@@ -305,7 +305,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
@@ -327,14 +327,14 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;

From b4b1f4525f8dc8f320d666c208bff5cb36777580 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Jun 2017 14:43:15 +0200
Subject: [PATCH 0379/2196] Fix coverage job

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index fbd644cd62b..aa2ee7290e6 100644
--- a/release.nix
+++ b/release.nix
@@ -173,7 +173,7 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl bzip2 openssl pkgconfig sqlite xz libsodium
+          [ curl bzip2 openssl pkgconfig sqlite xz libsodium libseccomp
             # These are for "make check" only:
             graphviz libxml2 libxslt
           ];

From 88acb6461045e4e565c7eef884ea7ab69c2e6de9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 5 Jun 2017 22:52:05 +0100
Subject: [PATCH 0380/2196] Add .editorconfig

- Automatically adjust editor to nix coding style
-> less nitpiks/styling issues in pull requests
-> profit(!)

see also nixpkgs' editorconfig: https://github.com/NixOS/nixpkgs/blob/master/.editorconfig
---
 .editorconfig | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 .editorconfig

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 00000000000..887ecadba59
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,26 @@
+# EditorConfig configuration for nix
+# http://EditorConfig.org
+
+# Top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file, utf-8 charset
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+# Match nix files, set indent to spaces with width of two
+[*.nix]
+indent_style = space
+indent_size = 2
+
+# Match c++/shell/perl, set indent to spaces with width of four
+[*.{hpp,cc,hh,sh,pl}]
+indent_style = space
+indent_size = 4
+
+# Match diffs, avoid to trim trailing whitespace
+[*.{diff,patch}]
+trim_trailing_whitespace = false

From 85e93d7b874f99730387714394bb60407cf138d5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Jun 2017 18:44:49 +0200
Subject: [PATCH 0381/2196] Always use the Darwin sandbox

Even with "build-use-sandbox = false", we now use sandboxing with a
permissive profile that allows everything except the creation of
setuid/setgid binaries.
---
 .gitignore                       |   4 +-
 src/libstore/build.cc            | 168 ++++++++++++++++---------------
 src/libstore/local.mk            |   6 +-
 src/libstore/sandbox-defaults.sb |   2 +
 src/libstore/sandbox-minimal.sb  |   5 +
 5 files changed, 99 insertions(+), 86 deletions(-)
 create mode 100644 src/libstore/sandbox-minimal.sb

diff --git a/.gitignore b/.gitignore
index 1cf7a3c3515..6163087384d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,9 +48,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/schema.sql.gen.hh
-/src/libstore/sandbox-defaults.sb.gen.hh
-/src/libstore/sandbox-network.sb.gen.hh
+/src/libstore/*.gen.hh
 
 /src/nix/nix
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 55c8ac58837..d12a1a7913b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2611,104 +2611,102 @@ void DerivationGoal::runChild()
 
         const char *builder = "invalid";
 
-        string sandboxProfile;
         if (drv->isBuiltin()) {
             ;
         }
 #if __APPLE__
-        else if (useChroot) {
-            /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-            PathSet ancestry;
+        else {
+            /* This has to appear before import statements. */
+            std::string sandboxProfile = "(version 1)\n";
+
+            if (useChroot) {
+
+                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
+                PathSet ancestry;
+
+                /* We build the ancestry before adding all inputPaths to the store because we know they'll
+                   all have the same parents (the store), and there might be lots of inputs. This isn't
+                   particularly efficient... I doubt it'll be a bottleneck in practice */
+                for (auto & i : dirsInChroot) {
+                    Path cur = i.first;
+                    while (cur.compare("/") != 0) {
+                        cur = dirOf(cur);
+                        ancestry.insert(cur);
+                    }
+                }
 
-            /* We build the ancestry before adding all inputPaths to the store because we know they'll
-               all have the same parents (the store), and there might be lots of inputs. This isn't
-               particularly efficient... I doubt it'll be a bottleneck in practice */
-            for (auto & i : dirsInChroot) {
-                Path cur = i.first;
+                /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
+                   path component this time, since it's typically /nix/store and we care about that. */
+                Path cur = worker.store.storeDir;
                 while (cur.compare("/") != 0) {
-                    cur = dirOf(cur);
                     ancestry.insert(cur);
+                    cur = dirOf(cur);
                 }
-            }
-
-            /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
-               path component this time, since it's typically /nix/store and we care about that. */
-            Path cur = worker.store.storeDir;
-            while (cur.compare("/") != 0) {
-                ancestry.insert(cur);
-                cur = dirOf(cur);
-            }
 
-            /* Add all our input paths to the chroot */
-            for (auto & i : inputPaths)
-                dirsInChroot[i] = i;
-
-            /* This has to appear before import statements */
-            sandboxProfile += "(version 1)\n";
-
-            /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-            if (settings.darwinLogSandboxViolations) {
-                sandboxProfile += "(deny default)\n";
-            } else {
-                sandboxProfile += "(deny default (with no-log))\n";
-            }
+                /* Add all our input paths to the chroot */
+                for (auto & i : inputPaths)
+                    dirsInChroot[i] = i;
 
-            sandboxProfile +=
-                #include "sandbox-defaults.sb.gen.hh"
-                ;
+                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
+                if (settings.darwinLogSandboxViolations) {
+                    sandboxProfile += "(deny default)\n";
+                } else {
+                    sandboxProfile += "(deny default (with no-log))\n";
+                }
 
-            if (fixedOutput)
                 sandboxProfile +=
-                    #include "sandbox-network.sb.gen.hh"
+                    #include "sandbox-defaults.sb.gen.hh"
                     ;
 
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
-            /* Our rwx outputs */
-            sandboxProfile += "(allow file-read* file-write* process-exec\n";
-            for (auto & i : missingPaths) {
-                sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
-            }
-            sandboxProfile += ")\n";
-
-            /* Our inputs (transitive dependencies and any impurities computed above)
+                if (fixedOutput)
+                    sandboxProfile +=
+                        #include "sandbox-network.sb.gen.hh"
+                        ;
 
-               without file-write* allowed, access() incorrectly returns EPERM
-             */
-            sandboxProfile += "(allow file-read* file-write* process-exec\n";
-            for (auto & i : dirsInChroot) {
-                if (i.first != i.second.source)
-                    throw Error(format(
-                        "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin")
-                        % i.first % i.second.source);
-
-                string path = i.first;
-                struct stat st;
-                if (lstat(path.c_str(), &st)) {
-                    if (i.second.optional && errno == ENOENT)
-                        continue;
-                    throw SysError(format("getting attributes of path ‘%1%’") % path);
+                /* Our rwx outputs */
+                sandboxProfile += "(allow file-read* file-write* process-exec\n";
+                for (auto & i : missingPaths) {
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
                 }
-                if (S_ISDIR(st.st_mode))
-                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
-                else
-                    sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str();
-            }
-            sandboxProfile += ")\n";
+                sandboxProfile += ")\n";
+
+                /* Our inputs (transitive dependencies and any impurities computed above)
+
+                   without file-write* allowed, access() incorrectly returns EPERM
+                 */
+                sandboxProfile += "(allow file-read* file-write* process-exec\n";
+                for (auto & i : dirsInChroot) {
+                    if (i.first != i.second.source)
+                        throw Error(format(
+                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin")
+                            % i.first % i.second.source);
+
+                    string path = i.first;
+                    struct stat st;
+                    if (lstat(path.c_str(), &st)) {
+                        if (i.second.optional && errno == ENOENT)
+                            continue;
+                        throw SysError(format("getting attributes of path ‘%1%’") % path);
+                    }
+                    if (S_ISDIR(st.st_mode))
+                        sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
+                    else
+                        sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str();
+                }
+                sandboxProfile += ")\n";
 
-            /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-            sandboxProfile += "(allow file-read*\n";
-            for (auto & i : ancestry) {
-                sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str();
-            }
-            sandboxProfile += ")\n";
+                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
+                sandboxProfile += "(allow file-read*\n";
+                for (auto & i : ancestry) {
+                    sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str();
+                }
+                sandboxProfile += ")\n";
 
-            sandboxProfile += additionalSandboxProfile;
+                sandboxProfile += additionalSandboxProfile;
+            } else
+                sandboxProfile +=
+                    #include "sandbox-minimal.sb.gen.hh"
+                    ;
 
             debug("Generated sandbox profile:");
             debug(sandboxProfile);
@@ -2717,6 +2715,13 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
+            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
+               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+            Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
+
+            /* They don't like trailing slashes on subpath directives */
+            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
+
             builder = "/usr/bin/sandbox-exec";
             args.push_back("sandbox-exec");
             args.push_back("-f");
@@ -2725,12 +2730,13 @@ void DerivationGoal::runChild()
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
             args.push_back(drv->builder);
         }
-#endif
+#else
         else {
             builder = drv->builder.c_str();
             string builderBasename = baseNameOf(drv->builder);
             args.push_back(builderBasename);
         }
+#endif
 
         for (auto & i : drv->args)
             args.push_back(rewriteStrings(i, inputRewrites));
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index c0cc91c2658..36b270f2e07 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -36,7 +36,9 @@ libstore_CXXFLAGS = \
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
-$(d)/build.cc: $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh
+sandbox-headers = $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh $(d)/sandbox-minimal.sb.gen.hh
+
+$(d)/build.cc: $(sandbox-headers)
 
 %.gen.hh: %
 	@echo 'R"foo(' >> $@.tmp
@@ -44,6 +46,6 @@ $(d)/build.cc: $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh
 	@echo ')foo"' >> $@.tmp
 	@mv $@.tmp $@
 
-clean-files += $(d)/schema.sql.gen.hh $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh
+clean-files += $(d)/schema.sql.gen.hh $(sandbox-headers)
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index 0292f5ee882..d63c8f813c9 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -1,5 +1,7 @@
 (define TMPDIR (param "_GLOBAL_TMP_DIR"))
 
+(deny default)
+
 ; Disallow creating setuid/setgid binaries, since that
 ; would allow breaking build user isolation.
 (deny file-write-setugid)
diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/sandbox-minimal.sb
new file mode 100644
index 00000000000..65f5108b399
--- /dev/null
+++ b/src/libstore/sandbox-minimal.sb
@@ -0,0 +1,5 @@
+(allow default)
+
+; Disallow creating setuid/setgid binaries, since that
+; would allow breaking build user isolation.
+(deny file-write-setugid)

From c8cc50d46e78de7ae02c2cb7a5159e995c993f61 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Jun 2017 18:52:15 +0200
Subject: [PATCH 0382/2196] Disable the build user mechanism on all platforms
 except Linux and OS X

---
 src/libstore/build.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d12a1a7913b..d5fe41d1b16 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1716,11 +1716,17 @@ void DerivationGoal::startBuilder()
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
     if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
         buildUser = std::make_unique<UserLock>();
 
         /* Make sure that no other processes are executing under this
            uid. */
         buildUser->kill();
+#else
+        /* Don't know how to block the creation of setuid/setgid
+           binaries on this platform. */
+        throw Error("build users are not supported on this platform for security reasons");
+#endif
     }
 
     /* Create a temporary directory where the build will take

From b8283773bd64d7da6859ed520ee19867742a03ba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jun 2017 16:17:17 +0200
Subject: [PATCH 0383/2196] nix: Make all options available as flags
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Thus, instead of ‘--option <name> <value>’, you can write ‘--<name>
<value>’. So

  --option http-connections 100

becomes

  --http-connections 100

Apart from brevity, the difference is that it's not an error to set a
non-existent option via --option, but unrecognized arguments are
fatal.

Boolean options have special treatment: they're mapped to the
argument-less flags ‘--<name>’ and ‘--no-<name>’. E.g.

  --option auto-optimise-store false

becomes

  --no-auto-optimise-store
---
 src/libstore/globals.cc |  7 +++++++
 src/libutil/config.cc   | 25 +++++++++++++++++++++++++
 src/libutil/config.hh   |  6 ++++++
 src/nix/main.cc         |  2 ++
 4 files changed, 40 insertions(+)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 3dd2508a26d..3f2bea8e7b1 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -98,6 +98,13 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
     AbstractSetting::toJSON(out);
 }
 
+template<> void BaseSetting<SandboxMode>::convertToArg(Args & args)
+{
+    args.mkFlag(0, name, {}, "Enable sandboxing.", 0, [=](Strings ss) { value = smEnabled; });
+    args.mkFlag(0, "no-" + name, {}, "Disable sandboxing.", 0, [=](Strings ss) { value = smDisabled; });
+    args.mkFlag(0, "relaxed-" + name, {}, "Enable sandboxing, but allow builds to disable it.", 0, [=](Strings ss) { value = smRelaxed; });
+}
+
 void MaxBuildJobsSetting::set(const std::string & str)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index f7a46bfee63..612fb6e6835 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -115,6 +115,13 @@ void Config::toJSON(JSONObject & out)
         }
 }
 
+void Config::convertToArgs(Args & args)
+{
+    for (auto & s : _settings)
+        if (!s.second.isAlias)
+            s.second.setting->convertToArg(args);
+}
+
 AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
@@ -128,12 +135,22 @@ void AbstractSetting::toJSON(JSONPlaceholder & out)
     out.write(to_string());
 }
 
+void AbstractSetting::convertToArg(Args & args)
+{
+}
+
 template<typename T>
 void BaseSetting<T>::toJSON(JSONPlaceholder & out)
 {
     out.write(value);
 }
 
+template<typename T>
+void BaseSetting<T>::convertToArg(Args & args)
+{
+    args.mkFlag(0, name, {}, description, 1, [=](Strings ss) { set(*ss.begin()); });
+}
+
 template<> void BaseSetting<std::string>::set(const std::string & str)
 {
     value = str;
@@ -174,6 +191,12 @@ template<> std::string BaseSetting<bool>::to_string()
     return value ? "true" : "false";
 }
 
+template<> void BaseSetting<bool>::convertToArg(Args & args)
+{
+    args.mkFlag(0, name, {}, description, 0, [=](Strings ss) { value = true; });
+    args.mkFlag(0, "no-" + name, {}, description, 0, [=](Strings ss) { value = false; });
+}
+
 template<> void BaseSetting<Strings>::set(const std::string & str)
 {
     value = tokenizeString<Strings>(str);
@@ -216,6 +239,8 @@ template class BaseSetting<long long>;
 template class BaseSetting<unsigned long long>;
 template class BaseSetting<bool>;
 template class BaseSetting<std::string>;
+template class BaseSetting<Strings>;
+template class BaseSetting<StringSet>;
 
 void PathSetting::set(const std::string & str)
 {
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 77620d47d37..2ca643fe0e7 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -63,6 +63,8 @@ public:
     void resetOverriden();
 
     void toJSON(JSONObject & out);
+
+    void convertToArgs(Args & args);
 };
 
 class AbstractSetting
@@ -99,6 +101,8 @@ protected:
 
     virtual void toJSON(JSONPlaceholder & out);
 
+    virtual void convertToArg(Args & args);
+
     bool isOverriden() { return overriden; }
 };
 
@@ -132,6 +136,8 @@ public:
 
     std::string to_string() override;
 
+    void convertToArg(Args & args) override;
+
     void toJSON(JSONPlaceholder & out) override;
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 216f0bccef1..f83843415e3 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -22,6 +22,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         });
 
         mkFlag(0, "version", "show version information", std::bind(printVersion, programName));
+
+        settings.convertToArgs(*this);
     }
 };
 

From aa952d5f0bc623a1584f2d589209f586e594c75f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jun 2017 16:49:54 +0200
Subject: [PATCH 0384/2196] nix: Add --help-config flag

---
 src/libutil/args.hh   |  2 +-
 src/libutil/config.hh | 10 +++++++++-
 src/nix/main.cc       | 20 ++++++++++++++++++++
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index ac12f8be633..f70bb7823dc 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -41,7 +41,7 @@ protected:
 
     virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
 
-    void printFlags(std::ostream & out);
+    virtual void printFlags(std::ostream & out);
 
     /* Positional arguments. */
     struct ExpectedArg
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 2ca643fe0e7..994eab91170 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -31,6 +31,8 @@ class Config
 {
     friend class AbstractSetting;
 
+public:
+
     struct SettingData
     {
         bool isAlias;
@@ -40,7 +42,11 @@ class Config
         { }
     };
 
-    std::map<std::string, SettingData> _settings;
+    typedef std::map<std::string, SettingData> Settings;
+
+private:
+
+    Settings _settings;
 
     StringMap initials;
 
@@ -58,6 +64,8 @@ public:
 
     StringMap getSettings(bool overridenOnly = false);
 
+    const Settings & _getSettings() { return _settings; }
+
     void applyConfigFile(const Path & path, bool fatal = false);
 
     void resetOverriden();
diff --git a/src/nix/main.cc b/src/nix/main.cc
index f83843415e3..ea6838cd76c 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -21,10 +21,30 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             throw Exit();
         });
 
+        mkFlag(0, "help-config", "show configuration options", [=]() {
+            std::cout << "The following configuration options are available:\n\n";
+            Table2 tbl;
+            for (const auto & s : settings._getSettings())
+                if (!s.second.isAlias)
+                    tbl.emplace_back(s.first, s.second.setting->description);
+            printTable(std::cout, tbl);
+            throw Exit();
+        });
+
         mkFlag(0, "version", "show version information", std::bind(printVersion, programName));
 
         settings.convertToArgs(*this);
     }
+
+    void printFlags(std::ostream & out) override
+    {
+        Args::printFlags(out);
+        std::cout <<
+            "\n"
+            "In addition, most configuration settings can be overriden using ‘--<name> <value>’.\n"
+            "Boolean settings can be overriden using ‘--<name>’ or ‘--no-<name>’. See ‘nix\n"
+            "--help-config’ for a list of configuration settings.\n";
+    }
 };
 
 void mainWrapped(int argc, char * * argv)

From 186571965dccf57d15b9f37c1cca92a57187b7b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Jun 2017 18:41:20 +0200
Subject: [PATCH 0385/2196] Don't show flags from config settings in "nix
 --help"

---
 src/libmain/common-args.cc |  8 +++-
 src/libstore/globals.cc    | 20 +++++++--
 src/libutil/args.cc        | 26 +++++++++---
 src/libutil/args.hh        | 83 ++++++++++++++++++++++++++------------
 src/libutil/config.cc      | 29 +++++++++----
 src/libutil/config.hh      |  6 +--
 src/nix/main.cc            |  4 +-
 src/nix/sigs.cc            |  9 ++++-
 src/nix/verify.cc          |  9 ++++-
 9 files changed, 140 insertions(+), 54 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 9a7a893138d..3fa42c2aafa 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -18,8 +18,12 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         verbosity = lvlDebug;
     });
 
-    mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2,
-        [](Strings ss) {
+    mkFlag()
+        .longName("option")
+        .labels({"name", "value"})
+        .description("set a Nix configuration option (overriding nix.conf)")
+        .arity(2)
+        .handler([](Strings ss) {
             auto name = ss.front(); ss.pop_front();
             auto value = ss.front();
             try {
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 3f2bea8e7b1..2aceed27051 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -98,11 +98,23 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
     AbstractSetting::toJSON(out);
 }
 
-template<> void BaseSetting<SandboxMode>::convertToArg(Args & args)
+template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag(0, name, {}, "Enable sandboxing.", 0, [=](Strings ss) { value = smEnabled; });
-    args.mkFlag(0, "no-" + name, {}, "Disable sandboxing.", 0, [=](Strings ss) { value = smDisabled; });
-    args.mkFlag(0, "relaxed-" + name, {}, "Enable sandboxing, but allow builds to disable it.", 0, [=](Strings ss) { value = smRelaxed; });
+    args.mkFlag()
+        .longName(name)
+        .description("Enable sandboxing.")
+        .handler([=](Strings ss) { value = smEnabled; })
+        .category(category);
+    args.mkFlag()
+        .longName("no-" + name)
+        .description("Disable sandboxing.")
+        .handler([=](Strings ss) { value = smDisabled; })
+        .category(category);
+    args.mkFlag()
+        .longName("relaxed-" + name)
+        .description("Enable sandboxing, but allow builds to disable it.")
+        .handler([=](Strings ss) { value = smRelaxed; })
+        .category(category);
 }
 
 void MaxBuildJobsSetting::set(const std::string & str)
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 115484f9e6c..df7e040875d 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -3,6 +3,18 @@
 
 namespace nix {
 
+Args::FlagMaker Args::mkFlag()
+{
+    return FlagMaker(*this);
+}
+
+Args::FlagMaker::~FlagMaker()
+{
+    assert(flag->longName != "");
+    args.longFlags[flag->longName] = flag;
+    if (flag->shortName) args.shortFlags[flag->shortName] = flag;
+}
+
 void Args::parseCmdline(const Strings & _cmdline)
 {
     Strings pendingArgs;
@@ -71,11 +83,13 @@ void Args::printHelp(const string & programName, std::ostream & out)
 void Args::printFlags(std::ostream & out)
 {
     Table2 table;
-    for (auto & flag : longFlags)
+    for (auto & flag : longFlags) {
+        if (hiddenCategories.count(flag.second->category)) continue;
         table.push_back(std::make_pair(
-                (flag.second.shortName ? std::string("-") + flag.second.shortName + ", " : "    ")
-                + "--" + flag.first + renderLabels(flag.second.labels),
-                flag.second.description));
+                (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : "    ")
+                + "--" + flag.first + renderLabels(flag.second->labels),
+                flag.second->description));
+    }
     printTable(out, table);
 }
 
@@ -99,14 +113,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     if (string(*pos, 0, 2) == "--") {
         auto i = longFlags.find(string(*pos, 2));
         if (i == longFlags.end()) return false;
-        return process("--" + i->first, i->second);
+        return process("--" + i->first, *i->second);
     }
 
     if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
         auto c = (*pos)[1];
         auto i = shortFlags.find(c);
         if (i == shortFlags.end()) return false;
-        return process(std::string("-") + c, i->second);
+        return process(std::string("-") + c, *i->second);
     }
 
     return false;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index f70bb7823dc..aa11373d5f1 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -29,15 +29,18 @@ protected:
     /* Flags. */
     struct Flag
     {
-        char shortName;
+        typedef std::shared_ptr<Flag> ptr;
+        std::string longName;
+        char shortName = 0;
         std::string description;
         Strings labels;
-        size_t arity;
+        size_t arity = 0;
         std::function<void(Strings)> handler;
+        std::string category;
     };
 
-    std::map<std::string, Flag> longFlags;
-    std::map<char, Flag> shortFlags;
+    std::map<std::string, Flag::ptr> longFlags;
+    std::map<char, Flag::ptr> shortFlags;
 
     virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
 
@@ -55,33 +58,53 @@ protected:
 
     virtual bool processArgs(const Strings & args, bool finish);
 
+    std::set<std::string> hiddenCategories;
+
 public:
 
+    class FlagMaker
+    {
+        Args & args;
+        Flag::ptr flag;
+        friend class Args;
+        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { };
+    public:
+        ~FlagMaker();
+        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
+        FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
+        FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
+        FlagMaker & labels(const Strings & ls) { flag->labels = ls; return *this; };
+        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
+        FlagMaker & handler(std::function<void(Strings)> handler) { flag->handler = handler; return *this; };
+        FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
+    };
+
+    FlagMaker mkFlag();
+
     /* Helper functions for constructing flags / positional
        arguments. */
 
-    void mkFlag(char shortName, const std::string & longName,
-        const Strings & labels, const std::string & description,
-        size_t arity, std::function<void(Strings)> handler)
-    {
-        auto flag = Flag{shortName, description, labels, arity, handler};
-        if (shortName) shortFlags[shortName] = flag;
-        longFlags[longName] = flag;
-    }
-
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void()> fun)
     {
-        mkFlag(shortName, longName, {}, description, 0, std::bind(fun));
+        mkFlag()
+            .shortName(shortName)
+            .longName(longName)
+            .description(description)
+            .handler(std::bind(fun));
     }
 
     void mkFlag1(char shortName, const std::string & longName,
         const std::string & label, const std::string & description,
         std::function<void(std::string)> fun)
     {
-        mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) {
-            fun(ss.front());
-        });
+        mkFlag()
+            .shortName(shortName)
+            .longName(longName)
+            .labels({label})
+            .description(description)
+            .arity(1)
+            .handler([=](Strings ss) { fun(ss.front()); });
     }
 
     void mkFlag(char shortName, const std::string & name,
@@ -105,9 +128,11 @@ public:
     void mkFlag(char shortName, const std::string & longName, const std::string & description,
         T * dest, const T & value)
     {
-        mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) {
-            *dest = value;
-        });
+        mkFlag()
+            .shortName(shortName)
+            .longName(longName)
+            .description(description)
+            .handler([=](Strings ss) { *dest = value; });
     }
 
     template<class I>
@@ -123,12 +148,18 @@ public:
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void(I)> fun)
     {
-        mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) {
-            I n;
-            if (!string2Int(ss.front(), n))
-                throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName);
-            fun(n);
-        });
+        mkFlag()
+            .shortName(shortName)
+            .longName(longName)
+            .labels({"N"})
+            .description(description)
+            .arity(1)
+            .handler([=](Strings ss) {
+                I n;
+                if (!string2Int(ss.front(), n))
+                    throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName);
+                fun(n);
+            });
     }
 
     /* Expect a string argument. */
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 612fb6e6835..0682bcd5dbc 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -115,11 +115,11 @@ void Config::toJSON(JSONObject & out)
         }
 }
 
-void Config::convertToArgs(Args & args)
+void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings)
         if (!s.second.isAlias)
-            s.second.setting->convertToArg(args);
+            s.second.setting->convertToArg(args, category);
 }
 
 AbstractSetting::AbstractSetting(
@@ -135,7 +135,7 @@ void AbstractSetting::toJSON(JSONPlaceholder & out)
     out.write(to_string());
 }
 
-void AbstractSetting::convertToArg(Args & args)
+void AbstractSetting::convertToArg(Args & args, const std::string & category)
 {
 }
 
@@ -146,9 +146,14 @@ void BaseSetting<T>::toJSON(JSONPlaceholder & out)
 }
 
 template<typename T>
-void BaseSetting<T>::convertToArg(Args & args)
+void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag(0, name, {}, description, 1, [=](Strings ss) { set(*ss.begin()); });
+    args.mkFlag()
+        .longName(name)
+        .description(description)
+        .arity(1)
+        .handler([=](Strings ss) { set(*ss.begin()); })
+        .category(category);
 }
 
 template<> void BaseSetting<std::string>::set(const std::string & str)
@@ -191,10 +196,18 @@ template<> std::string BaseSetting<bool>::to_string()
     return value ? "true" : "false";
 }
 
-template<> void BaseSetting<bool>::convertToArg(Args & args)
+template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag(0, name, {}, description, 0, [=](Strings ss) { value = true; });
-    args.mkFlag(0, "no-" + name, {}, description, 0, [=](Strings ss) { value = false; });
+    args.mkFlag()
+        .longName(name)
+        .description(description)
+        .handler([=](Strings ss) { value = true; })
+        .category(category);
+    args.mkFlag()
+        .longName("no-" + name)
+        .description(description)
+        .handler([=](Strings ss) { value = false; })
+        .category(category);
 }
 
 template<> void BaseSetting<Strings>::set(const std::string & str)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 994eab91170..99850c1cdfd 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -72,7 +72,7 @@ public:
 
     void toJSON(JSONObject & out);
 
-    void convertToArgs(Args & args);
+    void convertToArgs(Args & args, const std::string & category);
 };
 
 class AbstractSetting
@@ -109,7 +109,7 @@ protected:
 
     virtual void toJSON(JSONPlaceholder & out);
 
-    virtual void convertToArg(Args & args);
+    virtual void convertToArg(Args & args, const std::string & category);
 
     bool isOverriden() { return overriden; }
 };
@@ -144,7 +144,7 @@ public:
 
     std::string to_string() override;
 
-    void convertToArg(Args & args) override;
+    void convertToArg(Args & args, const std::string & category) override;
 
     void toJSON(JSONPlaceholder & out) override;
 };
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ea6838cd76c..88a602b8481 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -33,7 +33,9 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
         mkFlag(0, "version", "show version information", std::bind(printVersion, programName));
 
-        settings.convertToArgs(*this);
+        std::string cat = "config";
+        settings.convertToArgs(*this, cat);
+        hiddenCategories.insert(cat);
     }
 
     void printFlags(std::ostream & out) override
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 3dd03771619..0e3eb9b694b 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -13,8 +13,13 @@ struct CmdCopySigs : StorePathsCommand
 
     CmdCopySigs()
     {
-        mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1,
-            [&](Strings ss) { substituterUris.push_back(ss.front()); });
+        mkFlag()
+            .longName("substituter")
+            .shortName('s')
+            .labels({"store-uri"})
+            .description("use signatures from specified store")
+            .arity(1)
+            .handler([&](Strings ss) { substituterUris.push_back(ss.front()); });
     }
 
     std::string name() override
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 8facb4bef8a..18533e6066c 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -19,8 +19,13 @@ struct CmdVerify : StorePathsCommand
     {
         mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
         mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
-        mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1,
-            [&](Strings ss) { substituterUris.push_back(ss.front()); });
+        mkFlag()
+            .longName("substituter")
+            .shortName('s')
+            .labels({"store-uri"})
+            .description("use signatures from specified store")
+            .arity(1)
+            .handler([&](Strings ss) { substituterUris.push_back(ss.front()); });
         mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
     }
 

From 7f5b750b401e98e9e2a346552aba5bd2e0a9203f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 16:07:34 +0200
Subject: [PATCH 0386/2196] Don't run pre-build-hook if we don't have a
 derivation

This fixes a build failure on OS X when using Hydra or Nix 1.12's
build-remote (since they don't copy the derivation to the build
machine).
---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d5fe41d1b16..bdec30151b0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1950,7 +1950,7 @@ void DerivationGoal::startBuilder()
                 }
     }
 
-    if (settings.preBuildHook != "") {
+    if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
         printMsg(lvlChatty, format("executing pre-build hook ‘%1%’")
             % settings.preBuildHook);
         auto args = useChroot ? Strings({drvPath, chrootRootDir}) :

From 847f19a5f7a558252bbde9b4c70efa5f7fac1f4f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 16:44:43 +0200
Subject: [PATCH 0387/2196] Provide a builtin default for $NIX_SSL_CERT_FILE

This is mostly to ensure that when Nix is started on macOS via a
launchd service or sshd (for a remote build), it gets a certificate
bundle.
---
 misc/launchd/org.nixos.nix-daemon.plist.in |  5 -----
 src/libstore/download.cc                   |  7 ++++---
 src/libstore/globals.cc                    | 10 +++++++++-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index c5ef97ee9a3..66fcd155ee9 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -12,10 +12,5 @@
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>
     <string>/dev/null</string>
-    <key>EnvironmentVariables</key>
-    <dict>
-      <key>NIX_SSL_CERT_FILE</key>
-      <string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
-    </dict>
   </dict>
 </plist>
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 63e498f0603..33ab1f02782 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -221,9 +221,10 @@ struct CurlDownloader : public Downloader
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
-            if (request.verifyTLS)
-                curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
-            else {
+            if (request.verifyTLS) {
+                if (settings.caFile != "")
+                    curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
+            } else {
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0);
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 2aceed27051..935018132d3 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -41,7 +41,15 @@ Settings::Settings()
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
-    caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"));
+
+    caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
+    if (caFile == "") {
+        for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
+            if (pathExists(fn)) {
+                caFile = fn;
+                break;
+            }
+    }
 
     /* Backwards compatibility. */
     auto s = getEnv("NIX_REMOTE_SYSTEMS");

From 25230a17a9d0c22f97ed80b1a8f50566a4ff548d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 17:43:19 +0200
Subject: [PATCH 0388/2196] On macOS, don't use /var/folders for TMPDIR

This broke "nix-store --serve".
---
 src/libmain/shared.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d6c1c0c9cb4..915e7955014 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -138,6 +138,14 @@ void initNix()
     struct timeval tv;
     gettimeofday(&tv, 0);
     srandom(tv.tv_usec);
+
+    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
+       sshd). This breaks build users because they don't have access
+       to the TMPDIR, in particular in ‘nix-store --serve’. */
+#if __APPLE__
+    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/"))
+        unsetenv("TMPDIR");
+#endif
 }
 
 

From 177f3996e28967368791ba0e4ec036f3dbbb88d0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 18:34:48 +0200
Subject: [PATCH 0389/2196] Suppress spurious "killing process N: Operation not
 permitted" on macOS

---
 src/libutil/util.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 16f4b232e6c..6bf4b3d9180 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -724,8 +724,15 @@ int Pid::kill()
     /* Send the requested signal to the child.  If it has its own
        process group, send the signal to every process in the child
        process group (which hopefully includes *all* its children). */
-    if (::kill(separatePG ? -pid : pid, killSignal) != 0)
-        printError((SysError(format("killing process %1%") % pid).msg()));
+    if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
+        /* On BSDs, killing a process group will return EPERM if all
+           processes in the group are zombies (or something like
+           that). So try to detect and ignore that situation. */
+#if __FreeBSD__ || __APPLE__
+        if (errno != EPERM || ::kill(pid, 0) != 0)
+#endif
+            printError((SysError("killing process %d", pid).msg()));
+    }
 
     return wait();
 }

From 88b291ffc4aed550d3136a44580ba5f5d66dd922 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Jun 2017 11:41:03 +0200
Subject: [PATCH 0390/2196] canonicalisePathMetaData(): Ignore security.selinux
 attribute

Untested, hopefully fixes #1406.
---
 src/libstore/local-store.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ee36428af03..aa985ee53d9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -421,10 +421,14 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
             throw SysError("querying extended attributes of ‘%s’", path);
 
-        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1)))
+        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
+            /* Ignore SELinux security labels since these cannot be
+               removed even by root. */
+            if (eaName == "security.selinux") continue;
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
                 throw SysError("removing extended attribute ‘%s’ from ‘%s’", eaName, path);
-    }
+        }
+     }
 #endif
 
     /* Fail if the file is not owned by the build user.  This prevents

From 38b7d55af1402e2323ed22a03d6d97b16dec52ad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Jun 2017 13:45:38 +0200
Subject: [PATCH 0391/2196] Remove redundant debug line

---
 src/libstore/s3-binary-cache-store.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 24545529601..fb36dbc7be7 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -273,8 +273,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         std::function<void(std::exception_ptr exc)> failure) override
     {
         sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
-            debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
-
             stats.get++;
 
             auto res = s3Helper.getObject(bucketName, path);

From 1dcadadf745442e96db29eb652ed4e535b6352d6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 13:56:38 +0200
Subject: [PATCH 0392/2196] Add 1.11.10 release notes

(cherry picked from commit 0fb60e4e0f66cc42c7c274acfcf00b51f6c829c4)
---
 doc/manual/release-notes/release-notes.xml |  1 +
 doc/manual/release-notes/rl-1.11.10.xml    | 31 ++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 doc/manual/release-notes/rl-1.11.10.xml

diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index 8c2deb39418..c4b14bc5499 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -13,6 +13,7 @@
 -->
 
 <xi:include href="rl-1.12.xml" />
+<xi:include href="rl-1.11.10.xml" />
 <xi:include href="rl-1.11.xml" />
 <xi:include href="rl-1.10.xml" />
 <xi:include href="rl-1.9.xml" />
diff --git a/doc/manual/release-notes/rl-1.11.10.xml b/doc/manual/release-notes/rl-1.11.10.xml
new file mode 100644
index 00000000000..13cb497d921
--- /dev/null
+++ b/doc/manual/release-notes/rl-1.11.10.xml
@@ -0,0 +1,31 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-1.11.10">
+
+<title>Release 1.11.10 (2017-06-12)</title>
+
+<para>This release fixes a security bug in Nix’s “build user” build
+isolation mechanism. Previously, Nix builders had the ability to
+create setuid binaries owned by a <literal>nixbld</literal>
+user. Such a binary could then be used by an attacker to assume a
+<literal>nixbld</literal> identity and interfere with subsequent
+builds running under the same UID.</para>
+
+<para>To prevent this issue, Nix now disallows builders to create
+setuid and setgid binaries. On Linux, this is done using a seccomp BPF
+filter. Note that this imposes a small performance penalty (e.g. 1%
+when building GNU Hello). Using seccomp, we now also prevent the
+creation of extended attributes and POSIX ACLs since these cannot be
+represented in the NAR format and (in the case of POSIX ACLs) allow
+bypassing regular Nix store permissions. On OS X, the restriction is
+implemented using the existing sandbox mechanism, which now uses a
+minimal “allow all except the creation of setuid/setgid binaries”
+profile when regular sandboxing is disabled. On other platforms, the
+“build user” mechanism is now disabled.</para>
+
+<para>Thanks go to Linus Heckemann for discovering and reporting this
+bug.</para>
+
+</section>

From a10951de08117dd2f9e7117fdd6fa61a7b4e2b72 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Jun 2017 14:04:52 +0200
Subject: [PATCH 0393/2196] OS X -> macOS

(cherry picked from commit c20641ce569dc1fdeaeaa147b0292f258667f53b)
---
 doc/manual/release-notes/rl-1.11.10.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-1.11.10.xml b/doc/manual/release-notes/rl-1.11.10.xml
index 13cb497d921..415388b3e2d 100644
--- a/doc/manual/release-notes/rl-1.11.10.xml
+++ b/doc/manual/release-notes/rl-1.11.10.xml
@@ -19,7 +19,7 @@ filter. Note that this imposes a small performance penalty (e.g. 1%
 when building GNU Hello). Using seccomp, we now also prevent the
 creation of extended attributes and POSIX ACLs since these cannot be
 represented in the NAR format and (in the case of POSIX ACLs) allow
-bypassing regular Nix store permissions. On OS X, the restriction is
+bypassing regular Nix store permissions. On macOS, the restriction is
 implemented using the existing sandbox mechanism, which now uses a
 minimal “allow all except the creation of setuid/setgid binaries”
 profile when regular sandboxing is disabled. On other platforms, the

From b5bdfdef7320d74a98af1e53d64371e0cef2dbbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 14:19:21 +0200
Subject: [PATCH 0394/2196] macOS: Remove flags

In particular, UF_IMMUTABLE (uchg) needs to be cleared to allow the
path to be garbage-collected or optimised.

See https://github.com/NixOS/nixpkgs/issues/25819.
+       the file from being garbage-collected.
---
 src/libstore/local-store.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index aa985ee53d9..74c74e672e2 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -400,6 +400,16 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
 {
     checkInterrupt();
 
+#if __APPLE__
+    /* Remove flags, in particular UF_IMMUTABLE which would prevent
+       the file from being garbage-collected. FIXME: Use
+       setattrlist() to remove other attributes as well. */
+    if (lchflags(path.c_str(), 0)) {
+        if (errno != ENOTSUP)
+            throw SysError(format("clearing flags of path ‘%1%’") % path);
+    }
+#endif
+
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path ‘%1%’") % path);

From 1888f7889b107ecbca7ad47f3a1132c32a17d227 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 14:24:48 +0200
Subject: [PATCH 0395/2196] macOS: Ugly hack to make the tests succeed

Sandboxes cannot be nested, so if Nix's build runs inside a sandbox,
it cannot use a sandbox itself. I don't see a clean way to detect
whether we're in a sandbox, so use a test-specific hack.

https://github.com/NixOS/nix/issues/1413
---
 src/libstore/build.cc | 5 ++---
 tests/common.sh.in    | 4 ++++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index bdec30151b0..c34083d2e2f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2621,7 +2621,7 @@ void DerivationGoal::runChild()
             ;
         }
 #if __APPLE__
-        else {
+        else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") {
             /* This has to appear before import statements. */
             std::string sandboxProfile = "(version 1)\n";
 
@@ -2736,13 +2736,12 @@ void DerivationGoal::runChild()
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
             args.push_back(drv->builder);
         }
-#else
+#endif
         else {
             builder = drv->builder.c_str();
             string builderBasename = baseNameOf(drv->builder);
             args.push_back(builderBasename);
         }
-#endif
 
         for (auto & i : drv->args)
             args.push_back(rewriteStrings(i, inputRewrites));
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 4565a490adf..6c3804a257a 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -15,6 +15,10 @@ export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
 export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
+if [[ -n $NIX_STORE ]]; then
+    export _NIX_TEST_NO_SANDBOX=1
+fi
+export _NIX_IN_TEST=$TEST_ROOT/shared
 export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home

From 00aa7c6705c073aab8b24ae945ea9a09d5d256aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 17:01:01 +0200
Subject: [PATCH 0396/2196] Show aws-sdk-cpp log messages

---
 src/libstore/s3-binary-cache-store.cc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index fb36dbc7be7..39d98cd61e7 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -12,6 +12,8 @@
 #include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
+#include <aws/core/utils/logging/FormattedLogSystem.h>
+#include <aws/core/utils/logging/LogMacros.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetBucketLocationRequest.h>
@@ -41,6 +43,16 @@ R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
     return outcome.GetResultWithOwnership();
 }
 
+class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
+{
+    using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;
+
+    void ProcessFormattedStatement(Aws::String && statement) override
+    {
+        debug("AWS: %s", chomp(statement));
+    }
+};
+
 static void initAWS()
 {
     static std::once_flag flag;
@@ -51,6 +63,16 @@ static void initAWS()
            shared.cc), so don't let aws-sdk-cpp override it. */
         options.cryptoOptions.initAndCleanupOpenSSL = false;
 
+        if (verbosity >= lvlDebug) {
+            options.loggingOptions.logLevel =
+                verbosity == lvlDebug
+                ? Aws::Utils::Logging::LogLevel::Debug
+                : Aws::Utils::Logging::LogLevel::Trace;
+            options.loggingOptions.logger_create_fn = [options]() {
+                return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
+            };
+        }
+
         Aws::InitAPI(options);
     });
 }

From 1c969611ba962a860744b2718fa6f989e7be5165 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 18:13:32 +0200
Subject: [PATCH 0397/2196] Suppress "will retry in N ms" for non-retriable
 errors

Newer versions of aws-sdk-cpp call CalculateDelayBeforeNextRetry()
even for non-retriable errors (like NoSuchKey) whih causes log spam in
hydra-queue-runner.
---
 src/libstore/s3-binary-cache-store.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 39d98cd61e7..f57227f024a 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -86,12 +86,13 @@ S3Helper::S3Helper(const string & region)
 /* Log AWS retries. */
 class RetryStrategy : public Aws::Client::DefaultRetryStrategy
 {
-    long CalculateDelayBeforeNextRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
+    bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
     {
-        auto res = Aws::Client::DefaultRetryStrategy::CalculateDelayBeforeNextRetry(error, attemptedRetries);
-        printError("AWS error '%s' (%s), will retry in %d ms",
-            error.GetExceptionName(), error.GetMessage(), res);
-        return res;
+        auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
+        if (retry)
+            printError("AWS error '%s' (%s), will retry in %d ms",
+                error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
+        return retry;
     }
 };
 

From b33621d4253addc4a0cdf1fb7a4082109aaafa78 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 18:15:23 +0200
Subject: [PATCH 0398/2196] Handle S3Errors::RESOURCE_NOT_FOUND from
 aws-sdk-cpp

This is returned by recent versions. Also handle NO_SUCH_KEY even
though the library doesn't actually return that at the moment.
---
 src/libstore/s3-binary-cache-store.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f57227f024a..8029cd8cca0 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -233,8 +233,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
         if (!res.IsSuccess()) {
             auto & error = res.GetError();
-            if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
-                && error.GetMessage().find("404") != std::string::npos)
+            if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
+                || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
+                || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
+                    && error.GetMessage().find("404") != std::string::npos))
                 return false;
             throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage());
         }

From 82a0d614cf17b2997ba5f895c1db4c2cde41e88d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 18:39:57 +0200
Subject: [PATCH 0399/2196] Support creating S3 caches in other regions than
 us-east-1

---
 src/libstore/s3-binary-cache-store.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 8029cd8cca0..06622210cbf 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -187,14 +187,20 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
                     throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
 
+                printInfo("creating S3 bucket ‘%s’...", bucketName);
+
+                // Stupid S3 bucket locations.
+                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
+                if (s3Helper.config->region != "us-east-1")
+                    bucketConfig.SetLocationConstraint(
+                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
+                            s3Helper.config->region));
+
                 checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
                     s3Helper.client->CreateBucket(
                         Aws::S3::Model::CreateBucketRequest()
                         .WithBucket(bucketName)
-                        .WithCreateBucketConfiguration(
-                            Aws::S3::Model::CreateBucketConfiguration()
-                            /* .WithLocationConstraint(
-                               Aws::S3::Model::BucketLocationConstraint::US) */ )));
+                        .WithCreateBucketConfiguration(bucketConfig)));
             }
 
             BinaryCacheStore::init();

From a1355917ecf75e1f1f37101505a351743d590121 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2017 18:40:22 +0200
Subject: [PATCH 0400/2196] Disable use of virtual hosting in aws-sdk-cpp

Recently aws-sdk-cpp quietly switched to using S3 virtual host URIs
(https://github.com/aws/aws-sdk-cpp/commit/69d9c53882), i.e. it sends
requests to http://<bucket>.<region>.s3.amazonaws.com rather than
http://<region>.s3.amazonaws.com/<bucket>. However this interacts
badly with curl connection reuse. For example, if we do the following:

1) Check whether a bucket exists using GetBucketLocation.
2) If it doesn't, create it using CreateBucket.
3) Do operations on the bucket.

then 3) will fail for a minute or so with a NoSuchBucket exception,
presumably because the server being hit is a fallback for cases when
buckets don't exist.

Disabling the use of virtual hosts ensures that 3) succeeds
immediately. (I don't know what S3's consistency guarantees are for
bucket creation, but in practice buckets appear to be available
immediately.)
---
 src/libstore/s3-binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 06622210cbf..145a8191c55 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -79,7 +79,7 @@ static void initAWS()
 
 S3Helper::S3Helper(const string & region)
     : config(makeConfig(region))
-    , client(make_ref<Aws::S3::S3Client>(*config))
+    , client(make_ref<Aws::S3::S3Client>(*config, true, false))
 {
 }
 

From 04ed11a978bf70e04042df95f8b125c2977e070d Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 19 Jun 2017 14:21:06 -0400
Subject: [PATCH 0401/2196] Let hydra choose an alternate list of systems

---
 release.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/release.nix b/release.nix
index aa2ee7290e6..af7611fbe60 100644
--- a/release.nix
+++ b/release.nix
@@ -1,15 +1,13 @@
 { nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
 , nixpkgs ? { outPath = <nixpkgs>; revCount = 1234; shortRev = "abcdef"; }
 , officialRelease ? false
+, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
 
 let
 
   pkgs = import <nixpkgs> {};
 
-  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
-
-
   jobs = rec {
 
 

From c7346a275c4cdcb59b3961241ddc52b79452d716 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Jun 2017 12:11:22 +0200
Subject: [PATCH 0402/2196] Restore thunks on any exception

There's no reason to restrict this to Error exceptions. This shouldn't
matter to #1407 since the repl doesn't catch non-Error exceptions
anyway, but you never know...
---
 src/libexpr/eval-inline.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 0748fbd3f3e..8cc50e56135 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -33,7 +33,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
             v.type = tBlackhole;
             //checkInterrupt();
             expr->eval(*this, *env, v);
-        } catch (Error & e) {
+        } catch (...) {
             v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = expr;

From 596b0e0a045eb953b5ed3328d5ae8eb636e31373 Mon Sep 17 00:00:00 2001
From: David McFarland <corngood@gmail.com>
Date: Tue, 20 Jun 2017 09:10:10 -0300
Subject: [PATCH 0403/2196] Call SetDllDirectory("") after sqlite3 init on
 cygwin

Cygwin sqlite3 is patched to call SetDllDirectory("/usr/bin") on init, which
affects the current process and is inherited by child processes.  It causes
DLLs to be loaded from /usr/bin/ before $PATH, which breaks all sorts of
things.  A typical failures would be header/lib version mismatches (e.g.
openssl when running checkPhase on openssh).  We'll just set it back to the
default value.

Note that this is a problem with the cygwin version of sqlite3 (currently
3.18.0).  nixpkgs doesn't have the problematic patch.
---
 src/libstore/local-store.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 74c74e672e2..c76294dcccb 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -30,6 +30,10 @@
 #include <sys/xattr.h>
 #endif
 
+#ifdef __CYGWIN__
+#include <windows.h>
+#endif
+
 #include <sqlite3.h>
 
 
@@ -281,6 +285,16 @@ void LocalStore::openDB(State & state, bool create)
             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
         throw Error(format("cannot open Nix database ‘%1%’") % dbPath);
 
+#ifdef __CYGWIN__
+    /* The cygwin version of sqlite3 has a patch which calls
+       SetDllDirectory("/usr/bin") on init. It was intended to fix extension
+       loading, which we don't use, and the effect of SetDllDirectory is
+       inherited by child processes, and causes libraries to be loaded from
+       /usr/bin instead of $PATH. This breaks quite a few things (e.g.
+       checkPhase on openssh), so we set it back to default behaviour. */
+    SetDllDirectoryW(L"");
+#endif
+
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
         throwSQLiteError(db, "setting timeout");
 

From b591536e935a431b7f0a7a917ee04d62bd8b81d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sat, 24 Jun 2017 02:17:45 +0200
Subject: [PATCH 0404/2196] Fix potential crash/wrong result two hashes of
 unequal length are compared

---
 src/libutil/hash.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index fa1bb5d9718..e6f8aa85c6d 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -54,6 +54,8 @@ bool Hash::operator != (const Hash & h2) const
 
 bool Hash::operator < (const Hash & h) const
 {
+    if (hashSize < h.hashSize) return true;
+    if (hashSize > h.hashSize) return false;
     for (unsigned int i = 0; i < hashSize; i++) {
         if (hash[i] < h.hash[i]) return true;
         if (hash[i] > h.hash[i]) return false;

From 90da34e421607ad6c40f3dea08709ae89db7a7e1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Jun 2017 16:38:02 +0200
Subject: [PATCH 0405/2196] processGraph(): Call getEdges in parallel

---
 src/libutil/thread-pool.hh | 75 ++++++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index b64dc52d473..361a9d33a73 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -70,50 +70,69 @@ void processGraph(
     struct Graph {
         std::set<T> left;
         std::map<T, std::set<T>> refs, rrefs;
-        std::function<void(T)> wrap;
     };
 
-    ref<Sync<Graph>> graph_ = make_ref<Sync<Graph>>();
+    Sync<Graph> graph_(Graph{nodes, {}, {}});
 
-    auto wrapWork = [&pool, graph_, processNode](const T & node) {
+    std::function<void(const T &)> worker;
+
+    worker = [&](const T & node) {
+
+        {
+            auto graph(graph_.lock());
+            auto i = graph->refs.find(node);
+            if (i == graph->refs.end())
+                goto getRefs;
+            goto doWork;
+        }
+
+    getRefs:
+        {
+            auto refs = getEdges(node);
+            refs.erase(node);
+
+            {
+                auto graph(graph_.lock());
+                for (auto & ref : refs)
+                    if (graph->left.count(ref)) {
+                        graph->refs[node].insert(ref);
+                        graph->rrefs[ref].insert(node);
+                    }
+                if (graph->refs[node].empty())
+                    goto doWork;
+            }
+        }
+
+        return;
+
+    doWork:
         processNode(node);
 
-        /* Enqueue work for all nodes that were waiting on this one. */
+        /* Enqueue work for all nodes that were waiting on this one
+           and have no unprocessed dependencies. */
         {
-            auto graph(graph_->lock());
-            graph->left.erase(node);
+            auto graph(graph_.lock());
             for (auto & rref : graph->rrefs[node]) {
                 auto & refs(graph->refs[rref]);
                 auto i = refs.find(node);
                 assert(i != refs.end());
                 refs.erase(i);
                 if (refs.empty())
-                    pool.enqueue(std::bind(graph->wrap, rref));
+                    pool.enqueue(std::bind(worker, rref));
             }
+            graph->left.erase(node);
+            graph->refs.erase(node);
+            graph->rrefs.erase(node);
         }
     };
 
-    {
-        auto graph(graph_->lock());
-        graph->left = nodes;
-        graph->wrap = wrapWork;
-    }
-
-    /* Build the dependency graph; enqueue all nodes with no
-       dependencies. */
-    for (auto & node : nodes) {
-        auto refs = getEdges(node);
-        {
-            auto graph(graph_->lock());
-            for (auto & ref : refs)
-                if (ref != node && graph->left.count(ref)) {
-                    graph->refs[node].insert(ref);
-                    graph->rrefs[ref].insert(node);
-                }
-            if (graph->refs[node].empty())
-                pool.enqueue(std::bind(graph->wrap, node));
-        }
-    }
+    for (auto & node : nodes)
+        pool.enqueue(std::bind(worker, std::ref(node)));
+
+    pool.process();
+
+    if (!graph_.lock()->left.empty())
+        throw Error("graph processing incomplete (cyclic reference?)");
 }
 
 }

From fcca702a96a8ca0e73f6d035052c30121776aeba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Jun 2017 18:11:01 +0200
Subject: [PATCH 0406/2196] Replace a few bool flags with enums

Functions like copyClosure() had 3 bool arguments, which creates a
severe risk of mixing up arguments.

Also, implement copyClosure() using copyPaths().
---
 src/build-remote/build-remote.cc         |  4 +-
 src/libexpr/eval.cc                      |  1 +
 src/libexpr/eval.hh                      |  3 +-
 src/libstore/binary-cache-store.cc       | 10 +--
 src/libstore/binary-cache-store.hh       |  6 +-
 src/libstore/build.cc                    | 18 ++---
 src/libstore/derivations.cc              |  2 +-
 src/libstore/derivations.hh              |  3 +-
 src/libstore/download.cc                 |  4 +-
 src/libstore/export-import.cc            |  4 +-
 src/libstore/legacy-ssh-store.cc         |  9 +--
 src/libstore/local-store.cc              | 16 ++---
 src/libstore/local-store.hh              | 15 ++--
 src/libstore/remote-store.cc             | 12 ++--
 src/libstore/remote-store.hh             | 11 +--
 src/libstore/store-api.cc                | 91 +++++++++---------------
 src/libstore/store-api.hh                | 38 +++++++---
 src/nix-copy-closure/nix-copy-closure.cc |  6 +-
 src/nix-daemon/nix-daemon.cc             | 10 +--
 src/nix-env/nix-env.cc                   |  4 +-
 src/nix-instantiate/nix-instantiate.cc   |  4 +-
 src/nix-store/nix-store.cc               |  8 +--
 22 files changed, 138 insertions(+), 141 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 7ffbdca7c0f..8719959f0b6 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -201,7 +201,7 @@ int main (int argc, char * * argv)
             printError("somebody is hogging the upload lock for ‘%s’, continuing...");
         alarm(0);
         signal(SIGALRM, old);
-        copyPaths(store, ref<Store>(sshStore), inputs, false, true);
+        copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
         uploadLock = -1;
 
         BasicDerivation drv(readDerivation(drvPath));
@@ -219,7 +219,7 @@ int main (int argc, char * * argv)
 
         if (!missing.empty()) {
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, false, true);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs);
         }
 
         return;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0cdce602d7b..ca4c9a373a3 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -293,6 +293,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sWrong(symbols.create("wrong"))
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
     , sBuilder(symbols.create("builder"))
+    , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 46d5a1cc866..1e32db1e86b 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -14,6 +14,7 @@ namespace nix {
 
 class Store;
 class EvalState;
+enum RepairFlag : bool;
 
 
 typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v);
@@ -73,7 +74,7 @@ public:
 
     /* If set, force copying files to the Nix store even if they
        already exist there. */
-    bool repair = false;
+    RepairFlag repair;
 
     /* If set, don't allow access to files outside of the Nix search
        path or to environment variables. */
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 46c5aa21b2e..8ce5f5bbc7c 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -134,7 +134,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
 }
 
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     if (!repair && isValidPath(info.path)) return;
 
@@ -328,7 +328,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
 }
 
 Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
+    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -349,13 +349,13 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     ValidPathInfo info;
     info.path = makeFixedOutputPath(recursive, h, name);
 
-    addToStore(info, sink.s, repair, false, 0);
+    addToStore(info, sink.s, repair, CheckSigs, nullptr);
 
     return info.path;
 }
 
 Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, bool repair)
+    const PathSet & references, RepairFlag repair)
 {
     ValidPathInfo info;
     info.path = computeStorePathForText(name, s, references);
@@ -364,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, sink.s, repair, false, 0);
+        addToStore(info, sink.s, repair, CheckSigs, nullptr);
     }
 
     return info.path;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 87d4aa43838..bf5a56ab4dc 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -85,15 +85,15 @@ public:
     bool wantMassQuery() override { return wantMassQuery_; }
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        bool repair, bool dontCheckSigs,
+        RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
-        PathFilter & filter, bool repair) override;
+        PathFilter & filter, RepairFlag repair) override;
 
     Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair) override;
+        const PathSet & references, RepairFlag repair) override;
 
     void narFromPath(const Path & path, Sink & sink) override;
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index c34083d2e2f..6c740d99c58 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -262,7 +262,7 @@ class Worker
     GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath,
         const BasicDerivation & drv, BuildMode buildMode = bmNormal);
-    GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
+    GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);
@@ -1087,7 +1087,7 @@ void DerivationGoal::haveDerivation()
        them. */
     if (settings.useSubstitutes && drv->substitutesAllowed())
         for (auto & i : invalidOutputs)
-            addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair));
+            addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
@@ -1195,7 +1195,7 @@ void DerivationGoal::repairClosure()
         printError(format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath);
         Path drvPath2 = outputsToDrv[i];
         if (drvPath2 == "")
-            addWaitee(worker.makeSubstitutionGoal(i, true));
+            addWaitee(worker.makeSubstitutionGoal(i, Repair));
         else
             addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair));
     }
@@ -3291,7 +3291,7 @@ class SubstitutionGoal : public Goal
     std::promise<void> promise;
 
     /* Whether to try to repair a valid path. */
-    bool repair;
+    RepairFlag repair;
 
     /* Location where we're downloading the substitute.  Differs from
        storePath when doing a repair. */
@@ -3301,7 +3301,7 @@ class SubstitutionGoal : public Goal
     GoalState state;
 
 public:
-    SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
+    SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair);
     ~SubstitutionGoal();
 
     void timedOut() override { abort(); };
@@ -3337,7 +3337,7 @@ class SubstitutionGoal : public Goal
 };
 
 
-SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair)
+SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair)
     : Goal(worker)
     , hasSubstitute(false)
     , repair(repair)
@@ -3600,7 +3600,7 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drv
 }
 
 
-GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair)
+GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair)
 {
     GoalPtr goal = substitutionGoals[path].lock();
     if (!goal) {
@@ -3953,7 +3953,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
         if (isDerivation(i2.first))
             goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode));
         else
-            goals.insert(worker.makeSubstitutionGoal(i, buildMode));
+            goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
     }
 
     worker.run(goals);
@@ -4011,7 +4011,7 @@ void LocalStore::ensurePath(const Path & path)
 void LocalStore::repairPath(const Path & path)
 {
     Worker worker(*this);
-    GoalPtr goal = worker.makeSubstitutionGoal(path, true);
+    GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
     Goals goals = {goal};
 
     worker.run(goals);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 0c6ceb9f674..bb7b8fe628a 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -71,7 +71,7 @@ bool BasicDerivation::canBuildLocally() const
 
 
 Path writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, bool repair)
+    const Derivation & drv, const string & name, RepairFlag repair)
 {
     PathSet references;
     references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 9717a81e469..7b97730d3bf 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "hash.hh"
+#include "store-api.hh"
 
 #include <map>
 
@@ -85,7 +86,7 @@ class Store;
 
 /* Write a derivation to the Nix store, and return its path. */
 Path writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, bool repair = false);
+    const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Path & drvPath);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 33ab1f02782..4f3bf2d14f1 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -631,7 +631,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                 info.narHash = hashString(htSHA256, *sink.s);
                 info.narSize = sink.s->size();
                 info.ca = makeFixedOutputCA(false, hash);
-                store->addToStore(info, sink.s, false, true);
+                store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
                 storePath = info.path;
             }
 
@@ -660,7 +660,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
             runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"});
-            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false);
+            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
         storePath = unpackedStorePath;
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 6e8bc692cdf..1b3a43df32d 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -61,7 +61,7 @@ void Store::exportPath(const Path & path, Sink & sink)
     hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
 }
 
-Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, bool dontCheckSigs)
+Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
 {
     Paths res;
     while (true) {
@@ -95,7 +95,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, tee.source.data, false, dontCheckSigs, accessor);
+        addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
 
         res.push_back(info.path);
     }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index e09932e3d18..a84f85c1b95 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
     }
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        bool repair, bool dontCheckSigs,
+        RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override
     {
         debug("adding path ‘%s’ to remote host ‘%s’", info.path, host);
@@ -168,11 +168,11 @@ struct LegacySSHStore : public Store
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
-        PathFilter & filter, bool repair) override
+        PathFilter & filter, RepairFlag repair) override
     { unsupported(); }
 
     Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair) override
+        const PathSet & references, RepairFlag repair) override
     { unsupported(); }
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
@@ -249,7 +249,8 @@ struct LegacySSHStore : public Store
         out.insert(res.begin(), res.end());
     }
 
-    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override
+    PathSet queryValidPaths(const PathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override
     {
         auto conn(connections->get());
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c76294dcccb..a7a94a8b9e8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -718,7 +718,7 @@ bool LocalStore::isValidPathUncached(const Path & path)
 }
 
 
-PathSet LocalStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
+PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
     PathSet res;
     for (auto & i : paths)
@@ -961,7 +961,7 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 
 
 void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     assert(info.narHash);
 
@@ -974,7 +974,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         throw Error("size mismatch importing path ‘%s’; expected %s, got %s",
             info.path, info.narSize, nar->size());
 
-    if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys))
+    if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys))
         throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path);
 
     addTempRoot(info.path);
@@ -1012,7 +1012,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
 
 
 Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
-    bool recursive, HashType hashAlgo, bool repair)
+    bool recursive, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
@@ -1070,7 +1070,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 
 
 Path LocalStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
+    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
 
@@ -1088,7 +1088,7 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath,
 
 
 Path LocalStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, bool repair)
+    const PathSet & references, RepairFlag repair)
 {
     auto hash = hashString(htSHA256, s);
     auto dstPath = makeTextPath(name, hash, references);
@@ -1170,7 +1170,7 @@ void LocalStore::invalidatePathChecked(const Path & path)
 }
 
 
-bool LocalStore::verifyStore(bool checkContents, bool repair)
+bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 {
     printError(format("reading the Nix store..."));
 
@@ -1255,7 +1255,7 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
 
 
 void LocalStore::verifyPath(const Path & path, const PathSet & store,
-    PathSet & done, PathSet & validPaths, bool repair, bool & errors)
+    PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors)
 {
     checkInterrupt();
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index f2c40e96464..551c6b506fb 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -98,7 +98,8 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override;
+    PathSet queryValidPaths(const PathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
     PathSet queryAllValidPaths() override;
 
@@ -122,22 +123,22 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        bool repair, bool dontCheckSigs,
+        RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
-        PathFilter & filter, bool repair) override;
+        PathFilter & filter, RepairFlag repair) override;
 
     /* Like addToStore(), but the contents of the path are contained
        in `dump', which is either a NAR serialisation (if recursive ==
        true) or simply the contents of a regular file (if recursive ==
        false). */
     Path addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false);
+        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair);
 
     Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair) override;
+        const PathSet & references, RepairFlag repair) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
@@ -174,7 +175,7 @@ public:
     /* Optimise a single store path. */
     void optimisePath(const Path & path);
 
-    bool verifyStore(bool checkContents, bool repair) override;
+    bool verifyStore(bool checkContents, RepairFlag repair) override;
 
     /* Register the validity of a path, i.e., that `path' exists, that
        the paths referenced by it exists, and in the case of an output
@@ -212,7 +213,7 @@ private:
     void invalidatePathChecked(const Path & path);
 
     void verifyPath(const Path & path, const PathSet & store,
-        PathSet & done, PathSet & validPaths, bool repair, bool & errors);
+        PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors);
 
     void updatePathInfo(State & state, const ValidPathInfo & info);
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index be8819bbc00..7337e406d2e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -185,7 +185,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
 }
 
 
-PathSet RemoteStore::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
+PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
     auto conn(connections->get());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
@@ -357,7 +357,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 
 
 void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(connections->get());
 
@@ -390,7 +390,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
                  << info.path << info.deriver << printHash(info.narHash)
                  << info.references << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
-                 << repair << dontCheckSigs;
+                 << repair << !checkSigs;
         conn->to(*nar);
         conn->processStderr();
     }
@@ -398,7 +398,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
 
 
 Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
+    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
@@ -434,7 +434,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
 
 
 Path RemoteStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, bool repair)
+    const PathSet & references, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
@@ -570,7 +570,7 @@ void RemoteStore::optimiseStore()
 }
 
 
-bool RemoteStore::verifyStore(bool checkContents, bool repair)
+bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 {
     auto conn(connections->get());
     conn->to << wopVerifyStore << checkContents << repair;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ed430e4cabb..e370e4797d2 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -31,7 +31,8 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths, bool maybeSubstitute = false) override;
+    PathSet queryValidPaths(const PathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
     PathSet queryAllValidPaths() override;
 
@@ -55,15 +56,15 @@ public:
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        bool repair, bool dontCheckSigs,
+        RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, bool repair = false) override;
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
 
     Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair = false) override;
+        const PathSet & references, RepairFlag repair) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
@@ -84,7 +85,7 @@ public:
 
     void optimiseStore() override;
 
-    bool verifyStore(bool checkContents, bool repair) override;
+    bool verifyStore(bool checkContents, RepairFlag repair) override;
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 76ed9942256..39b9466162f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -378,7 +378,7 @@ void Store::queryPathInfo(const Path & storePath,
 }
 
 
-PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
+PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
     struct State
     {
@@ -537,14 +537,14 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 
 
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
-    const Path & storePath, bool repair, bool dontCheckSigs)
+    const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
     auto info = srcStore->queryPathInfo(storePath);
 
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
-    if (!info->narHash && dontCheckSigs) {
+    if (!info->narHash && !checkSigs) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
@@ -561,33 +561,47 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     assert(info->narHash);
 
-    dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
+    dstStore->addToStore(*info, sink.s, repair, checkSigs);
 }
 
 
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const PathSet & storePaths, bool repair, bool dontCheckSigs)
+void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
+    RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
-    PathSet closure;
+    PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
+
+    PathSet missing;
     for (auto & path : storePaths)
-        srcStore->computeFSClosure(path, closure);
+        if (!valid.count(path)) missing.insert(path);
 
-    // FIXME: use copyStorePaths()
+    ThreadPool pool;
 
-    PathSet valid = dstStore->queryValidPaths(closure);
+    processGraph<Path>(pool,
+        PathSet(missing.begin(), missing.end()),
 
-    if (valid.size() == closure.size()) return;
+        [&](const Path & storePath) {
+            if (dstStore->isValidPath(storePath)) return PathSet();
+            return srcStore->queryPathInfo(storePath)->references;
+        },
 
-    Paths sorted = srcStore->topoSortPaths(closure);
+        [&](const Path & storePath) {
+            checkInterrupt();
 
-    Paths missing;
-    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
-        if (!valid.count(*i)) missing.push_back(*i);
+            if (!dstStore->isValidPath(storePath)) {
+                printError("copying ‘%s’...", storePath);
+                copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+            }
+        });
+}
 
-    printMsg(lvlDebug, format("copying %1% missing paths") % missing.size());
 
-    for (auto & i : missing)
-        copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs);
+void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
+    const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
+    SubstituteFlag substitute)
+{
+    PathSet closure;
+    srcStore->computeFSClosure({storePaths}, closure);
+    copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
 }
 
 
@@ -812,45 +826,4 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
-    bool substitute, bool dontCheckSigs)
-{
-    PathSet valid = to->queryValidPaths(storePaths, substitute);
-
-    PathSet missing;
-    for (auto & path : storePaths)
-        if (!valid.count(path)) missing.insert(path);
-
-    std::string copiedLabel = "copied";
-
-    //logger->setExpected(copiedLabel, missing.size());
-
-    ThreadPool pool;
-
-    processGraph<Path>(pool,
-        PathSet(missing.begin(), missing.end()),
-
-        [&](const Path & storePath) {
-            if (to->isValidPath(storePath)) return PathSet();
-            return from->queryPathInfo(storePath)->references;
-        },
-
-        [&](const Path & storePath) {
-            checkInterrupt();
-
-            if (!to->isValidPath(storePath)) {
-                //Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
-
-                copyStorePath(from, to, storePath, false, dontCheckSigs);
-
-                //logger->incProgress(copiedLabel);
-            } else
-                ;
-                //logger->incExpected(copiedLabel, -1);
-        });
-
-    pool.process();
-}
-
-
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 929c95a0f2f..c625a363033 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -32,6 +32,11 @@ class Store;
 class JSONPlaceholder;
 
 
+enum RepairFlag : bool { NoRepair = false, Repair = true };
+enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
+enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
+
+
 /* Size of the hash part of store paths, in base-32 characters. */
 const size_t storePathHashLen = 32; // i.e. 160 bits
 
@@ -332,7 +337,7 @@ public:
     /* Query which of the given paths is valid. Optionally, try to
        substitute missing paths. */
     virtual PathSet queryValidPaths(const PathSet & paths,
-        bool maybeSubstitute = false);
+        SubstituteFlag maybeSubstitute = NoSubstitute);
 
     /* Query the set of all valid paths. Note that for some store
        backends, the name part of store paths may be omitted
@@ -392,7 +397,7 @@ public:
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        bool repair = false, bool dontCheckSigs = false,
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
         std::shared_ptr<FSAccessor> accessor = 0) = 0;
 
     /* Copy the contents of a path to the store and register the
@@ -401,12 +406,12 @@ public:
        libutil/archive.hh). */
     virtual Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, bool repair = false) = 0;
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */
     virtual Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, bool repair = false) = 0;
+        const PathSet & references, RepairFlag repair = NoRepair) = 0;
 
     /* Write a NAR dump of a store path. */
     virtual void narFromPath(const Path & path, Sink & sink) = 0;
@@ -496,7 +501,7 @@ public:
 
     /* Check the integrity of the Nix store.  Returns true if errors
        remain. */
-    virtual bool verifyStore(bool checkContents, bool repair) { return false; };
+    virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; };
 
     /* Return an object to access files in the Nix store. */
     virtual ref<FSAccessor> getFSAccessor() = 0;
@@ -548,7 +553,7 @@ public:
        preloaded into the specified FS accessor to speed up subsequent
        access. */
     Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
-        bool dontCheckSigs = false);
+        CheckSigsFlag checkSigs = CheckSigs);
 
     struct Stats
     {
@@ -650,12 +655,26 @@ void checkStoreName(const string & name);
 
 /* Copy a path from one store to another. */
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
-    const Path & storePath, bool repair = false, bool dontCheckSigs = false);
+    const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
+
+
+/* Copy store paths from one store to another. The paths may be copied
+   in parallel. They are copied in a topologically sorted order
+   (i.e. if A is a reference of B, then A is copied before B), but
+   the set of store paths is not automatically closed; use
+   copyClosure() for that. */
+void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
+    RepairFlag repair = NoRepair,
+    CheckSigsFlag checkSigs = CheckSigs,
+    SubstituteFlag substitute = NoSubstitute);
 
 
 /* Copy the closure of the specified paths from one store to another. */
 void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const PathSet & storePaths, bool repair = false, bool dontCheckSigs = false);
+    const PathSet & storePaths,
+    RepairFlag repair = NoRepair,
+    CheckSigsFlag checkSigs = CheckSigs,
+    SubstituteFlag substitute = NoSubstitute);
 
 
 /* Remove the temporary roots file for this process.  Any temporary
@@ -694,9 +713,6 @@ ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"),
     const Store::Params & extraParams = Store::Params());
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths,
-    bool substitute = false, bool dontCheckSigs = false);
-
 enum StoreType {
     tDaemon,
     tLocal,
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index dc324abcb3b..0c69bd41356 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -12,7 +12,7 @@ int main(int argc, char ** argv)
         auto toMode = true;
         auto includeOutputs = false;
         auto dryRun = false;
-        auto useSubstitutes = false;
+        auto useSubstitutes = NoSubstitute;
         std::string sshHost;
         PathSet storePaths;
 
@@ -36,7 +36,7 @@ int main(int argc, char ** argv)
             else if (*arg == "--dry-run")
                 dryRun = true;
             else if (*arg == "--use-substitutes" || *arg == "-s")
-                useSubstitutes = true;
+                useSubstitutes = Substitute;
             else if (sshHost.empty())
                 sshHost = *arg;
             else
@@ -58,6 +58,6 @@ int main(int argc, char ** argv)
         PathSet closure;
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
-        copyPaths(from, to, closure, useSubstitutes, true);
+        copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
     });
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 44127635ded..c9c16776634 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -304,7 +304,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         string s = readString(from);
         PathSet refs = readStorePaths<PathSet>(*store, from);
         startWork();
-        Path path = store->addTextToStore(suffix, s, refs, false);
+        Path path = store->addTextToStore(suffix, s, refs, NoRepair);
         stopWork();
         to << path;
         break;
@@ -324,7 +324,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopImportPaths: {
         startWork();
         TunnelSource source(from);
-        Paths paths = store->importPaths(source, 0, trusted);
+        Paths paths = store->importPaths(source, nullptr,
+            trusted ? NoCheckSigs : CheckSigs);
         stopWork();
         to << paths;
         break;
@@ -576,7 +577,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         startWork();
         if (repair && !trusted)
             throw Error("you are not privileged to repair paths");
-        bool errors = store->verifyStore(checkContents, repair);
+        bool errors = store->verifyStore(checkContents, (RepairFlag) repair);
         stopWork();
         to << errors;
         break;
@@ -623,7 +624,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         parseDump(tee, tee.source);
 
         startWork();
-        store->addToStore(info, tee.source.data, repair, dontCheckSigs, nullptr);
+        store->addToStore(info, tee.source.data, (RepairFlag) repair,
+            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
         stopWork();
         break;
     }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 464bcee4a84..10100d6a601 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1310,7 +1310,7 @@ int main(int argc, char * * argv)
         Strings opFlags, opArgs, searchPath;
         std::map<string, string> autoArgs_;
         Operation op = 0;
-        bool repair = false;
+        RepairFlag repair = NoRepair;
         string file;
 
         Globals globals;
@@ -1372,7 +1372,7 @@ int main(int argc, char * * argv)
             else if (*arg == "--prebuilt-only" || *arg == "-b")
                 globals.prebuiltOnly = true;
             else if (*arg == "--repair")
-                repair = true;
+                repair = Repair;
             else if (*arg != "" && arg->at(0) == '-') {
                 opFlags.push_back(*arg);
                 /* FIXME: hacky */
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 25f0b1bd692..a5d12c1466f 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -108,7 +108,7 @@ int main(int argc, char * * argv)
         Strings attrPaths;
         bool wantsReadWrite = false;
         std::map<string, string> autoArgs_;
-        bool repair = false;
+        RepairFlag repair = NoRepair;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
@@ -146,7 +146,7 @@ int main(int argc, char * * argv)
             else if (*arg == "--strict")
                 strict = true;
             else if (*arg == "--repair")
-                repair = true;
+                repair = Repair;
             else if (*arg == "--dry-run")
                 settings.readOnlyMode = true;
             else if (*arg != "" && arg->at(0) == '-')
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 950222812e2..314c9423907 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -677,7 +677,7 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    Paths paths = store->importPaths(source, nullptr, true);
+    Paths paths = store->importPaths(source, nullptr, NoCheckSigs);
 
     for (auto & i : paths)
         cout << format("%1%\n") % i << std::flush;
@@ -702,11 +702,11 @@ static void opVerify(Strings opFlags, Strings opArgs)
         throw UsageError("no arguments expected");
 
     bool checkContents = false;
-    bool repair = false;
+    RepairFlag repair = NoRepair;
 
     for (auto & i : opFlags)
         if (i == "--check-contents") checkContents = true;
-        else if (i == "--repair") repair = true;
+        else if (i == "--repair") repair = Repair;
         else throw UsageError(format("unknown flag ‘%1%’") % i);
 
     if (store->verifyStore(checkContents, repair)) {
@@ -871,7 +871,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdImportPaths: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
-                store->importPaths(in, 0, true); // FIXME: should we skip sig checking?
+                store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
                 out << 1; // indicate success
                 break;
             }

From 0a5a867758285ee07334a44ede336ae75fd51c50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 3 Jul 2017 11:54:30 +0200
Subject: [PATCH 0407/2196] nix-shell: Respect --dry-run

Fixes #824.
---
 src/nix-build/nix-build.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 42d68fdfdd7..dc80dd6a583 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -390,6 +390,8 @@ int main(int argc, char ** argv)
                     maybePrintExecError(e);
                 }
 
+                if (dryRun) return;
+
                 // Set the environment.
                 auto env = getEnv();
 

From fe97c6989841460efca37f0f3b9b470c98229283 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 14:45:50 +0200
Subject: [PATCH 0408/2196] <nix/fetchurl.nix>: Support sha512 argument

---
 corepkgs/fetchurl.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix
index 62359433971..e135b947fdb 100644
--- a/corepkgs/fetchurl.nix
+++ b/corepkgs/fetchurl.nix
@@ -1,10 +1,10 @@
 { system ? builtins.currentSystem
 , url
-, md5 ? "", sha1 ? "", sha256 ? ""
+, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
-    if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+    if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
 , outputHashAlgo ?
-    if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+    if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
 , executable ? false
 , unpack ? false
 , name ? baseNameOf (toString url)

From c0015e87af70f539f24d2aa2bc224a9d8b84276b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 14:47:59 +0200
Subject: [PATCH 0409/2196] Support base-64 hashes

Also simplify the Hash API.

Fixes #1437.
---
 perl/lib/Nix/Store.xs                    |  20 ++-
 src/libexpr/primops.cc                   |   8 +-
 src/libstore/binary-cache-store.cc       |   2 +-
 src/libstore/build.cc                    |   2 +-
 src/libstore/derivations.cc              |   6 +-
 src/libstore/download.cc                 |   2 +-
 src/libstore/export-import.cc            |   2 +-
 src/libstore/gc.cc                       |   2 +-
 src/libstore/local-store.cc              |  26 +---
 src/libstore/nar-info-disk-cache.cc      |   4 +-
 src/libstore/nar-info.cc                 |   6 +-
 src/libstore/optimise-store.cc           |   4 +-
 src/libstore/remote-store.cc             |   4 +-
 src/libstore/store-api.cc                |  17 +--
 src/libutil/hash.cc                      | 185 +++++++++++------------
 src/libutil/hash.hh                      |  51 +++----
 src/nix-daemon/nix-daemon.cc             |   6 +-
 src/nix-prefetch-url/nix-prefetch-url.cc |   2 +-
 src/nix-store/nix-store.cc               |   8 +-
 src/nix/hash.cc                          |  40 ++---
 src/nix/verify.cc                        |   2 +-
 tests/fetchurl.sh                        |   9 ++
 tests/hash.sh                            |  10 +-
 23 files changed, 205 insertions(+), 213 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index aa14bfa6270..bbfb2934315 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -81,8 +81,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto hash = store()->queryPathInfo(path)->narHash;
-            string s = "sha256:" + printHash32(hash);
+            auto s = store()->queryPathInfo(path)->narHash.to_string();
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -108,7 +107,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0)));
-            string s = "sha256:" + (base32 ? printHash32(info->narHash) : printHash(info->narHash));
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -184,7 +183,7 @@ void importPaths(int fd, int dontCheckSigs)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(source, 0, dontCheckSigs);
+            store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -194,7 +193,7 @@ SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashPath(parseHashType(algo), path).first;
-            string s = base32 ? printHash32(h) : printHash(h);
+            auto s = h.to_string(base32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -205,7 +204,7 @@ SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashFile(parseHashType(algo), path);
-            string s = base32 ? printHash32(h) : printHash(h);
+            auto s = h.to_string(base32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -216,7 +215,7 @@ SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
             Hash h = hashString(parseHashType(algo), s);
-            string s = base32 ? printHash32(h) : printHash(h);
+            auto s = h.to_string(base32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -226,8 +225,8 @@ SV * hashString(char * algo, int base32, char * s)
 SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
-            Hash h = parseHash16or32(parseHashType(algo), s);
-            string s = toBase32 ? printHash32(h) : printHash(h);
+            Hash h(s, parseHashType(algo));
+            string s = h.to_string(toBase32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -286,8 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
 SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
-            HashType ht = parseHashType(algo);
-            Hash h = parseHash16or32(ht, hash);
+            Hash h(hash, parseHashType(algo));
             Path path = store()->makeFixedOutputPath(recursive, h, name);
             XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
         } catch (Error & e) {
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 99ffddaeb80..b753d84e2e6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -708,8 +708,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = parseHashType(outputHashAlgo);
         if (ht == htUnknown)
             throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName);
-        Hash h = parseHash16or32(ht, *outputHash);
-        outputHash = printHash(h);
+        Hash h(*outputHash, ht);
+        outputHash = h.to_string(Base16, false);
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
@@ -1701,7 +1701,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, printHash(hashString(ht, s)), context);
+    mkString(v, hashString(ht, s).to_string(Base16, false), context);
 }
 
 
@@ -1852,7 +1852,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
-                expectedHash = parseHash16or32(htSHA256, state.forceStringNoCtx(*attr.value, *attr.pos));
+                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8ce5f5bbc7c..8147345c2e1 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -239,7 +239,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
         % duration);
 
     /* Atomically write the NAR file. */
-    narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar"
+    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "br" ? ".br" :
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6c740d99c58..f40a8c5498e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3236,7 +3236,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
 Path DerivationGoal::addHashRewrite(const Path & path)
 {
     string h1 = string(path, worker.store.storeDir.size() + 1, 32);
-    string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32);
+    string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32);
     Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33);
     deletePath(p);
     assert(path.size() == p.size());
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index bb7b8fe628a..48c0837ffaa 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -23,7 +23,7 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
     if (hashType == htUnknown)
         throw Error(format("unknown hash algorithm ‘%1%’") % algo);
 
-    hash = parseHash(hashType, this->hash);
+    hash = Hash(this->hash, hashType);
 }
 
 
@@ -354,7 +354,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
             h = hashDerivationModulo(store, drv2);
             drvHashes[i.first] = h;
         }
-        inputs2[printHash(h)] = i.second;
+        inputs2[h.to_string(Base16, false)] = i.second;
     }
     drv.inputDrvs = inputs2;
 
@@ -437,7 +437,7 @@ Sink & operator << (Sink & out, const BasicDerivation & drv)
 std::string hashPlaceholder(const std::string & outputName)
 {
     // FIXME: memoize?
-    return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName));
+    return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
 }
 
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4f3bf2d14f1..15eb68c69ea 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -581,7 +581,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path cacheDir = getCacheDir() + "/nix/tarballs";
     createDirs(cacheDir);
 
-    string urlHash = printHash32(hashString(htSHA256, url));
+    string urlHash = hashString(htSHA256, url).to_string(Base32, false);
 
     Path dataFile = cacheDir + "/" + urlHash + ".info";
     Path fileLink = cacheDir + "/" + urlHash + "-file";
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 1b3a43df32d..2cbcedc6fb0 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -56,7 +56,7 @@ void Store::exportPath(const Path & path, Sink & sink)
     Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path
-            % printHash(info->narHash) % printHash(hash));
+            % info->narHash.to_string() % hash.to_string());
 
     hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
 }
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 3cdbb114a79..0cf9f87cac3 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -76,7 +76,7 @@ void LocalStore::syncWithGC()
 
 void LocalStore::addIndirectRoot(const Path & path)
 {
-    string hash = printHash32(hashString(htSHA1, path));
+    string hash = hashString(htSHA1, path).to_string(Base32, false);
     Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
         % stateDir % gcRootsDir % hash).str());
     makeSymlink(realRoot, path);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a7a94a8b9e8..7c41dfca7f3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -572,7 +572,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     state.stmtRegisterValidPath.use()
         (info.path)
-        ("sha256:" + printHash(info.narHash))
+        (info.narHash.to_string(Base16))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
         (info.deriver, info.deriver != "")
         (info.narSize, info.narSize != 0)
@@ -614,20 +614,6 @@ uint64_t LocalStore::addValidPath(State & state,
 }
 
 
-Hash parseHashField(const Path & path, const string & s)
-{
-    string::size_type colon = s.find(':');
-    if (colon == string::npos)
-        throw Error(format("corrupt hash ‘%1%’ in valid-path entry for ‘%2%’")
-            % s % path);
-    HashType ht = parseHashType(string(s, 0, colon));
-    if (ht == htUnknown)
-        throw Error(format("unknown hash type ‘%1%’ in valid-path entry for ‘%2%’")
-            % string(s, 0, colon) % path);
-    return parseHash(ht, string(s, colon + 1));
-}
-
-
 void LocalStore::queryPathInfoUncached(const Path & path,
     std::function<void(std::shared_ptr<ValidPathInfo>)> success,
     std::function<void(std::exception_ptr exc)> failure)
@@ -650,7 +636,11 @@ void LocalStore::queryPathInfoUncached(const Path & path,
 
             info->id = useQueryPathInfo.getInt(0);
 
-            info->narHash = parseHashField(path, useQueryPathInfo.getStr(1));
+            try {
+                info->narHash = Hash(useQueryPathInfo.getStr(1));
+            } catch (BadHash & e) {
+                throw Error("in valid-path entry for ‘%s’: %s", path, e.what());
+            }
 
             info->registrationTime = useQueryPathInfo.getInt(2);
 
@@ -685,7 +675,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
     state.stmtUpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
-        ("sha256:" + printHash(info.narHash))
+        (info.narHash.to_string(Base16))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (info.ca, !info.ca.empty())
@@ -1211,7 +1201,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError(format("path ‘%1%’ was modified! "
                             "expected hash ‘%2%’, got ‘%3%’")
-                        % i % printHash(info->narHash) % printHash(current.first));
+                        % i % info->narHash.to_string() % current.first.to_string());
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 180a936edb8..6e155e87780 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -203,9 +203,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             narInfo->url = queryNAR.getStr(3);
             narInfo->compression = queryNAR.getStr(4);
             if (!queryNAR.isNull(5))
-                narInfo->fileHash = parseHash(queryNAR.getStr(5));
+                narInfo->fileHash = Hash(queryNAR.getStr(5));
             narInfo->fileSize = queryNAR.getInt(6);
-            narInfo->narHash = parseHash(queryNAR.getStr(7));
+            narInfo->narHash = Hash(queryNAR.getStr(7));
             narInfo->narSize = queryNAR.getInt(8);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
                 narInfo->references.insert(cache.storeDir + "/" + r);
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index d1042c6de25..660f6a42a19 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -11,7 +11,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
 
     auto parseHashField = [&](const string & s) {
         try {
-            return parseHash(s);
+            return Hash(s);
         } catch (BadHash &) {
             corrupt();
             return Hash(); // never reached
@@ -90,10 +90,10 @@ std::string NarInfo::to_string() const
     assert(compression != "");
     res += "Compression: " + compression + "\n";
     assert(fileHash.type == htSHA256);
-    res += "FileHash: sha256:" + printHash32(fileHash) + "\n";
+    res += "FileHash: " + fileHash.to_string(Base32) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.type == htSHA256);
-    res += "NarHash: sha256:" + printHash32(narHash) + "\n";
+    res += "NarHash: " + narHash.to_string(Base32) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 56167c4dfae..adaf313131f 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -135,10 +135,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(htSHA256, path).first;
-    debug(format("‘%1%’ has hash ‘%2%’") % path % printHash(hash));
+    debug(format("‘%1%’ has hash ‘%2%’") % path % hash.to_string());
 
     /* Check if this is a known hash. */
-    Path linkPath = linksDir + "/" + printHash32(hash);
+    Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
 
  retry:
     if (!pathExists(linkPath)) {
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 7337e406d2e..ab726e79534 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -294,7 +294,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
         info->path = path;
         info->deriver = readString(conn->from);
         if (info->deriver != "") assertStorePath(info->deriver);
-        info->narHash = parseHash(htSHA256, readString(conn->from));
+        info->narHash = Hash(readString(conn->from), htSHA256);
         info->references = readStorePaths<PathSet>(*this, conn->from);
         conn->from >> info->registrationTime >> info->narSize;
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
@@ -387,7 +387,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
 
     else {
         conn->to << wopAddToStoreNar
-                 << info.path << info.deriver << printHash(info.narHash)
+                 << info.path << info.deriver << info.narHash.to_string(Base16, false)
                  << info.references << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
                  << repair << !checkSigs;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 39b9466162f..d7b784cfbc2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -176,13 +176,12 @@ Path Store::makeStorePath(const string & type,
     const Hash & hash, const string & name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":sha256:" + printHash(hash) + ":"
-        + storeDir + ":" + name;
+    string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
 
     checkStoreName(name);
 
     return storeDir + "/"
-        + printHash32(compressHash(hashString(htSHA256, s), 20))
+        + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false)
         + "-" + name;
 }
 
@@ -202,7 +201,7 @@ Path Store::makeFixedOutputPath(bool recursive,
         ? makeStorePath("source", hash, name)
         : makeStorePath("output:out", hashString(htSHA256,
                 "fixed:out:" + (recursive ? (string) "r:" : "") +
-                printHashType(hash.type) + ":" + printHash(hash) + ":"),
+                hash.to_string(Base16) + ":"),
             name);
 }
 
@@ -438,7 +437,7 @@ string Store::makeValidityRegistration(const PathSet & paths,
         auto info = queryPathInfo(i);
 
         if (showHash) {
-            s += printHash(info->narHash) + "\n";
+            s += info->narHash.to_string(Base16, false) + "\n";
             s += (format("%1%\n") % info->narSize).str();
         }
 
@@ -613,7 +612,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
     if (hashGiven) {
         string s;
         getline(str, s);
-        info.narHash = parseHash(htSHA256, s);
+        info.narHash = Hash(s, htSHA256);
         getline(str, s);
         if (!string2Int(s, info.narSize)) throw Error("number expected");
     }
@@ -648,7 +647,7 @@ std::string ValidPathInfo::fingerprint() const
             % path);
     return
         "1;" + path + ";"
-        + printHashType(narHash.type) + ":" + printHash32(narHash) + ";"
+        + narHash.to_string(Base32) + ";"
         + std::to_string(narSize) + ";"
         + concatStringsSep(",", references);
 }
@@ -667,7 +666,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     };
 
     if (hasPrefix(ca, "text:")) {
-        auto hash = parseHash(std::string(ca, 5));
+        Hash hash(std::string(ca, 5));
         if (store.makeTextPath(storePathToName(path), hash, references) == path)
             return true;
         else
@@ -676,7 +675,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
 
     else if (hasPrefix(ca, "fixed:")) {
         bool recursive = ca.compare(6, 2, "r:") == 0;
-        auto hash = parseHash(std::string(ca, recursive ? 8 : 6));
+        Hash hash(std::string(ca, recursive ? 8 : 6));
         if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
             return true;
         else
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index fa1bb5d9718..6b45ac859d5 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -16,17 +16,8 @@
 namespace nix {
 
 
-Hash::Hash()
+void Hash::init()
 {
-    type = htUnknown;
-    hashSize = 0;
-    memset(hash, 0, maxHashSize);
-}
-
-
-Hash::Hash(HashType type)
-{
-    this->type = type;
     if (type == htMD5) hashSize = md5HashSize;
     else if (type == htSHA1) hashSize = sha1HashSize;
     else if (type == htSHA256) hashSize = sha256HashSize;
@@ -62,16 +53,10 @@ bool Hash::operator < (const Hash & h) const
 }
 
 
-std::string Hash::to_string(bool base32) const
-{
-    return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this));
-}
-
-
 const string base16Chars = "0123456789abcdef";
 
 
-string printHash(const Hash & hash)
+static string printHash16(const Hash & hash)
 {
     char buf[hash.hashSize * 2];
     for (unsigned int i = 0; i < hash.hashSize; i++) {
@@ -82,42 +67,11 @@ string printHash(const Hash & hash)
 }
 
 
-Hash parseHash(const string & s)
-{
-    string::size_type colon = s.find(':');
-    if (colon == string::npos)
-        throw BadHash(format("invalid hash ‘%s’") % s);
-    string hts = string(s, 0, colon);
-    HashType ht = parseHashType(hts);
-    if (ht == htUnknown)
-        throw BadHash(format("unknown hash type ‘%s’") % hts);
-    return parseHash16or32(ht, string(s, colon + 1));
-}
-
-
-Hash parseHash(HashType ht, const string & s)
-{
-    Hash hash(ht);
-    if (s.length() != hash.hashSize * 2)
-        throw BadHash(format("invalid hash ‘%1%’") % s);
-    for (unsigned int i = 0; i < hash.hashSize; i++) {
-        string s2(s, i * 2, 2);
-        if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
-            throw BadHash(format("invalid hash ‘%1%’") % s);
-        istringstream_nocopy str(s2);
-        int n;
-        str >> std::hex >> n;
-        hash.hash[i] = n;
-    }
-    return hash;
-}
-
-
 // omitted: E O U T
 const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
 
 
-string printHash32(const Hash & hash)
+static string printHash32(const Hash & hash)
 {
     assert(hash.hashSize);
     size_t len = hash.base32Len();
@@ -142,66 +96,103 @@ string printHash32(const Hash & hash)
 
 string printHash16or32(const Hash & hash)
 {
-    return hash.type == htMD5 ? printHash(hash) : printHash32(hash);
+    return hash.to_string(hash.type == htMD5 ? Base16 : Base32);
 }
 
 
-Hash parseHash32(HashType ht, const string & s)
+std::string Hash::to_string(Base base, bool includeType) const
 {
-    Hash hash(ht);
-    size_t len = hash.base32Len();
-    assert(s.size() == len);
-
-    for (unsigned int n = 0; n < len; ++n) {
-        char c = s[len - n - 1];
-        unsigned char digit;
-        for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
-            if (base32Chars[digit] == c) break;
-        if (digit >= 32)
-            throw BadHash(format("invalid base-32 hash ‘%1%’") % s);
-        unsigned int b = n * 5;
-        unsigned int i = b / 8;
-        unsigned int j = b % 8;
-        hash.hash[i] |= digit << j;
-
-        if (i < hash.hashSize - 1) {
-            hash.hash[i + 1] |= digit >> (8 - j);
-        } else {
-            if (digit >> (8 - j))
-                throw BadHash(format("invalid base-32 hash ‘%1%’") % s);
-        }
+    std::string s;
+    if (includeType) {
+        s += printHashType(type);
+        s += ':';
     }
-
-    return hash;
+    switch (base) {
+    case Base16:
+        s += printHash16(*this);
+        break;
+    case Base32:
+        s += printHash32(*this);
+        break;
+    case Base64:
+        s += base64Encode(std::string((const char *) hash, hashSize));
+        break;
+    }
+    return s;
 }
 
 
-Hash parseHash16or32(HashType ht, const string & s)
+Hash::Hash(const std::string & s, HashType type)
+    : type(type)
 {
-    Hash hash(ht);
-    if (s.size() == hash.hashSize * 2)
-        /* hexadecimal representation */
-        hash = parseHash(ht, s);
-    else if (s.size() == hash.base32Len())
-        /* base-32 representation */
-        hash = parseHash32(ht, s);
-    else
-        throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’")
-            % s % printHashType(ht));
-    return hash;
-}
+    auto colon = s.find(':');
+
+    size_t pos = 0;
+
+    if (colon == string::npos) {
+        if (type == htUnknown)
+            throw BadHash("hash ‘%s’ does not include a type", s);
+    } else {
+        string hts = string(s, 0, colon);
+        this->type = parseHashType(hts);
+        if (this->type == htUnknown)
+            throw BadHash("unknown hash type ‘%s’", hts);
+        if (type != htUnknown && type != this->type)
+            throw BadHash("hash ‘%s’ should have type ‘%s’", s, printHashType(type));
+        pos = colon + 1;
+    }
 
+    init();
 
-bool isHash(const string & s)
-{
-    if (s.length() != 32) return false;
-    for (int i = 0; i < 32; i++) {
-        char c = s[i];
-        if (!((c >= '0' && c <= '9') ||
-              (c >= 'a' && c <= 'f')))
-            return false;
+    size_t size = s.size() - pos;
+
+    if (size == base16Len()) {
+
+        auto parseHexDigit = [&](char c) {
+            if (c >= '0' && c <= '9') return c - '0';
+            if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+            if (c >= 'a' && c <= 'f') return c - 'a' + 10;
+            throw BadHash("invalid base-16 hash ‘%s’", s);
+        };
+
+        for (unsigned int i = 0; i < hashSize; i++) {
+            hash[i] =
+                parseHexDigit(s[pos + i * 2]) << 4
+                | parseHexDigit(s[pos + i * 2 + 1]);
+        }
     }
-    return true;
+
+    else if (size == base32Len()) {
+
+        for (unsigned int n = 0; n < size; ++n) {
+            char c = s[pos + size - n - 1];
+            unsigned char digit;
+            for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
+                if (base32Chars[digit] == c) break;
+            if (digit >= 32)
+                throw BadHash("invalid base-32 hash ‘%s’", s);
+            unsigned int b = n * 5;
+            unsigned int i = b / 8;
+            unsigned int j = b % 8;
+            hash[i] |= digit << j;
+
+            if (i < hashSize - 1) {
+                hash[i + 1] |= digit >> (8 - j);
+            } else {
+                if (digit >> (8 - j))
+                    throw BadHash("invalid base-32 hash ‘%s’", s);
+            }
+        }
+    }
+
+    else if (size == base64Len()) {
+        auto d = base64Decode(std::string(s, pos));
+        assert(d.size() == hashSize);
+        memcpy(hash, d.data(), hashSize);
+    }
+
+    else
+        throw BadHash("hash ‘%s’ has wrong length for hash type ‘%s’", s, printHashType(type));
 }
 
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 02e213fc7b3..b8b432256c9 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -20,20 +20,30 @@ const int sha512HashSize = 64;
 
 extern const string base32Chars;
 
+enum Base : int { Base64, Base32, Base16 };
+
 
 struct Hash
 {
     static const unsigned int maxHashSize = 64;
-    unsigned int hashSize;
-    unsigned char hash[maxHashSize];
+    unsigned int hashSize = 0;
+    unsigned char hash[maxHashSize] = {};
 
-    HashType type;
+    HashType type = htUnknown;
 
     /* Create an unset hash object. */
-    Hash();
+    Hash() { };
 
     /* Create a zero-filled hash object. */
-    Hash(HashType type);
+    Hash(HashType type) : type(type) { init(); };
+
+    /* Initialize the hash from a string representation, in the format
+       "[<type>:]<base16|base32|base64>". If the ‘type’ argument is
+       htUnknown, then the hash type must be specified in the
+       string. */
+    Hash(const std::string & s, HashType type = htUnknown);
+
+    void init();
 
     /* Check whether a hash is set. */
     operator bool () const { return type != htUnknown; }
@@ -59,33 +69,22 @@ struct Hash
         return (hashSize * 8 - 1) / 5 + 1;
     }
 
-    std::string to_string(bool base32 = true) const;
-};
-
-
-/* Convert a hash to a hexadecimal representation. */
-string printHash(const Hash & hash);
-
-Hash parseHash(const string & s);
+    /* Returns the length of a base-64 representation of this hash. */
+    size_t base64Len() const
+    {
+        return ((4 * hashSize / 3) + 3) & ~3;
+    }
 
-/* Parse a hexadecimal representation of a hash code. */
-Hash parseHash(HashType ht, const string & s);
+    /* Return a string representation of the hash, in base-16, base-32
+       or base-64. By default, this is prefixed by the hash type
+       (e.g. "sha256:"). */
+    std::string to_string(Base base = Base32, bool includeType = true) const;
+};
 
-/* Convert a hash to a base-32 representation. */
-string printHash32(const Hash & hash);
 
 /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
 string printHash16or32(const Hash & hash);
 
-/* Parse a base-32 representation of a hash code. */
-Hash parseHash32(HashType ht, const string & s);
-
-/* Parse a base-16 or base-32 representation of a hash code. */
-Hash parseHash16or32(HashType ht, const string & s);
-
-/* Verify that the given string is a valid hash code. */
-bool isHash(const string & s);
-
 /* Compute the hash of the given string. */
 Hash hashString(HashType ht, const string & s);
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index c9c16776634..b029b92db15 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -216,7 +216,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         startWork();
         auto hash = store->queryPathInfo(path)->narHash;
         stopWork();
-        to << printHash(hash);
+        to << hash.to_string(Base16, false);
         break;
     }
 
@@ -550,7 +550,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         if (info) {
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
-            to << info->deriver << printHash(info->narHash) << info->references
+            to << info->deriver << info->narHash.to_string(Base16, false) << info->references
                << info->registrationTime << info->narSize;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
@@ -610,7 +610,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         from >> info.deriver;
         if (!info.deriver.empty())
             store->assertStorePath(info.deriver);
-        info.narHash = parseHash(htSHA256, readString(from));
+        info.narHash = Hash(readString(from), htSHA256);
         info.references = readStorePaths<PathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index b3b2fcac713..47e66eaa651 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -145,7 +145,7 @@ int main(int argc, char * * argv)
         Hash hash, expectedHash(ht);
         Path storePath;
         if (args.size() == 2) {
-            expectedHash = parseHash16or32(ht, args[1]);
+            expectedHash = Hash(args[1], ht);
             storePath = store->makeFixedOutputPath(unpack, expectedHash, name);
             if (store->isValidPath(storePath))
                 hash = expectedHash;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 314c9423907..6cea57a7671 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -212,7 +212,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
     string name = *i++;
 
     cout << format("%1%\n") %
-        store->makeFixedOutputPath(recursive, parseHash16or32(hashAlgo, hash), name);
+        store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name);
 }
 
 
@@ -380,9 +380,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
                         assert(info->narHash.type == htSHA256);
-                        cout << format("sha256:%1%\n") % printHash32(info->narHash);
+                        cout << fmt("%s\n", info->narHash.to_string(Base32));
                     } else if (query == qSize)
-                        cout << format("%1%\n") % info->narSize;
+                        cout << fmt("%d\n", info->narSize);
                 }
             }
             break;
@@ -734,7 +734,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         if (current.first != info->narHash) {
             printError(
                 format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’")
-                % path % printHash(info->narHash) % printHash(current.first));
+                % path % info->narHash.to_string() % current.first.to_string());
             status = 1;
         }
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 5dd891e8add..98de8897112 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -9,15 +9,16 @@ struct CmdHash : Command
 {
     enum Mode { mFile, mPath };
     Mode mode;
-    bool base32 = false;
+    Base base = Base16;
     bool truncate = false;
     HashType ht = htSHA512;
     Strings paths;
 
     CmdHash(Mode mode) : mode(mode)
     {
-        mkFlag(0, "base32", "print hash in base-32", &base32);
-        mkFlag(0, "base16", "print hash in base-16", &base32, false);
+        mkFlag(0, "base64", "print hash in base-64", &base, Base64);
+        mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
+        mkFlag(0, "base16", "print hash in base-16", &base, Base16);
         mkHashTypeFlag("type", &ht);
         expectArgs("paths", &paths);
     }
@@ -40,7 +41,7 @@ struct CmdHash : Command
             Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
-                (base32 ? printHash32(h) : printHash(h));
+                h.to_string(base, false);
         }
     }
 };
@@ -50,11 +51,11 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath));
 
 struct CmdToBase : Command
 {
-    bool toBase32;
+    Base base;
     HashType ht = htSHA512;
     Strings args;
 
-    CmdToBase(bool toBase32) : toBase32(toBase32)
+    CmdToBase(Base base) : base(base)
     {
         mkHashTypeFlag("type", &ht);
         expectArgs("strings", &args);
@@ -62,28 +63,29 @@ struct CmdToBase : Command
 
     std::string name() override
     {
-        return toBase32 ? "to-base32" : "to-base16";
+        return
+            base == Base16 ? "to-base16" :
+            base == Base32 ? "to-base32" :
+            "to-base64";
     }
 
     std::string description() override
     {
-        return toBase32
-            ? "convert a hash to base-32 representation"
-            : "convert a hash to base-16 representation";
+        return fmt("convert a hash to base-%d representation",
+            base == Base16 ? 16 :
+            base == Base32 ? 32 : 64);
     }
 
     void run() override
     {
-        for (auto s : args) {
-            Hash h = parseHash16or32(ht, s);
-            std::cout << format("%1%\n") %
-                (toBase32 ? printHash32(h) : printHash(h));
-        }
+        for (auto s : args)
+            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false));
     }
 };
 
-static RegisterCommand r3(make_ref<CmdToBase>(false));
-static RegisterCommand r4(make_ref<CmdToBase>(true));
+static RegisterCommand r3(make_ref<CmdToBase>(Base16));
+static RegisterCommand r4(make_ref<CmdToBase>(Base32));
+static RegisterCommand r5(make_ref<CmdToBase>(Base64));
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
@@ -121,14 +123,14 @@ static int compatNixHash(int argc, char * * argv)
     if (op == opHash) {
         CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath);
         cmd.ht = ht;
-        cmd.base32 = base32;
+        cmd.base = base32 ? Base32 : Base16;
         cmd.truncate = truncate;
         cmd.paths = ss;
         cmd.run();
     }
 
     else {
-        CmdToBase cmd(op == opTo32);
+        CmdToBase cmd(op == opTo32 ? Base32 : Base16);
         cmd.args = ss;
         cmd.ht = ht;
         cmd.run();
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 18533e6066c..973f60a74ff 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -94,7 +94,7 @@ struct CmdVerify : StorePathsCommand
                         corrupted = 1;
                         printError(
                             format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
-                            % info->path % printHash(info->narHash) % printHash(hash.first));
+                            % info->path % info->narHash.to_string() % hash.first.to_string());
                     }
 
                 }
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index b6fa3a27edd..808f460258d 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -9,6 +9,15 @@ outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh
 
 cmp $outPath fetchurl.sh
 
+# Now using a base-64 hash.
+clearStore
+
+hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
+
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
+
+cmp $outPath fetchurl.sh
+
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive
 mkdir -p $TEST_ROOT/archive
diff --git a/tests/hash.sh b/tests/hash.sh
index a95c68683f8..9f234bc635b 100644
--- a/tests/hash.sh
+++ b/tests/hash.sh
@@ -63,11 +63,15 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
 # Conversion.
 try3() {
+    h64=$(nix to-base64 --type "$1" "$2")
+    [ "$h64" = "$4" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
+    h16=$(nix to-base16 --type "$1" "$h64")
+    [ "$h16" = "$2" ]
 }
-try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0"
-try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
-try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0"
+try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
+try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
+try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="

From ad8b96f1f2c80bf3f91ceab4d955aa368b7c85d4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 15:38:23 +0200
Subject: [PATCH 0410/2196] Fix handling of expression installables with a / in
 them

---
 src/nix/installables.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index f23308b9bc3..9982ff75f4f 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -189,7 +189,10 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
 
     for (auto & s : ss) {
 
-        if (s.find("/") != std::string::npos) {
+        if (s.compare(0, 1, "(") == 0)
+            result.push_back(std::make_shared<InstallableExpr>(*this, s));
+
+        else if (s.find("/") != std::string::npos) {
 
             auto path = store->toStorePath(store->followLinksToStore(s));
 
@@ -201,9 +204,6 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
             }
         }
 
-        else if (s.compare(0, 1, "(") == 0)
-            result.push_back(std::make_shared<InstallableExpr>(*this, s));
-
         else if (s == "" || std::regex_match(s, attrPathRegex))
             result.push_back(std::make_shared<InstallableAttrPath>(*this, s));
 

From 6cf23c3e8fa12e6bf297ca87d1b47889bf115d52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 15:43:06 +0200
Subject: [PATCH 0411/2196] Add allow-new-privileges option
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This allows builds to call setuid binaries. This was previously
possible until we started using seccomp. Turns out that seccomp by
default disallows processes from acquiring new privileges. Generally,
any use of setuid binaries (except those created by the builder
itself) is by definition impure, but some people were relying on this
ability for certain tests.

Example:

  $ nix build '(with import <nixpkgs> {}; runCommand "foo" {} "/run/wrappers/bin/ping -c 1 8.8.8.8; exit 1")' --no-allow-new-privileges
  builder for ‘/nix/store/j0nd8kv85hd6r4kxgnwzvr0k65ykf6fv-foo.drv’ failed with exit code 1; last 2 log lines:
    cannot raise the capability into the Ambient set
    : Operation not permitted

  $ nix build '(with import <nixpkgs> {}; runCommand "foo" {} "/run/wrappers/bin/ping -c 1 8.8.8.8; exit 1")' --allow-new-privileges
  builder for ‘/nix/store/j0nd8kv85hd6r4kxgnwzvr0k65ykf6fv-foo.drv’ failed with exit code 1; last 6 log lines:
    PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
    64 bytes from 8.8.8.8: icmp_seq=1 ttl=46 time=15.2 ms

Fixes #1429.
---
 doc/manual/command-ref/conf-file.xml | 17 +++++++++++++++++
 src/libstore/build.cc                |  3 +++
 src/libstore/globals.hh              |  6 ++++++
 3 files changed, 26 insertions(+)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 616983bc7f0..cde32b35f5b 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -643,6 +643,23 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term>
+
+    <listitem><para>(Linux-specific.) By default, builders on Linux
+    cannot acquire new privileges by calling setuid/setgid programs or
+    programs that have file capabilities. For example, programs such
+    as <command>sudo</command> or <command>ping</command> will
+    fail. (Note that in sandbox builds, no such programs are available
+    unless you bind-mount them into the sandbox via the
+    <option>build-sandbox-paths</option> option.) You can allow the
+    use of such programs by enabling this option. This is impure and
+    usually undesirable, but may be useful in certain scenarios
+    (e.g. to spin up containers or set up userspace network interfaces
+    in tests).</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f40a8c5498e..355fb3b7dfe 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2340,6 +2340,9 @@ void setupSeccomp()
         seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
         throw SysError("unable to add seccomp rule");
 
+    if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
+        throw SysError("unable to set 'no new privileges' seccomp attribute");
+
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
 #endif
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index af37ec61d7a..c8d67b07110 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -321,6 +321,12 @@ public:
 
     Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
         "String appended to the user agent in HTTP requests."};
+
+#if __linux__
+    Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
+        "Whether builders can acquire new privileges by calling programs with "
+        "setuid/setgid bits or with file capabilities."};
+#endif
 };
 
 

From b7203e853e3b928e1a7fb081fce379f023e935bb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 16:26:48 +0200
Subject: [PATCH 0412/2196] getDefaultSubstituters(): Simplify initialisation

As shlevy pointed out, static variables in C++11 have thread-safe
initialisation.
---
 src/libstore/store-api.cc | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index d7b784cfbc2..db2e023ab0b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -795,33 +795,27 @@ static RegisterStoreImplementation regStore([](
 
 std::list<ref<Store>> getDefaultSubstituters()
 {
-    struct State {
-        bool done = false;
+    static auto stores([]() {
         std::list<ref<Store>> stores;
-    };
-    static Sync<State> state_;
-
-    auto state(state_.lock());
 
-    if (state->done) return state->stores;
+        StringSet done;
 
-    StringSet done;
-
-    auto addStore = [&](const std::string & uri) {
-        if (done.count(uri)) return;
-        done.insert(uri);
-        state->stores.push_back(openStore(uri));
-    };
+        auto addStore = [&](const std::string & uri) {
+            if (done.count(uri)) return;
+            done.insert(uri);
+            stores.push_back(openStore(uri));
+        };
 
-    for (auto uri : settings.substituters.get())
-        addStore(uri);
+        for (auto uri : settings.substituters.get())
+            addStore(uri);
 
-    for (auto uri : settings.extraSubstituters.get())
-        addStore(uri);
+        for (auto uri : settings.extraSubstituters.get())
+            addStore(uri);
 
-    state->done = true;
+        return stores;
+    } ());
 
-    return state->stores;
+    return stores;
 }
 
 

From 42c5774e78a9f1422dee9c35adb9c056aa994d3b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 16:34:53 +0200
Subject: [PATCH 0413/2196] Sort substituters by priority

Fixes #1438.
---
 src/libstore/binary-cache-store.hh | 2 ++
 src/libstore/store-api.cc          | 4 ++++
 src/libstore/store-api.hh          | 5 +++++
 3 files changed, 11 insertions(+)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index bf5a56ab4dc..f9c1c2cbe8a 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -123,6 +123,8 @@ public:
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
 
+    int getPriority() override { return priority; }
+
 };
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index db2e023ab0b..108e2d4ce9b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -812,6 +812,10 @@ std::list<ref<Store>> getDefaultSubstituters()
         for (auto uri : settings.extraSubstituters.get())
             addStore(uri);
 
+        stores.sort([](ref<Store> & a, ref<Store> & b) {
+            return a->getPriority() < b->getPriority();
+        });
+
         return stores;
     } ());
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c625a363033..cada37653e6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -590,6 +590,11 @@ public:
        a notion of connection. Otherwise this is a no-op. */
     virtual void connect() { };
 
+    /* Get the priority of the store, used to order substituters. In
+       particular, binary caches can specify a priority field in their
+       "nix-cache-info" file. Lower value means higher priority. */
+    virtual int getPriority() { return 0; }
+
 protected:
 
     Stats stats;

From a3dc1e65abe29f9d0528d3b5ea45812f4dcd63ed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 4 Jul 2017 19:00:51 +0200
Subject: [PATCH 0414/2196] Add X32 to the seccomp filter

Fixes #1432.
---
 src/libstore/build.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 355fb3b7dfe..60b0a531f42 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2317,6 +2317,10 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
+    if (settings.thisSystem == "x86_64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
+        throw SysError("unable to add X32 seccomp architecture");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,

From 60da5d2b8f0f8586520566155172c1a18e4f04cc Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Thu, 6 Jul 2017 16:37:19 +0200
Subject: [PATCH 0415/2196] Fix nix-instantiate manpage indentation

The second command variant is now its own cmdsynopsis, which ensures
it's not indented as was the case using sbrk.
---
 doc/manual/command-ref/nix-instantiate.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 3d03358bea3..fe077ff6560 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -43,7 +43,8 @@
       <arg choice='plain'><option>-E</option></arg>
     </group>
     <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
-    <sbr/>
+  </cmdsynopsis>
+  <cmdsynopsis>
     <command>nix-instantiate</command>
     <arg choice='plain'><option>--find-file</option></arg>
     <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>

From 01722b3d2c10090f53290e68b585c50d042f0adb Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Thu, 6 Jul 2017 17:56:15 +0200
Subject: [PATCH 0416/2196] Remove unused variable from test script

---
 tests/common.sh.in | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/common.sh.in b/tests/common.sh.in
index 6c3804a257a..a8d56707e27 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -1,7 +1,5 @@
 set -e
 
-datadir="@datadir@"
-
 export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then

From 72e80c59b5176eb08986247ec0f1978d32993364 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 6 Jul 2017 13:42:12 -0700
Subject: [PATCH 0417/2196] =?UTF-8?q?Don=E2=80=99t=20hardlink=20disallowed?=
 =?UTF-8?q?=20paths=20in=20OS=20X.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #1443
---
 src/libstore/optimise-store.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 56167c4dfae..9e651ebeaf7 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -9,6 +9,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>
+#include <regex>
 
 
 namespace nix {
@@ -96,6 +97,19 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path ‘%1%’") % path);
 
+#if __APPLE__
+    /* HFS/OS X has some undocumented security feature disabling hardlinking for
+       special files within .app dirs. *.app/Contents/PkgInfo and
+       *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
+       https://github.com/NixOS/nix/issues/1443 for more discussion. */
+
+    if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
+        std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$"))) {
+        debug(format("‘%1%’ is not allowed to be linked in OS X") % path);
+        return;
+    }
+#endif
+
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectoryIgnoringInodes(path, inodeHash);
         for (auto & i : names)

From ce3095e14165ab0079f2c24002ae4c810b1c728d Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 20:38:33 +0200
Subject: [PATCH 0418/2196] glossary: Fix word order

---
 doc/manual/glossary/glossary.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml
index d74940c90b3..e0636044cc2 100644
--- a/doc/manual/glossary/glossary.xml
+++ b/doc/manual/glossary/glossary.xml
@@ -54,7 +54,7 @@
 
   <glossdef><para>A substitute is a command invocation stored in the
   Nix database that describes how to build a store object, bypassing
-  normal the build mechanism (i.e., derivations).  Typically, the
+  the normal build mechanism (i.e., derivations).  Typically, the
   substitute builds the store object by downloading a pre-built
   version of the store object from some server.</para></glossdef>
 

From 68c626c6b07e8676f7235c21efb94a0055f5cf3d Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 20:37:16 +0200
Subject: [PATCH 0419/2196] man pages: Grouping for option alternatives

---
 doc/manual/command-ref/nix-copy-closure.xml |  6 ++--
 doc/manual/command-ref/opt-common-syn.xml   | 32 +++++++++++++++------
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml
index 97e261ae993..800e1db6ab0 100644
--- a/doc/manual/command-ref/nix-copy-closure.xml
+++ b/doc/manual/command-ref/nix-copy-closure.xml
@@ -27,8 +27,10 @@
     <arg><option>- -show-progress</option></arg>
     -->
     <arg><option>--include-outputs</option></arg>
-    <arg><option>--use-substitutes</option></arg>
-    <arg><option>-s</option></arg>
+    <group>
+      <arg choice='plain'><option>--use-substitutes</option></arg>
+      <arg choice='plain'><option>-s</option></arg>
+    </group>
     <arg><option>-v</option></arg>
     <arg choice='plain'>
       <replaceable>user@</replaceable><replaceable>machine</replaceable>
diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
index 5b793639395..3aff4e1b635 100644
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ b/doc/manual/command-ref/opt-common-syn.xml
@@ -2,10 +2,18 @@
   
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
-<arg rep='repeat'><option>--verbose</option></arg>
-<arg rep='repeat'><option>-v</option></arg>
-<arg><option>--no-build-output</option></arg>
-<arg><option>-Q</option></arg>
+<arg rep='repeat'>
+  <group choice='req'>
+    <arg choice='plain'><option>--verbose</option></arg>
+    <arg choice='plain'><option>-v</option></arg>
+  </group>
+</arg>
+<arg>
+  <group choice='plain'>
+    <arg choice='plain'><option>--no-build-output</option></arg>
+    <arg choice='plain'><option>-Q</option></arg>
+  </group>
+</arg>
 <arg>
   <group choice='req'>
     <arg choice='plain'><option>--max-jobs</option></arg>
@@ -25,10 +33,18 @@
   <option>--timeout</option>
   <replaceable>number</replaceable>
 </arg>
-<arg><option>--keep-going</option></arg>
-<arg><option>-k</option></arg>
-<arg><option>--keep-failed</option></arg>
-<arg><option>-K</option></arg>
+<arg>
+  <group choice='plain'>
+    <arg choice='plain'><option>--keep-going</option></arg>
+    <arg choice='plain'><option>-k</option></arg>
+  </group>
+</arg>
+<arg>
+  <group choice='plain'>
+    <arg choice='plain'><option>--keep-failed</option></arg>
+    <arg choice='plain'><option>-K</option></arg>
+  </group>
+</arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
 <arg><option>--show-trace</option></arg>

From d1643bdaa2f66760b5dce3e42953b433e7eb49bc Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:11:39 +0200
Subject: [PATCH 0420/2196] man pages: Argument for --max-jobs

---
 doc/manual/command-ref/opt-common.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 2aa41c4d438..18c664d0973 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -89,8 +89,8 @@
 </varlistentry>
 
 
-<varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option></term>
-  <term><option>-j</option></term>
+<varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
+<replaceable>number</replaceable></term>
 
   <listitem><para>Sets the maximum number of build jobs that Nix will
   perform in parallel to the specified number.  Specify

From 56a1f8f4998cb6aa4ff2c8b61ca36634af4b3759 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:14:45 +0200
Subject: [PATCH 0421/2196] man pages: Consistently separate alternatives by /

---
 doc/manual/command-ref/nix-env.xml    |  6 ++----
 doc/manual/command-ref/opt-common.xml | 12 ++++--------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 8462cf8a027..d4563ac4755 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -146,8 +146,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><option>--file</option></term>
-    <term><option>-f</option></term>
+  <varlistentry><term><option>--file</option> / <option>-f</option> <replaceable>path</replaceable></term>
 
     <listitem><para>Specifies the Nix expression (designated below as
     the <emphasis>active Nix expression</emphasis>) used by the
@@ -166,8 +165,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--profile</option></term>
-    <term><option>-p</option></term>
+  <varlistentry><term><option>--profile</option> / <option>-p</option> <replaceable>path</replaceable></term>
 
     <listitem><para>Specifies the profile to be used by those
     operations that operate on a profile (designated below as the
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 18c664d0973..a930b4a0d78 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -22,8 +22,7 @@
 </varlistentry>
 
 
-<varlistentry><term><option>--verbose</option></term>
-  <term><option>-v</option></term>
+<varlistentry><term><option>--verbose</option> / <option>-v</option></term>
 
   <listitem>
 
@@ -76,8 +75,7 @@
 </varlistentry>
 
 
-<varlistentry><term><option>--no-build-output</option></term>
-  <term><option>-Q</option></term>
+<varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard
   output and standard error is echoed to the Nix command's standard
@@ -143,8 +141,7 @@
 
 </varlistentry>
 
-<varlistentry><term><option>--keep-going</option></term>
-  <term><option>-k</option></term>
+<varlistentry><term><option>--keep-going</option> / <option>-k</option></term>
 
   <listitem><para>Keep going in case of failed builds, to the
   greatest extent possible.  That is, if building an input of some
@@ -156,8 +153,7 @@
 </varlistentry>
 
 
-<varlistentry><term><option>--keep-failed</option></term>
-  <term><option>-K</option></term>
+<varlistentry><term><option>--keep-failed</option> / <option>-K</option></term>
 
   <listitem><para>Specifies that in case of a build failure, the
   temporary directory (usually in <filename>/tmp</filename>) in which

From b1f5995a20b5511bce614d1718816088bc7f0ec0 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 20:37:55 +0200
Subject: [PATCH 0422/2196] man page (nix-instantiate): Remove non-existent
 nix-build argument -r

---
 doc/manual/command-ref/nix-store.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index fb017b741da..19c99841a0a 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -234,7 +234,7 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
 <para>To test whether a previously-built derivation is deterministic:
 
 <screen>
-$ nix-build -r '&lt;nixpkgs>' -A hello --check -K
+$ nix-build '&lt;nixpkgs>' -A hello --check -K
 </screen>
 
 </para>

From 8ad898b2cd2f5d7bba29da48f67658d3a04888db Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:51:22 +0200
Subject: [PATCH 0423/2196] man page (nix-instantiate): Add --json to synopsis,
 order variables

---
 doc/manual/command-ref/nix-instantiate.xml | 33 +++++++++++-----------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index fe077ff6560..87422895470 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -24,6 +24,7 @@
       <arg choice='plain'>
         <option>--eval</option>
         <arg><option>--strict</option></arg>
+        <arg><option>--json</option></arg>
         <arg><option>--xml</option></arg>
       </arg>
     </group>
@@ -116,13 +117,17 @@ input.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--xml</option></term>
+  <varlistentry><term><option>--strict</option></term>
 
-    <listitem><para>When used with <option>--eval</option>, print the resulting
-    value as an XML representation of the abstract syntax tree rather than as
-    an ATerm. The schema is the same as that used by the <link
-    linkend="builtin-toXML"><function>toXML</function> built-in</link>.
-    </para></listitem>
+    <listitem><para>When used with <option>--eval</option>,
+    recursively evaluate list elements and attributes.  Normally, such
+    sub-expressions are left unevaluated (since the Nix expression
+    language is lazy).</para>
+
+    <warning><para>This option can cause non-termination, because lazy
+    data structures can be infinitely large.</para></warning>
+
+    </listitem>
 
   </varlistentry>
 
@@ -134,17 +139,13 @@ input.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--strict</option></term>
-
-    <listitem><para>When used with <option>--eval</option>,
-    recursively evaluate list elements and attributes.  Normally, such
-    sub-expressions are left unevaluated (since the Nix expression
-    language is lazy).</para>
-
-    <warning><para>This option can cause non-termination, because lazy
-    data structures can be infinitely large.</para></warning>
+  <varlistentry><term><option>--xml</option></term>
 
-    </listitem>
+    <listitem><para>When used with <option>--eval</option>, print the resulting
+    value as an XML representation of the abstract syntax tree rather than as
+    an ATerm. The schema is the same as that used by the <link
+    linkend="builtin-toXML"><function>toXML</function> built-in</link>.
+    </para></listitem>
 
   </varlistentry>
 

From 772ef22c25f6dac969ce7f13586a08d1191f4a44 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:51:54 +0200
Subject: [PATCH 0424/2196] man page (nix-instantiate): -E is optional

---
 doc/manual/command-ref/nix-instantiate.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 87422895470..39c1282fcc3 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -39,7 +39,7 @@
     </arg>
     <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
     <arg><option>--indirect</option></arg>
-    <group choice='req'>
+    <group>
       <arg choice='plain'><option>--expr</option></arg>
       <arg choice='plain'><option>-E</option></arg>
     </group>

From 89771a882157c22ff90ab6b1f9e93ed464aa3177 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:52:31 +0200
Subject: [PATCH 0425/2196] man page (nix-prefetch-url): Add some missing
 options

---
 doc/manual/command-ref/nix-prefetch-url.xml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml
index 016d8863a94..8ef748c74be 100644
--- a/doc/manual/command-ref/nix-prefetch-url.xml
+++ b/doc/manual/command-ref/nix-prefetch-url.xml
@@ -19,14 +19,16 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-prefetch-url</command>
+    <arg><option>--version</option></arg>
     <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg>
     <arg><option>--print-path</option></arg>
+    <arg><option>--unpack</option></arg>
+    <arg><option>--name</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><replaceable>url</replaceable></arg>
     <arg><replaceable>hash</replaceable></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
 
-
 <refsection><title>Description</title>
 
 <para>The command <command>nix-prefetch-url</command> downloads the
@@ -91,7 +93,7 @@ downloaded file in the Nix store is also printed.</para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--name</option></term>
+  <varlistentry><term><option>--name</option> <replaceable>name</replaceable></term>
 
     <listitem><para>Override the name of the file in the Nix store. By
     default, this is

From c85e662004d6f2881dde92991d3a2352e612e8e3 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Fri, 7 Jul 2017 21:52:56 +0200
Subject: [PATCH 0426/2196] man page (nix-shell): Fix grouping of -p option

Not sure about the raw ellipsis.
---
 doc/manual/command-ref/nix-shell.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index c64c93ec3ac..078fa38494b 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -33,12 +33,12 @@
     <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
     <arg><option>--pure</option></arg>
     <group choice='req'>
-      <group choice='plain'>
-        <group>
+      <arg choice='plain'>
+        <group choice='req'>
           <arg choice='plain'><option>--packages</option></arg>
           <arg choice='plain'><option>-p</option></arg>
         </group>
-        <replaceable>packages</replaceable>
+        <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg>
       </group>
       <arg><replaceable>path</replaceable></arg>
     </group>
@@ -144,7 +144,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
-  <varlistentry><term><option>--packages</option> / <option>-p</option></term>
+  <varlistentry><term><option>--packages</option> / <option>-p</option> <replaceable>packages</replaceable>…</term>
 
     <listitem><para>Set up an environment in which the specified
     packages are present.  The command line arguments are interpreted

From 17bb00d378aea9ea7f7f27618bd736f2c47e3de2 Mon Sep 17 00:00:00 2001
From: Rhys <rimmington@gmail.com>
Date: Fri, 30 Jun 2017 14:14:50 +1000
Subject: [PATCH 0427/2196] Clearer error message when regex exceeds space
 limit

---
 src/libexpr/primops.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 99ffddaeb80..f240e795ee7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1734,8 +1734,13 @@ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value
                 mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str());
         }
 
-    } catch (std::regex_error &) {
-        throw EvalError("invalid regular expression ‘%s’, at %s", re, pos);
+    } catch (std::regex_error &e) {
+        if (e.code() == std::regex_constants::error_space) {
+          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+          throw EvalError("memory limit exceeded by regular expression ‘%s’, at %s", re, pos);
+        } else {
+          throw EvalError("invalid regular expression ‘%s’, at %s", re, pos);
+        }
     }
 }
 

From 30117fb35bde6ba27d7de5037272ea23c8dcebf9 Mon Sep 17 00:00:00 2001
From: Robert Vollmert <rob@vllmrt.net>
Date: Mon, 10 Jul 2017 14:36:55 +0200
Subject: [PATCH 0428/2196] fix buggy nix-shell man page

---
 doc/manual/command-ref/nix-shell.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 078fa38494b..62d026ac238 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -39,7 +39,7 @@
           <arg choice='plain'><option>-p</option></arg>
         </group>
         <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg>
-      </group>
+      </arg>
       <arg><replaceable>path</replaceable></arg>
     </group>
   </cmdsynopsis>

From 8e8caf7f3e535d4e397f422f6c0effd81f497305 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 11 Jul 2017 23:11:35 +0200
Subject: [PATCH 0429/2196] fetchTarball: Prevent concurrent downloads of the
 same file

Fixes #849.
---
 src/libstore/download.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 15eb68c69ea..ac9a2b8fede 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "s3.hh"
 #include "compression.hh"
+#include "pathlocks.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -586,6 +587,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path dataFile = cacheDir + "/" + urlHash + ".info";
     Path fileLink = cacheDir + "/" + urlHash + "-file";
 
+    PathLocks lock({fileLink}, fmt("waiting for lock on ‘%1%’...", fileLink));
+
     Path storePath;
 
     string expectedETag;
@@ -647,6 +650,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     if (unpack) {
         Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
+        PathLocks lock2({unpackedLink}, fmt("waiting for lock on ‘%1%’...", unpackedLink));
         Path unpackedStorePath;
         if (pathExists(unpackedLink)) {
             unpackedStorePath = readLink(unpackedLink);

From 2965d40612e16388ef2177d2a13d168848c6ca8a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 11 Jul 2017 23:20:01 +0200
Subject: [PATCH 0430/2196] replaceSymlink(): Handle the case where the
 temporary file already exists

Not really necessary anymore for #849, but still nice to have.
---
 src/libutil/util.cc | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6bf4b3d9180..d9f8011b6fb 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -496,12 +496,21 @@ void createSymlink(const Path & target, const Path & link)
 
 void replaceSymlink(const Path & target, const Path & link)
 {
-    Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link));
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
 
-    createSymlink(target, tmp);
+        try {
+            createSymlink(target, tmp);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
 
-    if (rename(tmp.c_str(), link.c_str()) != 0)
-        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link);
+        if (rename(tmp.c_str(), link.c_str()) != 0)
+            throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link);
+
+        break;
+    }
 }
 
 

From 0681f8c9070f3a40899d3040370e460c249a74bd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 12 Jul 2017 19:24:20 +0200
Subject: [PATCH 0431/2196] Shut up a memory leak warning

---
 src/libmain/stack.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index 57b6a197c0f..ab3baf62c3d 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -52,7 +52,8 @@ void detectStackOverflow()
        delivered when we're out of stack space. */
     stack_t stack;
     stack.ss_size = 4096 * 4 + MINSIGSTKSZ;
-    stack.ss_sp = new char[stack.ss_size];
+    static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size);
+    stack.ss_sp = stackBuf->data();
     if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
     stack.ss_flags = 0;
     if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");

From 38374a9d35765a1c0b78bfeb02e6f22fc8643e83 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 11:40:09 +0200
Subject: [PATCH 0432/2196] Tarball job: Include libseccomp on Linux only

---
 release.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index af7611fbe60..1348ef7b93f 100644
--- a/release.nix
+++ b/release.nix
@@ -28,8 +28,7 @@ let
             docbook5 docbook5_xsl
             autoconf-archive
             git
-            libseccomp
-          ];
+          ] ++ lib.optional stdenv.isLinux libseccomp;
 
         configureFlags = "--enable-gc";
 

From 112ff7833d4f3a233755b2fe856b2eb2b3723254 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 13:44:45 +0200
Subject: [PATCH 0433/2196] nix: Show help when no arguments are given

Fixes #1464.
---
 src/libutil/args.cc |  2 +-
 src/libutil/args.hh |  5 +++--
 src/nix/command.cc  |  2 +-
 src/nix/main.cc     | 15 +++++++++------
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index df7e040875d..0eed4945418 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -146,7 +146,7 @@ bool Args::processArgs(const Strings & args, bool finish)
         res = true;
     }
 
-    if (finish && !expectedArgs.empty())
+    if (finish && !expectedArgs.empty() && !expectedArgs.front().optional)
         throw UsageError("more arguments are required");
 
     return res;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index aa11373d5f1..ef8a7953e52 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -51,6 +51,7 @@ protected:
     {
         std::string label;
         size_t arity; // 0 = any
+        bool optional;
         std::function<void(Strings)> handler;
     };
 
@@ -165,7 +166,7 @@ public:
     /* Expect a string argument. */
     void expectArg(const std::string & label, string * dest)
     {
-        expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) {
+        expectedArgs.push_back(ExpectedArg{label, 1, false, [=](Strings ss) {
             *dest = ss.front();
         }});
     }
@@ -173,7 +174,7 @@ public:
     /* Expect 0 or more arguments. */
     void expectArgs(const std::string & label, Strings * dest)
     {
-        expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) {
+        expectedArgs.push_back(ExpectedArg{label, 0, false, [=](Strings ss) {
             *dest = ss;
         }});
     }
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 3c82e0df57f..96b685a5b2e 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -24,7 +24,7 @@ void Command::printHelp(const string & programName, std::ostream & out)
 MultiCommand::MultiCommand(const Commands & _commands)
     : commands(_commands)
 {
-    expectedArgs.push_back(ExpectedArg{"command", 1, [=](Strings ss) {
+    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](Strings ss) {
         assert(!command);
         auto i = commands.find(ss.front());
         if (i == commands.end())
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 88a602b8481..4b51c5ee14a 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -15,11 +15,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
-        mkFlag('h', "help", "show usage information", [=]() {
-            printHelp(programName, std::cout);
-            std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
-            throw Exit();
-        });
+        mkFlag('h', "help", "show usage information", [&]() { showHelpAndExit(); });
 
         mkFlag(0, "help-config", "show configuration options", [=]() {
             std::cout << "The following configuration options are available:\n\n";
@@ -47,6 +43,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             "Boolean settings can be overriden using ‘--<name>’ or ‘--no-<name>’. See ‘nix\n"
             "--help-config’ for a list of configuration settings.\n";
     }
+
+    void showHelpAndExit()
+    {
+        printHelp(programName, std::cout);
+        std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
+        throw Exit();
+    }
 };
 
 void mainWrapped(int argc, char * * argv)
@@ -68,7 +71,7 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
-    assert(args.command);
+    if (!args.command) args.showHelpAndExit();
 
     StartProgressBar bar;
 

From a0ad8ba12ee26edf0bf5f221b080a61c0d24a874 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 9 Jul 2017 12:38:32 -0400
Subject: [PATCH 0434/2196] Shellcheck the existing installer

---
 release.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 1348ef7b93f..86a7350d5ea 100644
--- a/release.nix
+++ b/release.nix
@@ -137,7 +137,7 @@ let
 
       runCommand "nix-binary-tarball-${version}"
         { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ];
-          buildInputs = [ perl ];
+          buildInputs = [ perl shellcheck ];
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
@@ -146,6 +146,9 @@ let
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
+
+          shellcheck -e SC1090 $TMPDIR/install
+
           chmod +x $TMPDIR/install
           dir=nix-${version}-${system}
           fn=$out/$dir.tar.bz2

From fb40d73e23383224fc541911da95a894bbbcc2a4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 9 Jul 2017 13:07:28 -0400
Subject: [PATCH 0435/2196] Switch to a fancy multi-user installer on Darwin

---
 release.nix                          |   7 +-
 scripts/install-darwin-multi-user.sh | 808 +++++++++++++++++++++++++++
 scripts/install-nix-from-closure.sh  |  24 +-
 3 files changed, 831 insertions(+), 8 deletions(-)
 create mode 100644 scripts/install-darwin-multi-user.sh

diff --git a/release.nix b/release.nix
index 86a7350d5ea..796591cc167 100644
--- a/release.nix
+++ b/release.nix
@@ -146,10 +146,15 @@ let
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
 
           shellcheck -e SC1090 $TMPDIR/install
+          shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
 
           chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/install-darwin-multi-user
           dir=nix-${version}-${system}
           fn=$out/$dir.tar.bz2
           mkdir -p $out/nix-support
@@ -161,7 +166,7 @@ let
             --transform "s,$TMPDIR/install,$dir/install," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/reginfo $storePaths
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo $storePaths
         '');
 
 
diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
new file mode 100644
index 00000000000..87d55b2f017
--- /dev/null
+++ b/scripts/install-darwin-multi-user.sh
@@ -0,0 +1,808 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Sourced from:
+# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh
+# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0
+# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh
+# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh
+# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a
+#
+# however tracking which bits came from which would be impossible.
+
+readonly ESC='\033[0m'
+readonly BOLD='\033[38;1m'
+readonly BLUE='\033[38;34m'
+readonly BLUE_UL='\033[38;4;34m'
+readonly GREEN='\033[38;32m'
+readonly GREEN_UL='\033[38;4;32m'
+readonly RED='\033[38;31m'
+readonly RED_UL='\033[38;4;31m'
+readonly YELLOW='\033[38;33m'
+readonly YELLOW_UL='\033[38;4;33m'
+
+readonly CORES=$(sysctl -n hw.ncpu)
+readonly NIX_USER_COUNT="$CORES"
+readonly NIX_BUILD_GROUP_ID="30000"
+readonly NIX_BUILD_GROUP_NAME="nixbld"
+readonly NIX_FIRST_BUILD_UID="30001"
+# Please don't change this. We don't support it, because the
+# default shell profile that comes with Nix doesn't support it.
+readonly NIX_ROOT="/nix"
+readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
+
+readonly PROFILE_TARGETS=("/etc/profile" "/etc/bashrc" "/etc/zshrc")
+readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
+readonly PROFILE_NIX_FILE_DIR="/etc/"
+readonly PROFILE_NIX_FILE_NAME="profile-nix.sh"
+readonly PROFILE_NIX_FILE="$PROFILE_NIX_FILE_DIR$PROFILE_NIX_FILE_NAME"
+
+
+
+readonly NIX_INSTALLED_NIX="@nix@"
+readonly NIX_INSTALLED_CACERT="@cacert@"
+readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
+
+readonly ROOT_HOME="/var/root"
+
+contactme() {
+    echo "We'd love to help if you need it."
+    echo ""
+    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
+    echo ""
+    echo "Or feel free to contact the team,"
+    echo " - on IRC #nixos on irc.freenode.net"
+    echo " - on twitter @nixos_org"
+}
+
+uninstall_directions() {
+    subheader "Uninstalling nix:"
+    local step=1
+    cat <<EOF
+$step. If $PLIST_DEST exists:
+
+  sudo launchctl unload $PLIST_DEST
+  sudo rm $PLIST_DEST
+
+EOF
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ]; then
+            step=$((step + 1))
+            cat <<EOF
+$step. If $profile_target$PROFILE_BACKUP_SUFFIX exists:
+
+  sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+(after this one, you may need to re-open any terminals that were
+opened while it existed.)
+
+EOF
+        fi
+    done
+    step=$((step + 1))
+
+    cat <<EOF
+$step. Delete the files Nix added to your system:
+
+  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
+
+and that is it.
+
+EOF
+
+}
+
+nix_user_for_core() {
+    printf "nixbld%d" "$1"
+}
+
+nix_uid_for_core() {
+    echo $((NIX_FIRST_BUILD_UID + $1 - 1))
+}
+
+dsclattr() {
+    /usr/bin/dscl . -read "$1" \
+        | awk "/$2/ { print \$2 }"
+}
+
+_textout() {
+    echo -en "$1"
+    shift
+    if [ "$*" = "" ]; then
+        cat
+    else
+        echo "$@"
+    fi
+    echo -en "$ESC"
+}
+
+header() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$BLUE" "$header"
+}
+
+warningheader() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$RED" "$header"
+}
+
+subheader() {
+    echo ""
+    _textout "$BLUE_UL" "$*"
+}
+
+row() {
+    printf "$BOLD%s$ESC:\t%s\n" "$1" "$2"
+}
+
+task() {
+    echo ""
+    ok "~~> $1"
+}
+
+bold() {
+    echo "$BOLD$*$ESC"
+}
+
+ok() {
+    _textout "$GREEN" "$@"
+}
+
+warning() {
+    warningheader "warning!"
+    cat
+    echo ""
+}
+
+failure() {
+    header "oh no!"
+    _textout "$RED" "$@"
+    echo ""
+    _textout "$RED" "$(contactme)"
+    trap finish_cleanup EXIT
+    exit 1
+}
+
+ui_confirm() {
+    _textout "$GREEN$GREEN_UL" "$1"
+
+    local prompt="[y/n] "
+    echo -n "$prompt"
+    while read -r y; do
+        if [ "$y" = "y" ]; then
+            echo ""
+            return 0
+        elif [ "$y" = "n" ]; then
+            echo ""
+            return 1
+        else
+            _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n"
+            echo -n "$prompt"
+        fi
+    done
+    echo ""
+    return 1
+}
+
+__sudo() {
+    local expl="$1"
+    local cmd="$2"
+    shift
+    header "sudo execution"
+
+    echo "I am executing:"
+    echo ""
+    echo "    $ sudo $cmd"
+    echo ""
+    echo "$expl"
+    echo ""
+
+    return 0
+}
+
+_sudo() {
+    local expl="$1"
+    shift
+    if __sudo "$expl" "$*"; then
+        sudo "$@"
+    fi
+}
+
+
+readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
+function finish_cleanup {
+    rm -rf "$SCRATCH"
+}
+
+function finish_fail {
+    finish_cleanup
+
+    failure <<EOF
+Jeeze, something went wrong. If you can take all the output and open
+an issue, we'd love to fix the problem so nobody else has this issue.
+
+:(
+EOF
+}
+trap finish_fail EXIT
+
+function finish_success {
+    finish_cleanup
+
+    ok "Alright! We're done!"
+    cat <<EOF
+
+Before Nix will work in your existing shells, you'll need to close
+them and open them again. Other than that, you should be ready to go.
+
+Try it! Open a new terminal, and type:
+
+  $ nix-shell -p figlet -p lolcat --run "echo 'nix rules' | figlet | lolcat"
+
+Thank you for using this installer. If you have any feedback, don't
+hesitate:
+
+$(contactme)
+EOF
+}
+
+
+validate_starting_assumptions() {
+    if [ "$(uname -s)" != "Darwin" ]; then
+        failure "This script is for use with macOS!"
+    fi
+
+    if [ $EUID -eq 0 ]; then
+        failure <<EOF
+Please do not run this script with root privileges. We will call sudo
+when we need to.
+EOF
+    fi
+
+    if type nix-env 2> /dev/null >&2; then
+        failure <<EOF
+Nix already appears to be installed, and this tool assumes it is
+_not_ yet installed.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ "${NIX_REMOTE:-}" != "" ]; then
+        failure <<EOF
+For some reason, \$NIX_REMOTE is set. It really should not be set
+before this installer runs, and it hints that Nix is currently
+installed. Please delete the old Nix installation and start again.
+
+Note: You might need to close your shell window and open a new shell
+to clear the variable.
+EOF
+    fi
+
+    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
+        failure <<EOF
+It looks like \$SSL_CERT_FILE is set to a path that used to be part of
+the old Nix installation. Please unset that variable and try again:
+
+  $ unset SSL_CERT_FILE
+
+EOF
+    fi
+
+    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
+        if [ -f "$file" ]; then
+            if grep -l ".nix-profile" "$file"; then
+                failure <<EOF
+I found a reference to a ".nix-profile" in $file.
+This has a high chance of breaking a new nix installation. It was most
+likely put there by a previous Nix installer.
+
+Please remove this reference and try running this again. You should
+also look for similar references in:
+
+ - ~/.bash_profile
+ - ~/.bash_login
+ - ~/.profile
+
+or other shell init files that you may have.
+EOF
+            fi
+        fi
+    done
+
+    if [ -d /nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ -d /etc/nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /etc/nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+        failure <<EOF
+When this script runs, it backs up the current $profile_target to
+$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
+
+Please follow these instructions to clean up the old backup file:
+
+1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
+in case.
+
+2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
+it has anything nix-related in it. If it does, something is probably
+quite wrong. Please open an issue or get in touch immediately.
+
+3. Take care to make sure that $profile_target doesn't look like it has
+anything nix-related in it. If it does, and $profile_target _did not_,
+run:
+
+  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+and try again.
+EOF
+        fi
+
+        if grep -qi "nix" "$profile_target"; then
+            failure <<EOF
+It looks like $profile_target already has some Nix configuration in
+there. There should be no reason to run this again. If you're having
+trouble, please open an issue.
+EOF
+        fi
+    done
+
+    warning <<EOF
+I strongly recommend deleting all the following files, which are
+relics of a previous installation. You can check by running (yourself):
+
+  $ sudo find / -name .nix-profile  -o -name .nix-defexpr -o -name .nix-channels -o -name '*nix-daemon.plist'
+
+In particular, the following files and directories MUST NOT EXIST!
+
+ - $ROOT_HOME/.nix-profile
+ - $ROOT_HOME/.nix-defexpr
+ - $ROOT_HOME/.nix-channels
+
+If some relics are found and you want to keep them, it might be okay.
+If you're not sure, don't hesitate to ask for help.
+
+$(contactme)
+EOF
+
+    if ui_confirm "Does $ROOT_HOME/.nix-defexpr or $ROOT_HOME/.nix-channels or $ROOT_HOME/.nix-profile exist?"; then
+        failure <<EOF
+You must delete $ROOT_HOME/.nix-defexpr and $ROOT_HOME/.nix-channels
+before continuing.
+EOF
+    fi
+}
+
+setup_report() {
+    header "hardware report"
+    row "           Cores" "$CORES"
+
+    header "Nix config report"
+    row "        Temp Dir" "$SCRATCH"
+    row "        Nix Root" "$NIX_ROOT"
+    row "     Build Users" "$NIX_USER_COUNT"
+    row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
+    row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
+
+    subheader "build users:"
+
+    row "    Username" "UID"
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        row "     $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")"
+    done
+    echo ""
+}
+
+create_build_group() {
+    local primary_group_id
+
+    task "Setting up the build group $NIX_BUILD_GROUP_NAME"
+    if ! /usr/bin/dscl . -read "/Groups/$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1; then
+        _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
+              /usr/sbin/dseditgroup -o create \
+              -r "Nix build group for nix-daemon" \
+              -i "$NIX_BUILD_GROUP_ID" \
+              "$NIX_BUILD_GROUP_NAME" >&2
+        row "            Created" "Yes"
+    else
+        primary_group_id=$(dsclattr "/Groups/$NIX_BUILD_GROUP_NAME" "PrimaryGroupID")
+        if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then
+            failure <<EOF
+It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
+with the UID $primary_group_id. This script can't really handle
+that right now, so I'm going to give up.
+
+You can fix this by editing this script and changing the
+NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
+to $primary_group_id and re-run.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+}
+
+create_build_user_for_core() {
+    local coreid
+    local username
+    local uid
+
+    coreid="$1"
+    username=$(nix_user_for_core "$coreid")
+    uid=$(nix_uid_for_core "$coreid")
+    dsclpath="/Users/$username"
+
+    task "Setting up the build user $username"
+
+    if ! /usr/bin/dscl . -read "$dsclpath" > /dev/null 2>&1; then
+        _sudo "Creating the Nix build user, $username" \
+              /usr/sbin/sysadminctl -addUser -fullName "Nix build user $coreid" \
+	      -home /var/empty \
+	      -UID "${uid}" \
+              -addUser "${username}"
+        row "           Created" "Yes"
+    else
+        actual_uid=$(dsclattr "$dsclpath" "UniqueID")
+        if [ "$actual_uid" -ne "$uid" ]; then
+            failure <<EOF
+It seems the build user $username already exists, but with the UID
+with the UID $actual_uid. This script can't really handle that right
+now, so I'm going to give up.
+
+If you already created the users and you know they start from
+$actual_uid and go up from there, you can edit this script and change
+NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try
+again.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+
+    if [ "$(dsclattr "$dsclpath" "IsHidden")" = "1" ]; then
+        row "          IsHidden" "Yes"
+    else
+        _sudo "in order to make $username a hidden user" \
+              /usr/bin/dscl . -create "$dsclpath" "IsHidden" "1"
+        row "          IsHidden" "Yes"
+    fi
+
+    if [ "$(dsclattr "$dsclpath" "UserShell")" = "/sbin/nologin" ]; then
+        row "   Logins Disabled" "Yes"
+    else
+        _sudo "in order to prevent $username from logging in" \
+              /usr/bin/dscl . -create "$dsclpath" "UserShell" "/sbin/nologin"
+        row "   Logins Disabled" "Yes"
+    fi
+
+    if dseditgroup -o checkmember -m "$username" "$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1 ; then
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    else
+        _sudo "Add $username to the $NIX_BUILD_GROUP_NAME group"\
+            /usr/sbin/dseditgroup -o edit -t user \
+            -a "$username" "$NIX_BUILD_GROUP_NAME"
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    fi
+
+    if [ "$(dsclattr "$dsclpath" "PrimaryGroupId")" = "$NIX_BUILD_GROUP_ID" ]; then
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+    else
+        _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
+              /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupId" "$NIX_BUILD_GROUP_ID"
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+
+    fi
+}
+
+create_build_users() {
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        create_build_user_for_core "$i"
+    done
+}
+
+create_directories() {
+    _sudo "to make the basic directory structure of Nix (part 1)" \
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+
+    _sudo "to make the basic directory structure of Nix (part 2)" \
+          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
+
+    _sudo "to make the basic directory structure of Nix (part 3)" \
+          mkdir -pv -m 1775 /nix/store
+
+    _sudo "to make the basic directory structure of Nix (part 4)" \
+          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
+
+    _sudo "to set up the root user's profile (part 1)" \
+          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
+
+    _sudo "to set up the root user's profile (part 2)" \
+          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
+
+    _sudo "to place the default nix daemon configuration (part 1)" \
+          mkdir -pv -m 0555 /etc/nix
+}
+
+place_channel_configuration() {
+    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+    _sudo "to set up the default system channel (part 1)" \
+          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+}
+
+chat_about_sudo() {
+    header "let's talk about sudo"
+
+    cat <<EOF
+This script is going to call sudo a lot. Every time we do, it'll
+output exactly what it'll do, and why.
+
+Just like this:
+
+EOF
+
+    __sudo "to demonstrate how our sudo prompts look" \
+           echo "this is a sudo prompt"
+
+    cat <<EOF
+
+We're going to use sudo to set up Nix:
+
+ - create local users (see the list above for the users we'll make)
+ - create a local group ($NIX_BUILD_GROUP_NAME)
+ - install Nix in to $NIX_ROOT
+ - create a configuration file in /etc/nix
+ - set up the "default profile" by creating some Nix-related files in
+   $ROOT_HOME
+ - create $PROFILE_NIX_FILE
+EOF
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ]; then
+            cat <<EOF
+ - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
+ - update $ to include some Nix configuration
+EOF
+        fi
+    done
+    cat <<EOF
+ - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
+
+This might look scary, but everything can be undone by running just a
+few commands.
+
+$(uninstall_directions)
+
+We used to ask you to confirm each time sudo ran, but it was too many
+times. Instead, I'll just ask you this one time:
+
+EOF
+    if ui_confirm "Can we use sudo?"; then
+        ok "Yay! Thanks! Let's get going!"
+    else
+        failure <<EOF
+That is okay, but we can't install.
+EOF
+    fi
+}
+
+install_from_extracted_nix() {
+    (
+        cd "$EXTRACTED_NIX_PATH"
+
+        _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
+              rsync -a "$(pwd)/store/" "$NIX_ROOT/store/"
+
+        if [ -d "$NIX_INSTALLED_NIX" ]; then
+            echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
+        else
+            failure <<EOF
+Something went wrong, and I didn't find Nix installed at
+$NIX_INSTALLED_NIX.
+EOF
+        fi
+
+        _sudo "to initialize the Nix Database" \
+              $NIX_INSTALLED_NIX/bin/nix-store --init
+
+        cat ./.reginfo \
+            | _sudo "to load data for the first time in to the Nix Database" \
+                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
+
+        echo "      Just finished getting the nix database ready."
+    )
+}
+
+shell_source_lines() {
+    cat <<EOF
+
+# Nix
+if [ -e '$PROFILE_NIX_FILE' ]; then
+  . '$PROFILE_NIX_FILE'
+fi
+# End Nix
+
+EOF
+}
+configure_shell_profile() {
+    cat <<'EOF' > "$SCRATCH/$PROFILE_NIX_FILE_NAME"
+# Only execute this file once per shell.
+if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
+__ETC_PROFILE_NIX_SOURCED=1
+
+# Set up secure multi-user builds: non-root users build through the
+# Nix daemon.
+if [ "$USER" != root -o ! -w /nix/var/nix/db ]; then
+    export NIX_REMOTE=daemon
+fi
+
+export NIX_USER_PROFILE_DIR="/nix/var/nix/profiles/per-user/$USER"
+export NIX_PROFILES="/run/current-system/sw /nix/var/nix/profiles/default $HOME/.nix-profile"
+
+# Set up the per-user profile.
+mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
+if test "$(stat -f '%u' $NIX_USER_PROFILE_DIR)" != "$(id -u)"; then
+    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
+fi
+
+if test -w $HOME; then
+  if ! test -L $HOME/.nix-profile; then
+      if test "$USER" != root; then
+          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
+      else
+          # Root installs in the system-wide profile by default.
+          ln -s /nix/var/nix/profiles/default $HOME/.nix-profile
+      fi
+  fi
+
+  # Subscribe the root user to the NixOS channel by default.
+  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
+      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
+  fi
+
+  # Create the per-user garbage collector roots directory.
+  NIX_USER_GCROOTS_DIR=/nix/var/nix/gcroots/per-user/$USER
+  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
+  if test "$(stat -f '%u' $NIX_USER_GCROOTS_DIR)" != "$(id -u)"; then
+      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
+  fi
+
+  # Set up a default Nix expression from which to install stuff.
+  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
+      rm -f $HOME/.nix-defexpr
+      mkdir -p $HOME/.nix-defexpr
+      if [ "$USER" != root ]; then
+          ln -s /nix/var/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
+      fi
+  fi
+fi
+
+export NIX_SSL_CERT_FILE="/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+export NIX_PATH="/nix/var/nix/profiles/per-user/root/channels"
+export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/sbin:$HOME/.nix-profile/lib/kde4/libexec:/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/nix/var/nix/profiles/default/lib/kde4/libexec:$PATH"
+
+EOF
+
+    _sudo "put our custom profile code at $PROFILE_NIX_FILE with nix-daemon settings" \
+          install -m 0444 "$SCRATCH/$PROFILE_NIX_FILE_NAME" "$PROFILE_NIX_FILE_DIR"
+
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ]; then
+            _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
+                  cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
+
+            shell_source_lines \
+                | _sudo "extend your $profile_target with nix-daemon settings" \
+                        tee -a "$profile_target"
+        fi
+    done
+
+}
+
+setup_default_profile() {
+    _sudo "to installing a bootstrapping Nix in to the default Profile" \
+          -i "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
+
+    _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
+          -i "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+
+    _sudo "to update the default channel in the default profile" \
+          -i NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+
+    _sudo "to replace the bootstrapping Nix with a real Nix" \
+          -i NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-env" -iA nixpkgs.nix
+}
+
+
+place_nix_configuration() {
+    cat <<EOF > "$SCRATCH/nix.conf"
+build-users-group = $NIX_BUILD_GROUP_NAME
+
+build-max-jobs = $NIX_USER_COUNT
+build-cores = 1
+build-use-sandbox = false
+
+binary-caches = https://cache.nixos.org/
+trusted-binary-caches =
+binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+signed-binary-caches = *
+
+trusted-users = root
+allowed-users = *
+EOF
+    _sudo "to place the default nix daemon configuration (part 2)" \
+          install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
+}
+
+configure_nix_daemon_plist() {
+    _sudo "to set up the nix-daemon as a LaunchDaemon" \
+          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
+
+    _sudo "to load the LaunchDaemon plist for nix-daemon" \
+          launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+
+    _sudo "to start the nix-daemon" \
+          launchctl start org.nixos.nix-daemon
+
+}
+
+
+main() {
+    if [ "${PINCH_ME_IM_SILLY:-}" = "" ]; then
+        validate_starting_assumptions
+    fi
+
+    setup_report
+
+    if ! ui_confirm "Ready to continue?"; then
+        ok "Alright, no changes have been made :)"
+        contactme
+        trap finish_cleanup EXIT
+        exit 1
+    fi
+
+    chat_about_sudo
+
+    create_build_group
+    create_build_users
+    create_directories
+    place_channel_configuration
+    install_from_extracted_nix
+
+    configure_shell_profile
+
+    set +eu
+    . /etc/profile
+    set -eu
+
+    setup_default_profile
+    place_nix_configuration
+    configure_nix_daemon_plist
+
+    trap finish_success EXIT
+}
+
+
+main
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index d4eb1c6fbeb..dd826c42395 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -8,15 +8,8 @@ nix="@nix@"
 cacert="@cacert@"
 
 
-# macOS support for 10.10 or higher
-if [[ "$(uname -s)" = "Darwin" && $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]]; then
-    echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
-    exit 1
-fi
-
 if ! [ -e "$self/.reginfo" ]; then
     echo "$0: incomplete installer (.reginfo is missing)" >&2
-    exit 1
 fi
 
 if [ -z "$USER" ]; then
@@ -24,6 +17,23 @@ if [ -z "$USER" ]; then
     exit 1
 fi
 
+if [ -z "$HOME" ]; then
+    echo "$0: \$HOME is not set" >&2
+    exit 1
+fi
+
+# macOS support for 10.10 or higher
+if [ "$(uname -s)" = "Darwin" ]; then
+    if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then
+        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
+        exit 1
+    fi
+
+    printf '\e[1;31mSwitching to the Multi-User Darwin Installer\e[0m\n'
+    "$self/install-darwin-multi-user"
+    exit 0
+fi
+
 if [ "$(id -u)" -eq 0 ]; then
     printf '\e[1;31mwarning: installing Nix as root is not supported by this script!\e[0m\n'
 fi

From 2b5ab03524cb0a48757bb5fd86fe584f4d2c8f36 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 9 Jul 2017 14:44:44 -0400
Subject: [PATCH 0436/2196] multi-user install: move the profile in to the nix
 etc/profiles.d output

---
 scripts/install-darwin-multi-user.sh | 69 +---------------------------
 1 file changed, 1 insertion(+), 68 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 87d55b2f017..96728e98776 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -35,11 +35,7 @@ readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 readonly PROFILE_TARGETS=("/etc/profile" "/etc/bashrc" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
-readonly PROFILE_NIX_FILE_DIR="/etc/"
-readonly PROFILE_NIX_FILE_NAME="profile-nix.sh"
-readonly PROFILE_NIX_FILE="$PROFILE_NIX_FILE_DIR$PROFILE_NIX_FILE_NAME"
-
-
+readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
 readonly NIX_INSTALLED_NIX="@nix@"
 readonly NIX_INSTALLED_CACERT="@cacert@"
@@ -576,7 +572,6 @@ We're going to use sudo to set up Nix:
  - create a configuration file in /etc/nix
  - set up the "default profile" by creating some Nix-related files in
    $ROOT_HOME
- - create $PROFILE_NIX_FILE
 EOF
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target" ]; then
@@ -646,68 +641,6 @@ fi
 EOF
 }
 configure_shell_profile() {
-    cat <<'EOF' > "$SCRATCH/$PROFILE_NIX_FILE_NAME"
-# Only execute this file once per shell.
-if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
-__ETC_PROFILE_NIX_SOURCED=1
-
-# Set up secure multi-user builds: non-root users build through the
-# Nix daemon.
-if [ "$USER" != root -o ! -w /nix/var/nix/db ]; then
-    export NIX_REMOTE=daemon
-fi
-
-export NIX_USER_PROFILE_DIR="/nix/var/nix/profiles/per-user/$USER"
-export NIX_PROFILES="/run/current-system/sw /nix/var/nix/profiles/default $HOME/.nix-profile"
-
-# Set up the per-user profile.
-mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
-if test "$(stat -f '%u' $NIX_USER_PROFILE_DIR)" != "$(id -u)"; then
-    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
-fi
-
-if test -w $HOME; then
-  if ! test -L $HOME/.nix-profile; then
-      if test "$USER" != root; then
-          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
-      else
-          # Root installs in the system-wide profile by default.
-          ln -s /nix/var/nix/profiles/default $HOME/.nix-profile
-      fi
-  fi
-
-  # Subscribe the root user to the NixOS channel by default.
-  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
-      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
-  fi
-
-  # Create the per-user garbage collector roots directory.
-  NIX_USER_GCROOTS_DIR=/nix/var/nix/gcroots/per-user/$USER
-  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
-  if test "$(stat -f '%u' $NIX_USER_GCROOTS_DIR)" != "$(id -u)"; then
-      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
-  fi
-
-  # Set up a default Nix expression from which to install stuff.
-  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
-      rm -f $HOME/.nix-defexpr
-      mkdir -p $HOME/.nix-defexpr
-      if [ "$USER" != root ]; then
-          ln -s /nix/var/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
-      fi
-  fi
-fi
-
-export NIX_SSL_CERT_FILE="/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-export NIX_PATH="/nix/var/nix/profiles/per-user/root/channels"
-export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/sbin:$HOME/.nix-profile/lib/kde4/libexec:/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/nix/var/nix/profiles/default/lib/kde4/libexec:$PATH"
-
-EOF
-
-    _sudo "put our custom profile code at $PROFILE_NIX_FILE with nix-daemon settings" \
-          install -m 0444 "$SCRATCH/$PROFILE_NIX_FILE_NAME" "$PROFILE_NIX_FILE_DIR"
-
-
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target" ]; then
             _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \

From 6f639943c288a956a717a353a04f8e03442106c4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 9 Jul 2017 21:25:36 -0400
Subject: [PATCH 0437/2196] Prompt for sudo before validating assumptions, and
 check ourselves for root-owned files instead of making a scary warning.

---
 scripts/install-darwin-multi-user.sh | 38 ++++++++++------------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 96728e98776..30e5dec6667 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -366,30 +366,15 @@ EOF
         fi
     done
 
-    warning <<EOF
-I strongly recommend deleting all the following files, which are
-relics of a previous installation. You can check by running (yourself):
-
-  $ sudo find / -name .nix-profile  -o -name .nix-defexpr -o -name .nix-channels -o -name '*nix-daemon.plist'
-
-In particular, the following files and directories MUST NOT EXIST!
-
- - $ROOT_HOME/.nix-profile
- - $ROOT_HOME/.nix-defexpr
- - $ROOT_HOME/.nix-channels
-
-If some relics are found and you want to keep them, it might be okay.
-If you're not sure, don't hesitate to ask for help.
-
-$(contactme)
-EOF
-
-    if ui_confirm "Does $ROOT_HOME/.nix-defexpr or $ROOT_HOME/.nix-channels or $ROOT_HOME/.nix-profile exist?"; then
-        failure <<EOF
-You must delete $ROOT_HOME/.nix-defexpr and $ROOT_HOME/.nix-channels
-before continuing.
+    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
+    for danger_path in "${danger_paths[@]}"; do
+        if _sudo "foo"; then
+            failure <<EOF
+I found a file at $danger_path, which is a relic of a previous
+installation. You must first delete this file before continuing.
 EOF
-    fi
+        fi
+    done
 }
 
 setup_report() {
@@ -566,6 +551,8 @@ EOF
 
 We're going to use sudo to set up Nix:
 
+ - make sure your computer doesn't already have Nix files
+   (if it does, I  will tell you how to clean them up.)
  - create local users (see the list above for the users we'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
@@ -703,6 +690,9 @@ configure_nix_daemon_plist() {
 
 
 main() {
+
+    chat_about_sudo
+
     if [ "${PINCH_ME_IM_SILLY:-}" = "" ]; then
         validate_starting_assumptions
     fi
@@ -716,8 +706,6 @@ main() {
         exit 1
     fi
 
-    chat_about_sudo
-
     create_build_group
     create_build_users
     create_directories

From 092f447c6d9034687e10b356939ed0482bf65ed8 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jul 2017 06:08:14 +0400
Subject: [PATCH 0438/2196] Clean up issues around uninstall directions, and
 only show relevant directions

---
 scripts/install-darwin-multi-user.sh | 106 +++++++++++++++++++--------
 1 file changed, 77 insertions(+), 29 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 30e5dec6667..95222a7f61f 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -55,19 +55,24 @@ contactme() {
 
 uninstall_directions() {
     subheader "Uninstalling nix:"
-    local step=1
-    cat <<EOF
-$step. If $PLIST_DEST exists:
+    local step=0
+
+    if [ -e "$PLIST_DEST" ]; then
+        step=$((step + 1))
+        cat <<EOF
+$step. Delete $PLIST_DEST
 
   sudo launchctl unload $PLIST_DEST
   sudo rm $PLIST_DEST
 
 EOF
+    fi
+
     for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ]; then
+        if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
             step=$((step + 1))
             cat <<EOF
-$step. If $profile_target$PROFILE_BACKUP_SUFFIX exists:
+$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target
 
   sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
 
@@ -77,8 +82,8 @@ opened while it existed.)
 EOF
         fi
     done
-    step=$((step + 1))
 
+    step=$((step + 1))
     cat <<EOF
 $step. Delete the files Nix added to your system:
 
@@ -307,6 +312,8 @@ also look for similar references in:
  - ~/.profile
 
 or other shell init files that you may have.
+
+$(uninstall_directions)
 EOF
             fi
         fi
@@ -368,10 +375,13 @@ EOF
 
     danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
     for danger_path in "${danger_paths[@]}"; do
-        if _sudo "foo"; then
+        if _sudo "making sure that $danger_path doesn't exist" \
+           test -e "$danger_path"; then
             failure <<EOF
 I found a file at $danger_path, which is a relic of a previous
 installation. You must first delete this file before continuing.
+
+$(uninstall_directions)
 EOF
         fi
     done
@@ -533,23 +543,36 @@ place_channel_configuration() {
           install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
 }
 
-chat_about_sudo() {
-    header "let's talk about sudo"
+welcome_to_nix() {
+    ok "Welcome to the Multi-User Nix Installation"
 
     cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
-output exactly what it'll do, and why.
 
-Just like this:
+This installation tool will set up your computer with the Nix package
+manager. This will happen in a few stages:
 
-EOF
+1. Make sure your computer doesn't already have Nix. If it does, I
+   will show you instructions on how to clean up your old one.
 
-    __sudo "to demonstrate how our sudo prompts look" \
-           echo "this is a sudo prompt"
+2. Show you what we are going to install and where. Then we will ask
+   if you are ready to continue.
 
-    cat <<EOF
+3. Create the system users and groups that the Nix daemon uses to run
+   builds.
+
+4. Perform the basic installation of the Nix files daemon.
 
-We're going to use sudo to set up Nix:
+5. Configure your shell to import special Nix Profile files, so you
+   can use Nix.
+
+6. Start the Nix daemon.
+
+EOF
+
+    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+        cat <<EOF
+
+We will:
 
  - make sure your computer doesn't already have Nix files
    (if it does, I  will tell you how to clean them up.)
@@ -560,24 +583,44 @@ We're going to use sudo to set up Nix:
  - set up the "default profile" by creating some Nix-related files in
    $ROOT_HOME
 EOF
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ]; then
-            cat <<EOF
+        for profile_target in "${PROFILE_TARGETS[@]}"; do
+            if [ -e "$profile_target" ]; then
+                cat <<EOF
  - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
- - update $ to include some Nix configuration
+ - update $profile_target to include some Nix configuration
+EOF
+            fi
+        done
+        cat <<EOF
+ - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
+
+EOF
+        if ! ui_confirm "Ready to continue?"; then
+            failure <<EOF
+Okay, maybe you would like to talk to the team.
 EOF
         fi
-    done
+    fi
+}
+
+chat_about_sudo() {
+    header "let's talk about sudo"
+
     cat <<EOF
- - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
+This script is going to call sudo a lot. Every time we do, it'll
+output exactly what it'll do, and why.
 
-This might look scary, but everything can be undone by running just a
-few commands.
+Just like this:
+EOF
 
-$(uninstall_directions)
+    __sudo "to demonstrate how our sudo prompts look" \
+           echo "this is a sudo prompt"
+
+    cat <<EOF
 
-We used to ask you to confirm each time sudo ran, but it was too many
-times. Instead, I'll just ask you this one time:
+This might look scary, but everything can be undone by running just a
+few commands. We used to ask you to confirm each time sudo ran, but it
+was too many times. Instead, I'll just ask you this one time:
 
 EOF
     if ui_confirm "Can we use sudo?"; then
@@ -690,7 +733,7 @@ configure_nix_daemon_plist() {
 
 
 main() {
-
+    welcome_to_nix
     chat_about_sudo
 
     if [ "${PINCH_ME_IM_SILLY:-}" = "" ]; then
@@ -706,6 +749,11 @@ main() {
         exit 1
     fi
 
+    if [ "${PINCH_ME_IM_SILLY:-}" != "" ]; then
+        exit 1
+    fi
+
+
     create_build_group
     create_build_users
     create_directories

From 6a4037ca05cdfd04e468115be1905eace8f6f57c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jul 2017 19:38:45 -0400
Subject: [PATCH 0439/2196] Don't install a second nix after the initial
 installation, and the rsync change fixes a bug hidden by the nix replacement
 where the store files were being owned by the installing user due to rsync's
 -a implying -og.

---
 scripts/install-darwin-multi-user.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 95222a7f61f..3901ebe2d60 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -637,7 +637,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -a "$(pwd)/store/" "$NIX_ROOT/store/"
+              rsync -rlpt "$(pwd)/store/" "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
@@ -693,9 +693,6 @@ setup_default_profile() {
 
     _sudo "to update the default channel in the default profile" \
           -i NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
-
-    _sudo "to replace the bootstrapping Nix with a real Nix" \
-          -i NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-env" -iA nixpkgs.nix
 }
 
 

From 657b47e1b30d28760fab639532cd42535da9a097 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 11 Jul 2017 20:25:35 -0400
Subject: [PATCH 0440/2196] Address feedback around printf & exec

---
 scripts/install-darwin-multi-user.sh | 2 +-
 scripts/install-nix-from-closure.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 3901ebe2d60..01a0e4ffb4e 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -199,7 +199,7 @@ __sudo() {
 
     echo "I am executing:"
     echo ""
-    echo "    $ sudo $cmd"
+    printf "    $ sudo %s\n" "$cmd"
     echo ""
     echo "$expl"
     echo ""
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index dd826c42395..9d395020e60 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -30,7 +30,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     fi
 
     printf '\e[1;31mSwitching to the Multi-User Darwin Installer\e[0m\n'
-    "$self/install-darwin-multi-user"
+    exec "$self/install-darwin-multi-user"
     exit 0
 fi
 

From a0369b14f4ad17b021ce4b06f97fb1081f419c44 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 10:46:29 -0400
Subject: [PATCH 0441/2196] Test the installer

---
 .travis.yml             |  2 ++
 tests/install-darwin.sh | 76 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 .travis.yml
 create mode 100644 tests/install-darwin.sh

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 00000000000..99218a963cb
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,2 @@
+os: osx
+script: ./tests/install-darwin.sh
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
new file mode 100644
index 00000000000..30a3cc3d36b
--- /dev/null
+++ b/tests/install-darwin.sh
@@ -0,0 +1,76 @@
+#!/bin/sh
+
+cleanup() {
+    PLIST="/Library/LaunchDaemons/org.nixos.nix-daemon.plist"
+    if sudo launchctl list | grep -q nix-daemon; then
+        sudo launchctl unload "$PLIST"
+    fi
+
+    if [ -f "$PLIST" ]; then
+        sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+    fi
+
+    profiles=(/etc/profile /etc/bashrc /etc/zshrc)
+    for profile in "${profiles[@]}"; do
+        if [ -f "${profile}.backup-before-nix" ]; then
+            sudo mv "${profile}.backup-before-nix" "${profile}"
+        fi
+    done
+
+    for i in $(seq 1 $(sysctl -n hw.ncpu)); do
+        sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true
+    done
+    sudo /usr/bin/dscl . -delete "/Groups/nixbld" || true
+
+
+    sudo rm -rf /etc/nix \
+         /nix \
+         /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \
+         "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels"
+}
+
+verify() {
+    output=$(echo "nix-shell -p bash --run 'echo toow | rev'" | bash -l)
+    test "$output" = "woot"
+}
+
+scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
+function finish {
+    rm -rf "$scratch"
+}
+trap finish EXIT
+
+# First setup Nix
+cleanup
+curl https://nixos.org/nix/install | bash
+verify
+
+
+(
+    nix-build ./release.nix -A binaryTarball.x86_64-darwin
+    cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2
+)
+
+(
+    cd $scratch
+    tar -xf ./nix.tar.bz2
+
+    cd nix-*
+
+    set -eux
+
+    cat ~/.profile | grep -v nix-profile > ~/.profile-next
+    mv ~/.profile-next ~/.profile
+
+    cleanup
+
+    yes | ./install
+    verify
+
+    cleanup
+
+    yes | ./install
+    verify
+
+    cleanup
+)

From bc647fd2991deffcc1ab400912e641ac4e4d016f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 12:58:37 -0400
Subject: [PATCH 0442/2196] chmod

---
 tests/install-darwin.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tests/install-darwin.sh

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
old mode 100644
new mode 100755

From 61ea9e9867ad426eee064242963299553b8db9f3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 17:10:14 -0400
Subject: [PATCH 0443/2196] Run nix-build inside a fresh bash login

---
 tests/install-darwin.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index 30a3cc3d36b..4a6ab7d6471 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -eux
+
 cleanup() {
     PLIST="/Library/LaunchDaemons/org.nixos.nix-daemon.plist"
     if sudo launchctl list | grep -q nix-daemon; then
@@ -47,7 +49,10 @@ verify
 
 
 (
-    nix-build ./release.nix -A binaryTarball.x86_64-darwin
+    (
+        echo "cd $(pwd)"
+        echo nix-build ./release.nix -A binaryTarball.x86_64-darwin
+    ) | bash -l
     cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2
 )
 

From 92ca93528fe7d29b8909a89b0f9c7a9e23627fd5 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 20:29:58 -0400
Subject: [PATCH 0444/2196] Clean up nix hints from the old insstaller

---
 tests/install-darwin.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index 4a6ab7d6471..9084d92c23e 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -19,6 +19,13 @@ cleanup() {
         fi
     done
 
+    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
+        cat "$file" | grep -v nix-profile > "$file.next"
+        mv "$file.next" "$file"
+    done
+
+
+
     for i in $(seq 1 $(sysctl -n hw.ncpu)); do
         sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true
     done
@@ -64,9 +71,6 @@ verify
 
     set -eux
 
-    cat ~/.profile | grep -v nix-profile > ~/.profile-next
-    mv ~/.profile-next ~/.profile
-
     cleanup
 
     yes | ./install

From 3839dda2ecc0444db1469757b4a3e1f4f776fe9d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 20:31:33 -0400
Subject: [PATCH 0445/2196] Only clean if the file exists

---
 tests/install-darwin.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index 9084d92c23e..b553f710527 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -20,8 +20,10 @@ cleanup() {
     done
 
     for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        cat "$file" | grep -v nix-profile > "$file.next"
-        mv "$file.next" "$file"
+        if [ -e "$file" ]; then
+            cat "$file" | grep -v nix-profile > "$file.next"
+            mv "$file.next" "$file"
+        fi
     done
 
 

From 73a57a2f22668394740e1f642573b67a1316dee7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 12 Jul 2017 20:43:57 -0400
Subject: [PATCH 0446/2196] Cleanup and more specificity around set -e

---
 tests/install-darwin.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index b553f710527..d2a7ab456ea 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -26,14 +26,11 @@ cleanup() {
         fi
     done
 
-
-
     for i in $(seq 1 $(sysctl -n hw.ncpu)); do
         sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true
     done
     sudo /usr/bin/dscl . -delete "/Groups/nixbld" || true
 
-
     sudo rm -rf /etc/nix \
          /nix \
          /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \
@@ -41,7 +38,10 @@ cleanup() {
 }
 
 verify() {
+    set +e
     output=$(echo "nix-shell -p bash --run 'echo toow | rev'" | bash -l)
+    set -e
+
     test "$output" = "woot"
 }
 
@@ -58,10 +58,12 @@ verify
 
 
 (
+    set +e
     (
         echo "cd $(pwd)"
         echo nix-build ./release.nix -A binaryTarball.x86_64-darwin
     ) | bash -l
+    set -e
     cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2
 )
 

From 1c7ce2a018a081dd90dfefee6d1ca7e0c9fd66d9 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 13 Jul 2017 17:22:54 -0400
Subject: [PATCH 0447/2196] Assume yes if we have no TTY

Starve the TTY of input to ensure this works, but provide yes to the
current installer to handle the current broken case.
---
 scripts/install-darwin-multi-user.sh | 11 +++++++++++
 tests/install-darwin.sh              |  5 +++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 01a0e4ffb4e..368d9957e11 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -43,6 +43,12 @@ readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
 
 readonly ROOT_HOME="/var/root"
 
+if [ -t 0 ]; then
+    readonly IS_HEADLESS='no'
+else
+    readonly IS_HEADLESS='yes'
+fi
+
 contactme() {
     echo "We'd love to help if you need it."
     echo ""
@@ -173,6 +179,11 @@ failure() {
 ui_confirm() {
     _textout "$GREEN$GREEN_UL" "$1"
 
+    if [ "$IS_HEADLESS" = "yes" ]; then
+        echo "No TTY, assuming you would say yes :)"
+        return 0
+    fi
+
     local prompt="[y/n] "
     echo -n "$prompt"
     while read -r y; do
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index d2a7ab456ea..b7f87467287 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -53,7 +53,8 @@ trap finish EXIT
 
 # First setup Nix
 cleanup
-curl https://nixos.org/nix/install | bash
+curl -o install https://nixos.org/nix/install
+yes | bash ./install
 verify
 
 
@@ -82,7 +83,7 @@ verify
 
     cleanup
 
-    yes | ./install
+    echo -n "" | ./install
     verify
 
     cleanup

From ce2281e6d86aa1b461dd27f86b2a71145e85ac8d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 13 Jul 2017 18:01:22 -0400
Subject: [PATCH 0448/2196] Ensure PINCH_ME_IM_SILLY allows a /nix/store to
 stick around between builds

Also output in the status report that the user is very silly
---
 scripts/install-darwin-multi-user.sh | 8 +++-----
 tests/install-darwin.sh              | 7 ++++++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 368d9957e11..b1d258e0cad 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -408,6 +408,9 @@ setup_report() {
     row "     Build Users" "$NIX_USER_COUNT"
     row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
     row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
+    if [ "${PINCH_ME_IM_SILLY:-}" != "" ]; then
+        row "       Silliness" "Very Silly"
+    fi
 
     subheader "build users:"
 
@@ -757,11 +760,6 @@ main() {
         exit 1
     fi
 
-    if [ "${PINCH_ME_IM_SILLY:-}" != "" ]; then
-        exit 1
-    fi
-
-
     create_build_group
     create_build_users
     create_directories
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index b7f87467287..3f64944ef81 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -80,11 +80,16 @@ verify
 
     yes | ./install
     verify
-
     cleanup
 
     echo -n "" | ./install
     verify
+    cleanup
 
+    sudo mkdir -p /nix/store
+    sudo touch /nix/store/.silly-hint
+    echo -n "" | PINCH_ME_IM_SILLY=true ./install
+    verify
+    test -e /nix/store/.silly-hint
     cleanup
 )

From 12f6bb33d2cf56e8adf482950734a9af5f78a496 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 13 Jul 2017 19:03:35 -0400
Subject: [PATCH 0449/2196] If there is no TTY, also skip verbose sudo messages

---
 scripts/install-darwin-multi-user.sh | 36 +++++++++++++++++++++++++---
 tests/install-darwin.sh              |  1 +
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index b1d258e0cad..cde40a7f25e 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -49,6 +49,14 @@ else
     readonly IS_HEADLESS='yes'
 fi
 
+headless() {
+    if [ "$IS_HEADLESS" = "yes" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
 contactme() {
     echo "We'd love to help if you need it."
     echo ""
@@ -179,7 +187,7 @@ failure() {
 ui_confirm() {
     _textout "$GREEN$GREEN_UL" "$1"
 
-    if [ "$IS_HEADLESS" = "yes" ]; then
+    if headless; then
         echo "No TTY, assuming you would say yes :)"
         return 0
     fi
@@ -221,9 +229,10 @@ __sudo() {
 _sudo() {
     local expl="$1"
     shift
-    if __sudo "$expl" "$*"; then
-        sudo "$@"
+    if ! headless; then
+        __sudo "$expl" "$*"
     fi
+    sudo "$@"
 }
 
 
@@ -620,6 +629,27 @@ EOF
 chat_about_sudo() {
     header "let's talk about sudo"
 
+    if headless; then
+        cat <<EOF
+This script is going to call sudo a lot. Normally, it would show you
+exactly what commands it is running and why. However, the script is
+run in  a headless fashion, like this:
+
+  $ curl https://nixos.org/nix/install | sh
+
+or maybe in a CI pipeline. Because of that, we're going to skip the
+verbose output in the interest of brevity.
+
+If you would like to
+see the output, try like this:
+
+  $ curl -o install-nix https://nixos.org/nix/install
+  $ sh ./install-nix
+
+EOF
+        return 0
+    fi
+
     cat <<EOF
 This script is going to call sudo a lot. Every time we do, it'll
 output exactly what it'll do, and why.
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index 3f64944ef81..eb251fa3939 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -91,5 +91,6 @@ verify
     echo -n "" | PINCH_ME_IM_SILLY=true ./install
     verify
     test -e /nix/store/.silly-hint
+
     cleanup
 )

From e0d39c8dc47af203ef0647c277e0e26f3b9176d4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 14 Jul 2017 06:28:24 -0400
Subject: [PATCH 0450/2196] Rename PINCH_ME_IM_SILLY to
 ALLOW_PREEXISTING_INSTALLATION

---
 scripts/install-darwin-multi-user.sh | 6 +++---
 tests/install-darwin.sh              | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index cde40a7f25e..a2e120bf862 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -417,8 +417,8 @@ setup_report() {
     row "     Build Users" "$NIX_USER_COUNT"
     row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
     row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
-    if [ "${PINCH_ME_IM_SILLY:-}" != "" ]; then
-        row "       Silliness" "Very Silly"
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then
+        row "Preexisting Install" "Allowed"
     fi
 
     subheader "build users:"
@@ -777,7 +777,7 @@ main() {
     welcome_to_nix
     chat_about_sudo
 
-    if [ "${PINCH_ME_IM_SILLY:-}" = "" ]; then
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
         validate_starting_assumptions
     fi
 
diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index eb251fa3939..c99ce84acab 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -88,7 +88,7 @@ verify
 
     sudo mkdir -p /nix/store
     sudo touch /nix/store/.silly-hint
-    echo -n "" | PINCH_ME_IM_SILLY=true ./install
+    echo -n "" | ALLOW_PREEXISTING_INSTALLATION=true ./install
     verify
     test -e /nix/store/.silly-hint
 

From 6438ba22af57edc054e441053a7c3cd5d48e5597 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 14:23:38 +0200
Subject: [PATCH 0451/2196] StorePathsCommand: Don't build installables

On second though this was annoying. E.g. "nix log nixpkgs.hello" would
build/download Hello first, even though the log can be fetched
directly from the binary cache.

May need to revisit this.
---
 src/nix/build.cc        | 2 +-
 src/nix/command.cc      | 4 ++--
 src/nix/command.hh      | 4 +++-
 src/nix/installables.cc | 6 +++---
 src/nix/run.cc          | 2 +-
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 00bda1fd104..cc96ac48a97 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -23,7 +23,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto paths = buildInstallables(store, dryRun);
+        auto paths = toStorePaths(store, dryRun ? DryRun : Build);
 
         printInfo("build result: %s", showPaths(paths));
     }
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 96b685a5b2e..6b608a708ee 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -115,7 +115,7 @@ void StorePathsCommand::run(ref<Store> store)
     }
 
     else {
-        for (auto & p : buildInstallables(store, false))
+        for (auto & p : toStorePaths(store, NoBuild))
             storePaths.push_back(p);
 
         if (recursive) {
@@ -131,7 +131,7 @@ void StorePathsCommand::run(ref<Store> store)
 
 void StorePathCommand::run(ref<Store> store)
 {
-    auto storePaths = buildInstallables(store, false);
+    auto storePaths = toStorePaths(store, NoBuild);
 
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 4800b5c912e..eb736ce3a3e 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -80,7 +80,9 @@ struct InstallablesCommand : virtual Args, StoreCommand
 
     std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss);
 
-    PathSet buildInstallables(ref<Store> store, bool dryRun);
+    enum ToStorePathsMode { Build, NoBuild, DryRun };
+
+    PathSet toStorePaths(ref<Store> store, ToStorePathsMode mode);
 
     ref<EvalState> getEvalState();
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 9982ff75f4f..f0d5d547ca3 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -214,7 +214,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
     return result;
 }
 
-PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
+PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode)
 {
     PathSet buildables;
 
@@ -223,9 +223,9 @@ PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
         buildables.insert(b.begin(), b.end());
     }
 
-    if (dryRun)
+    if (mode == DryRun)
         printMissing(store, buildables);
-    else
+    else if (mode == Build)
         store->buildPaths(buildables);
 
     PathSet outPaths;
diff --git a/src/nix/run.cc b/src/nix/run.cc
index bcfa74eb5f5..49fe4e7a0c6 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -30,7 +30,7 @@ struct CmdRun : InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto outPaths = buildInstallables(store, false);
+        auto outPaths = toStorePaths(store, Build);
 
         auto store2 = store.dynamic_pointer_cast<LocalStore>();
 

From 3908d3929ceb07cee5983fa647817f0e7aecbd97 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 15:27:21 +0200
Subject: [PATCH 0452/2196] nix path-info: Don't barf on invalid paths

Now you get

  [
    {
      "path": "/nix/store/fzvliz4j5xzvnd0w5zgw2l0ksqh578yk-bla",
      "valid": false
    }
  ]
---
 src/libstore/store-api.cc | 62 +++++++++++++++++++++------------------
 src/libstore/store-api.hh |  4 ++-
 src/nix/path-info.cc      |  2 +-
 3 files changed, 38 insertions(+), 30 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 108e2d4ce9b..0440af95fa4 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -455,49 +455,55 @@ string Store::makeValidityRegistration(const PathSet & paths,
 
 
 void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
-    bool includeImpureInfo, bool showClosureSize)
+    bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
 {
     auto jsonList = jsonOut.list();
 
     for (auto storePath : storePaths) {
-        auto info = queryPathInfo(storePath);
-        storePath = info->path;
-
         auto jsonPath = jsonList.object();
-        jsonPath
-            .attr("path", storePath)
-            .attr("narHash", info->narHash.to_string())
-            .attr("narSize", info->narSize);
+        jsonPath.attr("path", storePath);
 
-        {
-            auto jsonRefs = jsonPath.list("references");
-            for (auto & ref : info->references)
-                jsonRefs.elem(ref);
-        }
+        try {
+            auto info = queryPathInfo(storePath);
+            storePath = info->path;
 
-        if (info->ca != "")
-            jsonPath.attr("ca", info->ca);
+            jsonPath
+                .attr("narHash", info->narHash.to_string())
+                .attr("narSize", info->narSize);
 
-        if (showClosureSize)
-            jsonPath.attr("closureSize", getClosureSize(storePath));
+            {
+                auto jsonRefs = jsonPath.list("references");
+                for (auto & ref : info->references)
+                    jsonRefs.elem(ref);
+            }
 
-        if (includeImpureInfo) {
+            if (info->ca != "")
+                jsonPath.attr("ca", info->ca);
 
-            if (info->deriver != "")
-                jsonPath.attr("deriver", info->deriver);
+            if (showClosureSize)
+                jsonPath.attr("closureSize", getClosureSize(storePath));
 
-            if (info->registrationTime)
-                jsonPath.attr("registrationTime", info->registrationTime);
+            if (includeImpureInfo) {
 
-            if (info->ultimate)
-                jsonPath.attr("ultimate", info->ultimate);
+                if (info->deriver != "")
+                    jsonPath.attr("deriver", info->deriver);
+
+                if (info->registrationTime)
+                    jsonPath.attr("registrationTime", info->registrationTime);
+
+                if (info->ultimate)
+                    jsonPath.attr("ultimate", info->ultimate);
+
+                if (!info->sigs.empty()) {
+                    auto jsonSigs = jsonPath.list("signatures");
+                    for (auto & sig : info->sigs)
+                        jsonSigs.elem(sig);
+                }
 
-            if (!info->sigs.empty()) {
-                auto jsonSigs = jsonPath.list("signatures");
-                for (auto & sig : info->sigs)
-                    jsonSigs.elem(sig);
             }
 
+        } catch (InvalidPath &) {
+            jsonPath.attr("valid", false);
         }
     }
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index cada37653e6..586d35e84ff 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -35,6 +35,7 @@ class JSONPlaceholder;
 enum RepairFlag : bool { NoRepair = false, Repair = true };
 enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
+enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 
 
 /* Size of the hash part of store paths, in base-32 characters. */
@@ -488,7 +489,8 @@ public:
        included. If ‘showClosureSize’ is true, the closure size of
        each path is included. */
     void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
-        bool includeImpureInfo, bool showClosureSize);
+        bool includeImpureInfo, bool showClosureSize,
+        AllowInvalidFlag allowInvalid = DisallowInvalid);
 
     /* Return the size of the closure of the specified path, that is,
        the sum of the size of the NAR serialisation of each path in
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index f1620923861..f7610ab0814 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -69,7 +69,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             store->pathInfoToJSON(jsonRoot,
                 // FIXME: preserve order?
                 PathSet(storePaths.begin(), storePaths.end()),
-                true, showClosureSize);
+                true, showClosureSize, AllowInvalid);
         }
 
         else {

From fdc9da034fd7e0cb9c5275209d991ed6ca38f1cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 17:10:13 +0200
Subject: [PATCH 0453/2196] Avoid a call to derivationFromPath()

This doesn't work in read-only mode, ensuring that operations like

  nix path-info --store https://cache.nixos.org -S nixpkgs.hello

(asking for the closure size of nixpkgs.hello in cache.nixos.org) work
when nixpkgs.hello doesn't exist in the local store.
---
 src/nix/command.hh      |  5 ++++-
 src/nix/installables.cc | 49 +++++++++++++++++++----------------------
 src/nix/log.cc          |  3 ++-
 3 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/src/nix/command.hh b/src/nix/command.hh
index eb736ce3a3e..ae7709b5dc8 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -44,11 +44,14 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+struct Whence { std::string outputName; Path drvPath; };
+typedef std::map<Path, Whence> Buildables;
+
 struct Installable
 {
     virtual std::string what() = 0;
 
-    virtual PathSet toBuildable()
+    virtual Buildables toBuildable()
     {
         throw Error("argument ‘%s’ cannot be built", what());
     }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index f0d5d547ca3..7fad8fe415c 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -67,9 +67,9 @@ struct InstallableStoreDrv : Installable
 
     std::string what() override { return storePath; }
 
-    PathSet toBuildable() override
+    Buildables toBuildable() override
     {
-        return {storePath};
+        return {{storePath, {}}};
     }
 };
 
@@ -81,9 +81,9 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return storePath; }
 
-    PathSet toBuildable() override
+    Buildables toBuildable() override
     {
-        return {storePath};
+        return {{storePath, {}}};
     }
 };
 
@@ -97,7 +97,7 @@ struct InstallableExpr : Installable
 
     std::string what() override { return text; }
 
-    PathSet toBuildable() override
+    Buildables toBuildable() override
     {
         auto state = installables.getEvalState();
 
@@ -110,10 +110,11 @@ struct InstallableExpr : Installable
         DrvInfos drvs;
         getDerivations(*state, *v, "", autoArgs, drvs, false);
 
-        PathSet res;
+        Buildables res;
 
-        for (auto & i : drvs)
-            res.insert(i.queryDrvPath());
+        for (auto & drv : drvs)
+            for (auto & output : drv.queryOutputs())
+                res.emplace(output.second, Whence{output.first, drv.queryDrvPath()});
 
         return res;
     }
@@ -137,7 +138,7 @@ struct InstallableAttrPath : Installable
 
     std::string what() override { return attrPath; }
 
-    PathSet toBuildable() override
+    Buildables toBuildable() override
     {
         auto state = installables.getEvalState();
 
@@ -150,10 +151,11 @@ struct InstallableAttrPath : Installable
         DrvInfos drvs;
         getDerivations(*state, *v, "", autoArgs, drvs, false);
 
-        PathSet res;
+        Buildables res;
 
-        for (auto & i : drvs)
-            res.insert(i.queryDrvPath());
+        for (auto & drv : drvs)
+            for (auto & output : drv.queryOutputs())
+                res.emplace(output.second, Whence{output.first, drv.queryDrvPath()});
 
         return res;
     }
@@ -216,27 +218,22 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
 
 PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode)
 {
-    PathSet buildables;
+    if (mode != DryRun)
+        settings.readOnlyMode = true;
 
-    for (auto & i : installables) {
-        auto b = i->toBuildable();
-        buildables.insert(b.begin(), b.end());
-    }
+    PathSet outPaths, buildables;
+
+    for (auto & i : installables)
+        for (auto & b : i->toBuildable()) {
+            outPaths.insert(b.first);
+            buildables.insert(b.second.drvPath != "" ? b.second.drvPath : b.first);
+        }
 
     if (mode == DryRun)
         printMissing(store, buildables);
     else if (mode == Build)
         store->buildPaths(buildables);
 
-    PathSet outPaths;
-    for (auto & path : buildables)
-        if (isDerivation(path)) {
-            Derivation drv = store->derivationFromPath(path);
-            for (auto & output : drv.outputs)
-                outPaths.insert(output.second.path);
-        } else
-            outPaths.insert(path);
-
     return outPaths;
 }
 
diff --git a/src/nix/log.cc b/src/nix/log.cc
index ed610261d1c..62ae6b8facf 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -28,7 +28,8 @@ struct CmdLog : InstallablesCommand
         subs.push_front(store);
 
         for (auto & inst : installables) {
-            for (auto & path : inst->toBuildable()) {
+            for (auto & b : inst->toBuildable()) {
+                auto path = b.second.drvPath != "" ? b.second.drvPath : b.first;
                 bool found = false;
                 for (auto & sub : subs) {
                     auto log = sub->getBuildLog(path);

From 766ad5db3b2e8678b649375e59191978671c821e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 14 Jul 2017 17:36:49 +0200
Subject: [PATCH 0454/2196] nix path-info: Show download sizes for binary cache
 stores

E.g.

  $ nix path-info --json --store https://cache.nixos.org nixpkgs.thunderbird -S
  ...
      "downloadHash": "sha256:1jlixpzi225wwa0f4xdrwrqgi47ip1qpj9p06fyxxg07sfmyi4q0",
      "downloadSize": 43047620,
      "closureDownloadSize": 84745960
    }
  ]
---
 src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++-------
 src/libstore/store-api.hh |  2 +-
 src/nix/path-info.cc      |  2 +-
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0440af95fa4..339445aa064 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -480,8 +480,12 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
             if (info->ca != "")
                 jsonPath.attr("ca", info->ca);
 
-            if (showClosureSize)
-                jsonPath.attr("closureSize", getClosureSize(storePath));
+            std::pair<uint64_t, uint64_t> closureSizes;
+
+            if (showClosureSize) {
+                closureSizes = getClosureSize(storePath);
+                jsonPath.attr("closureSize", closureSizes.first);
+            }
 
             if (includeImpureInfo) {
 
@@ -500,6 +504,17 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
                         jsonSigs.elem(sig);
                 }
 
+                auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
+                    std::shared_ptr<const ValidPathInfo>(info));
+
+                if (narInfo) {
+                    if (narInfo->fileHash)
+                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
+                    if (narInfo->fileSize)
+                        jsonPath.attr("downloadSize", narInfo->fileSize);
+                    if (showClosureSize)
+                        jsonPath.attr("closureDownloadSize", closureSizes.second);
+                }
             }
 
         } catch (InvalidPath &) {
@@ -509,14 +524,20 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
 }
 
 
-unsigned long long Store::getClosureSize(const Path & storePath)
+std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
 {
-    unsigned long long totalSize = 0;
+    uint64_t totalNarSize = 0, totalDownloadSize = 0;
     PathSet closure;
     computeFSClosure(storePath, closure, false, false);
-    for (auto & p : closure)
-        totalSize += queryPathInfo(p)->narSize;
-    return totalSize;
+    for (auto & p : closure) {
+        auto info = queryPathInfo(p);
+        totalNarSize += info->narSize;
+        auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
+            std::shared_ptr<const ValidPathInfo>(info));
+        if (narInfo)
+            totalDownloadSize += narInfo->fileSize;
+    }
+    return {totalNarSize, totalDownloadSize};
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 586d35e84ff..3247a67678c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -495,7 +495,7 @@ public:
     /* Return the size of the closure of the specified path, that is,
        the sum of the size of the NAR serialisation of each path in
        the closure. */
-    unsigned long long getClosureSize(const Path & storePath);
+    std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath);
 
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index f7610ab0814..ca02a4c929b 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -84,7 +84,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                     std::cout << '\t' << std::setw(11) << info->narSize;
 
                 if (showClosureSize)
-                    std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath);
+                    std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath).first;
 
                 if (showSigs) {
                     std::cout << '\t';

From 1e0f59ae14005c0dc3bcb0981bf6407e7ebaaf1a Mon Sep 17 00:00:00 2001
From: Harmen <harm@dds.nl>
Date: Sat, 15 Jul 2017 21:06:30 +0200
Subject: [PATCH 0455/2196] use sha256 hashes in the examples

And fix a dead link.
---
 doc/manual/expressions/advanced-attributes.xml | 4 ++--
 doc/manual/expressions/builtins.xml            | 4 ++--
 doc/manual/expressions/expression-syntax.xml   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 83ad6eefc8b..f3cf9837130 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -139,7 +139,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 <programlisting>
 fetchurl {
   url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
-  md5 = "70c9ccf9fac07f762c24f2df2290784d";
+  sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
 
@@ -150,7 +150,7 @@ fetchurl {
 <programlisting>
 fetchurl {
   url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
-  md5 = "70c9ccf9fac07f762c24f2df2290784d";
+  sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
 
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 125cbd78255..229a73fb12e 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -973,8 +973,8 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = http://nix.cs.uu.nl/dist/tarballs/hello-2.1.1.tar.gz;
-    md5 = "70c9ccf9fac07f762c24f2df2290784d";
+    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;
 }</programlisting>
diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
index 6f1a3a10c4e..b674b0d6000 100644
--- a/doc/manual/expressions/expression-syntax.xml
+++ b/doc/manual/expressions/expression-syntax.xml
@@ -16,7 +16,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
     url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
-    md5 = "70c9ccf9fac07f762c24f2df2290784d";
+    sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />
 }</programlisting>

From 4ec6eb1fdf513d93090d5898762d1186eb6feb0d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jul 2017 11:38:15 +0200
Subject: [PATCH 0456/2196] Fix accidental printError

---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 339445aa064..27bfbcb8f12 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -614,7 +614,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             checkInterrupt();
 
             if (!dstStore->isValidPath(storePath)) {
-                printError("copying ‘%s’...", storePath);
+                printInfo("copying ‘%s’...", storePath);
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
             }
         });

From 49304bae812f2c5334f304a4ea6a4e06360b33da Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jul 2017 13:07:08 +0200
Subject: [PATCH 0457/2196] Make the hashes mirrors used by builtins.fetchurl
 configurable

In particular, this allows it to be disabled in our tests.
---
 doc/manual/command-ref/conf-file.xml | 28 ++++++++++++++++++++++++++++
 src/libstore/builtins.cc             | 15 +++++++++------
 src/libstore/globals.hh              |  3 +++
 tests/fetchurl.sh                    |  4 ++--
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index cde32b35f5b..3512777dd71 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -660,6 +660,34 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
+
+    <listitem><para>A list of web servers used by
+    <function>builtins.fetchurl</function> to obtain files by
+    hash. The default is
+    <literal>http://tarballs.nixos.org/</literal>. Given a hash type
+    <replaceable>ht</replaceable> and a base-16 hash
+    <replaceable>h</replaceable>, Nix will try to download the file
+    from
+    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
+    This allows files to be downloaded even if they have disappeared
+    from their original URI. For example, given the default mirror
+    <literal>http://tarballs.nixos.org/</literal>, when building the derivation
+
+<programlisting>
+builtins.fetchurl {
+  url = https://example.org/foo-1.2.3.tar.xz;
+  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+}
+</programlisting>
+
+    Nix will attempt to download this file from
+    <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
+    first. If it is not available there, if will try the original URI.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index 8a5cf3327d4..b51b6f85cb0 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -38,12 +38,15 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     std::shared_ptr<std::string> data;
 
-    try {
-        if (getAttr("outputHashMode") == "flat")
-            data = fetch("http://tarballs.nixos.org/" + getAttr("outputHashAlgo") + "/" + getAttr("outputHash"));
-    } catch (Error & e) {
-        debug(e.what());
-    }
+    if (getAttr("outputHashMode") == "flat")
+        for (auto hashedMirror : settings.hashedMirrors.get())
+            try {
+                if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
+                data = fetch(hashedMirror + getAttr("outputHashAlgo") + "/" + getAttr("outputHash"));
+                break;
+            } catch (Error & e) {
+                debug(e.what());
+            }
 
     if (!data) data = fetch(getAttr("url"));
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index c8d67b07110..9ebbf7b477b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -327,6 +327,9 @@ public:
         "Whether builders can acquire new privileges by calling programs with "
         "setuid/setgid bits or with file capabilities."};
 #endif
+
+    Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
+        "A list of servers used by builtins.fetchurl to fetch files by hash."};
 };
 
 
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 808f460258d..02b83525bb3 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
 # Test fetching a flat file.
 hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --option hashed-mirrors '')
 
 cmp $outPath fetchurl.sh
 
@@ -14,7 +14,7 @@ clearStore
 
 hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --option hashed-mirrors '')
 
 cmp $outPath fetchurl.sh
 

From bf6792c0df2163a0501beb14feeb5135c4aa71c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jul 2017 13:13:18 +0200
Subject: [PATCH 0458/2196] Always use base-16 for hashed mirror lookups

In particular, don't use base-64, which we don't support. (We do have
base-32 redirects for hysterical reasons.)

Also, add a test for the hashed mirror feature.
---
 src/libstore/builtins.cc |  3 ++-
 tests/fetchurl.sh        | 13 +++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index b51b6f85cb0..a01dbba4ede 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -42,7 +42,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                data = fetch(hashedMirror + getAttr("outputHashAlgo") + "/" + getAttr("outputHash"));
+                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
                 break;
             } catch (Error & e) {
                 debug(e.what());
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 02b83525bb3..7f2de907049 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -18,6 +18,19 @@ outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh
 
 cmp $outPath fetchurl.sh
 
+# Test the hashed mirror feature.
+clearStore
+
+hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
+hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+
+mirror=$TMPDIR/hashed-mirror
+rm -rf $mirror
+mkdir -p $mirror/sha512
+ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --option hashed-mirrors "file://$mirror")
+
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive
 mkdir -p $TEST_ROOT/archive

From da2ad300545a838b2b6485481271dad9865a5e8f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 18 Jul 2017 08:02:53 -0400
Subject: [PATCH 0459/2196] Update the mailing list.

---
 doc/manual/release-notes/rl-1.2.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml
index dc272c420dd..748fd9e6702 100644
--- a/doc/manual/release-notes/rl-1.2.xml
+++ b/doc/manual/release-notes/rl-1.2.xml
@@ -40,7 +40,7 @@ $ nix-env -i thunderbird --option binary-caches http://cache.nixos.org
     <para>Binary caches are created using <command>nix-push</command>.
     For details on the operation and format of binary caches, see the
     <command>nix-push</command> manpage.  More details are provided in
-    <link xlink:href="http://lists.science.uu.nl/pipermail/nix-dev/2012-September/009826.html">this
+    <link xlink:href="https://nixos.org/nix-dev/2012-September/009826.html">this
     nix-dev posting</link>.</para>
   </listitem>
 

From 364f75e03a2169c794846fb12dd1d2fb49dcabd6 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 18 Jul 2017 08:04:01 -0400
Subject: [PATCH 0460/2196] Update mailing list.

---
 tests/lang/eval-fail-path-slash.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/lang/eval-fail-path-slash.nix
index 530105b3210..8c2e104c788 100644
--- a/tests/lang/eval-fail-path-slash.nix
+++ b/tests/lang/eval-fail-path-slash.nix
@@ -2,5 +2,5 @@
 # This restriction could be lifted sometime,
 # for example if we make '/' a path concatenation operator.
 # See https://github.com/NixOS/nix/issues/1138
-# and http://lists.science.uu.nl/pipermail/nix-dev/2016-June/020829.html
+# and https://nixos.org/nix-dev/2016-June/020829.html
 /nix/store/

From d5e1bffd2a5b5c39f14944fe3ba25414dcd0d508 Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Tue, 18 Jul 2017 17:51:50 -0400
Subject: [PATCH 0461/2196] Do not try to fill fd_set with fd>=FD_SETSIZE

This is UB and causes buffer overflow and crash on linux.
---
 src/libstore/build.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 60b0a531f42..d2a270259a8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3835,6 +3835,9 @@ void Worker::waitForInput()
     int fdMax = 0;
     for (auto & i : children) {
         for (auto & j : i.fds) {
+            if (j >= FD_SETSIZE) {
+                throw BuildError("reached FD_SETSIZE limit");
+            }
             FD_SET(j, &fds);
             if (j >= fdMax) fdMax = j + 1;
         }

From 90825dea518ea078f0783a72cc471a5b3716d198 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jul 2017 19:02:56 +0200
Subject: [PATCH 0462/2196] Add "nix search" command

---
 src/libexpr/get-drvs.cc |  44 ++++++++++----
 src/libexpr/get-drvs.hh |  39 ++++++------
 src/libutil/args.cc     |   1 +
 src/libutil/args.hh     |   4 +-
 src/nix-env/nix-env.cc  |  90 ++++++++++++++--------------
 src/nix-env/user-env.cc |   7 ++-
 src/nix/command.hh      |  34 +++++++----
 src/nix/installables.cc |  16 ++---
 src/nix/search.cc       | 130 ++++++++++++++++++++++++++++++++++++++++
 9 files changed, 263 insertions(+), 102 deletions(-)
 create mode 100644 src/nix/search.cc

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 4200e8fd675..b7e16de7fa4 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -9,7 +9,34 @@
 namespace nix {
 
 
-string DrvInfo::queryDrvPath()
+DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
+    : state(&state), attrs(attrs), attrPath(attrPath)
+{
+}
+
+
+string DrvInfo::queryName() const
+{
+    if (name == "" && attrs) {
+        auto i = attrs->find(state->sName);
+        if (i == attrs->end()) throw TypeError("derivation name missing");
+        name = state->forceStringNoCtx(*i->value);
+    }
+    return name;
+}
+
+
+string DrvInfo::querySystem() const
+{
+    if (system == "" && attrs) {
+        auto i = attrs->find(state->sSystem);
+        system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
+    }
+    return system;
+}
+
+
+string DrvInfo::queryDrvPath() const
 {
     if (drvPath == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
@@ -20,7 +47,7 @@ string DrvInfo::queryDrvPath()
 }
 
 
-string DrvInfo::queryOutPath()
+string DrvInfo::queryOutPath() const
 {
     if (outPath == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
@@ -76,7 +103,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 }
 
 
-string DrvInfo::queryOutputName()
+string DrvInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -225,17 +252,12 @@ static bool getDerivation(EvalState & state, Value & v,
         if (done.find(v.attrs) != done.end()) return false;
         done.insert(v.attrs);
 
-        Bindings::iterator i = v.attrs->find(state.sName);
-        /* !!! We really would like to have a decent back trace here. */
-        if (i == v.attrs->end()) throw TypeError("derivation name missing");
+        DrvInfo drv(state, attrPath, v.attrs);
 
-        Bindings::iterator i2 = v.attrs->find(state.sSystem);
-
-        DrvInfo drv(state, state.forceStringNoCtx(*i->value), attrPath,
-            i2 == v.attrs->end() ? "unknown" : state.forceStringNoCtx(*i2->value, *i2->pos),
-            v.attrs);
+        drv.queryName();
 
         drvs.push_back(drv);
+
         return false;
 
     } catch (AssertionError & e) {
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 37fcbe829d3..82fb8a3ac6a 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -17,31 +17,32 @@ public:
 private:
     EvalState * state;
 
-    string drvPath;
-    string outPath;
-    string outputName;
+    mutable string name;
+    mutable string system;
+    mutable string drvPath;
+    mutable string outPath;
+    mutable string outputName;
     Outputs outputs;
 
-    bool failed; // set if we get an AssertionError
+    bool failed = false; // set if we get an AssertionError
 
-    Bindings * attrs, * meta;
+    Bindings * attrs = nullptr, * meta = nullptr;
 
     Bindings * getMeta();
 
     bool checkMeta(Value & v);
 
 public:
-    string name;
     string attrPath; /* path towards the derivation */
-    string system;
 
-    DrvInfo(EvalState & state) : state(&state), failed(false), attrs(0), meta(0) { };
-    DrvInfo(EvalState & state, const string & name, const string & attrPath, const string & system, Bindings * attrs)
-        : state(&state), failed(false), attrs(attrs), meta(0), name(name), attrPath(attrPath), system(system) { };
+    DrvInfo(EvalState & state) : state(&state) { };
+    DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
 
-    string queryDrvPath();
-    string queryOutPath();
-    string queryOutputName();
+    string queryName() const;
+    string querySystem() const;
+    string queryDrvPath() const;
+    string queryOutPath() const;
+    string queryOutputName() const;
     /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
@@ -58,15 +59,9 @@ public:
     MetaValue queryMetaInfo(EvalState & state, const string & name) const;
     */
 
-    void setDrvPath(const string & s)
-    {
-        drvPath = s;
-    }
-
-    void setOutPath(const string & s)
-    {
-        outPath = s;
-    }
+    void setName(const string & s) { name = s; }
+    void setDrvPath(const string & s) { drvPath = s; }
+    void setOutPath(const string & s) { outPath = s; }
 
     void setFailed() { failed = true; };
     bool hasFailed() { return failed; };
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 0eed4945418..19a45d7e9b3 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -66,6 +66,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
         std::cout << renderLabels({exp.label});
         // FIXME: handle arity > 1
         if (exp.arity == 0) std::cout << "...";
+        if (exp.optional) std::cout << "?";
     }
     std::cout << "\n";
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index ef8a7953e52..37e780dd174 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -164,9 +164,9 @@ public:
     }
 
     /* Expect a string argument. */
-    void expectArg(const std::string & label, string * dest)
+    void expectArg(const std::string & label, string * dest, bool optional = false)
     {
-        expectedArgs.push_back(ExpectedArg{label, 1, false, [=](Strings ss) {
+        expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](Strings ss) {
             *dest = ss.front();
         }});
     }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 10100d6a601..8620cd25574 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -186,7 +186,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
        system. */
     for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) {
         j = i; j++;
-        if (systemFilter != "*" && i->system != systemFilter)
+        if (systemFilter != "*" && i->querySystem() != systemFilter)
             elems.erase(i);
     }
 }
@@ -247,7 +247,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
         for (DrvInfos::const_iterator j = allElems.begin();
              j != allElems.end(); ++j, ++n)
         {
-            DrvName drvName(j->name);
+            DrvName drvName(j->queryName());
             if (i.matches(drvName)) {
                 i.hits++;
                 matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n));
@@ -269,36 +269,36 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
             StringSet multiple;
 
             for (auto & j : matches) {
-                DrvName drvName(j.first.name);
+                DrvName drvName(j.first.queryName());
                 int d = 1;
 
                 Newest::iterator k = newest.find(drvName.name);
 
                 if (k != newest.end()) {
-                    d = j.first.system == k->second.first.system ? 0 :
-                        j.first.system == settings.thisSystem ? 1 :
-                        k->second.first.system == settings.thisSystem ? -1 : 0;
+                    d = j.first.querySystem() == k->second.first.querySystem() ? 0 :
+                        j.first.querySystem() == settings.thisSystem ? 1 :
+                        k->second.first.querySystem() == settings.thisSystem ? -1 : 0;
                     if (d == 0)
                         d = comparePriorities(state, j.first, k->second.first);
                     if (d == 0)
-                        d = compareVersions(drvName.version, DrvName(k->second.first.name).version);
+                        d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version);
                 }
 
                 if (d > 0) {
                     newest.erase(drvName.name);
                     newest.insert(Newest::value_type(drvName.name, j));
-                    multiple.erase(j.first.name);
+                    multiple.erase(j.first.queryName());
                 } else if (d == 0) {
-                    multiple.insert(j.first.name);
+                    multiple.insert(j.first.queryName());
                 }
             }
 
             matches.clear();
             for (auto & j : newest) {
-                if (multiple.find(j.second.first.name) != multiple.end())
+                if (multiple.find(j.second.first.queryName()) != multiple.end())
                     printInfo(
-                        format("warning: there are multiple derivations named ‘%1%’; using the first one")
-                        % j.second.first.name);
+                        "warning: there are multiple derivations named ‘%1%’; using the first one",
+                        j.second.first.queryName());
                 matches.push_back(j.second);
             }
         }
@@ -386,7 +386,8 @@ static void queryInstSources(EvalState & state,
                 if (dash != string::npos)
                     name = string(name, dash + 1);
 
-                DrvInfo elem(state, name, "", "", 0);
+                DrvInfo elem(state, "", nullptr);
+                elem.setName(name);
 
                 if (isDerivation(path)) {
                     elem.setDrvPath(path);
@@ -468,8 +469,8 @@ static void installDerivations(Globals & globals,
            path is not the one we want (e.g., `java-front' versus
            `java-front-0.9pre15899'). */
         if (globals.forceName != "")
-            i.name = globals.forceName;
-        newNames.insert(DrvName(i.name).name);
+            i.setName(globals.forceName);
+        newNames.insert(DrvName(i.queryName()).name);
     }
 
 
@@ -484,17 +485,17 @@ static void installDerivations(Globals & globals,
             DrvInfos installedElems = queryInstalled(*globals.state, profile);
 
             for (auto & i : installedElems) {
-                DrvName drvName(i.name);
+                DrvName drvName(i.queryName());
                 if (!globals.preserveInstalled &&
                     newNames.find(drvName.name) != newNames.end() &&
                     !keep(i))
-                    printInfo(format("replacing old ‘%1%’") % i.name);
+                    printInfo("replacing old ‘%s’", i.queryName());
                 else
                     allElems.push_back(i);
             }
 
             for (auto & i : newElems)
-                printInfo(format("installing ‘%1%’") % i.name);
+                printInfo("installing ‘%s’", i.queryName());
         }
 
         printMissing(*globals.state, newElems);
@@ -548,7 +549,7 @@ static void upgradeDerivations(Globals & globals,
         /* Go through all installed derivations. */
         DrvInfos newElems;
         for (auto & i : installedElems) {
-            DrvName drvName(i.name);
+            DrvName drvName(i.queryName());
 
             try {
 
@@ -569,7 +570,7 @@ static void upgradeDerivations(Globals & globals,
                 for (auto j = availElems.begin(); j != availElems.end(); ++j) {
                     if (comparePriorities(*globals.state, i, *j) > 0)
                         continue;
-                    DrvName newName(j->name);
+                    DrvName newName(j->queryName());
                     if (newName.name == drvName.name) {
                         int d = compareVersions(drvName.version, newName.version);
                         if ((upgradeType == utLt && d < 0) ||
@@ -596,14 +597,13 @@ static void upgradeDerivations(Globals & globals,
                 {
                     const char * action = compareVersions(drvName.version, bestVersion) <= 0
                         ? "upgrading" : "downgrading";
-                    printInfo(
-                        format("%1% ‘%2%’ to ‘%3%’")
-                        % action % i.name % bestElem->name);
+                    printInfo("%1% ‘%2%’ to ‘%3%’",
+                        action, i.queryName(), bestElem->queryName());
                     newElems.push_back(*bestElem);
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addPrefix(format("while trying to find an upgrade for ‘%1%’:\n") % i.name);
+                e.addPrefix(fmt("while trying to find an upgrade for ‘%s’:\n", i.queryName()));
                 throw;
             }
         }
@@ -663,10 +663,10 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs)
 
         /* Update all matching derivations. */
         for (auto & i : installedElems) {
-            DrvName drvName(i.name);
+            DrvName drvName(i.queryName());
             for (auto & j : selectors)
                 if (j.matches(drvName)) {
-                    printInfo(format("setting flag on ‘%1%’") % i.name);
+                    printInfo("setting flag on ‘%1%’", i.queryName());
                     j.hits++;
                     setMetaFlag(*globals.state, i, flagName, flagValue);
                     break;
@@ -702,7 +702,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     DrvInfo & drv(elems.front());
 
     if (globals.forceName != "")
-        drv.name = globals.forceName;
+        drv.setName(globals.forceName);
 
     if (drv.queryDrvPath() != "") {
         PathSet paths = {drv.queryDrvPath()};
@@ -732,7 +732,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
         DrvInfos newElems;
 
         for (auto & i : installedElems) {
-            DrvName drvName(i.name);
+            DrvName drvName(i.queryName());
             bool found = false;
             for (auto & j : selectors)
                 /* !!! the repeated calls to followLinksToStorePath()
@@ -740,7 +740,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
                 if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j))
                     || DrvName(j).matches(drvName))
                 {
-                    printInfo(format("uninstalling ‘%1%’") % i.name);
+                    printInfo("uninstalling ‘%s’", i.queryName());
                     found = true;
                     break;
                 }
@@ -771,9 +771,11 @@ static bool cmpChars(char a, char b)
 
 static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b)
 {
+    auto a_name = a.queryName();
+    auto b_name = b.queryName();
     return lexicographical_compare(
-        a.name.begin(), a.name.end(),
-        b.name.begin(), b.name.end(), cmpChars);
+        a_name.begin(), a_name.end(),
+        b_name.begin(), b_name.end(), cmpChars);
 }
 
 
@@ -822,13 +824,13 @@ typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff;
 static VersionDiff compareVersionAgainstSet(
     const DrvInfo & elem, const DrvInfos & elems, string & version)
 {
-    DrvName name(elem.name);
+    DrvName name(elem.queryName());
 
     VersionDiff diff = cvUnavail;
     version = "?";
 
     for (auto & i : elems) {
-        DrvName name2(i.name);
+        DrvName name2(i.queryName());
         if (name.name == name2.name) {
             int d = compareVersions(name.version, name2.version);
             if (d < 0) {
@@ -857,8 +859,8 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
     for (auto & i : elems) {
         JSONObject pkgObj = topObj.object(i.attrPath);
 
-        pkgObj.attr("name", i.name);
-        pkgObj.attr("system", i.system);
+        pkgObj.attr("name", i.queryName());
+        pkgObj.attr("system", i.querySystem());
 
         JSONObject metaObj = pkgObj.object("meta");
         StringSet metaNames = i.queryMetaNames();
@@ -866,7 +868,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                printError(format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j);
+                printError("derivation ‘%s’ has invalid meta attribute ‘%s’", i.queryName(), j);
                 placeholder.write(nullptr);
             } else {
                 PathSet context;
@@ -963,7 +965,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             try {
                 paths.insert(i.queryOutPath());
             } catch (AssertionError & e) {
-                printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name);
+                printMsg(lvlTalkative, "skipping derivation named ‘%s’ which gives an assertion failure", i.queryName());
                 i.setFailed();
             }
         validPaths = globals.state->store->queryValidPaths(paths);
@@ -1024,9 +1026,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                 columns.push_back(i.attrPath);
 
             if (xmlOutput)
-                attrs["name"] = i.name;
+                attrs["name"] = i.queryName();
             else if (printName)
-                columns.push_back(i.name);
+                columns.push_back(i.queryName());
 
             if (compareVersions) {
                 /* Compare this element against the versions of the
@@ -1059,10 +1061,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             }
 
             if (xmlOutput) {
-                if (i.system != "") attrs["system"] = i.system;
+                if (i.querySystem() != "") attrs["system"] = i.querySystem();
             }
             else if (printSystem)
-                columns.push_back(i.system);
+                columns.push_back(i.querySystem());
 
             if (printDrvPath) {
                 string drvPath = i.queryDrvPath();
@@ -1110,7 +1112,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
                             if (!v)
-                                printError(format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j);
+                                printError("derivation ‘%s’ has invalid meta attribute ‘%s’", i.queryName(), j);
                             else {
                                 if (v->type == tString) {
                                     attrs2["type"] = "string";
@@ -1161,9 +1163,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             cout.flush();
 
         } catch (AssertionError & e) {
-            printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name);
+            printMsg(lvlTalkative, "skipping derivation named ‘%1%’ which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addPrefix(format("while querying the derivation named ‘%1%’:\n") % i.name);
+            e.addPrefix(fmt("while querying the derivation named ‘%1%’:\n", i.queryName()));
             throw;
         }
     }
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index e9997fae57b..df5105f12c2 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -56,9 +56,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         state.mkAttrs(v, 16);
 
         mkString(*state.allocAttr(v, state.sType), "derivation");
-        mkString(*state.allocAttr(v, state.sName), i.name);
-        if (!i.system.empty())
-            mkString(*state.allocAttr(v, state.sSystem), i.system);
+        mkString(*state.allocAttr(v, state.sName), i.queryName());
+        auto system = i.querySystem();
+        if (!system.empty())
+            mkString(*state.allocAttr(v, state.sSystem), system);
         mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath());
         if (drvPath != "")
             mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath());
diff --git a/src/nix/command.hh b/src/nix/command.hh
index ae7709b5dc8..53680265381 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -62,17 +62,13 @@ struct Installable
     }
 };
 
-/* A command that operates on a list of "installables", which can be
-   store paths, attribute paths, Nix expressions, etc. */
-struct InstallablesCommand : virtual Args, StoreCommand
+struct SourceExprCommand : virtual Args, StoreCommand
 {
-    std::vector<std::shared_ptr<Installable>> installables;
     Path file;
 
-    InstallablesCommand()
+    SourceExprCommand()
     {
         mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file);
-        expectArgs("installables", &_installables);
     }
 
     /* Return a value representing the Nix expression from which we
@@ -81,14 +77,32 @@ struct InstallablesCommand : virtual Args, StoreCommand
        = import ...; bla = import ...; }’. */
     Value * getSourceExpr(EvalState & state);
 
+    ref<EvalState> getEvalState();
+
+private:
+
+    std::shared_ptr<EvalState> evalState;
+
+    Value * vSourceExpr = 0;
+};
+
+/* A command that operates on a list of "installables", which can be
+   store paths, attribute paths, Nix expressions, etc. */
+struct InstallablesCommand : virtual Args, SourceExprCommand
+{
+    std::vector<std::shared_ptr<Installable>> installables;
+
+    InstallablesCommand()
+    {
+        expectArgs("installables", &_installables);
+    }
+
     std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss);
 
     enum ToStorePathsMode { Build, NoBuild, DryRun };
 
     PathSet toStorePaths(ref<Store> store, ToStorePathsMode mode);
 
-    ref<EvalState> getEvalState();
-
     void prepare() override;
 
     virtual bool useDefaultInstallables() { return true; }
@@ -96,10 +110,6 @@ struct InstallablesCommand : virtual Args, StoreCommand
 private:
 
     Strings _installables;
-
-    std::shared_ptr<EvalState> evalState;
-
-    Value * vSourceExpr = 0;
 };
 
 /* A command that operates on zero or more store paths. */
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 7fad8fe415c..4da736f4d5c 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -12,7 +12,7 @@
 
 namespace nix {
 
-Value * InstallablesCommand::getSourceExpr(EvalState & state)
+Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
     if (vSourceExpr) return vSourceExpr;
 
@@ -59,6 +59,13 @@ Value * InstallablesCommand::getSourceExpr(EvalState & state)
     return vSourceExpr;
 }
 
+ref<EvalState> SourceExprCommand::getEvalState()
+{
+    if (!evalState)
+        evalState = std::make_shared<EvalState>(Strings{}, getStore());
+    return ref<EvalState>(evalState);
+}
+
 struct InstallableStoreDrv : Installable
 {
     Path storePath;
@@ -237,13 +244,6 @@ PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mod
     return outPaths;
 }
 
-ref<EvalState> InstallablesCommand::getEvalState()
-{
-    if (!evalState)
-        evalState = std::make_shared<EvalState>(Strings{}, getStore());
-    return ref<EvalState>(evalState);
-}
-
 void InstallablesCommand::prepare()
 {
     installables = parseInstallables(getStore(), _installables);
diff --git a/src/nix/search.cc b/src/nix/search.cc
new file mode 100644
index 00000000000..813f6d0a622
--- /dev/null
+++ b/src/nix/search.cc
@@ -0,0 +1,130 @@
+#include "command.hh"
+#include "globals.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "names.hh"
+#include "get-drvs.hh"
+
+#include <regex>
+
+using namespace nix;
+
+std::string hilite(const std::string & s, const std::smatch & m)
+{
+    return
+        m.empty()
+        ? s
+        : std::string(m.prefix())
+          + ANSI_RED + std::string(m.str()) + ANSI_NORMAL
+          + std::string(m.suffix());
+}
+
+struct CmdSearch : SourceExprCommand
+{
+    std::string re;
+
+    CmdSearch()
+    {
+        expectArg("regex", &re, true);
+    }
+
+    std::string name() override
+    {
+        return "search";
+    }
+
+    std::string description() override
+    {
+        return "query available packages";
+    }
+
+    void run(ref<Store> store) override
+    {
+        settings.readOnlyMode = true;
+
+        std::regex regex(re, std::regex::extended | std::regex::icase);
+
+        auto state = getEvalState();
+
+        std::function<void(Value *, std::string, bool)> doExpr;
+
+        bool first = true;
+
+        doExpr = [&](Value * v, std::string attrPath, bool toplevel) {
+            debug("at attribute ‘%s’", attrPath);
+
+            try {
+
+                state->forceValue(*v);
+
+                if (v->type == tLambda && toplevel) {
+                    Value * v2 = state->allocValue();
+                    state->autoCallFunction(*state->allocBindings(1), *v, *v2);
+                    v = v2;
+                    state->forceValue(*v);
+                }
+
+                if (state->isDerivation(*v)) {
+
+                    DrvInfo drv(*state, attrPath, v->attrs);
+
+                    DrvName parsed(drv.queryName());
+
+                    std::smatch attrPathMatch;
+                    std::regex_search(attrPath, attrPathMatch, regex);
+
+                    auto name = parsed.name;
+                    std::smatch nameMatch;
+                    std::regex_search(name, nameMatch, regex);
+
+                    std::string description = drv.queryMetaString("description");
+                    std::replace(description.begin(), description.end(), '\n', ' ');
+                    std::smatch descriptionMatch;
+                    std::regex_search(description, descriptionMatch, regex);
+
+                    if (!attrPathMatch.empty()
+                        || !nameMatch.empty()
+                        || !descriptionMatch.empty())
+                    {
+                        if (!first) std::cout << "\n";
+                        first = false;
+
+                        std::cout << fmt(
+                            "Attribute name: %s\n"
+                            "Package name: %s\n"
+                            "Version: %s\n"
+                            "Description: %s\n",
+                            hilite(attrPath, attrPathMatch),
+                            hilite(name, nameMatch),
+                            parsed.version,
+                            hilite(description, descriptionMatch));
+                    }
+                }
+
+                else if (v->type == tAttrs) {
+
+                    if (!toplevel) {
+                        auto attrs = v->attrs;
+                        Bindings::iterator j = attrs->find(state->symbols.create("recurseForDerivations"));
+                        if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) return;
+                    }
+
+                    Bindings::iterator j = v->attrs->find(state->symbols.create("_toplevel"));
+                    bool toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
+
+                    for (auto & i : *v->attrs) {
+                        doExpr(i.value,
+                            attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
+                            toplevel2);
+                    }
+                }
+
+            } catch (AssertionError & e) {
+            }
+        };
+
+        doExpr(getSourceExpr(*state), "", true);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdSearch>());

From b144c4d61738b9d9e3d741e77066a2be115f0ab5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 18 Jul 2017 17:30:09 +0200
Subject: [PATCH 0463/2196] nix search: Add --json flag

---
 src/nix/search.cc | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 813f6d0a622..8aac06ad2cd 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -4,6 +4,8 @@
 #include "eval-inline.hh"
 #include "names.hh"
 #include "get-drvs.hh"
+#include "common-args.hh"
+#include "json.hh"
 
 #include <regex>
 
@@ -19,7 +21,7 @@ std::string hilite(const std::string & s, const std::smatch & m)
           + std::string(m.suffix());
 }
 
-struct CmdSearch : SourceExprCommand
+struct CmdSearch : SourceExprCommand, MixJSON
 {
     std::string re;
 
@@ -50,6 +52,8 @@ struct CmdSearch : SourceExprCommand
 
         bool first = true;
 
+        auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr;
+
         doExpr = [&](Value * v, std::string attrPath, bool toplevel) {
             debug("at attribute ‘%s’", attrPath);
 
@@ -86,18 +90,28 @@ struct CmdSearch : SourceExprCommand
                         || !nameMatch.empty()
                         || !descriptionMatch.empty())
                     {
-                        if (!first) std::cout << "\n";
-                        first = false;
-
-                        std::cout << fmt(
-                            "Attribute name: %s\n"
-                            "Package name: %s\n"
-                            "Version: %s\n"
-                            "Description: %s\n",
-                            hilite(attrPath, attrPathMatch),
-                            hilite(name, nameMatch),
-                            parsed.version,
-                            hilite(description, descriptionMatch));
+                        if (json) {
+
+                            auto jsonElem = jsonOut->object(attrPath);
+
+                            jsonElem.attr("pkgName", parsed.name);
+                            jsonElem.attr("version", parsed.version);
+                            jsonElem.attr("description", description);
+
+                        } else {
+                            if (!first) std::cout << "\n";
+                            first = false;
+
+                            std::cout << fmt(
+                                "Attribute name: %s\n"
+                                "Package name: %s\n"
+                                "Version: %s\n"
+                                "Description: %s\n",
+                                hilite(attrPath, attrPathMatch),
+                                hilite(name, nameMatch),
+                                parsed.version,
+                                hilite(description, descriptionMatch));
+                        }
                     }
                 }
 

From fc3568e2633bf3faa234509cfc6034be95f10b6e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Jul 2017 11:18:06 +0200
Subject: [PATCH 0464/2196] FD_SETSIZE check: BuildError -> Error

BuildError denotes a permanent build failure, which is not the case
here.
---
 src/libstore/build.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d2a270259a8..8f483a90daa 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3835,9 +3835,8 @@ void Worker::waitForInput()
     int fdMax = 0;
     for (auto & i : children) {
         for (auto & j : i.fds) {
-            if (j >= FD_SETSIZE) {
-                throw BuildError("reached FD_SETSIZE limit");
-            }
+            if (j >= FD_SETSIZE)
+                throw Error("reached FD_SETSIZE limit");
             FD_SET(j, &fds);
             if (j >= fdMax) fdMax = j + 1;
         }

From 57a30e101b36a064f09619bf4a3f0b8a3fdcdcad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Jul 2017 16:06:10 +0200
Subject: [PATCH 0465/2196] nix search: Ignore top-level eval errors

$NIX_PATH may contain elements that don't evaluate to an attrset (like
"nixos-config"), so ignore those.
---
 src/nix/installables.cc | 6 +++++-
 src/nix/search.cc       | 9 ++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 4da736f4d5c..59162fdc116 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -16,6 +16,8 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
     if (vSourceExpr) return vSourceExpr;
 
+    auto sToplevel = state.symbols.create("_toplevel");
+
     vSourceExpr = state.allocValue();
 
     if (file != "") {
@@ -29,7 +31,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
         auto searchPath = state.getSearchPath();
 
-        state.mkAttrs(*vSourceExpr, searchPath.size());
+        state.mkAttrs(*vSourceExpr, searchPath.size() + 1);
+
+        mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true);
 
         std::unordered_set<std::string> seen;
 
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 8aac06ad2cd..970dcb9834b 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -54,6 +54,8 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr;
 
+        auto sToplevel = state->symbols.create("_toplevel");
+
         doExpr = [&](Value * v, std::string attrPath, bool toplevel) {
             debug("at attribute ‘%s’", attrPath);
 
@@ -123,7 +125,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                         if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) return;
                     }
 
-                    Bindings::iterator j = v->attrs->find(state->symbols.create("_toplevel"));
+                    Bindings::iterator j = v->attrs->find(sToplevel);
                     bool toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
 
                     for (auto & i : *v->attrs) {
@@ -134,6 +136,11 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 }
 
             } catch (AssertionError & e) {
+            } catch (Error & e) {
+                if (!toplevel) {
+                    e.addPrefix(fmt("While evaluating the attribute ‘%s’:\n", attrPath));
+                    throw;
+                }
             }
         };
 

From c94f3d5575d7af5403274d1e9e2f3c9d72989751 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 20 Jul 2017 13:32:01 +0200
Subject: [PATCH 0466/2196] nix-shell: Use bashInteractive from <nixpkgs>

This adds about 0.1s to nix-shell runtime in the case where
bashInteractive already exists.

See discussion at https://github.com/NixOS/nixpkgs/issues/27493.
---
 src/libexpr/get-drvs.cc    |  7 +++---
 src/libexpr/get-drvs.hh    |  8 +++----
 src/nix-build/local.mk     |  2 +-
 src/nix-build/nix-build.cc | 44 +++++++++++++++++++++++++++++++++++++-
 src/nix/repl.cc            |  6 +++---
 5 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index b7e16de7fa4..66689e3e889 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -267,15 +267,14 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-bool getDerivation(EvalState & state, Value & v, DrvInfo & drv,
+std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
     DrvInfos drvs;
     getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
-    if (drvs.size() != 1) return false;
-    drv = drvs.front();
-    return true;
+    if (drvs.size() != 1) return {};
+    return std::move(drvs.front());
 }
 
 
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 82fb8a3ac6a..32294e45875 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -75,10 +75,10 @@ typedef list<DrvInfo> DrvInfos;
 #endif
 
 
-/* If value `v' denotes a derivation, store information about the
-   derivation in `drv' and return true.  Otherwise, return false. */
-bool getDerivation(EvalState & state, Value & v, DrvInfo & drv,
-    bool ignoreAssertionFailures);
+/* If value `v' denotes a derivation, return a DrvInfo object
+   describing it. Otherwise return nothing. */
+std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+    Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
     Bindings & autoArgs, DrvInfos & drvs,
diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk
index 91532411a50..a2d1c91dfd9 100644
--- a/src/nix-build/local.mk
+++ b/src/nix-build/local.mk
@@ -4,6 +4,6 @@ nix-build_DIR := $(d)
 
 nix-build_SOURCES := $(d)/nix-build.cc
 
-nix-build_LIBS = libmain libstore libutil libformat
+nix-build_LIBS = libmain libexpr libstore libutil libformat
 
 $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell))
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index dc80dd6a583..72f89003d0b 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -13,6 +13,8 @@
 #include "affinity.hh"
 #include "util.hh"
 #include "shared.hh"
+#include "eval.hh"
+#include "get-drvs.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -75,6 +77,8 @@ int main(int argc, char ** argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+        initGC();
+
         auto store = openStore();
         auto dryRun = false;
         auto verbose = false;
@@ -88,6 +92,7 @@ int main(int argc, char ** argv)
         Strings instArgs;
         Strings buildArgs;
         Strings exprs;
+        Strings searchPath;
 
         auto shell = getEnv("SHELL", "/bin/sh");
         std::string envCommand; // interactive shell
@@ -320,6 +325,8 @@ int main(int argc, char ** argv)
             }
         }
 
+        EvalState state(searchPath, store);
+
         if (packages && fromArgs) {
             throw UsageError("‘-p’ and ‘-E’ are mutually exclusive");
         }
@@ -465,7 +472,42 @@ int main(int argc, char ** argv)
 
                 auto envPtrs = stringsToCharPtrs(envStrs);
 
-                auto shell = getEnv("NIX_BUILD_SHELL", "bash");
+                auto shell = getEnv("NIX_BUILD_SHELL", "");
+
+                if (shell == "") {
+
+                    try {
+
+                        auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
+
+                        Value v;
+                        state.eval(expr, v);
+
+                        auto drv = getDerivation(state, v, false);
+                        if (!drv)
+                            throw Error("the ‘bashInteractive’ attribute in <nixpkgs> did not evaluate to a derivation");
+
+                        auto drvPath = drv->queryDrvPath();
+
+                        unsigned long long downloadSize, narSize;
+                        PathSet willBuild, willSubstitute, unknown;
+                        store->queryMissing({drvPath},
+                            willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+                        if (settings.printMissing)
+                            printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+                        store->buildPaths({drvPath});
+
+                        shell = drv->queryOutPath() + "/bin/bash";
+                        if (!pathExists(shell))
+                            throw Error("expected shell ‘%s’ to exist, but it doesn't", shell);
+
+                    } catch (Error & e) {
+                        printError("warning: %s; will use bash from your environment", e.what());
+                        shell = "bash";
+                    }
+                }
 
                 environ = envPtrs.data();
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 437c7903ed4..7d5b8f4668a 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -292,10 +292,10 @@ bool isVarName(const string & s)
 
 
 Path NixRepl::getDerivationPath(Value & v) {
-    DrvInfo drvInfo(state);
-    if (!getDerivation(state, v, drvInfo, false))
+    auto drvInfo = getDerivation(state, v, false);
+    if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    Path drvPath = drvInfo.queryDrvPath();
+    Path drvPath = drvInfo->queryDrvPath();
     if (drvPath == "" || !state.store->isValidPath(drvPath))
         throw Error("expression did not evaluate to a valid derivation");
     return drvPath;

From 4c9ff89c261d84dcc4f88a79654daff2f4790e66 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Jul 2017 15:09:06 +0200
Subject: [PATCH 0467/2196] nix-build/nix-shell: Eliminate call to
 nix-instantiate / nix-store

Note that this removes the need for a derivation symlink, so the
--drv-path and --add-drv-link flags now do nothing.
---
 src/libexpr/eval.hh                    |   2 +
 src/libexpr/parser.y                   |   7 +
 src/libmain/shared.cc                  |   9 +-
 src/libmain/shared.hh                  |   3 +
 src/nix-build/nix-build.cc             | 830 +++++++++++--------------
 src/nix-instantiate/nix-instantiate.cc |  11 +-
 6 files changed, 400 insertions(+), 462 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 1e32db1e86b..04a36b14cef 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -118,6 +118,8 @@ public:
     Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
     Expr * parseExprFromString(const string & s, const Path & basePath);
 
+    Expr * parseStdin();
+
     /* Evaluate an expression read from the given file to normal
        form. */
     void evalFile(const Path & path, Value & v);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 62982650a22..ca3d057451a 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -601,6 +601,13 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 }
 
 
+Expr * EvalState::parseStdin()
+{
+    //Activity act(*logger, lvlTalkative, format("parsing standard input"));
+    return parseExprFromString(drainFD(0), absPath("."));
+}
+
+
 void EvalState::addToSearchPath(const string & s)
 {
     size_t pos = s.find('=');
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 915e7955014..48b00e1a399 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -222,7 +222,14 @@ struct LegacyArgs : public MixCommonArgs
 void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
 {
-    LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv));
+    parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
+}
+
+
+void parseCmdLine(const string & programName, const Strings & args,
+    std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
+{
+    LegacyArgs(programName, parseArg).parseCmdline(args);
 }
 
 
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 6d94a22f788..14e436cc8bb 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -25,6 +25,9 @@ void initNix();
 void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
 
+void parseCmdLine(const string & programName, const Strings & args,
+    std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
+
 void printVersion(const string & programName);
 
 /* Ugh.  No better place to put this. */
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 72f89003d0b..8a3c8a02506 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -5,8 +5,6 @@
 #include <sstream>
 #include <vector>
 
-#include <unistd.h>
-
 #include "store-api.hh"
 #include "globals.hh"
 #include "derivations.hh"
@@ -14,7 +12,10 @@
 #include "util.hh"
 #include "shared.hh"
 #include "eval.hh"
+#include "eval-inline.hh"
 #include "get-drvs.hh"
+#include "common-opts.hh"
+#include "attr-path.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -65,510 +66,435 @@ std::vector<string> shellwords(const string & s)
     return res;
 }
 
-static void maybePrintExecError(ExecError & e)
+void mainWrapped(int argc, char * * argv)
 {
-    if (WIFEXITED(e.status))
-        throw Exit(WEXITSTATUS(e.status));
-    else
-        throw e;
-}
-
-int main(int argc, char ** argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
-        auto store = openStore();
-        auto dryRun = false;
-        auto verbose = false;
-        auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
-        auto pure = false;
-        auto fromArgs = false;
-        auto packages = false;
-        // Same condition as bash uses for interactive shells
-        auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO);
-
-        Strings instArgs;
-        Strings buildArgs;
-        Strings exprs;
-        Strings searchPath;
-
-        auto shell = getEnv("SHELL", "/bin/sh");
-        std::string envCommand; // interactive shell
-        Strings envExclude;
-
-        auto myName = runEnv ? "nix-shell" : "nix-build";
-
-        auto inShebang = false;
-        std::string script;
-        std::vector<string> savedArgs;
-
-        AutoDelete tmpDir(createTempDir("", myName));
-
-        std::string outLink = "./result";
-        auto drvLink = (Path) tmpDir + "/derivation";
-
-        std::vector<string> args;
-        for (int i = 1; i < argc; ++i)
-            args.push_back(argv[i]);
-
-        // Heuristic to see if we're invoked as a shebang script, namely, if we
-        // have a single argument, it's the name of an executable file, and it
-        // starts with "#!".
-        if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) {
-            script = argv[1];
-            if (access(script.c_str(), F_OK) == 0 && access(script.c_str(), X_OK) == 0) {
-                auto lines = tokenizeString<Strings>(readFile(script), "\n");
-                if (std::regex_search(lines.front(), std::regex("^#!"))) {
-                    lines.pop_front();
-                    inShebang = true;
-                    for (int i = 2; i < argc; ++i)
-                        savedArgs.push_back(argv[i]);
-                    args.clear();
-                    for (auto line : lines) {
-                        line = chomp(line);
-                        std::smatch match;
-                        if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$")))
-                            for (const auto & word : shellwords(match[1].str()))
-                                args.push_back(word);
-                    }
+    initNix();
+    initGC();
+
+    auto dryRun = false;
+    auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
+    auto pure = false;
+    auto fromArgs = false;
+    auto packages = false;
+    // Same condition as bash uses for interactive shells
+    auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO);
+    Strings attrPaths;
+    Strings left;
+    Strings searchPath;
+    std::map<string, string> autoArgs_;
+    RepairFlag repair = NoRepair;
+    Path gcRoot;
+    BuildMode buildMode = bmNormal;
+    bool readStdin = false;
+
+    auto shell = getEnv("SHELL", "/bin/sh");
+    std::string envCommand; // interactive shell
+    Strings envExclude;
+
+    auto myName = runEnv ? "nix-shell" : "nix-build";
+
+    auto inShebang = false;
+    std::string script;
+    std::vector<string> savedArgs;
+
+    AutoDelete tmpDir(createTempDir("", myName));
+
+    std::string outLink = "./result";
+
+    Strings args;
+    for (int i = 1; i < argc; ++i)
+        args.push_back(argv[i]);
+
+    // Heuristic to see if we're invoked as a shebang script, namely, if we
+    // have a single argument, it's the name of an executable file, and it
+    // starts with "#!".
+    if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) {
+        script = argv[1];
+        if (access(script.c_str(), F_OK) == 0 && access(script.c_str(), X_OK) == 0) {
+            auto lines = tokenizeString<Strings>(readFile(script), "\n");
+            if (std::regex_search(lines.front(), std::regex("^#!"))) {
+                lines.pop_front();
+                inShebang = true;
+                for (int i = 2; i < argc; ++i)
+                    savedArgs.push_back(argv[i]);
+                args.clear();
+                for (auto line : lines) {
+                    line = chomp(line);
+                    std::smatch match;
+                    if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$")))
+                        for (const auto & word : shellwords(match[1].str()))
+                            args.push_back(word);
                 }
             }
         }
+    }
 
-        for (size_t n = 0; n < args.size(); ++n) {
-            auto arg = args[n];
+    parseCmdLine(myName, args, [&](Strings::iterator & arg, const Strings::iterator & end) {
+        if (*arg == "--help") {
+            deletePath(tmpDir);
+            showManPage(myName);
+        }
 
-            if (arg == "--help") {
-                deletePath(tmpDir);
-                showManPage(myName);
-            }
+        else if (*arg == "--version")
+            printVersion(myName);
 
-            else if (arg == "--version")
-                printVersion(myName);
+        else if (*arg == "--add-drv-link")
+            ; // obsolete
 
-            else if (arg == "--add-drv-link") {
-                drvLink = "./derivation";
-            }
+        else if (*arg == "--no-out-link" || *arg == "--no-link")
+            outLink = (Path) tmpDir + "/result";
 
-            else if (arg == "--no-out-link" || arg == "--no-link") {
-                outLink = (Path) tmpDir + "/result";
-            }
+        else if (*arg == "--attr" || *arg == "-A")
+            attrPaths.push_back(getArg(*arg, arg, end));
 
-            else if (arg == "--drv-link") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError("--drv-link requires an argument");
-                }
-                drvLink = args[n];
-            }
+        else if (*arg == "--drv-link")
+            getArg(*arg, arg, end); // obsolete
 
-            else if (arg == "--out-link" || arg == "-o") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                outLink = args[n];
-            }
+        else if (*arg == "--out-link" || *arg == "-o")
+            outLink = getArg(*arg, arg, end);
 
-            else if (arg == "--attr" || arg == "-A" || arg == "-I") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                instArgs.push_back(arg);
-                instArgs.push_back(args[n]);
-            }
+        else if (parseAutoArgs(arg, end, autoArgs_))
+            ;
 
-            else if (arg == "--arg" || arg == "--argstr") {
-                if (n + 2 >= args.size()) {
-                    throw UsageError(format("%1% requires two arguments") % arg);
-                }
-                instArgs.push_back(arg);
-                instArgs.push_back(args[n + 1]);
-                instArgs.push_back(args[n + 2]);
-                n += 2;
-            }
+        else if (parseSearchPathArg(arg, end, searchPath))
+            ;
 
-            else if (arg == "--option") {
-                if (n + 2 >= args.size()) {
-                    throw UsageError(format("%1% requires two arguments") % arg);
-                }
-                instArgs.push_back(arg);
-                instArgs.push_back(args[n + 1]);
-                instArgs.push_back(args[n + 2]);
-                buildArgs.push_back(arg);
-                buildArgs.push_back(args[n + 1]);
-                buildArgs.push_back(args[n + 2]);
-                settings.set(args[n + 1], args[n + 2]);
-                n += 2;
-            }
+        else if (*arg == "--add-root")
+            gcRoot = getArg(*arg, arg, end);
 
-            else if (arg == "--max-jobs" || arg == "-j" || arg == "--max-silent-time" || arg == "--cores" || arg == "--timeout" || arg == "--add-root") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                buildArgs.push_back(arg);
-                buildArgs.push_back(args[n]);
-            }
+        else if (*arg == "--dry-run")
+            dryRun = true;
 
-            else if (arg == "--dry-run") {
-                buildArgs.push_back("--dry-run");
-                dryRun = true;
-            }
+        else if (*arg == "--repair") {
+            repair = Repair;
+            buildMode = bmRepair;
+        }
 
-            else if (arg == "--show-trace") {
-                instArgs.push_back(arg);
-            }
+        else if (*arg == "--run-env") // obsolete
+            runEnv = true;
 
-            else if (arg == "-") {
-                exprs = Strings{"-"};
-            }
+        else if (*arg == "--command" || *arg == "--run") {
+            if (*arg == "--run")
+                interactive = false;
+            envCommand = getArg(*arg, arg, end) + "\nexit";
+        }
 
-            else if (arg == "--verbose" || (arg.size() >= 2 && arg.substr(0, 2) == "-v")) {
-                buildArgs.push_back(arg);
-                instArgs.push_back(arg);
-                verbose = true;
-            }
+        else if (*arg == "--check")
+            buildMode = bmCheck;
 
-            else if (arg == "--quiet" || arg == "--repair") {
-                buildArgs.push_back(arg);
-                instArgs.push_back(arg);
-            }
+        else if (*arg == "--exclude")
+            envExclude.push_back(getArg(*arg, arg, end));
 
-            else if (arg == "--check") {
-                buildArgs.push_back(arg);
-            }
+        else if (*arg == "--expr" || *arg == "-E")
+            fromArgs = true;
 
-            else if (arg == "--run-env") { // obsolete
-                runEnv = true;
-            }
+        else if (*arg == "--pure") pure = true;
+        else if (*arg == "--impure") pure = false;
 
-            else if (arg == "--command" || arg == "--run") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                envCommand = args[n] + "\nexit";
-                if (arg == "--run")
-                    interactive = false;
-            }
+        else if (*arg == "--packages" || *arg == "-p")
+            packages = true;
 
-            else if (arg == "--exclude") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                envExclude.push_back(args[n]);
-            }
+        else if (inShebang && *arg == "-i") {
+            auto interpreter = getArg(*arg, arg, end);
+            interactive = false;
+            auto execArgs = "";
 
-            else if (arg == "--pure") { pure = true; }
-            else if (arg == "--impure") { pure = false; }
+            auto shellEscape = [](const string & s) {
+                return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
+            };
 
-            else if (arg == "--expr" || arg == "-E") {
-                fromArgs = true;
-                instArgs.push_back("--expr");
-            }
+            // Überhack to support Perl. Perl examines the shebang and
+            // executes it unless it contains the string "perl" or "indir",
+            // or (undocumented) argv[0] does not contain "perl". Exploit
+            // the latter by doing "exec -a".
+            if (std::regex_search(interpreter, std::regex("perl")))
+                execArgs = "-a PERL";
 
-            else if (arg == "--packages" || arg == "-p") {
-                packages = true;
+            std::ostringstream joined;
+            for (const auto & i : savedArgs)
+                joined << shellEscape(i) << ' ';
+
+            if (std::regex_search(interpreter, std::regex("ruby"))) {
+                // Hack for Ruby. Ruby also examines the shebang. It tries to
+                // read the shebang to understand which packages to read from. Since
+                // this is handled via nix-shell -p, we wrap our ruby script execution
+                // in ruby -e 'load' which ignores the shebangs.
+                envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str();
+            } else {
+                envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
             }
+        }
 
-            else if (inShebang && arg == "-i") {
-                n++;
-                if (n >= args.size()) {
-                    throw UsageError(format("%1% requires an argument") % arg);
-                }
-                interactive = false;
-                auto interpreter = args[n];
-                auto execArgs = "";
-
-                auto shellEscape = [](const string & s) {
-                    return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
-                };
-
-                // Überhack to support Perl. Perl examines the shebang and
-                // executes it unless it contains the string "perl" or "indir",
-                // or (undocumented) argv[0] does not contain "perl". Exploit
-                // the latter by doing "exec -a".
-                if (std::regex_search(interpreter, std::regex("perl")))
-                    execArgs = "-a PERL";
-
-                std::ostringstream joined;
-                for (const auto & i : savedArgs)
-                    joined << shellEscape(i) << ' ';
-
-                if (std::regex_search(interpreter, std::regex("ruby"))) {
-                    // Hack for Ruby. Ruby also examines the shebang. It tries to
-                    // read the shebang to understand which packages to read from. Since
-                    // this is handled via nix-shell -p, we wrap our ruby script execution
-                    // in ruby -e 'load' which ignores the shebangs.
-                    envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str();
-                } else {
-                    envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
-                }
-            }
+        else if (*arg == "-")
+            readStdin = true;
 
-            else if (!arg.empty() && arg[0] == '-') {
-                buildArgs.push_back(arg);
-            }
+        else if (*arg != "" && arg->at(0) == '-')
+            return false;
 
-            else if (arg == "-Q" || arg == "--no-build-output") {
-                buildArgs.push_back(arg);
-                instArgs.push_back(arg);
-            }
+        else
+            left.push_back(*arg);
 
-            else {
-                exprs.push_back(arg);
-            }
-        }
+        return true;
+    });
 
-        EvalState state(searchPath, store);
+    if (packages && fromArgs)
+        throw UsageError("‘-p’ and ‘-E’ are mutually exclusive");
+
+    auto store = openStore();
+
+    EvalState state(searchPath, store);
+    state.repair = repair;
+
+    Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+
+    if (packages) {
+        std::ostringstream joined;
+        joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ ";
+        for (const auto & i : left)
+            joined << '(' << i << ") ";
+        joined << "]; } \"\"";
+        fromArgs = true;
+        left = {joined.str()};
+    } else if (!fromArgs) {
+        if (left.empty() && runEnv && pathExists("shell.nix"))
+            left = {"shell.nix"};
+        if (left.empty())
+            left = {"default.nix"};
+    }
 
-        if (packages && fromArgs) {
-            throw UsageError("‘-p’ and ‘-E’ are mutually exclusive");
-        }
+    if (runEnv)
+        setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1);
 
-        if (packages) {
-            instArgs.push_back("--expr");
-            std::ostringstream joined;
-            joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ ";
-            for (const auto & i : exprs)
-                joined << '(' << i << ") ";
-            joined << "]; } \"\"";
-            exprs = Strings{joined.str()};
-        } else if (!fromArgs) {
-            if (exprs.empty() && runEnv && access("shell.nix", F_OK) == 0)
-                exprs.push_back("shell.nix");
-            if (exprs.empty())
-                exprs.push_back("default.nix");
+    /* Parse the expressions. */
+    std::vector<Expr *> exprs;
+
+    if (readStdin)
+        exprs = {state.parseStdin()};
+    else
+        for (auto i : left) {
+            if (fromArgs)
+                exprs.push_back(state.parseExprFromString(i, absPath(".")));
+            else
+                /* If we're in a #! script, interpret filenames
+                   relative to the script. */
+                exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state,
+                    inShebang && !packages ? absPath(i, dirOf(script)) : i))));
         }
 
-        if (runEnv)
-            setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1);
-
-        for (auto & expr : exprs) {
-            // Instantiate.
-            std::vector<string> drvPaths;
-            if (!std::regex_match(expr, std::regex("^/.*\\.drv$"))) {
-                // If we're in a #! script, interpret filenames relative to the
-                // script.
-                if (inShebang && !packages)
-                    expr = absPath(expr, dirOf(script));
-
-                Strings instantiateArgs{"--add-root", drvLink, "--indirect"};
-                for (const auto & arg : instArgs)
-                    instantiateArgs.push_back(arg);
-                instantiateArgs.push_back(expr);
-                try {
-                    auto instOutput = runProgram(settings.nixBinDir + "/nix-instantiate", false, instantiateArgs);
-                    drvPaths = tokenizeString<std::vector<string>>(instOutput);
-                } catch (ExecError & e) {
-                    maybePrintExecError(e);
-                }
-            } else {
-                drvPaths.push_back(expr);
-            }
+    /* Evaluate them into derivations. */
+    DrvInfos drvs;
 
-            if (runEnv) {
-                if (drvPaths.size() != 1)
-                    throw UsageError("a single derivation is required");
-                auto drvPath = drvPaths[0];
-                drvPath = drvPath.substr(0, drvPath.find_first_of('!'));
-                if (isLink(drvPath))
-                    drvPath = readLink(drvPath);
-                auto drv = store->derivationFromPath(drvPath);
-
-                // Build or fetch all dependencies of the derivation.
-                Strings nixStoreArgs{"-r", "--no-output", "--no-gc-warning"};
-                for (const auto & arg : buildArgs)
-                    nixStoreArgs.push_back(arg);
-                for (const auto & input : drv.inputDrvs)
-                    if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); }))
-                        nixStoreArgs.push_back(input.first);
-                for (const auto & src : drv.inputSrcs)
-                    nixStoreArgs.push_back(src);
-
-                try {
-                    runProgram(settings.nixBinDir + "/nix-store", false, nixStoreArgs);
-                } catch (ExecError & e) {
-                    maybePrintExecError(e);
-                }
+    if (attrPaths.empty()) attrPaths = {""};
 
-                if (dryRun) return;
+    for (auto e : exprs) {
+        Value vRoot;
+        state.eval(e, vRoot);
 
-                // Set the environment.
-                auto env = getEnv();
+        for (auto & i : attrPaths) {
+            Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot));
+            state.forceValue(v);
+            getDerivations(state, v, "", autoArgs, drvs, false);
+        }
+    }
 
-                auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
+    auto buildPaths = [&](const PathSet & paths) {
+        /* Note: we do this even when !printMissing to efficiently
+           fetch binary cache data. */
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        store->queryMissing(paths,
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
 
-                if (pure) {
-                    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
-                    decltype(env) newEnv;
-                    for (auto & i : env)
-                        if (keepVars.count(i.first))
-                            newEnv.emplace(i);
-                    env = newEnv;
-                    // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile.
-                    env["__ETC_PROFILE_SOURCED"] = "1";
-                }
+        if (settings.printMissing)
+            printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
 
-                env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
-                env["NIX_STORE"] = store->storeDir;
-                env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
-
-                auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", ""));
-
-                bool keepTmp = false;
-                int fileNr = 0;
-
-                for (auto & var : drv.env)
-                    if (passAsFile.count(var.first)) {
-                        keepTmp = true;
-                        string fn = ".attr-" + std::to_string(fileNr++);
-                        Path p = (Path) tmpDir + "/" + fn;
-                        writeFile(p, var.second);
-                        env[var.first + "Path"] = p;
-                    } else
-                        env[var.first] = var.second;
-
-                restoreAffinity();
-
-                // Run a shell using the derivation's environment.  For
-                // convenience, source $stdenv/setup to setup additional
-                // environment variables and shell functions.  Also don't lose
-                // the current $PATH directories.
-                auto rcfile = (Path) tmpDir + "/rc";
-                writeFile(rcfile, fmt(
-                        (keepTmp ? "" : "rm -rf '%1%'; "s) +
-                        "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; "
-                        "%2%"
-                        "dontAddDisableDepTrack=1; "
-                        "[ -e $stdenv/setup ] && source $stdenv/setup; "
-                        "%3%"
-                        "set +e; "
-                        R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
-                        "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
-                        "unset NIX_ENFORCE_PURITY; "
-                        "unset NIX_INDENT_MAKE; "
-                        "shopt -u nullglob; "
-                        "unset TZ; %4%"
-                        "%5%",
-                        (Path) tmpDir,
-                        (pure ? "" : "p=$PATH; "),
-                        (pure ? "" : "PATH=$PATH:$p; unset p; "),
-                        (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
-                        envCommand));
-
-                Strings envStrs;
-                for (auto & i : env)
-                    envStrs.push_back(i.first + "=" + i.second);
-
-                auto args = interactive
-                    ? Strings{"bash", "--rcfile", rcfile}
-                    : Strings{"bash", rcfile};
-
-                auto envPtrs = stringsToCharPtrs(envStrs);
-
-                auto shell = getEnv("NIX_BUILD_SHELL", "");
-
-                if (shell == "") {
-
-                    try {
-
-                        auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
-
-                        Value v;
-                        state.eval(expr, v);
-
-                        auto drv = getDerivation(state, v, false);
-                        if (!drv)
-                            throw Error("the ‘bashInteractive’ attribute in <nixpkgs> did not evaluate to a derivation");
-
-                        auto drvPath = drv->queryDrvPath();
-
-                        unsigned long long downloadSize, narSize;
-                        PathSet willBuild, willSubstitute, unknown;
-                        store->queryMissing({drvPath},
-                            willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-                        if (settings.printMissing)
-                            printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
-
-                        store->buildPaths({drvPath});
-
-                        shell = drv->queryOutPath() + "/bin/bash";
-                        if (!pathExists(shell))
-                            throw Error("expected shell ‘%s’ to exist, but it doesn't", shell);
-
-                    } catch (Error & e) {
-                        printError("warning: %s; will use bash from your environment", e.what());
-                        shell = "bash";
-                    }
-                }
+        if (!dryRun)
+            store->buildPaths(paths, buildMode);
+    };
 
-                environ = envPtrs.data();
+    if (runEnv) {
+        if (drvs.size() != 1)
+            throw UsageError("nix-shell requires a single derivation");
 
-                auto argPtrs = stringsToCharPtrs(args);
+        auto & drvInfo = drvs.front();
+        auto drv = store->derivationFromPath(drvInfo.queryDrvPath());
 
-                restoreSignals();
+        PathSet pathsToBuild;
 
-                execvp(shell.c_str(), argPtrs.data());
+        /* Figure out what bash shell to use. If $NIX_BUILD_SHELL
+           is not set, then build bashInteractive from
+           <nixpkgs>. */
+        auto shell = getEnv("NIX_BUILD_SHELL", "");
 
-                throw SysError("executing shell ‘%s’", shell);
-            }
+        if (shell == "") {
 
-            // Ugly hackery to make "nix-build -A foo.all" produce symlinks
-            // ./result, ./result-dev, and so on, rather than ./result,
-            // ./result-2-dev, and so on.  This combines multiple derivation
-            // paths into one "/nix/store/drv-path!out1,out2,..." argument.
-            std::string prevDrvPath;
-            Strings drvPaths2;
-            for (const auto & drvPath : drvPaths) {
-                auto p = drvPath;
-                std::string output = "out";
-                std::smatch match;
-                if (std::regex_match(drvPath, match, std::regex("(.*)!(.*)"))) {
-                    p = match[1].str();
-                    output = match[2].str();
-                }
-                auto target = readLink(p);
-                if (verbose)
-                    std::cerr << "derivation is " << target << '\n';
-                if (target == prevDrvPath) {
-                    auto last = drvPaths2.back();
-                    drvPaths2.pop_back();
-                    drvPaths2.push_back(last + "," + output);
-                } else {
-                    drvPaths2.push_back(target + "!" + output);
-                    prevDrvPath = target;
-                }
-            }
-            // Build.
-            Strings outPaths;
-            Strings nixStoreArgs{"--add-root", outLink, "--indirect", "-r"};
-            for (const auto & arg : buildArgs)
-                nixStoreArgs.push_back(arg);
-            for (const auto & path : drvPaths2)
-                nixStoreArgs.push_back(path);
-
-            std::string nixStoreRes;
             try {
-                nixStoreRes = runProgram(settings.nixBinDir + "/nix-store", false, nixStoreArgs);
-            } catch (ExecError & e) {
-                maybePrintExecError(e);
+                auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
+
+                Value v;
+                state.eval(expr, v);
+
+                auto drv = getDerivation(state, v, false);
+                if (!drv)
+                    throw Error("the ‘bashInteractive’ attribute in <nixpkgs> did not evaluate to a derivation");
+
+                pathsToBuild.insert(drv->queryDrvPath());
+
+                shell = drv->queryOutPath() + "/bin/bash";
+
+            } catch (Error & e) {
+                printError("warning: %s; will use bash from your environment", e.what());
+                shell = "bash";
             }
+        }
+
+        // Build or fetch all dependencies of the derivation.
+        for (const auto & input : drv.inputDrvs)
+            if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); }))
+                pathsToBuild.insert(input.first);
+        for (const auto & src : drv.inputSrcs)
+            pathsToBuild.insert(src);
 
-            for (const auto & outpath : tokenizeString<std::vector<string>>(nixStoreRes))
-                outPaths.push_back(chomp(outpath));
+        buildPaths(pathsToBuild);
 
-            if (dryRun)
-                continue;
+        if (dryRun) return;
 
-            for (const auto & outPath : outPaths)
-                std::cout << readLink(outPath) << '\n';
+        // Set the environment.
+        auto env = getEnv();
+
+        auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
+
+        if (pure) {
+            std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+            decltype(env) newEnv;
+            for (auto & i : env)
+                if (keepVars.count(i.first))
+                    newEnv.emplace(i);
+            env = newEnv;
+            // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile.
+            env["__ETC_PROFILE_SOURCED"] = "1";
         }
+
+        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
+        env["NIX_STORE"] = store->storeDir;
+        env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
+
+        auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", ""));
+
+        bool keepTmp = false;
+        int fileNr = 0;
+
+        for (auto & var : drv.env)
+            if (passAsFile.count(var.first)) {
+                keepTmp = true;
+                string fn = ".attr-" + std::to_string(fileNr++);
+                Path p = (Path) tmpDir + "/" + fn;
+                writeFile(p, var.second);
+                env[var.first + "Path"] = p;
+            } else
+                env[var.first] = var.second;
+
+        restoreAffinity();
+
+        /* Run a shell using the derivation's environment.  For
+           convenience, source $stdenv/setup to setup additional
+           environment variables and shell functions.  Also don't
+           lose the current $PATH directories. */
+        auto rcfile = (Path) tmpDir + "/rc";
+        writeFile(rcfile, fmt(
+                (keepTmp ? "" : "rm -rf '%1%'; "s) +
+                "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; "
+                "%2%"
+                "dontAddDisableDepTrack=1; "
+                "[ -e $stdenv/setup ] && source $stdenv/setup; "
+                "%3%"
+                "set +e; "
+                R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
+                "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
+                "unset NIX_ENFORCE_PURITY; "
+                "unset NIX_INDENT_MAKE; "
+                "shopt -u nullglob; "
+                "unset TZ; %4%"
+                "%5%",
+                (Path) tmpDir,
+                (pure ? "" : "p=$PATH; "),
+                (pure ? "" : "PATH=$PATH:$p; unset p; "),
+                (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
+                envCommand));
+
+        Strings envStrs;
+        for (auto & i : env)
+            envStrs.push_back(i.first + "=" + i.second);
+
+        auto args = interactive
+            ? Strings{"bash", "--rcfile", rcfile}
+            : Strings{"bash", rcfile};
+
+        auto envPtrs = stringsToCharPtrs(envStrs);
+
+        environ = envPtrs.data();
+
+        auto argPtrs = stringsToCharPtrs(args);
+
+        restoreSignals();
+
+        execvp(shell.c_str(), argPtrs.data());
+
+        throw SysError("executing shell ‘%s’", shell);
+    }
+
+    else {
+
+        PathSet pathsToBuild;
+
+        std::map<Path, Path> drvPrefixes;
+        std::map<Path, Path> resultSymlinks;
+        std::vector<Path> outPaths;
+
+        for (auto & drvInfo : drvs) {
+            auto drvPath = drvInfo.queryDrvPath();
+            pathsToBuild.insert(drvPath);
+
+            auto outputName = drvInfo.queryOutputName();
+            if (outputName == "")
+                throw Error("derivation ‘%s’ lacks an ‘outputName’ attribute", drvPath);
+
+            pathsToBuild.insert(drvPath + (outputName != "out" ? "!" + outputName : ""));
+
+            std::string drvPrefix;
+            auto i = drvPrefixes.find(drvPath);
+            if (i != drvPrefixes.end())
+                drvPrefix = i->second;
+            else {
+                drvPrefix = outLink;
+                if (drvPrefixes.size())
+                    drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
+                drvPrefixes[drvPath] = drvPrefix;
+            }
+
+            std::string symlink = drvPrefix;
+            if (outputName != "out") symlink += "-" + outputName;
+
+            resultSymlinks[symlink] = drvInfo.queryOutPath();
+            outPaths.push_back(drvInfo.queryOutPath());
+        }
+
+        buildPaths(pathsToBuild);
+
+        if (dryRun) return;
+
+        for (auto & symlink : resultSymlinks)
+            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+                store2->addPermRoot(symlink.second, absPath(symlink.first), true);
+
+        for (auto & path : outPaths)
+            std::cout << path << '\n';
+    }
+}
+
+int main(int argc, char * * argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        return mainWrapped(argc, argv);
     });
 }
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index a5d12c1466f..49a14cb6445 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -17,13 +17,6 @@
 using namespace nix;
 
 
-static Expr * parseStdin(EvalState & state)
-{
-    //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return state.parseExprFromString(drainFD(0), absPath("."));
-}
-
-
 static Path gcRoot;
 static int rootNr = 0;
 static bool indirectRoot = false;
@@ -166,7 +159,7 @@ int main(int argc, char * * argv)
 
         Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
 
-        if (attrPaths.empty()) attrPaths.push_back("");
+        if (attrPaths.empty()) attrPaths = {""};
 
         if (findFile) {
             for (auto & i : files) {
@@ -178,7 +171,7 @@ int main(int argc, char * * argv)
         }
 
         if (readStdin) {
-            Expr * e = parseStdin(state);
+            Expr * e = state.parseStdin();
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         } else if (files.empty() && !fromArgs)

From 57b95057311d4dafb948c78889693a98ec349460 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Jul 2017 17:21:46 +0200
Subject: [PATCH 0468/2196] nix search: Add a cache

The package list is now cached in
~/.cache/nix/package-search.json. This gives a substantial speedup to
"nix search" queries. For example (on an SSD):

First run: (no package search cache, cold page cache)

  $ time nix search blender
  Attribute name: nixpkgs.blender
  Package name: blender
  Version: 2.78c
  Description: 3D Creation/Animation/Publishing System

  real    0m6.516s

Second run: (package search cache populated)

  $ time nix search blender
  Attribute name: nixpkgs.blender
  Package name: blender
  Version: 2.78c
  Description: 3D Creation/Animation/Publishing System

  real    0m0.143s
---
 src/libutil/json.cc | 20 +++++++-----
 src/libutil/json.hh | 14 ++++++--
 src/nix/search.cc   | 78 +++++++++++++++++++++++++++++++++++++++------
 3 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/src/libutil/json.cc b/src/libutil/json.cc
index b8b8ef9c8cc..813b257016e 100644
--- a/src/libutil/json.cc
+++ b/src/libutil/json.cc
@@ -50,20 +50,22 @@ template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t
 JSONWriter::JSONWriter(std::ostream & str, bool indent)
     : state(new JSONState(str, indent))
 {
-    state->stack.push_back(this);
+    state->stack++;
 }
 
 JSONWriter::JSONWriter(JSONState * state)
     : state(state)
 {
-    state->stack.push_back(this);
+    state->stack++;
 }
 
 JSONWriter::~JSONWriter()
 {
-    assertActive();
-    state->stack.pop_back();
-    if (state->stack.empty()) delete state;
+    if (state) {
+        assertActive();
+        state->stack--;
+        if (state->stack == 0) delete state;
+    }
 }
 
 void JSONWriter::comma()
@@ -121,9 +123,11 @@ void JSONObject::open()
 
 JSONObject::~JSONObject()
 {
-    state->depth--;
-    if (state->indent && !first) indent();
-    state->str << "}";
+    if (state) {
+        state->depth--;
+        if (state->indent && !first) indent();
+        state->str << "}";
+    }
 }
 
 void JSONObject::attr(const std::string & s)
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
index 595e9bbe349..02a39917fb5 100644
--- a/src/libutil/json.hh
+++ b/src/libutil/json.hh
@@ -21,11 +21,11 @@ protected:
         std::ostream & str;
         bool indent;
         size_t depth = 0;
-        std::vector<JSONWriter *> stack;
+        size_t stack = 0;
         JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
         ~JSONState()
         {
-            assert(stack.empty());
+            assert(stack == 0);
         }
     };
 
@@ -41,7 +41,7 @@ protected:
 
     void assertActive()
     {
-        assert(!state->stack.empty() && state->stack.back() == this);
+        assert(state->stack != 0);
     }
 
     void comma();
@@ -117,6 +117,14 @@ public:
         open();
     }
 
+    JSONObject(const JSONObject & obj) = delete;
+
+    JSONObject(JSONObject && obj)
+        : JSONWriter(obj.state)
+    {
+        obj.state = 0;
+    }
+
     ~JSONObject();
 
     template<typename T>
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 970dcb9834b..d3e018876a0 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -6,8 +6,10 @@
 #include "get-drvs.hh"
 #include "common-args.hh"
 #include "json.hh"
+#include "json-to-value.hh"
 
 #include <regex>
+#include <fstream>
 
 using namespace nix;
 
@@ -25,9 +27,23 @@ struct CmdSearch : SourceExprCommand, MixJSON
 {
     std::string re;
 
+    bool writeCache = true;
+    bool useCache = true;
+
     CmdSearch()
     {
         expectArg("regex", &re, true);
+
+        mkFlag()
+            .longName("update-cache")
+            .shortName('u')
+            .description("update the package search cache")
+            .handler([&](Strings ss) { writeCache = true; useCache = false; });
+
+        mkFlag()
+            .longName("no-cache")
+            .description("do not use or update the package search cache")
+            .handler([&](Strings ss) { writeCache = false; useCache = false; });
     }
 
     std::string name() override
@@ -48,15 +64,18 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         auto state = getEvalState();
 
-        std::function<void(Value *, std::string, bool)> doExpr;
-
         bool first = true;
 
         auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr;
 
         auto sToplevel = state->symbols.create("_toplevel");
+        auto sRecurse = state->symbols.create("recurseForDerivations");
+
+        bool fromCache = false;
 
-        doExpr = [&](Value * v, std::string attrPath, bool toplevel) {
+        std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
+
+        doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
             debug("at attribute ‘%s’", attrPath);
 
             try {
@@ -115,23 +134,41 @@ struct CmdSearch : SourceExprCommand, MixJSON
                                 hilite(description, descriptionMatch));
                         }
                     }
+
+                    if (cache) {
+                        cache->attr("type", "derivation");
+                        cache->attr("name", drv.queryName());
+                        cache->attr("system", drv.querySystem());
+                        if (description != "") {
+                            auto meta(cache->object("meta"));
+                            meta.attr("description", description);
+                        }
+                    }
                 }
 
                 else if (v->type == tAttrs) {
 
                     if (!toplevel) {
                         auto attrs = v->attrs;
-                        Bindings::iterator j = attrs->find(state->symbols.create("recurseForDerivations"));
-                        if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) return;
+                        Bindings::iterator j = attrs->find(sRecurse);
+                        if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
+                            debug("skip attribute ‘%s’", attrPath);
+                            return;
+                        }
                     }
 
-                    Bindings::iterator j = v->attrs->find(sToplevel);
-                    bool toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
+                    bool toplevel2 = false;
+                    if (!fromCache) {
+                        Bindings::iterator j = v->attrs->find(sToplevel);
+                        toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
+                    }
 
                     for (auto & i : *v->attrs) {
+                        auto cache2 =
+                            cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr;
                         doExpr(i.value,
                             attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
-                            toplevel2);
+                            toplevel2 || fromCache, cache2 ? cache2.get() : nullptr);
                     }
                 }
 
@@ -144,7 +181,30 @@ struct CmdSearch : SourceExprCommand, MixJSON
             }
         };
 
-        doExpr(getSourceExpr(*state), "", true);
+        Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json";
+
+        if (useCache && pathExists(jsonCacheFileName)) {
+
+            Value vRoot;
+            parseJSON(*state, readFile(jsonCacheFileName), vRoot);
+
+            fromCache = true;
+
+            doExpr(&vRoot, "", true, nullptr);
+        }
+
+        else {
+            Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
+
+            std::ofstream jsonCacheFile(tmpFile);
+
+            auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
+
+            doExpr(getSourceExpr(*state), "", true, cache.get());
+
+            if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
+                throw SysError("cannot rename ‘%s’ to ‘%s’", tmpFile, jsonCacheFileName);
+        }
     }
 };
 

From 6d7de7f3ded6254ea3a284b2c9b6a51fc8b041bb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Jul 2017 16:16:08 +0200
Subject: [PATCH 0469/2196] builtins.fetchgit: Cache hash -> store path
 mappings

This prevents an expensive call to addToStore() in the cached case.
---
 src/libexpr/primops/fetchgit.cc | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 3e4ece2cffd..1ac9b364f52 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -2,6 +2,7 @@
 #include "eval-inline.hh"
 #include "download.hh"
 #include "store-api.hh"
+#include "pathlocks.hh"
 
 namespace nix {
 
@@ -28,7 +29,18 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
 
     unlink(localRefFile.c_str());
 
-    debug(format("got revision ‘%s’") % commitHash);
+    printTalkative("using revision %s of repo ‘%s’", uri, commitHash);
+
+    Path storeLink = cacheDir + "/" + commitHash + ".link";
+    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on ‘%1%’...", storeLink));
+
+    if (pathExists(storeLink)) {
+        auto storePath = readLink(storeLink);
+        store->addTempRoot(storePath);
+        if (store->isValidPath(storePath)) {
+            return storePath;
+        }
+    }
 
     // FIXME: should pipe this, or find some better way to extract a
     // revision.
@@ -39,7 +51,11 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
 
     runProgram("tar", true, { "x", "-C", tmpDir }, tar);
 
-    return store->addToStore("git-export", tmpDir);
+    auto storePath = store->addToStore("git-export", tmpDir);
+
+    replaceSymlink(storePath, storeLink);
+
+    return storePath;
 }
 
 static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v)

From 69deca194ec789fa63d222bbd6549dab73328022 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Jul 2017 17:02:25 +0200
Subject: [PATCH 0470/2196] builtins.fetchgit: Use proper refs locally

---
 src/libexpr/primops/fetchgit.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 1ac9b364f52..b64a00b6146 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -20,15 +20,14 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
 
     //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri);
 
-    std::string localRef = "pid-" + std::to_string(getpid());
+    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, rev)).to_string(Base32, false);
+
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
-    runProgram("git", true, { "-C", cacheDir, "fetch", uri, rev + ":" + localRef });
+    runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, rev + ":" + localRef });
 
     std::string commitHash = chomp(readFile(localRefFile));
 
-    unlink(localRefFile.c_str());
-
     printTalkative("using revision %s of repo ‘%s’", uri, commitHash);
 
     Path storeLink = cacheDir + "/" + commitHash + ".link";

From 9f64cb89cbbd0cd0540ad99e3578b6cecd385a81 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Jul 2017 17:15:09 +0200
Subject: [PATCH 0471/2196] builtins.fetchgit: Respect tarball-ttl

I.e. if the local ref is more recent than tarball-ttl seconds, then
don't check the remote.
---
 src/libexpr/primops/fetchgit.cc | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index b64a00b6146..3ab2644c8b0 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -4,6 +4,8 @@
 #include "store-api.hh"
 #include "pathlocks.hh"
 
+#include <sys/time.h>
+
 namespace nix {
 
 Path exportGit(ref<Store> store, const std::string & uri, const std::string & rev)
@@ -24,7 +26,23 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
 
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
-    runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, rev + ":" + localRef });
+    /* If the local ref is older than ‘tarball-ttl’ seconds, do a git
+       fetch to update the local ref to the remote ref. */
+    time_t now = time(0);
+    struct stat st;
+    if (stat(localRefFile.c_str(), &st) != 0 ||
+        st.st_mtime < now - settings.tarballTtl)
+    {
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, rev + ":" + localRef });
+
+        struct timeval times[2];
+        times[0].tv_sec = now;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = now;
+        times[1].tv_usec = 0;
+
+        utimes(localRefFile.c_str(), times);
+    }
 
     std::string commitHash = chomp(readFile(localRefFile));
 

From 7480f4f9a478d294eaa095c101df7c0543c1fd71 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Jul 2017 18:08:23 +0200
Subject: [PATCH 0472/2196] builtins.fetchgit: Support specifying commit hashes

This adds an argument "rev" specififying the Git commit hash. The
existing argument "rev" is renamed to "ref". The default value for
"ref" is "master". When specifying a hash, it's necessary to specify a
ref since we're not cloning the entire repository but only fetching a
specific ref.

Example usage:

  builtins.fetchgit {
    url = https://github.com/NixOS/nixpkgs.git;
    ref = "release-16.03";
    rev = "c1c0484041ab6f9c6858c8ade80a8477c9ae4442";
  };
---
 src/libexpr/primops/fetchgit.cc | 34 ++++++++++++++++++++++++++-------
 src/libexpr/primops/fetchgit.hh |  4 ++--
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 3ab2644c8b0..1e01a452b7e 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -6,13 +6,27 @@
 
 #include <sys/time.h>
 
+#include <regex>
+
 namespace nix {
 
-Path exportGit(ref<Store> store, const std::string & uri, const std::string & rev)
+Path exportGit(ref<Store> store, const std::string & uri,
+    const std::string & ref, const std::string & rev)
 {
     if (!isUri(uri))
         throw EvalError(format("‘%s’ is not a valid URI") % uri);
 
+    if (rev != "") {
+        std::regex revRegex("^[0-9a-fA-F]{40}$");
+        if (!std::regex_match(rev, revRegex))
+            throw Error("invalid Git revision ‘%s’", rev);
+    }
+
+    // FIXME: too restrictive, but better safe than sorry.
+    std::regex refRegex("^[0-9a-zA-Z][0-9a-zA-Z.-]+$");
+    if (!std::regex_match(ref, refRegex))
+        throw Error("invalid Git ref ‘%s’", ref);
+
     Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {
@@ -22,7 +36,7 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
 
     //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri);
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, rev)).to_string(Base32, false);
+    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, ref)).to_string(Base32, false);
 
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
@@ -33,7 +47,7 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
     if (stat(localRefFile.c_str(), &st) != 0 ||
         st.st_mtime < now - settings.tarballTtl)
     {
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, rev + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, ref + ":" + localRef });
 
         struct timeval times[2];
         times[0].tv_sec = now;
@@ -44,7 +58,9 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re
         utimes(localRefFile.c_str(), times);
     }
 
-    std::string commitHash = chomp(readFile(localRefFile));
+    // FIXME: check whether rev is an ancestor of ref.
+    std::string commitHash =
+        rev != "" ? rev : chomp(readFile(localRefFile));
 
     printTalkative("using revision %s of repo ‘%s’", uri, commitHash);
 
@@ -81,7 +97,8 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
     if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode");
 
     std::string url;
-    std::string rev = "master";
+    std::string ref = "master";
+    std::string rev;
 
     state.forceValue(*args[0]);
 
@@ -95,7 +112,10 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
                 PathSet context;
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
                 if (hasPrefix(url, "/")) url = "file://" + url;
-            } else if (name == "rev")
+            }
+            else if (name == "ref")
+                ref = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (name == "rev")
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument ‘%s’ to ‘fetchgit’, at %s", attr.name, *attr.pos);
@@ -107,7 +127,7 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    Path storePath = exportGit(state.store, url, rev);
+    Path storePath = exportGit(state.store, url, ref, rev);
 
     mkString(v, storePath, PathSet({storePath}));
 }
diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh
index 6ffb21a96da..ff228f3b3c6 100644
--- a/src/libexpr/primops/fetchgit.hh
+++ b/src/libexpr/primops/fetchgit.hh
@@ -8,7 +8,7 @@ namespace nix {
 
 class Store;
 
-Path exportGit(ref<Store> store,
-    const std::string & uri, const std::string & rev);
+Path exportGit(ref<Store> store, const std::string & uri,
+    const std::string & ref, const std::string & rev = "");
 
 }

From af4689f9e9c846f8b8d902b05cabad88427d46b6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Jul 2017 14:56:39 +0200
Subject: [PATCH 0473/2196] nix-prefetch-url: Fix regression in hash printing

---
 src/libutil/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 817ddc0b8bc..e16c3b6ea93 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -98,7 +98,7 @@ static string printHash32(const Hash & hash)
 
 string printHash16or32(const Hash & hash)
 {
-    return hash.to_string(hash.type == htMD5 ? Base16 : Base32);
+    return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
 }
 
 

From c7654bc491d9ce7c1fbadecd7769418fa79a2060 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Jul 2017 15:03:59 +0200
Subject: [PATCH 0474/2196] nix-build: Fix regression causing all outputs to be
 built

---
 src/nix-build/nix-build.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 8a3c8a02506..769ca8cd03a 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -453,13 +453,13 @@ void mainWrapped(int argc, char * * argv)
 
         for (auto & drvInfo : drvs) {
             auto drvPath = drvInfo.queryDrvPath();
-            pathsToBuild.insert(drvPath);
+            auto outPath = drvInfo.queryOutPath();
 
             auto outputName = drvInfo.queryOutputName();
             if (outputName == "")
                 throw Error("derivation ‘%s’ lacks an ‘outputName’ attribute", drvPath);
 
-            pathsToBuild.insert(drvPath + (outputName != "out" ? "!" + outputName : ""));
+            pathsToBuild.insert(drvPath + "!" + outputName);
 
             std::string drvPrefix;
             auto i = drvPrefixes.find(drvPath);
@@ -475,8 +475,8 @@ void mainWrapped(int argc, char * * argv)
             std::string symlink = drvPrefix;
             if (outputName != "out") symlink += "-" + outputName;
 
-            resultSymlinks[symlink] = drvInfo.queryOutPath();
-            outPaths.push_back(drvInfo.queryOutPath());
+            resultSymlinks[symlink] = outPath;
+            outPaths.push_back(outPath);
         }
 
         buildPaths(pathsToBuild);

From 92bcb61127abd9da6ee1c11a53bf7b8da72f0c57 Mon Sep 17 00:00:00 2001
From: davidak <git@davidak.de>
Date: Sun, 30 Jul 2017 12:26:17 +0200
Subject: [PATCH 0475/2196] replace "Mac OS X" with "macOS"

except in older release notes where the name was actually Mac OS X.
---
 doc/manual/command-ref/conf-file.xml            | 2 +-
 doc/manual/installation/installing-binary.xml   | 4 ++--
 doc/manual/installation/multi-user.xml          | 2 +-
 doc/manual/installation/supported-platforms.xml | 2 +-
 doc/manual/introduction/about-nix.xml           | 2 +-
 src/libutil/archive.cc                          | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 3512777dd71..47ceff2625e 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -262,7 +262,7 @@ false</literal>.</para>
     system (except that fixed-output derivations do not run in private
     network namespace to ensure they can access the network).</para>
 
-    <para>Currently, sandboxing only work on Linux and Mac OS X. The use
+    <para>Currently, sandboxing only work on Linux and macOS. The use
     of a sandbox requires that Nix is run as root (so you should use
     the <link linkend='conf-build-users-group'>“build users”
     feature</link> to perform the actual builds under different users
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 2a9beec98c9..24e76eafeb1 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -6,7 +6,7 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or Mac OS X, the easiest way to install
+<para>If you are using Linux or macOS, the easiest way to install
 Nix is to run the following command:
 
 <screen>
@@ -39,7 +39,7 @@ behaviour.
 <!--
 <para>You can also manually download and install a binary package.
 Binary packages of the latest stable release are available for Fedora,
-Debian, Ubuntu, Mac OS X and various other systems from the <link
+Debian, Ubuntu, macOS and various other systems from the <link
 xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
 You can also get builds of the latest development release from our
 <link
diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml
index 49c4f723597..a13e3c89be7 100644
--- a/doc/manual/installation/multi-user.xml
+++ b/doc/manual/installation/multi-user.xml
@@ -52,7 +52,7 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if
 you expect to do many builds at the same time.</para>
 
-<para>On Mac OS X, you can create the required group and users by
+<para>On macOS, you can create the required group and users by
 running the following script:
 
 <programlisting>
diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml
index cbe528690cd..a468a564063 100644
--- a/doc/manual/installation/supported-platforms.xml
+++ b/doc/manual/installation/supported-platforms.xml
@@ -12,7 +12,7 @@
 
   <listitem><para>Linux (i686, x86_64).</para></listitem>
 
-  <listitem><para>Mac OS X (x86_64).</para></listitem>
+  <listitem><para>macOS (x86_64).</para></listitem>
 
   <!--
   <listitem><para>FreeBSD (only tested on Intel).</para></listitem>
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index be065da3eb2..7cb124033f3 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -236,7 +236,7 @@ href="[%root%]hydra">a continuous build system</a>.</para>
 
 <simplesect><title>Portability</title>
 
-<para>Nix runs on Linux and Mac OS X.</para>
+<para>Nix runs on Linux and macOS.</para>
 
 </simplesect>
 
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index e0e6f5dfd73..ed48c1349d8 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -72,7 +72,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
     else if (S_ISDIR(st.st_mode)) {
         sink << "type" << "directory";
 
-        /* If we're on a case-insensitive system like Mac OS X, undo
+        /* If we're on a case-insensitive system like macOS, undo
            the case hack applied by restorePath(). */
         std::map<string, string> unhacked;
         for (auto & i : readDirectory(path))

From fcb8d6a7a088622022364bde6534b2e4e804e4ed Mon Sep 17 00:00:00 2001
From: davidak <git@davidak.de>
Date: Sun, 30 Jul 2017 12:28:50 +0200
Subject: [PATCH 0476/2196] replace "OS X" with "macOS"

---
 src/build-remote/build-remote.cc | 2 +-
 src/libstore/optimise-store.cc   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8719959f0b6..7455160c732 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -57,7 +57,7 @@ int main (int argc, char * * argv)
         settings.builders = argv[5];
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
-           that gets cleared on reboot, but it wouldn't work on OS X. */
+           that gets cleared on reboot, but it wouldn't work on macOS. */
         currentLoad = settings.nixStateDir + "/current-load";
 
         std::shared_ptr<Store> sshStore;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 8e8002a30db..ba1ebc370fc 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -98,14 +98,14 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
         throw SysError(format("getting attributes of path ‘%1%’") % path);
 
 #if __APPLE__
-    /* HFS/OS X has some undocumented security feature disabling hardlinking for
+    /* HFS/macOS has some undocumented security feature disabling hardlinking for
        special files within .app dirs. *.app/Contents/PkgInfo and
        *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
        https://github.com/NixOS/nix/issues/1443 for more discussion. */
 
     if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
         std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$"))) {
-        debug(format("‘%1%’ is not allowed to be linked in OS X") % path);
+        debug(format("‘%1%’ is not allowed to be linked in macOS") % path);
         return;
     }
 #endif

From 2fd8f8bb99a2832b3684878c020ba47322e79332 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sun, 30 Jul 2017 12:27:57 +0100
Subject: [PATCH 0477/2196] Replace Unicode quotes in user-facing strings by
 ASCII
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Relevant RFC: NixOS/rfcs#4

$ ag -l | xargs sed -i -e "/\"/s/’/'/g;/\"/s/‘/'/g"
---
 perl/lib/Nix/Config.pm.in                     |   2 +-
 perl/lib/Nix/CopyClosure.pm                   |   4 +-
 perl/lib/Nix/Manifest.pm                      |  12 +-
 perl/lib/Nix/SSH.pm                           |   2 +-
 perl/lib/Nix/Utils.pm                         |   6 +-
 scripts/install-nix-from-closure.sh           |   4 +-
 src/build-remote/build-remote.cc              |  12 +-
 src/buildenv/buildenv.cc                      |  20 +-
 src/libexpr/attr-path.cc                      |  14 +-
 src/libexpr/common-opts.cc                    |   4 +-
 src/libexpr/eval.cc                           |  34 +--
 src/libexpr/get-drvs.cc                       |   4 +-
 src/libexpr/json-to-value.cc                  |   6 +-
 src/libexpr/lexer.l                           |   8 +-
 src/libexpr/nixexpr.cc                        |   4 +-
 src/libexpr/nixexpr.hh                        |   2 +-
 src/libexpr/parser.y                          |  16 +-
 src/libexpr/primops.cc                        | 108 +++++-----
 src/libexpr/primops/fetchgit.cc               |  18 +-
 src/libmain/shared.cc                         |  16 +-
 src/libmain/shared.hh                         |   6 +-
 src/libstore/binary-cache-store.cc            |  18 +-
 src/libstore/build.cc                         | 204 +++++++++---------
 src/libstore/builtins.cc                      |   4 +-
 src/libstore/derivations.cc                   |  12 +-
 src/libstore/download.cc                      |  30 +--
 src/libstore/export-import.cc                 |   8 +-
 src/libstore/gc.cc                            |  72 +++----
 src/libstore/globals.cc                       |   2 +-
 src/libstore/http-binary-cache-store.cc       |   2 +-
 src/libstore/legacy-ssh-store.cc              |  12 +-
 src/libstore/local-binary-cache-store.cc      |   2 +-
 src/libstore/local-fs-store.cc                |   8 +-
 src/libstore/local-store.cc                   |  86 ++++----
 src/libstore/machines.cc                      |   2 +-
 src/libstore/misc.cc                          |   2 +-
 src/libstore/nar-accessor.cc                  |  10 +-
 src/libstore/nar-info.cc                      |   2 +-
 src/libstore/optimise-store.cc                |  48 ++---
 src/libstore/pathlocks.cc                     |  16 +-
 src/libstore/profiles.cc                      |  10 +-
 src/libstore/references.cc                    |   4 +-
 src/libstore/remote-fs-accessor.cc            |   2 +-
 src/libstore/remote-store.cc                  |  10 +-
 src/libstore/s3-binary-cache-store.cc         |  24 +--
 src/libstore/sqlite.cc                        |  18 +-
 src/libstore/ssh.cc                           |   4 +-
 src/libstore/store-api.cc                     |  26 +--
 src/libstore/store-api.hh                     |   2 +-
 src/libutil/archive.cc                        |  18 +-
 src/libutil/args.cc                           |  10 +-
 src/libutil/args.hh                           |   2 +-
 src/libutil/compression.cc                    |   4 +-
 src/libutil/config.cc                         |   2 +-
 src/libutil/hash.cc                           |  18 +-
 src/libutil/hash.hh                           |   2 +-
 src/libutil/serialise.hh                      |   2 +-
 src/libutil/util.cc                           |  56 ++---
 src/nix-build/nix-build.cc                    |   8 +-
 src/nix-channel/nix-channel.cc                |  14 +-
 src/nix-copy-closure/nix-copy-closure.cc      |   4 +-
 src/nix-daemon/nix-daemon.cc                  |  12 +-
 src/nix-env/nix-env.cc                        |  62 +++---
 src/nix-env/user-env.cc                       |   2 +-
 src/nix-instantiate/nix-instantiate.cc        |   4 +-
 src/nix-prefetch-url/nix-prefetch-url.cc      |  16 +-
 src/nix-store/dotgraph.cc                     |   2 +-
 src/nix-store/nix-store.cc                    |  48 ++---
 src/nix/cat.cc                                |   4 +-
 src/nix/command.cc                            |   6 +-
 src/nix/command.hh                            |   4 +-
 src/nix/copy.cc                               |   2 +-
 src/nix/edit.cc                               |   6 +-
 src/nix/hash.cc                               |   2 +-
 src/nix/installables.cc                       |   2 +-
 src/nix/log.cc                                |   2 +-
 src/nix/ls.cc                                 |   2 +-
 src/nix/main.cc                               |   6 +-
 src/nix/repl.cc                               |   4 +-
 src/nix/run.cc                                |  16 +-
 src/nix/search.cc                             |   8 +-
 src/nix/sigs.cc                               |   6 +-
 src/nix/verify.cc                             |   6 +-
 .../resolve-system-dependencies.cc            |  16 +-
 tests/timeout.nix                             |   2 +-
 tests/timeout.sh                              |   2 +-
 86 files changed, 662 insertions(+), 662 deletions(-)

diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index 4bdee7fd89f..67a20c3f41d 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -23,7 +23,7 @@ sub readConfig {
     my $config = "$confDir/nix.conf";
     return unless -f $config;
 
-    open CONFIG, "<$config" or die "cannot open ‘$config’";
+    open CONFIG, "<$config" or die "cannot open '$config'";
     while (<CONFIG>) {
         /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next;
         $config{$1} = $2;
diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm
index affb3ea524a..902ee1a1bc9 100644
--- a/perl/lib/Nix/CopyClosure.pm
+++ b/perl/lib/Nix/CopyClosure.pm
@@ -35,14 +35,14 @@ sub copyToOpen {
     my $missingSize = 0;
     $missingSize += (queryPathInfo($_, 1))[3] foreach @missing;
 
-    printf STDERR "copying %d missing paths (%.2f MiB) to ‘$sshHost’...\n",
+    printf STDERR "copying %d missing paths (%.2f MiB) to '$sshHost'...\n",
         scalar(@missing), $missingSize / (1024**2);
     return if $dryRun;
 
     # Send the "import paths" command.
     syswrite($to, pack("L<x4", 4)) or die;
     exportPaths(fileno($to), @missing);
-    readInt($from) == 1 or die "remote machine ‘$sshHost’ failed to import closure\n";
+    readInt($from) == 1 or die "remote machine '$sshHost' failed to import closure\n";
 }
 
 
diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm
index 0da37676120..6438398e176 100644
--- a/perl/lib/Nix/Manifest.pm
+++ b/perl/lib/Nix/Manifest.pm
@@ -60,10 +60,10 @@ sub readManifest_ {
     # Decompress the manifest if necessary.
     if ($manifest =~ /\.bz2$/) {
         open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |"
-            or die "cannot decompress ‘$manifest’: $!";
+            or die "cannot decompress '$manifest': $!";
     } else {
         open MANIFEST, "<$manifest"
-            or die "cannot open ‘$manifest’: $!";
+            or die "cannot open '$manifest': $!";
     }
 
     my $inside = 0;
@@ -287,7 +287,7 @@ sub parseNARInfo {
         # FIXME: might be useful to support multiple signatures per .narinfo.
 
         if (!defined $sig) {
-            warn "NAR info file ‘$location’ lacks a signature; ignoring\n";
+            warn "NAR info file '$location' lacks a signature; ignoring\n";
             return undef;
         }
         my ($keyName, $sig64) = split ":", $sig;
@@ -295,7 +295,7 @@ sub parseNARInfo {
 
         my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName};
         if (!defined $publicKey) {
-            warn "NAR info file ‘$location’ is signed by unknown key ‘$keyName’; ignoring\n";
+            warn "NAR info file '$location' is signed by unknown key '$keyName'; ignoring\n";
             return undef;
         }
 
@@ -306,12 +306,12 @@ sub parseNARInfo {
                 [ map { "$Nix::Config::storeDir/$_" } @refs ]);
         };
         if ($@) {
-            warn "cannot compute fingerprint of ‘$location’; ignoring\n";
+            warn "cannot compute fingerprint of '$location'; ignoring\n";
             return undef;
         }
 
         if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) {
-            warn "NAR info file ‘$location’ has an incorrect signature; ignoring\n";
+            warn "NAR info file '$location' has an incorrect signature; ignoring\n";
             return undef;
         }
 
diff --git a/perl/lib/Nix/SSH.pm b/perl/lib/Nix/SSH.pm
index 95393d88145..490ba0ea991 100644
--- a/perl/lib/Nix/SSH.pm
+++ b/perl/lib/Nix/SSH.pm
@@ -97,7 +97,7 @@ sub connectToRemoteNix {
         syswrite($to, pack("L<x4L<x4", $SERVE_MAGIC_1, $clientVersion)) or die;
         $magic = readInt($from);
     };
-    die "unable to connect to ‘$sshHost’\n" if $@;
+    die "unable to connect to '$sshHost'\n" if $@;
     die "did not get valid handshake from remote host\n" if $magic  != 0x5452eecb;
 
     my $serverVersion = readInt($from);
diff --git a/perl/lib/Nix/Utils.pm b/perl/lib/Nix/Utils.pm
index 392c45f2fff..44955a70698 100644
--- a/perl/lib/Nix/Utils.pm
+++ b/perl/lib/Nix/Utils.pm
@@ -10,7 +10,7 @@ $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!
 
 sub checkURL {
     my ($url) = @_;
-    die "invalid URL ‘$url’\n" unless $url =~ /^ $urlRE $ /x;
+    die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x;
 }
 
 sub uniq {
@@ -26,7 +26,7 @@ sub uniq {
 
 sub writeFile {
     my ($fn, $s) = @_;
-    open TMP, ">$fn" or die "cannot create file ‘$fn’: $!";
+    open TMP, ">$fn" or die "cannot create file '$fn': $!";
     print TMP "$s" or die;
     close TMP or die;
 }
@@ -34,7 +34,7 @@ sub writeFile {
 sub readFile {
     local $/ = undef;
     my ($fn) = @_;
-    open TMP, "<$fn" or die "cannot open file ‘$fn’: $!";
+    open TMP, "<$fn" or die "cannot open file '$fn': $!";
     my $s = <TMP>;
     close TMP or die;
     return $s;
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 9d395020e60..3e5676f419b 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -44,13 +44,13 @@ if ! [ -e $dest ]; then
     cmd="mkdir -m 0755 $dest && chown $USER $dest"
     echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2
     if ! sudo sh -c "$cmd"; then
-        echo "$0: please manually run ‘$cmd’ as root to create $dest" >&2
+        echo "$0: please manually run '$cmd' as root to create $dest" >&2
         exit 1
     fi
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run ‘chown -R $USER $dest’ as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8719959f0b6..d433e24913f 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -77,7 +77,7 @@ int main (int argc, char * * argv)
             auto tokens = tokenizeString<std::vector<string>>(line);
             auto sz = tokens.size();
             if (sz != 3 && sz != 4)
-                throw Error("invalid build hook line ‘%1%’", line);
+                throw Error("invalid build hook line '%1%'", line);
             auto amWilling = tokens[0] == "1";
             auto neededSystem = tokens[1];
             drvPath = tokens[2];
@@ -99,7 +99,7 @@ int main (int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 unsigned long long bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on ‘%s’", m.storeUri);
+                    debug("considering building on '%s'", m.storeUri);
 
                     if (m.enabled && std::find(m.systemTypes.begin(),
                             m.systemTypes.end(),
@@ -171,7 +171,7 @@ int main (int argc, char * * argv)
                     storeUri = bestMachine->storeUri;
 
                 } catch (std::exception & e) {
-                    printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...",
+                    printError("unable to open SSH connection to '%s': %s; trying other available machines...",
                         bestMachine->storeUri, e.what());
                     bestMachine->enabled = false;
                     continue;
@@ -198,7 +198,7 @@ int main (int argc, char * * argv)
         auto old = signal(SIGALRM, handleAlarm);
         alarm(15 * 60);
         if (!lockFile(uploadLock.get(), ltWrite, true))
-            printError("somebody is hogging the upload lock for ‘%s’, continuing...");
+            printError("somebody is hogging the upload lock for '%s', continuing...");
         alarm(0);
         signal(SIGALRM, old);
         copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
@@ -207,11 +207,11 @@ int main (int argc, char * * argv)
         BasicDerivation drv(readDerivation(drvPath));
         drv.inputSrcs = inputs;
 
-        printError("building ‘%s’ on ‘%s’", drvPath, storeUri);
+        printError("building '%s' on '%s'", drvPath, storeUri);
         auto result = sshStore->buildDerivation(drvPath, drv);
 
         if (!result.success())
-            throw Error("build of ‘%s’ on ‘%s’ failed: %s", drvPath, storeUri, result.errorMsg);
+            throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
 
         PathSet missing;
         for (auto & path : outputs)
diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc
index f997096eddb..885c5e16903 100644
--- a/src/buildenv/buildenv.cc
+++ b/src/buildenv/buildenv.cc
@@ -12,7 +12,7 @@ static bool isDirectory (const Path & path)
 {
     struct stat st;
     if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting status of ‘%1%’") % path);
+        throw SysError(format("getting status of '%1%'") % path);
     return S_ISDIR(st.st_mode);
 }
 
@@ -53,18 +53,18 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                 } else if (S_ISLNK(dstSt.st_mode)) {
                     auto target = readLink(dstFile);
                     if (!isDirectory(target))
-                        throw Error(format("collision between ‘%1%’ and non-directory ‘%2%’")
+                        throw Error(format("collision between '%1%' and non-directory '%2%'")
                             % srcFile % target);
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking ‘%1%’") % dstFile);
+                        throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError(format("creating directory ‘%1%’"));
+                        throw SysError(format("creating directory '%1%'"));
                     createLinks(target, dstFile, priorities[dstFile]);
                     createLinks(srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of ‘%1%’") % dstFile);
+                throw SysError(format("getting status of '%1%'") % dstFile);
         } else {
             struct stat dstSt;
             auto res = lstat(dstFile.c_str(), &dstSt);
@@ -74,17 +74,17 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                     auto prevPriority = priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(format(
-                                "collision between ‘%1%’ and ‘%2%’; "
-                                "use ‘nix-env --set-flag priority NUMBER PKGNAME’ "
+                                "collision between '%1%' and '%2%'; "
+                                "use 'nix-env --set-flag priority NUMBER PKGNAME' "
                                 "to change the priority of one of the conflicting packages"
                                 ) % srcFile % target);
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking ‘%1%’") % dstFile);
+                        throw SysError(format("unlinking '%1%'") % dstFile);
                 }
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of ‘%1%’") % dstFile);
+                throw SysError(format("getting status of '%1%'") % dstFile);
         }
         createSymlink(srcFile, dstFile);
         priorities[dstFile] = priority;
@@ -112,7 +112,7 @@ static void addPkg(const Path & pkgDir, int priority)
         if (!fd) {
             if (errno == ENOENT)
                 return;
-            throw SysError(format("opening ‘%1%’") % propagatedFN);
+            throw SysError(format("opening '%1%'") % propagatedFN);
         }
         propagated = readLine(fd.get());
     }
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 55379f94b18..b0f80db32a8 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path ‘%1%’") % s);
+                    throw Error(format("missing closing quote in selection path '%1%'") % s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -38,7 +38,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
     Strings tokens = parseAttrPath(attrPath);
 
     Error attrError =
-        Error(format("attribute selection path ‘%1%’ does not match expression") % attrPath);
+        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
     Value * v = &vIn;
 
@@ -62,15 +62,15 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path ‘%1%’ should be a set but is %2%")
+                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
                     % attrPath % showType(*v));
 
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path ‘%1%’") % attrPath);
+                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute ‘%1%’ in selection path ‘%2%’ not found") % attr % attrPath);
+                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
             v = &*a->value;
         }
 
@@ -78,11 +78,11 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path ‘%1%’ should be a list but is %2%")
+                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path ‘%2%’ is out of range") % attrIndex % attrPath);
+                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
 
             v = v->listElems()[attrIndex];
         }
diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc
index 06d6ed87df9..6b31961d345 100644
--- a/src/libexpr/common-opts.cc
+++ b/src/libexpr/common-opts.cc
@@ -13,7 +13,7 @@ bool parseAutoArgs(Strings::iterator & i,
     string arg = *i;
     if (arg != "--arg" && arg != "--argstr") return false;
 
-    UsageError error(format("‘%1%’ requires two arguments") % arg);
+    UsageError error(format("'%1%' requires two arguments") % arg);
 
     if (++i == argsEnd) throw error;
     string name = *i;
@@ -46,7 +46,7 @@ bool parseSearchPathArg(Strings::iterator & i,
     const Strings::iterator & argsEnd, Strings & searchPath)
 {
     if (*i != "-I") return false;
-    if (++i == argsEnd) throw UsageError("‘-I’ requires an argument");
+    if (++i == argsEnd) throw UsageError("'-I' requires an argument");
     searchPath.push_back(*i);
     return true;
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ca4c9a373a3..78f6b001052 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -329,7 +329,7 @@ Path EvalState::checkSourcePath(const Path & path_)
     if (!restricted) return path_;
 
     /* Resolve symlinks. */
-    debug(format("checking access to ‘%s’") % path_);
+    debug(format("checking access to '%s'") % path_);
     Path path = canonPath(path_, true);
 
     for (auto & i : searchPath) {
@@ -351,7 +351,7 @@ Path EvalState::checkSourcePath(const Path & path_)
         return path;
 #endif
 
-    throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_);
+    throw RestrictedPathError(format("access to path '%1%' is forbidden in restricted mode") % path_);
 }
 
 
@@ -504,7 +504,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable ‘%1%’ at %2%", var.name, var.pos);
+            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
         for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -643,12 +643,12 @@ void EvalState::evalFile(const Path & path, Value & v)
         return;
     }
 
-    printTalkative("evaluating file ‘%1%’", path2);
+    printTalkative("evaluating file '%1%'", path2);
     Expr * e = parseExprFromFile(checkSourcePath(path2));
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the file ‘%1%’:\n", path2);
+        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
         throw;
     }
 
@@ -798,7 +798,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute ‘%1%’ at %2% already defined at %3%", nameSym, i.pos, *j->pos);
+            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -886,7 +886,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute ‘%1%’ missing, at %2%", name, pos);
+                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -897,7 +897,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n",
+            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
                 showAttrPath(state, env, attrPath), *pos2);
         throw;
     }
@@ -1039,7 +1039,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument ‘%2%’, at %3%",
+                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
                     lambda, i.name, pos);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
@@ -1055,7 +1055,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument ‘%2%’, at %3%", lambda, i.name, pos);
+                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
             abort(); // can't happen
         }
     }
@@ -1113,7 +1113,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         if (j != args.end())
             actualArgs->attrs->push_back(*j);
         else if (!i.def)
-            throwTypeError("cannot auto-call a function that has an argument without a default value (‘%1%’)", i.name);
+            throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
     }
 
     actualArgs->attrs->sort();
@@ -1342,7 +1342,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n", i.name, *i.pos);
+                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
                     throw;
                 }
         }
@@ -1434,10 +1434,10 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string ‘%1%’ is not allowed to refer to a store path (such as ‘%2%’), at %3%",
+            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
                 v.string.s, v.string.context[0], pos);
         else
-            throwEvalError("the string ‘%1%’ is not allowed to refer to a store path (such as ‘%2%’)",
+            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
     }
     return s;
@@ -1519,7 +1519,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 string EvalState::copyPathToStore(PathSet & context, const Path & path)
 {
     if (nix::isDerivation(path))
-        throwEvalError("file names are not allowed to end in ‘%1%’", drvExtension);
+        throwEvalError("file names are not allowed to end in '%1%'", drvExtension);
 
     Path dstPath;
     if (srcToStore[path] != "")
@@ -1529,7 +1529,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
             ? store->computeStorePathForPath(checkSourcePath(path)).first
             : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         srcToStore[path] = dstPath;
-        printMsg(lvlChatty, format("copied source ‘%1%’ -> ‘%2%’")
+        printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")
             % path % dstPath);
     }
 
@@ -1542,7 +1542,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string ‘%1%’ doesn't represent an absolute path, at %2%", path, pos);
+        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
     return path;
 }
 
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 66689e3e889..d5bc42352a2 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -89,7 +89,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
     const Value * outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
-    const auto errMsg = Error("this derivation has bad ‘meta.outputsToInstall’");
+    const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
@@ -310,7 +310,7 @@ static void getDerivations(EvalState & state, Value & vIn,
            bound to the attribute with the "lower" name should take
            precedence). */
         for (auto & i : v.attrs->lexicographicOrder()) {
-            debug("evaluating attribute ‘%1%’", i->name);
+            debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
             string pathPrefix2 = addToPath(pathPrefix, i->name);
diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index c189cdef35e..9380de3a66b 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -57,7 +57,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             values.push_back(v2);
             skipWhitespace(s);
             if (*s == ']') break;
-            if (*s != ',') throw JSONParseError("expected ‘,’ or ‘]’ after JSON array element");
+            if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
             s++;
         }
         s++;
@@ -74,14 +74,14 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             if (attrs.empty() && *s == '}') break;
             string name = parseJSONString(s);
             skipWhitespace(s);
-            if (*s != ':') throw JSONParseError("expected ‘:’ in JSON object");
+            if (*s != ':') throw JSONParseError("expected ':' in JSON object");
             s++;
             Value * v2 = state.allocValue();
             parseJSON(state, s, *v2);
             attrs[state.symbols.create(name)] = v2;
             skipWhitespace(s);
             if (*s == '}') break;
-            if (*s != ',') throw JSONParseError("expected ‘,’ or ‘}’ after JSON member");
+            if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
             s++;
         }
         state.mkAttrs(v, attrs.size());
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 40ca7725803..28a0a6a8789 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -124,13 +124,13 @@ or          { return OR_KW; }
 {INT}       { errno = 0;
               yylval->n = strtol(yytext, 0, 10);
               if (errno != 0)
-                  throw ParseError(format("invalid integer ‘%1%’") % yytext);
+                  throw ParseError(format("invalid integer '%1%'") % yytext);
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float ‘%1%’") % yytext);
+                  throw ParseError(format("invalid float '%1%'") % yytext);
               return FLOAT;
             }
 
@@ -191,12 +191,12 @@ or          { return OR_KW; }
 <INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path ‘%s’ has a trailing slash", yytext);
+                  throw ParseError("path '%s' has a trailing slash", yytext);
               yylval->path = strdup(yytext);
               return PATH;
             }
 {HPATH}     { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path ‘%s’ has a trailing slash", yytext);
+                  throw ParseError("path '%s' has a trailing slash", yytext);
               yylval->path = strdup(yytext);
               return HPATH;
             }
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index b2c9f0528ca..7b0a127cd41 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -267,7 +267,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable ‘%1%’ at %2%") % name % pos);
+    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
 
     fromWith = true;
     this->level = withLevel;
@@ -419,7 +419,7 @@ void ExprLambda::setName(Symbol & name)
 
 string ExprLambda::showNamePos() const
 {
-    return (format("%1% at %2%") % (name.set() ? "‘" + (string) name + "’" : "anonymous function") % pos).str();
+    return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
 }
 
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index d2ca09b3a5b..30be79bb57a 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -235,7 +235,7 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument ‘%1%’ at %2%")
+            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
                 % arg % pos);
     };
     void setName(Symbol & name);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index ca3d057451a..669312bb7cf 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -65,14 +65,14 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute ‘%1%’ at %2% already defined at %3%")
+    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
         % showAttrPath(attrPath) % pos % prevPos);
 }
 
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute ‘%1%’ at %2% already defined at %3%")
+    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
         % attr % pos % prevPos);
 }
 
@@ -121,7 +121,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (formals->argNames.find(formal.name) != formals->argNames.end())
-        throw ParseError(format("duplicate formal function argument ‘%1%’ at %2%")
+        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
             % formal.name % pos);
     formals->formals.push_front(formal);
     formals->argNames.insert(formal.name);
@@ -564,7 +564,7 @@ Path resolveExprPath(Path path)
     struct stat st;
     while (true) {
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting status of ‘%1%’") % path);
+            throw SysError(format("getting status of '%1%'") % path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }
@@ -649,7 +649,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         if (pathExists(res)) return canonPath(res);
     }
     format f = format(
-        "file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)"
+        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
         + string(pos ? ", at %2%" : ""));
     f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
     throw ThrownError(f % path % pos);
@@ -671,7 +671,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             else
                 res = { true, getDownloader()->downloadCached(store, elem.second, true) };
         } catch (DownloadError & e) {
-            printError(format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second);
+            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }
     } else {
@@ -679,12 +679,12 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second);
+            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
             res = { false, "" };
         }
     }
 
-    debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second);
+    debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second);
 
     searchPathResolved[elem.second] = res;
     return res;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 78d73e1f557..4e51e8ff256 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -44,7 +44,7 @@ std::pair<string, string> decodeContext(const string & s)
 
 
 InvalidPathError::InvalidPathError(const Path & path) :
-    EvalError(format("path ‘%1%’ is not valid") % path), path(path) {}
+    EvalError(format("path '%1%' is not valid") % path), path(path) {}
 
 void EvalState::realiseContext(const PathSet & context)
 {
@@ -60,7 +60,7 @@ void EvalState::realiseContext(const PathSet & context)
     }
     if (!drvs.empty()) {
         if (!settings.enableImportFromDerivation)
-            throw EvalError(format("attempted to realize ‘%1%’ during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+            throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
         /* For performance, prefetch all substitute info. */
         PathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;
@@ -80,7 +80,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
 
@@ -127,7 +127,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
                 env->values[displ++] = attr.value;
             }
 
-            printTalkative("evaluating file ‘%1%’", path);
+            printTalkative("evaluating file '%1%'", path);
             Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
 
             e->eval(state, *env, v);
@@ -149,7 +149,7 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
 
@@ -159,16 +159,16 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
-        throw EvalError(format("could not open ‘%1%’: %2%") % path % dlerror());
+        throw EvalError(format("could not open '%1%': %2%") % path % dlerror());
 
     dlerror();
     ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
     if(!func) {
         char *message = dlerror();
         if (message)
-            throw EvalError(format("could not load symbol ‘%1%’ from ‘%2%’: %3%") % sym % path % message);
+            throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message);
         else
-            throw EvalError(format("symbol ‘%1%’ from ‘%2%’ resolved to NULL when a function pointer was expected")
+            throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected")
                     % sym % path);
     }
 
@@ -196,7 +196,7 @@ static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot execute ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+        throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%")
             % program % e.path % pos);
     }
 
@@ -205,13 +205,13 @@ static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(format("While parsing the output from ‘%1%’, at %2%\n") % program % pos);
+        e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos);
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(format("While evaluating the output from ‘%1%’, at %2%\n") % program % pos);
+        e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos);
         throw;
     }
 }
@@ -332,7 +332,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError(format("attribute ‘startSet’ required, at %1%") % pos);
+        throw EvalError(format("attribute 'startSet' required, at %1%") % pos);
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -343,7 +343,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError(format("attribute ‘operator’ required, at %1%") % pos);
+        throw EvalError(format("attribute 'operator' required, at %1%") % pos);
     state.forceValue(*op->value);
 
     /* Construct the closure by applying the operator to element of
@@ -362,7 +362,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError(format("attribute ‘key’ required, at %1%") % pos);
+            throw EvalError(format("attribute 'key' required, at %1%") % pos);
         state.forceValue(*key->value);
 
         if (doneKeys.find(key->value) != doneKeys.end()) continue;
@@ -393,7 +393,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     string s = state.coerceToString(pos, *args[0], context);
-    throw Abort(format("evaluation aborted with the following error message: ‘%1%’") % s);
+    throw Abort(format("evaluation aborted with the following error message: '%1%'") % s);
 }
 
 
@@ -502,13 +502,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError(format("required attribute ‘name’ missing, at %1%") % pos);
+        throw EvalError(format("required attribute 'name' missing, at %1%") % pos);
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(format("while evaluating the derivation attribute ‘name’ at %1%:\n") % posDrvName);
+        e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName);
         throw;
     }
 
@@ -540,26 +540,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     for (auto & i : args[0]->attrs->lexicographicOrder()) {
         if (i->name == state.sIgnoreNulls) continue;
         string key = i->name;
-        vomit("processing attribute ‘%1%’", key);
+        vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
             if (s == "recursive") outputHashRecursive = true;
             else if (s == "flat") outputHashRecursive = false;
-            else throw EvalError("invalid value ‘%s’ for ‘outputHashMode’ attribute, at %s", s, posDrvName);
+            else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
         };
 
         auto handleOutputs = [&](const Strings & ss) {
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName);
+                    throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName);
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName);
+                    throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName);
                 outputs.insert(j);
             }
             if (outputs.empty())
@@ -622,7 +622,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     else if (i->name == state.sSystem) drv.platform = s;
                     else if (i->name == state.sName) {
                         drvName = s;
-                        printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName);
+                        printMsg(lvlVomit, format("derivation name is '%1%'") % drvName);
                     }
                     else if (key == "outputHash") outputHash = s;
                     else if (key == "outputHashAlgo") outputHashAlgo = s;
@@ -634,7 +634,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the attribute ‘%1%’ of the derivation ‘%2%’ at %3%:\n")
+            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
                 % key % drvName % posDrvName);
             throw;
         }
@@ -690,14 +690,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError(format("required attribute ‘builder’ missing, at %1%") % posDrvName);
+        throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName);
     if (drv.platform == "")
-        throw EvalError(format("required attribute ‘system’ missing, at %1%") % posDrvName);
+        throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
 
     /* Check whether the derivation name is valid. */
     checkStoreName(drvName);
     if (isDerivation(drvName))
-        throw EvalError(format("derivation names are not allowed to end in ‘%1%’, at %2%")
+        throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%")
             % drvExtension % posDrvName);
 
     if (outputHash) {
@@ -707,7 +707,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         HashType ht = parseHashType(outputHashAlgo);
         if (ht == htUnknown)
-            throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName);
+            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
         Hash h(*outputHash, ht);
         outputHash = h.to_string(Base16, false);
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
@@ -743,7 +743,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Write the resulting term into the Nix store directory. */
     Path drvPath = writeDerivation(state.store, drv, drvName, state.repair);
 
-    printMsg(lvlChatty, format("instantiated ‘%1%’ -> ‘%2%’")
+    printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'")
         % drvName % drvPath);
 
     /* Optimisation, but required in read-only mode! because in that
@@ -763,7 +763,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
 /* Return a placeholder string for the specified output that will be
    substituted by the corresponding output path at build time. For
-   example, ‘placeholder "out"’ returns the string
+   example, 'placeholder "out"' returns the string
    /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build
    time, any occurence of this string in an derivation attribute will
    be replaced with the concrete path in the Nix store of the output
@@ -805,7 +805,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError(format("path ‘%1%’ is not in the Nix store, at %2%") % path % pos);
+        throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
         state.store->ensurePath(path2);
@@ -819,7 +819,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     PathSet context;
     Path path = state.coerceToPath(pos, *args[0], context);
     if (!context.empty())
-        throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos);
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
     try {
         mkBool(v, pathExists(state.checkSourcePath(path)));
     } catch (SysError & e) {
@@ -860,18 +860,18 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
     string s = readFile(state.checkSourcePath(path));
     if (s.find((char) 0) != string::npos)
-        throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path);
+        throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
     mkString(v, s.c_str());
 }
 
 
 /* Find a file in the Nix search path. Used to implement <x> paths,
-   which are desugared to ‘findFile __nixPath "x"’. */
+   which are desugared to 'findFile __nixPath "x"'. */
 static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceList(*args[0], pos);
@@ -889,7 +889,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError(format("attribute ‘path’ missing, at %1%") % pos);
+            throw EvalError(format("attribute 'path' missing, at %1%") % pos);
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -897,7 +897,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+            throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%")
                 % path % e.path % pos);
         }
 
@@ -917,7 +917,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
 
@@ -991,7 +991,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
         if (isDerivation(path)) {
             /* See prim_unsafeDiscardOutputDependency. */
             if (path.at(0) != '~')
-                throw EvalError(format("in ‘toFile’: the file ‘%1%’ cannot refer to derivation outputs, at %2%") % name % pos);
+                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
             path = string(path, 1);
         }
         refs.insert(path);
@@ -1024,7 +1024,7 @@ struct FilterFromExpr : PathFilter
     {
         struct stat st;
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting attributes of path ‘%1%’") % path);
+            throw SysError(format("getting attributes of path '%1%'") % path);
 
         /* Call the filter function.  The first argument is the path,
            the second is a string indicating the type of the file. */
@@ -1054,11 +1054,11 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos);
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
 
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to ‘filterSource’ is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
     FilterFromExpr filter(state, *args[0], pos);
 
@@ -1119,7 +1119,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError(format("attribute ‘%1%’ missing, at %2%") % attr % pos);
+        throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value);
@@ -1199,14 +1199,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError(format("‘name’ attribute missing in a call to ‘listToAttrs’, at %1%") % pos);
+            throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos);
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.find(sym) == seen.end()) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError(format("‘value’ attribute missing in a call to ‘listToAttrs’, at %1%") % pos);
+                throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
             seen.insert(sym);
@@ -1281,7 +1281,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError(format("‘functionArgs’ requires a function, at %1%") % pos);
+        throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1340,7 +1340,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error(format("‘tail’ called on an empty list, at %1%") % pos);
+        throw Error(format("'tail' called on an empty list, at %1%") % pos);
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
         v.listElems()[n] = args[0]->listElems()[n + 1];
@@ -1649,7 +1649,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) throw EvalError(format("negative start position in ‘substring’, at %1%") % pos);
+    if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos);
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1696,7 +1696,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error(format("unknown hash type ‘%1%’, at %2%") % type % pos);
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
@@ -1737,9 +1737,9 @@ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
           // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression ‘%s’, at %s", re, pos);
+          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
         } else {
-          throw EvalError("invalid regular expression ‘%s’, at %s", re, pos);
+          throw EvalError("invalid regular expression '%s', at %s", re, pos);
         }
     }
 }
@@ -1770,7 +1770,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError(format("‘from’ and ‘to’ arguments to ‘replaceStrings’ have different lengths, at %1%") % pos);
+        throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos);
 
     vector<string> from;
     from.reserve(args[0]->listSize());
@@ -1861,17 +1861,17 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError(format("unsupported argument ‘%1%’ to ‘%2%’, at %3%") % attr.name % who % attr.pos);
+                throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
         }
 
         if (url.empty())
-            throw EvalError(format("‘url’ argument required, at %1%") % pos);
+            throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
     if (state.restricted && !expectedHash)
-        throw Error(format("‘%1%’ is not allowed in restricted mode") % who);
+        throw Error(format("'%1%' is not allowed in restricted mode") % who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
     mkString(v, res, PathSet({res}));
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 1e01a452b7e..e16c8235378 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -14,18 +14,18 @@ Path exportGit(ref<Store> store, const std::string & uri,
     const std::string & ref, const std::string & rev)
 {
     if (!isUri(uri))
-        throw EvalError(format("‘%s’ is not a valid URI") % uri);
+        throw EvalError(format("'%s' is not a valid URI") % uri);
 
     if (rev != "") {
         std::regex revRegex("^[0-9a-fA-F]{40}$");
         if (!std::regex_match(rev, revRegex))
-            throw Error("invalid Git revision ‘%s’", rev);
+            throw Error("invalid Git revision '%s'", rev);
     }
 
     // FIXME: too restrictive, but better safe than sorry.
     std::regex refRegex("^[0-9a-zA-Z][0-9a-zA-Z.-]+$");
     if (!std::regex_match(ref, refRegex))
-        throw Error("invalid Git ref ‘%s’", ref);
+        throw Error("invalid Git ref '%s'", ref);
 
     Path cacheDir = getCacheDir() + "/nix/git";
 
@@ -34,7 +34,7 @@ Path exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri);
+    //Activity act(*logger, lvlInfo, format("fetching Git repository '%s'") % uri);
 
     std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, ref)).to_string(Base32, false);
 
@@ -62,10 +62,10 @@ Path exportGit(ref<Store> store, const std::string & uri,
     std::string commitHash =
         rev != "" ? rev : chomp(readFile(localRefFile));
 
-    printTalkative("using revision %s of repo ‘%s’", uri, commitHash);
+    printTalkative("using revision %s of repo '%s'", uri, commitHash);
 
     Path storeLink = cacheDir + "/" + commitHash + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on ‘%1%’...", storeLink));
+    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink));
 
     if (pathExists(storeLink)) {
         auto storePath = readLink(storeLink);
@@ -94,7 +94,7 @@ Path exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     // FIXME: cut&paste from fetch().
-    if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode");
+    if (state.restricted) throw Error("'fetchgit' is not allowed in restricted mode");
 
     std::string url;
     std::string ref = "master";
@@ -118,11 +118,11 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (name == "rev")
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument ‘%s’ to ‘fetchgit’, at %s", attr.name, *attr.pos);
+                throw EvalError("unsupported argument '%s' to 'fetchgit', at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())
-            throw EvalError(format("‘url’ argument required, at %1%") % pos);
+            throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
         url = state.forceStringNoCtx(*args[0], pos);
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 48b00e1a399..67178b4a891 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -29,7 +29,7 @@ void printGCWarning()
     if (!gcWarning) return;
     static bool haveWarned = false;
     warnOnce(haveWarned,
-        "you did not specify ‘--add-root’; "
+        "you did not specify '--add-root'; "
         "the result might be removed by the garbage collector");
 }
 
@@ -76,7 +76,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError(format("‘%1%’ requires an argument") % opt);
+    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
     return *i;
 }
 
@@ -194,7 +194,7 @@ struct LegacyArgs : public MixCommonArgs
         mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
             &settings.showTrace);
 
-        mkFlag(0, "no-gc-warning", "disable warning about not using ‘--add-root’",
+        mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
             &gcWarning, false);
     }
 
@@ -213,7 +213,7 @@ struct LegacyArgs : public MixCommonArgs
         Strings ss(args);
         auto pos = ss.begin();
         if (!parseArg(pos, ss.end()))
-            throw UsageError(format("unexpected argument ‘%1%’") % args.front());
+            throw UsageError(format("unexpected argument '%1%'") % args.front());
         return true;
     }
 };
@@ -257,7 +257,7 @@ void showManPage(const string & name)
 {
     restoreSignals();
     execlp("man", "man", name.c_str(), NULL);
-    throw SysError(format("command ‘man %1%’ failed") % name.c_str());
+    throw SysError(format("command 'man %1%' failed") % name.c_str());
 }
 
 
@@ -281,13 +281,13 @@ int handleExceptions(const string & programName, std::function<void()> fun)
         return e.status;
     } catch (UsageError & e) {
         printError(
-            format(error + "%1%\nTry ‘%2% --help’ for more information.")
+            format(error + "%1%\nTry '%2% --help' for more information.")
             % e.what() % programName);
         return 1;
     } catch (BaseError & e) {
         printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         if (e.prefix() != "" && !settings.showTrace)
-            printError("(use ‘--show-trace’ to show detailed location information)");
+            printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
         printError(error + "out of memory");
@@ -322,7 +322,7 @@ RunPager::RunPager()
         execlp("pager", "pager", NULL);
         execlp("less", "less", NULL);
         execlp("more", "more", NULL);
-        throw SysError(format("executing ‘%1%’") % pager);
+        throw SysError(format("executing '%1%'") % pager);
     });
 
     pid.setKillSignal(SIGINT);
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 14e436cc8bb..becf258734e 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -48,7 +48,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError(format("‘%1%’ requires an argument") % opt);
+    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -58,13 +58,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier ‘%1%’") % u);
+            else throw UsageError(format("invalid unit specifier '%1%'") % u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError(format("‘%1%’ requires an integer argument") % opt);
+        throw UsageError(format("'%1%' requires an integer argument") % opt);
     return n * multiplier;
 }
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8147345c2e1..403b4687287 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -39,7 +39,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor
         std::string restPath = std::string(path, storePath.size());
 
         if (!store->isValidPath(storePath))
-            throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
+            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
 
         auto i = nars.find(storePath);
         if (i != nars.end()) return {i->second, restPath};
@@ -103,7 +103,7 @@ void BinaryCacheStore::init()
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error(format("binary cache ‘%s’ is for Nix stores with prefix ‘%s’, not ‘%s’")
+                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
                         % getUri() % value % storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery_ = value == "1";
@@ -145,7 +145,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             if (ref != info.path)
                 queryPathInfo(ref);
         } catch (InvalidPath &) {
-            throw Error(format("cannot add ‘%s’ to the binary cache because the reference ‘%s’ is not valid")
+            throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
                 % info.path % ref);
         }
 
@@ -159,7 +159,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
+        throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
 
     auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
 
@@ -233,7 +233,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->fileSize = narCompressed->size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
+    printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % narInfo->path % narInfo->narSize
         % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
@@ -286,7 +286,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 
     auto nar = getFile(info->url);
 
-    if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url);
+    if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url);
 
     stats.narRead++;
     stats.narReadCompressedBytes += nar->size();
@@ -296,13 +296,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
     try {
         nar = decompress(info->compression, *nar);
     } catch (UnknownCompressionMethod &) {
-        throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’")
+        throw Error(format("binary cache path '%s' uses unknown compression method '%s'")
             % storePath % info->compression);
     }
 
     stats.narReadBytes += nar->size();
 
-    printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar->size());
+    printMsg(lvlTalkative, format("exporting path '%1%' (%2% bytes)") % storePath % nar->size());
 
     assert(nar->size() % 8 == 0);
 
@@ -394,7 +394,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
 
     auto logPath = "log/" + baseNameOf(drvPath);
 
-    debug("fetching build log from binary cache ‘%s/%s’", getUri(), logPath);
+    debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
 
     return getFile(logPath);
 }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8f483a90daa..db5f606fa88 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -341,7 +341,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     assert(waitees.find(waitee) != waitees.end());
     waitees.erase(waitee);
 
-    trace(format("waitee ‘%1%’ done; %2% left") %
+    trace(format("waitee '%1%' done; %2% left") %
         waitee->name % waitees.size());
 
     if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
@@ -415,7 +415,7 @@ static void commonChildInit(Pipe & logPipe)
     /* Reroute stdin to /dev/null. */
     int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
     if (fdDevNull == -1)
-        throw SysError(format("cannot open ‘%1%’") % pathNullDevice);
+        throw SysError(format("cannot open '%1%'") % pathNullDevice);
     if (dup2(fdDevNull, STDIN_FILENO) == -1)
         throw SysError("cannot dup null device into stdin");
     close(fdDevNull);
@@ -468,29 +468,29 @@ UserLock::UserLock()
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
     if (!gr)
-        throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist")
+        throw Error(format("the group '%1%' specified in 'build-users-group' does not exist")
             % settings.buildUsersGroup);
     gid = gr->gr_gid;
 
     /* Copy the result of getgrnam. */
     Strings users;
     for (char * * p = gr->gr_mem; *p; ++p) {
-        debug(format("found build user ‘%1%’") % *p);
+        debug(format("found build user '%1%'") % *p);
         users.push_back(*p);
     }
 
     if (users.empty())
-        throw Error(format("the build users group ‘%1%’ has no members")
+        throw Error(format("the build users group '%1%' has no members")
             % settings.buildUsersGroup);
 
     /* Find a user account that isn't currently in use for another
        build. */
     for (auto & i : users) {
-        debug(format("trying user ‘%1%’") % i);
+        debug(format("trying user '%1%'") % i);
 
         struct passwd * pw = getpwnam(i.c_str());
         if (!pw)
-            throw Error(format("the user ‘%1%’ in the group ‘%2%’ does not exist")
+            throw Error(format("the user '%1%' in the group '%2%' does not exist")
                 % i % settings.buildUsersGroup);
 
         createDirs(settings.nixStateDir + "/userpool");
@@ -509,7 +509,7 @@ UserLock::UserLock()
 
             AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
             if (!fd)
-                throw SysError(format("opening user lock ‘%1%’") % fnUserLock);
+                throw SysError(format("opening user lock '%1%'") % fnUserLock);
 
             if (lockFile(fd.get(), ltWrite, false)) {
                 fdUserLock = std::move(fd);
@@ -518,7 +518,7 @@ UserLock::UserLock()
 
                 /* Sanity check... */
                 if (uid == getuid() || uid == geteuid())
-                    throw Error(format("the Nix user should not be a member of ‘%1%’")
+                    throw Error(format("the Nix user should not be a member of '%1%'")
                         % settings.buildUsersGroup);
 
 #if __linux__
@@ -529,7 +529,7 @@ UserLock::UserLock()
                 int err = getgrouplist(pw->pw_name, pw->pw_gid,
                     supplementaryGIDs.data(), &ngroups);
                 if (err == -1)
-                    throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name);
+                    throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name);
 
                 supplementaryGIDs.resize(ngroups);
 #endif
@@ -543,7 +543,7 @@ UserLock::UserLock()
     }
 
     throw Error(format("all build users are currently in use; "
-        "consider creating additional users and adding them to the ‘%1%’ group")
+        "consider creating additional users and adding them to the '%1%' group")
         % settings.buildUsersGroup);
 }
 
@@ -587,7 +587,7 @@ struct HookInstance
 
 HookInstance::HookInstance()
 {
-    debug("starting build hook ‘%s’", settings.buildHook);
+    debug("starting build hook '%s'", settings.buildHook);
 
     /* Create a pipe to get the output of the child. */
     fromHook.create();
@@ -624,7 +624,7 @@ HookInstance::HookInstance()
 
         execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("executing ‘%s’", settings.buildHook);
+        throw SysError("executing '%s'", settings.buildHook);
     });
 
     pid.setSeparatePG(true);
@@ -927,7 +927,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     , buildMode(buildMode)
 {
     state = &DerivationGoal::getDerivation;
-    name = (format("building of ‘%1%’") % drvPath).str();
+    name = (format("building of '%1%'") % drvPath).str();
     trace("created");
 
     logger->event(evBuildCreated, act, drvPath);
@@ -1042,7 +1042,7 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        printError(format("cannot build missing derivation ‘%1%’") % drvPath);
+        printError(format("cannot build missing derivation '%1%'") % drvPath);
         done(BuildResult::MiscFailure);
         return;
     }
@@ -1080,7 +1080,7 @@ void DerivationGoal::haveDerivation()
     /* Reject doing a hash build of anything other than a fixed-output
        derivation. */
     if (buildMode == bmHash && !drv->isFixedOutput())
-        throw Error("cannot do a hash build of non-fixed-output derivation ‘%1%’", drvPath);
+        throw Error("cannot do a hash build of non-fixed-output derivation '%1%'", drvPath);
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
@@ -1101,7 +1101,7 @@ void DerivationGoal::outputsSubstituted()
     trace("all outputs substituted (maybe)");
 
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation ‘%1%’ failed (usually happens due to networking issues); try ‘--fallback’ to build derivation from source ") % drvPath).str());
+        done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str());
         return;
     }
 
@@ -1128,7 +1128,7 @@ void DerivationGoal::outputsSubstituted()
         return;
     }
     if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error(format("some outputs of ‘%1%’ are not valid, so checking is not possible") % drvPath);
+        throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath);
 
     /* Otherwise, at least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
@@ -1145,7 +1145,7 @@ void DerivationGoal::outputsSubstituted()
     for (auto & i : drv->inputSrcs) {
         if (worker.store.isValidPath(i)) continue;
         if (!settings.useSubstitutes)
-            throw Error(format("dependency of ‘%1%’ of ‘%2%’ does not exist, and substitution is disabled")
+            throw Error(format("dependency of '%1%' of '%2%' does not exist, and substitution is disabled")
                 % i % drvPath);
         addWaitee(worker.makeSubstitutionGoal(i));
     }
@@ -1192,7 +1192,7 @@ void DerivationGoal::repairClosure()
     PathSet broken;
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
-        printError(format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath);
+        printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath);
         Path drvPath2 = outputsToDrv[i];
         if (drvPath2 == "")
             addWaitee(worker.makeSubstitutionGoal(i, Repair));
@@ -1213,7 +1213,7 @@ void DerivationGoal::closureRepaired()
 {
     trace("closure repaired");
     if (nrFailed > 0)
-        throw Error(format("some paths in the output closure of derivation ‘%1%’ could not be repaired") % drvPath);
+        throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath);
     done(BuildResult::AlreadyValid);
 }
 
@@ -1224,9 +1224,9 @@ void DerivationGoal::inputsRealised()
 
     if (nrFailed != 0) {
         if (!useDerivation)
-            throw Error(format("some dependencies of ‘%1%’ are missing") % drvPath);
+            throw Error(format("some dependencies of '%1%' are missing") % drvPath);
         printError(
-            format("cannot build derivation ‘%1%’: %2% dependencies couldn't be built")
+            format("cannot build derivation '%1%': %2% dependencies couldn't be built")
             % drvPath % nrFailed);
         done(BuildResult::DependencyFailed);
         return;
@@ -1242,7 +1242,7 @@ void DerivationGoal::inputsRealised()
 
     /* The outputs are referenceable paths. */
     for (auto & i : drv->outputs) {
-        debug(format("building path ‘%1%’") % i.second.path);
+        debug(format("building path '%1%'") % i.second.path);
         allPaths.insert(i.second.path);
     }
 
@@ -1261,7 +1261,7 @@ void DerivationGoal::inputsRealised()
                     worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths);
                 else
                     throw Error(
-                        format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’")
+                        format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'")
                         % drvPath % j % i.first);
         }
 
@@ -1302,7 +1302,7 @@ void DerivationGoal::tryToBuild()
        goal to sleep until another goal finishes, then try again. */
     for (auto & i : drv->outputs)
         if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
-            debug(format("putting derivation ‘%1%’ to sleep because ‘%2%’ is locked by another goal")
+            debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
                 % drvPath % i.second.path);
             worker.waitForAnyGoal(shared_from_this());
             return;
@@ -1331,7 +1331,7 @@ void DerivationGoal::tryToBuild()
        build this derivation, so no further checks are necessary. */
     validPaths = checkPathValidity(true, buildMode == bmRepair);
     if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) {
-        debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath);
+        debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath);
         outputLocks.setDeletion(true);
         done(BuildResult::AlreadyValid);
         return;
@@ -1346,7 +1346,7 @@ void DerivationGoal::tryToBuild()
     for (auto & i : drv->outputs) {
         Path path = i.second.path;
         if (worker.store.isValidPath(path)) continue;
-        debug(format("removing invalid path ‘%1%’") % path);
+        debug(format("removing invalid path '%1%'") % path);
         deletePath(worker.store.toRealPath(path));
     }
 
@@ -1416,7 +1416,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
     if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
-        throw SysError(format("moving ‘%1%’ to ‘%2%’") % tmpPath % storePath);
+        throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath);
     deletePath(oldPath);
 }
 
@@ -1439,7 +1439,7 @@ void DerivationGoal::buildDone()
        kill it. */
     int status = hook ? hook->pid.kill() : pid.kill();
 
-    debug(format("builder process for ‘%1%’ finished") % drvPath);
+    debug(format("builder process for '%1%' finished") % drvPath);
 
     result.timesBuilt++;
     result.stopTime = time(0);
@@ -1496,7 +1496,7 @@ void DerivationGoal::buildDone()
                     if (pathExists(chrootRootDir + i))
                         rename((chrootRootDir + i).c_str(), i.c_str());
 
-            std::string msg = (format("builder for ‘%1%’ %2%")
+            std::string msg = (format("builder for '%1%' %2%")
                 % drvPath % statusToString(status)).str();
 
             if (!settings.verboseBuild && !logTail.empty()) {
@@ -1606,7 +1606,7 @@ HookReply DerivationGoal::tryBuildHook()
             writeToStderr(s);
         }
 
-        debug(format("hook reply is ‘%1%’") % reply);
+        debug(format("hook reply is '%1%'") % reply);
 
         if (reply == "decline")
             return rpDecline;
@@ -1618,7 +1618,7 @@ HookReply DerivationGoal::tryBuildHook()
         else if (reply == "postpone")
             return rpPostpone;
         else if (reply != "accept")
-            throw Error(format("bad hook reply ‘%1%’") % reply);
+            throw Error(format("bad hook reply '%1%'") % reply);
 
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
@@ -1659,7 +1659,7 @@ HookReply DerivationGoal::tryBuildHook()
 void chmod_(const Path & path, mode_t mode)
 {
     if (chmod(path.c_str(), mode) == -1)
-        throw SysError(format("setting permissions on ‘%1%’") % path);
+        throw SysError(format("setting permissions on '%1%'") % path);
 }
 
 
@@ -1683,7 +1683,7 @@ void DerivationGoal::startBuilder()
     /* Right platform? */
     if (!drv->canBuildLocally()) {
         throw Error(
-            format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’")
+            format("a '%1%' is required to build '%3%', but I am a '%2%'")
             % drv->platform % settings.thisSystem % drvPath);
     }
 
@@ -1695,12 +1695,12 @@ void DerivationGoal::startBuilder()
     {
         if (settings.sandboxMode == smEnabled) {
             if (get(drv->env, "__noChroot") == "1")
-                throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, "
-                    "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath);
+                throw Error(format("derivation '%1%' has '__noChroot' set, "
+                    "but that's not allowed when 'build-use-sandbox' is 'true'") % drvPath);
 #if __APPLE__
             if (additionalSandboxProfile != "")
-                throw Error(format("derivation ‘%1%’ specifies a sandbox profile, "
-                    "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath);
+                throw Error(format("derivation '%1%' specifies a sandbox profile, "
+                    "but this is only allowed when 'build-use-sandbox' is 'relaxed'") % drvPath);
 #endif
             useChroot = true;
         }
@@ -1788,7 +1788,7 @@ void DerivationGoal::startBuilder()
                     worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
-                throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what());
+                throw Error(format("while processing 'build-sandbox-paths': %s") % e.what());
             }
         for (auto & i : closure)
             dirsInChroot[i] = i;
@@ -1813,7 +1813,7 @@ void DerivationGoal::startBuilder()
                 }
             }
             if (!found)
-                throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps") % drvPath % i);
+                throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i);
 
             dirsInChroot[i] = i;
         }
@@ -1829,13 +1829,13 @@ void DerivationGoal::startBuilder()
         /* Clean up the chroot directory automatically. */
         autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
 
-        printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir);
+        printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
         if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError(format("cannot create ‘%1%’") % chrootRootDir);
+            throw SysError(format("cannot create '%1%'") % chrootRootDir);
 
         if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir);
+            throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need
            this.  (Of course they should really respect $TMPDIR
@@ -1879,25 +1879,25 @@ void DerivationGoal::startBuilder()
         chmod_(chrootStoreDir, 01775);
 
         if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir);
+            throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir);
 
         for (auto & i : inputPaths) {
             Path r = worker.store.toRealPath(i);
             struct stat st;
             if (lstat(r.c_str(), &st))
-                throw SysError(format("getting attributes of path ‘%1%’") % i);
+                throw SysError(format("getting attributes of path '%1%'") % i);
             if (S_ISDIR(st.st_mode))
                 dirsInChroot[i] = r;
             else {
                 Path p = chrootRootDir + i;
-                debug("linking ‘%1%’ to ‘%2%’", p, r);
+                debug("linking '%1%' to '%2%'", p, r);
                 if (link(r.c_str(), p.c_str()) == -1) {
                     /* Hard-linking fails if we exceed the maximum
                        link count on a file (e.g. 32000 of ext3),
                        which is quite possible after a `nix-store
                        --optimise'. */
                     if (errno != EMLINK)
-                        throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % i);
+                        throw SysError(format("linking '%1%' to '%2%'") % p % i);
                     StringSink sink;
                     dumpPath(r, sink);
                     StringSource source(*sink.s);
@@ -1925,7 +1925,7 @@ void DerivationGoal::startBuilder()
     else {
 
         if (pathExists(homeDir))
-            throw Error(format("directory ‘%1%’ exists; please remove it") % homeDir);
+            throw Error(format("directory '%1%' exists; please remove it") % homeDir);
 
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete
@@ -1951,7 +1951,7 @@ void DerivationGoal::startBuilder()
     }
 
     if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
-        printMsg(lvlChatty, format("executing pre-build hook ‘%1%’")
+        printMsg(lvlChatty, format("executing pre-build hook '%1%'")
             % settings.preBuildHook);
         auto args = useChroot ? Strings({drvPath, chrootRootDir}) :
             Strings({ drvPath });
@@ -1970,7 +1970,7 @@ void DerivationGoal::startBuilder()
                 if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
                     state = stExtraChrootDirs;
                 } else {
-                    throw Error(format("unknown pre-build hook command ‘%1%’")
+                    throw Error(format("unknown pre-build hook command '%1%'")
                         % line);
                 }
             } else if (state == stExtraChrootDirs) {
@@ -1988,7 +1988,7 @@ void DerivationGoal::startBuilder()
     }
 
     /* Run the builder. */
-    printMsg(lvlChatty, format("executing builder ‘%1%’") % drv->builder);
+    printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder);
 
     /* Create the log file. */
     Path logFile = openLogFile();
@@ -2232,7 +2232,7 @@ void DerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;
     if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError(format("cannot change ownership of ‘%1%’") % path);
+        throw SysError(format("cannot change ownership of '%1%'") % path);
 }
 
 
@@ -2248,7 +2248,7 @@ void DerivationGoal::doExportReferencesGraph()
     string s = get(drv->env, "exportReferencesGraph");
     Strings ss = tokenizeString<Strings>(s);
     if (ss.size() % 2 != 0)
-        throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s);
+        throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
     for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
         string fileName = *i++;
         checkStoreName(fileName); /* !!! abuse of this function */
@@ -2256,11 +2256,11 @@ void DerivationGoal::doExportReferencesGraph()
         /* Check that the store path is valid. */
         Path storePath = *i++;
         if (!worker.store.isInStore(storePath))
-            throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’")
+            throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
                 % storePath);
         storePath = worker.store.toStorePath(storePath);
         if (!worker.store.isValidPath(storePath))
-            throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’")
+            throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
                 % storePath);
 
         /* If there are derivations in the graph, then include their
@@ -2418,13 +2418,13 @@ void DerivationGoal::runChild()
                local to the namespace, though, so setting MS_PRIVATE
                does not affect the outside world. */
             if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) {
-                throw SysError("unable to make ‘/’ private mount");
+                throw SysError("unable to make '/' private mount");
             }
 
             /* Bind-mount chroot directory to itself, to treat it as a
                different filesystem from /, as needed for pivot_root. */
             if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError(format("unable to bind mount ‘%1%’") % chrootRootDir);
+                throw SysError(format("unable to bind mount '%1%'") % chrootRootDir);
 
             /* Set up a nearly empty /dev, unless the user asked to
                bind-mount the host /dev. */
@@ -2464,13 +2464,13 @@ void DerivationGoal::runChild()
                filesystem that we want in the chroot
                environment. */
             auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target);
+                debug(format("bind mounting '%1%' to '%2%'") % source % target);
                 struct stat st;
                 if (stat(source.c_str(), &st) == -1) {
                     if (optional && errno == ENOENT)
                         return;
                     else
-                        throw SysError("getting attributes of path ‘%1%’", source);
+                        throw SysError("getting attributes of path '%1%'", source);
                 }
                 if (S_ISDIR(st.st_mode))
                     createDirs(target);
@@ -2479,7 +2479,7 @@ void DerivationGoal::runChild()
                     writeFile(target, "");
                 }
                 if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                    throw SysError("bind mount from ‘%1%’ to ‘%2%’ failed", source, target);
+                    throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
             };
 
             for (auto & i : dirsInChroot) {
@@ -2523,16 +2523,16 @@ void DerivationGoal::runChild()
 
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError(format("cannot change directory to ‘%1%’") % chrootRootDir);
+                throw SysError(format("cannot change directory to '%1%'") % chrootRootDir);
 
             if (mkdir("real-root", 0) == -1)
                 throw SysError("cannot create real-root directory");
 
             if (pivot_root(".", "real-root") == -1)
-                throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root"));
+                throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root"));
 
             if (chroot(".") == -1)
-                throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir);
+                throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir);
 
             if (umount2("real-root", MNT_DETACH) == -1)
                 throw SysError("cannot unmount real root filesystem");
@@ -2553,7 +2553,7 @@ void DerivationGoal::runChild()
 #endif
 
         if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError(format("changing into ‘%1%’") % tmpDir);
+            throw SysError(format("changing into '%1%'") % tmpDir);
 
         /* Close all other file descriptors. */
         closeMostFDs(set<int>());
@@ -2699,7 +2699,7 @@ void DerivationGoal::runChild()
                     if (lstat(path.c_str(), &st)) {
                         if (i.second.optional && errno == ENOENT)
                             continue;
-                        throw SysError(format("getting attributes of path ‘%1%’") % path);
+                        throw SysError(format("getting attributes of path '%1%'") % path);
                     }
                     if (S_ISDIR(st.st_mode))
                         sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
@@ -2762,7 +2762,7 @@ void DerivationGoal::runChild()
                 if (drv->builder == "builtin:fetchurl")
                     builtinFetchurl(*drv, netrcData);
                 else
-                    throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8));
+                    throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
             } catch (std::exception & e) {
                 writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n");
@@ -2772,7 +2772,7 @@ void DerivationGoal::runChild()
 
         execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 
-        throw SysError(format("executing ‘%1%’") % drv->builder);
+        throw SysError(format("executing '%1%'") % drv->builder);
 
     } catch (std::exception & e) {
         writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n");
@@ -2794,7 +2794,7 @@ PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, str
         else if (drv.outputs.find(i) != drv.outputs.end())
             result.insert(drv.outputs.find(i)->second.path);
         else throw BuildError(
-            format("derivation contains an illegal reference specifier ‘%1%’") % i);
+            format("derivation contains an illegal reference specifier '%1%'") % i);
     }
     return result;
 }
@@ -2841,7 +2841,7 @@ void DerivationGoal::registerOutputs()
                     replaceValidPath(path, actualPath);
                 else
                     if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1)
-                        throw SysError(format("moving build output ‘%1%’ from the sandbox to the Nix store") % path);
+                        throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
         } else {
@@ -2858,9 +2858,9 @@ void DerivationGoal::registerOutputs()
         if (lstat(actualPath.c_str(), &st) == -1) {
             if (errno == ENOENT)
                 throw BuildError(
-                    format("builder for ‘%1%’ failed to produce output path ‘%2%’")
+                    format("builder for '%1%' failed to produce output path '%2%'")
                     % drvPath % path);
-            throw SysError(format("getting attributes of path ‘%1%’") % actualPath);
+            throw SysError(format("getting attributes of path '%1%'") % actualPath);
         }
 
 #ifndef __CYGWIN__
@@ -2870,13 +2870,13 @@ void DerivationGoal::registerOutputs()
            user. */
         if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
             (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(format("suspicious ownership or permission on ‘%1%’; rejecting this build output") % path);
+            throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path);
 #endif
 
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
         if (!outputRewrites.empty()) {
-            printError(format("warning: rewriting hashes in ‘%1%’; cross fingers") % path);
+            printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path);
 
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
@@ -2907,7 +2907,7 @@ void DerivationGoal::registerOutputs()
                    execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
-                        format("output path ‘%1%’ should be a non-executable regular file") % path);
+                        format("output path '%1%' should be a non-executable regular file") % path);
             }
 
             /* Check the hash. In hash mode, move the path produced by
@@ -2915,7 +2915,7 @@ void DerivationGoal::registerOutputs()
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
             if (buildMode == bmHash) {
                 Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
-                printError(format("build produced path ‘%1%’ with %2% hash ‘%3%’")
+                printError(format("build produced path '%1%' with %2% hash '%3%'")
                     % dest % printHashType(h.type) % printHash16or32(h2));
                 if (worker.store.isValidPath(dest))
                     return;
@@ -2924,14 +2924,14 @@ void DerivationGoal::registerOutputs()
                     PathLocks outputLocks({actualDest});
                     deletePath(actualDest);
                     if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
-                        throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest);
+                        throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest);
                 }
                 path = dest;
                 actualPath = actualDest;
             } else {
                 if (h != h2)
                     throw BuildError(
-                        format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected")
+                        format("output path '%1%' has %2% hash '%3%' when '%4%' was expected")
                         % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h));
             }
 
@@ -2947,7 +2947,7 @@ void DerivationGoal::registerOutputs()
            contained in it.  Compute the SHA-256 NAR hash at the same
            time.  The hash is stored in the database so that we can
            verify later on whether nobody has messed with the store. */
-        debug("scanning for references inside ‘%1%’", path);
+        debug("scanning for references inside '%1%'", path);
         HashResult hash;
         PathSet references = scanForReferences(actualPath, allPaths, hash);
 
@@ -2959,11 +2959,11 @@ void DerivationGoal::registerOutputs()
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
-                        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst);
-                    throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’")
+                        throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
-                    throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs")
+                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs")
                         % drvPath % path);
             }
 
@@ -2983,9 +2983,9 @@ void DerivationGoal::registerOutputs()
         for (auto & i : inputPaths) {
             PathSet::iterator j = references.find(i);
             if (j == references.end())
-                debug(format("unreferenced input: ‘%1%’") % i);
+                debug(format("unreferenced input: '%1%'") % i);
             else
-                debug(format("referenced input: ‘%1%’") % i);
+                debug(format("referenced input: '%1%'") % i);
         }
 
         /* Enforce `allowedReferences' and friends. */
@@ -3021,7 +3021,7 @@ void DerivationGoal::registerOutputs()
                     badPathsStr += "\n\t";
                     badPathsStr += i;
                 }
-                throw BuildError(format("output ‘%1%’ is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr);
+                throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr);
             }
         };
 
@@ -3058,8 +3058,8 @@ void DerivationGoal::registerOutputs()
                 Path prev = i->path + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 auto msg = prevExists
-                    ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev)
-                    : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath);
+                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev)
+                    : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath);
 
                 auto diffHook = settings.diffHook;
                 if (prevExists && diffHook != "" && runDiffHook) {
@@ -3088,7 +3088,7 @@ void DerivationGoal::registerOutputs()
             deletePath(prev);
             Path dst = i.second.path + checkSuffix;
             if (rename(i.second.path.c_str(), dst.c_str()))
-                throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst);
+                throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst);
         }
     }
 
@@ -3129,7 +3129,7 @@ Path DerivationGoal::openLogFile()
         settings.compressLog ? ".bz2" : "");
 
     fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName);
+    if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName);
 
     logFileSink = std::make_shared<FdSink>(fdLogFile.get());
 
@@ -3159,7 +3159,7 @@ void DerivationGoal::deleteTmpDir(bool force)
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {
             printError(
-                format("note: keeping build directory ‘%2%’")
+                format("note: keeping build directory '%2%'")
                 % drvPath % tmpDir);
             chmod(tmpDir.c_str(), 0755);
         }
@@ -3351,7 +3351,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, Repa
 {
     this->storePath = storePath;
     state = &SubstitutionGoal::init;
-    name = (format("substitution of ‘%1%’") % storePath).str();
+    name = (format("substitution of '%1%'") % storePath).str();
     trace("created");
     logger->event(evSubstitutionCreated, act, storePath);
 }
@@ -3390,7 +3390,7 @@ void SubstitutionGoal::init()
     }
 
     if (settings.readOnlyMode)
-        throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath);
+        throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath);
 
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
 
@@ -3405,7 +3405,7 @@ void SubstitutionGoal::tryNext()
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
-        debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath);
+        debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath);
 
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
@@ -3436,7 +3436,7 @@ void SubstitutionGoal::tryNext()
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
     if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) {
-        printInfo(format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’")
+        printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
             % sub->getUri() % storePath);
         tryNext();
         return;
@@ -3460,7 +3460,7 @@ void SubstitutionGoal::referencesValid()
     trace("all references realised");
 
     if (nrFailed > 0) {
-        debug(format("some references of path ‘%1%’ could not be realised") % storePath);
+        debug(format("some references of path '%1%' could not be realised") % storePath);
         amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
         return;
     }
@@ -3487,7 +3487,7 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
-    printInfo(format("fetching path ‘%1%’...") % storePath);
+    printInfo(format("fetching path '%1%'...") % storePath);
 
     logger->event(evSubstitutionStarted, act);
 
@@ -3536,7 +3536,7 @@ void SubstitutionGoal::finished()
     worker.markContentsGood(storePath);
 
     printMsg(lvlChatty,
-        format("substitution of path ‘%1%’ succeeded") % storePath);
+        format("substitution of path '%1%' succeeded") % storePath);
 
     amDone(ecSuccess);
 }
@@ -3766,7 +3766,7 @@ void Worker::run(const Goals & _topGoals)
             waitForInput();
         else {
             if (awake.empty() && 0 == settings.maxBuildJobs) throw Error(
-                "unable to start any build; either increase ‘--max-jobs’ "
+                "unable to start any build; either increase '--max-jobs' "
                 "or enable distributed builds");
             assert(!awake.empty());
         }
@@ -3927,7 +3927,7 @@ bool Worker::pathContentsGood(const Path & path)
 {
     std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
     if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo(format("checking path ‘%1%’...") % path);
+    printInfo(format("checking path '%1%'...") % path);
     auto info = store.queryPathInfo(path);
     bool res;
     if (!pathExists(path))
@@ -3938,7 +3938,7 @@ bool Worker::pathContentsGood(const Path & path)
         res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache[path] = res;
-    if (!res) printError(format("path ‘%1%’ is corrupted or missing!") % path);
+    if (!res) printError(format("path '%1%' is corrupted or missing!") % path);
     return res;
 }
 
@@ -4013,7 +4013,7 @@ void LocalStore::ensurePath(const Path & path)
     worker.run(goals);
 
     if (goal->getExitCode() != Goal::ecSuccess)
-        throw Error(worker.exitStatus(), "path ‘%s’ does not exist and cannot be created", path);
+        throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path);
 }
 
 
@@ -4034,7 +4034,7 @@ void LocalStore::repairPath(const Path & path)
             goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair));
             worker.run(goals);
         } else
-            throw Error(worker.exitStatus(), "cannot repair path ‘%s’", path);
+            throw Error(worker.exitStatus(), "cannot repair path '%s'", path);
     }
 }
 
diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index a01dbba4ede..1075b45c00b 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error(format("attribute ‘%s’ missing") % name);
+        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
         return i->second;
     };
 
@@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto executable = drv.env.find("executable");
     if (executable != drv.env.end() && executable->second == "1") {
         if (chmod(storePath.c_str(), 0755) == -1)
-            throw SysError(format("making ‘%1%’ executable") % storePath);
+            throw SysError(format("making '%1%' executable") % storePath);
     }
 }
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 48c0837ffaa..a0a0d78b7d3 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -21,7 +21,7 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
 
     HashType hashType = parseHashType(algo);
     if (hashType == htUnknown)
-        throw Error(format("unknown hash algorithm ‘%1%’") % algo);
+        throw Error(format("unknown hash algorithm '%1%'") % algo);
 
     hash = Hash(this->hash, hashType);
 }
@@ -31,7 +31,7 @@ Path BasicDerivation::findOutput(const string & id) const
 {
     auto i = outputs.find(id);
     if (i == outputs.end())
-        throw Error(format("derivation has no output ‘%1%’") % id);
+        throw Error(format("derivation has no output '%1%'") % id);
     return i->second.path;
 }
 
@@ -94,7 +94,7 @@ static void expect(std::istream & str, const string & s)
     char s2[s.size()];
     str.read(s2, s.size());
     if (string(s2, s.size()) != s)
-        throw FormatError(format("expected string ‘%1%’") % s);
+        throw FormatError(format("expected string '%1%'") % s);
 }
 
 
@@ -121,7 +121,7 @@ static Path parsePath(std::istream & str)
 {
     string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
-        throw FormatError(format("bad path ‘%1%’ in derivation") % s);
+        throw FormatError(format("bad path '%1%' in derivation") % s);
     return s;
 }
 
@@ -204,7 +204,7 @@ Derivation readDerivation(const Path & drvPath)
     try {
         return parseDerivation(readFile(drvPath));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation ‘%1%’: %2%") % drvPath % e.msg());
+        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
     }
 }
 
@@ -217,7 +217,7 @@ Derivation Store::derivationFromPath(const Path & drvPath)
     try {
         return parseDerivation(accessor->readFile(drvPath));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation ‘%1%’: %2%") % drvPath % e.msg());
+        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
     }
 }
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index ac9a2b8fede..b731297a208 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -47,7 +47,7 @@ ref<std::string> decodeContent(const std::string & encoding, ref<std::string> da
     else if (encoding == "br")
         return decompress(encoding, *data);
     else
-        throw Error("unsupported Content-Encoding ‘%s’", encoding);
+        throw Error("unsupported Content-Encoding '%s'", encoding);
 }
 
 struct CurlDownloader : public Downloader
@@ -101,7 +101,7 @@ struct CurlDownloader : public Downloader
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri));
+                    fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -132,7 +132,7 @@ struct CurlDownloader : public Downloader
         {
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
-            printMsg(lvlVomit, format("got header for ‘%s’: %s") % request.uri % trim(line));
+            printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
             if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
                 result.etag = "";
                 auto ss = tokenizeString<vector<string>>(line, " ");
@@ -250,7 +250,7 @@ struct CurlDownloader : public Downloader
             if (effectiveUrlCStr)
                 result.effectiveUrl = effectiveUrlCStr;
 
-            debug(format("finished download of ‘%s’; curl status = %d, HTTP status = %d, body = %d bytes")
+            debug(format("finished download of '%s'; curl status = %d, HTTP status = %d, body = %d bytes")
                 % request.uri % code % httpStatus % (result.data ? result.data->size() : 0));
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
@@ -298,10 +298,10 @@ struct CurlDownloader : public Downloader
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri)
+                    ? DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)
                     : httpStatus != 0
-                      ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code))
-                      : DownloadError(err, format("unable to download ‘%s’: %s (%d)") % request.uri % curl_easy_strerror(code) % code);
+                      ? DownloadError(err, format("unable to download '%s': HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code))
+                      : DownloadError(err, format("unable to download '%s': %s (%d)") % request.uri % curl_easy_strerror(code) % code);
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. */
@@ -512,18 +512,18 @@ struct CurlDownloader : public Downloader
                 S3Helper s3Helper(Aws::Region::US_EAST_1); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
-                    throw nix::Error("bad S3 URI ‘%s’", request.uri);
+                    throw nix::Error("bad S3 URI '%s'", request.uri);
                 std::string bucketName(request.uri, 5, slash - 5);
                 std::string key(request.uri, slash + 1);
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;
                 if (!s3Res.data)
-                    throw DownloadError(NotFound, fmt("S3 object ‘%s’ does not exist", request.uri));
+                    throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
                 return res;
 #else
-                throw nix::Error("cannot download ‘%s’ because Nix is not built with S3 support", request.uri);
+                throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
 #endif
             });
             return;
@@ -587,7 +587,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path dataFile = cacheDir + "/" + urlHash + ".info";
     Path fileLink = cacheDir + "/" + urlHash + "-file";
 
-    PathLocks lock({fileLink}, fmt("waiting for lock on ‘%1%’...", fileLink));
+    PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
 
     Path storePath;
 
@@ -608,7 +608,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                     if (effectiveUrl)
                         *effectiveUrl = url_;
                 } else if (!ss[1].empty()) {
-                    debug(format("verifying previous ETag ‘%1%’") % ss[1]);
+                    debug(format("verifying previous ETag '%1%'") % ss[1]);
                     expectedETag = ss[1];
                 }
             }
@@ -650,7 +650,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     if (unpack) {
         Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
-        PathLocks lock2({unpackedLink}, fmt("waiting for lock on ‘%1%’...", unpackedLink));
+        PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
         Path unpackedStorePath;
         if (pathExists(unpackedLink)) {
             unpackedStorePath = readLink(unpackedLink);
@@ -659,7 +659,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                 unpackedStorePath = "";
         }
         if (unpackedStorePath.empty()) {
-            printInfo(format("unpacking ‘%1%’...") % url);
+            printInfo(format("unpacking '%1%'...") % url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
@@ -671,7 +671,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     }
 
     if (expectedStorePath != "" && storePath != expectedStorePath)
-        throw nix::Error("store path mismatch in file downloaded from ‘%s’", url);
+        throw nix::Error("store path mismatch in file downloaded from '%s'", url);
 
     return storePath;
 }
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 2cbcedc6fb0..ef3fea7c838 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -33,7 +33,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
     //logger->incExpected(doneLabel, sorted.size());
 
     for (auto & path : sorted) {
-        //Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path);
+        //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path);
         sink << 1;
         exportPath(path, sink);
         //logger->incProgress(doneLabel);
@@ -55,7 +55,7 @@ void Store::exportPath(const Path & path, Sink & sink)
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
-        throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path
+        throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path
             % info->narHash.to_string() % hash.to_string());
 
     hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
@@ -67,7 +67,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
     while (true) {
         auto n = readNum<uint64_t>(source);
         if (n == 0) break;
-        if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
+        if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
         TeeSink tee(source);
@@ -81,7 +81,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 
         info.path = readStorePath(*this, source);
 
-        //Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path);
+        //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
         info.references = readStorePaths<PathSet>(*this, source);
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0cf9f87cac3..bd335fbb49e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -32,11 +32,11 @@ int LocalStore::openGCLock(LockType lockType)
     Path fnGCLock = (format("%1%/%2%")
         % stateDir % gcLockName).str();
 
-    debug(format("acquiring global GC lock ‘%1%’") % fnGCLock);
+    debug(format("acquiring global GC lock '%1%'") % fnGCLock);
 
     AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
     if (!fdGCLock)
-        throw SysError(format("opening global GC lock ‘%1%’") % fnGCLock);
+        throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
         printError(format("waiting for the big garbage collector lock..."));
@@ -63,7 +63,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
     /* Atomically replace the old one. */
     if (rename(tempLink.c_str(), link.c_str()) == -1)
-        throw SysError(format("cannot rename ‘%1%’ to ‘%2%’")
+        throw SysError(format("cannot rename '%1%' to '%2%'")
             % tempLink % link);
 }
 
@@ -99,7 +99,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
         /* Don't clobber the link if it already exists and doesn't
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error(format("cannot create symlink ‘%1%’; already exists") % gcRoot);
+            throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
         makeSymlink(gcRoot, storePath);
         addIndirectRoot(gcRoot);
     }
@@ -110,8 +110,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
 
             if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
                 throw Error(format(
-                    "path ‘%1%’ is not a valid garbage collector root; "
-                    "it's not in the directory ‘%2%’")
+                    "path '%1%' is not a valid garbage collector root; "
+                    "it's not in the directory '%2%'")
                     % gcRoot % rootsDir);
         }
 
@@ -131,8 +131,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
         if (roots.find(gcRoot) == roots.end())
             printError(
                 format(
-                    "warning: ‘%1%’ is not in a directory where the garbage collector looks for roots; "
-                    "therefore, ‘%2%’ might be removed by the garbage collector")
+                    "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+                    "therefore, '%2%' might be removed by the garbage collector")
                 % gcRoot % storePath);
     }
 
@@ -169,14 +169,14 @@ void LocalStore::addTempRoot(const Path & path)
 
             fdGCLock = -1;
 
-            debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots);
+            debug(format("acquiring read lock on '%1%'") % state->fnTempRoots);
             lockFile(state->fdTempRoots.get(), ltRead, true);
 
             /* Check whether the garbage collector didn't get in our
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting ‘%1%’") % state->fnTempRoots);
+                throw SysError(format("statting '%1%'") % state->fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -188,14 +188,14 @@ void LocalStore::addTempRoot(const Path & path)
 
     /* Upgrade the lock to a write lock.  This will cause us to block
        if the garbage collector is holding our lock. */
-    debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots);
+    debug(format("acquiring write lock on '%1%'") % state->fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltWrite, true);
 
     string s = path + '\0';
     writeFull(state->fdTempRoots.get(), s);
 
     /* Downgrade to a read lock. */
-    debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots);
+    debug(format("downgrading to read lock on '%1%'") % state->fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltRead, true);
 }
 
@@ -210,12 +210,12 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
     for (auto & i : tempRootFiles) {
         Path path = (format("%1%/%2%/%3%") % stateDir % tempRootsDir % i.name).str();
 
-        debug(format("reading temporary root file ‘%1%’") % path);
+        debug(format("reading temporary root file '%1%'") % path);
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
         if (!*fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
-            throw SysError(format("opening temporary roots file ‘%1%’") % path);
+            throw SysError(format("opening temporary roots file '%1%'") % path);
         }
 
         /* This should work, but doesn't, for some reason. */
@@ -226,7 +226,7 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file ‘%1%’") % path);
+            printError(format("removing stale temporary roots file '%1%'") % path);
             unlink(path.c_str());
             writeFull(fd->get(), "d");
             continue;
@@ -235,7 +235,7 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
         /* Acquire a read lock.  This will prevent the owning process
            from upgrading to a write lock, therefore it will block in
            addTempRoot(). */
-        debug(format("waiting for read lock on ‘%1%’") % path);
+        debug(format("waiting for read lock on '%1%'") % path);
         lockFile(fd->get(), ltRead, true);
 
         /* Read the entire file. */
@@ -246,7 +246,7 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
 
         while ((end = contents.find((char) 0, pos)) != string::npos) {
             Path root(contents, pos, end - pos);
-            debug(format("got temporary root ‘%1%’") % root);
+            debug(format("got temporary root '%1%'") % root);
             assertStorePath(root);
             tempRoots.insert(root);
             pos = end + 1;
@@ -264,7 +264,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         if (isStorePath(storePath) && isValidPath(storePath))
             roots[path] = storePath;
         else
-            printInfo(format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath);
+            printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
     };
 
     try {
@@ -287,7 +287,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
                 target = absPath(target, dirOf(path));
                 if (!pathExists(target)) {
                     if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
-                        printInfo(format("removing stale link from ‘%1%’ to ‘%2%’") % path % target);
+                        printInfo(format("removing stale link from '%1%' to '%2%'") % path % target);
                         unlink(path.c_str());
                     }
                 } else {
@@ -310,7 +310,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     catch (SysError & e) {
         /* We only ignore permanent failures. */
         if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
-            printInfo(format("cannot read potential root ‘%1%’") % path);
+            printInfo(format("cannot read potential root '%1%'") % path);
         else
             throw;
     }
@@ -451,7 +451,7 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
         if (isInStore(i)) {
             Path path = toStorePath(i);
             if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
-                debug(format("got additional root ‘%1%’") % path);
+                debug(format("got additional root '%1%'") % path);
                 roots.insert(path);
             }
         }
@@ -517,7 +517,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         throw SysError(format("getting status of %1%") % realPath);
     }
 
-    printInfo(format("deleting ‘%1%’") % path);
+    printInfo(format("deleting '%1%'") % path);
 
     state.results.paths.insert(path);
 
@@ -532,14 +532,14 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         // size.
         try {
             if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError(format("making ‘%1%’ writable") % realPath);
+                throw SysError(format("making '%1%' writable") % realPath);
             Path tmp = trashDir + "/" + baseNameOf(path);
             if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError(format("unable to rename ‘%1%’ to ‘%2%’") % realPath % tmp);
+                throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
             state.bytesInvalidated += size;
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
-                printInfo(format("note: can't create move ‘%1%’: %2%") % realPath % e.msg());
+                printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
                 deleteGarbage(state, realPath);
             }
         }
@@ -566,7 +566,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
     }
 
     if (state.roots.find(path) != state.roots.end()) {
-        debug(format("cannot delete ‘%1%’ because it's a root") % path);
+        debug(format("cannot delete '%1%' because it's a root") % path);
         state.alive.insert(path);
         return true;
     }
@@ -615,7 +615,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
     auto realPath = realStoreDir + "/" + baseNameOf(path);
     if (realPath == linksDir || realPath == trashDir) return;
 
-    //Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
+    //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
 
     if (!isStorePath(path) || !isValidPath(path)) {
         /* A lock file belonging to a path that we're building right
@@ -635,7 +635,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
     PathSet visited;
 
     if (canReachRoot(state, visited, path)) {
-        debug(format("cannot delete ‘%1%’ because it's still reachable") % path);
+        debug(format("cannot delete '%1%' because it's still reachable") % path);
     } else {
         /* No path we visited was a root, so everything is garbage.
            But we only delete ‘path’ and its referrers here so that
@@ -656,7 +656,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 void LocalStore::removeUnusedLinks(const GCState & state)
 {
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir);
+    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
 
     long long actualSize = 0, unsharedSize = 0;
 
@@ -669,7 +669,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
 
         struct stat st;
         if (lstat(path.c_str(), &st) == -1)
-            throw SysError(format("statting ‘%1%’") % path);
+            throw SysError(format("statting '%1%'") % path);
 
         if (st.st_nlink != 1) {
             unsigned long long size = st.st_blocks * 512ULL;
@@ -678,17 +678,17 @@ void LocalStore::removeUnusedLinks(const GCState & state)
             continue;
         }
 
-        printMsg(lvlTalkative, format("deleting unused link ‘%1%’") % path);
+        printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
 
         if (unlink(path.c_str()) == -1)
-            throw SysError(format("deleting ‘%1%’") % path);
+            throw SysError(format("deleting '%1%'") % path);
 
         state.results.bytesFreed += st.st_blocks * 512ULL;
     }
 
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError(format("statting ‘%1%’") % linksDir);
+        throw SysError(format("statting '%1%'") % linksDir);
     long long overhead = st.st_blocks * 512ULL;
 
     printInfo(format("note: currently hard linking saves %.2f MiB")
@@ -768,7 +768,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             assertStorePath(i);
             tryToDelete(state, i);
             if (state.dead.find(i) == state.dead.end())
-                throw Error(format("cannot delete path ‘%1%’ since it is still alive") % i);
+                throw Error(format("cannot delete path '%1%' since it is still alive") % i);
         }
 
     } else if (options.maxFreed > 0) {
@@ -781,7 +781,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         try {
 
             AutoCloseDir dir(opendir(realStoreDir.c_str()));
-            if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir);
+            if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
@@ -834,7 +834,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     fds.clear();
 
     /* Delete the trash directory. */
-    printInfo(format("deleting ‘%1%’") % trashDir);
+    printInfo(format("deleting '%1%'") % trashDir);
     deleteGarbage(state, trashDir);
 
     /* Clean up the links directory. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 935018132d3..7da4bce8775 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -129,7 +129,7 @@ void MaxBuildJobsSetting::set(const std::string & str)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
     else if (!string2Int(str, value))
-        throw UsageError("configuration setting ‘%s’ should be ‘auto’ or an integer", name);
+        throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
 }
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index cead81514ab..05733768579 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -38,7 +38,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
-                throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
+                throw Error(format("'%s' does not appear to be a binary cache") % cacheUri);
             }
             diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
         }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index a84f85c1b95..65fe575d253 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -65,13 +65,13 @@ struct LegacySSHStore : public Store
 
             unsigned int magic = readInt(conn->from);
             if (magic != SERVE_MAGIC_2)
-                throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host);
+                throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
             conn->remoteVersion = readInt(conn->from);
             if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
-                throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host);
+                throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
 
         } catch (EndOfFile & e) {
-            throw Error("cannot connect to ‘%1%’", host);
+            throw Error("cannot connect to '%1%'", host);
         }
 
         return conn;
@@ -89,7 +89,7 @@ struct LegacySSHStore : public Store
         sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
             auto conn(connections->get());
 
-            debug("querying remote host ‘%s’ for info on ‘%s’", host, path);
+            debug("querying remote host '%s' for info on '%s'", host, path);
 
             conn->to << cmdQueryPathInfos << PathSet{path};
             conn->to.flush();
@@ -116,7 +116,7 @@ struct LegacySSHStore : public Store
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override
     {
-        debug("adding path ‘%s’ to remote host ‘%s’", info.path, host);
+        debug("adding path '%s' to remote host '%s'", info.path, host);
 
         auto conn(connections->get());
 
@@ -134,7 +134,7 @@ struct LegacySSHStore : public Store
         conn->to.flush();
 
         if (readInt(conn->from) != 1)
-            throw Error("failed to add path ‘%s’ to remote host ‘%s’, info.path, host");
+            throw Error("failed to add path '%s' to remote host '%s', info.path, host");
 
     }
 
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index aff22f9fcc2..2577e90aef2 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -76,7 +76,7 @@ static void atomicWrite(const Path & path, const std::string & s)
     AutoDelete del(tmp, false);
     writeFile(tmp, s);
     if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
+        throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
     del.cancel();
 }
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index bf28a1c70c6..642e4070d45 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor
     {
         Path storePath = store->toStorePath(path);
         if (!store->isValidPath(storePath))
-            throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
+            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
@@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor
         struct stat st;
         if (lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
-            throw SysError(format("getting status of ‘%1%’") % path);
+            throw SysError(format("getting status of '%1%'") % path);
         }
 
         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
-            throw Error(format("file ‘%1%’ has unsupported type") % path);
+            throw Error(format("file '%1%' has unsupported type") % path);
 
         return {
             S_ISREG(st.st_mode) ? Type::tRegular :
@@ -80,7 +80,7 @@ ref<FSAccessor> LocalFSStore::getFSAccessor()
 void LocalFSStore::narFromPath(const Path & path, Sink & sink)
 {
     if (!isValidPath(path))
-        throw Error(format("path ‘%s’ is not valid") % path);
+        throw Error(format("path '%s' is not valid") % path);
     dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
 }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7c41dfca7f3..95b05f8afd3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -76,24 +76,24 @@ LocalStore::LocalStore(const Params & params)
         Path perUserDir = profilesDir + "/per-user";
         createDirs(perUserDir);
         if (chmod(perUserDir.c_str(), 01777) == -1)
-            throw SysError(format("could not set permissions on ‘%1%’ to 1777") % perUserDir);
+            throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir);
 
         mode_t perm = 01775;
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            printError(format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist")
+            printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist")
                 % settings.buildUsersGroup);
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
-                throw SysError(format("getting attributes of path ‘%1%’") % realStoreDir);
+                throw SysError(format("getting attributes of path '%1%'") % realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
                 if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
-                    throw SysError(format("changing ownership of path ‘%1%’") % realStoreDir);
+                    throw SysError(format("changing ownership of path '%1%'") % realStoreDir);
                 if (chmod(realStoreDir.c_str(), perm) == -1)
-                    throw SysError(format("changing permissions on path ‘%1%’") % realStoreDir);
+                    throw SysError(format("changing permissions on path '%1%'") % realStoreDir);
             }
         }
     }
@@ -104,10 +104,10 @@ LocalStore::LocalStore(const Params & params)
         struct stat st;
         while (path != "/") {
             if (lstat(path.c_str(), &st))
-                throw SysError(format("getting status of ‘%1%’") % path);
+                throw SysError(format("getting status of '%1%'") % path);
             if (S_ISLNK(st.st_mode))
                 throw Error(format(
-                        "the path ‘%1%’ is a symlink; "
+                        "the path '%1%' is a symlink; "
                         "this is not allowed for the Nix store and its parent directories")
                     % path);
             path = dirOf(path);
@@ -267,7 +267,7 @@ int LocalStore::getSchema()
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
         if (!string2Int(s, curSchema))
-            throw Error(format("‘%1%’ is corrupt") % schemaPath);
+            throw Error(format("'%1%' is corrupt") % schemaPath);
     }
     return curSchema;
 }
@@ -276,14 +276,14 @@ int LocalStore::getSchema()
 void LocalStore::openDB(State & state, bool create)
 {
     if (access(dbDir.c_str(), R_OK | W_OK))
-        throw SysError(format("Nix database directory ‘%1%’ is not writable") % dbDir);
+        throw SysError(format("Nix database directory '%1%' is not writable") % dbDir);
 
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
     if (sqlite3_open_v2(dbPath.c_str(), &db.db,
             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open Nix database ‘%1%’") % dbPath);
+        throw Error(format("cannot open Nix database '%1%'") % dbPath);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls
@@ -378,7 +378,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
                  | 0444
                  | (st.st_mode & S_IXUSR ? 0111 : 0);
             if (chmod(path.c_str(), mode) == -1)
-                throw SysError(format("changing mode of ‘%1%’ to %2$o") % path % mode);
+                throw SysError(format("changing mode of '%1%' to %2$o") % path % mode);
         }
 
     }
@@ -396,7 +396,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 #else
         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 #endif
-            throw SysError(format("changing modification time of ‘%1%’") % path);
+            throw SysError(format("changing modification time of '%1%'") % path);
     }
 }
 
@@ -405,7 +405,7 @@ void canonicaliseTimestampAndPermissions(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
     canonicaliseTimestampAndPermissions(path, st);
 }
 
@@ -420,17 +420,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        setattrlist() to remove other attributes as well. */
     if (lchflags(path.c_str(), 0)) {
         if (errno != ENOTSUP)
-            throw SysError(format("clearing flags of path ‘%1%’") % path);
+            throw SysError(format("clearing flags of path '%1%'") % path);
     }
 #endif
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error(format("file ‘%1%’ has an unsupported type") % path);
+        throw Error(format("file '%1%' has an unsupported type") % path);
 
 #if __linux__
     /* Remove extended attributes / ACLs. */
@@ -438,19 +438,19 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
 
     if (eaSize < 0) {
         if (errno != ENOTSUP)
-            throw SysError("querying extended attributes of ‘%s’", path);
+            throw SysError("querying extended attributes of '%s'", path);
     } else if (eaSize > 0) {
         std::vector<char> eaBuf(eaSize);
 
         if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
-            throw SysError("querying extended attributes of ‘%s’", path);
+            throw SysError("querying extended attributes of '%s'", path);
 
         for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
             /* Ignore SELinux security labels since these cannot be
                removed even by root. */
             if (eaName == "security.selinux") continue;
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
-                throw SysError("removing extended attribute ‘%s’ from ‘%s’", eaName, path);
+                throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
         }
      }
 #endif
@@ -464,7 +464,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
         assert(!S_ISDIR(st.st_mode));
         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
-            throw BuildError(format("invalid ownership on file ‘%1%’") % path);
+            throw BuildError(format("invalid ownership on file '%1%'") % path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
         return;
@@ -488,7 +488,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         if (!S_ISLNK(st.st_mode) &&
             chown(path.c_str(), geteuid(), getegid()) == -1)
 #endif
-            throw SysError(format("changing owner of ‘%1%’ to %2%")
+            throw SysError(format("changing owner of '%1%' to %2%")
                 % path % geteuid());
     }
 
@@ -508,11 +508,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
        be a symlink, since we can't change its ownership. */
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
-        throw Error(format("wrong ownership of top-level store path ‘%1%’") % path);
+        throw Error(format("wrong ownership of top-level store path '%1%'") % path);
     }
 }
 
@@ -533,7 +533,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator out = drv.outputs.find("out");
         if (out == drv.outputs.end())
-            throw Error(format("derivation ‘%1%’ does not have an output named ‘out’") % drvPath);
+            throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath);
 
         bool recursive; Hash h;
         out->second.parseHashInfo(recursive, h);
@@ -541,7 +541,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 
         StringPairs::const_iterator j = drv.env.find("out");
         if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
-            throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’")
+            throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
                 % drvPath % out->second.path % outPath);
     }
 
@@ -558,7 +558,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
             Path outPath = makeOutputPath(i.first, h, drvName);
             StringPairs::const_iterator j = drv.env.find(i.first);
             if (i.second.path != outPath || j == drv.env.end() || j->second != outPath)
-                throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’")
+                throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
                     % drvPath % i.second.path % outPath);
         }
     }
@@ -639,7 +639,7 @@ void LocalStore::queryPathInfoUncached(const Path & path,
             try {
                 info->narHash = Hash(useQueryPathInfo.getStr(1));
             } catch (BadHash & e) {
-                throw Error("in valid-path entry for ‘%s’: %s", path, e.what());
+                throw Error("in valid-path entry for '%s': %s", path, e.what());
             }
 
             info->registrationTime = useQueryPathInfo.getInt(2);
@@ -688,7 +688,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
 {
     auto use(state.stmtQueryPathInfo.use()(path));
     if (!use.next())
-        throw Error(format("path ‘%1%’ is not valid") % path);
+        throw Error(format("path '%1%' is not valid") % path);
     return use.getInt(0);
 }
 
@@ -855,7 +855,7 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
         if (sub->storeDir != storeDir) continue;
         for (auto & path : paths) {
             if (infos.count(path)) continue;
-            debug(format("checking substituter ‘%s’ for path ‘%s’")
+            debug(format("checking substituter '%s' for path '%s'")
                 % sub->getUri() % path);
             try {
                 auto info = sub->queryPathInfo(path);
@@ -936,7 +936,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
    there are no referrers. */
 void LocalStore::invalidatePath(State & state, const Path & path)
 {
-    debug(format("invalidating path ‘%1%’") % path);
+    debug(format("invalidating path '%1%'") % path);
 
     state.stmtInvalidatePath.use()(path).exec();
 
@@ -957,15 +957,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
 
     Hash h = hashString(htSHA256, *nar);
     if (h != info.narHash)
-        throw Error("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’",
+        throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
             info.path, info.narHash.to_string(), h.to_string());
 
     if (nar->size() != info.narSize)
-        throw Error("size mismatch importing path ‘%s’; expected %s, got %s",
+        throw Error("size mismatch importing path '%s'; expected %s, got %s",
             info.path, info.narSize, nar->size());
 
     if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys))
-        throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path);
+        throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
 
     addTempRoot(info.path);
 
@@ -1150,7 +1150,7 @@ void LocalStore::invalidatePathChecked(const Path & path)
             PathSet referrers; queryReferrers(*state, path, referrers);
             referrers.erase(path); /* ignore self-references */
             if (!referrers.empty())
-                throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%")
+                throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%")
                     % path % showPaths(referrers));
             invalidatePath(*state, path);
         }
@@ -1195,12 +1195,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
 
                 /* Check the content hash (optionally - slow). */
-                printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i);
+                printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
                 HashResult current = hashPath(info->narHash.type, i);
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    printError(format("path ‘%1%’ was modified! "
-                            "expected hash ‘%2%’, got ‘%3%’")
+                    printError(format("path '%1%' was modified! "
+                            "expected hash '%2%', got '%3%'")
                         % i % info->narHash.to_string() % current.first.to_string());
                     if (repair) repairPath(i); else errors = true;
                 } else {
@@ -1209,14 +1209,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                     /* Fill in missing hashes. */
                     if (info->narHash == nullHash) {
-                        printError(format("fixing missing hash on ‘%1%’") % i);
+                        printError(format("fixing missing hash on '%1%'") % i);
                         info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
                     if (info->narSize == 0) {
-                        printError(format("updating size field on ‘%1%’ to %2%") % i % current.second);
+                        printError(format("updating size field on '%1%' to %2%") % i % current.second);
                         info->narSize = current.second;
                         update = true;
                     }
@@ -1253,7 +1253,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
     done.insert(path);
 
     if (!isStorePath(path)) {
-        printError(format("path ‘%1%’ is not in the Nix store") % path);
+        printError(format("path '%1%' is not in the Nix store") % path);
         auto state(_state.lock());
         invalidatePath(*state, path);
         return;
@@ -1272,11 +1272,11 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
             }
 
         if (canInvalidate) {
-            printError(format("path ‘%1%’ disappeared, removing from database...") % path);
+            printError(format("path '%1%' disappeared, removing from database...") % path);
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            printError(format("path ‘%1%’ disappeared, but it still has valid referrers!") % path);
+            printError(format("path '%1%' disappeared, but it still has valid referrers!") % path);
             if (repair)
                 try {
                     repairPath(path);
@@ -1315,7 +1315,7 @@ static void makeMutable(const Path & path)
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
     if (fd == -1) {
         if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file ‘%1%’") % path);
+        throw SysError(format("opening file '%1%'") % path);
     }
 
     unsigned int flags = 0, old;
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 7491037b2d7..076c3cab3e9 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -53,7 +53,7 @@ void parseMachines(const std::string & s, Machines & machines)
         auto tokens = tokenizeString<std::vector<string>>(line);
         auto sz = tokens.size();
         if (sz < 1)
-            throw FormatError("bad machine specification ‘%s’", line);
+            throw FormatError("bad machine specification '%s'", line);
 
         auto isSet = [&](size_t n) {
             return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 9a88cdc317b..8fa84d3a278 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -245,7 +245,7 @@ Paths Store::topoSortPaths(const PathSet & paths)
 
     dfsVisit = [&](const Path & path, const Path * parent) {
         if (parents.find(path) != parents.end())
-            throw BuildError(format("cycle detected in the references of ‘%1%’ from ‘%2%’") % path % *parent);
+            throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
 
         if (visited.find(path) != visited.end()) return;
         visited.insert(path);
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 82595e76a9b..2afdeb021a9 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -46,7 +46,7 @@ struct NarIndexer : ParseSink, StringSource
             parents.push(&root);
         } else {
             if(parents.top()->type != FSAccessor::Type::tDirectory) {
-                throw Error(format("NAR file missing parent directory of path ‘%1%’") % path);
+                throw Error(format("NAR file missing parent directory of path '%1%'") % path);
             }
             auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
             parents.push(&result.first->second);
@@ -120,7 +120,7 @@ struct NarIndexer : ParseSink, StringSource
     NarMember& at(const Path & path) {
         auto result = find(path);
         if(result == nullptr) {
-            throw Error(format("NAR file does not contain path ‘%1%’") % path);
+            throw Error(format("NAR file does not contain path '%1%'") % path);
         }
         return *result;
     }
@@ -149,7 +149,7 @@ struct NarAccessor : public FSAccessor
         auto i = indexer.at(path);
 
         if (i.type != FSAccessor::Type::tDirectory)
-            throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
+            throw Error(format("path '%1%' inside NAR file is not a directory") % path);
 
         StringSet res;
         for(auto&& child : i.children) {
@@ -163,7 +163,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = indexer.at(path);
         if (i.type != FSAccessor::Type::tRegular)
-            throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
+            throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
         return std::string(*nar, i.start, i.size);
     }
 
@@ -171,7 +171,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = indexer.at(path);
         if (i.type != FSAccessor::Type::tSymlink)
-            throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
+            throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
         return i.target;
     }
 };
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 660f6a42a19..cb568ccdc82 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -6,7 +6,7 @@ namespace nix {
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
 {
     auto corrupt = [&]() {
-        throw Error(format("NAR info file ‘%1%’ is corrupt") % whence);
+        throw Error(format("NAR info file '%1%' is corrupt") % whence);
     };
 
     auto parseHashField = [&](const string & s) {
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 8e8002a30db..4461fc6dd70 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -19,9 +19,9 @@ static void makeWritable(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-        throw SysError(format("changing writability of ‘%1%’") % path);
+        throw SysError(format("changing writability of '%1%'") % path);
 }
 
 
@@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
     InodeHash inodeHash;
 
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir);
+    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
         // We don't care if we hit non-hash files, anything goes
         inodeHash.insert(dirent->d_ino);
     }
-    if (errno) throw SysError(format("reading directory ‘%1%’") % linksDir);
+    if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
 
     printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
 
@@ -68,14 +68,14 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
     Strings names;
 
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory ‘%1%’") % path);
+    if (!dir) throw SysError(format("opening directory '%1%'") % path);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
         checkInterrupt();
 
         if (inodeHash.count(dirent->d_ino)) {
-            debug(format("‘%1%’ is already linked") % dirent->d_name);
+            debug(format("'%1%' is already linked") % dirent->d_name);
             continue;
         }
 
@@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
         if (name == "." || name == "..") continue;
         names.push_back(name);
     }
-    if (errno) throw SysError(format("reading directory ‘%1%’") % path);
+    if (errno) throw SysError(format("reading directory '%1%'") % path);
 
     return names;
 }
@@ -95,7 +95,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
 
 #if __APPLE__
     /* HFS/OS X has some undocumented security feature disabling hardlinking for
@@ -105,7 +105,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
 
     if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
         std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$"))) {
-        debug(format("‘%1%’ is not allowed to be linked in OS X") % path);
+        debug(format("'%1%' is not allowed to be linked in OS X") % path);
         return;
     }
 #endif
@@ -129,13 +129,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        printError(format("skipping suspicious writable file ‘%1%’") % path);
+        printError(format("skipping suspicious writable file '%1%'") % path);
         return;
     }
 
     /* This can still happen on top-level files. */
     if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
-        debug(format("‘%1%’ is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
+        debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
         return;
     }
 
@@ -149,7 +149,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(htSHA256, path).first;
-    debug(format("‘%1%’ has hash ‘%2%’") % path % hash.to_string());
+    debug(format("'%1%' has hash '%2%'") % path % hash.to_string());
 
     /* Check if this is a known hash. */
     Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
@@ -173,11 +173,11 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
                full.  When that happens, it's fine to ignore it: we
                just effectively disable deduplication of this
                file.  */
-            printInfo("cannot link ‘%s’ to ‘%s’: %s", linkPath, path, strerror(errno));
+            printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
             return;
 
         default:
-            throw SysError("cannot link ‘%1%’ to ‘%2%’", linkPath, path);
+            throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
         }
     }
 
@@ -185,20 +185,20 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
        current file with a hard link to that file. */
     struct stat stLink;
     if (lstat(linkPath.c_str(), &stLink))
-        throw SysError(format("getting attributes of path ‘%1%’") % linkPath);
+        throw SysError(format("getting attributes of path '%1%'") % linkPath);
 
     if (st.st_ino == stLink.st_ino) {
-        debug(format("‘%1%’ is already linked to ‘%2%’") % path % linkPath);
+        debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
         return;
     }
 
     if (st.st_size != stLink.st_size) {
-        printError(format("removing corrupted link ‘%1%’") % linkPath);
+        printError(format("removing corrupted link '%1%'") % linkPath);
         unlink(linkPath.c_str());
         goto retry;
     }
 
-    printMsg(lvlTalkative, format("linking ‘%1%’ to ‘%2%’") % path % linkPath);
+    printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
 
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
@@ -219,25 +219,25 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
                systems).  This is likely to happen with empty files.
                Just shrug and ignore. */
             if (st.st_size)
-                printInfo(format("‘%1%’ has maximum number of links") % linkPath);
+                printInfo(format("'%1%' has maximum number of links") % linkPath);
             return;
         }
-        throw SysError("cannot link ‘%1%’ to ‘%2%’", tempLink, linkPath);
+        throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
     }
 
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            printError(format("unable to unlink ‘%1%’") % tempLink);
+            printError(format("unable to unlink '%1%'") % tempLink);
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
                temporarily increases the st_nlink field before
                decreasing it again.) */
-            debug("‘%s’ has reached maximum number of links", linkPath);
+            debug("'%s' has reached maximum number of links", linkPath);
             return;
         }
-        throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path);
+        throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
     }
 
     stats.filesLinked++;
@@ -254,7 +254,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
     for (auto & i : paths) {
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
-        //Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
+        //Activity act(*logger, lvlChatty, format("hashing files in '%1%'") % i);
         optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
     }
 }
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index bf7ad3d2185..587f2959885 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -19,7 +19,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
 
     fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
     if (!fd && (create || errno != ENOENT))
-        throw SysError(format("opening lock file ‘%1%’") % path);
+        throw SysError(format("opening lock file '%1%'") % path);
 
     return fd;
 }
@@ -109,12 +109,12 @@ bool PathLocks::lockPaths(const PathSet & _paths,
         checkInterrupt();
         Path lockPath = path + ".lock";
 
-        debug(format("locking path ‘%1%’") % path);
+        debug(format("locking path '%1%'") % path);
 
         {
             auto lockedPaths(lockedPaths_.lock());
             if (lockedPaths->count(lockPath))
-                throw Error("deadlock: trying to re-acquire self-held lock ‘%s’", lockPath);
+                throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
             lockedPaths->insert(lockPath);
         }
 
@@ -141,19 +141,19 @@ bool PathLocks::lockPaths(const PathSet & _paths,
                     }
                 }
 
-                debug(format("lock acquired on ‘%1%’") % lockPath);
+                debug(format("lock acquired on '%1%'") % lockPath);
 
                 /* Check that the lock file hasn't become stale (i.e.,
                    hasn't been unlinked). */
                 struct stat st;
                 if (fstat(fd.get(), &st) == -1)
-                    throw SysError(format("statting lock file ‘%1%’") % lockPath);
+                    throw SysError(format("statting lock file '%1%'") % lockPath);
                 if (st.st_size != 0)
                     /* This lock file has been unlinked, so we're holding
                        a lock on a deleted file.  This means that other
                        processes may create and acquire a lock on
                        `lockPath', and proceed.  So we must retry. */
-                    debug(format("open lock file ‘%1%’ has become stale") % lockPath);
+                    debug(format("open lock file '%1%' has become stale") % lockPath);
                 else
                     break;
             }
@@ -191,9 +191,9 @@ void PathLocks::unlock()
 
         if (close(i.first) == -1)
             printError(
-                format("error (ignored): cannot close lock file on ‘%1%’") % i.second);
+                format("error (ignored): cannot close lock file on '%1%'") % i.second);
 
-        debug(format("lock released on ‘%1%’") % i.second);
+        debug(format("lock released on '%1%'") % i.second);
     }
 
     fds.clear();
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index f24daa8862a..4a607b58450 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen)
             gen.number = n;
             struct stat st;
             if (lstat(gen.path.c_str(), &st) != 0)
-                throw SysError(format("statting ‘%1%’") % gen.path);
+                throw SysError(format("statting '%1%'") % gen.path);
             gen.creationTime = st.st_mtime;
             gens.push_back(gen);
         }
@@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
 static void removeFile(const Path & path)
 {
     if (remove(path.c_str()) == -1)
-        throw SysError(format("cannot unlink ‘%1%’") % path);
+        throw SysError(format("cannot unlink '%1%'") % path);
 }
 
 
@@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     Generations gens = findGenerations(profile, curGen);
 
     if (gensToDelete.find(curGen) != gensToDelete.end())
-        throw Error(format("cannot delete current generation of profile %1%’") % profile);
+        throw Error(format("cannot delete current generation of profile %1%'") % profile);
 
     for (auto & i : gens) {
         if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@@ -203,7 +203,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
     int days;
 
     if (!string2Int(strDays, days) || days < 1)
-        throw Error(format("invalid number of days specifier ‘%1%’") % timeSpec);
+        throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
 
     time_t oldTime = curTime - days * 24 * 3600;
 
@@ -222,7 +222,7 @@ void switchLink(Path link, Path target)
 
 void lockProfile(PathLocks & lock, const Path & profile)
 {
-    lock.lockPaths({profile}, (format("waiting for lock on profile ‘%1%’") % profile).str());
+    lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());
     lock.setDeletion(true);
 }
 
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 33eab5a240b..ba9f18b9ca5 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -37,7 +37,7 @@ static void search(const unsigned char * s, unsigned int len,
         if (!match) continue;
         string ref((const char *) s + i, refLength);
         if (hashes.find(ref) != hashes.end()) {
-            debug(format("found reference to ‘%1%’ at offset ‘%2%’")
+            debug(format("found reference to '%1%' at offset '%2%'")
                   % ref % i);
             seen.insert(ref);
             hashes.erase(ref);
@@ -93,7 +93,7 @@ PathSet scanForReferences(const string & path,
         string baseName = baseNameOf(i);
         string::size_type pos = baseName.find('-');
         if (pos == string::npos)
-            throw Error(format("bad reference ‘%1%’") % i);
+            throw Error(format("bad reference '%1%'") % i);
         string s = string(baseName, 0, pos);
         assert(s.size() == refLength);
         assert(backMap.find(s) == backMap.end());
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index ca14057c2e2..098151f8c0f 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -17,7 +17,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     std::string restPath = std::string(path, storePath.size());
 
     if (!store->isValidPath(storePath))
-        throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
+        throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
 
     auto i = nars.find(storePath);
     if (i != nars.end()) return {i->second, restPath};
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ab726e79534..e9f2cee80db 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -54,7 +54,7 @@ RemoteStore::RemoteStore(const Params & params)
 ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 {
     if (failed)
-        throw Error("opening a connection to remote store ‘%s’ previously failed", getUri());
+        throw Error("opening a connection to remote store '%s' previously failed", getUri());
     try {
         return openConnection();
     } catch (...) {
@@ -97,11 +97,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
     if (socketPath.size() + 1 >= sizeof(addr.sun_path))
-        throw Error(format("socket path ‘%1%’ is too long") % socketPath);
+        throw Error(format("socket path '%1%' is too long") % socketPath);
     strcpy(addr.sun_path, socketPath.c_str());
 
     if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath);
+        throw SysError(format("cannot connect to daemon at '%1%'") % socketPath);
 
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();
@@ -142,7 +142,7 @@ void RemoteStore::initConnection(Connection & conn)
         conn.processStderr();
     }
     catch (Error & e) {
-        throw Error("cannot open connection to remote store ‘%s’: %s", getUri(), e.what());
+        throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
     }
 
     setOptions(conn);
@@ -288,7 +288,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
         }
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
             bool valid; conn->from >> valid;
-            if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
+            if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
         }
         auto info = std::make_shared<ValidPathInfo>();
         info->path = path;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 145a8191c55..5fc7371a519 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -110,7 +110,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
 S3Helper::DownloadResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
-    debug("fetching ‘s3://%s/%s’...", bucketName, key);
+    debug("fetching 's3://%s/%s'...", bucketName, key);
 
     auto request =
         Aws::S3::Model::GetObjectRequest()
@@ -127,7 +127,7 @@ S3Helper::DownloadResult S3Helper::getObject(
 
     try {
 
-        auto result = checkAws(fmt("AWS error fetching ‘%s’", key),
+        auto result = checkAws(fmt("AWS error fetching '%s'", key),
             client->GetObject(request));
 
         res.data = decodeContent(
@@ -185,9 +185,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             if (!res.IsSuccess()) {
                 if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
-                    throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
+                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());
 
-                printInfo("creating S3 bucket ‘%s’...", bucketName);
+                printInfo("creating S3 bucket '%s'...", bucketName);
 
                 // Stupid S3 bucket locations.
                 auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
@@ -196,7 +196,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                         Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
                             s3Helper.config->region));
 
-                checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
+                checkAws(format("AWS error creating bucket '%s'") % bucketName,
                     s3Helper.client->CreateBucket(
                         Aws::S3::Model::CreateBucketRequest()
                         .WithBucket(bucketName)
@@ -244,7 +244,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
                     && error.GetMessage().find("404") != std::string::npos))
                 return false;
-            throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage());
+            throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
         }
 
         return true;
@@ -273,14 +273,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
         auto now1 = std::chrono::steady_clock::now();
 
-        auto result = checkAws(format("AWS error uploading ‘%s’") % path,
+        auto result = checkAws(format("AWS error uploading '%s'") % path,
             s3Helper.client->PutObject(request));
 
         auto now2 = std::chrono::steady_clock::now();
 
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
 
-        printInfo(format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
+        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms")
             % bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
@@ -312,7 +312,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             stats.getTimeMs += res.durationMs;
 
             if (res.data)
-                printTalkative("downloaded ‘s3://%s/%s’ (%d bytes) in %d ms",
+                printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                     bucketName, path, res.data->size(), res.durationMs);
 
             return res.data;
@@ -325,9 +325,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         std::string marker;
 
         do {
-            debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker);
+            debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);
 
-            auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName,
+            auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
                 s3Helper.client->ListObjects(
                     Aws::S3::Model::ListObjectsRequest()
                     .WithBucket(bucketName)
@@ -336,7 +336,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             auto & contents = res.GetContents();
 
-            debug(format("got %d keys, next marker ‘%s’")
+            debug(format("got %d keys, next marker '%s'")
                 % contents.size() % res.GetNextMarker());
 
             for (auto object : contents) {
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index a81e62dbd6e..b13001b06d5 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -17,18 +17,18 @@ namespace nix {
     if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
         throw SQLiteBusy(
             err == SQLITE_PROTOCOL
-            ? fmt("SQLite database ‘%s’ is busy (SQLITE_PROTOCOL)", path)
-            : fmt("SQLite database ‘%s’ is busy", path));
+            ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path)
+            : fmt("SQLite database '%s' is busy", path));
     }
     else
-        throw SQLiteError("%s: %s (in ‘%s’)", f.str(), sqlite3_errstr(err), path);
+        throw SQLiteError("%s: %s (in '%s')", f.str(), sqlite3_errstr(err), path);
 }
 
 SQLite::SQLite(const Path & path)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
             SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
-        throw Error(format("cannot open SQLite database ‘%s’") % path);
+        throw Error(format("cannot open SQLite database '%s'") % path);
 }
 
 SQLite::~SQLite()
@@ -45,7 +45,7 @@ void SQLite::exec(const std::string & stmt)
 {
     retrySQLite<void>([&]() {
         if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
-            throwSQLiteError(db, format("executing SQLite statement ‘%s’") % stmt);
+            throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt);
     });
 }
 
@@ -54,7 +54,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & sql)
     checkInterrupt();
     assert(!stmt);
     if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK)
-        throwSQLiteError(db, fmt("creating statement ‘%s’", sql));
+        throwSQLiteError(db, fmt("creating statement '%s'", sql));
     this->db = db;
     this->sql = sql;
 }
@@ -63,7 +63,7 @@ SQLiteStmt::~SQLiteStmt()
 {
     try {
         if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
-            throwSQLiteError(db, fmt("finalizing statement ‘%s’", sql));
+            throwSQLiteError(db, fmt("finalizing statement '%s'", sql));
     } catch (...) {
         ignoreException();
     }
@@ -120,14 +120,14 @@ void SQLiteStmt::Use::exec()
     int r = step();
     assert(r != SQLITE_ROW);
     if (r != SQLITE_DONE)
-        throwSQLiteError(stmt.db, fmt("executing SQLite statement ‘%s’", stmt.sql));
+        throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
 }
 
 bool SQLiteStmt::Use::next()
 {
     int r = step();
     if (r != SQLITE_DONE && r != SQLITE_ROW)
-        throwSQLiteError(stmt.db, fmt("executing SQLite query ‘%s’", stmt.sql));
+        throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
     return r == SQLITE_ROW;
 }
 
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 6edabaa3a1d..776ffdb8343 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -41,7 +41,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("executing ‘%s’ on ‘%s’", command, host);
+        throw SysError("executing '%s' on '%s'", command, host);
     });
 
 
@@ -96,7 +96,7 @@ Path SSHMaster::startMaster()
     } catch (EndOfFile & e) { }
 
     if (reply != "started")
-        throw Error("failed to start SSH master connection to ‘%s’", host);
+        throw Error("failed to start SSH master connection to '%s'", host);
 
     return state->socketPath;
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 27bfbcb8f12..dd87a046e38 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -30,14 +30,14 @@ bool Store::isStorePath(const Path & path) const
 void Store::assertStorePath(const Path & path) const
 {
     if (!isStorePath(path))
-        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
+        throw Error(format("path '%1%' is not in the Nix store") % path);
 }
 
 
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
-        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
+        throw Error(format("path '%1%' is not in the Nix store") % path);
     Path::size_type slash = path.find('/', storeDir.size() + 1);
     if (slash == Path::npos)
         return path;
@@ -55,7 +55,7 @@ Path Store::followLinksToStore(const Path & _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
+        throw Error(format("path '%1%' is not in the Nix store") % path);
     return path;
 }
 
@@ -88,14 +88,14 @@ void checkStoreName(const string & name)
     /* Disallow names starting with a dot for possible security
        reasons (e.g., "." and ".."). */
     if (string(name, 0, 1) == ".")
-        throw Error(format("illegal name: ‘%1%’") % name);
+        throw Error(format("illegal name: '%1%'") % name);
     for (auto & i : name)
         if (!((i >= 'A' && i <= 'Z') ||
               (i >= 'a' && i <= 'z') ||
               (i >= '0' && i <= '9') ||
               validChars.find(i) != string::npos))
         {
-            throw Error(format("invalid character ‘%1%’ in name ‘%2%’")
+            throw Error(format("invalid character '%1%' in name '%2%'")
                 % i % name);
         }
 }
@@ -328,7 +328,7 @@ void Store::queryPathInfo(const Path & storePath,
             if (res) {
                 stats.narInfoReadAverted++;
                 if (!*res)
-                    throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+                    throw InvalidPath(format("path '%s' is not valid") % storePath);
                 return success(ref<ValidPathInfo>(*res));
             }
         }
@@ -343,7 +343,7 @@ void Store::queryPathInfo(const Path & storePath,
                         res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
                     if (res.first == NarInfoDiskCache::oInvalid ||
                         (res.second->path != storePath && storePathToName(storePath) != ""))
-                        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+                        throw InvalidPath(format("path '%s' is not valid") % storePath);
                 }
                 return success(ref<ValidPathInfo>(res.second));
             }
@@ -368,7 +368,7 @@ void Store::queryPathInfo(const Path & storePath,
                 || (info->path != storePath && storePathToName(storePath) != ""))
             {
                 stats.narInfoMissing++;
-                return failure(std::make_exception_ptr(InvalidPath(format("path ‘%s’ is not valid") % storePath)));
+                return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath)));
             }
 
             callSuccess(success, failure, ref<ValidPathInfo>(info));
@@ -614,7 +614,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             checkInterrupt();
 
             if (!dstStore->isValidPath(storePath)) {
-                printInfo("copying ‘%s’...", storePath);
+                printInfo("copying '%s'...", storePath);
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
             }
         });
@@ -661,7 +661,7 @@ string showPaths(const PathSet & paths)
     string s;
     for (auto & i : paths) {
         if (s.size() != 0) s += ", ";
-        s += "‘" + i + "’";
+        s += "'" + i + "'";
     }
     return s;
 }
@@ -670,7 +670,7 @@ string showPaths(const PathSet & paths)
 std::string ValidPathInfo::fingerprint() const
 {
     if (narSize == 0 || !narHash)
-        throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
+        throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known")
             % path);
     return
         "1;" + path + ";"
@@ -689,7 +689,7 @@ void ValidPathInfo::sign(const SecretKey & secretKey)
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
     auto warn = [&]() {
-        printError(format("warning: path ‘%s’ claims to be content-addressed but isn't") % path);
+        printError(format("warning: path '%s' claims to be content-addressed but isn't") % path);
     };
 
     if (hasPrefix(ca, "text:")) {
@@ -782,7 +782,7 @@ ref<Store> openStore(const std::string & uri_,
         }
     }
 
-    throw Error("don't know how to open Nix store ‘%s’", uri);
+    throw Error("don't know how to open Nix store '%s'", uri);
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3247a67678c..5f3d8c7b952 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -604,7 +604,7 @@ protected:
     /* Unsupported methods. */
     [[noreturn]] void unsupported()
     {
-        throw Unsupported("requested operation is not supported by store ‘%s’", getUri());
+        throw Unsupported("requested operation is not supported by store '%s'", getUri());
     }
 
 };
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index e0e6f5dfd73..be527450c46 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -38,7 +38,7 @@ static void dumpContents(const Path & path, size_t size,
     sink << "contents" << size;
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file ‘%1%’") % path);
+    if (!fd) throw SysError(format("opening file '%1%'") % path);
 
     unsigned char buf[65536];
     size_t left = size;
@@ -58,7 +58,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path ‘%1%’") % path);
+        throw SysError(format("getting attributes of path '%1%'") % path);
 
     sink << "(";
 
@@ -80,11 +80,11 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
                 string name(i.name);
                 size_t pos = i.name.find(caseHackSuffix);
                 if (pos != string::npos) {
-                    debug(format("removing case hack suffix from ‘%1%’") % (path + "/" + i.name));
+                    debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name));
                     name.erase(pos);
                 }
                 if (unhacked.find(name) != unhacked.end())
-                    throw Error(format("file name collision in between ‘%1%’ and ‘%2%’")
+                    throw Error(format("file name collision in between '%1%' and '%2%'")
                         % (path + "/" + unhacked[name]) % (path + "/" + i.name));
                 unhacked[name] = i.name;
             } else
@@ -101,7 +101,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
     else if (S_ISLNK(st.st_mode))
         sink << "type" << "symlink" << "target" << readLink(path);
 
-    else throw Error(format("file ‘%1%’ has an unsupported type") % path);
+    else throw Error(format("file '%1%' has an unsupported type") % path);
 
     sink << ")";
 }
@@ -237,14 +237,14 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                 } else if (s == "name") {
                     name = readString(source);
                     if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
-                        throw Error(format("NAR contains invalid file name ‘%1%’") % name);
+                        throw Error(format("NAR contains invalid file name '%1%'") % name);
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
                     if (useCaseHack) {
                         auto i = names.find(name);
                         if (i != names.end()) {
-                            debug(format("case collision between ‘%1%’ and ‘%2%’") % i->first % name);
+                            debug(format("case collision between '%1%' and '%2%'") % i->first % name);
                             name += caseHackSuffix;
                             name += std::to_string(++i->second);
                         } else
@@ -293,14 +293,14 @@ struct RestoreSink : ParseSink
     {
         Path p = dstPath + path;
         if (mkdir(p.c_str(), 0777) == -1)
-            throw SysError(format("creating directory ‘%1%’") % p);
+            throw SysError(format("creating directory '%1%'") % p);
     };
 
     void createRegularFile(const Path & path)
     {
         Path p = dstPath + path;
         fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
-        if (!fd) throw SysError(format("creating file ‘%1%’") % p);
+        if (!fd) throw SysError(format("creating file '%1%'") % p);
     }
 
     void isExecutable()
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 19a45d7e9b3..2b72079f2b4 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -47,7 +47,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         }
         else if (!dashDash && std::string(arg, 0, 1) == "-") {
             if (!processFlag(pos, cmdline.end()))
-                throw UsageError(format("unrecognised flag ‘%1%’") % arg);
+                throw UsageError(format("unrecognised flag '%1%'") % arg);
         }
         else {
             pendingArgs.push_back(*pos++);
@@ -103,7 +103,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         Strings args;
         for (size_t n = 0 ; n < flag.arity; ++n) {
             if (pos == end)
-                throw UsageError(format("flag ‘%1%’ requires %2% argument(s)")
+                throw UsageError(format("flag '%1%' requires %2% argument(s)")
                     % name % flag.arity);
             args.push_back(*pos++);
         }
@@ -131,7 +131,7 @@ bool Args::processArgs(const Strings & args, bool finish)
 {
     if (expectedArgs.empty()) {
         if (!args.empty())
-            throw UsageError(format("unexpected argument ‘%1%’") % args.front());
+            throw UsageError(format("unexpected argument '%1%'") % args.front());
         return true;
     }
 
@@ -155,10 +155,10 @@ bool Args::processArgs(const Strings & args, bool finish)
 
 void Args::mkHashTypeFlag(const std::string & name, HashType * ht)
 {
-    mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) {
+    mkFlag1(0, name, "TYPE", "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", [=](std::string s) {
         *ht = parseHashType(s);
         if (*ht == htUnknown)
-            throw UsageError(format("unknown hash type ‘%1%’") % s);
+            throw UsageError(format("unknown hash type '%1%'") % s);
     });
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 37e780dd174..fd910b965cc 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -158,7 +158,7 @@ public:
             .handler([=](Strings ss) {
                 I n;
                 if (!string2Int(ss.front(), n))
-                    throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName);
+                    throw UsageError(format("flag '--%1%' requires a integer argument") % longName);
                 fun(n);
             });
     }
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index b0b1d709fa4..2b3dff3a5ea 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -118,7 +118,7 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
     else if (method == "br")
         return decompressBrotli(in);
     else
-        throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
+        throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
 
 struct NoneSink : CompressionSink
@@ -309,7 +309,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "br")
         return make_ref<BrotliSink>(nextSink);
     else
-        throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
+        throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
 
 }
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 0682bcd5dbc..27157a83178 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -81,7 +81,7 @@ void Config::applyConfigFile(const Path & path, bool fatal)
             if (tokens.empty()) continue;
 
             if (tokens.size() < 2 || tokens[1] != "=")
-                throw UsageError("illegal configuration line ‘%1%’ in ‘%2%’", line, path);
+                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
             string name = tokens[0];
 
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index e16c3b6ea93..11e3c9dca58 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -133,14 +133,14 @@ Hash::Hash(const std::string & s, HashType type)
 
     if (colon == string::npos) {
         if (type == htUnknown)
-            throw BadHash("hash ‘%s’ does not include a type", s);
+            throw BadHash("hash '%s' does not include a type", s);
     } else {
         string hts = string(s, 0, colon);
         this->type = parseHashType(hts);
         if (this->type == htUnknown)
-            throw BadHash("unknown hash type ‘%s’", hts);
+            throw BadHash("unknown hash type '%s'", hts);
         if (type != htUnknown && type != this->type)
-            throw BadHash("hash ‘%s’ should have type ‘%s’", s, printHashType(type));
+            throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
         pos = colon + 1;
     }
 
@@ -154,7 +154,7 @@ Hash::Hash(const std::string & s, HashType type)
             if (c >= '0' && c <= '9') return c - '0';
             if (c >= 'A' && c <= 'F') return c - 'A' + 10;
             if (c >= 'a' && c <= 'f') return c - 'a' + 10;
-            throw BadHash("invalid base-16 hash ‘%s’", s);
+            throw BadHash("invalid base-16 hash '%s'", s);
         };
 
         for (unsigned int i = 0; i < hashSize; i++) {
@@ -172,7 +172,7 @@ Hash::Hash(const std::string & s, HashType type)
             for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
                 if (base32Chars[digit] == c) break;
             if (digit >= 32)
-                throw BadHash("invalid base-32 hash ‘%s’", s);
+                throw BadHash("invalid base-32 hash '%s'", s);
             unsigned int b = n * 5;
             unsigned int i = b / 8;
             unsigned int j = b % 8;
@@ -182,7 +182,7 @@ Hash::Hash(const std::string & s, HashType type)
                 hash[i + 1] |= digit >> (8 - j);
             } else {
                 if (digit >> (8 - j))
-                    throw BadHash("invalid base-32 hash ‘%s’", s);
+                    throw BadHash("invalid base-32 hash '%s'", s);
             }
         }
     }
@@ -194,7 +194,7 @@ Hash::Hash(const std::string & s, HashType type)
     }
 
     else
-        throw BadHash("hash ‘%s’ has wrong length for hash type ‘%s’", s, printHashType(type));
+        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
 }
 
 
@@ -253,13 +253,13 @@ Hash hashFile(HashType ht, const Path & path)
     start(ht, ctx);
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file ‘%1%’") % path);
+    if (!fd) throw SysError(format("opening file '%1%'") % path);
 
     unsigned char buf[8192];
     ssize_t n;
     while ((n = read(fd.get(), buf, sizeof(buf)))) {
         checkInterrupt();
-        if (n == -1) throw SysError(format("reading file ‘%1%’") % path);
+        if (n == -1) throw SysError(format("reading file '%1%'") % path);
         update(ht, ctx, buf, n);
     }
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index b8b432256c9..d83049b0236 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -38,7 +38,7 @@ struct Hash
     Hash(HashType type) : type(type) { init(); };
 
     /* Initialize the hash from a string representation, in the format
-       "[<type>:]<base16|base32|base64>". If the ‘type’ argument is
+       "[<type>:]<base16|base32|base64>". If the 'type' argument is
        htUnknown, then the hash type must be specified in the
        string. */
     Hash(const std::string & s, HashType type = htUnknown);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 2bdee70807b..70b19394163 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -198,7 +198,7 @@ T readNum(Source & source)
         ((unsigned long long) buf[7] << 56);
 
     if (n > std::numeric_limits<T>::max())
-        throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name());
+        throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
 
     return n;
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index d9f8011b6fb..27f4fea187d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -104,7 +104,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
     string s;
 
     if (path[0] != '/')
-        throw Error(format("not an absolute path: ‘%1%’") % path);
+        throw Error(format("not an absolute path: '%1%'") % path);
 
     string::const_iterator i = path.begin(), end = path.end();
     string temp;
@@ -140,7 +140,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
                the symlink target might contain new symlinks). */
             if (resolveSymlinks && isLink(s)) {
                 if (++followCount >= maxFollow)
-                    throw Error(format("infinite symlink recursion in path ‘%1%’") % path);
+                    throw Error(format("infinite symlink recursion in path '%1%'") % path);
                 temp = absPath(readLink(s), dirOf(s))
                     + string(i, end);
                 i = temp.begin(); /* restart */
@@ -158,7 +158,7 @@ Path dirOf(const Path & path)
 {
     Path::size_type pos = path.rfind('/');
     if (pos == string::npos)
-        throw Error(format("invalid file name ‘%1%’") % path);
+        throw Error(format("invalid file name '%1%'") % path);
     return pos == 0 ? "/" : Path(path, 0, pos);
 }
 
@@ -195,7 +195,7 @@ struct stat lstat(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting status of ‘%1%’") % path);
+        throw SysError(format("getting status of '%1%'") % path);
     return st;
 }
 
@@ -217,13 +217,13 @@ Path readLink(const Path & path)
     checkInterrupt();
     struct stat st = lstat(path);
     if (!S_ISLNK(st.st_mode))
-        throw Error(format("‘%1%’ is not a symlink") % path);
+        throw Error(format("'%1%' is not a symlink") % path);
     char buf[st.st_size];
     ssize_t rlsize = readlink(path.c_str(), buf, st.st_size);
     if (rlsize == -1)
-        throw SysError(format("reading symbolic link ‘%1%’") % path);
+        throw SysError(format("reading symbolic link '%1%'") % path);
     else if (rlsize > st.st_size)
-        throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%")
+        throw Error(format("symbolic link '%1%' size overflow %2% > %3%")
             % path % rlsize % st.st_size);
     return string(buf, rlsize);
 }
@@ -242,7 +242,7 @@ DirEntries readDirectory(const Path & path)
     entries.reserve(64);
 
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory ‘%1%’") % path);
+    if (!dir) throw SysError(format("opening directory '%1%'") % path);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -257,7 +257,7 @@ DirEntries readDirectory(const Path & path)
 #endif
         );
     }
-    if (errno) throw SysError(format("reading directory ‘%1%’") % path);
+    if (errno) throw SysError(format("reading directory '%1%'") % path);
 
     return entries;
 }
@@ -290,7 +290,7 @@ string readFile(const Path & path, bool drain)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd)
-        throw SysError(format("opening file ‘%1%’") % path);
+        throw SysError(format("opening file '%1%'") % path);
     return drain ? drainFD(fd.get()) : readFile(fd.get());
 }
 
@@ -299,7 +299,7 @@ void writeFile(const Path & path, const string & s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
-        throw SysError(format("opening file ‘%1%’") % path);
+        throw SysError(format("opening file '%1%'") % path);
     writeFull(fd.get(), s);
 }
 
@@ -338,7 +338,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
     struct stat st;
     if (lstat(path.c_str(), &st) == -1) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of ‘%1%’") % path);
+        throw SysError(format("getting status of '%1%'") % path);
     }
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
@@ -349,7 +349,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
         const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
         if ((st.st_mode & PERM_MASK) != PERM_MASK) {
             if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1)
-                throw SysError(format("chmod ‘%1%’") % path);
+                throw SysError(format("chmod '%1%'") % path);
         }
 
         for (auto & i : readDirectory(path))
@@ -358,7 +358,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
     if (remove(path.c_str()) == -1) {
         if (errno == ENOENT) return;
-        throw SysError(format("cannot unlink ‘%1%’") % path);
+        throw SysError(format("cannot unlink '%1%'") % path);
     }
 }
 
@@ -372,7 +372,7 @@ void deletePath(const Path & path)
 
 void deletePath(const Path & path, unsigned long long & bytesFreed)
 {
-    //Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path);
+    //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
     bytesFreed = 0;
     _deletePath(path, bytesFreed);
 }
@@ -410,12 +410,12 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
                "wheel", then "tar" will fail to unpack archives that
                have the setgid bit set on directories. */
             if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError(format("setting group of directory ‘%1%’") % tmpDir);
+                throw SysError(format("setting group of directory '%1%'") % tmpDir);
 #endif
             return tmpDir;
         }
         if (errno != EEXIST)
-            throw SysError(format("creating directory ‘%1%’") % tmpDir);
+            throw SysError(format("creating directory '%1%'") % tmpDir);
     }
 }
 
@@ -473,15 +473,15 @@ Paths createDirs(const Path & path)
     if (lstat(path.c_str(), &st) == -1) {
         created = createDirs(dirOf(path));
         if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
-            throw SysError(format("creating directory ‘%1%’") % path);
+            throw SysError(format("creating directory '%1%'") % path);
         st = lstat(path);
         created.push_back(path);
     }
 
     if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
-        throw SysError(format("statting symlink ‘%1%’") % path);
+        throw SysError(format("statting symlink '%1%'") % path);
 
-    if (!S_ISDIR(st.st_mode)) throw Error(format("‘%1%’ is not a directory") % path);
+    if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path);
 
     return created;
 }
@@ -490,7 +490,7 @@ Paths createDirs(const Path & path)
 void createSymlink(const Path & target, const Path & link)
 {
     if (symlink(target.c_str(), link.c_str()))
-        throw SysError(format("creating symlink from ‘%1%’ to ‘%2%’") % link % target);
+        throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target);
 }
 
 
@@ -507,7 +507,7 @@ void replaceSymlink(const Path & target, const Path & link)
         }
 
         if (rename(tmp.c_str(), link.c_str()) != 0)
-            throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link);
+            throw SysError(format("renaming '%1%' to '%2%'") % tmp % link);
 
         break;
     }
@@ -589,7 +589,7 @@ AutoDelete::~AutoDelete()
                 deletePath(path);
             else {
                 if (remove(path.c_str()) == -1)
-                    throw SysError(format("cannot unlink ‘%1%’") % path);
+                    throw SysError(format("cannot unlink '%1%'") % path);
             }
         }
     } catch (...) {
@@ -786,7 +786,7 @@ pid_t Pid::release()
 
 void killUser(uid_t uid)
 {
-    debug(format("killing all processes running under uid ‘%1%’") % uid);
+    debug(format("killing all processes running under uid '%1%'") % uid);
 
     assert(uid != 0); /* just to be safe... */
 
@@ -815,7 +815,7 @@ void killUser(uid_t uid)
 #endif
             if (errno == ESRCH) break; /* no more processes */
             if (errno != EINTR)
-                throw SysError(format("cannot kill processes for uid ‘%1%’") % uid);
+                throw SysError(format("cannot kill processes for uid '%1%'") % uid);
         }
 
         _exit(0);
@@ -823,7 +823,7 @@ void killUser(uid_t uid)
 
     int status = pid.wait();
     if (status != 0)
-        throw Error(format("cannot kill processes for uid ‘%1%’: %2%") % uid % statusToString(status));
+        throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status));
 
     /* !!! We should really do some check to make sure that there are
        no processes left running under `uid', but there is no portable
@@ -917,7 +917,7 @@ string runProgram(Path program, bool searchPath, const Strings & args,
         else
             execv(program.c_str(), stringsToCharPtrs(args_).data());
 
-        throw SysError(format("executing ‘%1%’") % program);
+        throw SysError(format("executing '%1%'") % program);
     });
 
     out.writeSide = -1;
@@ -950,7 +950,7 @@ string runProgram(Path program, bool searchPath, const Strings & args,
     /* Wait for the child to finish. */
     int status = pid.wait();
     if (!statusOk(status))
-        throw ExecError(status, format("program ‘%1%’ %2%")
+        throw ExecError(status, format("program '%1%' %2%")
             % program % statusToString(status));
 
     /* Wait for the writer thread to finish. */
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 769ca8cd03a..b36933a0ac2 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -238,7 +238,7 @@ void mainWrapped(int argc, char * * argv)
     });
 
     if (packages && fromArgs)
-        throw UsageError("‘-p’ and ‘-E’ are mutually exclusive");
+        throw UsageError("'-p' and '-E' are mutually exclusive");
 
     auto store = openStore();
 
@@ -336,7 +336,7 @@ void mainWrapped(int argc, char * * argv)
 
                 auto drv = getDerivation(state, v, false);
                 if (!drv)
-                    throw Error("the ‘bashInteractive’ attribute in <nixpkgs> did not evaluate to a derivation");
+                    throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation");
 
                 pathsToBuild.insert(drv->queryDrvPath());
 
@@ -440,7 +440,7 @@ void mainWrapped(int argc, char * * argv)
 
         execvp(shell.c_str(), argPtrs.data());
 
-        throw SysError("executing shell ‘%s’", shell);
+        throw SysError("executing shell '%s'", shell);
     }
 
     else {
@@ -457,7 +457,7 @@ void mainWrapped(int argc, char * * argv)
 
             auto outputName = drvInfo.queryOutputName();
             if (outputName == "")
-                throw Error("derivation ‘%s’ lacks an ‘outputName’ attribute", drvPath);
+                throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
 
             pathsToBuild.insert(drvPath + "!" + outputName);
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index f2742bc3bbd..370f216abcc 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -35,7 +35,7 @@ static void writeChannels()
 {
     auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)};
     if (!channelsFD)
-        throw SysError(format("opening ‘%1%’ for writing") % channelsList);
+        throw SysError(format("opening '%1%' for writing") % channelsList);
     for (const auto & channel : channels)
         writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n");
 }
@@ -44,9 +44,9 @@ static void writeChannels()
 static void addChannel(const string & url, const string & name)
 {
     if (!regex_search(url, std::regex("^(file|http|https)://")))
-        throw Error(format("invalid channel URL ‘%1%’") % url);
+        throw Error(format("invalid channel URL '%1%'") % url);
     if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$")))
-        throw Error(format("invalid channel identifier ‘%1%’") % name);
+        throw Error(format("invalid channel identifier '%1%'") % name);
     readChannels();
     channels[name] = url;
     writeChannels();
@@ -218,7 +218,7 @@ int main(int argc, char ** argv)
                 throw UsageError("no command specified");
             case cAdd:
                 if (args.size() < 1 || args.size() > 2)
-                    throw UsageError("‘--add’ requires one or two arguments");
+                    throw UsageError("'--add' requires one or two arguments");
                 {
                 auto url = args[0];
                 auto name = string{};
@@ -234,12 +234,12 @@ int main(int argc, char ** argv)
                 break;
             case cRemove:
                 if (args.size() != 1)
-                    throw UsageError("‘--remove’ requires one argument");
+                    throw UsageError("'--remove' requires one argument");
                 removeChannel(args[0]);
                 break;
             case cList:
                 if (!args.empty())
-                    throw UsageError("‘--list’ expects no arguments");
+                    throw UsageError("'--list' expects no arguments");
                 readChannels();
                 for (const auto & channel : channels)
                     std::cout << channel.first << ' ' << channel.second << '\n';
@@ -249,7 +249,7 @@ int main(int argc, char ** argv)
                 break;
             case cRollback:
                 if (args.size() > 1)
-                    throw UsageError("‘--rollback’ has at most one argument");
+                    throw UsageError("'--rollback' has at most one argument");
                 auto envArgs = Strings{"--profile", profile};
                 if (args.size() == 1) {
                     envArgs.push_back("--switch-generation");
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 0c69bd41356..861fc2e5cd6 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -23,7 +23,7 @@ int main(int argc, char ** argv)
                 printVersion("nix-copy-closure");
             else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
                 if (*arg != "--gzip")
-                    printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
+                    printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg);
                 gzip = true;
             } else if (*arg == "--from")
                 toMode = false;
@@ -32,7 +32,7 @@ int main(int argc, char ** argv)
             else if (*arg == "--include-outputs")
                 includeOutputs = true;
             else if (*arg == "--show-progress")
-                printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
+                printMsg(lvlError, "Warning: '--show-progress' is not implemented");
             else if (*arg == "--dry-run")
                 dryRun = true;
             else if (*arg == "--use-substitutes" || *arg == "-s")
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index b029b92db15..2b223c88264 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -682,7 +682,7 @@ static void processConnection(bool trusted)
         /* Prevent users from doing something very dangerous. */
         if (geteuid() == 0 &&
             querySetting("build-users-group", "") == "")
-            throw Error("if you run ‘nix-daemon’ as root, then you MUST set ‘build-users-group’!");
+            throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
 #endif
 
         /* Open the store. */
@@ -866,7 +866,7 @@ static void daemonLoop(char * * argv)
         struct sockaddr_un addr;
         addr.sun_family = AF_UNIX;
         if (socketPathRel.size() >= sizeof(addr.sun_path))
-            throw Error(format("socket path ‘%1%’ is too long") % socketPathRel);
+            throw Error(format("socket path '%1%' is too long") % socketPathRel);
         strcpy(addr.sun_path, socketPathRel.c_str());
 
         unlink(socketPath.c_str());
@@ -878,13 +878,13 @@ static void daemonLoop(char * * argv)
         int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr));
         umask(oldMode);
         if (res == -1)
-            throw SysError(format("cannot bind to socket ‘%1%’") % socketPath);
+            throw SysError(format("cannot bind to socket '%1%'") % socketPath);
 
         if (chdir("/") == -1) /* back to the root */
             throw SysError("cannot change current directory");
 
         if (listen(fdSocket.get(), 5) == -1)
-            throw SysError(format("cannot listen on socket ‘%1%’") % socketPath);
+            throw SysError(format("cannot listen on socket '%1%'") % socketPath);
     }
 
     closeOnExec(fdSocket.get());
@@ -923,7 +923,7 @@ static void daemonLoop(char * * argv)
                 trusted = true;
 
             if (!trusted && !matchUser(user, group, allowedUsers))
-                throw Error(format("user ‘%1%’ is not allowed to connect to the Nix daemon") % user);
+                throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
 
             printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""))
                 % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>")
@@ -998,7 +998,7 @@ int main(int argc, char * * argv)
 
                 auto socketDir = dirOf(socketPath);
                 if (chdir(socketDir.c_str()) == -1)
-                    throw SysError(format("changing to socket directory ‘%1%’") % socketDir);
+                    throw SysError(format("changing to socket directory '%1%'") % socketDir);
 
                 auto socketName = baseNameOf(socketPath);
                 auto addr = sockaddr_un{};
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 8620cd25574..94fbc09f664 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -70,7 +70,7 @@ static string needArg(Strings::iterator & i,
     Strings & args, const string & arg)
 {
     if (i == args.end()) throw UsageError(
-        format("‘%1%’ requires an argument") % arg);
+        format("'%1%' requires an argument") % arg);
     return *i++;
 }
 
@@ -124,7 +124,7 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             if (attrs.find(attrName) != attrs.end()) {
-                printError(format("warning: name collision in input Nix expressions, skipping ‘%1%’") % path2);
+                printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2);
                 continue;
             }
             attrs.insert(attrName);
@@ -133,7 +133,7 @@ static void getAllExprs(EvalState & state,
             Value & vArg(*state.allocValue());
             mkString(vArg, path2);
             if (v.attrs->size() == v.attrs->capacity())
-                throw Error(format("too many Nix expressions in directory ‘%1%’") % path);
+                throw Error(format("too many Nix expressions in directory '%1%'") % path);
             mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg);
         }
         else if (S_ISDIR(st.st_mode))
@@ -148,7 +148,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
 {
     struct stat st;
     if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting information about ‘%1%’") % path);
+        throw SysError(format("getting information about '%1%'") % path);
 
     if (isNixExpr(path, st)) {
         state.evalFile(path, v);
@@ -226,7 +226,7 @@ static void checkSelectorUse(DrvNames & selectors)
     /* Check that all selectors have been used. */
     for (auto & i : selectors)
         if (i.hits == 0 && i.fullName != "*")
-            throw Error(format("selector ‘%1%’ matches no derivations") % i.fullName);
+            throw Error(format("selector '%1%' matches no derivations") % i.fullName);
 }
 
 
@@ -297,7 +297,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
             for (auto & j : newest) {
                 if (multiple.find(j.second.first.queryName()) != multiple.end())
                     printInfo(
-                        "warning: there are multiple derivations named ‘%1%’; using the first one",
+                        "warning: there are multiple derivations named '%1%'; using the first one",
                         j.second.first.queryName());
                 matches.push_back(j.second);
             }
@@ -489,13 +489,13 @@ static void installDerivations(Globals & globals,
                 if (!globals.preserveInstalled &&
                     newNames.find(drvName.name) != newNames.end() &&
                     !keep(i))
-                    printInfo("replacing old ‘%s’", i.queryName());
+                    printInfo("replacing old '%s'", i.queryName());
                 else
                     allElems.push_back(i);
             }
 
             for (auto & i : newElems)
-                printInfo("installing ‘%s’", i.queryName());
+                printInfo("installing '%s'", i.queryName());
         }
 
         printMissing(*globals.state, newElems);
@@ -517,7 +517,7 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs)
             globals.preserveInstalled = true;
         else if (arg == "--remove-all" || arg == "-r")
             globals.removeAll = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % arg);
+        else throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
     installDerivations(globals, opArgs, globals.profile);
@@ -597,13 +597,13 @@ static void upgradeDerivations(Globals & globals,
                 {
                     const char * action = compareVersions(drvName.version, bestVersion) <= 0
                         ? "upgrading" : "downgrading";
-                    printInfo("%1% ‘%2%’ to ‘%3%’",
+                    printInfo("%1% '%2%' to '%3%'",
                         action, i.queryName(), bestElem->queryName());
                     newElems.push_back(*bestElem);
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addPrefix(fmt("while trying to find an upgrade for ‘%s’:\n", i.queryName()));
+                e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
                 throw;
             }
         }
@@ -628,7 +628,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs)
         else if (arg == "--leq") upgradeType = utLeq;
         else if (arg == "--eq") upgradeType = utEq;
         else if (arg == "--always") upgradeType = utAlways;
-        else throw UsageError(format("unknown flag ‘%1%’") % arg);
+        else throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
     upgradeDerivations(globals, opArgs, upgradeType);
@@ -647,9 +647,9 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv,
 static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     if (opArgs.size() < 2)
-        throw UsageError("not enough arguments to ‘--set-flag’");
+        throw UsageError("not enough arguments to '--set-flag'");
 
     Strings::iterator arg = opArgs.begin();
     string flagName = *arg++;
@@ -666,7 +666,7 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs)
             DrvName drvName(i.queryName());
             for (auto & j : selectors)
                 if (j.matches(drvName)) {
-                    printInfo("setting flag on ‘%1%’", i.queryName());
+                    printInfo("setting flag on '%1%'", i.queryName());
                     j.hits++;
                     setMetaFlag(*globals.state, i, flagName, flagValue);
                     break;
@@ -690,7 +690,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) {
         string arg = *i++;
         if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
-        else throw UsageError(format("unknown flag ‘%1%’") % arg);
+        else throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
     DrvInfos elems;
@@ -740,7 +740,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
                 if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j))
                     || DrvName(j).matches(drvName))
                 {
-                    printInfo("uninstalling ‘%s’", i.queryName());
+                    printInfo("uninstalling '%s'", i.queryName());
                     found = true;
                     break;
                 }
@@ -758,7 +758,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
 static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     uninstallDerivations(globals, opArgs, globals.profile);
 }
 
@@ -868,7 +868,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                printError("derivation ‘%s’ has invalid meta attribute ‘%s’", i.queryName(), j);
+                printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
                 placeholder.write(nullptr);
             } else {
                 PathSet context;
@@ -918,7 +918,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         else if (arg == "--attr" || arg == "-A")
             attrPath = needArg(i, opFlags, arg);
         else
-            throw UsageError(format("unknown flag ‘%1%’") % arg);
+            throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
 
@@ -965,7 +965,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             try {
                 paths.insert(i.queryOutPath());
             } catch (AssertionError & e) {
-                printMsg(lvlTalkative, "skipping derivation named ‘%s’ which gives an assertion failure", i.queryName());
+                printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName());
                 i.setFailed();
             }
         validPaths = globals.state->store->queryValidPaths(paths);
@@ -991,7 +991,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         try {
             if (i.hasFailed()) continue;
 
-            //Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath);
+            //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath);
 
             if (globals.prebuiltOnly &&
                 validPaths.find(i.queryOutPath()) == validPaths.end() &&
@@ -1112,7 +1112,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
                             if (!v)
-                                printError("derivation ‘%s’ has invalid meta attribute ‘%s’", i.queryName(), j);
+                                printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
                             else {
                                 if (v->type == tString) {
                                     attrs2["type"] = "string";
@@ -1163,9 +1163,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             cout.flush();
 
         } catch (AssertionError & e) {
-            printMsg(lvlTalkative, "skipping derivation named ‘%1%’ which gives an assertion failure", i.queryName());
+            printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addPrefix(fmt("while querying the derivation named ‘%1%’:\n", i.queryName()));
+            e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
             throw;
         }
     }
@@ -1177,7 +1177,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     if (opArgs.size() != 1)
         throw UsageError(format("exactly one argument expected"));
 
@@ -1225,7 +1225,7 @@ static void switchGeneration(Globals & globals, int dstGen)
 static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     if (opArgs.size() != 1)
         throw UsageError(format("exactly one argument expected"));
 
@@ -1240,7 +1240,7 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
 static void opRollback(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     if (opArgs.size() != 0)
         throw UsageError(format("no arguments expected"));
 
@@ -1251,7 +1251,7 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs)
 static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
     if (opArgs.size() != 0)
         throw UsageError(format("no arguments expected"));
 
@@ -1278,7 +1278,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs
 static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
+        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
 
     if (opArgs.size() == 1 && opArgs.front() == "old") {
         deleteOldGenerations(globals.profile, globals.dryRun);
@@ -1289,7 +1289,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         for (auto & i : opArgs) {
             unsigned int n;
             if (!string2Int(i, n))
-                throw UsageError(format("invalid generation number ‘%1%’") % i);
+                throw UsageError(format("invalid generation number '%1%'") % i);
             gens.insert(n);
         }
         deleteGenerations(globals.profile, gens, globals.dryRun);
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index df5105f12c2..7b9a8828117 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -140,7 +140,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         Path lockTokenCur = optimisticLockProfile(profile);
         if (lockToken != lockTokenCur) {
-            printError(format("profile ‘%1%’ changed while we were busy; restarting") % profile);
+            printError(format("profile '%1%' changed while we were busy; restarting") % profile);
             return false;
         }
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 49a14cb6445..2498df0f0d7 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -65,7 +65,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 /* What output do we want? */
                 string outputName = i.queryOutputName();
                 if (outputName == "")
-                    throw Error(format("derivation ‘%1%’ lacks an ‘outputName’ attribute ") % drvPath);
+                    throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath);
 
                 if (gcRoot == "")
                     printGCWarning();
@@ -164,7 +164,7 @@ int main(int argc, char * * argv)
         if (findFile) {
             for (auto & i : files) {
                 Path p = state.findFile(i);
-                if (p == "") throw Error(format("unable to find ‘%1%’") % i);
+                if (p == "") throw Error(format("unable to find '%1%'") % i);
                 std::cout << p << std::endl;
             }
             return;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 47e66eaa651..791a1eeb53c 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -29,11 +29,11 @@ string resolveMirrorUri(EvalState & state, string uri)
 
     auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
     if (mirrorList == vMirrors.attrs->end())
-        throw Error(format("unknown mirror name ‘%1%’") % mirrorName);
+        throw Error(format("unknown mirror name '%1%'") % mirrorName);
     state.forceList(*mirrorList->value);
 
     if (mirrorList->value->listSize() < 1)
-        throw Error(format("mirror URI ‘%1%’ did not expand to anything") % uri);
+        throw Error(format("mirror URI '%1%' did not expand to anything") % uri);
 
     string mirror = state.forceString(*mirrorList->value->listElems()[0]);
     return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
@@ -65,7 +65,7 @@ int main(int argc, char * * argv)
                 string s = getArg(*arg, arg, end);
                 ht = parseHashType(s);
                 if (ht == htUnknown)
-                    throw UsageError(format("unknown hash type ‘%1%’") % s);
+                    throw UsageError(format("unknown hash type '%1%'") % s);
             }
             else if (*arg == "--print-path")
                 printPath = true;
@@ -113,10 +113,10 @@ int main(int argc, char * * argv)
             /* Extract the URI. */
             auto attr = v.attrs->find(state.symbols.create("urls"));
             if (attr == v.attrs->end())
-                throw Error("attribute set does not contain a ‘urls’ attribute");
+                throw Error("attribute set does not contain a 'urls' attribute");
             state.forceList(*attr->value);
             if (attr->value->listSize() < 1)
-                throw Error("‘urls’ list is empty");
+                throw Error("'urls' list is empty");
             uri = state.forceString(*attr->value->listElems()[0]);
 
             /* Extract the hash mode. */
@@ -138,7 +138,7 @@ int main(int argc, char * * argv)
         if (name.empty())
             name = baseNameOf(uri);
         if (name.empty())
-            throw Error(format("cannot figure out file name for ‘%1%’") % uri);
+            throw Error(format("cannot figure out file name for '%1%'") % uri);
 
         /* If an expected hash is given, the file may already exist in
            the store. */
@@ -189,7 +189,7 @@ int main(int argc, char * * argv)
             hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
-                throw Error(format("hash mismatch for ‘%1%’") % uri);
+                throw Error(format("hash mismatch for '%1%'") % uri);
 
             /* Copy the file to the Nix store. FIXME: if RemoteStore
                implemented addToStoreFromDump() and downloadFile()
@@ -201,7 +201,7 @@ int main(int argc, char * * argv)
         }
 
         if (!printPath)
-            printInfo(format("path is ‘%1%’") % storePath);
+            printInfo(format("path is '%1%'") % storePath);
 
         std::cout << printHash16or32(hash) << std::endl;
         if (printPath)
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index 356a8251012..51dedcf0a09 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -77,7 +77,7 @@ void printClosure(const Path & nePath, const StoreExpr & fs)
 
             ClosureElems::const_iterator elem = fs.closure.elems.find(path);
             if (elem == fs.closure.elems.end())
-                throw Error(format("bad closure, missing path ‘%1%’") % path);
+                throw Error(format("bad closure, missing path '%1%'") % path);
 
             for (StringSet::const_iterator i = elem->second.refs.begin();
                  i != elem->second.refs.end(); ++i)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 6cea57a7671..1668fdb2d03 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -51,7 +51,7 @@ static Path useDeriver(Path path)
     if (isDerivation(path)) return path;
     Path drvPath = store->queryPathInfo(path)->deriver;
     if (drvPath == "")
-        throw Error(format("deriver of path ‘%1%’ is not known") % path);
+        throw Error(format("deriver of path '%1%' is not known") % path);
     return drvPath;
 }
 
@@ -76,7 +76,7 @@ static PathSet realisePath(Path path, bool build = true)
         for (auto & j : p.second) {
             DerivationOutputs::iterator i = drv.outputs.find(j);
             if (i == drv.outputs.end())
-                throw Error(format("derivation ‘%1%’ does not have an output named ‘%2%’") % p.first % j);
+                throw Error(format("derivation '%1%' does not have an output named '%2%'") % p.first % j);
             Path outPath = i->second.path;
             if (store2) {
                 if (gcRoot == "")
@@ -95,7 +95,7 @@ static PathSet realisePath(Path path, bool build = true)
 
     else {
         if (build) store->ensurePath(path);
-        else if (!store->isValidPath(path)) throw Error(format("path ‘%1%’ does not exist and cannot be created") % path);
+        else if (!store->isValidPath(path)) throw Error(format("path '%1%' does not exist and cannot be created") % path);
         if (store2) {
             if (gcRoot == "")
                 printGCWarning();
@@ -124,7 +124,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
         else if (i == "--check") buildMode = bmCheck;
         else if (i == "--hash") buildMode = bmHash;
         else if (i == "--ignore-unknown") ignoreUnknown = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     Paths paths;
     for (auto & i : opArgs) {
@@ -181,7 +181,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--recursive") recursive = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (opArgs.empty())
         throw UsageError("first argument must be hash algorithm");
@@ -201,10 +201,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 
     for (auto i : opFlags)
         if (i == "--recursive") recursive = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (opArgs.size() != 3)
-        throw UsageError(format("‘--print-fixed-path’ requires three arguments"));
+        throw UsageError(format("'--print-fixed-path' requires three arguments"));
 
     Strings::iterator i = opArgs.begin();
     HashType hashAlgo = parseHashType(*i++);
@@ -306,9 +306,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--use-output" || i == "-u") useOutput = true;
         else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true;
         else if (i == "--include-outputs") includeOutputs = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
         if (prev != qDefault && prev != query)
-            throw UsageError(format("query type ‘%1%’ conflicts with earlier flag") % i);
+            throw UsageError(format("query type '%1%' conflicts with earlier flag") % i);
     }
 
     if (query == qDefault) query = qOutputs;
@@ -366,7 +366,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 Derivation drv = store->derivationFromPath(path);
                 StringPairs::iterator j = drv.env.find(bindingName);
                 if (j == drv.env.end())
-                    throw Error(format("derivation ‘%1%’ has no environment binding named ‘%2%’")
+                    throw Error(format("derivation '%1%' has no environment binding named '%2%'")
                         % path % bindingName);
                 cout << format("%1%\n") % j->second;
             }
@@ -452,7 +452,7 @@ static string shellEscape(const string & s)
 static void opPrintEnv(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (opArgs.size() != 1) throw UsageError("‘--print-env’ requires one derivation store path");
+    if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path");
 
     Path drvPath = opArgs.front();
     Derivation drv = store->derivationFromPath(drvPath);
@@ -485,7 +485,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         auto log = store->getBuildLog(path);
         if (!log)
-            throw Error("build log of derivation ‘%s’ is not available", path);
+            throw Error("build log of derivation '%s' is not available", path);
         std::cout << *log;
     }
 }
@@ -543,7 +543,7 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         if (i == "--reregister") reregister = true;
         else if (i == "--hash-given") hashGiven = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -557,7 +557,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--print-invalid") printInvalid = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     for (auto & i : opArgs) {
         Path path = store->followLinksToStorePath(i);
@@ -565,7 +565,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
             if (printInvalid)
                 cout << format("%1%\n") % path;
             else
-                throw Error(format("path ‘%1%’ is not valid") % path);
+                throw Error(format("path '%1%' is not valid") % path);
         }
     }
 }
@@ -589,7 +589,7 @@ static void opGC(Strings opFlags, Strings opArgs)
             long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
             options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
         }
-        else throw UsageError(format("bad sub-operation ‘%1%’ in GC") % *i);
+        else throw UsageError(format("bad sub-operation '%1%' in GC") % *i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -620,7 +620,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--ignore-liveness") options.ignoreLiveness = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     for (auto & i : opArgs)
         options.pathsToDelete.insert(store->followLinksToStorePath(i));
@@ -659,7 +659,7 @@ static void opRestore(Strings opFlags, Strings opArgs)
 static void opExport(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag ‘%1%’") % i);
+        throw UsageError(format("unknown flag '%1%'") % i);
 
     for (auto & i : opArgs)
         i = store->followLinksToStorePath(i);
@@ -672,7 +672,7 @@ static void opExport(Strings opFlags, Strings opArgs)
 static void opImport(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag ‘%1%’") % i);
+        throw UsageError(format("unknown flag '%1%'") % i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -707,7 +707,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         if (i == "--check-contents") checkContents = true;
         else if (i == "--repair") repair = Repair;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (store->verifyStore(checkContents, repair)) {
         printError("warning: not all errors were fixed");
@@ -726,14 +726,14 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
 
     for (auto & i : opArgs) {
         Path path = store->followLinksToStorePath(i);
-        printMsg(lvlTalkative, format("checking path ‘%1%’...") % path);
+        printMsg(lvlTalkative, format("checking path '%1%'...") % path);
         auto info = store->queryPathInfo(path);
         HashSink sink(info->narHash.type);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError(
-                format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’")
+                format("path '%1%' was modified! expected hash '%2%', got '%3%'")
                 % path % info->narHash.to_string() % current.first.to_string());
             status = 1;
         }
@@ -772,7 +772,7 @@ static void opServe(Strings opFlags, Strings opArgs)
     bool writeAllowed = false;
     for (auto & i : opFlags)
         if (i == "--write") writeAllowed = true;
-        else throw UsageError(format("unknown flag ‘%1%’") % i);
+        else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -942,7 +942,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag ‘%1%’") % i);
+        throw UsageError(format("unknown flag '%1%'") % i);
 
     if (opArgs.size() != 3) throw UsageError("three arguments expected");
     auto i = opArgs.begin();
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 2405a8cb44e..a35f640d884 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -13,9 +13,9 @@ struct MixCat : virtual Args
     {
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
-            throw Error(format("path ‘%1%’ does not exist") % path);
+            throw Error(format("path '%1%' does not exist") % path);
         if (st.type != FSAccessor::Type::tRegular)
-            throw Error(format("path ‘%1%’ is not a regular file") % path);
+            throw Error(format("path '%1%' is not a regular file") % path);
 
         std::cout << accessor->readFile(path);
     }
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 6b608a708ee..e88c834abc5 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -28,7 +28,7 @@ MultiCommand::MultiCommand(const Commands & _commands)
         assert(!command);
         auto i = commands.find(ss.front());
         if (i == commands.end())
-            throw UsageError(format("‘%1%’ is not a recognised command") % ss.front());
+            throw UsageError(format("'%1%' is not a recognised command") % ss.front());
         command = i->second;
     }});
 }
@@ -55,7 +55,7 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out)
     printTable(out, table);
 
     out << "\n";
-    out << "For full documentation, run ‘man " << programName << "’ or ‘man " << programName << "-<COMMAND>’.\n";
+    out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
 }
 
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
@@ -109,7 +109,7 @@ void StorePathsCommand::run(ref<Store> store)
 
     if (all) {
         if (installables.size())
-            throw UsageError("‘--all’ does not expect arguments");
+            throw UsageError("'--all' does not expect arguments");
         for (auto & p : store->queryAllValidPaths())
             storePaths.push_back(p);
     }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 53680265381..b159408c3a9 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -53,12 +53,12 @@ struct Installable
 
     virtual Buildables toBuildable()
     {
-        throw Error("argument ‘%s’ cannot be built", what());
+        throw Error("argument '%s' cannot be built", what());
     }
 
     virtual Value * toValue(EvalState & state)
     {
-        throw Error("argument ‘%s’ cannot be evaluated", what());
+        throw Error("argument '%s' cannot be evaluated", what());
     }
 };
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index b2165cb8f85..9f1a80d07be 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -46,7 +46,7 @@ struct CmdCopy : StorePathsCommand
     void run(ref<Store> srcStore, Paths storePaths) override
     {
         if (srcUri.empty() && dstUri.empty())
-            throw UsageError("you must pass ‘--from’ and/or ‘--to’");
+            throw UsageError("you must pass '--from' and/or '--to'");
 
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 632b555771c..5cf9c7200d0 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -41,7 +41,7 @@ struct CmdEdit : InstallablesCommand
                 auto dummyArgs = state->allocBindings(0);
                 v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v);
             } catch (Error &) {
-                throw Error("package ‘%s’ has no source location information", i->what());
+                throw Error("package '%s' has no source location information", i->what());
             }
 
             auto pos = state->forceString(*v2);
@@ -49,7 +49,7 @@ struct CmdEdit : InstallablesCommand
 
             auto colon = pos.rfind(':');
             if (colon == std::string::npos)
-                throw Error("cannot parse meta.position attribute ‘%s’", pos);
+                throw Error("cannot parse meta.position attribute '%s'", pos);
 
             std::string filename(pos, 0, colon);
             int lineno = std::stoi(std::string(pos, colon + 1));
@@ -67,7 +67,7 @@ struct CmdEdit : InstallablesCommand
 
             execvp(editor.c_str(), stringsToCharPtrs(args).data());
 
-            throw SysError("cannot run editor ‘%s’", editor);
+            throw SysError("cannot run editor '%s'", editor);
         }
     }
 };
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 98de8897112..923dabb103d 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -109,7 +109,7 @@ static int compatNixHash(int argc, char * * argv)
             string s = getArg(*arg, arg, end);
             ht = parseHashType(s);
             if (ht == htUnknown)
-                throw UsageError(format("unknown hash type ‘%1%’") % s);
+                throw UsageError(format("unknown hash type '%1%'") % s);
         }
         else if (*arg == "--to-base16") op = opTo16;
         else if (*arg == "--to-base32") op = opTo32;
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 59162fdc116..54c9f7841fe 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -221,7 +221,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
             result.push_back(std::make_shared<InstallableAttrPath>(*this, s));
 
         else
-            throw UsageError("don't know what to do with argument ‘%s’", s);
+            throw UsageError("don't know what to do with argument '%s'", s);
     }
 
     return result;
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 62ae6b8facf..0fb45c1454c 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -39,7 +39,7 @@ struct CmdLog : InstallablesCommand
                     break;
                 }
                 if (!found)
-                    throw Error("build log of path ‘%s’ is not available", path);
+                    throw Error("build log of path '%s' is not available", path);
             }
         }
     }
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 417b7b421b1..a08e4c42c67 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -67,7 +67,7 @@ struct MixLs : virtual Args
 
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
-            throw Error(format("path ‘%1%’ does not exist") % path);
+            throw Error(format("path '%1%' does not exist") % path);
         doPath(st, path,
             st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path),
             showDirectory);
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 4b51c5ee14a..7fece5b87b0 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -39,9 +39,9 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         Args::printFlags(out);
         std::cout <<
             "\n"
-            "In addition, most configuration settings can be overriden using ‘--<name> <value>’.\n"
-            "Boolean settings can be overriden using ‘--<name>’ or ‘--no-<name>’. See ‘nix\n"
-            "--help-config’ for a list of configuration settings.\n";
+            "In addition, most configuration settings can be overriden using '--<name> <value>'.\n"
+            "Boolean settings can be overriden using '--<name>' or '--no-<name>'. See 'nix\n"
+            "--help-config' for a list of configuration settings.\n";
     }
 
     void showHelpAndExit()
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 7d5b8f4668a..e5a597addf4 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -397,7 +397,7 @@ bool NixRepl::processLine(string line)
         return false;
 
     else if (command != "")
-        throw Error(format("unknown command ‘%1%’") % command);
+        throw Error(format("unknown command '%1%'") % command);
 
     else {
         size_t p = line.find('=');
@@ -460,7 +460,7 @@ void NixRepl::reloadFiles()
     for (auto & i : old) {
         if (!first) std::cout << std::endl;
         first = false;
-        std::cout << format("Loading ‘%1%’...") % i << std::endl;
+        std::cout << format("Loading '%1%'...") % i << std::endl;
         loadFile(i);
     }
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 49fe4e7a0c6..f67fd637c4f 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -57,15 +57,15 @@ struct CmdRun : InstallablesCommand
                 createDirs(tmpDir + store->storeDir);
 
                 if (mount(store2->realStoreDir.c_str(), (tmpDir + store->storeDir).c_str(), "", MS_BIND, 0) == -1)
-                    throw SysError(format("mounting ‘%s’ on ‘%s’") % store2->realStoreDir % store->storeDir);
+                    throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
 
                 for (auto entry : readDirectory("/")) {
                     Path dst = tmpDir + "/" + entry.name;
                     if (pathExists(dst)) continue;
                     if (mkdir(dst.c_str(), 0700) == -1)
-                        throw SysError(format("creating directory ‘%s’") % dst);
+                        throw SysError(format("creating directory '%s'") % dst);
                     if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                        throw SysError(format("mounting ‘%s’ on ‘%s’") %  ("/" + entry.name) % dst);
+                        throw SysError(format("mounting '%s' on '%s'") %  ("/" + entry.name) % dst);
                 }
 
                 char * cwd = getcwd(0, 0);
@@ -73,19 +73,19 @@ struct CmdRun : InstallablesCommand
                 Finally freeCwd([&]() { free(cwd); });
 
                 if (chroot(tmpDir.c_str()) == -1)
-                    throw SysError(format("chrooting into ‘%s’") % tmpDir);
+                    throw SysError(format("chrooting into '%s'") % tmpDir);
 
                 if (chdir(cwd) == -1)
-                    throw SysError(format("chdir to ‘%s’ in chroot") % cwd);
+                    throw SysError(format("chdir to '%s' in chroot") % cwd);
             } else
                 if (mount(store2->realStoreDir.c_str(), store->storeDir.c_str(), "", MS_BIND, 0) == -1)
-                    throw SysError(format("mounting ‘%s’ on ‘%s’") % store2->realStoreDir % store->storeDir);
+                    throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
 
             writeFile("/proc/self/setgroups", "deny");
             writeFile("/proc/self/uid_map", (format("%d %d %d") % uid % uid % 1).str());
             writeFile("/proc/self/gid_map", (format("%d %d %d") % gid % gid % 1).str());
 #else
-            throw Error(format("mounting the Nix store on ‘%s’ is not supported on this platform") % store->storeDir);
+            throw Error(format("mounting the Nix store on '%s' is not supported on this platform") % store->storeDir);
 #endif
         }
 
@@ -97,7 +97,7 @@ struct CmdRun : InstallablesCommand
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
         if (execlp("bash", "bash", nullptr) == -1)
-            throw SysError("unable to exec ‘bash’");
+            throw SysError("unable to exec 'bash'");
     }
 };
 
diff --git a/src/nix/search.cc b/src/nix/search.cc
index d3e018876a0..5bb376f8f2d 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -76,7 +76,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
         std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
 
         doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
-            debug("at attribute ‘%s’", attrPath);
+            debug("at attribute '%s'", attrPath);
 
             try {
 
@@ -152,7 +152,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                         auto attrs = v->attrs;
                         Bindings::iterator j = attrs->find(sRecurse);
                         if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
-                            debug("skip attribute ‘%s’", attrPath);
+                            debug("skip attribute '%s'", attrPath);
                             return;
                         }
                     }
@@ -175,7 +175,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             } catch (AssertionError & e) {
             } catch (Error & e) {
                 if (!toplevel) {
-                    e.addPrefix(fmt("While evaluating the attribute ‘%s’:\n", attrPath));
+                    e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
                     throw;
                 }
             }
@@ -203,7 +203,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             doExpr(getSourceExpr(*state), "", true, cache.get());
 
             if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
-                throw SysError("cannot rename ‘%s’ to ‘%s’", tmpFile, jsonCacheFileName);
+                throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
     }
 };
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 0e3eb9b694b..992ff742835 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -35,7 +35,7 @@ struct CmdCopySigs : StorePathsCommand
     void run(ref<Store> store, Paths storePaths) override
     {
         if (substituterUris.empty())
-            throw UsageError("you must specify at least one substituter using ‘-s’");
+            throw UsageError("you must specify at least one substituter using '-s'");
 
         // FIXME: factor out commonality with MixVerify.
         std::vector<ref<Store>> substituters;
@@ -50,7 +50,7 @@ struct CmdCopySigs : StorePathsCommand
         //logger->setExpected(doneLabel, storePaths.size());
 
         auto doPath = [&](const Path & storePath) {
-            //Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
+            //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath);
 
             checkInterrupt();
 
@@ -117,7 +117,7 @@ struct CmdSignPaths : StorePathsCommand
     void run(ref<Store> store, Paths storePaths) override
     {
         if (secretKeyFile.empty())
-            throw UsageError("you must specify a secret key file using ‘-k’");
+            throw UsageError("you must specify a secret key file using '-k'");
 
         SecretKey secretKey(readFile(secretKeyFile));
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 973f60a74ff..0d9739314b5 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -78,7 +78,7 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                //Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
+                //Activity act(*logger, lvlInfo, format("checking '%s'") % storePath);
 
                 auto info = store->queryPathInfo(storePath);
 
@@ -93,7 +93,7 @@ struct CmdVerify : StorePathsCommand
                         //logger->incProgress(corruptedLabel);
                         corrupted = 1;
                         printError(
-                            format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
+                            format("path '%s' was modified! expected hash '%s', got '%s'")
                             % info->path % info->narHash.to_string() % hash.first.to_string());
                     }
 
@@ -144,7 +144,7 @@ struct CmdVerify : StorePathsCommand
                     if (!good) {
                         //logger->incProgress(untrustedLabel);
                         untrusted++;
-                        printError(format("path ‘%s’ is untrusted") % info->path);
+                        printError(format("path '%s' is untrusted") % info->path);
                     }
 
                 }
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 63e557ec5f1..e02cfc03ec9 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -32,25 +32,25 @@ std::set<std::string> runResolver(const Path & filename)
 {
     AutoCloseFD fd = open(filename.c_str(), O_RDONLY);
     if (!fd)
-        throw SysError("opening ‘%s’", filename);
+        throw SysError("opening '%s'", filename);
 
     struct stat st;
     if (fstat(fd.get(), &st))
-        throw SysError("statting ‘%s’", filename);
+        throw SysError("statting '%s'", filename);
 
     if (!S_ISREG(st.st_mode)) {
-        printError("file ‘%s’ is not a regular file", filename);
+        printError("file '%s' is not a regular file", filename);
         return {};
     }
 
     if (st.st_size < sizeof(mach_header_64)) {
-        printError("file ‘%s’ is too short for a MACH binary", filename);
+        printError("file '%s' is too short for a MACH binary", filename);
         return {};
     }
 
     char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0);
     if (!obj)
-        throw SysError("mmapping ‘%s’", filename);
+        throw SysError("mmapping '%s'", filename);
 
     ptrdiff_t mach64_offset = 0;
 
@@ -66,13 +66,13 @@ std::set<std::string> runResolver(const Path & filename)
             }
         }
         if (mach64_offset == 0) {
-            printError(format("Could not find any mach64 blobs in file ‘%1%’, continuing...") % filename);
+            printError(format("Could not find any mach64 blobs in file '%1%', continuing...") % filename);
             return {};
         }
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
-        printError(format("Object file has unknown magic number ‘%1%’, skipping it...") % magic);
+        printError(format("Object file has unknown magic number '%1%', skipping it...") % magic);
         return {};
     }
 
@@ -101,7 +101,7 @@ bool isSymlink(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st) == -1)
-        throw SysError("getting attributes of path ‘%1%’", path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     return S_ISLNK(st.st_mode);
 }
diff --git a/tests/timeout.nix b/tests/timeout.nix
index e18d717eff1..d0e949e3149 100644
--- a/tests/timeout.nix
+++ b/tests/timeout.nix
@@ -6,7 +6,7 @@ with import ./config.nix;
     name = "timeout";
     buildCommand = ''
       touch $out
-      echo "‘timeout’ builder entering an infinite loop"
+      echo "'timeout' builder entering an infinite loop"
       while true ; do echo -n .; done
     '';
   };
diff --git a/tests/timeout.sh b/tests/timeout.sh
index 77b227e89ba..2b864b86dca 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -5,7 +5,7 @@ source common.sh
 failed=0
 messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`"
 if [ $failed -ne 0 ]; then
-    echo "error: ‘nix-store’ succeeded; should have timed out"
+    echo "error: 'nix-store' succeeded; should have timed out"
     exit 1
 fi
 

From b39cc4fc813adddf2c9662b2c6e9d4f5a24c0f60 Mon Sep 17 00:00:00 2001
From: Brian McKenna <brian@brianmckenna.org>
Date: Thu, 3 Aug 2017 07:03:22 +1000
Subject: [PATCH 0478/2196] Include missing <cstdlib> for abort()

This is needed to get Nix compiled using Android NDK.
---
 src/libutil/sync.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
index 611c900e0a3..3b2710f6fd3 100644
--- a/src/libutil/sync.hh
+++ b/src/libutil/sync.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdlib>
 #include <mutex>
 #include <condition_variable>
 #include <cassert>

From c6184dec6c208d39a329586d0503b7a51bc2ded1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 31 Jul 2017 15:15:49 +0200
Subject: [PATCH 0479/2196] nix repl: Support printing floating-point numbers

---
 src/nix/repl.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index e5a597addf4..781b4463e54 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -652,6 +652,10 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         str << ESC_BLU "«primop-app»" ESC_END;
         break;
 
+    case tFloat:
+        str << v.fpoint;
+        break;
+
     default:
         str << ESC_RED "«unknown»" ESC_END;
         break;

From af765a8eab288eb638100e027b97a1d15e4e3026 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Aug 2017 16:22:05 +0200
Subject: [PATCH 0480/2196] Use /proc/self/fd to efficiently close all FDs on
 Linux

Issue #1506.
---
 src/libstore/build.cc |  2 +-
 src/libutil/util.cc   | 18 ++++++++++++++++--
 src/libutil/util.hh   |  4 ++--
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index db5f606fa88..9250a9b1778 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2556,7 +2556,7 @@ void DerivationGoal::runChild()
             throw SysError(format("changing into '%1%'") % tmpDir);
 
         /* Close all other file descriptors. */
-        closeMostFDs(set<int>());
+        closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
 
 #if __linux__
         /* Change the personality to 32-bit if we're doing an
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 27f4fea187d..55d3e1d16f6 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -310,6 +310,7 @@ string readLine(int fd)
     while (1) {
         checkInterrupt();
         char ch;
+        // FIXME: inefficient
         ssize_t rd = read(fd, &ch, 1);
         if (rd == -1) {
             if (errno != EINTR)
@@ -962,11 +963,24 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 
 void closeMostFDs(const set<int> & exceptions)
 {
+#if __linux__
+    try {
+        for (auto & s : readDirectory("/proc/self/fd")) {
+            auto fd = std::stoi(s.name);
+            if (!exceptions.count(fd)) {
+                debug("closing leaked FD %d", fd);
+                close(fd);
+            }
+        }
+        return;
+    } catch (SysError &) {
+    }
+#endif
+
     int maxFD = 0;
     maxFD = sysconf(_SC_OPEN_MAX);
     for (int fd = 0; fd < maxFD; ++fd)
-        if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
-            && exceptions.find(fd) == exceptions.end())
+        if (!exceptions.count(fd))
             close(fd); /* ignore result */
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 7ea32e8d9f1..f37f2c5d1be 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -261,8 +261,8 @@ public:
    list of strings. */
 std::vector<char *> stringsToCharPtrs(const Strings & ss);
 
-/* Close all file descriptors except stdin, stdout, stderr, and those
-   listed in the given set.  Good practice in child processes. */
+/* Close all file descriptors except those listed in the given set.
+   Good practice in child processes. */
 void closeMostFDs(const set<int> & exceptions);
 
 /* Set the close-on-exec flag for the given file descriptor. */

From b6ee5e5bf0a680d7f1e3df90b711f77a603c82c3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Aug 2017 13:51:07 +0200
Subject: [PATCH 0481/2196] Style fix

---
 src/libstore/optimise-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 462f8e72a8d..b736307b31b 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -104,7 +104,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
        https://github.com/NixOS/nix/issues/1443 for more discussion. */
 
     if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
-        std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$"))) {
+        std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$")))
+    {
         debug(format("'%1%' is not allowed to be linked in macOS") % path);
         return;
     }

From f76e85d8f581cc8f71b66386e86ed93c2c3d6992 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Aug 2017 18:44:45 +0200
Subject: [PATCH 0482/2196] Start of 1.12 release notes

---
 doc/manual/release-notes/rl-1.12.xml | 362 ++++++++++++++++++++++++++-
 1 file changed, 360 insertions(+), 2 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index b7f45fc44a3..6aab0ab906e 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -10,6 +10,91 @@
 
 <itemizedlist>
 
+  <listitem>
+    <para>Start of new <command>nix</command> command line
+    interface. This is a work in progress and the interface is subject
+    to change.</para>
+
+    <itemizedlist>
+
+      <listitem><para>Self-documenting: <option>--help</option> shows
+      all available command-line arguments.</para></listitem>
+
+      <listitem><para><option>--help-config</option> shows all
+      configuration options.</para></listitem>
+
+      <listitem><para><command>nix build</command>: Replacement for
+      <command>nix-build</command>.</para></listitem>
+
+      <listitem><para><command>nix ls-store</command> and <command>nix
+      ls-nar</command> allow listing the contents of a store path or
+      NAR file.</para></listitem>
+
+      <listitem><para><command>nix cat-store</command> and
+      <command>nix cat-nar</command> allow extracting a file from a
+      store path or NAR file.</para></listitem>
+
+      <listitem><para><command>nix verify</command> checks whether a
+      store path is unmodified and/or is trusted.</para></listitem>
+
+      <listitem><para><command>nix copy-sigs</command> copies
+      signatures from one store to another.</para></listitem>
+
+      <listitem><para><command>nix sign-paths</command> signs store
+      paths.</para></listitem>
+
+      <listitem><para><command>nix copy</command> copies paths between
+      arbitrary Nix stores, generalising
+      <command>nix-copy-closure</command> and
+      <command>nix-push</command>.</para></listitem>
+
+      <listitem><para><command>nix path-info</command> shows
+      information about store paths.</para></listitem>
+
+      <listitem><para><command>nix run</command> starts a shell in
+      which the specified packages are available. (TODO: currently
+      broken in chroot mode)</para></listitem>
+
+      <listitem><para><command>nix log</command> shows the build log
+      of a package or path. If the build log is not available locally,
+      it will try to obtain it from a binary cache.</para></listitem>
+
+      <listitem><para><command>nix eval</command> replaces
+      <command>nix-instantiate --eval</command>.</para></listitem>
+
+      <listitem><para><command>nix dump-path</command> to get a NAR
+      from a store path.</para></listitem>
+
+      <listitem><para><command>nix edit</command> opens the source
+      code of a package in an editor.</para></listitem>
+
+      <listitem><para><command>nix search</command> replaces
+      <command>nix-env -qa</command>. It searches the available
+      packages for occurences of a search string in the attribute
+      name, package name or description. It caches available packages
+      to speed up searches.</para></listitem>
+
+    </itemizedlist>
+
+  </listitem>
+
+  <listitem>
+    <para>The external program <command>nix-repl</command> has been
+    integrated into Nix as <command>nix repl</command>.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now supports floating-point numbers.</para>
+  </listitem>
+
+  <listitem>
+    <para>New build mode <command>nix-build --hash</command> that
+    builds a derivation, computes the hash of the output, and moves
+    the output to the store path corresponding to what a fixed-output
+    derivation with that hash would produce.
+    (Add docs and examples; see d367b8e7875161e655deaa96bf8a5dd0bcf8229e)</para>
+  </listitem>
+
   <listitem>
     <para>It is no longer necessary to set the
     <envar>NIX_REMOTE</envar> environment variable if you need to use
@@ -21,9 +106,282 @@
     <para>The Nix language now supports floating point numbers. They are
     based on regular C++ <literal>float</literal> and compatible with
     existing integers and number-related operations. Export and import to and
-    from JSON and XML works, too.
-  </para>
+    from JSON and XML works, too.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-shell</command> now sets the
+    <varname>IN_NIX_SHELL</varname> environment variable during
+    evaluation and in the shell itself. This can be used to perform
+    different actions depending on whether you’re in a Nix shell or in
+    a regular build. Nixpkgs provides
+    <varname>lib.inNixShell</varname> to check this variable during
+    evaluation. (bb36a1a3cf3fbe6bc9d0afcc5fa0f928bed03170)</para>
+  </listitem>
+
+  <listitem>
+    <para>Internal: all <classname>Store</classname> classes are now
+    thread-safe. <classname>RemoteStore</classname> supports multiple
+    concurrent connections to the daemon. This is primarily useful in
+    multi-threaded programs such as
+    <command>hydra-queue-runner</command>.</para>
+  </listitem>
+
+  <listitem>
+    <para>The dependency on Perl has been removed. As a result, some
+    (obsolete) programs have been removed: <command>nix-push</command>
+    (replaced by <command>nix copy</command>),
+    <command>nix-pull</command> (obsoleted by binary caches),
+    <command>nix-generate-patches</command>,
+    <command>bsdiff</command>, <command>bspatch</command>.</para>
+  </listitem>
+
+  <listitem>
+    <para>Improved store abstraction. Substituters
+    eliminated. BinaryCacheStore, LocalBinaryCacheStore,
+    HttpBinaryCacheStore, S3BinaryCacheStore (compile-time
+    optional), SSHStore. Add docs + examples?
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now stores signatures for local store
+    paths. Locally-built paths are now signed automatically using the
+    secret keys specified by the <option>secret-key-files</option>
+    store option.</para>
+
+    <para>In addition, store paths that have been built locally are
+    marked as “ultimately trusted”, and content-addressable store
+    paths carry a “content-addressability assertion” that allow them
+    to be trusted without any signatures.</para>
+  </listitem>
+
+  <listitem>
+    <para><envar>NIX_PATH</envar> is now lazy, so URIs in the path are
+    only downloaded if they are needed for evaluation.</para>
+  </listitem>
+
+  <listitem>
+    <para>You can now use
+    <uri>channel:<replaceable>channel-name</replaceable></uri> as a
+    short-hand for
+    <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For
+    example, <literal>nix-build channel:nixos-15.09 -A hello</literal>
+    will build the GNU Hello package from the
+    <literal>nixos-15.09</literal> channel.</para>
+  </listitem>
+
+  <listitem>
+    <para>“Nested” log output was removed. As a result,
+    <command>nix-log2xml</command> was also removed.</para>
+  </listitem>
+
+  <listitem>
+    <para>When <option>--no-build-output</option> is given, the last
+    10 lines of the build log will be shown if a build
+    fails.</para>
+  </listitem>
+
+  <listitem>
+    <para><function>builtins.fetchgit</function>.
+    (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para>
+  </listitem>
+
+  <listitem>
+    <para>Git repositories can now be specified in the Nix search
+    path,
+    e.g. <literal>nixpkgs=git://github.com/NixOS/nixpkgs</literal>.
+    (d8bf0d4859e28ddd23401fbe89f4e528aa09ddb3)</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para>
+  </listitem>
+
+  <listitem>
+    <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
+    content-addressable tarball cache at
+    <uri>http://tarballs.nixos.org/</uri>, just like
+    <function>fetchurl</function> in
+    Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para>
+  </listitem>
+
+  <listitem>
+    <para>Chroot Nix stores: allow the “physical” location of the Nix
+    store (e.g. <filename>/home/alice/nix/store</filename>) to differ
+    from its “logical” location (typically
+    <filename>/nix/store</filename>). This allows non-root users to
+    use Nix while still getting the benefits from prebuilt binaries
+    from
+    <uri>cache.nixos.org</uri>. (4494000e04122f24558e1436e66d20d89028b4bd,
+    3eb621750848e0e6b30e5a79f76afbb096bb6c8a)</para>
+  </listitem>
+
+  <listitem>
+    <para>On Linux, builds are now executed in a user
+    namespace with uid 1000 and gid 100.</para>
+  </listitem>
+
+  <listitem>
+    <para><function>builtins.fetchurl</function> and
+    <function>builtins.fetchTarball</function> now support
+    <varname>sha256</varname> and <varname>name</varname>
+    attributes. Using these functions is now allowed in restricted
+    mode if a hash is supplied.</para>
+  </listitem>
+
+  <listitem>
+    <para><literal>HttpBinaryCacheStore</literal> (the replacement of
+    <command>download-from-binary-cache</command>) now retries
+    automatically on certain HTTP error codes.</para>
+  </listitem>
+
+  <listitem>
+    <para>Derivation attributes can now reference the outputs of the
+    derivation using the <function>placeholder</function> builtin
+    function. For example, the attribute
+
+<programlisting>
+configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}";
+</programlisting>
+
+    will cause the <envar>configureFlags</envar> environment variable
+    to contain the actual store paths corresponding to the
+    <literal>out</literal> and <literal>dev</literal> outputs. TODO:
+    add docs.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added builtin function
+    <varname>builtins.partition</varname>.</para>
+  </listitem>
+
+  <listitem>
+    <para>Support for HTTP/2. This makes binary cache lookups much
+    more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para>
+  </listitem>
+
+  <listitem>
+    <para>Caching of failed builds has been
+    removed. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para>
+  </listitem>
+
+  <listitem>
+    <para>The <option>build-sandbox-paths</option> configuration
+    option can now specify optional paths by appending a
+    <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will
+    bind-mount <varname>/dev/nvidiactl</varname> only if it
+    exists.</para>
+  </listitem>
+
+  <listitem>
+    <para>More support for testing build reproducibility: when
+    <option>enforce-determinism</option> is set to
+    <literal>false</literal>, it’s no longer a fatal error build
+    rounds produce different output
+    (8bdf83f936adae6f2c907a6d2541e80d4120f051); add a hook to run
+    diffoscope when build rounds produce different output
+    (9a313469a4bdea2d1e8df24d16289dc2a172a169w).</para>
+  </listitem>
+
+  <listitem>
+    <para>Kill builds as soon as stdout/stderr is closed. This fixes a
+    bug that allowed builds to hang Nix indefinitely (regardless of
+    timeouts). (21948deed99a3295e4d5666e027a6ca42dc00b40)</para>
+  </listitem>
+
+  <listitem>
+    <para>Add support for passing structured data to builders. TODO:
+    document. (6de33a9c675b187437a2e1abbcb290981a89ecb1)</para>
+  </listitem>
+
+  <listitem>
+    <para><varname>exportReferencesGraph</varname>: Export more
+    complete info in JSON
+    format. (c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a)</para>
+  </listitem>
+
+  <listitem>
+    <para>Support for
+    netrc. (e6e74f987f0fa284d220432d426eb965269a97d6,
+    302386f775eea309679654e5ea7c972fb6e7b9af)</para>
+  </listitem>
+
+  <listitem>
+    <para>Support <uri>s3://</uri> URIs in all places where Nix allows
+    URIs. (9ff9c3f2f80ba4108e9c945bbfda2c64735f987b)</para>
+  </listitem>
+
+  <listitem>
+    <para>The <option>build-max-jobs</option> option can be set to
+    <literal>auto</literal> to use the number of CPUs in the
+    system. (7251d048fa812d2551b7003bc9f13a8f5d4c95a5)</para>
+  </listitem>
+
+  <listitem>
+    <para>Add support for Brotli compression.
+    <uri>cache.nixos.org</uri> compresses build logs using
+    Brotli.</para>
+  </listitem>
+
+  <listitem>
+    <para>Substitutions from binary caches now require signatures by
+    default. This was already the case on
+    NixOS. (ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b)</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-env</command> now ignores packages with bad
+    derivation names (in particular those starting with a digit or
+    containing a
+    dot). (b0cb11722626e906a73f10dd9a0c9eea29faf43a)</para>
+  </listitem>
+
+  <listitem>
+    <para>Renamed various configuration options. (TODO: in progress)</para>
   </listitem>
+
+  <listitem>
+    <para>Remote machines can now be specified on the command
+    line. TODO:
+    document. (1a68710d4dff609bbaf61db3e17a2573f0aadf17)</para>
+  </listitem>
+
+  <listitem>
+    <para>In Linux sandbox builds, we now use
+    <filename>/build</filename> instead of <filename>/tmp</filename>
+    as the temporary build directory. This fixes potential security
+    problems when a build accidentally stores its
+    <envar>TMPDIR</envar> in some critical place, such as an
+    RPATH. (eba840c8a13b465ace90172ff76a0db2899ab11b)</para>
+  </listitem>
+
+  <listitem>
+    <para>In Linux sandbox builds, we now provide a default
+    <filename>/bin/sh</filename> (namely <filename>ash</filename> from
+    BusyBox). (a2d92bb20e82a0957067ede60e91fab256948b41)</para>
+  </listitem>
+
+  <listitem>
+    <para>Make all configuration options available as command line
+    flags (b8283773bd64d7da6859ed520ee19867742a03ba).</para>
+  </listitem>
+
+  <listitem>
+    <para>Support base-64
+    hashes. (c0015e87af70f539f24d2aa2bc224a9d8b84276b)</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-shell</command> now used
+    <varname>bashInteractive</varname> from Nixpkgs, rather than the
+    <command>bash</command> command that happens to be in the caller’s
+    <envar>PATH</envar>. This is especially important on macOS where
+    the <command>bash</command> provided by the system is seriously
+    outdated and cannot execute <literal>stdenv</literal>’s setup
+    script.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>

From 27417c616073de0800514e79d58c84eaa8019099 Mon Sep 17 00:00:00 2001
From: "Nicolas B. Pierron" <nicolas.b.pierron@gmail.com>
Date: Tue, 15 Aug 2017 18:34:24 +0000
Subject: [PATCH 0483/2196] Move builtins.match documentation between map and
 mul.

---
 doc/manual/expressions/builtins.xml | 72 ++++++++++++++---------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 229a73fb12e..86c36da1b32 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -210,42 +210,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
-  <varlistentry><term><function>builtins.match</function>
-  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
-
-  <listitem><para>Returns a list if the <link
-  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-  POSIX regular expression</link> <replaceable>regex</replaceable>
-  matches <replaceable>str</replaceable> precisely, otherwise returns
-  <literal>null</literal>.  Each item in the list is a regex group.
-
-<programlisting>
-builtins.match "ab" "abc"
-</programlisting>
-
-Evaluates to <literal>null</literal>.
-
-<programlisting>
-builtins.match "abc" "abc"
-</programlisting>
-
-Evaluates to <literal>[ ]</literal>.
-
-<programlisting>
-builtins.match "a(b)(c)" "abc"
-</programlisting>
-
-Evaluates to <literal>[ "b" "c" ]</literal>.
-
-<programlisting>
-builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
-</programlisting>
-
-Evaluates to <literal>[ "foo" ]</literal>.
-
-  </para></listitem>
-  </varlistentry>
-
   <varlistentry><term><function>builtins.elem</function>
   <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
 
@@ -726,6 +690,42 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.match</function>
+  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+
+  <listitem><para>Returns a list if the <link
+  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+  POSIX regular expression</link> <replaceable>regex</replaceable>
+  matches <replaceable>str</replaceable> precisely, otherwise returns
+  <literal>null</literal>.  Each item in the list is a regex group.
+
+<programlisting>
+builtins.match "ab" "abc"
+</programlisting>
+
+Evaluates to <literal>null</literal>.
+
+<programlisting>
+builtins.match "abc" "abc"
+</programlisting>
+
+Evaluates to <literal>[ ]</literal>.
+
+<programlisting>
+builtins.match "a(b)(c)" "abc"
+</programlisting>
+
+Evaluates to <literal>[ "b" "c" ]</literal>.
+
+<programlisting>
+builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
+</programlisting>
+
+Evaluates to <literal>[ "foo" ]</literal>.
+
+  </para></listitem>
+  </varlistentry>
+
   <varlistentry><term><function>builtins.mul</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 

From b8867a0239b1930a16f9ef3f7f3e864b01416dff Mon Sep 17 00:00:00 2001
From: "Nicolas B. Pierron" <nicolas.b.pierron@gmail.com>
Date: Tue, 15 Aug 2017 18:08:41 +0000
Subject: [PATCH 0484/2196] Add builtins.string function.

The function 'builtins.split' takes a POSIX extended regular expression
and an arbitrary string. It returns a list of non-matching substring
interleaved by lists of matched groups of the regular expression.

```nix
with builtins;
assert split "(a)b" "abc"      == [ "" [ "a" ] "c" ];
assert split "([ac])" "abc"    == [ "" [ "a" ] "b" [ "c" ] "" ];
assert split "(a)|(c)" "abc"   == [ "" [ "a" null ] "b" [ null "c" ] "" ];
assert split "([[:upper:]]+)" "  FOO   "
                               == [ "  " [ "FOO" ] "   " ];
```
---
 doc/manual/expressions/builtins.xml  | 37 +++++++++++++++
 src/libexpr/primops.cc               | 68 ++++++++++++++++++++++++++++
 tests/lang/eval-okay-regex-split.nix | 48 ++++++++++++++++++++
 3 files changed, 153 insertions(+)
 create mode 100644 tests/lang/eval-okay-regex-split.nix

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 229a73fb12e..ac8ecedd0b1 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -873,6 +873,43 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.split</function>
+  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+
+  <listitem><para>Returns a list composed of non matched strings interleaved
+  with the lists of the <link
+  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+  POSIX regular expression</link> <replaceable>regex</replaceable> matches
+  of <replaceable>str</replaceable>. Each item in the lists of matched
+  sequences is a regex group.
+
+<programlisting>
+builtins.split "(a)b" "abc"
+</programlisting>
+
+Evaluates to <literal>[ "" [ "a" ] "c" ]</literal>.
+
+<programlisting>
+builtins.split "([ac])" "abc"
+</programlisting>
+
+Evaluates to <literal>[ "" [ "a" ] "b" [ "c" ] "" ]</literal>.
+
+<programlisting>
+builtins.split "(a)|(c)" "abc"
+</programlisting>
+
+Evaluates to <literal>[ "" [ "a" null ] "b" [ null "c" ] "" ]</literal>.
+
+<programlisting>
+builtins.split "([[:upper:]]+)" "  FOO   "
+</programlisting>
+
+Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
+
+  </para></listitem>
+  </varlistentry>
+
   <varlistentry><term><function>builtins.stringLength</function>
   <replaceable>e</replaceable></term>
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4e51e8ff256..fcd3f8efee3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1745,6 +1745,73 @@ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value
 }
 
 
+/* Split a string with a regular expression, and return a list of the
+   non-matching parts interleaved by the lists of the matching groups. */
+static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    auto re = state.forceStringNoCtx(*args[0], pos);
+
+    try {
+
+        std::regex regex(re, std::regex::extended);
+
+        PathSet context;
+        const std::string str = state.forceString(*args[1], context, pos);
+
+        auto begin = std::sregex_iterator(str.begin(), str.end(), regex);
+        auto end = std::sregex_iterator();
+
+        // Any matches results are surrounded by non-matching results.
+        const size_t len = std::distance(begin, end);
+        state.mkList(v, 2 * len + 1);
+        size_t idx = 0;
+        Value * elem;
+
+        if (len == 0) {
+            v.listElems()[idx++] = args[1];
+            return;
+        }
+
+        for (std::sregex_iterator i = begin; i != end; ++i) {
+            assert(idx <= 2 * len + 1 - 3);
+            std::smatch match = *i;
+
+            // Add a string for non-matched characters.
+            elem = v.listElems()[idx++] = state.allocValue();
+            mkString(*elem, match.prefix().str().c_str());
+
+            // Add a list for matched substrings.
+            const size_t slen = match.size() - 1;
+            elem = v.listElems()[idx++] = state.allocValue();
+
+            // Start at 1, beacause the first match is the whole string.
+            state.mkList(*elem, slen);
+            for (size_t si = 0; si < slen; ++si) {
+                if (!match[si + 1].matched)
+                    mkNull(*(elem->listElems()[si] = state.allocValue()));
+                else
+                    mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str());
+            }
+
+            // Add a string for non-matched suffix characters.
+            if (idx == 2 * len) {
+                elem = v.listElems()[idx++] = state.allocValue();
+                mkString(*elem, match.suffix().str().c_str());
+            }
+        }
+        assert(idx == 2 * len + 1);
+
+    } catch (std::regex_error &e) {
+        if (e.code() == std::regex_constants::error_space) {
+          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+        } else {
+          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+        }
+    }
+}
+
+
 static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
@@ -2039,6 +2106,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
+    addPrimOp("__split", 2, prim_split);
     addPrimOp("__concatStringsSep", 2, prim_concatStringSep);
     addPrimOp("__replaceStrings", 3, prim_replaceStrings);
 
diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/lang/eval-okay-regex-split.nix
new file mode 100644
index 00000000000..0073e057787
--- /dev/null
+++ b/tests/lang/eval-okay-regex-split.nix
@@ -0,0 +1,48 @@
+with builtins;
+
+# Non capturing regex returns empty lists
+assert  split "foobar" "foobar"  == ["" [] ""];
+assert  split "fo*" "f"          == ["" [] ""];
+assert  split "fo+" "f"          == ["f"];
+assert  split "fo*" "fo"         == ["" [] ""];
+assert  split "fo*" "foo"        == ["" [] ""];
+assert  split "fo+" "foo"        == ["" [] ""];
+assert  split "fo{1,2}" "foo"    == ["" [] ""];
+assert  split "fo{1,2}" "fooo"   == ["" [] "o"];
+assert  split "fo*" "foobar"     == ["" [] "bar"];
+
+# Capturing regex returns a list of sub-matches
+assert  split "(fo*)" "f"        == ["" ["f"] ""];
+assert  split "(fo+)" "f"        == ["f"];
+assert  split "(fo*)" "fo"       == ["" ["fo"] ""];
+assert  split "(f)(o*)" "f"      == ["" ["f" ""] ""];
+assert  split "(f)(o*)" "foo"    == ["" ["f" "oo"] ""];
+assert  split "(fo+)" "foo"      == ["" ["foo"] ""];
+assert  split "(fo{1,2})" "foo"  == ["" ["foo"] ""];
+assert  split "(fo{1,2})" "fooo" == ["" ["foo"] "o"];
+assert  split "(fo*)" "foobar"   == ["" ["foo"] "bar"];
+
+# Matches are greedy.
+assert  split "(o+)" "oooofoooo" == ["" ["oooo"] "f" ["oooo"] ""];
+
+# Matches multiple times.
+assert  split "(b)" "foobarbaz"  == ["foo" ["b"] "ar" ["b"] "az"];
+
+# Split large strings containing newlines. null are inserted when a
+# pattern within the current did not match anything.
+assert  split "[[:space:]]+|([',.!?])" ''
+  Nix Rocks!
+  That's why I use it.
+''  == [
+  "Nix" [ null ] "Rocks" ["!"] "" [ null ]
+  "That" ["'"] "s" [ null ] "why" [ null ] "I" [ null ] "use" [ null ] "it" ["."] "" [ null ]
+  ""
+];
+
+# Documentation examples
+assert  split  "(a)b" "abc"      == [ "" [ "a" ] "c" ];
+assert  split  "([ac])" "abc"    == [ "" [ "a" ] "b" [ "c" ] "" ];
+assert  split  "(a)|(c)" "abc"   == [ "" [ "a" null ] "b" [ null "c" ] "" ];
+assert  split  "([[:upper:]]+)" "  FOO   " == [ "  " [ "FOO" ] "   " ];
+
+true

From e4bd42f98f7da0ac49cdea579d1e43be64c0621f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 Aug 2017 13:55:41 +0200
Subject: [PATCH 0485/2196] Disallow SSH host names starting with a dash

---
 src/libstore/ssh.cc | 11 +++++++++++
 src/libstore/ssh.hh |  9 +--------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 776ffdb8343..7ff7a9bffc4 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -2,6 +2,17 @@
 
 namespace nix {
 
+SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
+    : host(host)
+    , keyFile(keyFile)
+    , useMaster(useMaster)
+    , compress(compress)
+    , logFD(logFD)
+{
+    if (host == "" || hasPrefix(host, "-"))
+        throw Error("invalid SSH host name '%s'", host);
+}
+
 void SSHMaster::addCommonSSHOpts(Strings & args)
 {
     for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 18dea227ad1..1268e6d0005 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -28,14 +28,7 @@ private:
 
 public:
 
-    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1)
-        : host(host)
-        , keyFile(keyFile)
-        , useMaster(useMaster)
-        , compress(compress)
-        , logFD(logFD)
-    {
-    }
+    SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
 
     struct Connection
     {

From dffc3fe43bd3cbf95945065ee7822727b9fcea90 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 15:24:04 +0200
Subject: [PATCH 0486/2196] nix copy: Add --no-check-sigs flag

---
 src/nix/copy.cc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 9f1a80d07be..d9e6c949352 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -12,10 +12,17 @@ struct CmdCopy : StorePathsCommand
 {
     std::string srcUri, dstUri;
 
+    CheckSigsFlag checkSigs = CheckSigs;
+
     CmdCopy()
     {
         mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri);
         mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri);
+
+        mkFlag()
+            .longName("no-check-sigs")
+            .description("do not require that paths are signed by trusted keys")
+            .handler([&](Strings ss) { checkSigs = NoCheckSigs; });
     }
 
     std::string name() override
@@ -50,7 +57,8 @@ struct CmdCopy : StorePathsCommand
 
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()));
+        copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()),
+            NoRepair, checkSigs);
     }
 };
 

From c5e4404580164d3edd043e79cf72bac5bdaecb42 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 15:28:16 +0200
Subject: [PATCH 0487/2196] nix copy: Revive progress bar

---
 src/libstore/legacy-ssh-store.cc |  1 -
 src/libstore/store-api.cc        | 26 +++++++++++++++++-
 src/libutil/archive.cc           |  2 ++
 src/libutil/logging.cc           | 11 ++++++++
 src/libutil/logging.hh           | 15 +++++++++++
 src/nix/progress-bar.cc          | 46 ++++++++++++++++++++++++++++++++
 6 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 65fe575d253..855a7c99aed 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -135,7 +135,6 @@ struct LegacySSHStore : public Store
 
         if (readInt(conn->from) != 1)
             throw Error("failed to add path '%s' to remote host '%s', info.path, host");
-
     }
 
     void narFromPath(const Path & path, Sink & sink) override
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index dd87a046e38..88f27e806dc 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -565,8 +565,12 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
+    Activity act(actCopyPath, fmt("copying path '%s'", storePath));
+
     auto info = srcStore->queryPathInfo(storePath);
 
+    //act->progress(0, info->size());
+
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
@@ -600,13 +604,28 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
+    Activity act;
+
+    logger->event(evCopyStarted, act);
+
+    std::atomic<size_t> nrCopied{0};
+    std::atomic<size_t> nrDone{storePaths.size() - missing.size()};
+
+    auto showProgress = [&]() {
+        logger->event(evCopyProgress, act, storePaths.size(), nrCopied, nrDone);
+    };
+
     ThreadPool pool;
 
     processGraph<Path>(pool,
         PathSet(missing.begin(), missing.end()),
 
         [&](const Path & storePath) {
-            if (dstStore->isValidPath(storePath)) return PathSet();
+            if (dstStore->isValidPath(storePath)) {
+                nrDone++;
+                showProgress();
+                return PathSet();
+            }
             return srcStore->queryPathInfo(storePath)->references;
         },
 
@@ -616,7 +635,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             if (!dstStore->isValidPath(storePath)) {
                 printInfo("copying '%s'...", storePath);
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                nrCopied++;
             }
+
+            nrDone++;
+
+            showProgress();
         });
 }
 
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 51b57a8f4e9..ea1deb924e6 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -56,6 +56,8 @@ static void dumpContents(const Path & path, size_t size,
 
 static void dump(const Path & path, Sink & sink, PathFilter & filter)
 {
+    checkInterrupt();
+
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path '%1%'") % path);
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 43245f61c60..c11271e6300 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -80,4 +80,15 @@ std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
 Activity::Activity() : id(nextId++) { };
 
+Activity::Activity(ActivityType type, std::string msg)
+    : Activity()
+{
+    logger->event(evStartActivity, id, msg);
+}
+
+Activity::~Activity()
+{
+    logger->event(evStopActivity, id);
+}
+
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 9ef6e3ee30e..aa407f60bb6 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -13,6 +13,10 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
+typedef enum {
+    actCopyPath = 100,
+} ActivityType;
+
 class Activity
 {
 public:
@@ -21,6 +25,10 @@ public:
     Activity();
     Activity(const Activity & act) : id(act.id) { };
     Activity(uint64_t id) : id(id) { };
+    Activity(ActivityType type, std::string msg = "");
+    ~Activity();
+
+    //void progress(...);
 };
 
 typedef enum {
@@ -35,6 +43,13 @@ typedef enum {
     evSubstitutionCreated = 8,
     evSubstitutionStarted = 9,
     evSubstitutionFinished = 10,
+
+    evCopyStarted = 100,
+    evCopyProgress = 101,
+
+    evStartActivity = 1000,
+    evStopActivity = 1001,
+
 } EventType;
 
 struct Event
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 2ecbea8eeea..7561a00843a 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -27,17 +27,29 @@ class ProgressBar : public Logger
         DownloadInfo(const std::string & uri) : uri(uri) { }
     };
 
+    struct CopyInfo
+    {
+        uint64_t expected = 0;
+        uint64_t copied = 0;
+        uint64_t done = 0;
+    };
+
     struct State
     {
         std::map<Activity::t, Path> builds;
         std::set<Activity::t> runningBuilds;
         uint64_t succeededBuilds = 0;
         uint64_t failedBuilds = 0;
+
         std::map<Activity::t, Path> substitutions;
         std::set<Activity::t> runningSubstitutions;
         uint64_t succeededSubstitutions = 0;
+
         uint64_t downloadedBytes = 0; // finished downloads
         std::map<Activity::t, DownloadInfo> downloads;
+
+        std::map<Activity::t, CopyInfo> runningCopies;
+
         std::list<ActInfo> activities;
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
     };
@@ -167,11 +179,35 @@ class ProgressBar : public Logger
             res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0);
         }
 
+        if (!state.runningCopies.empty()) {
+            if (!res.empty()) res += ", ";
+            uint64_t copied = 0, expected = 0;
+            for (auto & i : state.runningCopies) {
+                copied += i.second.copied;
+                expected += i.second.expected - (i.second.done - i.second.copied);
+            }
+            res += fmt("%d/%d copied", copied, expected);
+        }
+
         return res;
     }
 
     void event(const Event & ev) override
     {
+        if (ev.type == evStartActivity) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            createActivity(*state, act, ev.getS(1));
+            update(*state);
+        }
+
+        if (ev.type == evStopActivity) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
         if (ev.type == evBuildCreated) {
             auto state(state_.lock());
             state->builds[ev.getI(0)] = ev.getS(1);
@@ -280,6 +316,16 @@ class ProgressBar : public Logger
                 update(*state);
             }
         }
+
+        if (ev.type == evCopyProgress) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto & i = state->runningCopies[act];
+            i.expected = ev.getI(1);
+            i.copied = ev.getI(2);
+            i.done = ev.getI(3);
+            update(*state);
+        }
     }
 };
 

From b29b6feaba715432490e275a025b3361a021a99b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 19:00:03 +0200
Subject: [PATCH 0488/2196] nix copy: Improve progress indicator

It now shows the amount of data copied:

  [8/1038 copied, 160.4/1590.9 MiB copied] copying path '...'
---
 src/libstore/store-api.cc | 32 +++++++++++--
 src/libutil/logging.cc    |  2 +-
 src/libutil/logging.hh    | 16 ++++++-
 src/nix/progress-bar.cc   | 99 ++++++++++++++++++++++++++-------------
 4 files changed, 111 insertions(+), 38 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 88f27e806dc..2b99a07e2af 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -569,9 +569,26 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     auto info = srcStore->queryPathInfo(storePath);
 
-    //act->progress(0, info->size());
+    uint64_t total = 0;
 
-    StringSink sink;
+    auto progress = [&](size_t len) {
+        total += len;
+        act.progress(total, info->narSize);
+    };
+
+    struct MyStringSink : StringSink
+    {
+        typedef std::function<void(size_t)> Callback;
+        Callback callback;
+        MyStringSink(Callback callback) : callback(callback) { }
+        void operator () (const unsigned char * data, size_t len) override
+        {
+            StringSink::operator ()(data, len);
+            callback(len);
+        };
+    };
+
+    MyStringSink sink(progress);
     srcStore->narFromPath({storePath}, sink);
 
     if (!info->narHash && !checkSigs) {
@@ -604,12 +621,13 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
-    Activity act;
+    Activity act(actUnknown);
 
     logger->event(evCopyStarted, act);
 
     std::atomic<size_t> nrCopied{0};
     std::atomic<size_t> nrDone{storePaths.size() - missing.size()};
+    std::atomic<uint64_t> bytesExpected{0};
 
     auto showProgress = [&]() {
         logger->event(evCopyProgress, act, storePaths.size(), nrCopied, nrDone);
@@ -626,7 +644,13 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
                 showProgress();
                 return PathSet();
             }
-            return srcStore->queryPathInfo(storePath)->references;
+
+            auto info = srcStore->queryPathInfo(storePath);
+
+            bytesExpected += info->narSize;
+            logger->event(evSetExpected, act, actCopyPath, bytesExpected);
+
+            return info->references;
         },
 
         [&](const Path & storePath) {
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index c11271e6300..321c9071291 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -83,7 +83,7 @@ Activity::Activity() : id(nextId++) { };
 Activity::Activity(ActivityType type, std::string msg)
     : Activity()
 {
-    logger->event(evStartActivity, id, msg);
+    logger->event(evStartActivity, id, type, msg);
 }
 
 Activity::~Activity()
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index aa407f60bb6..6a4a3617156 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -14,6 +14,7 @@ typedef enum {
 } Verbosity;
 
 typedef enum {
+    actUnknown = 0,
     actCopyPath = 100,
 } ActivityType;
 
@@ -28,7 +29,8 @@ public:
     Activity(ActivityType type, std::string msg = "");
     ~Activity();
 
-    //void progress(...);
+    template<typename... Args>
+    void progress(const Args & ... args);
 };
 
 typedef enum {
@@ -49,6 +51,8 @@ typedef enum {
 
     evStartActivity = 1000,
     evStopActivity = 1001,
+    evProgress = 1002,
+    evSetExpected = 1003,
 
 } EventType;
 
@@ -149,4 +153,14 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs);
 
 void writeToStderr(const string & s);
 
+template<typename... Args>
+void Activity::progress(const Args & ... args)
+{
+    Event ev;
+    ev.type = evProgress;
+    ev.fields.emplace_back(id);
+    nop{(ev.fields.emplace_back(Event::Field(args)), 1)...};
+    logger->event(ev);
+}
+
 }
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 7561a00843a..b15f4488dd0 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -8,6 +8,8 @@
 
 #include <sys/ioctl.h>
 
+#include <iostream>
+
 namespace nix {
 
 class ProgressBar : public Logger
@@ -17,6 +19,10 @@ class ProgressBar : public Logger
     struct ActInfo
     {
         std::string s, s2;
+        ActivityType type = actUnknown;
+        uint64_t done = 0;
+        uint64_t expected = 0;
+        std::map<ActivityType, uint64_t> expectedByType;
     };
 
     struct DownloadInfo
@@ -34,6 +40,13 @@ class ProgressBar : public Logger
         uint64_t done = 0;
     };
 
+    struct ActivitiesByType
+    {
+        std::map<Activity::t, std::list<ActInfo>::iterator> its;
+        uint64_t done = 0;
+        uint64_t expected = 0;
+    };
+
     struct State
     {
         std::map<Activity::t, Path> builds;
@@ -52,6 +65,8 @@ class ProgressBar : public Logger
 
         std::list<ActInfo> activities;
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
+
+        std::map<ActivityType, ActivitiesByType> activitiesByType;
     };
 
     Sync<State> state_;
@@ -85,16 +100,25 @@ class ProgressBar : public Logger
         update(state);
     }
 
-    void createActivity(State & state, Activity::t activity, const std::string & s)
+    void createActivity(State & state, Activity::t activity, const std::string & s, ActivityType type = actUnknown)
     {
-        state.activities.emplace_back(ActInfo{s});
-        state.its.emplace(activity, std::prev(state.activities.end()));
+        state.activities.emplace_back(ActInfo{s, "", type});
+        auto i = std::prev(state.activities.end());
+        state.its.emplace(activity, i);
+        state.activitiesByType[type].its.emplace(activity, i);
     }
 
     void deleteActivity(State & state, Activity::t activity)
     {
         auto i = state.its.find(activity);
         if (i != state.its.end()) {
+            auto & act = state.activitiesByType[i->second->type];
+            act.done += i->second->done;
+
+            for (auto & j : i->second->expectedByType)
+                state.activitiesByType[j.first].expected -= j.second;
+
+            act.its.erase(activity);
             state.activities.erase(i->second);
             state.its.erase(i);
         }
@@ -144,6 +168,8 @@ class ProgressBar : public Logger
 
     std::string getStatus(State & state)
     {
+        auto MiB = 1024.0 * 1024.0;
+
         std::string res;
 
         if (state.failedBuilds) {
@@ -189,44 +215,70 @@ class ProgressBar : public Logger
             res += fmt("%d/%d copied", copied, expected);
         }
 
+        auto & act = state.activitiesByType[actCopyPath];
+        uint64_t done = act.done, expected = act.done;
+        for (auto & j : act.its) {
+            done += j.second->done;
+            expected += j.second->expected;
+        }
+
+        expected = std::max(expected, act.expected);
+
+        if (done || expected) {
+            if (!res.empty()) res += ", ";
+            res += fmt("%1$.1f/%2$.1f MiB copied", done / MiB, expected / MiB);
+        }
+
         return res;
     }
 
     void event(const Event & ev) override
     {
+        auto state(state_.lock());
+
         if (ev.type == evStartActivity) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
-            createActivity(*state, act, ev.getS(1));
-            update(*state);
+            createActivity(*state, act, ev.getS(2), (ActivityType) ev.getI(1));
         }
 
         if (ev.type == evStopActivity) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             deleteActivity(*state, act);
-            update(*state);
+        }
+
+        if (ev.type == evProgress) {
+            auto i = state->its.find(ev.getI(0));
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            actInfo.done = ev.getI(1);
+            actInfo.expected = ev.getI(2);
+        }
+
+        if (ev.type == evSetExpected) {
+            auto i = state->its.find(ev.getI(0));
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            auto type = (ActivityType) ev.getI(1);
+            auto & j = actInfo.expectedByType[type];
+            state->activitiesByType[type].expected -= j;
+            j = ev.getI(2);
+            state->activitiesByType[type].expected += j;
         }
 
         if (ev.type == evBuildCreated) {
-            auto state(state_.lock());
             state->builds[ev.getI(0)] = ev.getS(1);
-            update(*state);
         }
 
         if (ev.type == evBuildStarted) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             state->runningBuilds.insert(act);
             auto name = storePathToName(state->builds[act]);
             if (hasSuffix(name, ".drv"))
                 name.resize(name.size() - 4);
             createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name));
-            update(*state);
         }
 
         if (ev.type == evBuildFinished) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             if (ev.getI(1)) {
                 if (state->runningBuilds.count(act))
@@ -236,34 +288,26 @@ class ProgressBar : public Logger
             state->runningBuilds.erase(act);
             state->builds.erase(act);
             deleteActivity(*state, act);
-            update(*state);
         }
 
         if (ev.type == evBuildOutput) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             assert(state->runningBuilds.count(act));
             updateActivity(*state, act, ev.getS(1));
-            update(*state);
         }
 
         if (ev.type == evSubstitutionCreated) {
-            auto state(state_.lock());
             state->substitutions[ev.getI(0)] = ev.getS(1);
-            update(*state);
         }
 
         if (ev.type == evSubstitutionStarted) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             state->runningSubstitutions.insert(act);
             auto name = storePathToName(state->substitutions[act]);
             createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name));
-            update(*state);
         }
 
         if (ev.type == evSubstitutionFinished) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             if (ev.getI(1)) {
                 if (state->runningSubstitutions.count(act))
@@ -272,60 +316,51 @@ class ProgressBar : public Logger
             state->runningSubstitutions.erase(act);
             state->substitutions.erase(act);
             deleteActivity(*state, act);
-            update(*state);
         }
 
         if (ev.type == evDownloadCreated) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             std::string uri = ev.getS(1);
             state->downloads.emplace(act, DownloadInfo{uri});
             if (state->runningSubstitutions.empty()) // FIXME: hack
                 createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri));
-            update(*state);
         }
 
         if (ev.type == evDownloadProgress) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             auto i = state->downloads.find(act);
             assert(i != state->downloads.end());
             i->second.expected = ev.getI(1);
             i->second.current = ev.getI(2);
-            update(*state);
         }
 
         if (ev.type == evDownloadSucceeded) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             auto i = state->downloads.find(act);
             assert(i != state->downloads.end());
             state->downloadedBytes += ev.getI(1);
             state->downloads.erase(i);
             deleteActivity(*state, act);
-            update(*state);
         }
 
         if (ev.type == evDownloadDestroyed) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             auto i = state->downloads.find(act);
             if (i != state->downloads.end()) {
                 state->downloads.erase(i);
                 deleteActivity(*state, act);
-                update(*state);
             }
         }
 
         if (ev.type == evCopyProgress) {
-            auto state(state_.lock());
             Activity::t act = ev.getI(0);
             auto & i = state->runningCopies[act];
             i.expected = ev.getI(1);
             i.copied = ev.getI(2);
             i.done = ev.getI(3);
-            update(*state);
         }
+
+        update(*state);
     }
 };
 

From c36467ad2e2e27d04e681144e0e10417c53c10c1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 20:14:55 +0200
Subject: [PATCH 0489/2196] Improve substitution progress indicator

E.g.

  $ nix build --store local?root=/tmp/nix nixpkgs.firefox-unwrapped
  [0/1 built, 1/97/98 fetched, 65.8/92.8 MiB DL, 203.2/309.2 MiB copied] downloading 'https://cache.nixos.org/nar/1czm9fk0svacy4h6a3fzkpafi4f7a9gml36kk8cq1igaghbspg3k.nar.xz'
---
 src/libstore/build.cc    |  55 +++++++++++++++++++-
 src/libstore/download.cc |  11 ++--
 src/libutil/logging.hh   |   6 +--
 src/nix/installables.cc  |   2 +-
 src/nix/progress-bar.cc  | 105 ++++++++++++---------------------------
 5 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9250a9b1778..d5f48237bd7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -9,6 +9,7 @@
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
+#include "nar-info.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -201,6 +202,16 @@ struct Child
 };
 
 
+template<typename T>
+struct MaintainCount
+{
+    T & counter;
+    T delta;
+    MaintainCount(T & counter, T delta) : counter(counter), delta(delta) { counter += delta; }
+    ~MaintainCount() { counter -= delta; }
+};
+
+
 /* The worker class. */
 class Worker
 {
@@ -244,6 +255,8 @@ class Worker
 
 public:
 
+    const Activity act;
+
     /* Set if at least one derivation had a BuildError (i.e. permanent
        failure). */
     bool permanentFailure;
@@ -255,6 +268,11 @@ class Worker
 
     std::unique_ptr<HookInstance> hook;
 
+    uint64_t expectedDownloadSize = 0;
+    uint64_t doneDownloadSize = 0;
+    uint64_t expectedNarSize = 0;
+    uint64_t doneNarSize = 0;
+
     Worker(LocalStore & store);
     ~Worker();
 
@@ -313,6 +331,12 @@ class Worker
     bool pathContentsGood(const Path & path);
 
     void markContentsGood(const Path & path);
+
+    void updateProgress()
+    {
+        logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
+        logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
+    }
 };
 
 
@@ -3304,6 +3328,8 @@ class SubstitutionGoal : public Goal
        storePath when doing a repair. */
     Path destPath;
 
+    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedNar, maintainExpectedDownload;
+
     typedef void (SubstitutionGoal::*GoalState)();
     GoalState state;
 
@@ -3430,6 +3456,18 @@ void SubstitutionGoal::tryNext()
         return;
     }
 
+    /* Update the total expected download size. */
+    auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
+
+    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize);
+
+    maintainExpectedDownload =
+        narInfo && narInfo->fileSize
+        ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
+        : nullptr;
+
+    worker.updateProgress();
+
     hasSubstitute = true;
 
     /* Bail out early if this substituter lacks a valid
@@ -3538,6 +3576,17 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path '%1%' succeeded") % storePath);
 
+    if (maintainExpectedDownload) {
+        auto fileSize = maintainExpectedDownload->delta;
+        maintainExpectedDownload.reset();
+        worker.doneDownloadSize += fileSize;
+    }
+
+    worker.doneNarSize += maintainExpectedNar->delta;
+    maintainExpectedNar.reset();
+
+    worker.updateProgress();
+
     amDone(ecSuccess);
 }
 
@@ -3560,7 +3609,8 @@ static bool working = false;
 
 
 Worker::Worker(LocalStore & store)
-    : store(store)
+    : act(actRealise)
+    , store(store)
 {
     /* Debugging: prevent recursive workers. */
     if (working) abort();
@@ -3581,6 +3631,9 @@ Worker::~Worker()
        are in trouble, since goals may call childTerminated() etc. in
        their destructors). */
     topGoals.clear();
+
+    assert(expectedDownloadSize == 0);
+    assert(expectedNarSize == 0);
 }
 
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index b731297a208..9ac4a65d570 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -83,12 +83,12 @@ struct CurlDownloader : public Downloader
         std::string encoding;
 
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
-            : downloader(downloader), request(request)
+            : downloader(downloader)
+            , request(request)
+            , act(actDownload, fmt("downloading '%s'", request.uri))
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
-
-            logger->event(evDownloadCreated, act, request.uri);
         }
 
         ~DownloadItem()
@@ -105,7 +105,6 @@ struct CurlDownloader : public Downloader
             } catch (...) {
                 ignoreException();
             }
-            logger->event(evDownloadDestroyed, act);
         }
 
         template<class T>
@@ -168,7 +167,7 @@ struct CurlDownloader : public Downloader
 
         int progressCallback(double dltotal, double dlnow)
         {
-            logger->event(evDownloadProgress, act, dltotal, dlnow);
+            act.progress(dlnow, dltotal);
             return _isInterrupted;
         }
 
@@ -267,7 +266,7 @@ struct CurlDownloader : public Downloader
                 try {
                     result.data = decodeContent(encoding, ref<std::string>(result.data));
                     callSuccess(success, failure, const_cast<const DownloadResult &>(result));
-                    logger->event(evDownloadSucceeded, act, result.data->size());
+                    act.progress(result.data->size(), result.data->size());
                 } catch (...) {
                     done = true;
                     callFailure(failure, std::current_exception());
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 6a4a3617156..7633408ead0 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -16,6 +16,8 @@ typedef enum {
 typedef enum {
     actUnknown = 0,
     actCopyPath = 100,
+    actDownload = 101,
+    actRealise = 102,
 } ActivityType;
 
 class Activity
@@ -38,10 +40,6 @@ typedef enum {
     evBuildStarted = 1,
     evBuildOutput = 2,
     evBuildFinished = 3,
-    evDownloadCreated = 4,
-    evDownloadDestroyed = 5,
-    evDownloadProgress = 6,
-    evDownloadSucceeded = 7,
     evSubstitutionCreated = 8,
     evSubstitutionStarted = 9,
     evSubstitutionFinished = 10,
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 54c9f7841fe..9d812fee47e 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -229,7 +229,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
 
 PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode)
 {
-    if (mode != DryRun)
+    if (mode != Build)
         settings.readOnlyMode = true;
 
     PathSet outPaths, buildables;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index b15f4488dd0..736eb5ede22 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -25,14 +25,6 @@ class ProgressBar : public Logger
         std::map<ActivityType, uint64_t> expectedByType;
     };
 
-    struct DownloadInfo
-    {
-        std::string uri;
-        uint64_t current = 0;
-        uint64_t expected = 0;
-        DownloadInfo(const std::string & uri) : uri(uri) { }
-    };
-
     struct CopyInfo
     {
         uint64_t expected = 0;
@@ -58,9 +50,6 @@ class ProgressBar : public Logger
         std::set<Activity::t> runningSubstitutions;
         uint64_t succeededSubstitutions = 0;
 
-        uint64_t downloadedBytes = 0; // finished downloads
-        std::map<Activity::t, DownloadInfo> downloads;
-
         std::map<Activity::t, CopyInfo> runningCopies;
 
         std::list<ActInfo> activities;
@@ -155,10 +144,16 @@ class ProgressBar : public Logger
         if (!state.activities.empty()) {
             if (!status.empty()) line += " ";
             auto i = state.activities.rbegin();
-            line += i->s;
-            if (!i->s2.empty()) {
-                line += ": ";
-                line += i->s2;
+
+            while (i != state.activities.rend() && i->s.empty() && i->s2.empty())
+                ++i;
+
+            if (i != state.activities.rend()) {
+                line += i->s;
+                if (!i->s2.empty()) {
+                    line += ": ";
+                    line += i->s2;
+                }
             }
         }
 
@@ -172,9 +167,13 @@ class ProgressBar : public Logger
 
         std::string res;
 
-        if (state.failedBuilds) {
+        auto add = [&](const std::string & s) {
             if (!res.empty()) res += ", ";
-            res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds);
+            res += s;
+        };
+
+        if (state.failedBuilds) {
+            add(fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds));
         }
 
         if (!state.builds.empty() || state.succeededBuilds)
@@ -195,39 +194,31 @@ class ProgressBar : public Logger
                 state.succeededSubstitutions + state.substitutions.size());
         }
 
-        if (!state.downloads.empty() || state.downloadedBytes) {
-            if (!res.empty()) res += ", ";
-            uint64_t expected = state.downloadedBytes, current = state.downloadedBytes;
-            for (auto & i : state.downloads) {
-                expected += i.second.expected;
-                current += i.second.current;
-            }
-            res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0);
-        }
-
         if (!state.runningCopies.empty()) {
-            if (!res.empty()) res += ", ";
             uint64_t copied = 0, expected = 0;
             for (auto & i : state.runningCopies) {
                 copied += i.second.copied;
                 expected += i.second.expected - (i.second.done - i.second.copied);
             }
-            res += fmt("%d/%d copied", copied, expected);
+            add(fmt("%d/%d copied", copied, expected));
         }
 
-        auto & act = state.activitiesByType[actCopyPath];
-        uint64_t done = act.done, expected = act.done;
-        for (auto & j : act.its) {
-            done += j.second->done;
-            expected += j.second->expected;
-        }
+        auto showActivity = [&](ActivityType type, const std::string & f) {
+            auto & act = state.activitiesByType[type];
+            uint64_t done = act.done, expected = act.done;
+            for (auto & j : act.its) {
+                done += j.second->done;
+                expected += j.second->expected;
+            }
 
-        expected = std::max(expected, act.expected);
+            expected = std::max(expected, act.expected);
 
-        if (done || expected) {
-            if (!res.empty()) res += ", ";
-            res += fmt("%1$.1f/%2$.1f MiB copied", done / MiB, expected / MiB);
-        }
+            if (done || expected)
+                add(fmt(f, done / MiB, expected / MiB));
+        };
+
+        showActivity(actDownload, "%1$.1f/%2$.1f MiB DL");
+        showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied");
 
         return res;
     }
@@ -318,40 +309,6 @@ class ProgressBar : public Logger
             deleteActivity(*state, act);
         }
 
-        if (ev.type == evDownloadCreated) {
-            Activity::t act = ev.getI(0);
-            std::string uri = ev.getS(1);
-            state->downloads.emplace(act, DownloadInfo{uri});
-            if (state->runningSubstitutions.empty()) // FIXME: hack
-                createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri));
-        }
-
-        if (ev.type == evDownloadProgress) {
-            Activity::t act = ev.getI(0);
-            auto i = state->downloads.find(act);
-            assert(i != state->downloads.end());
-            i->second.expected = ev.getI(1);
-            i->second.current = ev.getI(2);
-        }
-
-        if (ev.type == evDownloadSucceeded) {
-            Activity::t act = ev.getI(0);
-            auto i = state->downloads.find(act);
-            assert(i != state->downloads.end());
-            state->downloadedBytes += ev.getI(1);
-            state->downloads.erase(i);
-            deleteActivity(*state, act);
-        }
-
-        if (ev.type == evDownloadDestroyed) {
-            Activity::t act = ev.getI(0);
-            auto i = state->downloads.find(act);
-            if (i != state->downloads.end()) {
-                state->downloads.erase(i);
-                deleteActivity(*state, act);
-            }
-        }
-
         if (ev.type == evCopyProgress) {
             Activity::t act = ev.getI(0);
             auto & i = state->runningCopies[act];

From 0e0dcf2c7ec054f1b30629e275f53f56039b8257 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 22:12:36 +0200
Subject: [PATCH 0490/2196] Progress indicator: Unify "copying" and
 "substituting"

They're the same thing after all.

Example:

  $ nix build --store local?root=/tmp/nix nixpkgs.firefox-unwrapped
  [0/1 built, 49/98 copied, 16.3/92.8 MiB DL, 55.8/309.2 MiB copied] downloading 'https://cache.nixos.org/nar/0pl9li1jigcj2dany47hpmn0r3r48wc4nz48v5mqhh426lgz3bz6.nar.xz'
---
 src/libstore/build.cc     | 19 +++++++----
 src/libstore/store-api.cc | 11 ++----
 src/libutil/logging.hh    | 11 ++----
 src/nix/progress-bar.cc   | 70 +++------------------------------------
 4 files changed, 24 insertions(+), 87 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d5f48237bd7..9ea4e4b88ef 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -207,7 +207,7 @@ struct MaintainCount
 {
     T & counter;
     T delta;
-    MaintainCount(T & counter, T delta) : counter(counter), delta(delta) { counter += delta; }
+    MaintainCount(T & counter, T delta = 1) : counter(counter), delta(delta) { counter += delta; }
     ~MaintainCount() { counter -= delta; }
 };
 
@@ -256,6 +256,7 @@ class Worker
 public:
 
     const Activity act;
+    const Activity actSubstitutions;
 
     /* Set if at least one derivation had a BuildError (i.e. permanent
        failure). */
@@ -268,6 +269,8 @@ class Worker
 
     std::unique_ptr<HookInstance> hook;
 
+    uint64_t expectedSubstitutions = 0;
+    uint64_t doneSubstitutions = 0;
     uint64_t expectedDownloadSize = 0;
     uint64_t doneDownloadSize = 0;
     uint64_t expectedNarSize = 0;
@@ -334,6 +337,7 @@ class Worker
 
     void updateProgress()
     {
+        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions);
         logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
         logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
     }
@@ -3328,7 +3332,8 @@ class SubstitutionGoal : public Goal
        storePath when doing a repair. */
     Path destPath;
 
-    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedNar, maintainExpectedDownload;
+    std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
+        maintainExpectedNar, maintainExpectedDownload;
 
     typedef void (SubstitutionGoal::*GoalState)();
     GoalState state;
@@ -3364,7 +3369,6 @@ class SubstitutionGoal : public Goal
 
     void amDone(ExitCode result) override
     {
-        logger->event(evSubstitutionFinished, act, result == ecSuccess);
         Goal::amDone(result);
     }
 };
@@ -3379,7 +3383,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, Repa
     state = &SubstitutionGoal::init;
     name = (format("substitution of '%1%'") % storePath).str();
     trace("created");
-    logger->event(evSubstitutionCreated, act, storePath);
+    maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
 }
 
 
@@ -3527,8 +3531,6 @@ void SubstitutionGoal::tryToRun()
 
     printInfo(format("fetching path '%1%'...") % storePath);
 
-    logger->event(evSubstitutionStarted, act);
-
     outPipe.create();
 
     promise = std::promise<void>();
@@ -3576,6 +3578,9 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path '%1%' succeeded") % storePath);
 
+    maintainExpectedSubstitutions.reset();
+    worker.doneSubstitutions++;
+
     if (maintainExpectedDownload) {
         auto fileSize = maintainExpectedDownload->delta;
         maintainExpectedDownload.reset();
@@ -3610,6 +3615,7 @@ static bool working = false;
 
 Worker::Worker(LocalStore & store)
     : act(actRealise)
+    , actSubstitutions(actCopyPaths)
     , store(store)
 {
     /* Debugging: prevent recursive workers. */
@@ -3632,6 +3638,7 @@ Worker::~Worker()
        their destructors). */
     topGoals.clear();
 
+    assert(expectedSubstitutions == 0);
     assert(expectedDownloadSize == 0);
     assert(expectedNarSize == 0);
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 2b99a07e2af..1e11e54eba7 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -621,16 +621,13 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
-    Activity act(actUnknown);
+    Activity act(actCopyPaths, fmt("copying %d paths", missing.size()));
 
-    logger->event(evCopyStarted, act);
-
-    std::atomic<size_t> nrCopied{0};
-    std::atomic<size_t> nrDone{storePaths.size() - missing.size()};
+    std::atomic<size_t> nrDone{0};
     std::atomic<uint64_t> bytesExpected{0};
 
     auto showProgress = [&]() {
-        logger->event(evCopyProgress, act, storePaths.size(), nrCopied, nrDone);
+        act.progress(nrDone, missing.size());
     };
 
     ThreadPool pool;
@@ -659,11 +656,9 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             if (!dstStore->isValidPath(storePath)) {
                 printInfo("copying '%s'...", storePath);
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
-                nrCopied++;
             }
 
             nrDone++;
-
             showProgress();
         });
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 7633408ead0..f3ff099f0ce 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -18,6 +18,7 @@ typedef enum {
     actCopyPath = 100,
     actDownload = 101,
     actRealise = 102,
+    actCopyPaths = 103,
 } ActivityType;
 
 class Activity
@@ -32,7 +33,7 @@ public:
     ~Activity();
 
     template<typename... Args>
-    void progress(const Args & ... args);
+    void progress(const Args & ... args) const;
 };
 
 typedef enum {
@@ -40,12 +41,6 @@ typedef enum {
     evBuildStarted = 1,
     evBuildOutput = 2,
     evBuildFinished = 3,
-    evSubstitutionCreated = 8,
-    evSubstitutionStarted = 9,
-    evSubstitutionFinished = 10,
-
-    evCopyStarted = 100,
-    evCopyProgress = 101,
 
     evStartActivity = 1000,
     evStopActivity = 1001,
@@ -152,7 +147,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs);
 void writeToStderr(const string & s);
 
 template<typename... Args>
-void Activity::progress(const Args & ... args)
+void Activity::progress(const Args & ... args) const
 {
     Event ev;
     ev.type = evProgress;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 736eb5ede22..21d89e47709 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -25,13 +25,6 @@ class ProgressBar : public Logger
         std::map<ActivityType, uint64_t> expectedByType;
     };
 
-    struct CopyInfo
-    {
-        uint64_t expected = 0;
-        uint64_t copied = 0;
-        uint64_t done = 0;
-    };
-
     struct ActivitiesByType
     {
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
@@ -46,12 +39,6 @@ class ProgressBar : public Logger
         uint64_t succeededBuilds = 0;
         uint64_t failedBuilds = 0;
 
-        std::map<Activity::t, Path> substitutions;
-        std::set<Activity::t> runningSubstitutions;
-        uint64_t succeededSubstitutions = 0;
-
-        std::map<Activity::t, CopyInfo> runningCopies;
-
         std::list<ActInfo> activities;
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
 
@@ -185,25 +172,7 @@ class ProgressBar : public Logger
                 state.succeededBuilds, state.succeededBuilds + state.builds.size());
         }
 
-        if (!state.substitutions.empty() || state.succeededSubstitutions) {
-            if (!res.empty()) res += ", ";
-            if (!state.runningSubstitutions.empty())
-                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size());
-            res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL,
-                state.succeededSubstitutions,
-                state.succeededSubstitutions + state.substitutions.size());
-        }
-
-        if (!state.runningCopies.empty()) {
-            uint64_t copied = 0, expected = 0;
-            for (auto & i : state.runningCopies) {
-                copied += i.second.copied;
-                expected += i.second.expected - (i.second.done - i.second.copied);
-            }
-            add(fmt("%d/%d copied", copied, expected));
-        }
-
-        auto showActivity = [&](ActivityType type, const std::string & f) {
+        auto showActivity = [&](ActivityType type, const std::string & f, double unit) {
             auto & act = state.activitiesByType[type];
             uint64_t done = act.done, expected = act.done;
             for (auto & j : act.its) {
@@ -214,11 +183,12 @@ class ProgressBar : public Logger
             expected = std::max(expected, act.expected);
 
             if (done || expected)
-                add(fmt(f, done / MiB, expected / MiB));
+                add(fmt(f, done / unit, expected / unit));
         };
 
-        showActivity(actDownload, "%1$.1f/%2$.1f MiB DL");
-        showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied");
+        showActivity(actCopyPaths, ANSI_GREEN "%d" ANSI_NORMAL "/%d copied", 1);
+        showActivity(actDownload, "%1$.1f/%2$.1f MiB DL", MiB);
+        showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied", MiB);
 
         return res;
     }
@@ -287,36 +257,6 @@ class ProgressBar : public Logger
             updateActivity(*state, act, ev.getS(1));
         }
 
-        if (ev.type == evSubstitutionCreated) {
-            state->substitutions[ev.getI(0)] = ev.getS(1);
-        }
-
-        if (ev.type == evSubstitutionStarted) {
-            Activity::t act = ev.getI(0);
-            state->runningSubstitutions.insert(act);
-            auto name = storePathToName(state->substitutions[act]);
-            createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name));
-        }
-
-        if (ev.type == evSubstitutionFinished) {
-            Activity::t act = ev.getI(0);
-            if (ev.getI(1)) {
-                if (state->runningSubstitutions.count(act))
-                    state->succeededSubstitutions++;
-            }
-            state->runningSubstitutions.erase(act);
-            state->substitutions.erase(act);
-            deleteActivity(*state, act);
-        }
-
-        if (ev.type == evCopyProgress) {
-            Activity::t act = ev.getI(0);
-            auto & i = state->runningCopies[act];
-            i.expected = ev.getI(1);
-            i.copied = ev.getI(2);
-            i.done = ev.getI(3);
-        }
-
         update(*state);
     }
 };

From bf1f123b09ec7402b0565808619e11b5bfe6b16b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 14 Aug 2017 22:42:17 +0200
Subject: [PATCH 0491/2196] Progress indicator: Show number of active items

---
 src/libstore/build.cc     | 20 ++++++++------------
 src/libstore/store-api.cc |  5 ++++-
 src/libutil/logging.cc    |  5 +++++
 src/libutil/logging.hh    | 13 +------------
 src/libutil/types.hh      |  1 +
 src/libutil/util.hh       | 14 ++++++++++++++
 src/nix/progress-bar.cc   |  9 ++++++---
 7 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9ea4e4b88ef..a45ca892301 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -202,16 +202,6 @@ struct Child
 };
 
 
-template<typename T>
-struct MaintainCount
-{
-    T & counter;
-    T delta;
-    MaintainCount(T & counter, T delta = 1) : counter(counter), delta(delta) { counter += delta; }
-    ~MaintainCount() { counter -= delta; }
-};
-
-
 /* The worker class. */
 class Worker
 {
@@ -271,6 +261,7 @@ class Worker
 
     uint64_t expectedSubstitutions = 0;
     uint64_t doneSubstitutions = 0;
+    uint64_t runningSubstitutions = 0;
     uint64_t expectedDownloadSize = 0;
     uint64_t doneDownloadSize = 0;
     uint64_t expectedNarSize = 0;
@@ -337,7 +328,7 @@ class Worker
 
     void updateProgress()
     {
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions);
+        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions);
         logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
         logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
     }
@@ -3333,7 +3324,7 @@ class SubstitutionGoal : public Goal
     Path destPath;
 
     std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
-        maintainExpectedNar, maintainExpectedDownload;
+        maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
 
     typedef void (SubstitutionGoal::*GoalState)();
     GoalState state;
@@ -3531,6 +3522,9 @@ void SubstitutionGoal::tryToRun()
 
     printInfo(format("fetching path '%1%'...") % storePath);
 
+    maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
+    worker.updateProgress();
+
     outPipe.create();
 
     promise = std::promise<void>();
@@ -3578,6 +3572,8 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path '%1%' succeeded") % storePath);
 
+    maintainRunningSubstitutions.reset();
+
     maintainExpectedSubstitutions.reset();
     worker.doneSubstitutions++;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1e11e54eba7..5aa3dcfd26a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -625,9 +625,10 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
 
     std::atomic<size_t> nrDone{0};
     std::atomic<uint64_t> bytesExpected{0};
+    std::atomic<uint64_t> nrRunning{0};
 
     auto showProgress = [&]() {
-        act.progress(nrDone, missing.size());
+        act.progress(nrDone, missing.size(), nrRunning);
     };
 
     ThreadPool pool;
@@ -655,6 +656,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
 
             if (!dstStore->isValidPath(storePath)) {
                 printInfo("copying '%s'...", storePath);
+                MaintainCount<decltype(nrRunning)> mc(nrRunning);
+                showProgress();
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
             }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 321c9071291..ed83770a14a 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -91,4 +91,9 @@ Activity::~Activity()
     logger->event(evStopActivity, id);
 }
 
+void Activity::progress(uint64_t done, uint64_t expected, uint64_t running, uint64_t failed) const
+{
+    logger->event(evProgress, id, done, expected, running, failed);
+}
+
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index f3ff099f0ce..b6ab3d7d3b8 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -32,8 +32,7 @@ public:
     Activity(ActivityType type, std::string msg = "");
     ~Activity();
 
-    template<typename... Args>
-    void progress(const Args & ... args) const;
+    void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const;
 };
 
 typedef enum {
@@ -146,14 +145,4 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs);
 
 void writeToStderr(const string & s);
 
-template<typename... Args>
-void Activity::progress(const Args & ... args) const
-{
-    Event ev;
-    ev.type = evProgress;
-    ev.fields.emplace_back(id);
-    nop{(ev.fields.emplace_back(Event::Field(args)), 1)...};
-    logger->event(ev);
-}
-
 }
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 9f32d31addb..92bf469b5c6 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -70,6 +70,7 @@ template<typename... Args>
 inline std::string fmt(const std::string & fs, Args... args)
 {
     boost::format f(fs);
+    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
     nop{boost::io::detail::feed(f, args)...};
     return f.str();
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index f37f2c5d1be..35909c8d5b3 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -462,4 +462,18 @@ struct ReceiveInterrupts
     { }
 };
 
+
+
+/* A RAII helper that increments a counter on construction and
+   decrements it on destruction. */
+template<typename T>
+struct MaintainCount
+{
+    T & counter;
+    long delta;
+    MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; }
+    ~MaintainCount() { counter -= delta; }
+};
+
+
 }
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 21d89e47709..f4fc79fca16 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -22,6 +22,7 @@ class ProgressBar : public Logger
         ActivityType type = actUnknown;
         uint64_t done = 0;
         uint64_t expected = 0;
+        uint64_t running = 0;
         std::map<ActivityType, uint64_t> expectedByType;
     };
 
@@ -174,19 +175,20 @@ class ProgressBar : public Logger
 
         auto showActivity = [&](ActivityType type, const std::string & f, double unit) {
             auto & act = state.activitiesByType[type];
-            uint64_t done = act.done, expected = act.done;
+            uint64_t done = act.done, expected = act.done, running = 0;
             for (auto & j : act.its) {
                 done += j.second->done;
                 expected += j.second->expected;
+                running += j.second->running;
             }
 
             expected = std::max(expected, act.expected);
 
             if (done || expected)
-                add(fmt(f, done / unit, expected / unit));
+                add(fmt(f, done / unit, expected / unit, running));
         };
 
-        showActivity(actCopyPaths, ANSI_GREEN "%d" ANSI_NORMAL "/%d copied", 1);
+        showActivity(actCopyPaths, ANSI_BLUE "%3$d" ANSI_NORMAL "/" ANSI_GREEN "%1$d" ANSI_NORMAL "/%2$d copied", 1);
         showActivity(actDownload, "%1$.1f/%2$.1f MiB DL", MiB);
         showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied", MiB);
 
@@ -213,6 +215,7 @@ class ProgressBar : public Logger
             ActInfo & actInfo = *i->second;
             actInfo.done = ev.getI(1);
             actInfo.expected = ev.getI(2);
+            actInfo.running = ev.getI(3);
         }
 
         if (ev.type == evSetExpected) {

From dff12b38f9d836fd0a58abc41286a9ad6b602aa5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 15 Aug 2017 15:31:59 +0200
Subject: [PATCH 0492/2196] Progress indicator: More improvements

---
 src/libstore/build.cc   |  56 +++++++++++++++++----
 src/libutil/logging.hh  |   5 +-
 src/nix/progress-bar.cc | 109 +++++++++++++++++++---------------------
 3 files changed, 100 insertions(+), 70 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a45ca892301..257f02a4919 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -122,8 +122,6 @@ class Goal : public std::enable_shared_from_this<Goal>
     /* Whether the goal is finished. */
     ExitCode exitCode;
 
-    Activity act;
-
     Goal(Worker & worker) : worker(worker)
     {
         nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
@@ -246,6 +244,7 @@ class Worker
 public:
 
     const Activity act;
+    const Activity actDerivations;
     const Activity actSubstitutions;
 
     /* Set if at least one derivation had a BuildError (i.e. permanent
@@ -259,8 +258,14 @@ class Worker
 
     std::unique_ptr<HookInstance> hook;
 
+    uint64_t expectedBuilds = 0;
+    uint64_t doneBuilds = 0;
+    uint64_t failedBuilds = 0;
+    uint64_t runningBuilds = 0;
+
     uint64_t expectedSubstitutions = 0;
     uint64_t doneSubstitutions = 0;
+    uint64_t failedSubstitutions = 0;
     uint64_t runningSubstitutions = 0;
     uint64_t expectedDownloadSize = 0;
     uint64_t doneDownloadSize = 0;
@@ -328,7 +333,8 @@ class Worker
 
     void updateProgress()
     {
-        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions);
+        actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
+        actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
         logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
         logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
     }
@@ -829,6 +835,10 @@ class DerivationGoal : public Goal
 
     const static Path homeDir;
 
+    std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
+
+    std::unique_ptr<Activity> act;
+
 public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
@@ -926,7 +936,6 @@ class DerivationGoal : public Goal
 
     void amDone(ExitCode result) override
     {
-        logger->event(evBuildFinished, act, result == ecSuccess);
         Goal::amDone(result);
     }
 
@@ -949,7 +958,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     name = (format("building of '%1%'") % drvPath).str();
     trace("created");
 
-    logger->event(evBuildCreated, act, drvPath);
+    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
+    worker.updateProgress();
 }
 
 
@@ -965,7 +975,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv
     name = (format("building of %1%") % showPaths(drv.outputPaths())).str();
     trace("created");
 
-    logger->event(evBuildCreated, act, drvPath);
+    mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
+    worker.updateProgress();
 
     /* Prevent the .chroot directory from being
        garbage-collected. (See isActiveTempFile() in gc.cc.) */
@@ -1374,6 +1385,12 @@ void DerivationGoal::tryToBuild()
        supported for local builds. */
     bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
+    auto started = [&]() {
+        act = std::make_unique<Activity>(actBuild, fmt("building '%s'", drvPath));
+        mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+        worker.updateProgress();
+    };
+
     /* Is the build hook willing to accept this job? */
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -1382,6 +1399,7 @@ void DerivationGoal::tryToBuild()
                    EOF from the hook. */
                 result.startTime = time(0); // inexact
                 state = &DerivationGoal::buildDone;
+                started();
                 return;
             case rpPostpone:
                 /* Not now; wait until at least one child finishes or
@@ -1422,6 +1440,8 @@ void DerivationGoal::tryToBuild()
     /* This state will be reached when we get EOF on the child's
        log pipe. */
     state = &DerivationGoal::buildDone;
+
+    started();
 }
 
 
@@ -2147,8 +2167,6 @@ void DerivationGoal::startBuilder()
         }
         debug(msg);
     }
-
-    logger->event(evBuildStarted, act);
 }
 
 
@@ -3239,7 +3257,7 @@ void DerivationGoal::flushLine()
         logTail.push_back(currentLogLine);
         if (logTail.size() > settings.logLines) logTail.pop_front();
     }
-    logger->event(evBuildOutput, act, currentLogLine);
+    logger->event(evBuildOutput, *act, currentLogLine);
     currentLogLine = "";
     currentLogLinePos = 0;
 }
@@ -3282,6 +3300,19 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg)
         worker.timedOut = true;
     if (result.status == BuildResult::PermanentFailure)
         worker.permanentFailure = true;
+
+    mcExpectedBuilds.reset();
+    mcRunningBuilds.reset();
+
+    if (result.success()) {
+        if (status == BuildResult::Built)
+            worker.doneBuilds++;
+    } else {
+        if (status != BuildResult::DependencyFailed)
+            worker.failedBuilds++;
+    }
+
+    worker.updateProgress();
 }
 
 
@@ -3432,6 +3463,12 @@ void SubstitutionGoal::tryNext()
            In that case the calling derivation should just do a
            build. */
         amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
+
+        if (hasSubstitute) {
+            worker.failedSubstitutions++;
+            worker.updateProgress();
+        }
+
         return;
     }
 
@@ -3611,6 +3648,7 @@ static bool working = false;
 
 Worker::Worker(LocalStore & store)
     : act(actRealise)
+    , actDerivations(actBuilds)
     , actSubstitutions(actCopyPaths)
     , store(store)
 {
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index b6ab3d7d3b8..097d486eff9 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -19,6 +19,8 @@ typedef enum {
     actDownload = 101,
     actRealise = 102,
     actCopyPaths = 103,
+    actBuilds = 104,
+    actBuild = 105,
 } ActivityType;
 
 class Activity
@@ -36,10 +38,7 @@ public:
 };
 
 typedef enum {
-    evBuildCreated = 0,
-    evBuildStarted = 1,
     evBuildOutput = 2,
-    evBuildFinished = 3,
 
     evStartActivity = 1000,
     evStopActivity = 1001,
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index f4fc79fca16..c79a0d6d1db 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -23,6 +23,7 @@ class ProgressBar : public Logger
         uint64_t done = 0;
         uint64_t expected = 0;
         uint64_t running = 0;
+        uint64_t failed = 0;
         std::map<ActivityType, uint64_t> expectedByType;
     };
 
@@ -31,15 +32,11 @@ class ProgressBar : public Logger
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
         uint64_t done = 0;
         uint64_t expected = 0;
+        uint64_t failed = 0;
     };
 
     struct State
     {
-        std::map<Activity::t, Path> builds;
-        std::set<Activity::t> runningBuilds;
-        uint64_t succeededBuilds = 0;
-        uint64_t failedBuilds = 0;
-
         std::list<ActInfo> activities;
         std::map<Activity::t, std::list<ActInfo>::iterator> its;
 
@@ -62,7 +59,10 @@ class ProgressBar : public Logger
     ~ProgressBar()
     {
         auto state(state_.lock());
+        std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
+        if (status != "")
+            writeToStderr("[" + status + "]\n");
     }
 
     void log(Verbosity lvl, const FormatOrString & fs) override
@@ -91,6 +91,7 @@ class ProgressBar : public Logger
         if (i != state.its.end()) {
             auto & act = state.activitiesByType[i->second->type];
             act.done += i->second->done;
+            act.failed += i->second->failed;
 
             for (auto & j : i->second->expectedByType)
                 state.activitiesByType[j.first].expected -= j.second;
@@ -139,7 +140,7 @@ class ProgressBar : public Logger
             if (i != state.activities.rend()) {
                 line += i->s;
                 if (!i->s2.empty()) {
-                    line += ": ";
+                    if (!i->s.empty()) line += ": ";
                     line += i->s2;
                 }
             }
@@ -155,42 +156,57 @@ class ProgressBar : public Logger
 
         std::string res;
 
-        auto add = [&](const std::string & s) {
-            if (!res.empty()) res += ", ";
-            res += s;
-        };
-
-        if (state.failedBuilds) {
-            add(fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds));
-        }
-
-        if (!state.builds.empty() || state.succeededBuilds)
-        {
-            if (!res.empty()) res += ", ";
-            if (!state.runningBuilds.empty())
-                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size());
-            res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL,
-                state.succeededBuilds, state.succeededBuilds + state.builds.size());
-        }
-
-        auto showActivity = [&](ActivityType type, const std::string & f, double unit) {
+        auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt, double unit) {
             auto & act = state.activitiesByType[type];
-            uint64_t done = act.done, expected = act.done, running = 0;
+            uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
             for (auto & j : act.its) {
                 done += j.second->done;
                 expected += j.second->expected;
                 running += j.second->running;
+                failed += j.second->failed;
             }
 
             expected = std::max(expected, act.expected);
 
-            if (done || expected)
-                add(fmt(f, done / unit, expected / unit, running));
+            std::string s;
+
+            if (running || done || expected || failed) {
+                if (running)
+                    s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                        running / unit, done / unit, expected / unit);
+                else if (expected != done)
+                    s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                        done / unit, expected / unit);
+                else
+                    s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit);
+                s = fmt(itemFmt, s);
+
+                if (failed)
+                    s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit);
+            }
+
+            return s;
+        };
+
+        auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt, double unit) {
+            auto s = renderActivity(type, itemFmt, numberFmt, unit);
+            if (s.empty()) return;
+            if (!res.empty()) res += ", ";
+            res += s;
         };
 
-        showActivity(actCopyPaths, ANSI_BLUE "%3$d" ANSI_NORMAL "/" ANSI_GREEN "%1$d" ANSI_NORMAL "/%2$d copied", 1);
-        showActivity(actDownload, "%1$.1f/%2$.1f MiB DL", MiB);
-        showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied", MiB);
+        showActivity(actBuilds, "%s built", "%d", 1);
+
+        auto s1 = renderActivity(actCopyPaths, "%s copied", "%d", 1);
+        auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB);
+
+        if (!s1.empty() || !s2.empty()) {
+            if (!res.empty()) res += ", ";
+            if (s1.empty()) res += "0 copied"; else res += s1;
+            if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
+        }
+
+        showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
 
         return res;
     }
@@ -216,6 +232,7 @@ class ProgressBar : public Logger
             actInfo.done = ev.getI(1);
             actInfo.expected = ev.getI(2);
             actInfo.running = ev.getI(3);
+            actInfo.failed = ev.getI(4);
         }
 
         if (ev.type == evSetExpected) {
@@ -229,35 +246,11 @@ class ProgressBar : public Logger
             state->activitiesByType[type].expected += j;
         }
 
-        if (ev.type == evBuildCreated) {
-            state->builds[ev.getI(0)] = ev.getS(1);
-        }
-
-        if (ev.type == evBuildStarted) {
-            Activity::t act = ev.getI(0);
-            state->runningBuilds.insert(act);
-            auto name = storePathToName(state->builds[act]);
-            if (hasSuffix(name, ".drv"))
-                name.resize(name.size() - 4);
-            createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name));
-        }
-
-        if (ev.type == evBuildFinished) {
-            Activity::t act = ev.getI(0);
-            if (ev.getI(1)) {
-                if (state->runningBuilds.count(act))
-                    state->succeededBuilds++;
-            } else
-                state->failedBuilds++;
-            state->runningBuilds.erase(act);
-            state->builds.erase(act);
-            deleteActivity(*state, act);
-        }
-
         if (ev.type == evBuildOutput) {
             Activity::t act = ev.getI(0);
-            assert(state->runningBuilds.count(act));
-            updateActivity(*state, act, ev.getS(1));
+            auto s = trim(ev.getS(1));
+            if (!s.empty())
+                updateActivity(*state, act, s);
         }
 
         update(*state);

From 40bffe0a43e5f2f320c6bae7e39ea9c26906451d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Aug 2017 16:38:23 +0200
Subject: [PATCH 0493/2196] Progress indicator: Cleanup

---
 src/libstore/build.cc        |  14 ++---
 src/libstore/download.cc     |   2 +-
 src/libstore/store-api.cc    |   7 +--
 src/libutil/logging.cc       |  22 +------
 src/libutil/logging.hh       |  98 ++++++++++---------------------
 src/nix-daemon/nix-daemon.cc |   4 --
 src/nix/progress-bar.cc      | 109 +++++++++++++++++++----------------
 7 files changed, 105 insertions(+), 151 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 257f02a4919..1a1fc1dee54 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -335,8 +335,8 @@ class Worker
     {
         actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
         actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
-        logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
+        act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize);
+        act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
     }
 };
 
@@ -1386,7 +1386,7 @@ void DerivationGoal::tryToBuild()
     bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
     auto started = [&]() {
-        act = std::make_unique<Activity>(actBuild, fmt("building '%s'", drvPath));
+        act = std::make_unique<Activity>(*logger, actBuild, fmt("building '%s'", drvPath));
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
     };
@@ -3257,7 +3257,7 @@ void DerivationGoal::flushLine()
         logTail.push_back(currentLogLine);
         if (logTail.size() > settings.logLines) logTail.pop_front();
     }
-    logger->event(evBuildOutput, *act, currentLogLine);
+    act->progress(currentLogLine);
     currentLogLine = "";
     currentLogLinePos = 0;
 }
@@ -3647,9 +3647,9 @@ static bool working = false;
 
 
 Worker::Worker(LocalStore & store)
-    : act(actRealise)
-    , actDerivations(actBuilds)
-    , actSubstitutions(actCopyPaths)
+    : act(*logger, actRealise)
+    , actDerivations(*logger, actBuilds)
+    , actSubstitutions(*logger, actCopyPaths)
     , store(store)
 {
     /* Debugging: prevent recursive workers. */
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 9ac4a65d570..71b720a3b59 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -85,7 +85,7 @@ struct CurlDownloader : public Downloader
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader)
             , request(request)
-            , act(actDownload, fmt("downloading '%s'", request.uri))
+            , act(*logger, actDownload, fmt("downloading '%s'", request.uri))
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 5aa3dcfd26a..f520210615a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -565,7 +565,7 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    Activity act(actCopyPath, fmt("copying path '%s'", storePath));
+    Activity act(*logger, actCopyPath, fmt("copying path '%s'", storePath));
 
     auto info = srcStore->queryPathInfo(storePath);
 
@@ -621,7 +621,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
-    Activity act(actCopyPaths, fmt("copying %d paths", missing.size()));
+    Activity act(*logger, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};
     std::atomic<uint64_t> bytesExpected{0};
@@ -646,7 +646,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             auto info = srcStore->queryPathInfo(storePath);
 
             bytesExpected += info->narSize;
-            logger->event(evSetExpected, act, actCopyPath, bytesExpected);
+            act.setExpected(actCopyPath, bytesExpected);
 
             return info->references;
         },
@@ -655,7 +655,6 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             checkInterrupt();
 
             if (!dstStore->isValidPath(storePath)) {
-                printInfo("copying '%s'...", storePath);
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
                 copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index ed83770a14a..900f24e4cbd 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -43,10 +43,6 @@ class SimpleLogger : public Logger
 
         writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
     }
-
-    void event(const Event & ev) override
-    {
-    }
 };
 
 Verbosity verbosity = lvlInfo;
@@ -78,22 +74,10 @@ Logger * makeDefaultLogger()
 
 std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
-Activity::Activity() : id(nextId++) { };
-
-Activity::Activity(ActivityType type, std::string msg)
-    : Activity()
-{
-    logger->event(evStartActivity, id, type, msg);
-}
-
-Activity::~Activity()
-{
-    logger->event(evStopActivity, id);
-}
-
-void Activity::progress(uint64_t done, uint64_t expected, uint64_t running, uint64_t failed) const
+Activity::Activity(Logger & logger, ActivityType type, const std::string & s)
+    : logger(logger), id(nextId++)
 {
-    logger->event(evProgress, id, done, expected, running, failed);
+    logger.startActivity(id, type, s);
 }
 
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 097d486eff9..edc3553f694 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -23,63 +23,7 @@ typedef enum {
     actBuild = 105,
 } ActivityType;
 
-class Activity
-{
-public:
-    typedef uint64_t t;
-    const t id;
-    Activity();
-    Activity(const Activity & act) : id(act.id) { };
-    Activity(uint64_t id) : id(id) { };
-    Activity(ActivityType type, std::string msg = "");
-    ~Activity();
-
-    void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const;
-};
-
-typedef enum {
-    evBuildOutput = 2,
-
-    evStartActivity = 1000,
-    evStopActivity = 1001,
-    evProgress = 1002,
-    evSetExpected = 1003,
-
-} EventType;
-
-struct Event
-{
-    struct Field
-    {
-        // FIXME: use std::variant.
-        enum { tInt, tString } type;
-        uint64_t i = 0;
-        std::string s;
-        Field(const std::string & s) : type(tString), s(s) { }
-        Field(const char * s) : type(tString), s(s) { }
-        Field(const uint64_t & i) : type(tInt), i(i) { }
-        Field(const Activity & act) : type(tInt), i(act.id) { }
-    };
-
-    typedef std::vector<Field> Fields;
-
-    EventType type;
-    Fields fields;
-
-    std::string getS(size_t n) const
-    {
-        assert(n < fields.size());
-        assert(fields[n].type == Field::tString);
-        return fields[n].s;
-    }
-
-    uint64_t getI(size_t n) const
-    {
-        assert(n < fields.size());
-        assert(fields[n].type == Field::tInt);
-        return fields[n].i;
-    }
-};
+typedef uint64_t ActivityId;
 
 class Logger
 {
@@ -98,16 +42,38 @@ public:
 
     virtual void warn(const std::string & msg);
 
-    template<typename... Args>
-    void event(EventType type, const Args & ... args)
-    {
-        Event ev;
-        ev.type = type;
-        nop{(ev.fields.emplace_back(Event::Field(args)), 1)...};
-        event(ev);
-    }
+    virtual void startActivity(ActivityId act, ActivityType type, const std::string & s) { };
+
+    virtual void stopActivity(ActivityId act) { };
+
+    virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
+
+    virtual void progress(ActivityId act, const std::string & s) { };
+
+    virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
+};
+
+struct Activity
+{
+    Logger & logger;
+
+    const ActivityId id;
+
+    Activity(Logger & logger, ActivityType type, const std::string & s = "");
+
+    ~Activity()
+    { logger.stopActivity(id); }
+
+    void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
+    { logger.progress(id, done, expected, running, failed); }
+
+    void progress(const std::string & s) const
+    { logger.progress(id, s); }
+
+    void setExpected(ActivityType type2, uint64_t expected) const
+    { logger.setExpected(id, type2, expected); }
 
-    virtual void event(const Event & ev) = 0;
+    friend class Logger;
 };
 
 extern Logger * logger;
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 2b223c88264..7e6f3aa25b5 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -81,10 +81,6 @@ class TunnelLogger : public Logger
         } else
             defaultLogger->log(lvl, fs);
     }
-
-    void event(const Event & ev) override
-    {
-    }
 };
 
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index c79a0d6d1db..6288737088a 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -29,7 +29,7 @@ class ProgressBar : public Logger
 
     struct ActivitiesByType
     {
-        std::map<Activity::t, std::list<ActInfo>::iterator> its;
+        std::map<ActivityId, std::list<ActInfo>::iterator> its;
         uint64_t done = 0;
         uint64_t expected = 0;
         uint64_t failed = 0;
@@ -38,7 +38,7 @@ class ProgressBar : public Logger
     struct State
     {
         std::list<ActInfo> activities;
-        std::map<Activity::t, std::list<ActInfo>::iterator> its;
+        std::map<ActivityId, std::list<ActInfo>::iterator> its;
 
         std::map<ActivityType, ActivitiesByType> activitiesByType;
     };
@@ -77,7 +77,61 @@ class ProgressBar : public Logger
         update(state);
     }
 
-    void createActivity(State & state, Activity::t activity, const std::string & s, ActivityType type = actUnknown)
+    void startActivity(ActivityId act, ActivityType type, const std::string & s) override
+    {
+        auto state(state_.lock());
+        createActivity(*state, act, s, type);
+        update(*state);
+    }
+
+    void stopActivity(ActivityId act) override
+    {
+        auto state(state_.lock());
+        deleteActivity(*state, act);
+        update(*state);
+    }
+
+    void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override
+    {
+        auto state(state_.lock());
+
+        auto i = state->its.find(act);
+        assert(i != state->its.end());
+        ActInfo & actInfo = *i->second;
+        actInfo.done = done;
+        actInfo.expected = expected;
+        actInfo.running = running;
+        actInfo.failed = failed;
+
+        update(*state);
+    }
+
+    void progress(ActivityId act, const std::string & s) override
+    {
+        auto state(state_.lock());
+        auto s2 = trim(s);
+        if (!s2.empty()) {
+            updateActivity(*state, act, s2);
+            update(*state);
+        }
+    }
+
+    void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
+    {
+        auto state(state_.lock());
+
+        auto i = state->its.find(act);
+        assert(i != state->its.end());
+        ActInfo & actInfo = *i->second;
+        auto & j = actInfo.expectedByType[type];
+        state->activitiesByType[type].expected -= j;
+        j = expected;
+        state->activitiesByType[type].expected += j;
+
+        update(*state);
+    }
+
+    void createActivity(State & state, ActivityId activity, const std::string & s, ActivityType type = actUnknown)
     {
         state.activities.emplace_back(ActInfo{s, "", type});
         auto i = std::prev(state.activities.end());
@@ -85,7 +139,7 @@ class ProgressBar : public Logger
         state.activitiesByType[type].its.emplace(activity, i);
     }
 
-    void deleteActivity(State & state, Activity::t activity)
+    void deleteActivity(State & state, ActivityId activity)
     {
         auto i = state.its.find(activity);
         if (i != state.its.end()) {
@@ -102,7 +156,7 @@ class ProgressBar : public Logger
         }
     }
 
-    void updateActivity(State & state, Activity::t activity, const std::string & s2)
+    void updateActivity(State & state, ActivityId activity, const std::string & s2)
     {
         auto i = state.its.find(activity);
         assert(i != state.its.end());
@@ -210,51 +264,6 @@ class ProgressBar : public Logger
 
         return res;
     }
-
-    void event(const Event & ev) override
-    {
-        auto state(state_.lock());
-
-        if (ev.type == evStartActivity) {
-            Activity::t act = ev.getI(0);
-            createActivity(*state, act, ev.getS(2), (ActivityType) ev.getI(1));
-        }
-
-        if (ev.type == evStopActivity) {
-            Activity::t act = ev.getI(0);
-            deleteActivity(*state, act);
-        }
-
-        if (ev.type == evProgress) {
-            auto i = state->its.find(ev.getI(0));
-            assert(i != state->its.end());
-            ActInfo & actInfo = *i->second;
-            actInfo.done = ev.getI(1);
-            actInfo.expected = ev.getI(2);
-            actInfo.running = ev.getI(3);
-            actInfo.failed = ev.getI(4);
-        }
-
-        if (ev.type == evSetExpected) {
-            auto i = state->its.find(ev.getI(0));
-            assert(i != state->its.end());
-            ActInfo & actInfo = *i->second;
-            auto type = (ActivityType) ev.getI(1);
-            auto & j = actInfo.expectedByType[type];
-            state->activitiesByType[type].expected -= j;
-            j = ev.getI(2);
-            state->activitiesByType[type].expected += j;
-        }
-
-        if (ev.type == evBuildOutput) {
-            Activity::t act = ev.getI(0);
-            auto s = trim(ev.getS(1));
-            if (!s.empty())
-                updateActivity(*state, act, s);
-        }
-
-        update(*state);
-    }
 };
 
 StartProgressBar::StartProgressBar()

From 23b8b7e096d3a2a784387a09f68115706b1e9552 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Aug 2017 17:00:24 +0200
Subject: [PATCH 0494/2196] nix optimise-store: Add

This replaces "nix-store --optimise". Main difference is that it has a
progress indicator.
---
 src/libstore/optimise-store.cc | 14 ++++++++++--
 src/libutil/logging.hh         |  1 +
 src/nix/optimise-store.cc      | 41 ++++++++++++++++++++++++++++++++++
 src/nix/progress-bar.cc        | 10 +++++----
 4 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 src/nix/optimise-store.cc

diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b736307b31b..5093305a13e 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -249,14 +249,24 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
 
 void LocalStore::optimiseStore(OptimiseStats & stats)
 {
+    Activity act(*logger, actOptimiseStore);
+
     PathSet paths = queryAllValidPaths();
     InodeHash inodeHash = loadInodeHash();
 
+    act.progress(0, paths.size());
+
+    uint64_t done = 0;
+
     for (auto & i : paths) {
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
-        //Activity act(*logger, lvlChatty, format("hashing files in '%1%'") % i);
-        optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
+        {
+            Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
+            optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
+        }
+        done++;
+        act.progress(done, paths.size());
     }
 }
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index edc3553f694..bb89643d0a0 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -21,6 +21,7 @@ typedef enum {
     actCopyPaths = 103,
     actBuilds = 104,
     actBuild = 105,
+    actOptimiseStore = 106,
 } ActivityType;
 
 typedef uint64_t ActivityId;
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
new file mode 100644
index 00000000000..725fb75a19e
--- /dev/null
+++ b/src/nix/optimise-store.cc
@@ -0,0 +1,41 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdOptimiseStore : StoreCommand
+{
+    CmdOptimiseStore()
+    {
+    }
+
+    std::string name() override
+    {
+        return "optimise-store";
+    }
+
+    std::string description() override
+    {
+        return "replace identical files in the store by hard links";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To optimise the Nix store:",
+                "nix optimise-store"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        store->optimiseStore();
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdOptimiseStore>());
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 6288737088a..15ef95370a8 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -210,7 +210,7 @@ class ProgressBar : public Logger
 
         std::string res;
 
-        auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt, double unit) {
+        auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
             auto & act = state.activitiesByType[type];
             uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
             for (auto & j : act.its) {
@@ -242,16 +242,16 @@ class ProgressBar : public Logger
             return s;
         };
 
-        auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt, double unit) {
+        auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
             auto s = renderActivity(type, itemFmt, numberFmt, unit);
             if (s.empty()) return;
             if (!res.empty()) res += ", ";
             res += s;
         };
 
-        showActivity(actBuilds, "%s built", "%d", 1);
+        showActivity(actBuilds, "%s built");
 
-        auto s1 = renderActivity(actCopyPaths, "%s copied", "%d", 1);
+        auto s1 = renderActivity(actCopyPaths, "%s copied");
         auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB);
 
         if (!s1.empty() || !s2.empty()) {
@@ -262,6 +262,8 @@ class ProgressBar : public Logger
 
         showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
 
+        showActivity(actOptimiseStore, "%s paths optimised");
+
         return res;
     }
 };

From b4ed97e3a3116909fcaa79f5ce84487ed3838112 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Aug 2017 17:32:18 +0200
Subject: [PATCH 0495/2196] nix optimise-store: Show how much space has been
 freed

---
 src/libstore/build.cc          |   2 +-
 src/libstore/local-store.hh    |   2 +-
 src/libstore/optimise-store.cc |  14 +++--
 src/libutil/logging.hh         |  31 ++++++++--
 src/nix/progress-bar.cc        | 107 ++++++++++++++++++++-------------
 5 files changed, 101 insertions(+), 55 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1a1fc1dee54..fc039a8e52b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3257,7 +3257,7 @@ void DerivationGoal::flushLine()
         logTail.push_back(currentLogLine);
         if (logTail.size() > settings.logLines) logTail.pop_front();
     }
-    act->progress(currentLogLine);
+    act->result(resBuildLogLine, currentLogLine);
     currentLogLine = "";
     currentLogLinePos = 0;
 }
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 551c6b506fb..2af1bfbb389 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -251,7 +251,7 @@ private:
 
     InodeHash loadInodeHash();
     Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
-    void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
+    void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
 
     // Internal versions that are not wrapped in retry_sqlite.
     bool isValidPath_(State & state, const Path & path);
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 5093305a13e..f1325ba5a1c 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -89,7 +89,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
 }
 
 
-void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
+void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
+    const Path & path, InodeHash & inodeHash)
 {
     checkInterrupt();
 
@@ -114,7 +115,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectoryIgnoringInodes(path, inodeHash);
         for (auto & i : names)
-            optimisePath_(stats, path + "/" + i, inodeHash);
+            optimisePath_(act, stats, path + "/" + i, inodeHash);
         return;
     }
 
@@ -244,6 +245,9 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
     stats.filesLinked++;
     stats.bytesFreed += st.st_size;
     stats.blocksFreed += st.st_blocks;
+
+    if (act)
+        act->result(resFileLinked, st.st_size, st.st_blocks);
 }
 
 
@@ -263,7 +267,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
         {
             Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
-            optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
+            optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
         }
         done++;
         act.progress(done, paths.size());
@@ -281,7 +285,7 @@ void LocalStore::optimiseStore()
 
     optimiseStore(stats);
 
-    printError(
+    printInfo(
         format("%1% freed by hard-linking %2% files")
         % showBytes(stats.bytesFreed)
         % stats.filesLinked);
@@ -292,7 +296,7 @@ void LocalStore::optimisePath(const Path & path)
     OptimiseStats stats;
     InodeHash inodeHash;
 
-    if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
+    if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
 }
 
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index bb89643d0a0..e0b8c0855ea 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -24,6 +24,11 @@ typedef enum {
     actOptimiseStore = 106,
 } ActivityType;
 
+typedef enum {
+    resFileLinked = 100,
+    resBuildLogLine = 101,
+} ResultType;
+
 typedef uint64_t ActivityId;
 
 class Logger
@@ -49,9 +54,20 @@ public:
 
     virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
 
-    virtual void progress(ActivityId act, const std::string & s) { };
-
     virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
+
+    struct Field
+    {
+        // FIXME: use std::variant.
+        enum { tInt, tString } type;
+        uint64_t i = 0;
+        std::string s;
+        Field(const std::string & s) : type(tString), s(s) { }
+        Field(const char * s) : type(tString), s(s) { }
+        Field(const uint64_t & i) : type(tInt), i(i) { }
+    };
+
+    virtual void result(ActivityId act, ResultType type, const std::vector<Field> & fields) { };
 };
 
 struct Activity
@@ -68,12 +84,17 @@ struct Activity
     void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
     { logger.progress(id, done, expected, running, failed); }
 
-    void progress(const std::string & s) const
-    { logger.progress(id, s); }
-
     void setExpected(ActivityType type2, uint64_t expected) const
     { logger.setExpected(id, type2, expected); }
 
+    template<typename... Args>
+    void result(ResultType type, const Args & ... args)
+    {
+        std::vector<Logger::Field> fields;
+        nop{(fields.emplace_back(Logger::Field(args)), 1)...};
+        logger.result(id, type, fields);
+    }
+
     friend class Logger;
 };
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 15ef95370a8..cc2928472d1 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -12,6 +12,20 @@
 
 namespace nix {
 
+static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
+{
+    assert(n < fields.size());
+    assert(fields[n].type == Logger::Field::tString);
+    return fields[n].s;
+}
+
+static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
+{
+    assert(n < fields.size());
+    assert(fields[n].type == Logger::Field::tInt);
+    return fields[n].i;
+}
+
 class ProgressBar : public Logger
 {
 private:
@@ -41,6 +55,8 @@ class ProgressBar : public Logger
         std::map<ActivityId, std::list<ActInfo>::iterator> its;
 
         std::map<ActivityType, ActivitiesByType> activitiesByType;
+
+        uint64_t filesLinked = 0, bytesLinked = 0;
     };
 
     Sync<State> state_;
@@ -80,14 +96,33 @@ class ProgressBar : public Logger
     void startActivity(ActivityId act, ActivityType type, const std::string & s) override
     {
         auto state(state_.lock());
-        createActivity(*state, act, s, type);
+
+        state->activities.emplace_back(ActInfo{s, "", type});
+        auto i = std::prev(state->activities.end());
+        state->its.emplace(act, i);
+        state->activitiesByType[type].its.emplace(act, i);
+
         update(*state);
     }
 
     void stopActivity(ActivityId act) override
     {
         auto state(state_.lock());
-        deleteActivity(*state, act);
+
+        auto i = state->its.find(act);
+        if (i != state->its.end()) {
+            auto & actByType = state->activitiesByType[i->second->type];
+            actByType.done += i->second->done;
+            actByType.failed += i->second->failed;
+
+            for (auto & j : i->second->expectedByType)
+                state->activitiesByType[j.first].expected -= j.second;
+
+            actByType.its.erase(act);
+            state->activities.erase(i->second);
+            state->its.erase(i);
+        }
+
         update(*state);
     }
 
@@ -106,16 +141,6 @@ class ProgressBar : public Logger
         update(*state);
     }
 
-    void progress(ActivityId act, const std::string & s) override
-    {
-        auto state(state_.lock());
-        auto s2 = trim(s);
-        if (!s2.empty()) {
-            updateActivity(*state, act, s2);
-            update(*state);
-        }
-    }
-
     void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
     {
         auto state(state_.lock());
@@ -131,40 +156,29 @@ class ProgressBar : public Logger
         update(*state);
     }
 
-    void createActivity(State & state, ActivityId activity, const std::string & s, ActivityType type = actUnknown)
+    void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
     {
-        state.activities.emplace_back(ActInfo{s, "", type});
-        auto i = std::prev(state.activities.end());
-        state.its.emplace(activity, i);
-        state.activitiesByType[type].its.emplace(activity, i);
-    }
-
-    void deleteActivity(State & state, ActivityId activity)
-    {
-        auto i = state.its.find(activity);
-        if (i != state.its.end()) {
-            auto & act = state.activitiesByType[i->second->type];
-            act.done += i->second->done;
-            act.failed += i->second->failed;
-
-            for (auto & j : i->second->expectedByType)
-                state.activitiesByType[j.first].expected -= j.second;
+        auto state(state_.lock());
 
-            act.its.erase(activity);
-            state.activities.erase(i->second);
-            state.its.erase(i);
+        if (type == resFileLinked) {
+            state->filesLinked++;
+            state->bytesLinked += getI(fields, 0);
+            update(*state);
         }
-    }
 
-    void updateActivity(State & state, ActivityId activity, const std::string & s2)
-    {
-        auto i = state.its.find(activity);
-        assert(i != state.its.end());
-        ActInfo info = *i->second;
-        state.activities.erase(i->second);
-        info.s2 = s2;
-        state.activities.emplace_back(info);
-        i->second = std::prev(state.activities.end());
+        else if (type == resBuildLogLine) {
+            auto s2 = trim(getS(fields, 0));
+            if (!s2.empty()) {
+                auto i = state->its.find(act);
+                assert(i != state->its.end());
+                ActInfo info = *i->second;
+                state->activities.erase(i->second);
+                info.s2 = s2;
+                state->activities.emplace_back(info);
+                i->second = std::prev(state->activities.end());
+                update(*state);
+            }
+        }
     }
 
     void update()
@@ -262,7 +276,14 @@ class ProgressBar : public Logger
 
         showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
 
-        showActivity(actOptimiseStore, "%s paths optimised");
+        {
+            auto s = renderActivity(actOptimiseStore, "%s paths optimised");
+            if (s != "") {
+                s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
+                if (!res.empty()) res += ", ";
+                res += s;
+            }
+        }
 
         return res;
     }

From c2cab207320672fb6ed4af40a99fc9082ff55234 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Aug 2017 19:23:46 +0200
Subject: [PATCH 0496/2196] nix verify: Restore the progress indicator

---
 src/libutil/logging.hh  |  3 +++
 src/nix/progress-bar.cc | 25 +++++++++++++++++++++++++
 src/nix/verify.cc       | 29 +++++++++++++++--------------
 3 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index e0b8c0855ea..1e0c735676b 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -22,11 +22,14 @@ typedef enum {
     actBuilds = 104,
     actBuild = 105,
     actOptimiseStore = 106,
+    actVerifyPaths = 107,
 } ActivityType;
 
 typedef enum {
     resFileLinked = 100,
     resBuildLogLine = 101,
+    resUntrustedPath = 102,
+    resCorruptedPath = 103,
 } ResultType;
 
 typedef uint64_t ActivityId;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index cc2928472d1..1616c420551 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -57,6 +57,8 @@ class ProgressBar : public Logger
         std::map<ActivityType, ActivitiesByType> activitiesByType;
 
         uint64_t filesLinked = 0, bytesLinked = 0;
+
+        uint64_t corruptedPaths = 0, untrustedPaths = 0;
     };
 
     Sync<State> state_;
@@ -179,6 +181,16 @@ class ProgressBar : public Logger
                 update(*state);
             }
         }
+
+        else if (type == resUntrustedPath) {
+            state->untrustedPaths++;
+            update(*state);
+        }
+
+        else if (type == resCorruptedPath) {
+            state->corruptedPaths++;
+            update(*state);
+        }
     }
 
     void update()
@@ -285,6 +297,19 @@ class ProgressBar : public Logger
             }
         }
 
+        // FIXME: don't show "done" paths in green.
+        showActivity(actVerifyPaths, "%s paths verified");
+
+        if (state.corruptedPaths) {
+            if (!res.empty()) res += ", ";
+            res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths);
+        }
+
+        if (state.untrustedPaths) {
+            if (!res.empty()) res += ", ";
+            res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths);
+        }
+
         return res;
     }
 };
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 0d9739314b5..7156341396d 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -61,16 +61,17 @@ struct CmdVerify : StorePathsCommand
 
         auto publicKeys = getDefaultPublicKeys();
 
+        Activity act(*logger, actVerifyPaths);
+
         std::atomic<size_t> done{0};
         std::atomic<size_t> untrusted{0};
         std::atomic<size_t> corrupted{0};
         std::atomic<size_t> failed{0};
+        std::atomic<size_t> active{0};
 
-        std::string doneLabel("paths checked");
-        std::string untrustedLabel("untrusted");
-        std::string corruptedLabel("corrupted");
-        std::string failedLabel("failed");
-        //logger->setExpected(doneLabel, storePaths.size());
+        auto update = [&]() {
+            act.progress(done, storePaths.size(), active, failed);
+        };
 
         ThreadPool pool;
 
@@ -78,7 +79,10 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                //Activity act(*logger, lvlInfo, format("checking '%s'") % storePath);
+                Activity act2(*logger, actUnknown, fmt("checking '%s'", storePath));
+
+                MaintainCount<std::atomic<size_t>> mcActive(active);
+                update();
 
                 auto info = store->queryPathInfo(storePath);
 
@@ -90,8 +94,8 @@ struct CmdVerify : StorePathsCommand
                     auto hash = sink.finish();
 
                     if (hash.first != info->narHash) {
-                        //logger->incProgress(corruptedLabel);
-                        corrupted = 1;
+                        corrupted++;
+                        act2.result(resCorruptedPath, info->path);
                         printError(
                             format("path '%s' was modified! expected hash '%s', got '%s'")
                             % info->path % info->narHash.to_string() % hash.first.to_string());
@@ -142,21 +146,21 @@ struct CmdVerify : StorePathsCommand
                     }
 
                     if (!good) {
-                        //logger->incProgress(untrustedLabel);
                         untrusted++;
+                        act2.result(resUntrustedPath, info->path);
                         printError(format("path '%s' is untrusted") % info->path);
                     }
 
                 }
 
-                //logger->incProgress(doneLabel);
                 done++;
 
             } catch (Error & e) {
                 printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
-                //logger->incProgress(failedLabel);
                 failed++;
             }
+
+            update();
         };
 
         for (auto & storePath : storePaths)
@@ -164,9 +168,6 @@ struct CmdVerify : StorePathsCommand
 
         pool.process();
 
-        printInfo(format("%d paths checked, %d untrusted, %d corrupted, %d failed")
-            % done % untrusted % corrupted % failed);
-
         throw Exit(
             (corrupted ? 1 : 0) |
             (untrusted ? 2 : 0) |

From e2f9a61dc95f8f0ee51995ad88f4c5d5bd0e7740 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Aug 2017 21:36:47 +0200
Subject: [PATCH 0497/2196] Update release notes

---
 doc/manual/release-notes/rl-1.12.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 6aab0ab906e..551139f3a66 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -382,6 +382,11 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     script.</para>
   </listitem>
 
+  <listitem>
+    <para><function>builtins.split</function>.
+    (b8867a0239b1930a16f9ef3f7f3e864b01416dff))</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>

From 6bb4e3e8fe30a09d174a04295ba074ee006ebb4f Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Fri, 18 Aug 2017 11:07:00 +0200
Subject: [PATCH 0498/2196] Remove unused decodeOctalEscaped

Besides being unused, this function has a bug that it will incorrectly
decode the path component Ubuntu\04016.04.2\040LTS\040amd64 as
"Ubuntu.04.2 LTS amd64" instead of "Ubuntu 16.04.2 LTS amd64".
---
 src/libutil/util.cc | 15 ---------------
 src/libutil/util.hh |  6 ------
 2 files changed, 21 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 55d3e1d16f6..c2bceea3d9d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1140,21 +1140,6 @@ std::string toLower(const std::string & s)
 }
 
 
-string decodeOctalEscaped(const string & s)
-{
-    string r;
-    for (string::const_iterator i = s.begin(); i != s.end(); ) {
-        if (*i != '\\') { r += *i++; continue; }
-        unsigned char c = 0;
-        ++i;
-        while (i != s.end() && *i >= '0' && *i < '8')
-            c = c * 8 + (*i++ - '0');
-        r += c;
-    }
-    return r;
-}
-
-
 void ignoreException()
 {
     try {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 35909c8d5b3..7340ca6c747 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -349,12 +349,6 @@ bool hasSuffix(const string & s, const string & suffix);
 std::string toLower(const std::string & s);
 
 
-/* Escape a string that contains octal-encoded escape codes such as
-   used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to
-   "foo bar"). */
-string decodeOctalEscaped(const string & s);
-
-
 /* Exception handling in destructors: print an error message, then
    ignore the exception. */
 void ignoreException();

From 09a38f91257c7ef482368c24b38dd7098d4589b8 Mon Sep 17 00:00:00 2001
From: Chase Adams <realchaseadams@gmail.com>
Date: Fri, 18 Aug 2017 16:07:33 -0700
Subject: [PATCH 0499/2196] update MD5 to SHA-256 in expression-syntax

---
 doc/manual/expressions/expression-syntax.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
index b674b0d6000..42b9dca3623 100644
--- a/doc/manual/expressions/expression-syntax.xml
+++ b/doc/manual/expressions/expression-syntax.xml
@@ -108,7 +108,7 @@ the single Nix expression in that directory
     <para>The builder has to know what the sources of the package
     are.  Here, the attribute <varname>src</varname> is bound to the
     result of a call to the <command>fetchurl</command> function.
-    Given a URL and an MD5 hash of the expected contents of the file
+    Given a URL and a SHA-256 hash of the expected contents of the file
     at that URL, this function builds a derivation that downloads the
     file and checks its hash.  So the sources are a dependency that
     like all other dependencies is built before Hello itself is
@@ -145,4 +145,4 @@ perl = perl;</programlisting>
 
 </para>
 
-</section>
\ No newline at end of file
+</section>

From 2c75945de58defcfae23cb30a9003695fbcd2f3d Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 19 Aug 2017 21:16:30 -0700
Subject: [PATCH 0500/2196] Remove nix-mode.el from Nix.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This removes the file nix-mode.el from Nix. The file is now available within the
repository https://github.com/NixOS/nix-mode.

Fixes #662
Fixes #1040
Fixes #1054
Fixes #1055
Closes #1119
Fixes #1419

NOTE: all of the above should be fixed within NixOS/nix-mode. If one of those
hasn’t please reopen within NixOS/nix-mode and not within NixOS/nix.
---
 Makefile                             |   1 -
 doc/manual/release-notes/rl-1.12.xml |   6 +
 misc/emacs/README                    |  10 --
 misc/emacs/local.mk                  |   1 -
 misc/emacs/nix-mode.el               | 177 ---------------------------
 nix.spec.in                          |  39 ------
 release.nix                          |   2 +-
 7 files changed, 7 insertions(+), 229 deletions(-)
 delete mode 100644 misc/emacs/README
 delete mode 100644 misc/emacs/local.mk
 delete mode 100644 misc/emacs/nix-mode.el

diff --git a/Makefile b/Makefile
index 40ac4e72dbc..5d8e990cc5c 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,6 @@ makefiles = \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
   misc/upstart/local.mk \
-  misc/emacs/local.mk \
   doc/manual/local.mk \
   tests/local.mk
 
diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 551139f3a66..d9bdd9eddd4 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -387,6 +387,12 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     (b8867a0239b1930a16f9ef3f7f3e864b01416dff))</para>
   </listitem>
 
+  <listitem>
+    <para><filename>nix-mode.el</filename> has been removed from Nix. It is now a separate repository
+    in <uri>https://github.com/NixOS/nix-mode</uri> and can be installed through the MELPA
+    package repository.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>
diff --git a/misc/emacs/README b/misc/emacs/README
deleted file mode 100644
index 8c87f67d571..00000000000
--- a/misc/emacs/README
+++ /dev/null
@@ -1,10 +0,0 @@
-The Nix Emacs mode supports syntax highlighting, somewhat sensible
-indenting, and refilling of comments.
-
-To enable Nix mode in Emacs, add something like this to your ~/.emacs
-file:
-
-  (load "/nix/share/emacs/site-lisp/nix-mode.el")
-
-This automatically causes Nix mode to be activated for all files with
-extension `.nix'.
diff --git a/misc/emacs/local.mk b/misc/emacs/local.mk
deleted file mode 100644
index 8e06b881bcd..00000000000
--- a/misc/emacs/local.mk
+++ /dev/null
@@ -1 +0,0 @@
-$(eval $(call install-data-in,$(d)/nix-mode.el,$(datadir)/emacs/site-lisp))
diff --git a/misc/emacs/nix-mode.el b/misc/emacs/nix-mode.el
deleted file mode 100644
index e129e9efe1d..00000000000
--- a/misc/emacs/nix-mode.el
+++ /dev/null
@@ -1,177 +0,0 @@
-;;; nix-mode.el --- Major mode for editing Nix expressions
-
-;; Author: Eelco Dolstra
-;; URL: https://github.com/NixOS/nix/tree/master/misc/emacs
-;; Version: 1.0
-
-;;; Commentary:
-
-;;; Code:
-
-(defun nix-syntax-match-antiquote (limit)
-  (let ((pos (next-single-char-property-change (point) 'nix-syntax-antiquote
-                                               nil limit)))
-    (when (and pos (> pos (point)))
-      (goto-char pos)
-      (let ((char (char-after pos)))
-        (pcase char
-          (`?$
-           (forward-char 2))
-          (`?}
-           (forward-char 1)))
-        (set-match-data (list pos (point)))
-        t))))
-
-(defconst nix-font-lock-keywords
-  '("\\_<if\\_>" "\\_<then\\_>" "\\_<else\\_>" "\\_<assert\\_>" "\\_<with\\_>"
-    "\\_<let\\_>" "\\_<in\\_>" "\\_<rec\\_>" "\\_<inherit\\_>" "\\_<or\\_>"
-    ("\\_<true\\_>" . font-lock-builtin-face)
-    ("\\_<false\\_>" . font-lock-builtin-face)
-    ("\\_<null\\_>" . font-lock-builtin-face)
-    ("\\_<import\\_>" . font-lock-builtin-face)
-    ("\\_<derivation\\_>" . font-lock-builtin-face)
-    ("\\_<baseNameOf\\_>" . font-lock-builtin-face)
-    ("\\_<toString\\_>" . font-lock-builtin-face)
-    ("\\_<isNull\\_>" . font-lock-builtin-face)
-    ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+"
-     . font-lock-constant-face)
-    ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\-\.]*\\)[ \t]*="
-     (1 font-lock-variable-name-face nil nil))
-    ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>"
-     . font-lock-constant-face)
-    ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+"
-     . font-lock-constant-face)
-    (nix-syntax-match-antiquote 0 font-lock-preprocessor-face t))
-  "Font lock keywords for nix.")
-
-(defvar nix-mode-syntax-table
-  (let ((table (make-syntax-table)))
-    (modify-syntax-entry ?/ ". 14" table)
-    (modify-syntax-entry ?* ". 23" table)
-    (modify-syntax-entry ?# "< b" table)
-    (modify-syntax-entry ?\n "> b" table)
-    table)
-  "Syntax table for Nix mode.")
-
-(defun nix-syntax-propertize-escaped-antiquote ()
-  "Set syntax properies for escaped antiquote marks."
-  nil)
-
-(defun nix-syntax-propertize-multiline-string ()
-  "Set syntax properies for multiline string delimiters."
-  (let* ((start (match-beginning 0))
-         (end (match-end 0))
-         (context (save-excursion (save-match-data (syntax-ppss start))))
-         (string-type (nth 3 context)))
-    (pcase string-type
-      (`t
-       ;; inside a multiline string
-       ;; ending multi-line string delimiter
-       (put-text-property (1- end) end
-                          'syntax-table (string-to-syntax "|")))
-      (`nil
-       ;; beginning multi-line string delimiter
-       (put-text-property start (1+ start)
-                          'syntax-table (string-to-syntax "|"))))))
-
-(defun nix-syntax-propertize-antiquote ()
-  "Set syntax properties for antiquote marks."
-  (let* ((start (match-beginning 0)))
-    (put-text-property start (1+ start)
-                       'syntax-table (string-to-syntax "|"))
-    (put-text-property start (+ start 2)
-                       'nix-syntax-antiquote t)))
-
-(defun nix-syntax-propertize-close-brace ()
-  "Set syntax properties for close braces.
-If a close brace `}' ends an antiquote, the next character begins a string."
-  (let* ((start (match-beginning 0))
-         (end (match-end 0))
-         (context (save-excursion (save-match-data (syntax-ppss start))))
-         (open (nth 1 context)))
-    (when open ;; a corresponding open-brace was found
-      (let* ((antiquote (get-text-property open 'nix-syntax-antiquote)))
-        (when antiquote
-          (put-text-property (+ start 1) (+ start 2)
-                             'syntax-table (string-to-syntax "|"))
-          (put-text-property start (1+ start)
-                             'nix-syntax-antiquote t))))))
-
-(defun nix-syntax-propertize (start end)
-  "Special syntax properties for Nix."
-  ;; search for multi-line string delimiters
-  (goto-char start)
-  (remove-text-properties start end '(syntax-table nil nix-syntax-antiquote nil))
-  (funcall
-   (syntax-propertize-rules
-    ("''\\${"
-     (0 (ignore (nix-syntax-propertize-escaped-antiquote))))
-    ("''"
-     (0 (ignore (nix-syntax-propertize-multiline-string))))
-    ("\\${"
-     (0 (ignore (nix-syntax-propertize-antiquote))))
-    ("}"
-     (0 (ignore (nix-syntax-propertize-close-brace)))))
-   start end))
-
-(defun nix-indent-line ()
-  "Indent current line in a Nix expression."
-  (interactive)
-  (indent-relative-maybe))
-
-
-;;;###autoload
-(define-derived-mode nix-mode prog-mode "Nix"
-  "Major mode for editing Nix expressions.
-
-The following commands may be useful:
-
-  '\\[newline-and-indent]'
-    Insert a newline and move the cursor to align with the previous
-    non-empty line.
-
-  '\\[fill-paragraph]'
-    Refill a paragraph so that all lines are at most `fill-column'
-    lines long.  This should do the right thing for comments beginning
-    with `#'.  However, this command doesn't work properly yet if the
-    comment is adjacent to code (i.e., no intervening empty lines).
-    In that case, select the text to be refilled and use
-    `\\[fill-region]' instead.
-
-The hook `nix-mode-hook' is run when Nix mode is started.
-
-\\{nix-mode-map}
-"
-  (set-syntax-table nix-mode-syntax-table)
-
-  ;; Font lock support.
-  (setq-local font-lock-defaults '(nix-font-lock-keywords nil nil nil nil))
-
-  ;; Special syntax properties for Nix
-  (setq-local syntax-propertize-function 'nix-syntax-propertize)
-
-  ;; Look at text properties when parsing
-  (setq-local parse-sexp-lookup-properties t)
-
-  ;; Automatic indentation [C-j].
-  (set (make-local-variable 'indent-line-function) 'nix-indent-line)
-
-  ;; Indenting of comments.
-  (set (make-local-variable 'comment-start) "# ")
-  (set (make-local-variable 'comment-end) "")
-  (set (make-local-variable 'comment-start-skip) "\\(^\\|\\s-\\);?#+ *")
-
-  ;; Filling of comments.
-  (set (make-local-variable 'adaptive-fill-mode) t)
-  (set (make-local-variable 'paragraph-start) "[ \t]*\\(#+[ \t]*\\)?$")
-  (set (make-local-variable 'paragraph-separate) paragraph-start))
-
-
-;;;###autoload
-(progn
-  (add-to-list 'auto-mode-alist '("\\.nix\\'" . nix-mode))
-  (add-to-list 'auto-mode-alist '("\\.nix.in\\'" . nix-mode)))
-
-(provide 'nix-mode)
-
-;;; nix-mode.el ends here
diff --git a/nix.spec.in b/nix.spec.in
index 88f0d04d390..241e28f3678 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -60,33 +60,6 @@ Requires:       %{name} = %{version}-%{release}
 %description   doc
 The %{name}-doc package contains documentation files for %{name}.
 
-
-%package -n emacs-%{name}
-Summary:        Nix mode for Emacs
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Applications/Editors
-%endif
-BuildArch:      noarch
-BuildRequires:  emacs
-Requires:       emacs(bin) >= %{_emacs_version}
-
-%description -n emacs-%{name}
-This package provides a major mode for editing Nix expressions.
-
-%package -n emacs-%{name}-el
-Summary:        Elisp source files for emacs-%{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Applications/Editors
-%endif
-BuildArch:      noarch
-Requires:       emacs-%{name} = %{version}-%{release}
-
-%description -n emacs-%{name}-el
-This package contains the elisp source file for the Nix major mode for
-GNU Emacs. You do not need to install this package to run Nix. Install
-the emacs-%{name} package to edit Nix expressions with GNU Emacs.
-
-
 %prep
 %setup -q
 
@@ -100,7 +73,6 @@ extraFlags=
            --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
            $extraFlags
 make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
-%{_emacs_bytecompile} misc/emacs/nix-mode.el
 
 
 %install
@@ -126,9 +98,6 @@ done
 # (until this is fixed in the relevant Makefile)
 chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh
 
-# Copy the byte-compiled mode file by hand
-cp -p misc/emacs/nix-mode.elc $RPM_BUILD_ROOT%{_emacs_sitelispdir}/
-
 # we ship this file in the base package
 rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README
 
@@ -166,7 +135,6 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
-%{_datadir}/emacs/site-lisp/nix-mode.el
 %{_datadir}/nix
 %{_mandir}/man1/*.1*
 %{_mandir}/man5/*.5*
@@ -181,10 +149,3 @@ systemctl start  nix-daemon.socket
 %files doc
 %docdir %{_defaultdocdir}/%{name}-doc-%{version}
 %{_defaultdocdir}/%{name}-doc-%{version}
-
-%files -n emacs-%{name}
-%{_emacs_sitelispdir}/*.elc
-#{_emacs_sitestartdir}/*.el
-
-%files -n emacs-%{name}-el
-%{_emacs_sitelispdir}/*.el
diff --git a/release.nix b/release.nix
index 796591cc167..834269ff7dc 100644
--- a/release.nix
+++ b/release.nix
@@ -310,7 +310,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;

From 9400cb36b7b7764e742f94fbe7f4fd8082fa1ae3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Aug 2017 12:00:41 +0200
Subject: [PATCH 0501/2196] Disallow accidental copy construction

---
 src/libutil/logging.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 1e0c735676b..08a11617591 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -81,6 +81,8 @@ struct Activity
 
     Activity(Logger & logger, ActivityType type, const std::string & s = "");
 
+    Activity(const Activity & act) = delete;
+
     ~Activity()
     { logger.stopActivity(id); }
 

From 4c6a26539c8c5d1163c1a6a6d22caf818376b36a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Aug 2017 12:01:05 +0200
Subject: [PATCH 0502/2196] Remove debug line

---
 src/nix/progress-bar.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 1616c420551..f0db3be6026 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -8,8 +8,6 @@
 
 #include <sys/ioctl.h>
 
-#include <iostream>
-
 namespace nix {
 
 static std::string getS(const std::vector<Logger::Field> & fields, size_t n)

From 4af2611bd105f365e568069463b1700d141dd0a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Aug 2017 12:01:21 +0200
Subject: [PATCH 0503/2196] Allow builders to create activities

Actually, currently they can only create download activities. Thus,
downloads by builtins.fetchurl show up in the progress bar.
---
 local.mk              |  2 +-
 release.nix           |  2 ++
 shell.nix             |  1 +
 src/libstore/build.cc | 78 +++++++++++++++++++++++++++++++++++++++----
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/local.mk b/local.mk
index 0a225423741..5c80451bd57 100644
--- a/local.mk
+++ b/local.mk
@@ -3,7 +3,7 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
 endif
 
-dist-files += configure config.h.in nix.spec perl/configure
+dist-files += configure config.h.in nix.spec perl/configure src/nlohmann/json.hpp
 
 clean-files += Makefile.config
 
diff --git a/release.nix b/release.nix
index 834269ff7dc..40df200fdc5 100644
--- a/release.nix
+++ b/release.nix
@@ -43,6 +43,8 @@ let
           (cd perl ; autoreconf --install --force --verbose)
           # TeX needs a writable font cache.
           export VARTEXFONTS=$TMPDIR/texfonts
+
+          cp -rv ${nlohmann_json}/include/nlohmann src/nlohmann
         '';
 
         distPhase =
diff --git a/shell.nix b/shell.nix
index 435399f0b25..2b4d0dd201e 100644
--- a/shell.nix
+++ b/shell.nix
@@ -18,6 +18,7 @@ with import ./release-common.nix { inherit pkgs; };
         customMemoryManagement = false;
       })
       autoreconfHook
+      nlohmann_json
 
       # For nix-perl
       perl
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index fc039a8e52b..3663c33b012 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -55,6 +55,8 @@
 #include <sys/statvfs.h>
 #endif
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -839,6 +841,8 @@ class DerivationGoal : public Goal
 
     std::unique_ptr<Activity> act;
 
+    std::map<ActivityId, Activity> builderActivities;
+
 public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
@@ -2390,6 +2394,34 @@ void setupSeccomp()
 }
 
 
+struct BuilderLogger : Logger
+{
+    Logger & prevLogger;
+
+    BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
+
+    void log(Verbosity lvl, const FormatOrString & fs) override
+    {
+        prevLogger.log(lvl, fs);
+    }
+
+    void startActivity(ActivityId act, ActivityType type, const std::string & s) override
+    {
+        log(lvlError, fmt("@nix {\"action\": \"start\", \"id\": %d, \"type\": %d, \"text\": \"%s\"}", act, type, s));
+    }
+
+    void stopActivity(ActivityId act) override
+    {
+        log(lvlError, fmt("@nix {\"action\": \"stop\", \"id\": %d}", act));
+    }
+
+    void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override
+    {
+        log(lvlError, fmt("@nix {\"action\": \"progress\", \"id\": %d, \"done\": %d, \"expected\": %d, \"running\": %d, \"failed\": %d}", act, done, expected, running, failed));
+    }
+};
+
+
 void DerivationGoal::runChild()
 {
     /* Warning: in the child we should absolutely not make any SQLite
@@ -2796,6 +2828,7 @@ void DerivationGoal::runChild()
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
             try {
+                logger = new BuilderLogger(*logger);
                 if (drv->builder == "builtin:fetchurl")
                     builtinFetchurl(*drv, netrcData);
                 else
@@ -3250,14 +3283,47 @@ void DerivationGoal::handleEOF(int fd)
 
 void DerivationGoal::flushLine()
 {
-    if (settings.verboseBuild &&
-        (settings.printRepeatedBuilds || curRound == 1))
-        printError(filterANSIEscapes(currentLogLine, true));
+    if (hasPrefix(currentLogLine, "@nix ")) {
+
+        try {
+            auto json = nlohmann::json::parse(std::string(currentLogLine, 5));
+
+            std::string action = json["action"];
+
+            if (action == "start") {
+                auto type = (ActivityType) json["type"];
+                if (type == actDownload)
+                    builderActivities.emplace(std::piecewise_construct,
+                        std::forward_as_tuple(json["id"]),
+                        std::forward_as_tuple(*logger, type, json["text"]));
+            }
+
+            else if (action == "stop")
+                builderActivities.erase((ActivityId) json["id"]);
+
+            else if (action == "progress") {
+                auto i = builderActivities.find((ActivityId) json["id"]);
+                if (i != builderActivities.end())
+                    i->second.progress(json.value("done", 0), json.value("expected", 0), json.value("running", 0), json.value("failed", 0));
+            }
+
+        } catch (std::exception & e) {
+            printError("bad log message from builder: %s", e.what());
+        }
+    }
+
     else {
-        logTail.push_back(currentLogLine);
-        if (logTail.size() > settings.logLines) logTail.pop_front();
+        if (settings.verboseBuild &&
+            (settings.printRepeatedBuilds || curRound == 1))
+            printError(filterANSIEscapes(currentLogLine, true));
+        else {
+            logTail.push_back(currentLogLine);
+            if (logTail.size() > settings.logLines) logTail.pop_front();
+        }
+
+        act->result(resBuildLogLine, currentLogLine);
     }
-    act->result(resBuildLogLine, currentLogLine);
+
     currentLogLine = "";
     currentLogLinePos = 0;
 }

From 1f56235438984d8079159d7c81ad4127c318b2dc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Aug 2017 12:10:04 +0200
Subject: [PATCH 0504/2196] Clean up JSON construction

---
 src/libstore/build.cc | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 3663c33b012..e97fcc9e269 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2407,17 +2407,32 @@ struct BuilderLogger : Logger
 
     void startActivity(ActivityId act, ActivityType type, const std::string & s) override
     {
-        log(lvlError, fmt("@nix {\"action\": \"start\", \"id\": %d, \"type\": %d, \"text\": \"%s\"}", act, type, s));
+        nlohmann::json json;
+        json["action"] = "start";
+        json["id"] = act;
+        json["type"] = type;
+        json["text"] = s;
+        log(lvlError, "@nix " + json.dump());
     }
 
     void stopActivity(ActivityId act) override
     {
-        log(lvlError, fmt("@nix {\"action\": \"stop\", \"id\": %d}", act));
+        nlohmann::json json;
+        json["action"] = "stop";
+        json["id"] = act;
+        log(lvlError, "@nix " + json.dump());
     }
 
     void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override
     {
-        log(lvlError, fmt("@nix {\"action\": \"progress\", \"id\": %d, \"done\": %d, \"expected\": %d, \"running\": %d, \"failed\": %d}", act, done, expected, running, failed));
+        nlohmann::json json;
+        json["action"] = "progress";
+        json["id"] = act;
+        json["done"] = done;
+        json["expected"] = expected;
+        json["running"] = running;
+        json["failed"] = failed;
+        log(lvlError, "@nix " + json.dump());
     }
 };
 

From 0e9ddcc306f0900fc38472a2c8b9d9aa886b279e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 14:53:50 +0200
Subject: [PATCH 0505/2196] Restore activity metadata

This allows the progress bar to display "building perl-5.22.3" instead
of "building /nix/store/<hash>-perl-5.22.3.drv".
---
 src/libstore/build.cc   |  7 +++++--
 src/libutil/logging.cc  |  5 +++--
 src/libutil/logging.hh  | 34 +++++++++++++++++++---------------
 src/nix/progress-bar.cc | 10 +++++++++-
 4 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e97fcc9e269..77dee291467 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1390,7 +1390,8 @@ void DerivationGoal::tryToBuild()
     bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
     auto started = [&]() {
-        act = std::make_unique<Activity>(*logger, actBuild, fmt("building '%s'", drvPath));
+        act = std::make_unique<Activity>(*logger, actBuild,
+            fmt("building '%s'", drvPath), Logger::Fields{drvPath});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
     };
@@ -2405,13 +2406,15 @@ struct BuilderLogger : Logger
         prevLogger.log(lvl, fs);
     }
 
-    void startActivity(ActivityId act, ActivityType type, const std::string & s) override
+    void startActivity(ActivityId act, ActivityType type,
+        const std::string & s, const Fields & fields) override
     {
         nlohmann::json json;
         json["action"] = "start";
         json["id"] = act;
         json["type"] = type;
         json["text"] = s;
+        // FIXME: handle fields
         log(lvlError, "@nix " + json.dump());
     }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 900f24e4cbd..87f20664ef0 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -74,10 +74,11 @@ Logger * makeDefaultLogger()
 
 std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
-Activity::Activity(Logger & logger, ActivityType type, const std::string & s)
+Activity::Activity(Logger & logger, ActivityType type,
+    const std::string & s, const Logger::Fields & fields)
     : logger(logger), id(nextId++)
 {
-    logger.startActivity(id, type, s);
+    logger.startActivity(id, type, s, fields);
 }
 
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 08a11617591..567af361e8e 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -40,6 +40,19 @@ class Logger
 
 public:
 
+    struct Field
+    {
+        // FIXME: use std::variant.
+        enum { tInt, tString } type;
+        uint64_t i = 0;
+        std::string s;
+        Field(const std::string & s) : type(tString), s(s) { }
+        Field(const char * s) : type(tString), s(s) { }
+        Field(const uint64_t & i) : type(tInt), i(i) { }
+    };
+
+    typedef std::vector<Field> Fields;
+
     virtual ~Logger() { }
 
     virtual void log(Verbosity lvl, const FormatOrString & fs) = 0;
@@ -51,7 +64,8 @@ public:
 
     virtual void warn(const std::string & msg);
 
-    virtual void startActivity(ActivityId act, ActivityType type, const std::string & s) { };
+    virtual void startActivity(ActivityId act, ActivityType type,
+        const std::string & s, const Fields & fields) { };
 
     virtual void stopActivity(ActivityId act) { };
 
@@ -59,18 +73,7 @@ public:
 
     virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
 
-    struct Field
-    {
-        // FIXME: use std::variant.
-        enum { tInt, tString } type;
-        uint64_t i = 0;
-        std::string s;
-        Field(const std::string & s) : type(tString), s(s) { }
-        Field(const char * s) : type(tString), s(s) { }
-        Field(const uint64_t & i) : type(tInt), i(i) { }
-    };
-
-    virtual void result(ActivityId act, ResultType type, const std::vector<Field> & fields) { };
+    virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
 };
 
 struct Activity
@@ -79,7 +82,8 @@ struct Activity
 
     const ActivityId id;
 
-    Activity(Logger & logger, ActivityType type, const std::string & s = "");
+    Activity(Logger & logger, ActivityType type, const std::string & s = "",
+        const Logger::Fields & fields = {});
 
     Activity(const Activity & act) = delete;
 
@@ -95,7 +99,7 @@ struct Activity
     template<typename... Args>
     void result(ResultType type, const Args & ... args)
     {
-        std::vector<Logger::Field> fields;
+        Logger::Fields fields;
         nop{(fields.emplace_back(Logger::Field(args)), 1)...};
         logger.result(id, type, fields);
     }
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index f0db3be6026..a692250f826 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -93,7 +93,8 @@ class ProgressBar : public Logger
         update(state);
     }
 
-    void startActivity(ActivityId act, ActivityType type, const std::string & s) override
+    void startActivity(ActivityId act, ActivityType type, const std::string & s,
+        const Fields & fields) override
     {
         auto state(state_.lock());
 
@@ -102,6 +103,13 @@ class ProgressBar : public Logger
         state->its.emplace(act, i);
         state->activitiesByType[type].its.emplace(act, i);
 
+        if (type == actBuild) {
+            auto name = storePathToName(getS(fields, 0));
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
+        }
+
         update(*state);
     }
 

From a3015db6c388d1b62064e10bac77d4abe2a457ef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 15:44:15 +0200
Subject: [PATCH 0506/2196] Typo

---
 src/libutil/util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index c2bceea3d9d..b7b32b3d384 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1171,7 +1171,7 @@ string filterANSIEscapes(const string & s, bool nixOnly)
             }
         } else {
             r += c;
-            if (c >= 0x40 && c != 0x7e) {
+            if (c >= 0x40 && c <= 0x7e) {
                 if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b'))
                     t += r;
                 state = stTop;

From ec9e0c03c398ca48fba81fd6e870dc396da01b08 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 15:44:36 +0200
Subject: [PATCH 0507/2196] When truncating the progress bar, take ANSI escape
 sequences into account

---
 src/nix/progress-bar.cc | 45 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index a692250f826..cd5a8fca26e 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -24,6 +24,44 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
     return fields[n].i;
 }
 
+/* Truncate a string to 'width' printable characters. ANSI escape
+   sequences are copied but not included in the character count. Also,
+   tabs are expanded to spaces. */
+static std::string ansiTruncate(const std::string & s, int width)
+{
+    if (width <= 0) return s;
+
+    std::string t;
+    size_t w = 0;
+    auto i = s.begin();
+
+    while (w < (size_t) width && i != s.end()) {
+        if (*i == '\e') {
+            t += *i++;
+            if (i != s.end() && *i == '[') {
+                t += *i++;
+                while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
+                    t += *i++;
+                }
+                if (i != s.end()) t += *i++;
+            }
+        }
+
+        else if (*i == '\t') {
+            t += ' '; w++;
+            while (w < (size_t) width && w & 8) {
+                t += ' '; w++;
+            }
+        }
+
+        else {
+            t += *i++; w++;
+        }
+    }
+
+    return t;
+}
+
 class ProgressBar : public Logger
 {
 private:
@@ -89,7 +127,7 @@ class ProgressBar : public Logger
 
     void log(State & state, Verbosity lvl, const std::string & s)
     {
-        writeToStderr("\r\e[K" + s + "\n");
+        writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
         update(state);
     }
 
@@ -207,7 +245,7 @@ class ProgressBar : public Logger
 
     void update(State & state)
     {
-        std::string line = "\r";
+        std::string line;
 
         std::string status = getStatus(state);
         if (!status.empty()) {
@@ -232,8 +270,7 @@ class ProgressBar : public Logger
             }
         }
 
-        line += "\e[K";
-        writeToStderr(std::string(line, 0, width - 1));
+        writeToStderr("\r" + ansiTruncate(line, width) + "\e[K");
     }
 
     std::string getStatus(State & state)

From db1d45037cf923bf470a8581d5546bef8247c756 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 15:57:49 +0200
Subject: [PATCH 0508/2196] Handle SIGWINCH

---
 src/libutil/util.cc     | 28 ++++++++++++++++++++++++++++
 src/libutil/util.hh     | 10 ++++++++++
 src/nix/progress-bar.cc |  9 +--------
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index b7b32b3d384..2f410f7e3fd 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -17,6 +17,7 @@
 #include <fcntl.h>
 #include <limits.h>
 #include <pwd.h>
+#include <sys/ioctl.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
@@ -1254,6 +1255,26 @@ void callFailure(const std::function<void(std::exception_ptr exc)> & failure, st
 }
 
 
+static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
+
+
+static void updateWindowSize()
+{
+    struct winsize ws;
+    if (ioctl(1, TIOCGWINSZ, &ws) == 0) {
+        auto windowSize_(windowSize.lock());
+        windowSize_->first = ws.ws_row;
+        windowSize_->second = ws.ws_col;
+    }
+}
+
+
+std::pair<unsigned short, unsigned short> getWindowSize()
+{
+    return *windowSize.lock();
+}
+
+
 static Sync<std::list<std::function<void()>>> _interruptCallbacks;
 
 static void signalHandlerThread(sigset_t set)
@@ -1264,6 +1285,10 @@ static void signalHandlerThread(sigset_t set)
 
         if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
             triggerInterrupt();
+
+        else if (signal == SIGWINCH) {
+            updateWindowSize();
+        }
     }
 }
 
@@ -1287,6 +1312,8 @@ static sigset_t savedSignalMask;
 
 void startSignalHandlerThread()
 {
+    updateWindowSize();
+
     if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
         throw SysError("quering signal mask");
 
@@ -1296,6 +1323,7 @@ void startSignalHandlerThread()
     sigaddset(&set, SIGTERM);
     sigaddset(&set, SIGHUP);
     sigaddset(&set, SIGPIPE);
+    sigaddset(&set, SIGWINCH);
     if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
         throw SysError("blocking signals");
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 7340ca6c747..30e3c0df1c1 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -349,6 +349,12 @@ bool hasSuffix(const string & s, const string & suffix);
 std::string toLower(const std::string & s);
 
 
+/* Escape a string that contains octal-encoded escape codes such as
+   used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to
+   "foo bar"). */
+string decodeOctalEscaped(const string & s);
+
+
 /* Exception handling in destructors: print an error message, then
    ignore the exception. */
 void ignoreException();
@@ -470,4 +476,8 @@ struct MaintainCount
 };
 
 
+/* Return the number of rows and columns of the terminal. */
+std::pair<unsigned short, unsigned short> getWindowSize();
+
+
 }
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index cd5a8fca26e..2ad0994508f 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -6,8 +6,6 @@
 #include <map>
 #include <atomic>
 
-#include <sys/ioctl.h>
-
 namespace nix {
 
 static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
@@ -99,15 +97,10 @@ class ProgressBar : public Logger
 
     Sync<State> state_;
 
-    int width = 0;
-
 public:
 
     ProgressBar()
     {
-        struct winsize ws;
-        if (ioctl(1, TIOCGWINSZ, &ws) == 0)
-            width = ws.ws_col;
     }
 
     ~ProgressBar()
@@ -270,7 +263,7 @@ class ProgressBar : public Logger
             }
         }
 
-        writeToStderr("\r" + ansiTruncate(line, width) + "\e[K");
+        writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K");
     }
 
     std::string getStatus(State & state)

From f194629f96d4bdbded897e7e88ac0d03211c959d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 16:11:18 +0200
Subject: [PATCH 0509/2196] Fix Debian build

https://hydra.nixos.org/build/59390148
---
 src/nix/progress-bar.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 2ad0994508f..17d7109e716 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -129,8 +129,10 @@ class ProgressBar : public Logger
     {
         auto state(state_.lock());
 
-        state->activities.emplace_back(ActInfo{s, "", type});
+        state->activities.emplace_back(ActInfo());
         auto i = std::prev(state->activities.end());
+        i->s = s;
+        i->type = type;
         state->its.emplace(act, i);
         state->activitiesByType[type].its.emplace(act, i);
 

From c137c0a5ebc0d58c53f86986ab66967ac8629cbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 17:49:40 +0200
Subject: [PATCH 0510/2196] Allow activities to be nested

In particular, this allows more relevant activities ("substituting X")
to supersede inferior ones ("downloading X").
---
 src/libstore/build.cc     |  9 ++++++---
 src/libstore/download.cc  |  2 +-
 src/libstore/download.hh  |  4 +++-
 src/libstore/store-api.cc |  4 +++-
 src/libutil/logging.cc    |  6 ++++--
 src/libutil/logging.hh    | 14 ++++++++++++--
 src/nix/progress-bar.cc   | 28 ++++++++++++++++++++++++++--
 7 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 77dee291467..cb67d7a6c14 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2407,14 +2407,14 @@ struct BuilderLogger : Logger
     }
 
     void startActivity(ActivityId act, ActivityType type,
-        const std::string & s, const Fields & fields) override
+        const std::string & s, const Fields & fields, ActivityId parent) override
     {
         nlohmann::json json;
         json["action"] = "start";
         json["id"] = act;
         json["type"] = type;
         json["text"] = s;
-        // FIXME: handle fields
+        // FIXME: handle fields, parent
         log(lvlError, "@nix " + json.dump());
     }
 
@@ -3313,7 +3313,7 @@ void DerivationGoal::flushLine()
                 if (type == actDownload)
                     builderActivities.emplace(std::piecewise_construct,
                         std::forward_as_tuple(json["id"]),
-                        std::forward_as_tuple(*logger, type, json["text"]));
+                        std::forward_as_tuple(*logger, type, json["text"], Logger::Fields{}, act->id));
             }
 
             else if (action == "stop")
@@ -3655,6 +3655,9 @@ void SubstitutionGoal::tryToRun()
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide = -1; });
 
+            Activity act(*logger, actSubstitute, "", Logger::Fields{storePath, sub->getUri()});
+            PushActivity pact(act.id);
+
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
                 storePath, repair);
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 71b720a3b59..a2099917677 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -85,7 +85,7 @@ struct CurlDownloader : public Downloader
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader)
             , request(request)
-            , act(*logger, actDownload, fmt("downloading '%s'", request.uri))
+            , act(*logger, actDownload, fmt("downloading '%s'", request.uri), {}, request.parentAct)
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 7d8982d64c4..752bf3723cf 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -16,8 +16,10 @@ struct DownloadRequest
     bool head = false;
     size_t tries = 5;
     unsigned int baseRetryTimeMs = 250;
+    ActivityId parentAct;
 
-    DownloadRequest(const std::string & uri) : uri(uri) { }
+    DownloadRequest(const std::string & uri)
+        : uri(uri), parentAct(curActivity) { }
 };
 
 struct DownloadResult
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f520210615a..f0737685271 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -565,7 +565,9 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    Activity act(*logger, actCopyPath, fmt("copying path '%s'", storePath));
+    Activity act(*logger, actCopyPath, fmt("copying path '%s'", storePath),
+        {storePath, srcStore->getUri(), dstStore->getUri()});
+    PushActivity pact(act.id);
 
     auto info = srcStore->queryPathInfo(storePath);
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 87f20664ef0..b103b902eab 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -5,6 +5,8 @@
 
 namespace nix {
 
+thread_local ActivityId curActivity = 0;
+
 Logger * logger = makeDefaultLogger();
 
 void Logger::warn(const std::string & msg)
@@ -75,10 +77,10 @@ Logger * makeDefaultLogger()
 std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
 Activity::Activity(Logger & logger, ActivityType type,
-    const std::string & s, const Logger::Fields & fields)
+    const std::string & s, const Logger::Fields & fields, ActivityId parent)
     : logger(logger), id(nextId++)
 {
-    logger.startActivity(id, type, s, fields);
+    logger.startActivity(id, type, s, fields, parent);
 }
 
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 567af361e8e..9427f268267 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -23,6 +23,7 @@ typedef enum {
     actBuild = 105,
     actOptimiseStore = 106,
     actVerifyPaths = 107,
+    actSubstitute = 108,
 } ActivityType;
 
 typedef enum {
@@ -65,7 +66,7 @@ public:
     virtual void warn(const std::string & msg);
 
     virtual void startActivity(ActivityId act, ActivityType type,
-        const std::string & s, const Fields & fields) { };
+        const std::string & s, const Fields & fields, ActivityId parent) { };
 
     virtual void stopActivity(ActivityId act) { };
 
@@ -76,6 +77,8 @@ public:
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
 };
 
+extern thread_local ActivityId curActivity;
+
 struct Activity
 {
     Logger & logger;
@@ -83,7 +86,7 @@ struct Activity
     const ActivityId id;
 
     Activity(Logger & logger, ActivityType type, const std::string & s = "",
-        const Logger::Fields & fields = {});
+        const Logger::Fields & fields = {}, ActivityId parent = curActivity);
 
     Activity(const Activity & act) = delete;
 
@@ -107,6 +110,13 @@ struct Activity
     friend class Logger;
 };
 
+struct PushActivity
+{
+    const ActivityId prevAct;
+    PushActivity(ActivityId act) : prevAct(curActivity) { curActivity = act; }
+    ~PushActivity() { curActivity = prevAct; }
+};
+
 extern Logger * logger;
 
 Logger * makeDefaultLogger();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 17d7109e716..1b597433b30 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -73,6 +73,8 @@ class ProgressBar : public Logger
         uint64_t running = 0;
         uint64_t failed = 0;
         std::map<ActivityType, uint64_t> expectedByType;
+        bool visible = true;
+        ActivityId parent;
     };
 
     struct ActivitiesByType
@@ -125,7 +127,7 @@ class ProgressBar : public Logger
     }
 
     void startActivity(ActivityId act, ActivityType type, const std::string & s,
-        const Fields & fields) override
+        const Fields & fields, ActivityId parent) override
     {
         auto state(state_.lock());
 
@@ -133,6 +135,7 @@ class ProgressBar : public Logger
         auto i = std::prev(state->activities.end());
         i->s = s;
         i->type = type;
+        i->parent = parent;
         state->its.emplace(act, i);
         state->activitiesByType[type].its.emplace(act, i);
 
@@ -143,9 +146,30 @@ class ProgressBar : public Logger
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
         }
 
+        if (type == actSubstitute) {
+            auto name = storePathToName(getS(fields, 0));
+            i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1));
+        }
+
+        if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))
+            || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
+            i->visible = false;
+
         update(*state);
     }
 
+    /* Check whether an activity has an ancestore with the specified
+       type. */
+    bool hasAncestor(State & state, ActivityType type, ActivityId act)
+    {
+        while (act != 0) {
+            auto i = state.its.find(act);
+            if (i == state.its.end()) break;
+            if (i->second->type == type) return true;
+        }
+        return false;
+    }
+
     void stopActivity(ActivityId act) override
     {
         auto state(state_.lock());
@@ -253,7 +277,7 @@ class ProgressBar : public Logger
             if (!status.empty()) line += " ";
             auto i = state.activities.rbegin();
 
-            while (i != state.activities.rend() && i->s.empty() && i->s2.empty())
+            while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->s2.empty())))
                 ++i;
 
             if (i != state.activities.rend()) {

From 0ac35b67b8035ad1166df3fb199590bc346d71f2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 18:04:05 +0200
Subject: [PATCH 0511/2196] Allow derivations to update the build phase

So the progress bar can show

  [1/0/1 built, 0.0 MiB DL] building hello-2.10 (configuring): checking whether pread is declared without a macro... yes
---
 src/libstore/build.cc   |  5 +++++
 src/libutil/logging.hh  |  1 +
 src/nix/progress-bar.cc | 25 ++++++++++++++++++-------
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cb67d7a6c14..59013c0349b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3325,6 +3325,11 @@ void DerivationGoal::flushLine()
                     i->second.progress(json.value("done", 0), json.value("expected", 0), json.value("running", 0), json.value("failed", 0));
             }
 
+            else if (action == "setPhase") {
+                std::string phase = json["phase"];
+                act->result(resSetPhase, phase);
+            }
+
         } catch (std::exception & e) {
             printError("bad log message from builder: %s", e.what());
         }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 9427f268267..57157dce15c 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -31,6 +31,7 @@ typedef enum {
     resBuildLogLine = 101,
     resUntrustedPath = 102,
     resCorruptedPath = 103,
+    resSetPhase = 104,
 } ResultType;
 
 typedef uint64_t ActivityId;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 1b597433b30..90e54ed48a2 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -66,7 +66,7 @@ class ProgressBar : public Logger
 
     struct ActInfo
     {
-        std::string s, s2;
+        std::string s, lastLine, phase;
         ActivityType type = actUnknown;
         uint64_t done = 0;
         uint64_t expected = 0;
@@ -232,13 +232,13 @@ class ProgressBar : public Logger
         }
 
         else if (type == resBuildLogLine) {
-            auto s2 = trim(getS(fields, 0));
-            if (!s2.empty()) {
+            auto lastLine = trim(getS(fields, 0));
+            if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
                 state->activities.erase(i->second);
-                info.s2 = s2;
+                info.lastLine = lastLine;
                 state->activities.emplace_back(info);
                 i->second = std::prev(state->activities.end());
                 update(*state);
@@ -254,6 +254,12 @@ class ProgressBar : public Logger
             state->corruptedPaths++;
             update(*state);
         }
+
+        else if (type == resSetPhase) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            i->second->phase = getS(fields, 0);
+        }
     }
 
     void update()
@@ -277,14 +283,19 @@ class ProgressBar : public Logger
             if (!status.empty()) line += " ";
             auto i = state.activities.rbegin();
 
-            while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->s2.empty())))
+            while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty())))
                 ++i;
 
             if (i != state.activities.rend()) {
                 line += i->s;
-                if (!i->s2.empty()) {
+                if (!i->phase.empty()) {
+                    line += " (";
+                    line += i->phase;
+                    line += ")";
+                }
+                if (!i->lastLine.empty()) {
                     if (!i->s.empty()) line += ": ";
-                    line += i->s2;
+                    line += i->lastLine;
                 }
             }
         }

From 9b845e6936929711fe03d23adc1059acdc84a4fd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 20:52:34 +0200
Subject: [PATCH 0512/2196] Doh

---
 src/nix/progress-bar.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 90e54ed48a2..f638c7cfb19 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -166,6 +166,7 @@ class ProgressBar : public Logger
             auto i = state.its.find(act);
             if (i == state.its.end()) break;
             if (i->second->type == type) return true;
+            act = i->second->parent;
         }
         return false;
     }

From 94a0548dc420e23c64b18d36ea8d50a2afbce6fb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 25 Aug 2017 21:26:37 +0200
Subject: [PATCH 0513/2196] Simplify

---
 src/libstore/build.cc   | 46 ++++++++++++++++++++++++++++++++---------
 src/libutil/logging.hh  | 12 +++++++----
 src/nix/progress-bar.cc | 27 +++++++++++-------------
 3 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 59013c0349b..92626738ab2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2401,6 +2401,19 @@ struct BuilderLogger : Logger
 
     BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
 
+    void addFields(nlohmann::json & json, const Fields & fields)
+    {
+        if (fields.empty()) return;
+        auto & arr = json["fields"] = nlohmann::json::array();
+        for (auto & f : fields)
+            if (f.type == Logger::Field::tInt)
+                arr.push_back(f.i);
+            else if (f.type == Logger::Field::tString)
+                arr.push_back(f.s);
+            else
+                abort();
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         prevLogger.log(lvl, fs);
@@ -2414,7 +2427,8 @@ struct BuilderLogger : Logger
         json["id"] = act;
         json["type"] = type;
         json["text"] = s;
-        // FIXME: handle fields, parent
+        addFields(json, fields);
+        // FIXME: handle parent
         log(lvlError, "@nix " + json.dump());
     }
 
@@ -2426,15 +2440,13 @@ struct BuilderLogger : Logger
         log(lvlError, "@nix " + json.dump());
     }
 
-    void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override
+    void result(ActivityId act, ResultType type, const Fields & fields) override
     {
         nlohmann::json json;
-        json["action"] = "progress";
+        json["action"] = "result";
         json["id"] = act;
-        json["done"] = done;
-        json["expected"] = expected;
-        json["running"] = running;
-        json["failed"] = failed;
+        json["type"] = type;
+        addFields(json, fields);
         log(lvlError, "@nix " + json.dump());
     }
 };
@@ -3299,6 +3311,20 @@ void DerivationGoal::handleEOF(int fd)
 }
 
 
+static Logger::Fields getFields(nlohmann::json & json)
+{
+    Logger::Fields fields;
+    for (auto & f : json) {
+        if (f.type() == nlohmann::json::value_t::number_unsigned)
+            fields.emplace_back(Logger::Field(f.get<uint64_t>()));
+        else if (f.type() == nlohmann::json::value_t::string)
+            fields.emplace_back(Logger::Field(f.get<std::string>()));
+        else throw Error("unsupported JSON type %d", (int) f.type());
+    }
+    return fields;
+}
+
+
 void DerivationGoal::flushLine()
 {
     if (hasPrefix(currentLogLine, "@nix ")) {
@@ -3313,16 +3339,16 @@ void DerivationGoal::flushLine()
                 if (type == actDownload)
                     builderActivities.emplace(std::piecewise_construct,
                         std::forward_as_tuple(json["id"]),
-                        std::forward_as_tuple(*logger, type, json["text"], Logger::Fields{}, act->id));
+                        std::forward_as_tuple(*logger, type, json["text"], getFields(json["fields"]), act->id));
             }
 
             else if (action == "stop")
                 builderActivities.erase((ActivityId) json["id"]);
 
-            else if (action == "progress") {
+            else if (action == "result") {
                 auto i = builderActivities.find((ActivityId) json["id"]);
                 if (i != builderActivities.end())
-                    i->second.progress(json.value("done", 0), json.value("expected", 0), json.value("running", 0), json.value("failed", 0));
+                    i->second.result((ResultType) json["type"], getFields(json["fields"]));
             }
 
             else if (action == "setPhase") {
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 57157dce15c..4b1f3b5f70c 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -32,6 +32,7 @@ typedef enum {
     resUntrustedPath = 102,
     resCorruptedPath = 103,
     resSetPhase = 104,
+    resProgress = 105,
 } ResultType;
 
 typedef uint64_t ActivityId;
@@ -71,8 +72,6 @@ public:
 
     virtual void stopActivity(ActivityId act) { };
 
-    virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
-
     virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
 
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
@@ -95,16 +94,21 @@ struct Activity
     { logger.stopActivity(id); }
 
     void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
-    { logger.progress(id, done, expected, running, failed); }
+    { result(resProgress, done, expected, running, failed); }
 
     void setExpected(ActivityType type2, uint64_t expected) const
     { logger.setExpected(id, type2, expected); }
 
     template<typename... Args>
-    void result(ResultType type, const Args & ... args)
+    void result(ResultType type, const Args & ... args) const
     {
         Logger::Fields fields;
         nop{(fields.emplace_back(Logger::Field(args)), 1)...};
+        result(type, fields);
+    }
+
+    void result(ResultType type, const Logger::Fields & fields) const
+    {
         logger.result(id, type, fields);
     }
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index f638c7cfb19..054127f6277 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -192,21 +192,6 @@ class ProgressBar : public Logger
         update(*state);
     }
 
-    void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) override
-    {
-        auto state(state_.lock());
-
-        auto i = state->its.find(act);
-        assert(i != state->its.end());
-        ActInfo & actInfo = *i->second;
-        actInfo.done = done;
-        actInfo.expected = expected;
-        actInfo.running = running;
-        actInfo.failed = failed;
-
-        update(*state);
-    }
-
     void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
     {
         auto state(state_.lock());
@@ -260,6 +245,18 @@ class ProgressBar : public Logger
             auto i = state->its.find(act);
             assert(i != state->its.end());
             i->second->phase = getS(fields, 0);
+            update(*state);
+        }
+
+        else if (type == resProgress) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            actInfo.done = getI(fields, 0);
+            actInfo.expected = getI(fields, 1);
+            actInfo.running = getI(fields, 2);
+            actInfo.failed = getI(fields, 3);
+            update(*state);
         }
     }
 

From 8fff3e7bb50377e48fb9c672e6551abae0fdf03d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Aug 2017 14:17:07 +0200
Subject: [PATCH 0514/2196] Make TunnelLogger thread-safe

Now that we use threads in lots of places, it's possible for
TunnelLogger::log() to be called asynchronously from other threads
than the main loop. So we need to ensure that STDERR_NEXT messages
don't clobber other messages.
---
 src/nix-daemon/nix-daemon.cc | 238 +++++++++++++++++++----------------
 1 file changed, 129 insertions(+), 109 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 7e6f3aa25b5..3237333f4d0 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -8,6 +8,7 @@
 #include "globals.hh"
 #include "monitor-fd.hh"
 #include "derivations.hh"
+#include "finally.hh"
 
 #include <algorithm>
 
@@ -54,57 +55,73 @@ static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t
 static FdSource from(STDIN_FILENO);
 static FdSink to(STDOUT_FILENO);
 
-static bool canSendStderr;
-
-static Logger * defaultLogger;
-
 
 /* Logger that forwards log messages to the client, *if* we're in a
    state where the protocol allows it (i.e., when canSendStderr is
    true). */
-class TunnelLogger : public Logger
+struct TunnelLogger : public Logger
 {
+    struct State
+    {
+        bool canSendStderr = false;
+        std::vector<std::string> pendingMsgs;
+    };
+
+    Sync<State> state_;
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         if (lvl > verbosity) return;
 
-        if (canSendStderr) {
+        auto state(state_.lock());
+
+        if (state->canSendStderr) {
             try {
                 to << STDERR_NEXT << (fs.s + "\n");
                 to.flush();
             } catch (...) {
                 /* Write failed; that means that the other side is
                    gone. */
-                canSendStderr = false;
+                state->canSendStderr = false;
                 throw;
             }
         } else
-            defaultLogger->log(lvl, fs);
+            state->pendingMsgs.push_back(fs.s);
     }
-};
 
+    /* startWork() means that we're starting an operation for which we
+      want to send out stderr to the client. */
+    void startWork()
+    {
+        std::vector<std::string> pendingMsgs;
 
-/* startWork() means that we're starting an operation for which we
-   want to send out stderr to the client. */
-static void startWork()
-{
-    canSendStderr = true;
-}
+        auto state(state_.lock());
+        state->canSendStderr = true;
 
+        for (auto & msg : state->pendingMsgs)
+            to << STDERR_NEXT << (msg + "\n");
 
-/* stopWork() means that we're done; stop sending stderr to the
-   client. */
-static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
-{
-    canSendStderr = false;
+        state->pendingMsgs.clear();
 
-    if (success)
-        to << STDERR_LAST;
-    else {
-        to << STDERR_ERROR << msg;
-        if (status != 0) to << status;
+        to.flush();
     }
-}
+
+    /* stopWork() means that we're done; stop sending stderr to the
+       client. */
+    void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
+    {
+        auto state(state_.lock());
+
+        state->canSendStderr = false;
+
+        if (success)
+            to << STDERR_LAST;
+        else {
+            to << STDERR_ERROR << msg;
+            if (status != 0) to << status;
+        }
+    }
+};
 
 
 struct TunnelSink : Sink
@@ -160,7 +177,8 @@ struct RetrieveRegularNARSink : ParseSink
 };
 
 
-static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVersion,
+static void performOp(TunnelLogger * logger, ref<LocalStore> store,
+    bool trusted, unsigned int clientVersion,
     Source & from, Sink & to, unsigned int op)
 {
     switch (op) {
@@ -172,46 +190,46 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
            that the 'Error' exception handler doesn't close the
            connection.  */
         Path path = readString(from);
-        startWork();
+        logger->startWork();
         store->assertStorePath(path);
         bool result = store->isValidPath(path);
-        stopWork();
+        logger->stopWork();
         to << result;
         break;
     }
 
     case wopQueryValidPaths: {
         PathSet paths = readStorePaths<PathSet>(*store, from);
-        startWork();
+        logger->startWork();
         PathSet res = store->queryValidPaths(paths);
-        stopWork();
+        logger->stopWork();
         to << res;
         break;
     }
 
     case wopHasSubstitutes: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         PathSet res = store->querySubstitutablePaths({path});
-        stopWork();
+        logger->stopWork();
         to << (res.find(path) != res.end());
         break;
     }
 
     case wopQuerySubstitutablePaths: {
         PathSet paths = readStorePaths<PathSet>(*store, from);
-        startWork();
+        logger->startWork();
         PathSet res = store->querySubstitutablePaths(paths);
-        stopWork();
+        logger->stopWork();
         to << res;
         break;
     }
 
     case wopQueryPathHash: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         auto hash = store->queryPathInfo(path)->narHash;
-        stopWork();
+        logger->stopWork();
         to << hash.to_string(Base16, false);
         break;
     }
@@ -221,7 +239,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopQueryValidDerivers:
     case wopQueryDerivationOutputs: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         PathSet paths;
         if (op == wopQueryReferences)
             paths = store->queryPathInfo(path)->references;
@@ -230,35 +248,35 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         else if (op == wopQueryValidDerivers)
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
-        stopWork();
+        logger->stopWork();
         to << paths;
         break;
     }
 
     case wopQueryDerivationOutputNames: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         StringSet names;
         names = store->queryDerivationOutputNames(path);
-        stopWork();
+        logger->stopWork();
         to << names;
         break;
     }
 
     case wopQueryDeriver: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         auto deriver = store->queryPathInfo(path)->deriver;
-        stopWork();
+        logger->stopWork();
         to << deriver;
         break;
     }
 
     case wopQueryPathFromHashPart: {
         string hashPart = readString(from);
-        startWork();
+        logger->startWork();
         Path path = store->queryPathFromHashPart(hashPart);
-        stopWork();
+        logger->stopWork();
         to << path;
         break;
     }
@@ -286,10 +304,10 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         } else
             parseDump(savedRegular, from);
 
-        startWork();
+        logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
         Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
-        stopWork();
+        logger->stopWork();
 
         to << path;
         break;
@@ -299,9 +317,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         string suffix = readString(from);
         string s = readString(from);
         PathSet refs = readStorePaths<PathSet>(*store, from);
-        startWork();
+        logger->startWork();
         Path path = store->addTextToStore(suffix, s, refs, NoRepair);
-        stopWork();
+        logger->stopWork();
         to << path;
         break;
     }
@@ -309,20 +327,20 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopExportPath: {
         Path path = readStorePath(*store, from);
         readInt(from); // obsolete
-        startWork();
+        logger->startWork();
         TunnelSink sink(to);
         store->exportPath(path, sink);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopImportPaths: {
-        startWork();
+        logger->startWork();
         TunnelSource source(from);
         Paths paths = store->importPaths(source, nullptr,
             trusted ? NoCheckSigs : CheckSigs);
-        stopWork();
+        logger->stopWork();
         to << paths;
         break;
     }
@@ -338,9 +356,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
             if (mode == bmRepair && !trusted)
                 throw Error("repairing is not supported when building through the Nix daemon");
         }
-        startWork();
+        logger->startWork();
         store->buildPaths(drvs, mode);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
@@ -350,54 +368,54 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         BasicDerivation drv;
         readDerivation(from, *store, drv);
         BuildMode buildMode = (BuildMode) readInt(from);
-        startWork();
+        logger->startWork();
         if (!trusted)
             throw Error("you are not privileged to build derivations");
         auto res = store->buildDerivation(drvPath, drv, buildMode);
-        stopWork();
+        logger->stopWork();
         to << res.status << res.errorMsg;
         break;
     }
 
     case wopEnsurePath: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         store->ensurePath(path);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopAddTempRoot: {
         Path path = readStorePath(*store, from);
-        startWork();
+        logger->startWork();
         store->addTempRoot(path);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopAddIndirectRoot: {
         Path path = absPath(readString(from));
-        startWork();
+        logger->startWork();
         store->addIndirectRoot(path);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopSyncWithGC: {
-        startWork();
+        logger->startWork();
         store->syncWithGC();
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopFindRoots: {
-        startWork();
+        logger->startWork();
         Roots roots = store->findRoots();
-        stopWork();
+        logger->stopWork();
         to << roots.size();
         for (auto & i : roots)
             to << i.first << i.second;
@@ -416,11 +434,11 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
 
         GCResults results;
 
-        startWork();
+        logger->startWork();
         if (options.ignoreLiveness)
             throw Error("you are not allowed to ignore liveness");
         store->collectGarbage(options, results);
-        stopWork();
+        logger->stopWork();
 
         to << results.paths << results.bytesFreed << 0 /* obsolete */;
 
@@ -451,7 +469,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
             }
         }
 
-        startWork();
+        logger->startWork();
 
         for (auto & i : overrides) {
             auto & name(i.first);
@@ -492,16 +510,16 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
             }
         }
 
-        stopWork();
+        logger->stopWork();
         break;
     }
 
     case wopQuerySubstitutablePathInfo: {
         Path path = absPath(readString(from));
-        startWork();
+        logger->startWork();
         SubstitutablePathInfos infos;
         store->querySubstitutablePathInfos({path}, infos);
-        stopWork();
+        logger->stopWork();
         SubstitutablePathInfos::iterator i = infos.find(path);
         if (i == infos.end())
             to << 0;
@@ -513,10 +531,10 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
 
     case wopQuerySubstitutablePathInfos: {
         PathSet paths = readStorePaths<PathSet>(*store, from);
-        startWork();
+        logger->startWork();
         SubstitutablePathInfos infos;
         store->querySubstitutablePathInfos(paths, infos);
-        stopWork();
+        logger->stopWork();
         to << infos.size();
         for (auto & i : infos) {
             to << i.first << i.second.deriver << i.second.references
@@ -526,9 +544,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopQueryAllValidPaths: {
-        startWork();
+        logger->startWork();
         PathSet paths = store->queryAllValidPaths();
-        stopWork();
+        logger->stopWork();
         to << paths;
         break;
     }
@@ -536,13 +554,13 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopQueryPathInfo: {
         Path path = readStorePath(*store, from);
         std::shared_ptr<const ValidPathInfo> info;
-        startWork();
+        logger->startWork();
         try {
             info = store->queryPathInfo(path);
         } catch (InvalidPath &) {
             if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw;
         }
-        stopWork();
+        logger->stopWork();
         if (info) {
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
@@ -561,20 +579,20 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopOptimiseStore:
-        startWork();
+        logger->startWork();
         store->optimiseStore();
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
 
     case wopVerifyStore: {
         bool checkContents, repair;
         from >> checkContents >> repair;
-        startWork();
+        logger->startWork();
         if (repair && !trusted)
             throw Error("you are not privileged to repair paths");
         bool errors = store->verifyStore(checkContents, (RepairFlag) repair);
-        stopWork();
+        logger->stopWork();
         to << errors;
         break;
     }
@@ -582,19 +600,19 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopAddSignatures: {
         Path path = readStorePath(*store, from);
         StringSet sigs = readStrings<StringSet>(from);
-        startWork();
+        logger->startWork();
         if (!trusted)
             throw Error("you are not privileged to add signatures");
         store->addSignatures(path, sigs);
-        stopWork();
+        logger->stopWork();
         to << 1;
         break;
     }
 
     case wopNarFromPath: {
         auto path = readStorePath(*store, from);
-        startWork();
-        stopWork();
+        logger->startWork();
+        logger->stopWork();
         dumpPath(path, to);
         break;
     }
@@ -619,20 +637,20 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         TeeSink tee(from);
         parseDump(tee, tee.source);
 
-        startWork();
+        logger->startWork();
         store->addToStore(info, tee.source.data, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
-        stopWork();
+        logger->stopWork();
         break;
     }
 
     case wopQueryMissing: {
         PathSet targets = readStorePaths<PathSet>(*store, from);
-        startWork();
+        logger->startWork();
         PathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
-        stopWork();
+        logger->stopWork();
         to << willBuild << willSubstitute << unknown << downloadSize << narSize;
         break;
     }
@@ -647,9 +665,17 @@ static void processConnection(bool trusted)
 {
     MonitorFdHup monitor(from.fd);
 
-    canSendStderr = false;
-    defaultLogger = logger;
-    logger = new TunnelLogger();
+    TunnelLogger tunnelLogger;
+    auto prevLogger = nix::logger;
+    logger = &tunnelLogger;
+
+    unsigned int opCount = 0;
+
+    Finally finally([&]() {
+        logger = prevLogger;
+        _isInterrupted = false;
+        debug("%d operations", opCount);
+    });
 
     /* Exchange the greeting. */
     unsigned int magic = readInt(from);
@@ -667,7 +693,7 @@ static void processConnection(bool trusted)
     readInt(from); // obsolete reserveSpace
 
     /* Send startup error messages to the client. */
-    startWork();
+    tunnelLogger.startWork();
 
     try {
 
@@ -687,12 +713,10 @@ static void processConnection(bool trusted)
         params["path-info-cache-size"] = "0";
         auto store = make_ref<LocalStore>(params);
 
-        stopWork();
+        tunnelLogger.stopWork();
         to.flush();
 
         /* Process client requests. */
-        unsigned int opCount = 0;
-
         while (true) {
             WorkerOp op;
             try {
@@ -706,32 +730,28 @@ static void processConnection(bool trusted)
             opCount++;
 
             try {
-                performOp(store, trusted, clientVersion, from, to, op);
+                performOp(&tunnelLogger, store, trusted, clientVersion, from, to, op);
             } catch (Error & e) {
                 /* If we're not in a state where we can send replies, then
                    something went wrong processing the input of the
                    client.  This can happen especially if I/O errors occur
                    during addTextToStore() / importPath().  If that
                    happens, just send the error message and exit. */
-                bool errorAllowed = canSendStderr;
-                stopWork(false, e.msg(), e.status);
+                bool errorAllowed = tunnelLogger.state_.lock()->canSendStderr;
+                tunnelLogger.stopWork(false, e.msg(), e.status);
                 if (!errorAllowed) throw;
             } catch (std::bad_alloc & e) {
-                stopWork(false, "Nix daemon out of memory", 1);
+                tunnelLogger.stopWork(false, "Nix daemon out of memory", 1);
                 throw;
             }
 
             to.flush();
 
-            assert(!canSendStderr);
+            assert(!tunnelLogger.state_.lock()->canSendStderr);
         };
 
-        canSendStderr = false;
-        _isInterrupted = false;
-        debug(format("%1% operations") % opCount);
-
-    } catch (Error & e) {
-        stopWork(false, e.msg(), 1);
+    } catch (std::exception & e) {
+        tunnelLogger.stopWork(false, e.what(), 1);
         to.flush();
         return;
     }

From e681b1f064f88b46cd75d374937bd07e5d6e95d1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Aug 2017 14:30:35 +0200
Subject: [PATCH 0515/2196] Simplify

---
 src/libutil/logging.hh  |  5 ++---
 src/nix/progress-bar.cc | 27 ++++++++++++---------------
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 4b1f3b5f70c..2ec15cb68d2 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -33,6 +33,7 @@ typedef enum {
     resCorruptedPath = 103,
     resSetPhase = 104,
     resProgress = 105,
+    resSetExpected = 106,
 } ResultType;
 
 typedef uint64_t ActivityId;
@@ -72,8 +73,6 @@ public:
 
     virtual void stopActivity(ActivityId act) { };
 
-    virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
-
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
 };
 
@@ -97,7 +96,7 @@ struct Activity
     { result(resProgress, done, expected, running, failed); }
 
     void setExpected(ActivityType type2, uint64_t expected) const
-    { logger.setExpected(id, type2, expected); }
+    { result(resSetExpected, type2, expected); }
 
     template<typename... Args>
     void result(ResultType type, const Args & ... args) const
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 054127f6277..70dbfd4e596 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -192,21 +192,6 @@ class ProgressBar : public Logger
         update(*state);
     }
 
-    void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
-    {
-        auto state(state_.lock());
-
-        auto i = state->its.find(act);
-        assert(i != state->its.end());
-        ActInfo & actInfo = *i->second;
-        auto & j = actInfo.expectedByType[type];
-        state->activitiesByType[type].expected -= j;
-        j = expected;
-        state->activitiesByType[type].expected += j;
-
-        update(*state);
-    }
-
     void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
     {
         auto state(state_.lock());
@@ -258,6 +243,18 @@ class ProgressBar : public Logger
             actInfo.failed = getI(fields, 3);
             update(*state);
         }
+
+        else if (type == resSetExpected) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            auto type = (ActivityType) getI(fields, 0);
+            auto & j = actInfo.expectedByType[type];
+            state->activitiesByType[type].expected -= j;
+            j = getI(fields, 1);
+            state->activitiesByType[type].expected += j;
+            update(*state);
+        }
     }
 
     void update()

From fe34b91289793178323b4924ee63c9e3ed636d11 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Aug 2017 18:49:42 +0200
Subject: [PATCH 0516/2196] Tunnel progress messages from the daemon to the
 client

This makes the progress bar work for non-root users.
---
 src/libstore/remote-store.cc    | 69 +++++++++++++++++++++++-----
 src/libstore/worker-protocol.hh |  3 ++
 src/libutil/logging.hh          |  2 +-
 src/nix-daemon/nix-daemon.cc    | 80 +++++++++++++++++++++++++--------
 4 files changed, 124 insertions(+), 30 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e9f2cee80db..f0e3502bf79 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -629,17 +629,37 @@ RemoteStore::Connection::~Connection()
 }
 
 
+static Logger::Fields readFields(Source & from)
+{
+    Logger::Fields fields;
+    size_t size = readInt(from);
+    for (size_t n = 0; n < size; n++) {
+        auto type = (decltype(Logger::Field::type)) readInt(from);
+        if (type == Logger::Field::tInt)
+            fields.push_back(readNum<uint64_t>(from));
+        else if (type == Logger::Field::tString)
+            fields.push_back(readString(from));
+        else
+            throw Error("got unsupported field type %x from Nix daemon", (int) type);
+    }
+    return fields;
+}
+
+
 void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
 {
     to.flush();
-    unsigned int msg;
-    while ((msg = readInt(from)) == STDERR_NEXT
-        || msg == STDERR_READ || msg == STDERR_WRITE) {
+
+    while (true) {
+
+        auto msg = readNum<uint64_t>(from);
+
         if (msg == STDERR_WRITE) {
             string s = readString(from);
             if (!sink) throw Error("no sink");
             (*sink)(s);
         }
+
         else if (msg == STDERR_READ) {
             if (!source) throw Error("no source");
             size_t len = readNum<size_t>(from);
@@ -647,16 +667,43 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
             writeString(buf.get(), source->read(buf.get(), len), to);
             to.flush();
         }
-        else
+
+        else if (msg == STDERR_ERROR) {
+            string error = readString(from);
+            unsigned int status = readInt(from);
+            throw Error(status, error);
+        }
+
+        else if (msg == STDERR_NEXT)
             printError(chomp(readString(from)));
+
+        else if (msg == STDERR_START_ACTIVITY) {
+            auto act = readNum<ActivityId>(from);
+            auto type = (ActivityType) readInt(from);
+            auto s = readString(from);
+            auto fields = readFields(from);
+            auto parent = readNum<ActivityId>(from);
+            logger->startActivity(act, type, s, fields, parent);
+        }
+
+        else if (msg == STDERR_STOP_ACTIVITY) {
+            auto act = readNum<ActivityId>(from);
+            logger->stopActivity(act);
+        }
+
+        else if (msg == STDERR_RESULT) {
+            auto act = readNum<ActivityId>(from);
+            auto type = (ResultType) readInt(from);
+            auto fields = readFields(from);
+            logger->result(act, type, fields);
+        }
+
+        else if (msg == STDERR_LAST)
+            break;
+
+        else
+            throw Error("got unknown message type %x from Nix daemon", msg);
     }
-    if (msg == STDERR_ERROR) {
-        string error = readString(from);
-        unsigned int status = readInt(from);
-        throw Error(status, error);
-    }
-    else if (msg != STDERR_LAST)
-        throw Error("protocol error processing standard error");
 }
 
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6c6766b3612..9daeb46add1 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -57,6 +57,9 @@ typedef enum {
 #define STDERR_WRITE 0x64617416 // data for sink
 #define STDERR_LAST  0x616c7473
 #define STDERR_ERROR 0x63787470
+#define STDERR_START_ACTIVITY 0x53545254
+#define STDERR_STOP_ACTIVITY  0x53544f50
+#define STDERR_RESULT         0x52534c54
 
 
 Path readStorePath(Store & store, Source & from);
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 2ec15cb68d2..e3e7c8e6f33 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -47,7 +47,7 @@ public:
     struct Field
     {
         // FIXME: use std::variant.
-        enum { tInt, tString } type;
+        enum { tInt = 0, tString = 1 } type;
         uint64_t i = 0;
         std::string s;
         Field(const std::string & s) : type(tString), s(s) { }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 3237333f4d0..65c88562cf0 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -56,6 +56,21 @@ static FdSource from(STDIN_FILENO);
 static FdSink to(STDOUT_FILENO);
 
 
+Sink & operator << (Sink & sink, const Logger::Fields & fields)
+{
+    sink << fields.size();
+    for (auto & f : fields) {
+        sink << f.type;
+        if (f.type == Logger::Field::tInt)
+            sink << f.i;
+        else if (f.type == Logger::Field::tString)
+            sink << f.s;
+        else abort();
+    }
+    return sink;
+}
+
+
 /* Logger that forwards log messages to the client, *if* we're in a
    state where the protocol allows it (i.e., when canSendStderr is
    true). */
@@ -69,15 +84,14 @@ struct TunnelLogger : public Logger
 
     Sync<State> state_;
 
-    void log(Verbosity lvl, const FormatOrString & fs) override
+    void enqueueMsg(const std::string & s)
     {
-        if (lvl > verbosity) return;
-
         auto state(state_.lock());
 
         if (state->canSendStderr) {
+            assert(state->pendingMsgs.empty());
             try {
-                to << STDERR_NEXT << (fs.s + "\n");
+                to(s);
                 to.flush();
             } catch (...) {
                 /* Write failed; that means that the other side is
@@ -86,7 +100,16 @@ struct TunnelLogger : public Logger
                 throw;
             }
         } else
-            state->pendingMsgs.push_back(fs.s);
+            state->pendingMsgs.push_back(s);
+    }
+
+    void log(Verbosity lvl, const FormatOrString & fs) override
+    {
+        if (lvl > verbosity) return;
+
+        StringSink buf;
+        buf << STDERR_NEXT << (fs.s + "\n");
+        enqueueMsg(*buf.s);
     }
 
     /* startWork() means that we're starting an operation for which we
@@ -99,7 +122,7 @@ struct TunnelLogger : public Logger
         state->canSendStderr = true;
 
         for (auto & msg : state->pendingMsgs)
-            to << STDERR_NEXT << (msg + "\n");
+            to(msg);
 
         state->pendingMsgs.clear();
 
@@ -121,6 +144,28 @@ struct TunnelLogger : public Logger
             if (status != 0) to << status;
         }
     }
+
+    void startActivity(ActivityId act, ActivityType type,
+        const std::string & s, const Fields & fields, ActivityId parent) override
+    {
+        StringSink buf;
+        buf << STDERR_START_ACTIVITY << act << type << s << fields << parent;
+        enqueueMsg(*buf.s);
+    }
+
+    void stopActivity(ActivityId act) override
+    {
+        StringSink buf;
+        buf << STDERR_STOP_ACTIVITY << act;
+        enqueueMsg(*buf.s);
+    }
+
+    void result(ActivityId act, ResultType type, const Fields & fields) override
+    {
+        StringSink buf;
+        buf << STDERR_RESULT << act << type << fields;
+        enqueueMsg(*buf.s);
+    }
 };
 
 
@@ -665,16 +710,15 @@ static void processConnection(bool trusted)
 {
     MonitorFdHup monitor(from.fd);
 
-    TunnelLogger tunnelLogger;
+    auto tunnelLogger = new TunnelLogger();
     auto prevLogger = nix::logger;
-    logger = &tunnelLogger;
+    logger = tunnelLogger;
 
     unsigned int opCount = 0;
 
     Finally finally([&]() {
-        logger = prevLogger;
         _isInterrupted = false;
-        debug("%d operations", opCount);
+        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
     });
 
     /* Exchange the greeting. */
@@ -693,7 +737,7 @@ static void processConnection(bool trusted)
     readInt(from); // obsolete reserveSpace
 
     /* Send startup error messages to the client. */
-    tunnelLogger.startWork();
+    tunnelLogger->startWork();
 
     try {
 
@@ -713,7 +757,7 @@ static void processConnection(bool trusted)
         params["path-info-cache-size"] = "0";
         auto store = make_ref<LocalStore>(params);
 
-        tunnelLogger.stopWork();
+        tunnelLogger->stopWork();
         to.flush();
 
         /* Process client requests. */
@@ -730,28 +774,28 @@ static void processConnection(bool trusted)
             opCount++;
 
             try {
-                performOp(&tunnelLogger, store, trusted, clientVersion, from, to, op);
+                performOp(tunnelLogger, store, trusted, clientVersion, from, to, op);
             } catch (Error & e) {
                 /* If we're not in a state where we can send replies, then
                    something went wrong processing the input of the
                    client.  This can happen especially if I/O errors occur
                    during addTextToStore() / importPath().  If that
                    happens, just send the error message and exit. */
-                bool errorAllowed = tunnelLogger.state_.lock()->canSendStderr;
-                tunnelLogger.stopWork(false, e.msg(), e.status);
+                bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
+                tunnelLogger->stopWork(false, e.msg(), e.status);
                 if (!errorAllowed) throw;
             } catch (std::bad_alloc & e) {
-                tunnelLogger.stopWork(false, "Nix daemon out of memory", 1);
+                tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
                 throw;
             }
 
             to.flush();
 
-            assert(!tunnelLogger.state_.lock()->canSendStderr);
+            assert(!tunnelLogger->state_.lock()->canSendStderr);
         };
 
     } catch (std::exception & e) {
-        tunnelLogger.stopWork(false, e.what(), 1);
+        tunnelLogger->stopWork(false, e.what(), 1);
         to.flush();
         return;
     }

From cfc813239128fc69a9228b39b5c0abb7e7a67b11 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Aug 2017 18:54:23 +0200
Subject: [PATCH 0517/2196] Don't send progress messages to older clients

---
 src/libstore/worker-protocol.hh |  2 +-
 src/nix-daemon/nix-daemon.cc    | 29 ++++++++++++++++++-----------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 9daeb46add1..996e1d25355 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x113
+#define PROTOCOL_VERSION 0x114
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 65c88562cf0..9f1d619e59b 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -84,6 +84,10 @@ struct TunnelLogger : public Logger
 
     Sync<State> state_;
 
+    unsigned int clientVersion;
+
+    TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { }
+
     void enqueueMsg(const std::string & s)
     {
         auto state(state_.lock());
@@ -148,6 +152,7 @@ struct TunnelLogger : public Logger
     void startActivity(ActivityId act, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) override
     {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
         StringSink buf;
         buf << STDERR_START_ACTIVITY << act << type << s << fields << parent;
         enqueueMsg(*buf.s);
@@ -155,6 +160,7 @@ struct TunnelLogger : public Logger
 
     void stopActivity(ActivityId act) override
     {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
         StringSink buf;
         buf << STDERR_STOP_ACTIVITY << act;
         enqueueMsg(*buf.s);
@@ -162,6 +168,7 @@ struct TunnelLogger : public Logger
 
     void result(ActivityId act, ResultType type, const Fields & fields) override
     {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
         StringSink buf;
         buf << STDERR_RESULT << act << type << fields;
         enqueueMsg(*buf.s);
@@ -710,17 +717,6 @@ static void processConnection(bool trusted)
 {
     MonitorFdHup monitor(from.fd);
 
-    auto tunnelLogger = new TunnelLogger();
-    auto prevLogger = nix::logger;
-    logger = tunnelLogger;
-
-    unsigned int opCount = 0;
-
-    Finally finally([&]() {
-        _isInterrupted = false;
-        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
-    });
-
     /* Exchange the greeting. */
     unsigned int magic = readInt(from);
     if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
@@ -731,6 +727,17 @@ static void processConnection(bool trusted)
     if (clientVersion < 0x10a)
         throw Error("the Nix client version is too old");
 
+    auto tunnelLogger = new TunnelLogger(clientVersion);
+    auto prevLogger = nix::logger;
+    logger = tunnelLogger;
+
+    unsigned int opCount = 0;
+
+    Finally finally([&]() {
+        _isInterrupted = false;
+        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
+    });
+
     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
         setAffinityTo(readInt(from));
 

From 2cc345b95f19334e7050c7203a5f6823af24344f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 28 Aug 2017 19:13:24 +0200
Subject: [PATCH 0518/2196] Give activities a verbosity level again

And print them (separately from the progress bar) given sufficient -v
flags.
---
 src/libstore/build.cc          | 10 ++++++----
 src/libstore/download.cc       |  2 +-
 src/libstore/optimise-store.cc |  2 +-
 src/libstore/remote-store.cc   |  3 ++-
 src/libstore/store-api.cc      |  4 ++--
 src/libutil/logging.cc         | 11 +++++++++--
 src/libutil/logging.hh         |  8 ++++++--
 src/nix-daemon/nix-daemon.cc   | 11 ++++++++---
 src/nix/progress-bar.cc        |  7 +++++--
 src/nix/verify.cc              |  2 +-
 10 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 92626738ab2..02dcd4536a5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1390,7 +1390,7 @@ void DerivationGoal::tryToBuild()
     bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
     auto started = [&]() {
-        act = std::make_unique<Activity>(*logger, actBuild,
+        act = std::make_unique<Activity>(*logger, lvlInfo, actBuild,
             fmt("building '%s'", drvPath), Logger::Fields{drvPath});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
@@ -2419,12 +2419,13 @@ struct BuilderLogger : Logger
         prevLogger.log(lvl, fs);
     }
 
-    void startActivity(ActivityId act, ActivityType type,
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) override
     {
         nlohmann::json json;
         json["action"] = "start";
         json["id"] = act;
+        json["level"] = lvl;
         json["type"] = type;
         json["text"] = s;
         addFields(json, fields);
@@ -3339,7 +3340,8 @@ void DerivationGoal::flushLine()
                 if (type == actDownload)
                     builderActivities.emplace(std::piecewise_construct,
                         std::forward_as_tuple(json["id"]),
-                        std::forward_as_tuple(*logger, type, json["text"], getFields(json["fields"]), act->id));
+                        std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
+                            json["text"], getFields(json["fields"]), act->id));
             }
 
             else if (action == "stop")
@@ -3686,7 +3688,7 @@ void SubstitutionGoal::tryToRun()
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide = -1; });
 
-            Activity act(*logger, actSubstitute, "", Logger::Fields{storePath, sub->getUri()});
+            Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()});
             PushActivity pact(act.id);
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index a2099917677..625e781b452 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -85,7 +85,7 @@ struct CurlDownloader : public Downloader
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader)
             , request(request)
-            , act(*logger, actDownload, fmt("downloading '%s'", request.uri), {}, request.parentAct)
+            , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {}, request.parentAct)
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index f1325ba5a1c..891540ae4c1 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -266,7 +266,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
         {
-            Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i));
             optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
         }
         done++;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f0e3502bf79..1af84cff5ba 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -679,11 +679,12 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
 
         else if (msg == STDERR_START_ACTIVITY) {
             auto act = readNum<ActivityId>(from);
+            auto lvl = (Verbosity) readInt(from);
             auto type = (ActivityType) readInt(from);
             auto s = readString(from);
             auto fields = readFields(from);
             auto parent = readNum<ActivityId>(from);
-            logger->startActivity(act, type, s, fields, parent);
+            logger->startActivity(act, lvl, type, s, fields, parent);
         }
 
         else if (msg == STDERR_STOP_ACTIVITY) {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f0737685271..7a8a3f7dee0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -565,7 +565,7 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    Activity act(*logger, actCopyPath, fmt("copying path '%s'", storePath),
+    Activity act(*logger, lvlInfo, actCopyPath, fmt("copying path '%s'", storePath),
         {storePath, srcStore->getUri(), dstStore->getUri()});
     PushActivity pact(act.id);
 
@@ -623,7 +623,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
-    Activity act(*logger, actCopyPaths, fmt("copying %d paths", missing.size()));
+    Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};
     std::atomic<uint64_t> bytesExpected{0};
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index b103b902eab..e38a460537e 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -45,6 +45,13 @@ class SimpleLogger : public Logger
 
         writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
     }
+
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
+        const std::string & s, const Fields & fields, ActivityId parent)
+    {
+        if (lvl <= verbosity && !s.empty())
+            log(lvl, s + "...");
+    }
 };
 
 Verbosity verbosity = lvlInfo;
@@ -76,11 +83,11 @@ Logger * makeDefaultLogger()
 
 std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
 
-Activity::Activity(Logger & logger, ActivityType type,
+Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     const std::string & s, const Logger::Fields & fields, ActivityId parent)
     : logger(logger), id(nextId++)
 {
-    logger.startActivity(id, type, s, fields, parent);
+    logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index e3e7c8e6f33..84fffa820f4 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -68,7 +68,7 @@ public:
 
     virtual void warn(const std::string & msg);
 
-    virtual void startActivity(ActivityId act, ActivityType type,
+    virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) { };
 
     virtual void stopActivity(ActivityId act) { };
@@ -84,9 +84,13 @@ struct Activity
 
     const ActivityId id;
 
-    Activity(Logger & logger, ActivityType type, const std::string & s = "",
+    Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "",
         const Logger::Fields & fields = {}, ActivityId parent = curActivity);
 
+    Activity(Logger & logger, ActivityType type,
+        const Logger::Fields & fields = {}, ActivityId parent = curActivity)
+        : Activity(logger, lvlError, type, "", fields, parent) { };
+
     Activity(const Activity & act) = delete;
 
     ~Activity()
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 9f1d619e59b..dbf301a9153 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -149,12 +149,17 @@ struct TunnelLogger : public Logger
         }
     }
 
-    void startActivity(ActivityId act, ActivityType type,
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) override
     {
-        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) {
+            if (!s.empty())
+                log(lvl, s + "...");
+            return;
+        }
+
         StringSink buf;
-        buf << STDERR_START_ACTIVITY << act << type << s << fields << parent;
+        buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
         enqueueMsg(*buf.s);
     }
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 70dbfd4e596..34186667918 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -126,9 +126,12 @@ class ProgressBar : public Logger
         update(state);
     }
 
-    void startActivity(ActivityId act, ActivityType type, const std::string & s,
-        const Fields & fields, ActivityId parent) override
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
+        const std::string & s, const Fields & fields, ActivityId parent) override
     {
+        if (lvl <= verbosity && !s.empty())
+            log(lvl, s + "...");
+
         auto state(state_.lock());
 
         state->activities.emplace_back(ActInfo());
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 7156341396d..4913d990097 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -79,7 +79,7 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                Activity act2(*logger, actUnknown, fmt("checking '%s'", storePath));
+                Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
 
                 MaintainCount<std::atomic<size_t>> mcActive(active);
                 update();

From 05c45f301de6f4b5528969b97b9cc1891b1a985c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 11:51:16 +0200
Subject: [PATCH 0519/2196] readLink(): Handle symlinks in /proc

Symlinks like /proc/self/exe report a stat() size of 0, so use a
buffer of at least PATH_MAX instead.
---
 src/libutil/util.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 2f410f7e3fd..605e89ce27f 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -219,13 +219,14 @@ Path readLink(const Path & path)
     struct stat st = lstat(path);
     if (!S_ISLNK(st.st_mode))
         throw Error(format("'%1%' is not a symlink") % path);
-    char buf[st.st_size];
-    ssize_t rlsize = readlink(path.c_str(), buf, st.st_size);
+    auto bufSize = std::max(st.st_size, (off_t) PATH_MAX + 1);
+    char buf[bufSize];
+    ssize_t rlsize = readlink(path.c_str(), buf, bufSize);
     if (rlsize == -1)
         throw SysError(format("reading symbolic link '%1%'") % path);
-    else if (rlsize > st.st_size)
+    else if (rlsize > bufSize)
         throw Error(format("symbolic link '%1%' size overflow %2% > %3%")
-            % path % rlsize % st.st_size);
+            % path % rlsize % bufSize);
     return string(buf, rlsize);
 }
 

From 1c58e13bee6715c668e93cd4ec93bc5d5fd1a4ad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 11:52:55 +0200
Subject: [PATCH 0520/2196] Hide commands that don't have a description

These are assumed to be internal.
---
 src/nix/command.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index e88c834abc5..f90d848f2a4 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -50,8 +50,11 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out)
     out << "Available commands:\n";
 
     Table2 table;
-    for (auto & command : commands)
-        table.push_back(std::make_pair(command.second->name(), command.second->description()));
+    for (auto & command : commands) {
+        auto descr = command.second->description();
+        if (!descr.empty())
+            table.push_back(std::make_pair(command.second->name(), descr));
+    }
     printTable(out, table);
 
     out << "\n";

From 93a5ef0516881ec133fc5e54069add6dd64ffbdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 13:21:07 +0200
Subject: [PATCH 0521/2196] nix run: Fix chroot execution

Running "nix run" with a diverted store, e.g.

  $ nix run --store local?root=/tmp/nix nixpkgs.hello

stopped working when Nix became multithreaded, because
unshare(CLONE_NEWUSER) doesn't work in multithreaded processes. The
obvious solution is to terminate all other threads first, but 1) there
is no way to terminate Boehm GC marker threads; and 2) it appears that
the kernel has a race where unshare(CLONE_NEWUSER) will still fail for
some indeterminate amount of time after joining other threads.

So instead, "nix run" will now exec() a single-threaded helper ("nix
__run_in_chroot") that performs the actual unshare()/chroot()/exec().
---
 src/nix/main.cc |  11 ++++
 src/nix/run.cc  | 159 ++++++++++++++++++++++++++++++------------------
 2 files changed, 110 insertions(+), 60 deletions(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 7fece5b87b0..0990549aeec 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -9,6 +9,10 @@
 #include "store-api.hh"
 #include "progress-bar.hh"
 
+extern std::string chrootHelperName;
+
+void chrootHelper(int argc, char * * argv);
+
 namespace nix {
 
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
@@ -57,6 +61,13 @@ void mainWrapped(int argc, char * * argv)
     verbosity = lvlError;
     settings.verboseBuild = false;
 
+    /* The chroot helper needs to be run before any threads have been
+       started. */
+    if (argc > 0 && argv[0] == chrootHelperName) {
+        chrootHelper(argc, argv);
+        return;
+    }
+
     initNix();
     initGC();
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f67fd637c4f..e58e2f187a2 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -5,6 +5,7 @@
 #include "derivations.hh"
 #include "local-store.hh"
 #include "finally.hh"
+#include "fs-accessor.hh"
 
 #if __linux__
 #include <sys/mount.h>
@@ -12,6 +13,8 @@
 
 using namespace nix;
 
+std::string chrootHelperName = "__run_in_chroot";
+
 struct CmdRun : InstallablesCommand
 {
     CmdRun()
@@ -32,73 +35,109 @@ struct CmdRun : InstallablesCommand
     {
         auto outPaths = toStorePaths(store, Build);
 
-        auto store2 = store.dynamic_pointer_cast<LocalStore>();
-
-        if (store2 && store->storeDir != store2->realStoreDir) {
-#if __linux__
-            uid_t uid = getuid();
-            uid_t gid = getgid();
-
-            if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
-                throw SysError("setting up a private mount namespace");
-
-            /* Bind-mount realStoreDir on /nix/store. If the latter
-               mount point doesn't already exists, we have to create a
-               chroot environment containing the mount point and bind
-               mounts for the children of /. Would be nice if we could
-               use overlayfs here, but that doesn't work in a user
-               namespace yet (Ubuntu has a patch for this:
-               https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */
-            if (!pathExists(store->storeDir)) {
-                // FIXME: Use overlayfs?
-
-                Path tmpDir = createTempDir();
-
-                createDirs(tmpDir + store->storeDir);
-
-                if (mount(store2->realStoreDir.c_str(), (tmpDir + store->storeDir).c_str(), "", MS_BIND, 0) == -1)
-                    throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
-
-                for (auto entry : readDirectory("/")) {
-                    Path dst = tmpDir + "/" + entry.name;
-                    if (pathExists(dst)) continue;
-                    if (mkdir(dst.c_str(), 0700) == -1)
-                        throw SysError(format("creating directory '%s'") % dst);
-                    if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                        throw SysError(format("mounting '%s' on '%s'") %  ("/" + entry.name) % dst);
-                }
-
-                char * cwd = getcwd(0, 0);
-                if (!cwd) throw SysError("getting current directory");
-                Finally freeCwd([&]() { free(cwd); });
-
-                if (chroot(tmpDir.c_str()) == -1)
-                    throw SysError(format("chrooting into '%s'") % tmpDir);
-
-                if (chdir(cwd) == -1)
-                    throw SysError(format("chdir to '%s' in chroot") % cwd);
-            } else
-                if (mount(store2->realStoreDir.c_str(), store->storeDir.c_str(), "", MS_BIND, 0) == -1)
-                    throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
-
-            writeFile("/proc/self/setgroups", "deny");
-            writeFile("/proc/self/uid_map", (format("%d %d %d") % uid % uid % 1).str());
-            writeFile("/proc/self/gid_map", (format("%d %d %d") % gid % gid % 1).str());
-#else
-            throw Error(format("mounting the Nix store on '%s' is not supported on this platform") % store->storeDir);
-#endif
-        }
-
+        auto accessor = store->getFSAccessor();
 
         auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
         for (auto & path : outPaths)
-            if (pathExists(path + "/bin"))
+            if (accessor->stat(path + "/bin").type != FSAccessor::tMissing)
                 unixPath.push_front(path + "/bin");
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
-        if (execlp("bash", "bash", nullptr) == -1)
-            throw SysError("unable to exec 'bash'");
+        std::string cmd = "bash";
+        Strings args = { cmd };
+
+        /* If this is a diverted store (i.e. its "logical" location
+           (typically /nix/store) differs from its "physical" location
+           (e.g. /home/eelco/nix/store), then run the command in a
+           chroot. For non-root users, this requires running it in new
+           mount and user namespaces. Unfortunately,
+           unshare(CLONE_NEWUSER) doesn't work in a multithreaded
+           program (which "nix" is), so we exec() a single-threaded
+           helper program (chrootHelper() below) to do the work. */
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+
+        if (store2 && store->storeDir != store2->realStoreDir) {
+            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
+            for (auto & arg : args) helperArgs.push_back(arg);
+
+            execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
+
+            throw SysError("could not execute chroot helper");
+        }
+
+        execvp(cmd.c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("unable to exec '%s'", cmd);
     }
 };
 
 static RegisterCommand r1(make_ref<CmdRun>());
+
+void chrootHelper(int argc, char * * argv)
+{
+    int p = 1;
+    std::string storeDir = argv[p++];
+    std::string realStoreDir = argv[p++];
+    std::string cmd = argv[p++];
+    Strings args;
+    while (p < argc)
+        args.push_back(argv[p++]);
+
+#if __linux__
+    uid_t uid = getuid();
+    uid_t gid = getgid();
+
+    if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
+        throw SysError("setting up a private mount namespace");
+
+    /* Bind-mount realStoreDir on /nix/store. If the latter mount
+       point doesn't already exists, we have to create a chroot
+       environment containing the mount point and bind mounts for the
+       children of /. Would be nice if we could use overlayfs here,
+       but that doesn't work in a user namespace yet (Ubuntu has a
+       patch for this:
+       https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */
+    if (true /* !pathExists(storeDir) */) {
+        // FIXME: Use overlayfs?
+
+        Path tmpDir = createTempDir();
+
+        createDirs(tmpDir + storeDir);
+
+        if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1)
+            throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
+
+        for (auto entry : readDirectory("/")) {
+            Path dst = tmpDir + "/" + entry.name;
+            if (pathExists(dst)) continue;
+            if (mkdir(dst.c_str(), 0700) == -1)
+                throw SysError(format("creating directory '%s'") % dst);
+            if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+                throw SysError(format("mounting '%s' on '%s'") %  ("/" + entry.name) % dst);
+        }
+
+        char * cwd = getcwd(0, 0);
+        if (!cwd) throw SysError("getting current directory");
+        Finally freeCwd([&]() { free(cwd); });
+
+        if (chroot(tmpDir.c_str()) == -1)
+            throw SysError(format("chrooting into '%s'") % tmpDir);
+
+        if (chdir(cwd) == -1)
+            throw SysError(format("chdir to '%s' in chroot") % cwd);
+    } else
+        if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1)
+            throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
+
+    writeFile("/proc/self/setgroups", "deny");
+    writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1));
+    writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1));
+
+    execvp(cmd.c_str(), stringsToCharPtrs(args).data());
+
+    throw SysError("unable to exec '%s'", cmd);
+
+#else
+    throw Error("mounting the Nix store on '%s' is not supported on this platform", >storeDir);
+#endif
+}

From 5cc8609e30ab4f76053f3159c51ea78d9215bc8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 14:28:57 +0200
Subject: [PATCH 0522/2196] nix run: Allow passing a command to execute

E.g.

  nix run nixpkgs.hello -c hello --greeting Hallo

Note that unlike "nix-shell --command", no quoting of arguments is
necessary.

"-c" (short for "--command") cannot be combined with "--" because they
both consume all remaining arguments. But since installables shouldn't
start with a dash, this is unlikely to cause problems.
---
 src/libutil/args.cc |  4 +++-
 src/libutil/args.hh |  2 ++
 src/nix/run.cc      | 16 ++++++++++++++--
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 2b72079f2b4..d17a1e7a9ab 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -102,9 +102,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         ++pos;
         Strings args;
         for (size_t n = 0 ; n < flag.arity; ++n) {
-            if (pos == end)
+            if (pos == end) {
+                if (flag.arity == ArityAny) break;
                 throw UsageError(format("flag '%1%' requires %2% argument(s)")
                     % name % flag.arity);
+            }
             args.push_back(*pos++);
         }
         flag.handler(args);
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index fd910b965cc..044ed209e3b 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -26,6 +26,8 @@ public:
 
 protected:
 
+    static const size_t ArityAny = std::numeric_limits<size_t>::max();
+
     /* Flags. */
     struct Flag
     {
diff --git a/src/nix/run.cc b/src/nix/run.cc
index e58e2f187a2..39fbcc8acf9 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -17,8 +17,20 @@ std::string chrootHelperName = "__run_in_chroot";
 
 struct CmdRun : InstallablesCommand
 {
+    Strings command = { "bash" };
+
     CmdRun()
     {
+        mkFlag()
+            .longName("command")
+            .shortName('c')
+            .description("command and arguments to be executed; defaults to 'bash'")
+            .arity(ArityAny)
+            .labels({"command", "args"})
+            .handler([&](Strings ss) {
+                if (ss.empty()) throw UsageError("--command requires at least one argument");
+                command = ss;
+            });
     }
 
     std::string name() override
@@ -43,8 +55,8 @@ struct CmdRun : InstallablesCommand
                 unixPath.push_front(path + "/bin");
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
-        std::string cmd = "bash";
-        Strings args = { cmd };
+        std::string cmd = *command.begin();
+        Strings args = command;
 
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location

From 05d68a6e23127e490193599f75489006830b302c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 15:00:08 +0200
Subject: [PATCH 0523/2196] nix run: Add some flags for clearing/keeping the
 environment

This is useful for testing commands in isolation.

For example,

  $ nix run nixpkgs.geeqie -i -k DISPLAY -k XAUTHORITY -c geeqie

runs geeqie in an empty environment, except for $DISPLAY and
$XAUTHORITY.
---
 src/nix/run.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 39fbcc8acf9..8175d38e8ca 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -18,6 +18,8 @@ std::string chrootHelperName = "__run_in_chroot";
 struct CmdRun : InstallablesCommand
 {
     Strings command = { "bash" };
+    StringSet keep, unset;
+    bool ignoreEnvironment = false;
 
     CmdRun()
     {
@@ -31,6 +33,28 @@ struct CmdRun : InstallablesCommand
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
             });
+
+        mkFlag()
+            .longName("ignore-environment")
+            .shortName('i')
+            .description("clear the entire environment (except those specified with --keep)")
+            .handler([&](Strings ss) { ignoreEnvironment = true; });
+
+        mkFlag()
+            .longName("keep")
+            .shortName('k')
+            .description("keep specified environment variable")
+            .arity(1)
+            .labels({"name"})
+            .handler([&](Strings ss) { keep.insert(ss.front()); });
+
+        mkFlag()
+            .longName("unset")
+            .shortName('u')
+            .description("unset specified environment variable")
+            .arity(1)
+            .labels({"name"})
+            .handler([&](Strings ss) { unset.insert(ss.front()); });
     }
 
     std::string name() override
@@ -49,6 +73,31 @@ struct CmdRun : InstallablesCommand
 
         auto accessor = store->getFSAccessor();
 
+        if (ignoreEnvironment) {
+
+            if (!unset.empty())
+                throw UsageError("--unset does not make sense with --ignore-environment");
+
+            std::map<std::string, std::string> kept;
+            for (auto & var : keep) {
+                auto s = getenv(var.c_str());
+                if (s) kept[var] = s;
+            }
+
+            clearenv();
+
+            for (auto & var : kept)
+                setenv(var.first.c_str(), var.second.c_str(), 1);
+
+        } else {
+
+            if (!keep.empty())
+                throw UsageError("--keep does not make sense without --ignore-environment");
+
+            for (auto & var : unset)
+                unsetenv(var.c_str());
+        }
+
         auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
         for (auto & path : outPaths)
             if (accessor->stat(path + "/bin").type != FSAccessor::tMissing)

From c8235c531355d612d48f633f8a6ace5fcd97f5fe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 15:13:30 +0200
Subject: [PATCH 0524/2196] nix run: Flush the progress bar before starting the
 command

---
 src/nix/edit.cc         |  3 +++
 src/nix/main.cc         |  3 ++-
 src/nix/progress-bar.cc | 28 +++++++++++++++++-----------
 src/nix/progress-bar.hh | 10 +++-------
 src/nix/run.cc          |  3 +++
 5 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 5cf9c7200d0..fd07913bcfa 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "eval.hh"
 #include "attr-path.hh"
+#include "progress-bar.hh"
 
 #include <unistd.h>
 
@@ -65,6 +66,8 @@ struct CmdEdit : InstallablesCommand
 
             args.push_back(filename);
 
+            stopProgressBar();
+
             execvp(editor.c_str(), stringsToCharPtrs(args).data());
 
             throw SysError("cannot run editor '%s'", editor);
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 0990549aeec..9781b58544f 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -84,7 +84,8 @@ void mainWrapped(int argc, char * * argv)
 
     if (!args.command) args.showHelpAndExit();
 
-    StartProgressBar bar;
+    if (isatty(STDERR_FILENO))
+        startProgressBar();
 
     args.command->prepare();
     args.command->run();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 34186667918..f48cfa13b4c 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -95,6 +95,8 @@ class ProgressBar : public Logger
         uint64_t filesLinked = 0, bytesLinked = 0;
 
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
+
+        bool active = true;
     };
 
     Sync<State> state_;
@@ -106,8 +108,15 @@ class ProgressBar : public Logger
     }
 
     ~ProgressBar()
+    {
+        stop();
+    }
+
+    void stop()
     {
         auto state(state_.lock());
+        if (!state->active) return;
+        state->active = true;
         std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
         if (status != "")
@@ -268,6 +277,8 @@ class ProgressBar : public Logger
 
     void update(State & state)
     {
+        if (!state.active) return;
+
         std::string line;
 
         std::string status = getStatus(state);
@@ -385,21 +396,16 @@ class ProgressBar : public Logger
     }
 };
 
-StartProgressBar::StartProgressBar()
+void startProgressBar()
 {
-    if (isatty(STDERR_FILENO)) {
-        prev = logger;
-        logger = new ProgressBar();
-    }
+    logger = new ProgressBar();
 }
 
-StartProgressBar::~StartProgressBar()
+void stopProgressBar()
 {
-    if (prev) {
-        auto bar = logger;
-        logger = prev;
-        delete bar;
-    }
+    auto progressBar = dynamic_cast<ProgressBar *>(logger);
+    if (progressBar) progressBar->stop();
+
 }
 
 }
diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh
index d2e44f7c4fd..af8eda5a84f 100644
--- a/src/nix/progress-bar.hh
+++ b/src/nix/progress-bar.hh
@@ -4,12 +4,8 @@
 
 namespace nix {
 
-class StartProgressBar
-{
-    Logger * prev = 0;
-public:
-    StartProgressBar();
-    ~StartProgressBar();
-};
+void startProgressBar();
+
+void stopProgressBar();
 
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 8175d38e8ca..2fc93a05c51 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -6,6 +6,7 @@
 #include "local-store.hh"
 #include "finally.hh"
 #include "fs-accessor.hh"
+#include "progress-bar.hh"
 
 #if __linux__
 #include <sys/mount.h>
@@ -107,6 +108,8 @@ struct CmdRun : InstallablesCommand
         std::string cmd = *command.begin();
         Strings args = command;
 
+        stopProgressBar();
+
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location
            (e.g. /home/eelco/nix/store), then run the command in a

From 9b82ecbae04e36cb8f009510716268fa72a067b8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 15:22:05 +0200
Subject: [PATCH 0525/2196] nix search: Warn about cached results

---
 src/nix/search.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 5bb376f8f2d..6e099ce377f 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -185,6 +185,8 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         if (useCache && pathExists(jsonCacheFileName)) {
 
+            warn("using cached results; pass '-u' to update the cache");
+
             Value vRoot;
             parseJSON(*state, readFile(jsonCacheFileName), vRoot);
 

From e9c07a3b26a1e3056538a8fce49c9f7d9f1d8aba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 29 Aug 2017 16:18:00 +0200
Subject: [PATCH 0526/2196] nix edit / log: Operate on a single Installable

---
 src/nix/command.hh      | 20 ++++++++--
 src/nix/edit.cc         | 54 +++++++++++++-------------
 src/nix/installables.cc | 84 +++++++++++++++++++----------------------
 src/nix/log.cc          | 26 +++++++------
 src/nix/main.cc         |  3 ++
 src/nix/progress-bar.cc |  2 +-
 6 files changed, 100 insertions(+), 89 deletions(-)

diff --git a/src/nix/command.hh b/src/nix/command.hh
index b159408c3a9..479e6fd3742 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -51,7 +51,7 @@ struct Installable
 {
     virtual std::string what() = 0;
 
-    virtual Buildables toBuildable()
+    virtual Buildables toBuildable(bool singular = false)
     {
         throw Error("argument '%s' cannot be built", what());
     }
@@ -97,8 +97,6 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
         expectArgs("installables", &_installables);
     }
 
-    std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss);
-
     enum ToStorePathsMode { Build, NoBuild, DryRun };
 
     PathSet toStorePaths(ref<Store> store, ToStorePathsMode mode);
@@ -112,6 +110,22 @@ private:
     Strings _installables;
 };
 
+struct InstallableCommand : virtual Args, SourceExprCommand
+{
+    std::shared_ptr<Installable> installable;
+
+    InstallableCommand()
+    {
+        expectArg("installable", &_installable);
+    }
+
+    void prepare() override;
+
+private:
+
+    std::string _installable;
+};
+
 /* A command that operates on zero or more store paths. */
 struct StorePathsCommand : public InstallablesCommand
 {
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index fd07913bcfa..127be321eee 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdEdit : InstallablesCommand
+struct CmdEdit : InstallableCommand
 {
     std::string name() override
     {
@@ -34,44 +34,42 @@ struct CmdEdit : InstallablesCommand
     {
         auto state = getEvalState();
 
-        for (auto & i : installables) {
-            auto v = i->toValue(*state);
+        auto v = installable->toValue(*state);
 
-            Value * v2;
-            try {
-                auto dummyArgs = state->allocBindings(0);
-                v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v);
-            } catch (Error &) {
-                throw Error("package '%s' has no source location information", i->what());
-            }
+        Value * v2;
+        try {
+            auto dummyArgs = state->allocBindings(0);
+            v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v);
+        } catch (Error &) {
+            throw Error("package '%s' has no source location information", installable->what());
+        }
 
-            auto pos = state->forceString(*v2);
-            debug("position is %s", pos);
+        auto pos = state->forceString(*v2);
+        debug("position is %s", pos);
 
-            auto colon = pos.rfind(':');
-            if (colon == std::string::npos)
-                throw Error("cannot parse meta.position attribute '%s'", pos);
+        auto colon = pos.rfind(':');
+        if (colon == std::string::npos)
+            throw Error("cannot parse meta.position attribute '%s'", pos);
 
-            std::string filename(pos, 0, colon);
-            int lineno = std::stoi(std::string(pos, colon + 1));
+        std::string filename(pos, 0, colon);
+        int lineno = std::stoi(std::string(pos, colon + 1));
 
-            auto editor = getEnv("EDITOR", "cat");
+        auto editor = getEnv("EDITOR", "cat");
 
-            Strings args{editor};
+        Strings args{editor};
 
-            if (editor.find("emacs") != std::string::npos ||
-                editor.find("nano") != std::string::npos ||
-                editor.find("vim") != std::string::npos)
-                args.push_back(fmt("+%d", lineno));
+        if (editor.find("emacs") != std::string::npos ||
+            editor.find("nano") != std::string::npos ||
+            editor.find("vim") != std::string::npos)
+            args.push_back(fmt("+%d", lineno));
 
-            args.push_back(filename);
+        args.push_back(filename);
 
-            stopProgressBar();
+        stopProgressBar();
 
-            execvp(editor.c_str(), stringsToCharPtrs(args).data());
+        execvp(editor.c_str(), stringsToCharPtrs(args).data());
 
-            throw SysError("cannot run editor '%s'", editor);
-        }
+        throw SysError("cannot run editor '%s'", editor);
     }
 };
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 9d812fee47e..bdd273d509c 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -78,7 +78,7 @@ struct InstallableStoreDrv : Installable
 
     std::string what() override { return storePath; }
 
-    Buildables toBuildable() override
+    Buildables toBuildable(bool singular) override
     {
         return {{storePath, {}}};
     }
@@ -92,25 +92,21 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return storePath; }
 
-    Buildables toBuildable() override
+    Buildables toBuildable(bool singular) override
     {
         return {{storePath, {}}};
     }
 };
 
-struct InstallableExpr : Installable
+struct InstallableValue : Installable
 {
-    InstallablesCommand & installables;
-    std::string text;
-
-    InstallableExpr(InstallablesCommand & installables, const std::string & text)
-         : installables(installables), text(text) { }
+    SourceExprCommand & cmd;
 
-    std::string what() override { return text; }
+    InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
 
-    Buildables toBuildable() override
+    Buildables toBuildable(bool singular) override
     {
-        auto state = installables.getEvalState();
+        auto state = cmd.getEvalState();
 
         auto v = toValue(*state);
 
@@ -121,6 +117,9 @@ struct InstallableExpr : Installable
         DrvInfos drvs;
         getDerivations(*state, *v, "", autoArgs, drvs, false);
 
+        if (singular && drvs.size() != 1)
+            throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), drvs.size());
+
         Buildables res;
 
         for (auto & drv : drvs)
@@ -129,6 +128,16 @@ struct InstallableExpr : Installable
 
         return res;
     }
+};
+
+struct InstallableExpr : InstallableValue
+{
+    std::string text;
+
+    InstallableExpr(SourceExprCommand & cmd, const std::string & text)
+         : InstallableValue(cmd), text(text) { }
+
+    std::string what() override { return text; }
 
     Value * toValue(EvalState & state) override
     {
@@ -138,42 +147,19 @@ struct InstallableExpr : Installable
     }
 };
 
-struct InstallableAttrPath : Installable
+struct InstallableAttrPath : InstallableValue
 {
-    InstallablesCommand & installables;
     std::string attrPath;
 
-    InstallableAttrPath(InstallablesCommand & installables, const std::string & attrPath)
-        : installables(installables), attrPath(attrPath)
+    InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath)
+        : InstallableValue(cmd), attrPath(attrPath)
     { }
 
     std::string what() override { return attrPath; }
 
-    Buildables toBuildable() override
-    {
-        auto state = installables.getEvalState();
-
-        auto v = toValue(*state);
-
-        // FIXME
-        std::map<string, string> autoArgs_;
-        Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_));
-
-        DrvInfos drvs;
-        getDerivations(*state, *v, "", autoArgs, drvs, false);
-
-        Buildables res;
-
-        for (auto & drv : drvs)
-            for (auto & output : drv.queryOutputs())
-                res.emplace(output.second, Whence{output.first, drv.queryDrvPath()});
-
-        return res;
-    }
-
     Value * toValue(EvalState & state) override
     {
-        auto source = installables.getSourceExpr(state);
+        auto source = cmd.getSourceExpr(state);
 
         // FIXME
         std::map<string, string> autoArgs_;
@@ -190,20 +176,21 @@ struct InstallableAttrPath : Installable
 std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
 static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
 
-std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss)
+static std::vector<std::shared_ptr<Installable>> parseInstallables(
+    SourceExprCommand & cmd, ref<Store> store, Strings ss, bool useDefaultInstallables)
 {
     std::vector<std::shared_ptr<Installable>> result;
 
-    if (ss.empty() && useDefaultInstallables()) {
-        if (file == "")
-            file = ".";
+    if (ss.empty() && useDefaultInstallables) {
+        if (cmd.file == "")
+            cmd.file = ".";
         ss = Strings{""};
     }
 
     for (auto & s : ss) {
 
         if (s.compare(0, 1, "(") == 0)
-            result.push_back(std::make_shared<InstallableExpr>(*this, s));
+            result.push_back(std::make_shared<InstallableExpr>(cmd, s));
 
         else if (s.find("/") != std::string::npos) {
 
@@ -218,7 +205,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))
-            result.push_back(std::make_shared<InstallableAttrPath>(*this, s));
+            result.push_back(std::make_shared<InstallableAttrPath>(cmd, s));
 
         else
             throw UsageError("don't know what to do with argument '%s'", s);
@@ -250,7 +237,14 @@ PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mod
 
 void InstallablesCommand::prepare()
 {
-    installables = parseInstallables(getStore(), _installables);
+    installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables());
+}
+
+void InstallableCommand::prepare()
+{
+    auto installables = parseInstallables(*this, getStore(), {_installable}, false);
+    assert(installables.size() == 1);
+    installable = installables.front();
 }
 
 }
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 0fb45c1454c..ba5e71c10a0 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -2,10 +2,11 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
-struct CmdLog : InstallablesCommand
+struct CmdLog : InstallableCommand
 {
     CmdLog()
     {
@@ -27,21 +28,22 @@ struct CmdLog : InstallablesCommand
 
         subs.push_front(store);
 
-        for (auto & inst : installables) {
-            for (auto & b : inst->toBuildable()) {
-                auto path = b.second.drvPath != "" ? b.second.drvPath : b.first;
-                bool found = false;
-                for (auto & sub : subs) {
-                    auto log = sub->getBuildLog(path);
+        for (auto & b : installable->toBuildable(true)) {
+
+            for (auto & sub : subs) {
+                auto log = b.second.drvPath != "" ? sub->getBuildLog(b.second.drvPath) : nullptr;
+                if (!log) {
+                    log = sub->getBuildLog(b.first);
                     if (!log) continue;
-                    std::cout << *log;
-                    found = true;
-                    break;
                 }
-                if (!found)
-                    throw Error("build log of path '%s' is not available", path);
+                stopProgressBar();
+                printInfo("got build log for '%s' from '%s'", b.first, sub->getUri());
+                std::cout << *log;
+                return;
             }
         }
+
+        throw Error("build log of '%s' is not available", installable->what());
     }
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 9781b58544f..ec9b58b20fe 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -8,6 +8,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
+#include "finally.hh"
 
 extern std::string chrootHelperName;
 
@@ -84,6 +85,8 @@ void mainWrapped(int argc, char * * argv)
 
     if (!args.command) args.showHelpAndExit();
 
+    Finally f([]() { stopProgressBar(); });
+
     if (isatty(STDERR_FILENO))
         startProgressBar();
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index f48cfa13b4c..9bc40523970 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -116,7 +116,7 @@ class ProgressBar : public Logger
     {
         auto state(state_.lock());
         if (!state->active) return;
-        state->active = true;
+        state->active = false;
         std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
         if (status != "")

From 9f47eac92bd80604e51cb710656133b5cd6be3ed Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 11:04:52 +0200
Subject: [PATCH 0527/2196] Fix mismatched tag warning

https://hydra.nixos.org/build/59649086
---
 src/libutil/logging.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 84fffa820f4..414309a5246 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -40,7 +40,7 @@ typedef uint64_t ActivityId;
 
 class Logger
 {
-    friend class Activity;
+    friend struct Activity;
 
 public:
 

From fabde432dc046183b1eeaf9265effa47e89c3236 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 11:05:18 +0200
Subject: [PATCH 0528/2196] Fix build failure on non-Linux

https://hydra.nixos.org/build/59649086
---
 src/nix/run.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 2fc93a05c51..fcde540fc2d 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -202,6 +202,6 @@ void chrootHelper(int argc, char * * argv)
     throw SysError("unable to exec '%s'", cmd);
 
 #else
-    throw Error("mounting the Nix store on '%s' is not supported on this platform", >storeDir);
+    throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir);
 #endif
 }

From 7d4a7136dba7ceb3d5b080b092eb76d23e2bb549 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 12:52:07 +0200
Subject: [PATCH 0529/2196] More macOS build fixes

---
 src/nix/run.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index fcde540fc2d..f016b04a1ec 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -16,6 +16,8 @@ using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
 
+extern char * * environ;
+
 struct CmdRun : InstallablesCommand
 {
     Strings command = { "bash" };
@@ -85,7 +87,7 @@ struct CmdRun : InstallablesCommand
                 if (s) kept[var] = s;
             }
 
-            clearenv();
+            environ = nullptr;
 
             for (auto & var : kept)
                 setenv(var.first.c_str(), var.second.c_str(), 1);

From c2154d4c8422ddc1c201d503bb52edff854af2ad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 14:28:25 +0200
Subject: [PATCH 0530/2196] Rename a few configuration options

In particular, drop the "build-" and "gc-" prefixes which are
pointless. So now you can say

  nix build --no-sandbox

instead of

  nix build --no-build-use-sandbox
---
 doc/manual/command-ref/conf-file.xml  | 57 +++++++++---------
 doc/manual/command-ref/nix-store.xml  |  4 +-
 doc/manual/command-ref/opt-common.xml |  8 +--
 scripts/install-darwin-multi-user.sh  |  6 +-
 src/libmain/shared.cc                 |  8 +--
 src/libstore/build.cc                 |  6 +-
 src/libstore/gc.cc                    | 10 ++--
 src/libstore/globals.hh               | 84 +++++++++++++++------------
 tests/build-remote.sh                 |  2 +-
 tests/init.sh                         |  4 +-
 tests/linux-sandbox.sh                |  2 +-
 tests/logging.sh                      |  2 +-
 tests/multiple-outputs.sh             |  2 +-
 tests/setuid.nix                      | 16 ++---
 tests/shell.shebang.sh                |  2 +-
 tests/timeout.sh                      |  4 +-
 16 files changed, 113 insertions(+), 104 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 47ceff2625e..6b90083f087 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -45,13 +45,12 @@ Comments start with a <literal>#</literal> character.  Here is an
 example configuration file:</para>
 
 <programlisting>
-gc-keep-outputs = true       # Nice for developers
-gc-keep-derivations = true   # Idem
-env-keep-derivations = false
+keep-outputs = true       # Nice for developers
+keep-derivations = true   # Idem
 </programlisting>
 
 <para>You can override settings on the command line using the
-<option>--option</option> flag, e.g. <literal>--option gc-keep-outputs
+<option>--option</option> flag, e.g. <literal>--option keep-outputs
 false</literal>.</para>
 
 <para>The following settings are currently available:
@@ -59,7 +58,7 @@ false</literal>.</para>
 <variablelist>
 
 
-  <varlistentry xml:id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
+  <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
     will keep the outputs of non-garbage derivations.  If
@@ -76,7 +75,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
+  <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term>
 
     <listitem><para>If <literal>true</literal> (default), the garbage
     collector will keep the derivations from which non-garbage store
@@ -88,12 +87,12 @@ false</literal>.</para>
     traceability (e.g., it allows you to ask with what dependencies or
     options a store path was built), so by default this option is on.
     Turn it off to save a bit of disk space (or a lot if
-    <literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
+    <literal>keep-outputs</literal> is also turned on).</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>env-keep-derivations</literal></term>
+  <varlistentry><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
     are not stored in Nix user environments.  That is, the derivation
@@ -105,19 +104,19 @@ false</literal>.</para>
     garbage-collected until the user environment generation is deleted
     (<command>nix-env --delete-generations</command>).  To prevent
     build-time-only dependencies from being collected, you should also
-    turn on <literal>gc-keep-outputs</literal>.</para>
+    turn on <literal>keep-outputs</literal>.</para>
 
     <para>The difference between this option and
-    <literal>gc-keep-derivations</literal> is that this one is
+    <literal>keep-derivations</literal> is that this one is
     “sticky”: it applies to any user environment created while this
-    option was enabled, while <literal>gc-keep-derivations</literal>
+    option was enabled, while <literal>keep-derivations</literal>
     only applies at the moment the garbage collector is
     run.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-max-jobs"><term><literal>build-max-jobs</literal></term>
+  <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
 
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
@@ -130,7 +129,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-cores"><term><literal>build-cores</literal></term>
+  <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
 
     <listitem><para>Sets the value of the
     <envar>NIX_BUILD_CORES</envar> environment variable in the
@@ -149,7 +148,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-max-silent-time"><term><literal>build-max-silent-time</literal></term>
+  <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
 
     <listitem>
 
@@ -170,7 +169,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-timeout"><term><literal>build-timeout</literal></term>
+  <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
     <listitem>
 
@@ -190,7 +189,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-max-log-size"><term><literal>build-max-log-size</literal></term>
+  <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
 
     <listitem>
 
@@ -245,7 +244,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>build-use-sandbox</literal></term>
+  <varlistentry><term><literal>sandbox</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, builds will be
     performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
@@ -254,7 +253,7 @@ false</literal>.</para>
     directory, private versions of <filename>/proc</filename>,
     <filename>/dev</filename>, <filename>/dev/shm</filename> and
     <filename>/dev/pts</filename> (on Linux), and the paths configured with the
-    <link linkend='conf-build-sandbox-paths'><literal>build-sandbox-paths</literal>
+    <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal>
     option</link>. This is useful to prevent undeclared dependencies
     on files in directories such as <filename>/usr/bin</filename>. In
     addition, on Linux, builds run in private PID, mount, network, IPC
@@ -280,8 +279,8 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-sandbox-paths">
-    <term><literal>build-sandbox-paths</literal></term>
+  <varlistentry xml:id="conf-sandbox-paths">
+    <term><literal>sandbox-paths</literal></term>
 
     <listitem><para>A list of paths bind-mounted into Nix sandbox
     environments. You can use the syntax
@@ -303,17 +302,17 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-extra-sandbox-paths">
+  <varlistentry xml:id="conf-extra-sandbox-paths">
     <term><literal>build-extra-sandbox-paths</literal></term>
 
     <listitem><para>A list of additional paths appended to
-    <option>build-sandbox-paths</option>. Useful if you want to extend
+    <option>sandbox-paths</option>. Useful if you want to extend
     its default value.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>build-use-substitutes</literal></term>
+  <varlistentry><term><literal>use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
@@ -322,7 +321,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>build-fallback</literal></term>
+  <varlistentry><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
     back to building from source if a binary substitute fails.  This
@@ -332,7 +331,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>build-keep-log</literal></term>
+  <varlistentry><term><literal>keep-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     Nix will write the build log of a derivation (i.e. the standard
@@ -344,7 +343,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>build-compress-log</literal></term>
+  <varlistentry><term><literal>compress-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     build logs written to <filename>/nix/var/log/nix/drvs</filename>
@@ -597,7 +596,7 @@ password <replaceable>my-password</replaceable>
             <para>Pass a list of files and directories to be included in the
             sandbox for this build. One entry per line, terminated by an empty
             line. Entries have the same format as
-            <literal>build-sandbox-paths</literal>.</para>
+            <literal>sandbox-paths</literal>.</para>
 
           </listitem>
 
@@ -608,7 +607,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-build-repeat"><term><literal>build-repeat</literal></term>
+  <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
     they are deterministic. The default value is 0. If the value is
@@ -651,7 +650,7 @@ password <replaceable>my-password</replaceable>
     as <command>sudo</command> or <command>ping</command> will
     fail. (Note that in sandbox builds, no such programs are available
     unless you bind-mount them into the sandbox via the
-    <option>build-sandbox-paths</option> option.) You can allow the
+    <option>sandbox-paths</option> option.) You can allow the
     use of such programs by enabling this option. This is impure and
     usually undesirable, but may be useful in certain scenarios
     (e.g. to spin up containers or set up userspace network interfaces
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 19c99841a0a..a5f615b0c26 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -397,9 +397,9 @@ options control what gets deleted and in what order:
 </para>
 
 <para>The behaviour of the collector is also influenced by the <link
-linkend="conf-gc-keep-outputs"><literal>gc-keep-outputs</literal></link>
+linkend="conf-keep-outputs"><literal>keep-outputs</literal></link>
 and <link
-linkend="conf-gc-keep-derivations"><literal>gc-keep-derivations</literal></link>
+linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
 <para>With <option>--delete</option>, the collector prints the total
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index a930b4a0d78..32d53c753a2 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -94,7 +94,7 @@
   perform in parallel to the specified number.  Specify
   <literal>auto</literal> to use the number of CPUs in the system.
   The default is specified by the <link
-  linkend='conf-build-max-jobs'><literal>build-max-jobs</literal></link>
+  linkend='conf-max-jobs'><literal>max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
   exploit I/O latency.</para></listitem>
@@ -112,7 +112,7 @@
   <literal>true</literal>, the builder passes the
   <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
   It defaults to the value of the <link
-  linkend='conf-build-cores'><literal>build-cores</literal></link>
+  linkend='conf-cores'><literal>cores</literal></link>
   configuration setting, if set, or <literal>1</literal> otherwise.
   The value <literal>0</literal> means that the builder should use all
   available CPU cores in the system.</para></listitem>
@@ -125,7 +125,7 @@
   <listitem><para>Sets the maximum number of seconds that a builder
   can go without producing any data on standard output or standard
   error.  The default is specified by the <link
-  linkend='conf-build-max-silent-time'><literal>build-max-silent-time</literal></link>
+  linkend='conf-max-silent-time'><literal>max-silent-time</literal></link>
   configuration setting.  <literal>0</literal> means no
   time-out.</para></listitem>
 
@@ -135,7 +135,7 @@
 
   <listitem><para>Sets the maximum number of seconds that a builder
   can run.  The default is specified by the <link
-  linkend='conf-build-timeout'><literal>build-timeout</literal></link>
+  linkend='conf-timeout'><literal>timeout</literal></link>
   configuration setting.  <literal>0</literal> means no
   timeout.</para></listitem>
 
diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index a2e120bf862..5b466ac3c88 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -744,9 +744,9 @@ place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
 build-users-group = $NIX_BUILD_GROUP_NAME
 
-build-max-jobs = $NIX_USER_COUNT
-build-cores = 1
-build-use-sandbox = false
+max-jobs = $NIX_USER_COUNT
+cores = 1
+sandbox = false
 
 binary-caches = https://cache.nixos.org/
 trusted-binary-caches =
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 67178b4a891..aa28ff2e53a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -171,7 +171,7 @@ struct LegacyArgs : public MixCommonArgs
         });
 
         mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
-            settings.set("build-max-jobs", s);
+            settings.set("max-jobs", s);
         });
 
         auto intSettingAlias = [&](char shortName, const std::string & longName,
@@ -181,9 +181,9 @@ struct LegacyArgs : public MixCommonArgs
             });
         };
 
-        intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores");
-        intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time");
-        intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout");
+        intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
+        intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
+        intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
 
         mkFlag(0, "readonly-mode", "do not write to the Nix store",
             &settings.readOnlyMode);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 02dcd4536a5..39f6128aafd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1740,11 +1740,11 @@ void DerivationGoal::startBuilder()
         if (settings.sandboxMode == smEnabled) {
             if (get(drv->env, "__noChroot") == "1")
                 throw Error(format("derivation '%1%' has '__noChroot' set, "
-                    "but that's not allowed when 'build-use-sandbox' is 'true'") % drvPath);
+                    "but that's not allowed when 'sandbox' is 'true'") % drvPath);
 #if __APPLE__
             if (additionalSandboxProfile != "")
                 throw Error(format("derivation '%1%' specifies a sandbox profile, "
-                    "but this is only allowed when 'build-use-sandbox' is 'relaxed'") % drvPath);
+                    "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath);
 #endif
             useChroot = true;
         }
@@ -1832,7 +1832,7 @@ void DerivationGoal::startBuilder()
                     worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
-                throw Error(format("while processing 'build-sandbox-paths': %s") % e.what());
+                throw Error(format("while processing 'sandbox-paths': %s") % e.what());
             }
         for (auto & i : closure)
             dirsInChroot[i] = i;
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index bd335fbb49e..5e3958ea52a 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -580,7 +580,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
     /* Don't delete this path if any of its referrers are alive. */
     queryReferrers(path, incoming);
 
-    /* If gc-keep-derivations is set and this is a derivation, then
+    /* If keep-derivations is set and this is a derivation, then
        don't delete the derivation if any of the outputs are alive. */
     if (state.gcKeepDerivations && isDerivation(path)) {
         PathSet outputs = queryDerivationOutputs(path);
@@ -589,7 +589,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
                 incoming.insert(i);
     }
 
-    /* If gc-keep-outputs is set, then don't delete this path if there
+    /* If keep-outputs is set, then don't delete this path if there
        are derivers of this path that are not garbage. */
     if (state.gcKeepOutputs) {
         PathSet derivers = queryValidDerivers(path);
@@ -704,9 +704,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     state.gcKeepDerivations = settings.gcKeepDerivations;
 
     /* Using `--ignore-liveness' with `--delete' can have unintended
-       consequences if `gc-keep-outputs' or `gc-keep-derivations' are
-       true (the garbage collector will recurse into deleting the
-       outputs or derivers, respectively).  So disable them. */
+       consequences if `keep-outputs' or `keep-derivations' are true
+       (the garbage collector will recurse into deleting the outputs
+       or derivers, respectively).  So disable them. */
     if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
         state.gcKeepOutputs = false;
         state.gcKeepDerivations = false;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 9ebbf7b477b..c20d147f5d3 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -89,8 +89,9 @@ public:
     Setting<bool> keepGoing{this, false, "keep-going",
         "Whether to keep building derivations when another build fails."};
 
-    Setting<bool> tryFallback{this, false, "build-fallback",
-        "Whether to fall back to building when substitution fails."};
+    Setting<bool> tryFallback{this, false, "fallback",
+        "Whether to fall back to building when substitution fails.",
+        {"build-fallback"}};
 
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
@@ -99,14 +100,15 @@ public:
        the log to show if a build fails. */
     size_t logLines = 10;
 
-    MaxBuildJobsSetting maxBuildJobs{this, 1, "build-max-jobs",
-        "Maximum number of parallel build jobs. \"auto\" means use number of cores."};
+    MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
+        "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
+        {"build-max-jobs"}};
 
-    Setting<unsigned int> buildCores{this, getDefaultCores(), "build-cores",
+    Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
         "Number of CPU cores to utilize in parallel within a build, "
         "i.e. by passing this number to Make via '-j'. 0 means that the "
         "number of actual CPU cores on the local host ought to be "
-        "auto-detected."};
+        "auto-detected.", {"build-cores"}};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
@@ -115,14 +117,15 @@ public:
     Setting<std::string> thisSystem{this, SYSTEM, "system",
         "The canonical Nix system name."};
 
-    Setting<time_t> maxSilentTime{this, 0, "build-max-silent-time",
+    Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
         "The maximum time in seconds that a builer can go without "
         "producing any output on stdout/stderr before it is killed. "
-        "0 means infinity."};
+        "0 means infinity.",
+        {"build-max-silent-time"}};
 
-    Setting<time_t> buildTimeout{this, 0, "build-timeout",
+    Setting<time_t> buildTimeout{this, 0, "timeout",
         "The maximum duration in seconds that a builder can run. "
-        "0 means infinity."};
+        "0 means infinity.", {"build-timeout"}};
 
     Setting<bool> useBuildHook{this, true, "remote-builds",
         "Whether to use build hooks (for distributed builds)."};
@@ -149,27 +152,32 @@ public:
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
         "Whether to call sync() before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "build-use-substitutes",
-        "Whether to use substitutes."};
+    Setting<bool> useSubstitutes{this, true, "use-substitutes",
+        "Whether to use substitutes.",
+        {"build-use-substitutes"}};
 
     Setting<std::string> buildUsersGroup{this, "", "build-users-group",
         "The Unix group that contains the build users."};
 
-    Setting<bool> impersonateLinux26{this, false, "build-impersonate-linux-26",
-        "Whether to impersonate a Linux 2.6 machine on newer kernels."};
+    Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
+        "Whether to impersonate a Linux 2.6 machine on newer kernels.",
+        {"build-impersonate-linux-26"}};
 
-    Setting<bool> keepLog{this, true, "build-keep-log",
-        "Whether to store build logs."};
+    Setting<bool> keepLog{this, true, "keep-build-log",
+        "Whether to store build logs.",
+        {"build-keep-log"}};
 
-    Setting<bool> compressLog{this, true, "build-compress-log",
-        "Whether to compress logs."};
+    Setting<bool> compressLog{this, true, "compress-build-log",
+        "Whether to compress logs.",
+        {"build-compress-log"}};
 
-    Setting<unsigned long> maxLogSize{this, 0, "build-max-log-size",
+    Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
         "Maximum number of bytes a builder can write to stdout/stderr "
-        "before being killed (0 means no limit)."};
+        "before being killed (0 means no limit).",
+        {"build-max-log-size"}};
 
-    /* When build-repeat > 0 and verboseBuild == true, whether to
-       print repeated builds (i.e. builds other than the first one) to
+    /* When buildRepeat > 0 and verboseBuild == true, whether to print
+       repeated builds (i.e. builds other than the first one) to
        stderr. Hack to prevent Hydra logs from being polluted. */
     bool printRepeatedBuilds = true;
 
@@ -180,18 +188,21 @@ public:
         "Whether to check if new GC roots can in fact be found by the "
         "garbage collector."};
 
-    Setting<bool> gcKeepOutputs{this, false, "gc-keep-outputs",
-        "Whether the garbage collector should keep outputs of live derivations."};
+    Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
+        "Whether the garbage collector should keep outputs of live derivations.",
+        {"gc-keep-outputs"}};
 
-    Setting<bool> gcKeepDerivations{this, true, "gc-keep-derivations",
-        "Whether the garbage collector should keep derivers of live paths."};
+    Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
+        "Whether the garbage collector should keep derivers of live paths.",
+        {"gc-keep-derivations"}};
 
     Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
         "Whether to automatically replace files with identical contents with hard links."};
 
-    Setting<bool> envKeepDerivations{this, false, "env-keep-derivations",
+    Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
         "Whether to add derivations as a dependency of user environments "
-        "(to prevent them from being GCed)."};
+        "(to prevent them from being GCed).",
+        {"env-keep-derivations"}};
 
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
@@ -202,24 +213,25 @@ public:
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<SandboxMode> sandboxMode{this, smDisabled, "build-use-sandbox",
+    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
-        {"build-use-chroot"}};
+        {"build-use-chroot", "build-use-sandbox"}};
 
-    Setting<PathSet> sandboxPaths{this, {}, "build-sandbox-paths",
+    Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
         "The paths to make available inside the build sandbox.",
-        {"build-chroot-dirs"}};
+        {"build-chroot-dirs", "build-sandbox-paths"}};
 
-    Setting<PathSet> extraSandboxPaths{this, {}, "build-extra-sandbox-paths",
+    Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
         "Additional paths to make available inside the build sandbox.",
-        {"build-extra-chroot-dirs"}};
+        {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
     Setting<bool> restrictEval{this, false, "restrict-eval",
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and to disallow fetching files from the network."};
 
-    Setting<size_t> buildRepeat{this, 0, "build-repeat",
-        "The number of times to repeat a build in order to verify determinism."};
+    Setting<size_t> buildRepeat{this, 0, "repeat",
+        "The number of times to repeat a build in order to verify determinism.",
+        {"build-repeat"}};
 
 #if __linux__
     Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 927a217f376..e27ce7e25a8 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -13,7 +13,7 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 export NIX_CONF_DIR=$TEST_ROOT/etc2
 mkdir -p $NIX_CONF_DIR
 echo "
-build-sandbox-paths = /nix/store
+sandbox-paths = /nix/store
 sandbox-build-dir = /build-tmp
 " > $NIX_CONF_DIR/nix.conf
 
diff --git a/tests/init.sh b/tests/init.sh
index 4571b75b859..41cca047d8f 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -15,9 +15,7 @@ mkdir "$NIX_CONF_DIR"
 
 cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
-gc-keep-outputs = false
-gc-keep-derivations = false
-env-keep-derivations = false
+keep-derivations = false
 fsync-metadata = false
 EOF
 
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 54cdef8178e..7f4e83b560b 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -16,7 +16,7 @@ rm -rf $TEST_ROOT/store0
 export NIX_STORE_DIR=/my/store
 export NIX_REMOTE="local?root=$TEST_ROOT/store0"
 
-outPath=$(nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store)
+outPath=$(nix-build dependencies.nix --no-out-link --option sandbox-paths /nix/store)
 
 [[ $outPath =~ /my/store/.*-dependencies ]]
 
diff --git a/tests/logging.sh b/tests/logging.sh
index 86f32bade94..d38136531ba 100644
--- a/tests/logging.sh
+++ b/tests/logging.sh
@@ -11,5 +11,5 @@ path=$(nix-build dependencies.nix --no-out-link)
 clearStore
 rm -rf $NIX_LOG_DIR
 (! nix-store -l $path)
-nix-build dependencies.nix --no-out-link --option build-compress-log true
+nix-build dependencies.nix --no-out-link --option compress-build-log true
 [ "$(nix-store -l $path)" = FOO ]
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index 7ac77908151..47698b10381 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -59,5 +59,5 @@ fi
 
 echo "collecting garbage..."
 rm $TEST_ROOT/result*
-nix-store --gc --option gc-keep-derivations true --option gc-keep-outputs true
+nix-store --gc --option keep-derivations true --option keep-outputs true
 nix-store --gc --print-roots
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 2508549c546..7a12b4fb316 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -20,7 +20,7 @@ makeTest {
       startAll;
 
       # Copying to /tmp should succeed.
-      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
       ")\' ');
@@ -30,7 +30,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # Creating a setuid binary should fail.
-      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 4755 /tmp/id
@@ -41,7 +41,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # Creating a setgid binary should fail.
-      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 2755 /tmp/id
@@ -52,7 +52,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # The checks should also work on 32-bit binaries.
-      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
+      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 2755 /tmp/id
@@ -63,7 +63,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # The tests above use fchmodat(). Test chmod() as well.
-      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"chmod 0666, qw(/tmp/id) or die\"
@@ -73,7 +73,7 @@ makeTest {
 
       $machine->succeed("rm /tmp/id");
 
-      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"chmod 04755, qw(/tmp/id) or die\"
@@ -84,7 +84,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # And test fchmod().
-      $machine->succeed('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\"
@@ -94,7 +94,7 @@ makeTest {
 
       $machine->succeed("rm /tmp/id");
 
-      $machine->fail('nix-build --option build-use-sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\"
diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh
index c9a83aaf83d..a6c4bc94592 100755
--- a/tests/shell.shebang.sh
+++ b/tests/shell.shebang.sh
@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --option build-use-substitutes false
+#! nix-shell -I nixpkgs=shell.nix --option use-substitutes false
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"
diff --git a/tests/timeout.sh b/tests/timeout.sh
index 2b864b86dca..d3d85200fa5 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -15,7 +15,7 @@ if ! echo "$messages" | grep -q "timed out"; then
     exit 1
 fi
 
-if nix-build -Q timeout.nix -A infiniteLoop --option build-max-log-size 100; then
+if nix-build -Q timeout.nix -A infiniteLoop --option max-build-log-size 100; then
     echo "build should have failed"
     exit 1
 fi
@@ -30,7 +30,7 @@ if nix-build timeout.nix -A closeLog; then
     exit 1
 fi
 
-if nix build -f timeout.nix silent --option build-max-silent-time 2; then
+if nix build -f timeout.nix silent --max-silent-time 2; then
     echo "build should have failed"
     exit 1
 fi

From fd73c1e20a7aaefcb69db3e6b2f081e1a5e20406 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 15:25:58 +0200
Subject: [PATCH 0531/2196] Add an activity for binary cache queries

---
 src/libstore/binary-cache-store.cc |  7 +++++++
 src/libstore/download.cc           |  2 +-
 src/libutil/logging.hh             |  1 +
 src/nix/progress-bar.cc            | 13 ++++++++++---
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 403b4687287..556fa3d5935 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -313,6 +313,11 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
         std::function<void(std::shared_ptr<ValidPathInfo>)> success,
         std::function<void(std::exception_ptr exc)> failure)
 {
+    auto uri = getUri();
+    auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
+        fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
+    PushActivity pact(act->id);
+
     auto narInfoFile = narInfoFileFor(storePath);
 
     getFile(narInfoFile,
@@ -323,6 +328,8 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
 
             callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>)
                 std::make_shared<NarInfo>(*this, *data, narInfoFile));
+
+            (void) act; // force Activity into this lambda to ensure it stays alive
         },
         failure);
 }
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 625e781b452..054244c7b8b 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -85,7 +85,7 @@ struct CurlDownloader : public Downloader
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {}, request.parentAct)
+            , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct)
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 414309a5246..21898c03a27 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -24,6 +24,7 @@ typedef enum {
     actOptimiseStore = 106,
     actVerifyPaths = 107,
     actSubstitute = 108,
+    actQueryPathInfo = 109,
 } ActivityType;
 
 typedef enum {
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 9bc40523970..0c42fe5cc13 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -138,11 +138,11 @@ class ProgressBar : public Logger
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) override
     {
-        if (lvl <= verbosity && !s.empty())
-            log(lvl, s + "...");
-
         auto state(state_.lock());
 
+        if (lvl <= verbosity && !s.empty())
+            log(*state, lvl, s + "...");
+
         state->activities.emplace_back(ActInfo());
         auto i = std::prev(state->activities.end());
         i->s = s;
@@ -163,7 +163,13 @@ class ProgressBar : public Logger
             i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1));
         }
 
+        if (type == actQueryPathInfo) {
+            auto name = storePathToName(getS(fields, 0));
+            i->s = fmt("querying about " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
+        }
+
         if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))
+            || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent))
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
@@ -189,6 +195,7 @@ class ProgressBar : public Logger
 
         auto i = state->its.find(act);
         if (i != state->its.end()) {
+
             auto & actByType = state->activitiesByType[i->second->type];
             actByType.done += i->second->done;
             actByType.failed += i->second->failed;

From bbdf08bc0facb5157a10c794712dae7e5902be03 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 16:02:36 +0200
Subject: [PATCH 0532/2196] Call queryMissing() prior to building

Without this, substitute info is fetched sequentially, which is
superslow. In the old UI (e.g. nix-build), we call printMissing(),
which calls queryMissing(), thereby preheating the binary cache
cache. But the new UI doesn't do that.
---
 src/libstore/build.cc | 12 ++++++++++++
 src/libstore/misc.cc  |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 39f6128aafd..ce41752e623 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4166,10 +4166,20 @@ void Worker::markContentsGood(const Path & path)
 //////////////////////////////////////////////////////////////////////
 
 
+static void primeCache(Store & store, const PathSet & paths)
+{
+    PathSet willBuild, willSubstitute, unknown;
+    unsigned long long downloadSize, narSize;
+    store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+}
+
+
 void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 {
     Worker worker(*this);
 
+    primeCache(*this, drvPaths);
+
     Goals goals;
     for (auto & i : drvPaths) {
         DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i);
@@ -4220,6 +4230,8 @@ void LocalStore::ensurePath(const Path & path)
     /* If the path is already valid, we're done. */
     if (isValidPath(path)) return;
 
+    primeCache(*this, {path});
+
     Worker worker(*this);
     GoalPtr goal = worker.makeSubstitutionGoal(path);
     Goals goals = {goal};
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 8fa84d3a278..a82aa4e9cfa 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -107,6 +107,8 @@ void Store::queryMissing(const PathSet & targets,
     PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
     unsigned long long & downloadSize_, unsigned long long & narSize_)
 {
+    Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
+
     downloadSize_ = narSize_ = 0;
 
     ThreadPool pool;

From c9857ef262c391672b47e7714e61f1686b684e4d Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyfox@gentoo.org>
Date: Thu, 31 Aug 2017 21:41:36 +0100
Subject: [PATCH 0533/2196] src/libmain/stack.cc: fix 'ucontext' usage on
 glibc-2.26

Build fails as:

$ make
  CXX    src/libmain/stack.o
src/libmain/stack.cc: In function 'void nix::sigsegvHandler(int, siginfo_t*, void*)':
src/libmain/stack.cc:21:21: error: 'ucontext' was not declared in this scope
     sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
                     ^~~~~~~~
src/libmain/stack.cc:21:21: note: suggested alternative: 'ucontext_t'
     sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
                     ^~~~~~~~
                     ucontext_t

It's caused by upstream rename:
https://sourceware.org/git/?p=glibc.git;a=commitdiff;h=251287734e89a52da3db682a8241eb6bccc050c9

which basically changes
    typedef struct ucontext {} ucontext_t;
to
    typedef struct ucontext_t {} ucontext_t;

The change uses ucontext_t.

Signed-off-by: Sergei Trofimovich <slyfox@gentoo.org>
---
 src/libmain/stack.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index ab3baf62c3d..cc0eea68fca 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -18,9 +18,9 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
     bool haveSP = true;
     char * sp = 0;
 #if defined(__x86_64__) && defined(REG_RSP)
-    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
+    sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP];
 #elif defined(REG_ESP)
-    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP];
+    sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP];
 #else
     haveSP = false;
 #endif

From 7a108d904e9d3bb9ef94487c37b35a050c315a52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 Aug 2017 17:57:04 +0200
Subject: [PATCH 0534/2196] Fix verbosity level for nix build --dry-run

---
 src/libmain/shared.cc   | 24 ++++++++++++------------
 src/libmain/shared.hh   |  4 ++--
 src/nix/build.cc        |  2 +-
 src/nix/installables.cc |  2 +-
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index aa28ff2e53a..6393d80bbf0 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -34,40 +34,40 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const PathSet & paths)
+void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
 {
     unsigned long long downloadSize, narSize;
     PathSet willBuild, willSubstitute, unknown;
     store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-    printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
 }
 
 
 void printMissing(ref<Store> store, const PathSet & willBuild,
     const PathSet & willSubstitute, const PathSet & unknown,
-    unsigned long long downloadSize, unsigned long long narSize)
+    unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
-        printInfo(format("these derivations will be built:"));
+        printMsg(lvl, "these derivations will be built:");
         Paths sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
-            printInfo(format("  %1%") % i);
+            printMsg(lvl, fmt("  %s", i));
     }
 
     if (!willSubstitute.empty()) {
-        printInfo(format("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):")
-            % (downloadSize / (1024.0 * 1024.0))
-            % (narSize / (1024.0 * 1024.0)));
+        printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                downloadSize / (1024.0 * 1024.0),
+                narSize / (1024.0 * 1024.0)));
         for (auto & i : willSubstitute)
-            printInfo(format("  %1%") % i);
+            printMsg(lvl, fmt("  %s", i));
     }
 
     if (!unknown.empty()) {
-        printInfo(format("don't know how to build these paths%1%:")
-            % (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""));
+        printMsg(lvl, fmt("don't know how to build these paths%s:",
+                (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
         for (auto & i : unknown)
-            printInfo(format("  %1%") % i);
+            printMsg(lvl, fmt("  %s", i));
     }
 }
 
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index becf258734e..2a1e42dd977 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -35,11 +35,11 @@ void printGCWarning();
 
 class Store;
 
-void printMissing(ref<Store> store, const PathSet & paths);
+void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo);
 
 void printMissing(ref<Store> store, const PathSet & willBuild,
     const PathSet & willSubstitute, const PathSet & unknown,
-    unsigned long long downloadSize, unsigned long long narSize);
+    unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
 
 string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index cc96ac48a97..4239dd3fff9 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -25,7 +25,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
     {
         auto paths = toStorePaths(store, dryRun ? DryRun : Build);
 
-        printInfo("build result: %s", showPaths(paths));
+        printError("build result: %s", showPaths(paths));
     }
 };
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index bdd273d509c..fdb6004fb31 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -228,7 +228,7 @@ PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mod
         }
 
     if (mode == DryRun)
-        printMissing(store, buildables);
+        printMissing(store, buildables, lvlError);
     else if (mode == Build)
         store->buildPaths(buildables);
 

From 8215b75d36a6c60649dfc8721b8ddd44fbcf697c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 Sep 2017 16:51:26 +0200
Subject: [PATCH 0535/2196] Abort curl downloads if there is no progress for 5
 minutes

Maybe this will fix the curl hangs on macOS. (We could also use
CURLOPT_TIMEOUT but that seems more of a sledgehammer.)
---
 src/libstore/download.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 054244c7b8b..f5304daeefd 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -183,6 +183,8 @@ struct CurlDownloader : public Downloader
             return 0;
         }
 
+        long lowSpeedTimeout = 300;
+
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -231,6 +233,9 @@ struct CurlDownloader : public Downloader
 
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get());
 
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
+
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
@@ -422,7 +427,7 @@ struct CurlDownloader : public Downloader
             auto sleepTimeMs =
                 nextWakeup != std::chrono::steady_clock::time_point()
                 ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
-                : 1000000000;
+                : 10000;
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)

From b932ea58ec610830ed3141bb14fbd812aa66b2c1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Sep 2017 20:39:57 +0200
Subject: [PATCH 0536/2196] GC: Don't delete own temproots file

Since file locks are per-process rather than per-file-descriptor, the
garbage collector would always acquire a lock on its own temproots
file and conclude that it's stale.
---
 src/libstore/gc.cc          | 57 ++++++++++++++++++-------------------
 src/libstore/local-store.cc |  8 ++++--
 src/libstore/local-store.hh |  3 +-
 3 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 5e3958ea52a..534db8c6e4f 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -18,7 +18,6 @@ namespace nix {
 
 
 static string gcLockName = "gc.lock";
-static string tempRootsDir = "temproots";
 static string gcRootsDir = "gcroots";
 
 
@@ -153,30 +152,25 @@ void LocalStore::addTempRoot(const Path & path)
     if (!state->fdTempRoots) {
 
         while (1) {
-            Path dir = (format("%1%/%2%") % stateDir % tempRootsDir).str();
-            createDirs(dir);
-
-            state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str();
-
             AutoCloseFD fdGCLock = openGCLock(ltRead);
 
-            if (pathExists(state->fnTempRoots))
+            if (pathExists(fnTempRoots))
                 /* It *must* be stale, since there can be no two
                    processes with the same pid. */
-                unlink(state->fnTempRoots.c_str());
+                unlink(fnTempRoots.c_str());
 
-            state->fdTempRoots = openLockFile(state->fnTempRoots, true);
+            state->fdTempRoots = openLockFile(fnTempRoots, true);
 
             fdGCLock = -1;
 
-            debug(format("acquiring read lock on '%1%'") % state->fnTempRoots);
+            debug(format("acquiring read lock on '%1%'") % fnTempRoots);
             lockFile(state->fdTempRoots.get(), ltRead, true);
 
             /* Check whether the garbage collector didn't get in our
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting '%1%'") % state->fnTempRoots);
+                throw SysError(format("statting '%1%'") % fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -188,14 +182,14 @@ void LocalStore::addTempRoot(const Path & path)
 
     /* Upgrade the lock to a write lock.  This will cause us to block
        if the garbage collector is holding our lock. */
-    debug(format("acquiring write lock on '%1%'") % state->fnTempRoots);
+    debug(format("acquiring write lock on '%1%'") % fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltWrite, true);
 
     string s = path + '\0';
     writeFull(state->fdTempRoots.get(), s);
 
     /* Downgrade to a read lock. */
-    debug(format("downgrading to read lock on '%1%'") % state->fnTempRoots);
+    debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltRead, true);
 }
 
@@ -204,11 +198,10 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
-    DirEntries tempRootFiles = readDirectory(
-        (format("%1%/%2%") % stateDir % tempRootsDir).str());
+    DirEntries tempRootFiles = readDirectory(tempRootsDir);
 
     for (auto & i : tempRootFiles) {
-        Path path = (format("%1%/%2%/%3%") % stateDir % tempRootsDir % i.name).str();
+        Path path = tempRootsDir + "/" + i.name;
 
         debug(format("reading temporary root file '%1%'") % path);
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
@@ -222,21 +215,25 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-        /* Try to acquire a write lock without blocking.  This can
-           only succeed if the owning process has died.  In that case
-           we don't care about its temporary roots. */
-        if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file '%1%'") % path);
-            unlink(path.c_str());
-            writeFull(fd->get(), "d");
-            continue;
-        }
+        if (path != fnTempRoots) {
 
-        /* Acquire a read lock.  This will prevent the owning process
-           from upgrading to a write lock, therefore it will block in
-           addTempRoot(). */
-        debug(format("waiting for read lock on '%1%'") % path);
-        lockFile(fd->get(), ltRead, true);
+            /* Try to acquire a write lock without blocking.  This can
+               only succeed if the owning process has died.  In that case
+               we don't care about its temporary roots. */
+            if (lockFile(fd->get(), ltWrite, false)) {
+                printError(format("removing stale temporary roots file '%1%'") % path);
+                unlink(path.c_str());
+                writeFull(fd->get(), "d");
+                continue;
+            }
+
+            /* Acquire a read lock.  This will prevent the owning process
+               from upgrading to a write lock, therefore it will block in
+               addTempRoot(). */
+            debug(format("waiting for read lock on '%1%'") % path);
+            lockFile(fd->get(), ltRead, true);
+
+        }
 
         /* Read the entire file. */
         string contents = readFile(fd->get());
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 95b05f8afd3..5ca776099df 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -51,6 +51,8 @@ LocalStore::LocalStore(const Params & params)
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
     , trashDir(realStoreDir + "/trash")
+    , tempRootsDir(stateDir + "/temproots")
+    , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
     , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());
@@ -61,7 +63,7 @@ LocalStore::LocalStore(const Params & params)
     createDirs(linksDir);
     Path profilesDir = stateDir + "/profiles";
     createDirs(profilesDir);
-    createDirs(stateDir + "/temproots");
+    createDirs(tempRootsDir);
     createDirs(dbDir);
     Path gcRootsDir = stateDir + "/gcroots";
     if (!pathExists(gcRootsDir)) {
@@ -242,12 +244,12 @@ LocalStore::LocalStore(const Params & params)
 
 LocalStore::~LocalStore()
 {
-    auto state(_state.lock());
 
     try {
+        auto state(_state.lock());
         if (state->fdTempRoots) {
             state->fdTempRoots = -1;
-            unlink(state->fnTempRoots.c_str());
+            unlink(fnTempRoots.c_str());
         }
     } catch (...) {
         ignoreException();
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 2af1bfbb389..04519bfca61 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -59,7 +59,6 @@ private:
         SQLiteStmt stmtQueryValidPaths;
 
         /* The file to which we write our temporary roots. */
-        Path fnTempRoots;
         AutoCloseFD fdTempRoots;
     };
 
@@ -75,6 +74,8 @@ public:
     const Path reservedPath;
     const Path schemaPath;
     const Path trashDir;
+    const Path tempRootsDir;
+    const Path fnTempRoots;
 
 private:
 

From 0b606aad46e1d96da36d4831df63ad90f11d21c3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Sep 2017 20:43:42 +0200
Subject: [PATCH 0537/2196] Add automatic garbage collection

Nix can now automatically run the garbage collector during builds or
while adding paths to the store. The option "min-free = <bytes>"
specifies that Nix should run the garbage collector whenever free
space in the Nix store drops below <bytes>. It will then delete
garbage until "max-free" bytes are available.

Garbage collection during builds is asynchronous; running builds are
not paused and new builds are not blocked. However, there also is a
synchronous GC run prior to the first build/substitution.

Currently, no old GC roots are deleted (as in "nix-collect-garbage
-d").
---
 doc/manual/release-notes/rl-1.12.xml |  4 ++
 src/libstore/build.cc                |  5 ++
 src/libstore/gc.cc                   | 70 ++++++++++++++++++++++++++++
 src/libstore/globals.hh              | 10 +++-
 src/libstore/local-store.cc          | 18 +++++++
 src/libstore/local-store.hh          | 21 +++++++++
 6 files changed, 127 insertions(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index d9bdd9eddd4..033c9b97135 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -393,6 +393,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     package repository.</para>
   </listitem>
 
+  <listitem>
+    <para>Automatic garbage collection.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ce41752e623..ddf4bf00d8c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3957,6 +3957,8 @@ void Worker::run(const Goals & _topGoals)
 
         checkInterrupt();
 
+        store.autoGC(false);
+
         /* Call every wake goal (in the ordering established by
            CompareGoalPtrs). */
         while (!awake.empty() && !topGoals.empty()) {
@@ -4014,6 +4016,9 @@ void Worker::waitForInput()
        is a build timeout, then wait for input until the first
        deadline for any child. */
     auto nearest = steady_time_point::max(); // nearest deadline
+    if (settings.minFree.get() != 0)
+        // Periodicallty wake up to see if we need to run the garbage collector.
+        nearest = before + std::chrono::seconds(10);
     for (auto & i : children) {
         if (!i.respectTimeouts) continue;
         if (0 != settings.maxSilentTime)
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 534db8c6e4f..cf95f7f450b 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -1,6 +1,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "local-store.hh"
+#include "finally.hh"
 
 #include <functional>
 #include <queue>
@@ -9,6 +10,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/statvfs.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
@@ -845,4 +847,72 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 }
 
 
+void LocalStore::autoGC(bool sync)
+{
+    auto getAvail = [this]() {
+        struct statvfs st;
+        if (statvfs(realStoreDir.c_str(), &st))
+            throw SysError("getting filesystem info about '%s'", realStoreDir);
+
+        return (uint64_t) st.f_bavail * st.f_bsize;
+    };
+
+    std::shared_future<void> future;
+
+    {
+        auto state(_state.lock());
+
+        if (state->gcRunning) {
+            future = state->gcFuture;
+            debug("waiting for auto-GC to finish");
+            goto sync;
+        }
+
+        auto now = std::chrono::steady_clock::now();
+
+        if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
+
+        auto avail = getAvail();
+
+        state->lastGCCheck = now;
+
+        if (avail >= settings.minFree || avail >= settings.maxFree) return;
+
+        if (avail > state->availAfterGC * 0.97) return;
+
+        state->gcRunning = true;
+
+        std::promise<void> promise;
+        future = state->gcFuture = promise.get_future().share();
+
+        std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
+
+            /* Wake up any threads waiting for the auto-GC to finish. */
+            Finally wakeup([&]() {
+                auto state(_state.lock());
+                state->gcRunning = false;
+                state->lastGCCheck = std::chrono::steady_clock::now();
+                promise.set_value();
+            });
+
+            printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
+
+            GCOptions options;
+            options.maxFreed = settings.maxFree - avail;
+
+            GCResults results;
+
+            collectGarbage(options, results);
+
+            _state.lock()->availAfterGC = getAvail();
+
+        }).detach();
+    }
+
+ sync:
+    // Wait for the future outside of the state lock.
+    if (sync) future.get();
+}
+
+
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index c20d147f5d3..41d3323117b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -4,8 +4,9 @@
 #include "config.hh"
 
 #include <map>
-#include <sys/types.h>
+#include <limits>
 
+#include <sys/types.h>
 
 namespace nix {
 
@@ -342,6 +343,13 @@ public:
 
     Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
         "A list of servers used by builtins.fetchurl to fetch files by hash."};
+
+    Setting<uint64_t> minFree{this, 0, "min-free",
+        "Automatically run the garbage collector when free disk space drops below the specified amount."};
+
+    Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
+        "Stop deleting garbage when free disk space is above the specified amount."};
+
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5ca776099df..7afecc1cfc6 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -244,6 +244,18 @@ LocalStore::LocalStore(const Params & params)
 
 LocalStore::~LocalStore()
 {
+    std::shared_future<void> future;
+
+    {
+        auto state(_state.lock());
+        if (state->gcRunning)
+            future = state->gcFuture;
+    }
+
+    if (future.valid()) {
+        printError("waiting for auto-GC to finish on exit...");
+        future.get();
+    }
 
     try {
         auto state(_state.lock());
@@ -991,6 +1003,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
             StringSource source(*nar);
             restorePath(realPath, source);
 
+            autoGC();
+
             canonicalisePathMetaData(realPath, -1);
 
             optimisePath(realPath); // FIXME: combine with hashPath()
@@ -1025,6 +1039,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 
             deletePath(realPath);
 
+            autoGC();
+
             if (recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
@@ -1097,6 +1113,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             deletePath(realPath);
 
+            autoGC();
+
             writeFile(realPath, s);
 
             canonicalisePathMetaData(realPath, -1);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 04519bfca61..4973bd9a984 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -7,6 +7,8 @@
 #include "sync.hh"
 #include "util.hh"
 
+#include <chrono>
+#include <future>
 #include <string>
 #include <unordered_set>
 
@@ -60,6 +62,21 @@ private:
 
         /* The file to which we write our temporary roots. */
         AutoCloseFD fdTempRoots;
+
+        /* The last time we checked whether to do an auto-GC, or an
+           auto-GC finished. */
+        std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
+
+        /* Whether auto-GC is running. If so, get gcFuture to wait for
+           the GC to finish. */
+        bool gcRunning = false;
+        std::shared_future<void> gcFuture;
+
+        /* How much disk space was available after the previous
+           auto-GC. If the current available disk space is below
+           minFree but not much below availAfterGC, then there is no
+           point in starting a new GC. */
+        uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
     };
 
     Sync<State, std::recursive_mutex> _state;
@@ -196,6 +213,10 @@ public:
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
+    /* If free disk space in /nix/store if below minFree, delete
+       garbage until it exceeds maxFree. */
+    void autoGC(bool sync = true);
+
 private:
 
     int getSchema();

From 1277aab219c83beae856024eea6e868677f7523b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Sep 2017 11:37:58 +0200
Subject: [PATCH 0538/2196] Fix abort when the GC thread gets an exception

---
 src/libstore/gc.cc | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index cf95f7f450b..45f28c328a5 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -887,24 +887,32 @@ void LocalStore::autoGC(bool sync)
 
         std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
 
-            /* Wake up any threads waiting for the auto-GC to finish. */
-            Finally wakeup([&]() {
-                auto state(_state.lock());
-                state->gcRunning = false;
-                state->lastGCCheck = std::chrono::steady_clock::now();
-                promise.set_value();
-            });
+            try {
 
-            printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
+                /* Wake up any threads waiting for the auto-GC to finish. */
+                Finally wakeup([&]() {
+                    auto state(_state.lock());
+                    state->gcRunning = false;
+                    state->lastGCCheck = std::chrono::steady_clock::now();
+                    promise.set_value();
+                });
 
-            GCOptions options;
-            options.maxFreed = settings.maxFree - avail;
+                printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
 
-            GCResults results;
+                GCOptions options;
+                options.maxFreed = settings.maxFree - avail;
 
-            collectGarbage(options, results);
+                GCResults results;
 
-            _state.lock()->availAfterGC = getAvail();
+                collectGarbage(options, results);
+
+                _state.lock()->availAfterGC = getAvail();
+
+            } catch (...) {
+                // FIXME: we could propagate the exception to the
+                // future, but we don't really care.
+                ignoreException();
+            }
 
         }).detach();
     }

From df4342bc175a153986ebcf5bce2a758d5353adeb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Sep 2017 16:03:22 +0200
Subject: [PATCH 0539/2196] nix build: Create result symlinks

---
 src/nix/build.cc        | 15 ++++++-
 src/nix/command.hh      | 19 ++++++---
 src/nix/installables.cc | 87 ++++++++++++++++++++++++++++++-----------
 src/nix/log.cc          | 26 ++++++------
 4 files changed, 105 insertions(+), 42 deletions(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 4239dd3fff9..64bcafd2d93 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -23,9 +23,20 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto paths = toStorePaths(store, dryRun ? DryRun : Build);
+        auto buildables = toBuildables(store, dryRun ? DryRun : Build);
 
-        printError("build result: %s", showPaths(paths));
+        for (size_t i = 0; i < buildables.size(); ++i) {
+            auto & b(buildables[i]);
+
+            for (auto & output : b.outputs) {
+                if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                    std::string symlink = "result";
+                    if (i) symlink += fmt("-%d", i);
+                    if (output.first != "out") symlink += fmt("-%s", output.first);
+                    store2->addPermRoot(output.second, absPath(symlink), true);
+                }
+            }
+        }
     }
 };
 
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 479e6fd3742..b052c42b1de 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -44,18 +44,25 @@ private:
     std::shared_ptr<Store> _store;
 };
 
-struct Whence { std::string outputName; Path drvPath; };
-typedef std::map<Path, Whence> Buildables;
+struct Buildable
+{
+    Path drvPath; // may be empty
+    std::map<std::string, Path> outputs;
+};
+
+typedef std::vector<Buildable> Buildables;
 
 struct Installable
 {
     virtual std::string what() = 0;
 
-    virtual Buildables toBuildable(bool singular = false)
+    virtual Buildables toBuildables()
     {
         throw Error("argument '%s' cannot be built", what());
     }
 
+    Buildable toBuildable();
+
     virtual Value * toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
@@ -97,9 +104,11 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
         expectArgs("installables", &_installables);
     }
 
-    enum ToStorePathsMode { Build, NoBuild, DryRun };
+    enum RealiseMode { Build, NoBuild, DryRun };
+
+    Buildables toBuildables(ref<Store> store, RealiseMode mode);
 
-    PathSet toStorePaths(ref<Store> store, ToStorePathsMode mode);
+    PathSet toStorePaths(ref<Store> store, RealiseMode mode);
 
     void prepare() override;
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index fdb6004fb31..f3c7d3075fe 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -70,17 +70,27 @@ ref<EvalState> SourceExprCommand::getEvalState()
     return ref<EvalState>(evalState);
 }
 
+Buildable Installable::toBuildable()
+{
+    auto buildables = toBuildables();
+    if (buildables.size() != 1)
+        throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
+    return std::move(buildables[0]);
+}
+
 struct InstallableStoreDrv : Installable
 {
-    Path storePath;
+    Path drvPath;
 
-    InstallableStoreDrv(const Path & storePath) : storePath(storePath) { }
+    InstallableStoreDrv(const Path & drvPath) : drvPath(drvPath) { }
 
-    std::string what() override { return storePath; }
+    std::string what() override { return drvPath; }
 
-    Buildables toBuildable(bool singular) override
+    Buildables toBuildables() override
     {
-        return {{storePath, {}}};
+        Buildable b = {drvPath};
+        // FIXME: add outputs?
+        return {b};
     }
 };
 
@@ -92,9 +102,9 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return storePath; }
 
-    Buildables toBuildable(bool singular) override
+    Buildables toBuildables() override
     {
-        return {{storePath, {}}};
+        return {{"", {{"out", storePath}}}};
     }
 };
 
@@ -104,7 +114,7 @@ struct InstallableValue : Installable
 
     InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
 
-    Buildables toBuildable(bool singular) override
+    Buildables toBuildables() override
     {
         auto state = cmd.getEvalState();
 
@@ -117,16 +127,32 @@ struct InstallableValue : Installable
         DrvInfos drvs;
         getDerivations(*state, *v, "", autoArgs, drvs, false);
 
-        if (singular && drvs.size() != 1)
-            throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), drvs.size());
-
         Buildables res;
 
-        for (auto & drv : drvs)
-            for (auto & output : drv.queryOutputs())
-                res.emplace(output.second, Whence{output.first, drv.queryDrvPath()});
+        PathSet drvPaths;
+
+        for (auto & drv : drvs) {
+            Buildable b{drv.queryDrvPath()};
+            drvPaths.insert(b.drvPath);
+
+            auto outputName = drv.queryOutputName();
+            if (outputName == "")
+                throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath);
+
+            b.outputs.emplace(outputName, drv.queryOutPath());
 
-        return res;
+            res.push_back(std::move(b));
+        }
+
+        // Hack to recognize .all: if all drvs have the same drvPath,
+        // merge the buildables.
+        if (drvPaths.size() == 1) {
+            Buildable b{*drvPaths.begin()};
+            for (auto & b2 : res)
+                b.outputs.insert(b2.outputs.begin(), b2.outputs.end());
+            return {b};
+        } else
+            return res;
     }
 };
 
@@ -214,23 +240,38 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
     return result;
 }
 
-PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode)
+Buildables InstallablesCommand::toBuildables(ref<Store> store, RealiseMode mode)
 {
     if (mode != Build)
         settings.readOnlyMode = true;
 
-    PathSet outPaths, buildables;
+    Buildables buildables;
+
+    PathSet pathsToBuild;
 
-    for (auto & i : installables)
-        for (auto & b : i->toBuildable()) {
-            outPaths.insert(b.first);
-            buildables.insert(b.second.drvPath != "" ? b.second.drvPath : b.first);
+    for (auto & i : installables) {
+        for (auto & b : i->toBuildables()) {
+            if (b.drvPath != "")
+                pathsToBuild.insert(b.drvPath);
+            buildables.push_back(std::move(b));
         }
+    }
 
     if (mode == DryRun)
-        printMissing(store, buildables, lvlError);
+        printMissing(store, pathsToBuild, lvlError);
     else if (mode == Build)
-        store->buildPaths(buildables);
+        store->buildPaths(pathsToBuild);
+
+    return buildables;
+}
+
+PathSet InstallablesCommand::toStorePaths(ref<Store> store, RealiseMode mode)
+{
+    PathSet outPaths;
+
+    for (auto & b : toBuildables(store, mode))
+        for (auto & output : b.outputs)
+            outPaths.insert(output.second);
 
     return outPaths;
 }
diff --git a/src/nix/log.cc b/src/nix/log.cc
index ba5e71c10a0..15ae61d08cc 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -24,23 +24,25 @@ struct CmdLog : InstallableCommand
 
     void run(ref<Store> store) override
     {
+        settings.readOnlyMode = true;
+
         auto subs = getDefaultSubstituters();
 
         subs.push_front(store);
 
-        for (auto & b : installable->toBuildable(true)) {
-
-            for (auto & sub : subs) {
-                auto log = b.second.drvPath != "" ? sub->getBuildLog(b.second.drvPath) : nullptr;
-                if (!log) {
-                    log = sub->getBuildLog(b.first);
-                    if (!log) continue;
-                }
-                stopProgressBar();
-                printInfo("got build log for '%s' from '%s'", b.first, sub->getUri());
-                std::cout << *log;
-                return;
+        auto b = installable->toBuildable();
+
+        for (auto & sub : subs) {
+            auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr;
+            for (auto & output : b.outputs) {
+                if (log) break;
+                log = sub->getBuildLog(output.second);
             }
+            if (!log) continue;
+            stopProgressBar();
+            printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());
+            std::cout << *log;
+            return;
         }
 
         throw Error("build log of '%s' is not available", installable->what());

From dff440aab32131b3459cadc2c4cc60c6f42674c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Sep 2017 16:20:34 +0200
Subject: [PATCH 0540/2196] nix build: Add --out-link and --no-link options

---
 src/libutil/args.hh | 13 +++++++++++++
 src/nix/build.cc    | 30 ++++++++++++++++++++++--------
 src/nix/copy.cc     |  2 +-
 src/nix/run.cc      |  2 +-
 4 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 044ed209e3b..7f8ea647c90 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -80,6 +80,19 @@ public:
         FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
         FlagMaker & handler(std::function<void(Strings)> handler) { flag->handler = handler; return *this; };
         FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
+
+        FlagMaker & dest(std::string * dest) {
+            assert(flag->arity == 1);
+            flag->handler = [=](Strings ss) { *dest = ss.front(); };
+            return *this;
+        };
+
+        template<class T>
+        FlagMaker & set(T * dest, const T & val) {
+            assert(flag->arity == 0);
+            flag->handler = [=](Strings ss) { *dest = val; };
+            return *this;
+        };
     };
 
     FlagMaker mkFlag();
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 64bcafd2d93..4240f06ade7 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -7,8 +7,22 @@ using namespace nix;
 
 struct CmdBuild : MixDryRun, InstallablesCommand
 {
+    Path outLink = "result";
+
     CmdBuild()
     {
+        mkFlag()
+            .longName("out-link")
+            .shortName('o')
+            .description("path of the symlink to the build result")
+            .arity(1)
+            .labels({"path"})
+            .dest(&outLink);
+
+        mkFlag()
+            .longName("no-link")
+            .description("do not create a symlink to the build result")
+            .set(&outLink, Path(""));
     }
 
     std::string name() override
@@ -28,14 +42,14 @@ struct CmdBuild : MixDryRun, InstallablesCommand
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);
 
-            for (auto & output : b.outputs) {
-                if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
-                    std::string symlink = "result";
-                    if (i) symlink += fmt("-%d", i);
-                    if (output.first != "out") symlink += fmt("-%s", output.first);
-                    store2->addPermRoot(output.second, absPath(symlink), true);
-                }
-            }
+            if (outLink != "")
+                for (auto & output : b.outputs)
+                    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+                        std::string symlink = outLink;
+                        if (i) symlink += fmt("-%d", i);
+                        if (output.first != "out") symlink += fmt("-%s", output.first);
+                        store2->addPermRoot(output.second, absPath(symlink), true);
+                    }
         }
     }
 };
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index d9e6c949352..fe963e79427 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -22,7 +22,7 @@ struct CmdCopy : StorePathsCommand
         mkFlag()
             .longName("no-check-sigs")
             .description("do not require that paths are signed by trusted keys")
-            .handler([&](Strings ss) { checkSigs = NoCheckSigs; });
+            .set(&checkSigs, NoCheckSigs);
     }
 
     std::string name() override
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f016b04a1ec..64a5cbd304d 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -41,7 +41,7 @@ struct CmdRun : InstallablesCommand
             .longName("ignore-environment")
             .shortName('i')
             .description("clear the entire environment (except those specified with --keep)")
-            .handler([&](Strings ss) { ignoreEnvironment = true; });
+            .set(&ignoreEnvironment, true);
 
         mkFlag()
             .longName("keep")

From 2e9b7c4cb2168f0aac8948d3212e50726b37acdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Sep 2017 20:09:04 +0200
Subject: [PATCH 0541/2196] nix run: Add some examples

---
 src/nix/run.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 64a5cbd304d..31c3c1464f1 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -70,6 +70,24 @@ struct CmdRun : InstallablesCommand
         return "run a shell in which the specified packages are available";
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To start a shell providing GNU Hello from NixOS 17.03:",
+                "nix run -f channel:nixos-17.03 hello"
+            },
+            Example{
+                "To start a shell providing youtube-dl from your 'nixpkgs' channel:",
+                "nix run nixpkgs.youtube-dl"
+            },
+            Example{
+                "To run GNU Hello:",
+                "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"
+            },
+        };
+    }
+
     void run(ref<Store> store) override
     {
         auto outPaths = toStorePaths(store, Build);

From 17afc42895a586d6eefa76583c55022261b0d923 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Sep 2017 20:14:04 +0200
Subject: [PATCH 0542/2196] nix build: Add examples

---
 src/nix/build.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 4240f06ade7..42b3e42c69e 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -35,6 +35,20 @@ struct CmdBuild : MixDryRun, InstallablesCommand
         return "build a derivation or fetch a store path";
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To build and run GNU Hello from NixOS 17.03:",
+                "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"
+            },
+            Example{
+                "To build the build.x86_64-linux attribute from release.nix:",
+                "nix build -f release.nix build.x86_64-linux"
+            },
+        };
+    }
+
     void run(ref<Store> store) override
     {
         auto buildables = toBuildables(store, dryRun ? DryRun : Build);

From 2ebeffcfd46f4a471cf4e49130ec106cf8a88ff1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Sep 2017 20:18:29 +0200
Subject: [PATCH 0543/2196] nix log: Add examples

---
 src/nix/log.cc | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/nix/log.cc b/src/nix/log.cc
index 15ae61d08cc..966ad8b6508 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -19,7 +19,25 @@ struct CmdLog : InstallableCommand
 
     std::string description() override
     {
-        return "show the build log of the specified packages or paths";
+        return "show the build log of the specified packages or paths, if available";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To get the build log of GNU Hello:",
+                "nix log nixpkgs.hello"
+            },
+            Example{
+                "To get the build log of a specific path:",
+                "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1"
+            },
+            Example{
+                "To get a build log from a specific binary cache:",
+                "nix log --store https://cache.nixos.org nixpkgs.hello"
+            },
+        };
     }
 
     void run(ref<Store> store) override

From 4967f0509aca0001156d937f36ed8eb17c72d752 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Sep 2017 20:37:46 +0200
Subject: [PATCH 0544/2196] nix eval: Add examples

---
 src/nix/eval.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 981cefa800c..e2212869263 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -27,6 +27,24 @@ struct CmdEval : MixJSON, InstallablesCommand
         return "evaluate a Nix expression";
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To evaluate a Nix expression given on the command line:",
+                "nix eval '(1 + 2)'"
+            },
+            Example{
+                "To evaluate a Nix expression from a file or URI:",
+                "nix eval -f channel:nixos-17.09 hello.name"
+            },
+            Example{
+                "To get the current version of Nixpkgs:",
+                "nix eval --raw nixpkgs.lib.nixpkgsVersion"
+            },
+        };
+    }
+
     void run(ref<Store> store) override
     {
         if (raw && json)

From 24e23a1a73d573c5d0cda86c34e0e1fd5aef9171 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Sep 2017 20:42:11 +0200
Subject: [PATCH 0545/2196] nix search: Add examples

---
 src/nix/search.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 6e099ce377f..9476b79fbc1 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -56,6 +56,24 @@ struct CmdSearch : SourceExprCommand, MixJSON
         return "query available packages";
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To show all available packages:",
+                "nix search"
+            },
+            Example{
+                "To show any packages containing 'blender' in its name or description:",
+                "nix search blender"
+            },
+            Example{
+                "To search for Firefox and Chromium:",
+                "nix search 'firefox|chromium'"
+            },
+        };
+    }
+
     void run(ref<Store> store) override
     {
         settings.readOnlyMode = true;

From 88e6bb76de5564b3217be9688677d1c89101b2a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 11:27:10 +0200
Subject: [PATCH 0546/2196] Communicate the structured log FD to builders

Since we may use a dedicated file descriptor in the future, this
allows us to change it. So builders can do

  if [[ -n $NIX_LOG_FD ]]; then
    echo "@nix { message... }" >&$NIX_LOG_FD
  fi
---
 src/libstore/build.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ddf4bf00d8c..3b3cebfb18f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2202,7 +2202,7 @@ void DerivationGoal::initEnv()
     env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
     /* In non-structured mode, add all bindings specified in the
-       derivation via the environments, except those listed in the
+       derivation via the environment, except those listed in the
        passAsFile attribute. Those are passed as file names pointing
        to temporary files containing the contents. Note that
        passAsFile is ignored in structure mode because it's not
@@ -2258,6 +2258,11 @@ void DerivationGoal::initEnv()
         Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
         for (auto & i : varNames) env[i] = getEnv(i);
     }
+
+    /* Currently structured log messages piggyback on stderr, but we
+       may change that in the future. So tell the builder which file
+       descriptor to use for that. */
+    env["NIX_LOG_FD"] = "2";
 }
 
 

From a2740c9ca23e748c3ab8ea61a135594a94c86aae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 11:33:46 +0200
Subject: [PATCH 0547/2196] nix copy: Add examples

---
 src/nix/copy.cc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index fe963e79427..bf9ffe75569 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -39,9 +39,17 @@ struct CmdCopy : StorePathsCommand
     {
         return {
             Example{
-                "To copy Firefox to the local store to a binary cache in file:///tmp/cache:",
+                "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
                 "nix copy --to file:///tmp/cache -r $(type -p firefox)"
             },
+            Example{
+                "To copy the entire current NixOS system closure to another machine via SSH:",
+                "nix copy --to ssh://server -r /run/current-system"
+            },
+            Example{
+                "To copy a closure from another machine via SSH:",
+                "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
+            },
         };
     }
 

From 8f6b347abd93706e36cf67fc4c53a46c7ad1ad49 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 14:40:27 +0200
Subject: [PATCH 0548/2196] ThreadPool: Improve exception handling

In particular, process() won't return as long as there are active
items. This prevents work item lambdas from referring to stack frames
that no longer exist.
---
 src/libutil/thread-pool.cc | 75 ++++++++++++++++++++++++++------------
 src/libutil/thread-pool.hh |  2 +-
 2 files changed, 53 insertions(+), 24 deletions(-)

diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index 0a3a407240f..f43dbe0c371 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -46,11 +46,17 @@ void ThreadPool::enqueue(const work_t & t)
 
 void ThreadPool::process()
 {
+    /* Loop until there are no active work items *and* there either
+       are no queued items or there is an exception. The
+       post-condition is that no new items will become active. */
     while (true) {
         auto state(state_.lock());
-        if (state->exception)
-            std::rethrow_exception(state->exception);
-        if (state->left.empty() && !state->pending) break;
+        if (!state->active) {
+            if (state->exception)
+                std::rethrow_exception(state->exception);
+            if (state->left.empty())
+                break;
+        }
         state.wait(done);
     }
 }
@@ -58,41 +64,64 @@ void ThreadPool::process()
 void ThreadPool::workerEntry()
 {
     bool didWork = false;
+    std::exception_ptr exc;
 
     while (true) {
         work_t w;
         {
             auto state(state_.lock());
+
+            if (didWork) {
+                assert(state->active);
+                state->active--;
+
+                if (exc) {
+
+                    if (!state->exception) {
+                        state->exception = exc;
+                        // Tell the other workers to quit.
+                        state->quit = true;
+                        work.notify_all();
+                    } else {
+                        /* Print the exception, since we can't
+                           propagate it. */
+                        try {
+                            std::rethrow_exception(exc);
+                        } catch (std::exception & e) {
+                            if (!dynamic_cast<Interrupted*>(&e) &&
+                                !dynamic_cast<ThreadPoolShutDown*>(&e))
+                                ignoreException();
+                        } catch (...) {
+                        }
+                    }
+                }
+            }
+
+            /* Wait until a work item is available or another thread
+               had an exception or we're asked to quit. */
             while (true) {
-                if (state->quit || state->exception) return;
-                if (didWork) {
-                    assert(state->pending);
-                    state->pending--;
-                    didWork = false;
+                if (state->quit) {
+                    if (!state->active)
+                        done.notify_one();
+                    return;
                 }
                 if (!state->left.empty()) break;
-                if (!state->pending)
-                    done.notify_all();
+                if (!state->active) {
+                    done.notify_one();
+                    return;
+                }
                 state.wait(work);
             }
-            w = state->left.front();
+
+            w = std::move(state->left.front());
             state->left.pop();
-            state->pending++;
+            state->active++;
         }
 
         try {
             w();
-        } catch (std::exception & e) {
-            auto state(state_.lock());
-            if (state->exception) {
-                if (!dynamic_cast<Interrupted*>(&e) &&
-                    !dynamic_cast<ThreadPoolShutDown*>(&e))
-                    printError(format("error: %s") % e.what());
-            } else {
-                state->exception = std::current_exception();
-                work.notify_all();
-                done.notify_all();
-            }
+        } catch (...) {
+            exc = std::current_exception();
         }
 
         didWork = true;
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index 361a9d33a73..835dfb4b83a 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -44,7 +44,7 @@ private:
     struct State
     {
         std::queue<work_t> left;
-        size_t pending = 0;
+        size_t active = 0;
         std::exception_ptr exception;
         std::vector<std::thread> workers;
         bool quit = false;

From 3cf1705583ee64137cdebf8f4e50ccffdaf7c756 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 14:46:55 +0200
Subject: [PATCH 0549/2196] nix copy: Don't open the --from store twice

---
 src/nix/command.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index f90d848f2a4..058a7548c89 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -97,7 +97,7 @@ ref<Store> StoreCommand::createStore()
 
 void StoreCommand::run()
 {
-    run(createStore());
+    run(getStore());
 }
 
 StorePathsCommand::StorePathsCommand()

From 6a888ec29a6724f916f96508b3a94a86c643c18e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 14:48:08 +0200
Subject: [PATCH 0550/2196] copyStorePath(): Fill in missing narHash regardless
 of checkSigs

I don't remember what the reasoning was here, but security is provided
by the signatures, not by whether the hash is provided by the other
store.
---
 src/libstore/store-api.cc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7a8a3f7dee0..fa6ade75002 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -593,23 +593,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     MyStringSink sink(progress);
     srcStore->narFromPath({storePath}, sink);
 
-    if (!info->narHash && !checkSigs) {
+    if (!info->narHash) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
         info = info2;
     }
 
-    assert(info->narHash);
-
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->ultimate = false;
         info = info2;
     }
 
-    assert(info->narHash);
-
     dstStore->addToStore(*info, sink.s, repair, checkSigs);
 }
 

From b7376edf06e3836394503e16ec14af23c7615f22 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 15:31:24 +0200
Subject: [PATCH 0551/2196] ThreadPool: On exception, interrupt the other
 worker threads

---
 src/libutil/thread-pool.cc | 10 ++++++----
 src/libutil/thread-pool.hh |  4 +++-
 src/libutil/util.cc        |  2 +-
 src/libutil/util.hh        |  5 ++++-
 4 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index f43dbe0c371..ce126d36db8 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -21,7 +21,7 @@ ThreadPool::~ThreadPool()
     std::vector<std::thread> workers;
     {
         auto state(state_.lock());
-        state->quit = true;
+        quit = true;
         std::swap(workers, state->workers);
     }
 
@@ -36,7 +36,7 @@ ThreadPool::~ThreadPool()
 void ThreadPool::enqueue(const work_t & t)
 {
     auto state(state_.lock());
-    if (state->quit)
+    if (quit)
         throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down");
     state->left.push(t);
     if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads)
@@ -63,6 +63,8 @@ void ThreadPool::process()
 
 void ThreadPool::workerEntry()
 {
+    interruptCheck = [&]() { return (bool) quit; };
+
     bool didWork = false;
     std::exception_ptr exc;
 
@@ -80,7 +82,7 @@ void ThreadPool::workerEntry()
                     if (!state->exception) {
                         state->exception = exc;
                         // Tell the other workers to quit.
-                        state->quit = true;
+                        quit = true;
                         work.notify_all();
                     } else {
                         /* Print the exception, since we can't
@@ -100,7 +102,7 @@ void ThreadPool::workerEntry()
             /* Wait until a work item is available or another thread
                had an exception or we're asked to quit. */
             while (true) {
-                if (state->quit) {
+                if (quit) {
                     if (!state->active)
                         done.notify_one();
                     return;
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index 835dfb4b83a..06a097ab5ea 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -7,6 +7,7 @@
 #include <functional>
 #include <thread>
 #include <map>
+#include <atomic>
 
 namespace nix {
 
@@ -47,9 +48,10 @@ private:
         size_t active = 0;
         std::exception_ptr exception;
         std::vector<std::thread> workers;
-        bool quit = false;
     };
 
+    std::atomic_bool quit{false};
+
     Sync<State> state_;
 
     std::condition_variable work, done;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 605e89ce27f..3c98a61f9e5 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1002,6 +1002,7 @@ void closeOnExec(int fd)
 bool _isInterrupted = false;
 
 static thread_local bool interruptThrown = false;
+thread_local std::function<bool()> interruptCheck;
 
 void setInterruptThrown()
 {
@@ -1020,7 +1021,6 @@ void _interrupted()
 }
 
 
-
 //////////////////////////////////////////////////////////////////////
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 30e3c0df1c1..6a66576e96c 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -273,13 +273,16 @@ void closeOnExec(int fd);
 
 extern bool _isInterrupted;
 
+extern thread_local std::function<bool()> interruptCheck;
+
 void setInterruptThrown();
 
 void _interrupted();
 
 void inline checkInterrupt()
 {
-    if (_isInterrupted) _interrupted();
+    if (_isInterrupted || (interruptCheck && interruptCheck()))
+        _interrupted();
 }
 
 MakeError(Interrupted, BaseError)

From e02edb14836fda29019adf95073092d874ebd925 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 15:32:07 +0200
Subject: [PATCH 0552/2196] nix copy: Add --substitute flag

---
 src/nix/copy.cc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index bf9ffe75569..8d7c6a0e8e4 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -14,6 +14,8 @@ struct CmdCopy : StorePathsCommand
 
     CheckSigsFlag checkSigs = CheckSigs;
 
+    SubstituteFlag substitute = NoSubstitute;
+
     CmdCopy()
     {
         mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri);
@@ -23,6 +25,12 @@ struct CmdCopy : StorePathsCommand
             .longName("no-check-sigs")
             .description("do not require that paths are signed by trusted keys")
             .set(&checkSigs, NoCheckSigs);
+
+        mkFlag()
+            .longName("substitute")
+            .shortName('s')
+            .description("whether to try substitutes on the destination store (only supported by SSH)")
+            .set(&substitute, Substitute);
     }
 
     std::string name() override
@@ -66,7 +74,7 @@ struct CmdCopy : StorePathsCommand
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
         copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()),
-            NoRepair, checkSigs);
+            NoRepair, checkSigs, substitute);
     }
 };
 

From 3ed8290e5331011609de462bda67857837998583 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Sep 2017 16:55:27 +0200
Subject: [PATCH 0553/2196] LegacySSHStore: Include signatures etc.

---
 src/libstore/legacy-ssh-store.cc | 7 +++++++
 src/libstore/serve-protocol.hh   | 2 +-
 src/nix-store/nix-store.cc       | 2 ++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 855a7c99aed..dfefdb9bc87 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -105,6 +105,13 @@ struct LegacySSHStore : public Store
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
+            if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
+                auto s = readString(conn->from);
+                info->narHash = s.empty() ? Hash() : Hash(s);
+                conn->from >> info->ca;
+                info->sigs = readStrings<StringSet>(conn->from);
+            }
+
             auto s = readString(conn->from);
             assert(s == "");
 
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index f8cc9a4b6eb..f67d1e2580a 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x203
+#define SERVE_PROTOCOL_VERSION 0x204
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 1668fdb2d03..85bcbc22e9d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -858,6 +858,8 @@ static void opServe(Strings opFlags, Strings opArgs)
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
+                        if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
+                            out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs;
                     } catch (InvalidPath &) {
                     }
                 }

From ad228d84e56aeb1b5f77ae3bd3f8272f00782a8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 10 Sep 2017 15:58:30 +0200
Subject: [PATCH 0554/2196] nix build: Only download the requested derivation
 outputs

Also some refactoring.
---
 src/nix/build.cc        |  2 +-
 src/nix/command.cc      |  4 ++--
 src/nix/command.hh      | 21 +++++++++++++++------
 src/nix/installables.cc | 41 +++++++++++++++++++++++++++++++++--------
 src/nix/run.cc          |  2 +-
 5 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index 42b3e42c69e..915a066497b 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -51,7 +51,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto buildables = toBuildables(store, dryRun ? DryRun : Build);
+        auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
 
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 058a7548c89..f69c5689656 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -118,7 +118,7 @@ void StorePathsCommand::run(ref<Store> store)
     }
 
     else {
-        for (auto & p : toStorePaths(store, NoBuild))
+        for (auto & p : toStorePaths(store, NoBuild, installables))
             storePaths.push_back(p);
 
         if (recursive) {
@@ -134,7 +134,7 @@ void StorePathsCommand::run(ref<Store> store)
 
 void StorePathCommand::run(ref<Store> store)
 {
-    auto storePaths = toStorePaths(store, NoBuild);
+    auto storePaths = toStorePaths(store, NoBuild, installables);
 
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
diff --git a/src/nix/command.hh b/src/nix/command.hh
index b052c42b1de..182b01ef92e 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -93,6 +93,8 @@ private:
     Value * vSourceExpr = 0;
 };
 
+enum RealiseMode { Build, NoBuild, DryRun };
+
 /* A command that operates on a list of "installables", which can be
    store paths, attribute paths, Nix expressions, etc. */
 struct InstallablesCommand : virtual Args, SourceExprCommand
@@ -104,12 +106,6 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
         expectArgs("installables", &_installables);
     }
 
-    enum RealiseMode { Build, NoBuild, DryRun };
-
-    Buildables toBuildables(ref<Store> store, RealiseMode mode);
-
-    PathSet toStorePaths(ref<Store> store, RealiseMode mode);
-
     void prepare() override;
 
     virtual bool useDefaultInstallables() { return true; }
@@ -198,4 +194,17 @@ struct RegisterCommand
     }
 };
 
+std::shared_ptr<Installable> parseInstallable(
+    SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
+    bool useDefaultInstallables);
+
+Buildables toBuildables(ref<Store> store, RealiseMode mode,
+    std::vector<std::shared_ptr<Installable>> installables);
+
+PathSet toStorePaths(ref<Store> store, RealiseMode mode,
+    std::vector<std::shared_ptr<Installable>> installables);
+
+Path toStorePath(ref<Store> store, RealiseMode mode,
+    std::shared_ptr<Installable> installable);
+
 }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index f3c7d3075fe..76df05fa32d 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -240,7 +240,17 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
     return result;
 }
 
-Buildables InstallablesCommand::toBuildables(ref<Store> store, RealiseMode mode)
+std::shared_ptr<Installable> parseInstallable(
+    SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
+    bool useDefaultInstallables)
+{
+    auto installables = parseInstallables(cmd, store, {installable}, false);
+    assert(installables.size() == 1);
+    return installables.front();
+}
+
+Buildables toBuildables(ref<Store> store, RealiseMode mode,
+    std::vector<std::shared_ptr<Installable>> installables)
 {
     if (mode != Build)
         settings.readOnlyMode = true;
@@ -251,8 +261,13 @@ Buildables InstallablesCommand::toBuildables(ref<Store> store, RealiseMode mode)
 
     for (auto & i : installables) {
         for (auto & b : i->toBuildables()) {
-            if (b.drvPath != "")
-                pathsToBuild.insert(b.drvPath);
+            if (b.drvPath != "") {
+                StringSet outputNames;
+                for (auto & output : b.outputs)
+                    outputNames.insert(output.first);
+                pathsToBuild.insert(
+                    b.drvPath + "!" + concatStringsSep(",", outputNames));
+            }
             buildables.push_back(std::move(b));
         }
     }
@@ -265,17 +280,29 @@ Buildables InstallablesCommand::toBuildables(ref<Store> store, RealiseMode mode)
     return buildables;
 }
 
-PathSet InstallablesCommand::toStorePaths(ref<Store> store, RealiseMode mode)
+PathSet toStorePaths(ref<Store> store, RealiseMode mode,
+    std::vector<std::shared_ptr<Installable>> installables)
 {
     PathSet outPaths;
 
-    for (auto & b : toBuildables(store, mode))
+    for (auto & b : toBuildables(store, mode, installables))
         for (auto & output : b.outputs)
             outPaths.insert(output.second);
 
     return outPaths;
 }
 
+Path toStorePath(ref<Store> store, RealiseMode mode,
+    std::shared_ptr<Installable> installable)
+{
+    auto paths = toStorePaths(store, mode, {installable});
+
+    if (paths.size() != 1)
+            throw Error("argument '%s' should evaluate to one store path", installable->what());
+
+    return *paths.begin();
+}
+
 void InstallablesCommand::prepare()
 {
     installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables());
@@ -283,9 +310,7 @@ void InstallablesCommand::prepare()
 
 void InstallableCommand::prepare()
 {
-    auto installables = parseInstallables(*this, getStore(), {_installable}, false);
-    assert(installables.size() == 1);
-    installable = installables.front();
+    installable = parseInstallable(*this, getStore(), _installable, false);
 }
 
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 31c3c1464f1..c72ede99c1c 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -90,7 +90,7 @@ struct CmdRun : InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto outPaths = toStorePaths(store, Build);
+        auto outPaths = toStorePaths(store, Build, installables);
 
         auto accessor = store->getFSAccessor();
 

From 8af704eef1ee9a73b8d120b863223c015189e943 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 10 Sep 2017 22:36:32 +0200
Subject: [PATCH 0555/2196] Typo

---
 src/nix/ls.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index a08e4c42c67..5a5fa8f62d9 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -50,7 +50,7 @@ struct MixLs : virtual Args
             }
         };
 
-        doPath = [&](const FSAccessor::Stat & st , const Path & curPath,
+        doPath = [&](const FSAccessor::Stat & st, const Path & curPath,
             const std::string & relPath, bool showDirectory)
         {
             if (st.type == FSAccessor::Type::tDirectory && !showDirectory) {

From d41c5eb13f4f3a37d80dbc6d3888644170c3b44a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 10 Sep 2017 22:36:59 +0200
Subject: [PATCH 0556/2196] Add command "nix why-depends"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This command shows why a package has another package in its runtime
closure. For example, to see why VLC has libdrm.dev in its closure:

  $ nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev
  /nix/store/g901z9pcj0n5yy5n6ykxk3qm4ina1d6z-vlc-2.2.5.1:
    lib/libvlccore.so.8.0.0: …nfig:/nix/store/405lmx6jl8lp0ad1vrr6j498chrqhz8g-libdrm-2.4.75-d…

  /nix/store/s3nm7kd8hlcg0facn2q1ff2n7wrwdi2l-mesa-noglu-17.0.7-dev:
    nix-support/propagated-native-build-inputs: …-dev /nix/store/405lmx6jl8lp0ad1vrr6j498chrqhz8g-libdrm-2.4.75-d…

Thus, VLC's lib/libvlccore.so.8.0.0 as well as mesa-noglu's
nix-support/propagated-native-build-inputs cause the dependency.
---
 src/nix/why-depends.cc | 133 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 src/nix/why-depends.cc

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
new file mode 100644
index 00000000000..da37e5eb04f
--- /dev/null
+++ b/src/nix/why-depends.cc
@@ -0,0 +1,133 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "progress-bar.hh"
+#include "fs-accessor.hh"
+
+using namespace nix;
+
+static std::string hilite(const std::string & s, size_t pos, size_t len)
+{
+    return
+        std::string(s, 0, pos)
+        + ANSI_RED
+        + std::string(s, pos, len)
+        + ANSI_NORMAL
+        + std::string(s, pos + len);
+}
+
+struct CmdWhyDepends : SourceExprCommand
+{
+    std::string _package, _dependency;
+
+    CmdWhyDepends()
+    {
+        expectArg("package", &_package);
+        expectArg("dependency", &_dependency);
+    }
+
+    std::string name() override
+    {
+        return "why-depends";
+    }
+
+    std::string description() override
+    {
+        return "show why a package has another package in its closure";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To show which files in Hello's closure depend on Glibc:",
+                "nix why-depends nixpkgs.hello nixpkgs.glibc"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto package = parseInstallable(*this, store, _package, false);
+        auto packagePath = toStorePath(store, Build, package);
+        auto dependency = parseInstallable(*this, store, _dependency, false);
+        auto dependencyPath = toStorePath(store, NoBuild, dependency);
+        auto dependencyPathHash = storePathToHash(dependencyPath);
+
+        PathSet closure;
+        store->computeFSClosure({packagePath}, closure, false, false);
+
+        if (!closure.count(dependencyPath)) {
+            printError("'%s' does not depend on '%s'", package->what(), dependency->what());
+            return;
+        }
+
+        stopProgressBar(); // FIXME
+
+        auto accessor = store->getFSAccessor();
+
+        // FIXME: show the path through the dependency graph.
+
+        bool first = true;
+
+        for (auto & path : closure) {
+
+            if (path == dependencyPath && packagePath != dependencyPath)
+                continue;
+
+            if (!store->queryPathInfo(path)->references.count(dependencyPath))
+                continue;
+
+            if (!first) std::cerr << "\n";
+            first = false;
+
+            std::cerr << fmt("%s:\n", path);
+
+            std::function<void(const Path &)> recurse;
+
+            recurse = [&](const Path & p) {
+                auto st = accessor->stat(p);
+
+                auto p2 = p == path ? "/" : std::string(p, path.size() + 1);
+
+                if (st.type == FSAccessor::Type::tDirectory) {
+                    auto names = accessor->readDirectory(p);
+                    for (auto & name : names)
+                        recurse(p + "/" + name);
+                }
+
+                else if (st.type == FSAccessor::Type::tRegular) {
+                    auto contents = accessor->readFile(p);
+                    auto pos = contents.find(dependencyPathHash);
+                    if (pos != std::string::npos) {
+                        size_t margin = 16;
+                        auto pos2 = pos >= margin ? pos - margin : 0;
+                        std::string fragment;
+                        for (char c : std::string(contents, pos2,
+                                pos - pos2 + dependencyPathHash.size() + margin))
+                        {
+                            fragment += isprint(c) ? c : '.';
+                        }
+
+                        std::cerr << fmt("  %s: …%s…\n",
+                            p2,
+                            hilite(fragment, pos - pos2, storePathHashLen));
+                    }
+                }
+
+                else if (st.type == FSAccessor::Type::tSymlink) {
+                    auto target = accessor->readLink(p);
+                    auto pos = target.find(dependencyPathHash);
+                    if (pos != std::string::npos) {
+                        std::cerr << fmt("  %s -> %s\n", p2, hilite(target, pos, storePathHashLen));
+                    }
+                }
+            };
+
+            recurse(path);
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdWhyDepends>());

From 1511f01351c992b2b5b3108e6f1854bac50df71d Mon Sep 17 00:00:00 2001
From: Peter Simons <simons@cryp.to>
Date: Fri, 8 Sep 2017 13:34:08 +0200
Subject: [PATCH 0557/2196] docker: update to Nix 1.11.14

- Use the latest Nix version 1.11.14.

- Attempts to download the Nix installation tarball from http://nixos.org
  redirect to https these days, which wget doesn't support unless OpenSSL is
  available.

- Use addgroup and adduser commands to create the Nix build users.

- Link the Nix profile script into /etc/profile.d, where it's run
  automatically.

- Dropped installation of bash and tar. Neither tool is essential for running
  Nix.

Use the command "docker build -t nix . && docker run -it --rm nix sh -"
to build and run the Nix docker container.
---
 misc/docker/Dockerfile | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 85bd32e199a..fb6f73517bb 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -1,13 +1,16 @@
 FROM alpine
 
-RUN wget -O- http://nixos.org/releases/nix/nix-1.11.7/nix-1.11.7-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
-    && echo "nixbld:x:30000:nixbld1,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld30" >> /etc/group \
-    && for i in $(seq 1 30); do echo "nixbld$i:x:$((30000 + $i)):30000:::" >> /etc/passwd; done \
-    && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
-    && echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile \
-    && rm -r /nix-*-x86_64-linux \
-    && apk --update add bash tar \
-    && rm -rf /var/cache/apk/*
+# Enable HTTPS support in wget.
+RUN apk add --update openssl
+
+# Download Nix and install it into the system.
+RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
+  && addgroup -g 30000 -S nixbld \
+  && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
+  && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
+  && ln -s /root/.nix-profile/etc/profile.d/nix.sh /etc/profile.d/ \
+  && rm -r /nix-*-x86_64-linux \
+  && rm -r /var/cache/apk/*
 
 ONBUILD ENV \
     ENV=/etc/profile \

From fc0ded3408ac1e17924f1901b384bb70125be62f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Sep 2017 16:21:27 +0200
Subject: [PATCH 0558/2196] nix why-depends: Add option to show all edges
 causing a dependency
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For example, without --all:

  $ nix why-depends nixpkgs.nixUnstable nixpkgs.libssh2
  /nix/store/s9n5gvj2l49b4n19nz6xl832654nf7n7-nix-1.12pre5511_c94f3d55
  └───lib/libnixstore.so: …/lib:/nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0/lib…
      => /nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0
      └───lib/libcurl.la: …ib -L/nix/store/4mbayl1y5hpjbjzkx8ndyhkv98kqw1wi-libssh2-1.8.0/l…
          => /nix/store/4mbayl1y5hpjbjzkx8ndyhkv98kqw1wi-libssh2-1.8.0

but with --all:

  $ nix why-depends -a nixpkgs.nixUnstable nixpkgs.libssh2
  /nix/store/s9n5gvj2l49b4n19nz6xl832654nf7n7-nix-1.12pre5511_c94f3d55
  ├───lib/libnixstore.so: …/lib:/nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0/lib…
  │   => /nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0
  │   └───lib/libcurl.la: …ib -L/nix/store/4mbayl1y5hpjbjzkx8ndyhkv98kqw1wi-libssh2-1.8.0/l…
  │       lib/libcurl.so.4.4.0: …/lib:/nix/store/4mbayl1y5hpjbjzkx8ndyhkv98kqw1wi-libssh2-1.8.0/l…
  │       => /nix/store/4mbayl1y5hpjbjzkx8ndyhkv98kqw1wi-libssh2-1.8.0
  └───lib/libnixstore.so: …/lib:/nix/store/bx2i9vi76lps6w9rr73fxf6my31s4dg5-aws-sdk-cpp-1.0…
      => /nix/store/bx2i9vi76lps6w9rr73fxf6my31s4dg5-aws-sdk-cpp-1.0.153
      └───lib/libaws-cpp-sdk-core.so: …e.so./nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0/lib…
          lib/libaws-cpp-sdk-s3.so: …/lib:/nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0/lib…
          => /nix/store/w9ykqpl5v0r3vfwsgn408jqhs72cx96x-curl-7.55.0
---
 src/nix/why-depends.cc | 190 +++++++++++++++++++++++++++++++++--------
 1 file changed, 156 insertions(+), 34 deletions(-)

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index da37e5eb04f..a6750ac95ca 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -5,26 +5,44 @@
 #include "progress-bar.hh"
 #include "fs-accessor.hh"
 
+#include <queue>
+
 using namespace nix;
 
-static std::string hilite(const std::string & s, size_t pos, size_t len)
+static std::string hilite(const std::string & s, size_t pos, size_t len,
+    const std::string & colour = ANSI_RED)
 {
     return
         std::string(s, 0, pos)
-        + ANSI_RED
+        + colour
         + std::string(s, pos, len)
         + ANSI_NORMAL
         + std::string(s, pos + len);
 }
 
+static std::string filterPrintable(const std::string & s)
+{
+    std::string res;
+    for (char c : s)
+        res += isprint(c) ? c : '.';
+    return res;
+}
+
 struct CmdWhyDepends : SourceExprCommand
 {
     std::string _package, _dependency;
+    bool all = false;
 
     CmdWhyDepends()
     {
         expectArg("package", &_package);
         expectArg("dependency", &_dependency);
+
+        mkFlag()
+            .longName("all")
+            .shortName('a')
+            .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path")
+            .set(&all, true);
     }
 
     std::string name() override
@@ -67,66 +85,170 @@ struct CmdWhyDepends : SourceExprCommand
 
         auto accessor = store->getFSAccessor();
 
-        // FIXME: show the path through the dependency graph.
+        auto const inf = std::numeric_limits<size_t>::max();
+
+        struct Node
+        {
+            Path path;
+            PathSet refs;
+            PathSet rrefs;
+            size_t dist = inf;
+            Node * prev = nullptr;
+            bool queued = false;
+            bool visited = false;
+        };
+
+        std::map<Path, Node> graph;
+
+        for (auto & path : closure)
+            graph.emplace(path, Node{path, store->queryPathInfo(path)->references});
+
+        // Transpose the graph.
+        for (auto & node : graph)
+            for (auto & ref : node.second.refs)
+                graph[ref].rrefs.insert(node.first);
 
-        bool first = true;
+        /* Run Dijkstra's shortest path algorithm to get the distance
+           of every path in the closure to 'dependency'. */
+        graph[dependencyPath].dist = 0;
+
+        std::priority_queue<Node *> queue;
+
+        queue.push(&graph.at(dependencyPath));
+
+        while (!queue.empty()) {
+            auto & node = *queue.top();
+            queue.pop();
+
+            for (auto & rref : node.rrefs) {
+                auto & node2 = graph.at(rref);
+                auto dist = node.dist + 1;
+                if (dist < node2.dist) {
+                    node2.dist = dist;
+                    node2.prev = &node;
+                    if (!node2.queued) {
+                        node2.queued = true;
+                        queue.push(&node2);
+                    }
+                }
+
+            }
+        }
 
-        for (auto & path : closure) {
+        /* Print the subgraph of nodes that have 'dependency' in their
+           closure (i.e., that have a non-infinite distance to
+           'dependency'). Print every edge on a path between `package`
+           and `dependency`. */
+        std::function<void(Node &, const string &, const string &)> printNode;
 
-            if (path == dependencyPath && packagePath != dependencyPath)
-                continue;
+        const string treeConn = "├───";
+        const string treeLast = "└───";
+        const string treeLine = "│   ";
+        const string treeNull = "    ";
 
-            if (!store->queryPathInfo(path)->references.count(dependencyPath))
-                continue;
+        struct BailOut { };
 
-            if (!first) std::cerr << "\n";
-            first = false;
+        printNode = [&](Node & node, const string & firstPad, const string & tailPad) {
+            assert(node.dist != inf);
+            std::cerr << fmt("%s%s%s%s" ANSI_NORMAL "\n",
+                firstPad,
+                node.visited ? "\e[38;5;244m" : "",
+                firstPad != "" ? "=> " : "",
+                node.path);
 
-            std::cerr << fmt("%s:\n", path);
+            if (node.path == dependencyPath && !all) throw BailOut();
 
-            std::function<void(const Path &)> recurse;
+            if (node.visited) return;
+            node.visited = true;
 
-            recurse = [&](const Path & p) {
+            /* Sort the references by distance to `dependency` to
+               ensure that the shortest path is printed first. */
+            std::multimap<size_t, Node *> refs;
+            std::set<std::string> hashes;
+
+            for (auto & ref : node.refs) {
+                if (ref == node.path) continue;
+                auto & node2 = graph.at(ref);
+                if (node2.dist == inf) continue;
+                refs.emplace(node2.dist, &node2);
+                hashes.insert(storePathToHash(node2.path));
+            }
+
+            /* For each reference, find the files and symlinks that
+               contain the reference. */
+            std::map<std::string, Strings> hits;
+
+            std::function<void(const Path &)> visitPath;
+
+            visitPath = [&](const Path & p) {
                 auto st = accessor->stat(p);
 
-                auto p2 = p == path ? "/" : std::string(p, path.size() + 1);
+                auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1);
+
+                auto getColour = [&](const std::string & hash) {
+                    return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE;
+                };
 
                 if (st.type == FSAccessor::Type::tDirectory) {
                     auto names = accessor->readDirectory(p);
                     for (auto & name : names)
-                        recurse(p + "/" + name);
+                        visitPath(p + "/" + name);
                 }
 
                 else if (st.type == FSAccessor::Type::tRegular) {
                     auto contents = accessor->readFile(p);
-                    auto pos = contents.find(dependencyPathHash);
-                    if (pos != std::string::npos) {
-                        size_t margin = 16;
-                        auto pos2 = pos >= margin ? pos - margin : 0;
-                        std::string fragment;
-                        for (char c : std::string(contents, pos2,
-                                pos - pos2 + dependencyPathHash.size() + margin))
-                        {
-                            fragment += isprint(c) ? c : '.';
-                        }
 
-                        std::cerr << fmt("  %s: …%s…\n",
-                            p2,
-                            hilite(fragment, pos - pos2, storePathHashLen));
+                    for (auto & hash : hashes) {
+                        auto pos = contents.find(hash);
+                        if (pos != std::string::npos) {
+                            size_t margin = 16;
+                            auto pos2 = pos >= margin ? pos - margin : 0;
+                            hits[hash].emplace_back(fmt("%s: …%s…\n",
+                                    p2,
+                                    hilite(filterPrintable(
+                                            std::string(contents, pos2, pos - pos2 + hash.size() + margin)),
+                                        pos - pos2, storePathHashLen,
+                                        getColour(hash))));
+                        }
                     }
                 }
 
                 else if (st.type == FSAccessor::Type::tSymlink) {
                     auto target = accessor->readLink(p);
-                    auto pos = target.find(dependencyPathHash);
-                    if (pos != std::string::npos) {
-                        std::cerr << fmt("  %s -> %s\n", p2, hilite(target, pos, storePathHashLen));
+
+                    for (auto & hash : hashes) {
+                        auto pos = target.find(hash);
+                        if (pos != std::string::npos)
+                            hits[hash].emplace_back(fmt("%s -> %s\n", p2,
+                                    hilite(target, pos, storePathHashLen, getColour(hash))));
                     }
                 }
             };
 
-            recurse(path);
-        }
+            visitPath(node.path);
+
+            for (auto & ref : refs) {
+                auto hash = storePathToHash(ref.second->path);
+
+                bool last = all ? ref == *refs.rbegin() : true;
+
+                for (auto & hit : hits[hash]) {
+                    bool first = hit == *hits[hash].begin();
+                    std::cerr << tailPad
+                              << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine))
+                              << hit;
+                    if (!all) break;
+                }
+
+                printNode(*ref.second,
+                    tailPad + (last ? treeNull : treeLine),
+                    tailPad + (last ? treeNull : treeLine));
+            }
+        };
+
+        try {
+            printNode(graph.at(packagePath), "", "");
+        } catch (BailOut & ) { }
     }
 };
 

From 1c58ad2ffa698d17b7be56eab69a40ace64d0e21 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 11:17:32 +0200
Subject: [PATCH 0559/2196] nix why-depends: Fix showing self-references

---
 src/nix/why-depends.cc | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index a6750ac95ca..f31469cbc96 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -1,6 +1,4 @@
 #include "command.hh"
-#include "common-args.hh"
-#include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
 #include "fs-accessor.hh"
@@ -59,9 +57,17 @@ struct CmdWhyDepends : SourceExprCommand
     {
         return {
             Example{
-                "To show which files in Hello's closure depend on Glibc:",
+                "To show one path through the dependency graph leading from Hello to Glibc:",
                 "nix why-depends nixpkgs.hello nixpkgs.glibc"
             },
+            Example{
+                "To show all files and paths in the dependency graph leading from Thunderbird to libX11:",
+                "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11"
+            },
+            Example{
+                "To show why Glibc depends on itself:",
+                "nix why-depends nixpkgs.glibc nixpkgs.glibc"
+            },
         };
     }
 
@@ -141,9 +147,9 @@ struct CmdWhyDepends : SourceExprCommand
            and `dependency`. */
         std::function<void(Node &, const string &, const string &)> printNode;
 
-        const string treeConn = "├───";
-        const string treeLast = "└───";
-        const string treeLine = "│   ";
+        const string treeConn = "╠═══";
+        const string treeLast = "╚═══";
+        const string treeLine = "║   ";
         const string treeNull = "    ";
 
         struct BailOut { };
@@ -156,7 +162,9 @@ struct CmdWhyDepends : SourceExprCommand
                 firstPad != "" ? "=> " : "",
                 node.path);
 
-            if (node.path == dependencyPath && !all) throw BailOut();
+            if (node.path == dependencyPath && !all
+                && packagePath != dependencyPath)
+                throw BailOut();
 
             if (node.visited) return;
             node.visited = true;
@@ -167,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand
             std::set<std::string> hashes;
 
             for (auto & ref : node.refs) {
-                if (ref == node.path) continue;
+                if (ref == node.path && packagePath != dependencyPath) continue;
                 auto & node2 = graph.at(ref);
                 if (node2.dist == inf) continue;
                 refs.emplace(node2.dist, &node2);

From 970366266b8df712f5f9cedb45af183ef5a8357f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 13:22:32 +0200
Subject: [PATCH 0560/2196] Add "nix add-to-store" command

---
 src/libutil/args.hh     |  7 +++--
 src/nix/add-to-store.cc | 61 +++++++++++++++++++++++++++++++++++++++++
 src/nix/build.cc        |  1 -
 src/nix/why-depends.cc  |  2 ++
 4 files changed, 67 insertions(+), 4 deletions(-)
 create mode 100644 src/nix/add-to-store.cc

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 7f8ea647c90..401bebbabc0 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -81,15 +81,16 @@ public:
         FlagMaker & handler(std::function<void(Strings)> handler) { flag->handler = handler; return *this; };
         FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
 
-        FlagMaker & dest(std::string * dest) {
-            assert(flag->arity == 1);
+        template<class T>
+        FlagMaker & dest(T * dest) {
+            flag->arity = 1;
             flag->handler = [=](Strings ss) { *dest = ss.front(); };
             return *this;
         };
 
         template<class T>
         FlagMaker & set(T * dest, const T & val) {
-            assert(flag->arity == 0);
+            flag->arity = 1;
             flag->handler = [=](Strings ss) { *dest = val; };
             return *this;
         };
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
new file mode 100644
index 00000000000..d0003790d3b
--- /dev/null
+++ b/src/nix/add-to-store.cc
@@ -0,0 +1,61 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "store-api.hh"
+#include "archive.hh"
+
+using namespace nix;
+
+struct CmdAddToStore : MixDryRun, StoreCommand
+{
+    Path path;
+    std::experimental::optional<std::string> namePart;
+
+    CmdAddToStore()
+    {
+        expectArg("path", &path);
+
+        mkFlag()
+            .longName("name")
+            .shortName('n')
+            .description("name component of the store path")
+            .labels({"name"})
+            .dest(&namePart);
+    }
+
+    std::string name() override
+    {
+        return "add-to-store";
+    }
+
+    std::string description() override
+    {
+        return "add a path to the Nix store";
+    }
+
+    Examples examples() override
+    {
+        return {
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        if (!namePart) namePart = baseNameOf(path);
+
+        StringSink sink;
+        dumpPath(path, sink);
+
+        ValidPathInfo info;
+        info.narHash = hashString(htSHA256, *sink.s);
+        info.narSize = sink.s->size();
+        info.path = store->makeFixedOutputPath(true, info.narHash, *namePart);
+        info.ca = makeFixedOutputCA(true, info.narHash);
+
+        if (!dryRun)
+            store->addToStore(info, sink.s);
+
+        std::cout << fmt("%s\n", info.path);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdAddToStore>());
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 915a066497b..f7c99f12dbb 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -15,7 +15,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand
             .longName("out-link")
             .shortName('o')
             .description("path of the symlink to the build result")
-            .arity(1)
             .labels({"path"})
             .dest(&outLink);
 
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index f31469cbc96..a90d07ed26e 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -233,6 +233,8 @@ struct CmdWhyDepends : SourceExprCommand
                 }
             };
 
+            // FIXME: should use scanForReferences().
+
             visitPath(node.path);
 
             for (auto & ref : refs) {

From da1e4fdfb54e0a69dd145180cae1eb7a0afd1b26 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 14:38:36 +0200
Subject: [PATCH 0561/2196] nix-store -q --roots / --gc --print-roots: Print
 temporary / in-memory roots

For example,

  $ nix-store -q --roots /nix/store/7phd2sav7068nivgvmj2vpm3v47fd27l-patchelf-0.8pre845_0315148
  {temp:1}

denotes that the path is only being kept alive by a temporary root
(i.e. /nix/var/nix/temproots/). Similarly,

  $ nix-store --gc --print-roots
  ...
  {memory:9} -> /nix/store/094gpjn9f15ip17wzxhma4r51nvsj17p-curl-7.53.1

shows that curl is being used by some process.
---
 src/libstore/gc.cc          | 57 ++++++++++++++++++++++++-------------
 src/libstore/local-store.hh |  6 ++--
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 45f28c328a5..fbc96f583ef 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -196,8 +196,10 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
+PathSet LocalStore::readTempRoots(FDs & fds)
 {
+    PathSet tempRoots;
+
     /* Read the `temproots' directory for per-process temporary root
        files. */
     DirEntries tempRootFiles = readDirectory(tempRootsDir);
@@ -253,6 +255,8 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
 
         fds.push_back(fd); /* keep open */
     }
+
+    return tempRoots;
 }
 
 
@@ -316,7 +320,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-Roots LocalStore::findRoots()
+Roots LocalStore::findRootsNoTemp()
 {
     Roots roots;
 
@@ -326,6 +330,27 @@ Roots LocalStore::findRoots()
         findRoots(stateDir + "/manifests", DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
+    /* Add additional roots returned by the program specified by the
+       NIX_ROOT_FINDER environment variable.  This is typically used
+       to add running programs to the set of roots (to prevent them
+       from being garbage collected). */
+    size_t n = 0;
+    for (auto & root : findRuntimeRoots())
+        roots[fmt("{memory:%d}", n++)] = root;
+
+    return roots;
+}
+
+
+Roots LocalStore::findRoots()
+{
+    Roots roots = findRootsNoTemp();
+
+    FDs fds;
+    size_t n = 0;
+    for (auto & root : readTempRoots(fds))
+        roots[fmt("{temp:%d}", n++)] = root;
+
     return roots;
 }
 
@@ -369,8 +394,9 @@ static void readFileRoots(const char * path, StringSet & paths)
     }
 }
 
-void LocalStore::findRuntimeRoots(PathSet & roots)
+PathSet LocalStore::findRuntimeRoots()
 {
+    PathSet roots;
     StringSet paths;
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
@@ -454,6 +480,8 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
                 roots.insert(path);
             }
         }
+
+    return roots;
 }
 
 
@@ -554,17 +582,13 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 
 bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
 {
-    if (visited.find(path) != visited.end()) return false;
+    if (visited.count(path)) return false;
 
-    if (state.alive.find(path) != state.alive.end()) {
-        return true;
-    }
+    if (state.alive.count(path)) return true;
 
-    if (state.dead.find(path) != state.dead.end()) {
-        return false;
-    }
+    if (state.dead.count(path)) return false;
 
-    if (state.roots.find(path) != state.roots.end()) {
+    if (state.roots.count(path)) {
         debug(format("cannot delete '%1%' because it's a root") % path);
         state.alive.insert(path);
         return true;
@@ -724,22 +748,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printError(format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : findRoots();
+    Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
 
     for (auto & i : rootMap) state.roots.insert(i.second);
 
-    /* Add additional roots returned by the program specified by the
-       NIX_ROOT_FINDER environment variable.  This is typically used
-       to add running programs to the set of roots (to prevent them
-       from being garbage collected). */
-    if (!options.ignoreLiveness)
-        findRuntimeRoots(state.roots);
-
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    readTempRoots(state.tempRoots, fds);
+    state.tempRoots = readTempRoots(fds);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 4973bd9a984..62a11a94a5e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -176,7 +176,7 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    void readTempRoots(PathSet & tempRoots, FDs & fds);
+    PathSet readTempRoots(FDs & fds);
 
 public:
 
@@ -261,7 +261,9 @@ private:
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    void findRuntimeRoots(PathSet & roots);
+    Roots findRootsNoTemp();
+
+    PathSet findRuntimeRoots();
 
     void removeUnusedLinks(const GCState & state);
 

From 308ecf63611c858327f163a154a3216fd6195ed9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 15:02:52 +0200
Subject: [PATCH 0562/2196] Show the pid of temporary roots

---
 src/libstore/gc.cc          | 25 +++++++++++++++----------
 src/libstore/local-store.hh |  2 +-
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index fbc96f583ef..ab2c5ca0274 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -196,17 +196,17 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-PathSet LocalStore::readTempRoots(FDs & fds)
+std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
 {
-    PathSet tempRoots;
+    std::set<std::pair<pid_t, Path>> tempRoots;
 
     /* Read the `temproots' directory for per-process temporary root
        files. */
-    DirEntries tempRootFiles = readDirectory(tempRootsDir);
-
-    for (auto & i : tempRootFiles) {
+    for (auto & i : readDirectory(tempRootsDir)) {
         Path path = tempRootsDir + "/" + i.name;
 
+        pid_t pid = std::stoi(i.name);
+
         debug(format("reading temporary root file '%1%'") % path);
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
         if (!*fd) {
@@ -247,9 +247,9 @@ PathSet LocalStore::readTempRoots(FDs & fds)
 
         while ((end = contents.find((char) 0, pos)) != string::npos) {
             Path root(contents, pos, end - pos);
-            debug(format("got temporary root '%1%'") % root);
+            debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots.insert(root);
+            tempRoots.emplace(pid, root);
             pos = end + 1;
         }
 
@@ -347,9 +347,13 @@ Roots LocalStore::findRoots()
     Roots roots = findRootsNoTemp();
 
     FDs fds;
+    pid_t prev = -1;
     size_t n = 0;
-    for (auto & root : readTempRoots(fds))
-        roots[fmt("{temp:%d}", n++)] = root;
+    for (auto & root : readTempRoots(fds)) {
+        if (prev != root.first) n = 0;
+        prev = root.first;
+        roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
+    }
 
     return roots;
 }
@@ -756,7 +760,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    state.tempRoots = readTempRoots(fds);
+    for (auto & root : readTempRoots(fds))
+        state.tempRoots.insert(root.second);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 62a11a94a5e..0a3841133e5 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -176,7 +176,7 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    PathSet readTempRoots(FDs & fds);
+    std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
 
 public:
 

From 89dc62c1745950748084a763ea325bc427045e07 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 18:10:38 +0200
Subject: [PATCH 0563/2196] RemoteStore: Add option to drop old connections
 from the pool

This is a hack to make hydra-queue-runner free its temproots
periodically, thereby ensuring that garbage collection of the
corresponding paths is not blocked until the queue runner is
restarted.

It would be better if temproots could be released earlier than at
process exit. I started working on a RAII object returned by functions
like addToStore() that releases temproots. However, this would be a
pretty massive change so I gave up on it for now.
---
 src/libstore/remote-store.cc | 16 +++++++++++++++-
 src/libstore/remote-store.hh |  6 ++++++
 src/libutil/pool.hh          | 10 ++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1af84cff5ba..b9076c0474d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -45,7 +45,13 @@ RemoteStore::RemoteStore(const Params & params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() { return openConnectionWrapper(); },
-            [](const ref<Connection> & r) { return r->to.good() && r->from.good(); }
+            [this](const ref<Connection> & r) {
+                return
+                    r->to.good()
+                    && r->from.good()
+                    && std::chrono::duration_cast<std::chrono::seconds>(
+                        std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
+            }
             ))
 {
 }
@@ -106,6 +112,8 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();
 
+    conn->startTime = std::chrono::steady_clock::now();
+
     initConnection(*conn);
 
     return conn;
@@ -619,6 +627,12 @@ void RemoteStore::connect()
 }
 
 
+void RemoteStore::flushBadConnections()
+{
+    connections->flushBad();
+}
+
+
 RemoteStore::Connection::~Connection()
 {
     try {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index e370e4797d2..30c6beae6ff 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -25,6 +25,9 @@ public:
     const Setting<int> maxConnections{(Store*) this, 1,
             "max-connections", "maximum number of concurrent connections to the Nix daemon"};
 
+    const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
+            "max-connection-age", "number of seconds to reuse a connection"};
+
     RemoteStore(const Params & params);
 
     /* Implementations of abstract store API methods. */
@@ -95,6 +98,8 @@ public:
 
     void connect() override;
 
+    void flushBadConnections();
+
 protected:
 
     struct Connection
@@ -102,6 +107,7 @@ protected:
         FdSink to;
         FdSource from;
         unsigned int daemonVersion;
+        std::chrono::time_point<std::chrono::steady_clock> startTime;
 
         virtual ~Connection();
 
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index 7033090020e..0b142b0597c 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -168,6 +168,16 @@ public:
     {
         return state.lock()->max;
     }
+
+    void flushBad()
+    {
+        auto state_(state.lock());
+        std::vector<ref<R>> left;
+        for (auto & p : state_->idle)
+            if (validator(p))
+                left.push_back(p);
+        std::swap(state_->idle, left);
+    }
 };
 
 }

From 346aeee1cb21b5cab5ddc3e8658c88321f513761 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Sep 2017 18:56:33 +0200
Subject: [PATCH 0564/2196] Remove Debian 8 and Ubuntu 14.10

These have a GCC (4.9) that is too old.

https://hydra.nixos.org/eval/1391740
---
 release.nix | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/release.nix b/release.nix
index 40df200fdc5..29cd36860ad 100644
--- a/release.nix
+++ b/release.nix
@@ -206,11 +206,9 @@ let
     rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
 
 
-    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
+    #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
+    #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
 
-    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] [];
-    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] [];
     deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
@@ -282,8 +280,8 @@ let
           binaryTarball.i686-linux
           binaryTarball.x86_64-darwin
           binaryTarball.x86_64-linux
-          deb_debian8i386
-          deb_debian8x86_64
+          #deb_debian8i386
+          #deb_debian8x86_64
           deb_ubuntu1604i386
           deb_ubuntu1604x86_64
           rpm_fedora25i386

From 5324bb9399b87782227cc98d83bc3c2d55b102fa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Sep 2017 11:07:17 +0200
Subject: [PATCH 0565/2196] Update release notes

---
 doc/manual/release-notes/rl-1.12.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 033c9b97135..e1529696530 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -397,6 +397,16 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     <para>Automatic garbage collection.</para>
   </listitem>
 
+  <listitem>
+    <para><command>nix why-depends</command></para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-store -q --roots</command> and
+    <command>nix-store --gc --print-roots</command> now show temporary
+    and in-memory roots.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>

From aca4f7dff0ee615164668e923deaf5cc96ab1c8a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Sep 2017 11:07:28 +0200
Subject: [PATCH 0566/2196] Don't remove Content-Encoding in fetchurl /
 nix-prefetch-url

Fixes #1568.
---
 src/libstore/builtins.cc                 | 1 +
 src/libstore/download.cc                 | 3 ++-
 src/libstore/download.hh                 | 1 +
 src/nix-prefetch-url/nix-prefetch-url.cc | 4 +++-
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index 1075b45c00b..4ca4a838e3c 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -27,6 +27,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
            the result anyway. */
         DownloadRequest request(url);
         request.verifyTLS = false;
+        request.decompress = false;
 
         /* Note: have to use a fresh downloader here because we're in
            a forked process. */
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index f5304daeefd..3f5e744dde1 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -269,7 +269,8 @@ struct CurlDownloader : public Downloader
                 done = true;
 
                 try {
-                    result.data = decodeContent(encoding, ref<std::string>(result.data));
+                    if (request.decompress)
+                        result.data = decodeContent(encoding, ref<std::string>(result.data));
                     callSuccess(success, failure, const_cast<const DownloadResult &>(result));
                     act.progress(result.data->size(), result.data->size());
                 } catch (...) {
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 752bf3723cf..f2d65ad8d61 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -17,6 +17,7 @@ struct DownloadRequest
     size_t tries = 5;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
+    bool decompress = true;
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(curActivity) { }
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 791a1eeb53c..7e62a033b45 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -158,7 +158,9 @@ int main(int argc, char * * argv)
             auto actualUri = resolveMirrorUri(state, uri);
 
             /* Download the file. */
-            auto result = getDownloader()->download(DownloadRequest(actualUri));
+            DownloadRequest req(actualUri);
+            req.decompress = false;
+            auto result = getDownloader()->download(req);
 
             AutoDelete tmpDir(createTempDir(), true);
             Path tmpFile = (Path) tmpDir + "/tmp";

From 84f112b1c8d3c5181b7a9b11d309f14f1709480d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Sep 2017 13:28:00 +0200
Subject: [PATCH 0567/2196] nix-shell: Ignore readFile() errors

Fixes #1563.
---
 src/nix-build/nix-build.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b36933a0ac2..a3d3c8007be 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -105,12 +105,12 @@ void mainWrapped(int argc, char * * argv)
     for (int i = 1; i < argc; ++i)
         args.push_back(argv[i]);
 
-    // Heuristic to see if we're invoked as a shebang script, namely, if we
-    // have a single argument, it's the name of an executable file, and it
-    // starts with "#!".
+    // Heuristic to see if we're invoked as a shebang script, namely,
+    // if we have at least one argument, it's the name of an
+    // executable file, and it starts with "#!".
     if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) {
         script = argv[1];
-        if (access(script.c_str(), F_OK) == 0 && access(script.c_str(), X_OK) == 0) {
+        try {
             auto lines = tokenizeString<Strings>(readFile(script), "\n");
             if (std::regex_search(lines.front(), std::regex("^#!"))) {
                 lines.pop_front();
@@ -126,7 +126,7 @@ void mainWrapped(int argc, char * * argv)
                             args.push_back(word);
                 }
             }
-        }
+        } catch (SysError &) { }
     }
 
     parseCmdLine(myName, args, [&](Strings::iterator & arg, const Strings::iterator & end) {

From 6920c237016e5f33d06f948a6f8485e23c1900f6 Mon Sep 17 00:00:00 2001
From: Matthieu Coudron <mattator@gmail.com>
Date: Thu, 21 Sep 2017 10:25:20 +0900
Subject: [PATCH 0568/2196] Improve error message for conflicting priorities

I find the error message 'nix-env --set-flag priority NUMBER PKGNAME'
not as helpful as it could be :
- doesn't share the current priorities
- doesn't say that the command must be run on the already installed
PKGNAME (which is confusing the first time)
- the doc needs careful reading:
"If there are multiple derivations matching a name in args that have the same name (e.g., gcc-3.3.6 and gcc-4.1.1), then the derivation with the highest priority is used."
if one stops reading there, he is screwed. Salvation comes with reading "A derivation can define a priority by declaring the meta.priority attribute. This attribute should be a number, with a higher value denoting a lower priority. The default priority is 0."
To sum it up, lower number wins. I tried to convey this idea in the
message too.
---
 src/buildenv/buildenv.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc
index 885c5e16903..f05aa7bf2fb 100644
--- a/src/buildenv/buildenv.cc
+++ b/src/buildenv/buildenv.cc
@@ -74,10 +74,11 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                     auto prevPriority = priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(format(
-                                "collision between '%1%' and '%2%'; "
-                                "use 'nix-env --set-flag priority NUMBER PKGNAME' "
+                                "Packages '%1%' and '%2%' have the same priority '%3%'"
+                                "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
                                 "to change the priority of one of the conflicting packages"
-                                ) % srcFile % target);
+                                " ('0' being the highest priority)"
+                                ) % srcFile % target % priority);
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)

From e350671737fc7bb1093cc89cc89ec9884dbbb6c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Sep 2017 13:25:55 +0200
Subject: [PATCH 0569/2196] Doh

---
 src/libutil/args.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 401bebbabc0..37e31825ab3 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -90,7 +90,7 @@ public:
 
         template<class T>
         FlagMaker & set(T * dest, const T & val) {
-            flag->arity = 1;
+            flag->arity = 0;
             flag->handler = [=](Strings ss) { *dest = val; };
             return *this;
         };

From e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Sep 2017 13:43:35 +0200
Subject: [PATCH 0570/2196] Add "nix show-derivation"

This debug command prints a store derivation in JSON format. For
example:

  $ nix show-derivation nixpkgs.hello
  {
    "/nix/store/ayjwpwwiyy04nh9z71rsdgd3q7bra7ch-hello-2.10.drv": {
      "outputs": {
        "out": {
          "path": "/nix/store/w5w4v29ql0qwqhczkdxs94ix2lh7ibgs-hello-2.10"
        }
      },
      "inputSrcs": [
        "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
      ],
      "inputDrvs": {
        "/nix/store/13839aqdf6x4k3b785rw5f2l7857l6y3-bash-4.4-p12.drv": [
          "out"
        ],
        "/nix/store/vgdx7fdc7d4iirmnwj2py1nrvr5qwzj7-hello-2.10.tar.gz.drv": [
          "out"
        ],
        "/nix/store/x3kkd0vsqfflbvwf1055l9mr39bg0ms0-stdenv.drv": [
          "out"
        ]
      },
      "platform": "x86_64-linux",
      "builder": "/nix/store/qp5fw57d38bd1n07ss4zxh88zg67c3vg-bash-4.4-p12/bin/bash",
      "args": [
        "-e",
        "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
      ],
      "env": {
        "buildInputs": "",
        "builder": "/nix/store/qp5fw57d38bd1n07ss4zxh88zg67c3vg-bash-4.4-p12/bin/bash",
        "configureFlags": "",
        "doCheck": "1",
        "name": "hello-2.10",
        "nativeBuildInputs": "",
        "out": "/nix/store/w5w4v29ql0qwqhczkdxs94ix2lh7ibgs-hello-2.10",
        "propagatedBuildInputs": "",
        "propagatedNativeBuildInputs": "",
        "src": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz",
        "stdenv": "/nix/store/6zngq1rdh0ans9qyckqimqibgnlvlfrm-stdenv",
        "system": "x86_64-linux"
      }
    }
  }

This removes the need for pp-aterm.
---
 src/nix/command.hh         |   4 ++
 src/nix/installables.cc    |  24 ++++++++
 src/nix/show-derivation.cc | 119 +++++++++++++++++++++++++++++++++++++
 3 files changed, 147 insertions(+)
 create mode 100644 src/nix/show-derivation.cc

diff --git a/src/nix/command.hh b/src/nix/command.hh
index 182b01ef92e..82400f31c07 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -207,4 +207,8 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode,
 Path toStorePath(ref<Store> store, RealiseMode mode,
     std::shared_ptr<Installable> installable);
 
+PathSet toDerivations(ref<Store> store,
+    std::vector<std::shared_ptr<Installable>> installables,
+    bool useDeriver = false);
+
 }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 76df05fa32d..b60f71b1488 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -303,6 +303,30 @@ Path toStorePath(ref<Store> store, RealiseMode mode,
     return *paths.begin();
 }
 
+PathSet toDerivations(ref<Store> store,
+    std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
+{
+    PathSet drvPaths;
+
+    for (auto & i : installables)
+        for (auto & b : i->toBuildables()) {
+            if (b.drvPath.empty()) {
+                if (!useDeriver)
+                    throw Error("argument '%s' did not evaluate to a derivation", i->what());
+                for (auto & output : b.outputs) {
+                    auto derivers = store->queryValidDerivers(output.second);
+                    if (derivers.empty())
+                        throw Error("'%s' does not have a known deriver", i->what());
+                    // FIXME: use all derivers?
+                    drvPaths.insert(*derivers.begin());
+                }
+            } else
+                drvPaths.insert(b.drvPath);
+        }
+
+    return drvPaths;
+}
+
 void InstallablesCommand::prepare()
 {
     installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables());
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
new file mode 100644
index 00000000000..ee94fded364
--- /dev/null
+++ b/src/nix/show-derivation.cc
@@ -0,0 +1,119 @@
+// FIXME: integrate this with nix path-info?
+
+#include "command.hh"
+#include "common-args.hh"
+#include "store-api.hh"
+#include "archive.hh"
+#include "json.hh"
+#include "derivations.hh"
+
+using namespace nix;
+
+struct CmdShowDerivation : InstallablesCommand
+{
+    bool recursive = false;
+
+    CmdShowDerivation()
+    {
+        mkFlag()
+            .longName("recursive")
+            .shortName('r')
+            .description("include the dependencies of the specified derivations")
+            .set(&recursive, true);
+    }
+
+    std::string name() override
+    {
+        return "show-derivation";
+    }
+
+    std::string description() override
+    {
+        return "show the contents of a store derivation";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To show the store derivation that results from evaluating the Hello package:",
+                "nix show-derivation nixpkgs.hello"
+            },
+            Example{
+                "To show the full derivation graph (if available) that produced your NixOS system:",
+                "nix show-derivation -r /run/current-system"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto drvPaths = toDerivations(store, installables, true);
+
+        if (recursive) {
+            PathSet closure;
+            store->computeFSClosure(drvPaths, closure);
+            drvPaths = closure;
+        }
+
+        {
+
+        JSONObject jsonRoot(std::cout, true);
+
+        for (auto & drvPath : drvPaths) {
+            if (!isDerivation(drvPath)) continue;
+
+            auto drvObj(jsonRoot.object(drvPath));
+
+            auto drv = readDerivation(drvPath);
+
+            {
+                auto outputsObj(drvObj.object("outputs"));
+                for (auto & output : drv.outputs) {
+                    auto outputObj(outputsObj.object(output.first));
+                    outputObj.attr("path", output.second.path);
+                    if (output.second.hash != "") {
+                        outputObj.attr("hashAlgo", output.second.hashAlgo);
+                        outputObj.attr("hash", output.second.hash);
+                    }
+                }
+            }
+
+            {
+                auto inputsList(drvObj.list("inputSrcs"));
+                for (auto & input : drv.inputSrcs)
+                    inputsList.elem(input);
+            }
+
+            {
+                auto inputDrvsObj(drvObj.object("inputDrvs"));
+                for (auto & input : drv.inputDrvs) {
+                    auto inputList(inputDrvsObj.list(input.first));
+                    for (auto & outputId : input.second)
+                        inputList.elem(outputId);
+                }
+            }
+
+            drvObj.attr("platform", drv.platform);
+            drvObj.attr("builder", drv.builder);
+
+            {
+                auto argsList(drvObj.list("args"));
+                for (auto & arg : drv.args)
+                    argsList.elem(arg);
+            }
+
+            {
+                auto envObj(drvObj.object("env"));
+                for (auto & var : drv.env)
+                    envObj.attr(var.first, var.second);
+            }
+        }
+
+        }
+
+        std::cout << "\n";
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdShowDerivation>());

From d1c6e0fe8c98106b85b4bbd5c7a775ff1972d2ff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Sep 2017 16:59:16 +0200
Subject: [PATCH 0571/2196] nix why-depends: Write to stdout

---
 src/nix/why-depends.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index a90d07ed26e..17e0595ae88 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -156,7 +156,7 @@ struct CmdWhyDepends : SourceExprCommand
 
         printNode = [&](Node & node, const string & firstPad, const string & tailPad) {
             assert(node.dist != inf);
-            std::cerr << fmt("%s%s%s%s" ANSI_NORMAL "\n",
+            std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n",
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "=> " : "",
@@ -209,7 +209,7 @@ struct CmdWhyDepends : SourceExprCommand
                     for (auto & hash : hashes) {
                         auto pos = contents.find(hash);
                         if (pos != std::string::npos) {
-                            size_t margin = 16;
+                            size_t margin = 32;
                             auto pos2 = pos >= margin ? pos - margin : 0;
                             hits[hash].emplace_back(fmt("%s: …%s…\n",
                                     p2,
@@ -244,7 +244,7 @@ struct CmdWhyDepends : SourceExprCommand
 
                 for (auto & hit : hits[hash]) {
                     bool first = hit == *hits[hash].begin();
-                    std::cerr << tailPad
+                    std::cout << tailPad
                               << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine))
                               << hit;
                     if (!all) break;

From bfef3ec855f23930b907fd7700d9d20a820dae66 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Sep 2017 13:14:45 +0200
Subject: [PATCH 0572/2196] nix build: Fix realising a store path

---
 src/nix/installables.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index b60f71b1488..c83d6316d3f 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -267,7 +267,9 @@ Buildables toBuildables(ref<Store> store, RealiseMode mode,
                     outputNames.insert(output.first);
                 pathsToBuild.insert(
                     b.drvPath + "!" + concatStringsSep(",", outputNames));
-            }
+            } else
+                for (auto & output : b.outputs)
+                    pathsToBuild.insert(output.second);
             buildables.push_back(std::move(b));
         }
     }

From b8875213dcedd60f419089392cd5a9b3cfaf6057 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Sep 2017 13:21:08 +0200
Subject: [PATCH 0573/2196] Tweak message

---
 src/nix/progress-bar.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 0c42fe5cc13..76138b2cce2 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -165,7 +165,7 @@ class ProgressBar : public Logger
 
         if (type == actQueryPathInfo) {
             auto name = storePathToName(getS(fields, 0));
-            i->s = fmt("querying about " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
+            i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
         }
 
         if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))

From 5421ad243e261d35bf8cb1000aa389399cf7657f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Sep 2017 18:28:54 +0200
Subject: [PATCH 0574/2196] nix copy: make recursive by default

---
 src/nix/command.cc | 16 ++++++++++++++--
 src/nix/command.hh |  2 +-
 src/nix/copy.cc    |  1 +
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index f69c5689656..0f6bb294b38 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -100,9 +100,21 @@ void StoreCommand::run()
     run(getStore());
 }
 
-StorePathsCommand::StorePathsCommand()
+StorePathsCommand::StorePathsCommand(bool recursive)
+    : recursive(recursive)
 {
-    mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive);
+    if (recursive)
+        mkFlag()
+            .longName("no-recursive")
+            .description("apply operation to specified paths only")
+            .set(&this->recursive, false);
+    else
+        mkFlag()
+            .longName("recursive")
+            .shortName('r')
+            .description("apply operation to closure of the specified paths")
+            .set(&this->recursive, true);
+
     mkFlag(0, "all", "apply operation to the entire store", &all);
 }
 
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 82400f31c07..bf897f620db 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -141,7 +141,7 @@ private:
 
 public:
 
-    StorePathsCommand();
+    StorePathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 8d7c6a0e8e4..071ac3890aa 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -17,6 +17,7 @@ struct CmdCopy : StorePathsCommand
     SubstituteFlag substitute = NoSubstitute;
 
     CmdCopy()
+        : StorePathsCommand(true)
     {
         mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri);
         mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri);

From f3e0d468218994343d0e595a50304cc122ad1406 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 28 Sep 2017 17:58:59 +0200
Subject: [PATCH 0575/2196] nix run: Restore signals

Otherwise Ctrl-C doesn't work.
---
 src/nix/run.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index c72ede99c1c..2f93ca35150 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -130,6 +130,8 @@ struct CmdRun : InstallablesCommand
 
         stopProgressBar();
 
+        restoreSignals();
+
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location
            (e.g. /home/eelco/nix/store), then run the command in a

From 92f9d18aa030805189b7bf31bb1d129c49edc83f Mon Sep 17 00:00:00 2001
From: Anthony Cowley <acowley@gmail.com>
Date: Mon, 2 Oct 2017 20:07:56 -0400
Subject: [PATCH 0576/2196] install-darwin-multi-user: relax assumption check

The installer will error out if a user's shell configuration includes any mention of ~nix-profile~, even if this is in a comment. This change is designed to do the bare minimum to ignore lines beginning with a `#`.
---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 5b466ac3c88..9bb33d4177e 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -318,7 +318,7 @@ EOF
 
     for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
         if [ -f "$file" ]; then
-            if grep -l ".nix-profile" "$file"; then
+            if grep -l "^[^#].*.nix-profile" "$file"; then
                 failure <<EOF
 I found a reference to a ".nix-profile" in $file.
 This has a high chance of breaking a new nix installation. It was most

From 476493dbf5d5f66db3fcfe305430972c5accf10f Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Mon, 25 Sep 2017 12:36:01 -0700
Subject: [PATCH 0577/2196] Reverse retry logic to retry in all but a few cases

It was getting too much like whac-a-mole listing all the retriable error
conditions, so we now retry by default and list the cases where retrying
is almost certainly hopeless.
---
 .gitignore               |  4 +++
 src/libstore/download.cc | 57 ++++++++++++++++++++++++++--------------
 2 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6163087384d..ce22fa007dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,16 +99,20 @@ perl/Makefile.config
 /misc/systemd/nix-daemon.socket
 /misc/upstart/nix-daemon.conf
 
+/src/resolve-system-dependencies/resolve-system-dependencies
+
 inst/
 
 *.a
 *.o
 *.so
+*.dylib
 *.dll
 *.exe
 *.dep
 *~
 *.pc
+*.plist
 
 # GNU Global
 GPATH
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 3f5e744dde1..608b8fd399b 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -278,26 +278,43 @@ struct CurlDownloader : public Downloader
                     callFailure(failure, std::current_exception());
                 }
             } else {
-                Error err =
-                    (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound :
-                    httpStatus == 403 ? Forbidden :
-                    (httpStatus == 408 || httpStatus == 500 || httpStatus == 503
-                        || httpStatus == 504  || httpStatus == 522 || httpStatus == 524
-                        || code == CURLE_COULDNT_RESOLVE_HOST
-                        || code == CURLE_RECV_ERROR
-
-                        // this seems to occur occasionally for retriable reasons, and shows up in an error like this:
-                        //   curl: (23) Failed writing body (315 != 16366)
-                        || code == CURLE_WRITE_ERROR
-
-                        // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable
-                        || code == CURLE_SSL_CONNECT_ERROR
-#if LIBCURL_VERSION_NUM >= 0x073200
-                        || code == CURLE_HTTP2
-                        || code == CURLE_HTTP2_STREAM
-#endif
-                        ) ? Transient :
-                    Misc;
+                // We treat most errors as transient, but won't retry when hopeless
+                Error err = Transient;
+
+                if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) {
+                    // The file is definitely not there
+                    err = NotFound;
+                } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
+                    // Don't retry on authentication/authorization failures
+                    err = Forbidden;
+                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) {
+                    // Most 4xx errors are client errors and are probably not worth retrying:
+                    //   * 408 means the server timed out waiting for us, so we try again
+                    err = Misc;
+                } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
+                    // Let's treat most 5xx (server) errors as transient, except for a handful:
+                    //   * 501 not implemented
+                    //   * 505 http version not supported
+                    //   * 511 we're behind a captive portal
+                    err = Misc;
+                } else {
+                    // Don't bother retrying on certain cURL errors either
+                    switch (code) {
+                        case CURLE_FAILED_INIT:
+                        case CURLE_NOT_BUILT_IN:
+                        case CURLE_REMOTE_ACCESS_DENIED:
+                        case CURLE_FILE_COULDNT_READ_FILE:
+                        case CURLE_FUNCTION_NOT_FOUND:
+                        case CURLE_ABORTED_BY_CALLBACK:
+                        case CURLE_BAD_FUNCTION_ARGUMENT:
+                        case CURLE_INTERFACE_FAILED:
+                        case CURLE_UNKNOWN_OPTION:
+                        err = Misc;
+                        break;
+                        default: // Shut up warnings
+                        break;
+                    }
+                }
 
                 attempt++;
 

From 3139cad9cd3aab9e1fe06b6fb5e9e11b3524aca2 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Mon, 2 Oct 2017 23:41:29 -0400
Subject: [PATCH 0578/2196] Fix tests

`nix copy` no longer accepts a `--recursive` argument
---
 tests/binary-cache.sh | 4 ++--
 tests/nix-channel.sh  | 2 +-
 tests/repair.sh       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 532099d0214..30d7cc6ba88 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -6,7 +6,7 @@ clearCache
 # Create the binary cache.
 outPath=$(nix-build dependencies.nix --no-out-link)
 
-nix copy --recursive --to file://$cacheDir $outPath
+nix copy --to file://$cacheDir $outPath
 
 
 basicTests() {
@@ -117,7 +117,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
 res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
 otherKey="$(cat $TEST_ROOT/pk3)"
 
-nix copy --recursive --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
+nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 
 
 # Downloading should fail if we don't provide a key.
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 553ada51d9f..55f1695c441 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -15,7 +15,7 @@ nix-channel --remove xyzzy
 # Create a channel.
 rm -rf $TEST_ROOT/foo
 mkdir -p $TEST_ROOT/foo
-nix copy --recursive --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix))
+nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix))
 rm -rf $TEST_ROOT/nixexprs
 mkdir -p $TEST_ROOT/nixexprs
 cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
diff --git a/tests/repair.sh b/tests/repair.sh
index 57152d450a1..7c928e3be73 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -46,7 +46,7 @@ fi
 # --verify can fix it.
 clearCache
 
-nix copy --recursive --to file://$cacheDir $path
+nix copy --to file://$cacheDir $path
 
 chmod u+w $path2
 rm -rf $path2

From 60ecbd793457d0951273ff35df90aaeb950188c0 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Tue, 3 Oct 2017 00:54:00 -0400
Subject: [PATCH 0579/2196] More elegant test output

I got sick of trying to find the failures in the sea of debug output, so
we now:
- Hide test output unless it fails
- Sprinkle in some simple color
- Pad results for a more tabular look

If Nix is getting a more friendly user interface, we might as well get
a friendlier developer interface, right? :)
---
 mk/tests.mk | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/mk/tests.mk b/mk/tests.mk
index 004a4802861..aa97d6427cc 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -7,20 +7,38 @@ define run-install-test
 
 endef
 
+# Color code from https://unix.stackexchange.com/a/10065
 installcheck:
-	@total=0; failed=0; for i in $(_installcheck-list); do \
+	@total=0; failed=0; \
+	pad="                           "; \
+	red=""; \
+	green=""; \
+	normal=""; \
+	if [ -t 1 ]; then \
+		ncolors="$$(tput colors)"; \
+		if [[ -n "$$ncolors" && $$ncolors -ge 8 ]]; then \
+			red="$$(tput setaf 1)"; \
+			green="$$(tput setaf 2)"; \
+			normal="$$(tput sgr0)"; \
+		fi; \
+	fi; \
+	for i in $(_installcheck-list); do \
 	  total=$$((total + 1)); \
-	  echo "running test $$i"; \
-	  if (cd $$(dirname $$i) && $(tests-environment) $$(basename $$i)); then \
-	    echo "PASS: $$i"; \
+	  printf "running test $$i... $${pad:$${#i}}"; \
+	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
+	  if [ $$? == 0 ]; then \
+	    echo "[$${green}PASS$$normal]"; \
 	  else \
-	    echo "FAIL: $$i"; \
+	    echo "[$${red}FAIL$$normal]"; \
+	    echo "$$log" | sed 's/^/    /'; \
 	    failed=$$((failed + 1)); \
 	  fi; \
 	done; \
 	if [ "$$failed" != 0 ]; then \
-	  echo "$$failed out of $$total tests failed "; \
+	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
 	  exit 1; \
+	else \
+		echo "$${green}All tests succeeded"; \
 	fi
 
 .PHONY: check installcheck

From fb98e29067bb3899b3ebf876128a7032dba295e5 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 5 Oct 2017 09:07:55 -0700
Subject: [PATCH 0580/2196] darwin installer: Fix on systems where sudo -i is
 disabled.

---
 scripts/install-darwin-multi-user.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 5b466ac3c88..cb255e83c63 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -730,13 +730,13 @@ configure_shell_profile() {
 
 setup_default_profile() {
     _sudo "to installing a bootstrapping Nix in to the default Profile" \
-          -i "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
+          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
 
     _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
-          -i "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
 
     _sudo "to update the default channel in the default profile" \
-          -i NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+          HOME=$ROOT_HOME NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
 }
 
 

From e94fc238cf87b90547803d29b04f55e0bb211daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 6 Oct 2017 06:04:12 -0500
Subject: [PATCH 0581/2196] fixing bashisms in test code

This fixed the build on ubuntu/debian, where dash is the sh.
---
 mk/tests.mk | 7 +++----
 release.nix | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/mk/tests.mk b/mk/tests.mk
index aa97d6427cc..d18b910584f 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -10,13 +10,12 @@ endef
 # Color code from https://unix.stackexchange.com/a/10065
 installcheck:
 	@total=0; failed=0; \
-	pad="                           "; \
 	red=""; \
 	green=""; \
 	normal=""; \
 	if [ -t 1 ]; then \
 		ncolors="$$(tput colors)"; \
-		if [[ -n "$$ncolors" && $$ncolors -ge 8 ]]; then \
+		if [ -n "$$ncolors" ] && [ "$$ncolors" -ge 8 ]; then \
 			red="$$(tput setaf 1)"; \
 			green="$$(tput setaf 2)"; \
 			normal="$$(tput sgr0)"; \
@@ -24,9 +23,9 @@ installcheck:
 	fi; \
 	for i in $(_installcheck-list); do \
 	  total=$$((total + 1)); \
-	  printf "running test $$i... $${pad:$${#i}}"; \
+	  printf "running test $$i..."; \
 	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  if [ $$? == 0 ]; then \
+	  if [ $$? -eq 0 ]; then \
 	    echo "[$${green}PASS$$normal]"; \
 	  else \
 	    echo "[$${red}FAIL$$normal]"; \
diff --git a/release.nix b/release.nix
index 29cd36860ad..a9819925884 100644
--- a/release.nix
+++ b/release.nix
@@ -332,7 +332,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "ncurses-bin" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;

From 2f1a1c5a49b50594a4aa9d4729a5b64255dd0932 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 7 Oct 2017 07:43:55 -0500
Subject: [PATCH 0582/2196] release-common: Fix busybox builtins (busybox >=
 1.27)

See https://github.com/NixOS/nixpkgs/pull/28261
---
 release-common.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index c64fc619df6..4553118e1f5 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -7,8 +7,8 @@ rec {
     enableMinimal = true;
     extraConfig = ''
       CONFIG_ASH y
-      CONFIG_ASH_BUILTIN_ECHO y
-      CONFIG_ASH_BUILTIN_TEST y
+      CONFIG_ASH_ECHO y
+      CONFIG_ASH_TEST y
       CONFIG_ASH_OPTIMIZE_FOR_SIZE y
     '';
   };

From 37164ae389f0118aaec75f2566198e0b5129d9d7 Mon Sep 17 00:00:00 2001
From: Peter Simons <simons@cryp.to>
Date: Sat, 7 Oct 2017 17:28:34 +0200
Subject: [PATCH 0583/2196] docker: ensure that the installation works for
 users other than 'root', too

The path /root/.nix-profile is a sym-link to /nix/var/nix/profiles/default. The
latter path, however, works for everyone while the former path works only for
root, so we prefer the public path whenever possible.
---
 misc/docker/Dockerfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index fb6f73517bb..d6b88c7e91a 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -8,19 +8,19 @@ RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
   && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
-  && ln -s /root/.nix-profile/etc/profile.d/nix.sh /etc/profile.d/ \
+  && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \
   && rm -r /nix-*-x86_64-linux \
   && rm -r /var/cache/apk/*
 
 ONBUILD ENV \
     ENV=/etc/profile \
-    PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt
+    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
+    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
 
 ENV \
     ENV=/etc/profile \
-    PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \
-    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels/
+    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
+    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
+    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels

From 838509d1a085b8f7dc5474e13cfce3fbe3c47166 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Oct 2017 15:03:15 +0200
Subject: [PATCH 0584/2196] Whitespace

---
 mk/tests.mk | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mk/tests.mk b/mk/tests.mk
index d18b910584f..1138857c3c1 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -26,9 +26,9 @@ installcheck:
 	  printf "running test $$i..."; \
 	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
 	  if [ $$? -eq 0 ]; then \
-	    echo "[$${green}PASS$$normal]"; \
+	    echo " [$${green}PASS$$normal]"; \
 	  else \
-	    echo "[$${red}FAIL$$normal]"; \
+	    echo " [$${red}FAIL$$normal]"; \
 	    echo "$$log" | sed 's/^/    /'; \
 	    failed=$$((failed + 1)); \
 	  fi; \

From fda7b95cb08c447a7ee4ec18ea3574d76e6264df Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Oct 2017 15:07:07 +0200
Subject: [PATCH 0585/2196] Fix a hang in ThreadPool

The worker threads could exit prematurely if they finished processing
all items while the main thread was still adding items. In particular,
this caused hanging nix-store --serve processes in the build farm.

Also, process items from the main thread.
---
 src/libutil/thread-pool.cc | 81 ++++++++++++++++++++++++--------------
 src/libutil/thread-pool.hh |  9 +++--
 2 files changed, 57 insertions(+), 33 deletions(-)

diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index ce126d36db8..857ee91f87d 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -13,10 +13,15 @@ ThreadPool::ThreadPool(size_t _maxThreads)
         if (!maxThreads) maxThreads = 1;
     }
 
-    debug(format("starting pool of %d threads") % maxThreads);
+    debug("starting pool of %d threads", maxThreads - 1);
 }
 
 ThreadPool::~ThreadPool()
+{
+    shutdown();
+}
+
+void ThreadPool::shutdown()
 {
     std::vector<std::thread> workers;
     {
@@ -25,7 +30,9 @@ ThreadPool::~ThreadPool()
         std::swap(workers, state->workers);
     }
 
-    debug(format("reaping %d worker threads") % workers.size());
+    if (workers.empty()) return;
+
+    debug("reaping %d worker threads", workers.size());
 
     work.notify_all();
 
@@ -38,32 +45,43 @@ void ThreadPool::enqueue(const work_t & t)
     auto state(state_.lock());
     if (quit)
         throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down");
-    state->left.push(t);
-    if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads)
-        state->workers.emplace_back(&ThreadPool::workerEntry, this);
+    state->pending.push(t);
+    /* Note: process() also executes items, so count it as a worker. */
+    if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads)
+        state->workers.emplace_back(&ThreadPool::doWork, this, false);
     work.notify_one();
 }
 
 void ThreadPool::process()
 {
-    /* Loop until there are no active work items *and* there either
-       are no queued items or there is an exception. The
-       post-condition is that no new items will become active. */
-    while (true) {
+    state_.lock()->draining = true;
+
+    /* Do work until no more work is pending or active. */
+    try {
+        doWork(true);
+
         auto state(state_.lock());
-        if (!state->active) {
-            if (state->exception)
-                std::rethrow_exception(state->exception);
-            if (state->left.empty())
-                break;
-        }
-        state.wait(done);
+
+        assert(quit);
+
+        if (state->exception)
+            std::rethrow_exception(state->exception);
+
+    } catch (...) {
+        /* In the exceptional case, some workers may still be
+           active. They may be referencing the stack frame of the
+           caller. So wait for them to finish. (~ThreadPool also does
+           this, but it might be destroyed after objects referenced by
+           the work item lambdas.) */
+        shutdown();
+        throw;
     }
 }
 
-void ThreadPool::workerEntry()
+void ThreadPool::doWork(bool mainThread)
 {
-    interruptCheck = [&]() { return (bool) quit; };
+    if (!mainThread)
+        interruptCheck = [&]() { return (bool) quit; };
 
     bool didWork = false;
     std::exception_ptr exc;
@@ -99,24 +117,27 @@ void ThreadPool::workerEntry()
                 }
             }
 
-            /* Wait until a work item is available or another thread
-               had an exception or we're asked to quit. */
+            /* Wait until a work item is available or we're asked to
+               quit. */
             while (true) {
-                if (quit) {
-                    if (!state->active)
-                        done.notify_one();
-                    return;
-                }
-                if (!state->left.empty()) break;
-                if (!state->active) {
-                    done.notify_one();
+                if (quit) return;
+
+                if (!state->pending.empty()) break;
+
+                /* If there are no active or pending items, and the
+                   main thread is running process(), then no new items
+                   can be added. So exit. */
+                if (!state->active && state->draining) {
+                    quit = true;
+                    work.notify_all();
                     return;
                 }
+
                 state.wait(work);
             }
 
-            w = std::move(state->left.front());
-            state->left.pop();
+            w = std::move(state->pending.front());
+            state->pending.pop();
             state->active++;
         }
 
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index 06a097ab5ea..bb16b639a59 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -44,19 +44,22 @@ private:
 
     struct State
     {
-        std::queue<work_t> left;
+        std::queue<work_t> pending;
         size_t active = 0;
         std::exception_ptr exception;
         std::vector<std::thread> workers;
+        bool draining = false;
     };
 
     std::atomic_bool quit{false};
 
     Sync<State> state_;
 
-    std::condition_variable work, done;
+    std::condition_variable work;
 
-    void workerEntry();
+    void doWork(bool mainThread);
+
+    void shutdown();
 };
 
 /* Process in parallel a set of items of type T that have a partial

From 1dd29d7aebae706f3e90a18bbfae727f2ed03c70 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 12 Oct 2017 18:21:55 +0200
Subject: [PATCH 0586/2196] Add option to disable the seccomp filter

I needed this to test ACL/xattr removal in
canonicalisePathMetaData(). Might also be useful if you need to build
old Nixpkgs that doesn't have the required patches to remove
setuid/setgid creation.
---
 src/libstore/build.cc   | 2 ++
 src/libstore/globals.hh | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 3b3cebfb18f..64cbc19bd96 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2351,6 +2351,8 @@ void DerivationGoal::doExportReferencesGraph()
 void setupSeccomp()
 {
 #if __linux__
+    if (!settings.filterSyscalls) return;
+
     scmp_filter_ctx ctx;
 
     if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 41d3323117b..264e82a16e2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -336,6 +336,12 @@ public:
         "String appended to the user agent in HTTP requests."};
 
 #if __linux__
+    Setting<bool> filterSyscalls{this, true, "filter-syscalls",
+            "Whether to prevent certain dangerous system calls, such as "
+            "creation of setuid/setgid files or adding ACLs or extended "
+            "attributes. Only disable this if you're aware of the "
+            "security implications."};
+
     Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
         "Whether builders can acquire new privileges by calling programs with "
         "setuid/setgid bits or with file capabilities."};

From 4e58294ae62075fae1a2e82e1f62f627c2a0bd80 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 16 Oct 2017 12:56:58 -0400
Subject: [PATCH 0587/2196] fetchgit: Remove incomplete/unneeded isURI check.

This check spuriously fails for e.g. git@github.com:NixOS/nixpkgs.git,
and even for ssh://git@github.com/NixOS/nixpkgs.git, and is made
redundant by the checks git itself will do when fetching the repo. We
instead pass a -- before passing the URI to git to avoid injection.
---
 src/libexpr/primops/fetchgit.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index e16c8235378..545954f5843 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -13,9 +13,6 @@ namespace nix {
 Path exportGit(ref<Store> store, const std::string & uri,
     const std::string & ref, const std::string & rev)
 {
-    if (!isUri(uri))
-        throw EvalError(format("'%s' is not a valid URI") % uri);
-
     if (rev != "") {
         std::regex revRegex("^[0-9a-fA-F]{40}$");
         if (!std::regex_match(rev, revRegex))
@@ -47,7 +44,7 @@ Path exportGit(ref<Store> store, const std::string & uri,
     if (stat(localRefFile.c_str(), &st) != 0 ||
         st.st_mtime < now - settings.tarballTtl)
     {
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", "--", uri, ref + ":" + localRef });
 
         struct timeval times[2];
         times[0].tv_sec = now;

From 6a037a738a931c51e0112a945bc2d319b5ae9052 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 16 Oct 2017 14:51:39 -0400
Subject: [PATCH 0588/2196] Pull nix-profile-daemon from 1.11

---
 scripts/local.mk                 |  1 +
 scripts/nix-profile-daemon.sh.in | 54 ++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 scripts/nix-profile-daemon.sh.in

diff --git a/scripts/local.mk b/scripts/local.mk
index 9524baf81b0..2a00558521b 100644
--- a/scripts/local.mk
+++ b/scripts/local.mk
@@ -8,5 +8,6 @@ noinst-scripts += $(nix_noinst_scripts)
 profiledir = $(sysconfdir)/profile.d
 
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
+$(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644))
 
 clean-files += $(nix_noinst_scripts)
diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
new file mode 100644
index 00000000000..43c7156062d
--- /dev/null
+++ b/scripts/nix-profile-daemon.sh.in
@@ -0,0 +1,54 @@
+# Only execute this file once per shell.
+if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
+__ETC_PROFILE_NIX_SOURCED=1
+
+# Set up secure multi-user builds: non-root users build through the
+# Nix daemon.
+if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then
+    export NIX_REMOTE=daemon
+fi
+
+export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
+export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
+
+# Set up the per-user profile.
+mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
+if ! test -O "$NIX_USER_PROFILE_DIR"; then
+    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
+fi
+
+if test -w $HOME; then
+  if ! test -L $HOME/.nix-profile; then
+      if test "$USER" != root; then
+          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
+      else
+          # Root installs in the system-wide profile by default.
+          ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile
+      fi
+  fi
+
+  # Subscribe the root user to the NixOS channel by default.
+  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
+      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
+  fi
+
+  # Create the per-user garbage collector roots directory.
+  NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER
+  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
+  if ! test -O "$NIX_USER_GCROOTS_DIR"; then
+      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
+  fi
+
+  # Set up a default Nix expression from which to install stuff.
+  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
+      rm -f $HOME/.nix-defexpr
+      mkdir -p $HOME/.nix-defexpr
+      if [ "$USER" != root ]; then
+          ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
+      fi
+  fi
+fi
+
+export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels"
+export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"

From cbc216911dbda23c3bc050c969bc725fe60760ef Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 16 Oct 2017 13:25:24 -0400
Subject: [PATCH 0589/2196] fetchgit: Remove incomplete/unneeded ref check.

This check fails for tags and branches, and is made redundant by the
checks git itself will do when fetching the repo.
---
 src/libexpr/primops/fetchgit.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 545954f5843..9db4adbc9da 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -19,11 +19,6 @@ Path exportGit(ref<Store> store, const std::string & uri,
             throw Error("invalid Git revision '%s'", rev);
     }
 
-    // FIXME: too restrictive, but better safe than sorry.
-    std::regex refRegex("^[0-9a-zA-Z][0-9a-zA-Z.-]+$");
-    if (!std::regex_match(ref, refRegex))
-        throw Error("invalid Git ref '%s'", ref);
-
     Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {

From 11ba4302e3666d3860ca83b734217afe07f22de2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Oct 2017 20:51:42 +0200
Subject: [PATCH 0590/2196] Remove BinaryCacheStoreAccessor

Probably as a result of a bad merge in
4b8f1b0ec066a5b994747b1afd050f5f62d857f6, we had both a
BinaryCacheStoreAccessor and a
RemoteFSAccessor. BinaryCacheStore::getFSAccessor() returned the
latter, but BinaryCacheStore::addToStore() checked for the
former. This probably caused hydra-queue-runner to download paths that
it just uploaded.
---
 src/libstore/binary-cache-store.cc | 62 +-----------------------------
 src/libstore/remote-fs-accessor.hh |  3 ++
 2 files changed, 4 insertions(+), 61 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 556fa3d5935..9d497110f62 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -17,66 +17,6 @@
 
 namespace nix {
 
-/* Given requests for a path /nix/store/<x>/<y>, this accessor will
-   first download the NAR for /nix/store/<x> from the binary cache,
-   build a NAR accessor for that NAR, and use that to access <y>. */
-struct BinaryCacheStoreAccessor : public FSAccessor
-{
-    ref<BinaryCacheStore> store;
-
-    std::map<Path, ref<FSAccessor>> nars;
-
-    BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
-        : store(store)
-    {
-    }
-
-    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
-    {
-        auto path = canonPath(path_);
-
-        auto storePath = store->toStorePath(path);
-        std::string restPath = std::string(path, storePath.size());
-
-        if (!store->isValidPath(storePath))
-            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
-
-        auto i = nars.find(storePath);
-        if (i != nars.end()) return {i->second, restPath};
-
-        StringSink sink;
-        store->narFromPath(storePath, sink);
-
-        auto accessor = makeNarAccessor(sink.s);
-        nars.emplace(storePath, accessor);
-        return {accessor, restPath};
-    }
-
-    Stat stat(const Path & path) override
-    {
-        auto res = fetch(path);
-        return res.first->stat(res.second);
-    }
-
-    StringSet readDirectory(const Path & path) override
-    {
-        auto res = fetch(path);
-        return res.first->readDirectory(res.second);
-    }
-
-    std::string readFile(const Path & path) override
-    {
-        auto res = fetch(path);
-        return res.first->readFile(res.second);
-    }
-
-    std::string readLink(const Path & path) override
-    {
-        auto res = fetch(path);
-        return res.first->readLink(res.second);
-    }
-};
-
 BinaryCacheStore::BinaryCacheStore(const Params & params)
     : Store(params)
 {
@@ -161,7 +101,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
 
-    auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
+    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
 
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 28f36c8296e..df8b7b16291 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -13,6 +13,9 @@ class RemoteFSAccessor : public FSAccessor
     std::map<Path, ref<FSAccessor>> nars;
 
     std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
+
+    friend class BinaryCacheStore;
+
 public:
 
     RemoteFSAccessor(ref<Store> store);

From ca580bec35ea4d1984e36864158d7be99cfcb34b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Oct 2017 21:15:33 +0200
Subject: [PATCH 0591/2196] BinaryCacheStore: Support local caching of NARs

This speeds up commands like "nix cat-store". For example:

  $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf
  real    0m4.336s

  $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf
  real    0m0.045s

The primary motivation is to allow hydra-server to serve files from S3
binary caches. Previously Hydra had a hack to do "nix-store -r
<path>", but that fetches the entire closure so is prohibitively
expensive.

There is no garbage collection of the NAR cache yet. Also, the entire
NAR is read when accessing a single member file. We could generate the
NAR listing to provide random access.

Note: the NAR cache is indexed by the store path hash, not the content
hash, so NAR caches should not be shared between binary caches, unless
you're sure that all your builds are binary-reproducible.
---
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/binary-cache-store.hh |  1 +
 src/libstore/remote-fs-accessor.cc | 22 +++++++++++++++++++---
 src/libstore/remote-fs-accessor.hh |  5 ++++-
 4 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9d497110f62..f1179f1890a 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -319,7 +319,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
 }
 
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index f9c1c2cbe8a..d3b0e0bd933 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -18,6 +18,7 @@ public:
     const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
+    const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
 
 private:
 
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 098151f8c0f..da4e30b2263 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -3,10 +3,12 @@
 
 namespace nix {
 
-
-RemoteFSAccessor::RemoteFSAccessor(ref<Store> store)
+RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
     : store(store)
+    , cacheDir(cacheDir)
 {
+    if (cacheDir != "")
+        createDirs(cacheDir);
 }
 
 std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
@@ -23,7 +25,21 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
-    store->narFromPath(storePath, sink);
+
+    Path cacheFile = cacheDir != "" ? fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)) : "";
+
+    try {
+        if (cacheFile != "")
+            *sink.s = nix::readFile(cacheFile);
+    } catch (SysError &) { }
+
+    if (sink.s->empty()) {
+        store->narFromPath(storePath, sink);
+
+        if (cacheFile != "")
+            /* FIXME: do this asynchronously. */
+            writeFile(cacheFile, *sink.s);
+    }
 
     auto accessor = makeNarAccessor(sink.s);
     nars.emplace(storePath, accessor);
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index df8b7b16291..d359ecc9c87 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -12,13 +12,16 @@ class RemoteFSAccessor : public FSAccessor
 
     std::map<Path, ref<FSAccessor>> nars;
 
+    Path cacheDir;
+
     std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
 
     friend class BinaryCacheStore;
 
 public:
 
-    RemoteFSAccessor(ref<Store> store);
+    RemoteFSAccessor(ref<Store> store,
+        const /* FIXME: use std::optional */ Path & cacheDir = "");
 
     Stat stat(const Path & path) override;
 

From b24b8ef77c63727b9d6494fe25cfce45d57581be Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Oct 2017 21:39:48 +0200
Subject: [PATCH 0592/2196] BinaryCacheStore::addToStore(): Add NARs to the
 local cache

---
 src/libstore/binary-cache-store.cc |  8 ++++++--
 src/libstore/remote-fs-accessor.cc | 24 ++++++++++++++++--------
 src/libstore/remote-fs-accessor.hh |  4 ++++
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f1179f1890a..67607ab3d43 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -114,8 +114,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             auto narAccessor = makeNarAccessor(nar);
 
-            if (accessor_)
+            if (accessor_) {
                 accessor_->nars.emplace(info.path, narAccessor);
+                accessor_->addToCache(info.path, *nar);
+            }
 
             std::function<void(const Path &, JSONPlaceholder &)> recurse;
 
@@ -160,8 +162,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     }
 
     else {
-        if (accessor_)
+        if (accessor_) {
             accessor_->nars.emplace(info.path, makeNarAccessor(nar));
+            accessor_->addToCache(info.path, *nar);
+        }
     }
 
     /* Compress the NAR. */
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index da4e30b2263..93969120462 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -11,6 +11,19 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
+Path RemoteFSAccessor::makeCacheFile(const Path & storePath)
+{
+    assert(cacheDir != "");
+    return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath));
+}
+
+void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar)
+{
+    if (cacheDir != "")
+        /* FIXME: do this asynchronously. */
+        writeFile(makeCacheFile(storePath), nar);
+}
+
 std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
 {
     auto path = canonPath(path_);
@@ -26,19 +39,14 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
 
     StringSink sink;
 
-    Path cacheFile = cacheDir != "" ? fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)) : "";
-
     try {
-        if (cacheFile != "")
-            *sink.s = nix::readFile(cacheFile);
+        if (cacheDir != "")
+            *sink.s = nix::readFile(makeCacheFile(storePath));
     } catch (SysError &) { }
 
     if (sink.s->empty()) {
         store->narFromPath(storePath, sink);
-
-        if (cacheFile != "")
-            /* FIXME: do this asynchronously. */
-            writeFile(cacheFile, *sink.s);
+        addToCache(storePath, *sink.s);
     }
 
     auto accessor = makeNarAccessor(sink.s);
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index d359ecc9c87..2a3fc01eff5 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -18,6 +18,10 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
+    Path makeCacheFile(const Path & storePath);
+
+    void addToCache(const Path & storePath, const std::string & nar);
+
 public:
 
     RemoteFSAccessor(ref<Store> store,

From 6e5165b77370c76bfa39d4b55e9f83673f3bd466 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Tue, 17 Oct 2017 19:15:51 -0400
Subject: [PATCH 0593/2196] Shift Darwin sandbox to separate installed files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This makes it slightly more manageable to see at a glance what in a
build's sandbox profile is unique to the build and what is standard. Also
a first step to factoring more of our Darwin logic into scheme functions
that will allow us a bit more flexibility. And of course less of that
nasty codegen in C++! 😀
---
 src/libstore/build.cc | 14 +++++---------
 src/libstore/local.mk | 10 ++++++----
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 64cbc19bd96..88c51654614 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2774,14 +2774,10 @@ void DerivationGoal::runChild()
                     sandboxProfile += "(deny default (with no-log))\n";
                 }
 
-                sandboxProfile +=
-                    #include "sandbox-defaults.sb.gen.hh"
-                    ;
+                sandboxProfile += "(import \"sandbox-defaults.sb\")";
 
                 if (fixedOutput)
-                    sandboxProfile +=
-                        #include "sandbox-network.sb.gen.hh"
-                        ;
+                    sandboxProfile += "(import \"sandbox-network.sb\")";
 
                 /* Our rwx outputs */
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
@@ -2824,9 +2820,7 @@ void DerivationGoal::runChild()
 
                 sandboxProfile += additionalSandboxProfile;
             } else
-                sandboxProfile +=
-                    #include "sandbox-minimal.sb.gen.hh"
-                    ;
+                sandboxProfile += "(import \"sandbox-minimal.sb\")";
 
             debug("Generated sandbox profile:");
             debug(sandboxProfile);
@@ -2848,6 +2842,8 @@ void DerivationGoal::runChild()
             args.push_back(sandboxFile);
             args.push_back("-D");
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
+            args.push_back("-D");
+            args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
             args.push_back(drv->builder);
         }
 #endif
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 36b270f2e07..50c46ce6fe9 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -10,6 +10,10 @@ libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 
+libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
+
+$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
+
 ifeq ($(ENABLE_S3), 1)
 	libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
@@ -36,9 +40,7 @@ libstore_CXXFLAGS = \
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
-sandbox-headers = $(d)/sandbox-defaults.sb.gen.hh $(d)/sandbox-network.sb.gen.hh $(d)/sandbox-minimal.sb.gen.hh
-
-$(d)/build.cc: $(sandbox-headers)
+$(d)/build.cc:
 
 %.gen.hh: %
 	@echo 'R"foo(' >> $@.tmp
@@ -46,6 +48,6 @@ $(d)/build.cc: $(sandbox-headers)
 	@echo ')foo"' >> $@.tmp
 	@mv $@.tmp $@
 
-clean-files += $(d)/schema.sql.gen.hh $(sandbox-headers)
+clean-files += $(d)/schema.sql.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))

From d8306148e04919b5ecf0e6fb58682a38d99ef944 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Oct 2017 15:02:58 +0200
Subject: [PATCH 0594/2196] Suppress "copying 0 paths" message

---
 src/libstore/store-api.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index fa6ade75002..3bc5e4c57c9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -619,6 +619,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
+    if (missing.empty()) return;
+
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};

From 75cd75b1ae3d32267ec5b92f7e047dc9ea33f075 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Oct 2017 15:41:14 +0200
Subject: [PATCH 0595/2196] Errors writing to the NAR cache should not be fatal

---
 src/libstore/remote-fs-accessor.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 93969120462..ba9620a175b 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -19,9 +19,13 @@ Path RemoteFSAccessor::makeCacheFile(const Path & storePath)
 
 void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar)
 {
-    if (cacheDir != "")
+    try {
+        if (cacheDir == "") return;
         /* FIXME: do this asynchronously. */
         writeFile(makeCacheFile(storePath), nar);
+    } catch (...) {
+        ignoreException();
+    }
 }
 
 std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)

From 908590dc6cfcca3a98755b194d93b2da39aee95c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Oct 2017 15:45:39 +0200
Subject: [PATCH 0596/2196] Let's not populate the NAR cache from
 hydra-queue-runner for now

---
 src/libstore/binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 67607ab3d43..1a565c60b23 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -116,7 +116,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             if (accessor_) {
                 accessor_->nars.emplace(info.path, narAccessor);
-                accessor_->addToCache(info.path, *nar);
+                //accessor_->addToCache(info.path, *nar);
             }
 
             std::function<void(const Path &, JSONPlaceholder &)> recurse;
@@ -164,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     else {
         if (accessor_) {
             accessor_->nars.emplace(info.path, makeNarAccessor(nar));
-            accessor_->addToCache(info.path, *nar);
+            //accessor_->addToCache(info.path, *nar);
         }
     }
 

From ef1d0142a056d4441fb6ba8ecb42da833ebb3ba0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Oct 2017 12:11:07 +0200
Subject: [PATCH 0597/2196] Revert "Let's not populate the NAR cache from
 hydra-queue-runner for now"

This reverts commit 908590dc6cfcca3a98755b194d93b2da39aee95c. Since
hydra-server can have a different store URI from hydra-queue-runner
now, we don't really need this.
---
 src/libstore/binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 1a565c60b23..67607ab3d43 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -116,7 +116,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             if (accessor_) {
                 accessor_->nars.emplace(info.path, narAccessor);
-                //accessor_->addToCache(info.path, *nar);
+                accessor_->addToCache(info.path, *nar);
             }
 
             std::function<void(const Path &, JSONPlaceholder &)> recurse;
@@ -164,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     else {
         if (accessor_) {
             accessor_->nars.emplace(info.path, makeNarAccessor(nar));
-            //accessor_->addToCache(info.path, *nar);
+            accessor_->addToCache(info.path, *nar);
         }
     }
 

From c7af84ce846a9deefa5b4db1b1bce1c091ca2a1e Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 19 Oct 2017 15:02:40 -0400
Subject: [PATCH 0598/2196] nix-shell: Fix file name resolution in shebangs

---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index a3d3c8007be..4541c89a700 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -278,7 +278,7 @@ void mainWrapped(int argc, char * * argv)
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
                 exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state,
-                    inShebang && !packages ? absPath(i, dirOf(script)) : i))));
+                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))));
         }
 
     /* Evaluate them into derivations. */

From 32940702fc4b39f361fbed8fbf92d6250bef7aad Mon Sep 17 00:00:00 2001
From: Keshav Kini <keshav.kini@gmail.com>
Date: Thu, 19 Oct 2017 18:03:26 -0700
Subject: [PATCH 0599/2196] Mention C++14 dependency in the manual.

A couple makefiles in the sources have -std=c++14 in the CFLAGS.
---
 doc/manual/installation/prerequisites-source.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index 7311e4885e7..49660c36e39 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -10,14 +10,14 @@
 
   <listitem><para>GNU Make.</para></listitem>
 
-  <listitem><para>A version of GCC or Clang that supports C++11.</para></listitem>
+  <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem>
 
   <listitem><para><command>pkg-config</command> to locate
   dependencies.  If your distribution does not provide it, you can get
   it from <link
   xlink:href="http://www.freedesktop.org/wiki/Software/pkg-config"
   />.</para></listitem>
-  
+
   <listitem><para>The OpenSSL library to calculate cryptographic hashes.
   If your distribution does not provide it, you can get it from <link
   xlink:href="https://www.openssl.org"/>.</para></listitem>

From 863a45f1f3ac3ac5db6afcf906aa2c485418d9ac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 23 Oct 2017 19:06:55 +0200
Subject: [PATCH 0600/2196] build-remote: Put current load under the store
 state directory

Fixes the error

  error: opening lock file '/nix/var/nix/current-load/main-lock': Permission denied

when using a chroot store.
---
 src/build-remote/build-remote.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 4b7a24d03e9..ea002cdccab 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -16,6 +16,7 @@
 #include "serialise.hh"
 #include "store-api.hh"
 #include "derivations.hh"
+#include "local-store.hh"
 
 using namespace nix;
 using std::cin;
@@ -48,7 +49,7 @@ int main (int argc, char * * argv)
         if (argc != 6)
             throw UsageError("called without required arguments");
 
-        auto store = openStore();
+        auto store = openStore().cast<LocalStore>();
 
         auto localSystem = argv[1];
         settings.maxSilentTime = std::stoll(argv[2]);
@@ -58,7 +59,7 @@ int main (int argc, char * * argv)
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
-        currentLoad = settings.nixStateDir + "/current-load";
+        currentLoad = store->stateDir + "/current-load";
 
         std::shared_ptr<Store> sshStore;
         AutoCloseFD bestSlotLock;

From 38dcd0c4b8789f3ae5c9384e103123ec44f76b71 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 23 Oct 2017 19:08:16 +0200
Subject: [PATCH 0601/2196] Release note updates

---
 doc/manual/release-notes/rl-1.12.xml | 84 ++++++++++++++++------------
 1 file changed, 48 insertions(+), 36 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index e1529696530..adfbaa157f2 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -52,8 +52,7 @@
       information about store paths.</para></listitem>
 
       <listitem><para><command>nix run</command> starts a shell in
-      which the specified packages are available. (TODO: currently
-      broken in chroot mode)</para></listitem>
+      which the specified packages are available.</para></listitem>
 
       <listitem><para><command>nix log</command> shows the build log
       of a package or path. If the build log is not available locally,
@@ -74,6 +73,17 @@
       name, package name or description. It caches available packages
       to speed up searches.</para></listitem>
 
+      <listitem><para><command>nix why-depends</command> (d41c5eb13f4f3a37d80dbc6d3888644170c3b44a).</para></listitem>
+
+      <listitem><para><command>nix show-derivation</command> (e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2).</para></listitem>
+
+      <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem>
+
+      <listitem><para>Progress indicator.</para></listitem>
+
+      <listitem><para>All options are available as flags now
+      (b8283773bd64d7da6859ed520ee19867742a03ba).</para></listitem>
+
     </itemizedlist>
 
   </listitem>
@@ -83,10 +93,6 @@
     integrated into Nix as <command>nix repl</command>.</para>
   </listitem>
 
-  <listitem>
-    <para>Nix now supports floating-point numbers.</para>
-  </listitem>
-
   <listitem>
     <para>New build mode <command>nix-build --hash</command> that
     builds a derivation, computes the hash of the output, and moves
@@ -171,11 +177,6 @@
     <literal>nixos-15.09</literal> channel.</para>
   </listitem>
 
-  <listitem>
-    <para>“Nested” log output was removed. As a result,
-    <command>nix-log2xml</command> was also removed.</para>
-  </listitem>
-
   <listitem>
     <para>When <option>--no-build-output</option> is given, the last
     10 lines of the build log will be shown if a build
@@ -194,10 +195,6 @@
     (d8bf0d4859e28ddd23401fbe89f4e528aa09ddb3)</para>
   </listitem>
 
-  <listitem>
-    <para>Removed OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para>
-  </listitem>
-
   <listitem>
     <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
     content-addressable tarball cache at
@@ -251,21 +248,11 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     add docs.</para>
   </listitem>
 
-  <listitem>
-    <para>Added builtin function
-    <varname>builtins.partition</varname>.</para>
-  </listitem>
-
   <listitem>
     <para>Support for HTTP/2. This makes binary cache lookups much
     more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para>
   </listitem>
 
-  <listitem>
-    <para>Caching of failed builds has been
-    removed. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para>
-  </listitem>
-
   <listitem>
     <para>The <option>build-sandbox-paths</option> configuration
     option can now specify optional paths by appending a
@@ -373,7 +360,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
   </listitem>
 
   <listitem>
-    <para><command>nix-shell</command> now used
+    <para><command>nix-shell</command> now uses
     <varname>bashInteractive</varname> from Nixpkgs, rather than the
     <command>bash</command> command that happens to be in the caller’s
     <envar>PATH</envar>. This is especially important on macOS where
@@ -383,28 +370,53 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
   </listitem>
 
   <listitem>
-    <para><function>builtins.split</function>.
-    (b8867a0239b1930a16f9ef3f7f3e864b01416dff))</para>
+    <para>New builtin functions: <function>builtins.split</function>
+    (b8867a0239b1930a16f9ef3f7f3e864b01416dff),
+    <function>builtins.partition</function>.</para>
   </listitem>
 
   <listitem>
-    <para><filename>nix-mode.el</filename> has been removed from Nix. It is now a separate repository
-    in <uri>https://github.com/NixOS/nix-mode</uri> and can be installed through the MELPA
-    package repository.</para>
+    <para>Automatic garbage collection.</para>
   </listitem>
 
   <listitem>
-    <para>Automatic garbage collection.</para>
+    <para><command>nix-store -q --roots</command> and
+    <command>nix-store --gc --print-roots</command> now show temporary
+    and in-memory roots.</para>
   </listitem>
 
   <listitem>
-    <para><command>nix why-depends</command></para>
+    <para>Builders can now communicate what build phase they are in by
+    writing messages to the file descriptor specified in
+    <envar>NIX_LOG_FD</envar>. (88e6bb76de5564b3217be9688677d1c89101b2a3)
+    </para>
   </listitem>
 
+</itemizedlist>
+
+<para>Some features were removed:</para>
+
+<itemizedlist>
+
   <listitem>
-    <para><command>nix-store -q --roots</command> and
-    <command>nix-store --gc --print-roots</command> now show temporary
-    and in-memory roots.</para>
+    <para>“Nested” log output. As a result,
+    <command>nix-log2xml</command> was also removed.</para>
+  </listitem>
+
+  <listitem>
+    <para>OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para>
+  </listitem>
+
+  <listitem>
+    <para>Caching of failed
+    builds. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para>
+  </listitem>
+
+  <listitem>
+    <para><filename>nix-mode.el</filename> has been removed from
+    Nix. It is now a separate repository in
+    <uri>https://github.com/NixOS/nix-mode</uri> and can be installed
+    through the MELPA package repository.</para>
   </listitem>
 
 </itemizedlist>

From f32cdc4fab1bff919c3d074a95e32a9934eb7386 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 23 Oct 2017 19:34:49 +0200
Subject: [PATCH 0602/2196] Turn $NIX_REMOTE into a configuration option

---
 src/libstore/globals.hh   | 4 ++++
 src/libstore/store-api.hh | 5 +++--
 src/nix/command.cc        | 5 +----
 src/nix/command.hh        | 1 -
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 264e82a16e2..12d57b3e764 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "config.hh"
+#include "util.hh"
 
 #include <map>
 #include <limits>
@@ -84,6 +85,9 @@ public:
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
+    Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
+        "The default Nix store to use."};
+
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5f3d8c7b952..d1e1b5d6f45 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -716,7 +716,7 @@ void removeTempRoots();
    You can pass parameters to the store implementation by appending
    ‘?key=value&key=value&...’ to the URI.
 */
-ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"),
+ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
     const Store::Params & extraParams = Store::Params());
 
 
@@ -727,7 +727,8 @@ enum StoreType {
 };
 
 
-StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir);
+StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
+    const std::string & stateDir = settings.nixStateDir);
 
 /* Return the default substituter stores, defined by the
    ‘substituters’ option and various legacy options like
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 0f6bb294b38..70d642605d8 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -78,9 +78,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
 
 StoreCommand::StoreCommand()
 {
-    storeUri = getEnv("NIX_REMOTE");
-
-    mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri);
 }
 
 ref<Store> StoreCommand::getStore()
@@ -92,7 +89,7 @@ ref<Store> StoreCommand::getStore()
 
 ref<Store> StoreCommand::createStore()
 {
-    return openStore(storeUri);
+    return openStore();
 }
 
 void StoreCommand::run()
diff --git a/src/nix/command.hh b/src/nix/command.hh
index bf897f620db..77ca8cfb64b 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -33,7 +33,6 @@ class Store;
 /* A command that require a Nix store. */
 struct StoreCommand : virtual Command
 {
-    std::string storeUri;
     StoreCommand();
     void run() override;
     ref<Store> getStore();

From 37fbfffd8e23cf9ca4038e6c4145583a811e91aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 23 Oct 2017 20:43:04 +0200
Subject: [PATCH 0603/2196] Pass all settings to build-remote

This ensures that command line flags such as --builders get passed
correctly.
---
 src/build-remote/build-remote.cc | 59 +++++++++++++++++---------------
 src/libstore/build.cc            | 28 +++++++++------
 src/libutil/serialise.hh         | 22 +++++++++++-
 tests/build-hook.hook.sh         | 23 -------------
 tests/build-hook.sh              | 10 ------
 tests/build-remote.sh            | 13 +++----
 tests/local.mk                   |  2 +-
 7 files changed, 75 insertions(+), 82 deletions(-)
 delete mode 100755 tests/build-hook.hook.sh
 delete mode 100644 tests/build-hook.sh

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index ea002cdccab..f023aedb190 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -46,16 +46,23 @@ int main (int argc, char * * argv)
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
 
-        if (argc != 6)
+        if (argc != 2)
             throw UsageError("called without required arguments");
 
-        auto store = openStore().cast<LocalStore>();
+        verbosity = (Verbosity) std::stoll(argv[1]);
+
+        FdSource source(STDIN_FILENO);
+
+        /* Read the parent's settings. */
+        while (readInt(source)) {
+            auto name = readString(source);
+            auto value = readString(source);
+            settings.set(name, value);
+        }
+
+        settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
-        auto localSystem = argv[1];
-        settings.maxSilentTime = std::stoll(argv[2]);
-        settings.buildTimeout = std::stoll(argv[3]);
-        verbosity = (Verbosity) std::stoll(argv[4]);
-        settings.builders = argv[5];
+        auto store = openStore().cast<LocalStore>();
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
            that gets cleared on reboot, but it wouldn't work on macOS. */
@@ -74,18 +81,20 @@ int main (int argc, char * * argv)
 
         string drvPath;
         string storeUri;
-        for (string line; getline(cin, line);) {
-            auto tokens = tokenizeString<std::vector<string>>(line);
-            auto sz = tokens.size();
-            if (sz != 3 && sz != 4)
-                throw Error("invalid build hook line '%1%'", line);
-            auto amWilling = tokens[0] == "1";
-            auto neededSystem = tokens[1];
-            drvPath = tokens[2];
-            auto requiredFeatures = sz == 3 ?
-                std::set<string>{} :
-                tokenizeString<std::set<string>>(tokens[3], ",");
-            auto canBuildLocally = amWilling && (neededSystem == localSystem);
+
+        while (true) {
+
+            try {
+                auto s = readString(source);
+                if (s != "try") return;
+            } catch (EndOfFile &) { return; }
+
+            auto amWilling = readInt(source);
+            auto neededSystem = readString(source);
+            source >> drvPath;
+            auto requiredFeatures = readStrings<std::set<std::string>>(source);
+
+            auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -100,7 +109,7 @@ int main (int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 unsigned long long bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on '%s'", m.storeUri);
+                    debug("considering building on remote machine '%s'", m.storeUri);
 
                     if (m.enabled && std::find(m.systemTypes.begin(),
                             m.systemTypes.end(),
@@ -184,15 +193,9 @@ int main (int argc, char * * argv)
 
 connected:
         std::cerr << "# accept\n";
-        string line;
-        if (!getline(cin, line))
-            throw Error("hook caller didn't send inputs");
-
-        auto inputs = tokenizeString<PathSet>(line);
-        if (!getline(cin, line))
-            throw Error("hook caller didn't send outputs");
 
-        auto outputs = tokenizeString<PathSet>(line);
+        auto inputs = readStrings<PathSet>(source);
+        auto outputs = readStrings<PathSet>(source);
 
         AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 88c51654614..057ad2bdf2b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -606,6 +606,8 @@ struct HookInstance
     /* The process ID of the hook. */
     Pid pid;
 
+    FdSink sink;
+
     HookInstance();
 
     ~HookInstance();
@@ -642,11 +644,7 @@ HookInstance::HookInstance()
 
         Strings args = {
             baseNameOf(settings.buildHook),
-            settings.thisSystem,
-            std::to_string(settings.maxSilentTime),
-            std::to_string(settings.buildTimeout),
             std::to_string(verbosity),
-            settings.builders
         };
 
         execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
@@ -657,6 +655,11 @@ HookInstance::HookInstance()
     pid.setSeparatePG(true);
     fromHook.writeSide = -1;
     toHook.readSide = -1;
+
+    sink = FdSink(toHook.writeSide.get());
+    for (auto & setting : settings.getSettings())
+        sink << 1 << setting.first << setting.second;
+    sink << 0;
 }
 
 
@@ -1633,9 +1636,13 @@ HookReply DerivationGoal::tryBuildHook()
         for (auto & i : features) checkStoreName(i); /* !!! abuse */
 
         /* Send the request to the hook. */
-        writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%")
-                % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
-                % drv->platform % drvPath % concatStringsSep(",", features)).str());
+        worker.hook->sink
+            << "try"
+            << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
+            << drv->platform
+            << drvPath
+            << features;
+        worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
            whether the hook wishes to perform the build. */
@@ -1680,12 +1687,13 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths));
+    hook->sink << inputPaths;
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths));
+    hook->sink << missingPaths;
 
+    hook->sink = FdSink();
     hook->toHook.writeSide = -1;
 
     /* Create the log file and pipe. */
@@ -3986,7 +3994,7 @@ void Worker::run(const Goals & _topGoals)
         else {
             if (awake.empty() && 0 == settings.maxBuildJobs) throw Error(
                 "unable to start any build; either increase '--max-jobs' "
-                "or enable distributed builds");
+                "or enable remote builds");
             assert(!awake.empty());
         }
     }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 70b19394163..2ea5b6354ee 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -92,7 +92,17 @@ struct FdSink : BufferedSink
     FdSink() : fd(-1) { }
     FdSink(int fd) : fd(fd) { }
     FdSink(FdSink&&) = default;
-    FdSink& operator=(FdSink&&) = default;
+
+    FdSink& operator=(FdSink && s)
+    {
+        flush();
+        fd = s.fd;
+        s.fd = -1;
+        warn = s.warn;
+        written = s.written;
+        return *this;
+    }
+
     ~FdSink();
 
     void write(const unsigned char * data, size_t len) override;
@@ -112,6 +122,16 @@ struct FdSource : BufferedSource
 
     FdSource() : fd(-1) { }
     FdSource(int fd) : fd(fd) { }
+    FdSource(FdSource&&) = default;
+
+    FdSource& operator=(FdSource && s)
+    {
+        fd = s.fd;
+        s.fd = -1;
+        read = s.read;
+        return *this;
+    }
+
     size_t readUnbuffered(unsigned char * data, size_t len) override;
     bool good() override;
 private:
diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh
deleted file mode 100755
index c7472eab760..00000000000
--- a/tests/build-hook.hook.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#! /bin/sh
-
-#set -x
-
-while read x y drv rest; do
-
-    echo "HOOK for $drv" >&2
-
-    outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv`
-
-    echo "output path is $outPath" >&2
-
-    if `echo $outPath | grep -q input-1`; then
-        echo "# accept" >&2
-        read inputs
-        read outputs
-        mkdir $outPath
-        echo "BAR" > $outPath/foo
-    else
-        echo "# decline" >&2
-    fi
-
-done
diff --git a/tests/build-hook.sh b/tests/build-hook.sh
deleted file mode 100644
index 2005c7cebdc..00000000000
--- a/tests/build-hook.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-source common.sh
-
-clearStore
-
-outPath=$(nix-build build-hook.nix --no-out-link --option build-hook $(pwd)/build-hook.hook.sh)
-
-echo "output path is $outPath"
-
-text=$(cat "$outPath"/foobar)
-if test "$text" != "BARBAR"; then exit 1; fi
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index e27ce7e25a8..603b57c59ed 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -9,16 +9,11 @@ chmod -R u+w $TEST_ROOT/store0 || true
 chmod -R u+w $TEST_ROOT/store1 || true
 rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 
-# FIXME: --option is not passed to build-remote, so have to create a config file.
-export NIX_CONF_DIR=$TEST_ROOT/etc2
-mkdir -p $NIX_CONF_DIR
-echo "
-sandbox-paths = /nix/store
-sandbox-build-dir = /build-tmp
-" > $NIX_CONF_DIR/nix.conf
+nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
+  --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
+  --builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo"
 
-outPath=$(nix-build build-hook.nix --no-out-link -j0 \
-  --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo")
+outPath=$TEST_ROOT/result
 
 cat $outPath/foobar | grep FOOBAR
 
diff --git a/tests/local.mk b/tests/local.mk
index 7d99a0fc767..19d6f1893d2 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -3,7 +3,7 @@ check:
 
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
-  build-hook.sh gc.sh gc-concurrent.sh \
+  gc.sh gc-concurrent.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh check-refs.sh filter-source.sh \
   remote-store.sh export.sh export-graph.sh \

From d4609bb3af14dc0e19f67caa262a47792c3bd73d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 23 Oct 2017 20:44:55 +0200
Subject: [PATCH 0604/2196] NIX_BUILD_HOOK variable is gone

---
 tests/common.sh.in | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/common.sh.in b/tests/common.sh.in
index a8d56707e27..04d605e3443 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -26,7 +26,6 @@ mkdir -p $TEST_HOME
 export PATH=@bindir@:$PATH
 coreutils=@coreutils@
 
-export NIX_BUILD_HOOK=
 export dot=@dot@
 export xmllint="@xmllint@"
 export SHELL="@bash@"

From af241ae7d3d2a9975d43c9137806a6ffcb96e95b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 10:52:34 +0200
Subject: [PATCH 0605/2196] Remove the builder-files option

You can now include files via the "builders" option, using the syntax
"@<filename>". Having only one option makes it easier to override
builders completely.

For backward compatibility, the default is "@/etc/nix/machines", or
"@<filename>" for each file name in NIX_REMOTE_SYSTEMS.
---
 src/libstore/globals.cc  |  7 ++++++-
 src/libstore/globals.hh  |  6 +-----
 src/libstore/machines.cc | 24 ++++++++++++++----------
 3 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 7da4bce8775..bef2510d949 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -53,7 +53,12 @@ Settings::Settings()
 
     /* Backwards compatibility. */
     auto s = getEnv("NIX_REMOTE_SYSTEMS");
-    if (s != "") builderFiles = tokenizeString<Strings>(s, ":");
+    if (s != "") {
+        Strings ss;
+        for (auto & p : tokenizeString<Strings>(s, ":"))
+            ss.push_back("@" + p);
+        builders = concatStringsSep(" ", ss);
+    }
 
 #if defined(__linux__) && defined(SANDBOX_SHELL)
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 12d57b3e764..4980615dc3f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -138,13 +138,9 @@ public:
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
-    Setting<std::string> builders{this, "", "builders",
+    Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
-    Setting<Strings> builderFiles{this,
-        {nixConfDir + "/machines"}, "builder-files",
-        "A list of files specifying build machines."};
-
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 076c3cab3e9..d2faf45357e 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -47,9 +47,22 @@ bool Machine::mandatoryMet(const std::set<string> & features) const {
 void parseMachines(const std::string & s, Machines & machines)
 {
     for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
-        chomp(line);
+        trim(line);
         line.erase(std::find(line.begin(), line.end(), '#'), line.end());
         if (line.empty()) continue;
+
+        if (line[0] == '@') {
+            auto file = trim(std::string(line, 1));
+            try {
+                parseMachines(readFile(file), machines);
+            } catch (const SysError & e) {
+                if (e.errNo != ENOENT)
+                    throw;
+                debug("cannot find machines file '%s'", file);
+            }
+            continue;
+        }
+
         auto tokens = tokenizeString<std::vector<string>>(line);
         auto sz = tokens.size();
         if (sz < 1)
@@ -74,15 +87,6 @@ Machines getMachines()
 {
     Machines machines;
 
-    for (auto & file : settings.builderFiles.get()) {
-        try {
-            parseMachines(readFile(file), machines);
-        } catch (const SysError & e) {
-            if (e.errNo != ENOENT)
-                throw;
-        }
-    }
-
     parseMachines(settings.builders, machines);
 
     return machines;

From 25f32625e2f2a3a1e1b3a3811da82f21c3a3b880 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 11:00:16 +0200
Subject: [PATCH 0606/2196] Remove the remote-builds option

This is superfluous since you can now just set "builders" to empty,
e.g. "--builders ''".
---
 src/libmain/shared.cc        | 3 ---
 src/libstore/build.cc        | 8 ++++++--
 src/libstore/globals.hh      | 3 ---
 src/libstore/remote-store.cc | 2 +-
 src/nix-daemon/nix-daemon.cc | 2 +-
 5 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 6393d80bbf0..aef5d88bf46 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -188,9 +188,6 @@ struct LegacyArgs : public MixCommonArgs
         mkFlag(0, "readonly-mode", "do not write to the Nix store",
             &settings.readOnlyMode);
 
-        mkFlag(0, "no-build-hook", "disable use of the build hook mechanism",
-            &(bool&) settings.useBuildHook, false);
-
         mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
             &settings.showTrace);
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 057ad2bdf2b..fc800f430c9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -274,6 +274,10 @@ class Worker
     uint64_t expectedNarSize = 0;
     uint64_t doneNarSize = 0;
 
+    /* Whether to ask the build hook if it can build a derivation. If
+       it answers with "decline-permanently", we don't try again. */
+    bool tryBuildHook = true;
+
     Worker(LocalStore & store);
     ~Worker();
 
@@ -1622,7 +1626,7 @@ void DerivationGoal::buildDone()
 
 HookReply DerivationGoal::tryBuildHook()
 {
-    if (!settings.useBuildHook || !useDerivation) return rpDecline;
+    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
 
     if (!worker.hook)
         worker.hook = std::make_unique<HookInstance>();
@@ -1662,7 +1666,7 @@ HookReply DerivationGoal::tryBuildHook()
         if (reply == "decline")
             return rpDecline;
         else if (reply == "decline-permanently") {
-            settings.useBuildHook = false;
+            worker.tryBuildHook = false;
             worker.hook = 0;
             return rpDecline;
         }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 4980615dc3f..88052732279 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -132,9 +132,6 @@ public:
         "The maximum duration in seconds that a builder can run. "
         "0 means infinity.", {"build-timeout"}};
 
-    Setting<bool> useBuildHook{this, true, "remote-builds",
-        "Whether to use build hooks (for distributed builds)."};
-
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b9076c0474d..77b41b6bf8a 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -166,7 +166,7 @@ void RemoteStore::setOptions(Connection & conn)
        << verbosity
        << settings.maxBuildJobs
        << settings.maxSilentTime
-       << settings.useBuildHook
+       << true
        << (settings.verboseBuild ? lvlError : lvlVomit)
        << 0 // obsolete log type
        << 0 /* obsolete print build trace */
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index dbf301a9153..5629cc64b96 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -509,7 +509,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
         verbosity = (Verbosity) readInt(from);
         settings.maxBuildJobs.assign(readInt(from));
         settings.maxSilentTime = readInt(from);
-        settings.useBuildHook = readInt(from) != 0;
+        readInt(from); // obsolete useBuildHook
         settings.verboseBuild = lvlError == (Verbosity) readInt(from);
         readInt(from); // obsolete logType
         readInt(from); // obsolete printBuildTrace

From 0d59f1ca49c9f7f3b2edaadcf590360ec66a6257 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 12:45:11 +0200
Subject: [PATCH 0607/2196] nix: Respect -I, --arg, --argstr

Also, random cleanup to argument handling.
---
 src/libexpr/common-eval-args.cc          |  57 +++++++++++
 src/libexpr/common-eval-args.hh          |  26 +++++
 src/libexpr/common-opts.cc               |  67 -------------
 src/libexpr/common-opts.hh               |  20 ----
 src/libmain/common-args.cc               |  28 +++---
 src/libmain/shared.cc                    | 116 ++++++++++++-----------
 src/libmain/shared.hh                    |  14 +++
 src/libstore/globals.cc                  |   6 +-
 src/libutil/args.cc                      |  18 ++--
 src/libutil/args.hh                      |  68 ++++++-------
 src/libutil/config.cc                    |   6 +-
 src/nix-build/nix-build.cc               |  23 +++--
 src/nix-env/nix-env.cc                   |  22 +++--
 src/nix-instantiate/nix-instantiate.cc   |  22 +++--
 src/nix-prefetch-url/nix-prefetch-url.cc |  21 ++--
 src/nix/command.cc                       |   6 +-
 src/nix/command.hh                       |  11 +--
 src/nix/copy.cc                          |  12 ++-
 src/nix/hash.cc                          |  14 ++-
 src/nix/installables.cc                  |  26 +++--
 src/nix/main.cc                          |  36 ++++---
 src/nix/repl.cc                          |  13 ++-
 src/nix/run.cc                           |  11 ++-
 src/nix/search.cc                        |   4 +-
 src/nix/sigs.cc                          |   9 +-
 src/nix/verify.cc                        |   2 +-
 26 files changed, 354 insertions(+), 304 deletions(-)
 create mode 100644 src/libexpr/common-eval-args.cc
 create mode 100644 src/libexpr/common-eval-args.hh
 delete mode 100644 src/libexpr/common-opts.cc
 delete mode 100644 src/libexpr/common-opts.hh

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
new file mode 100644
index 00000000000..3e0c78f280f
--- /dev/null
+++ b/src/libexpr/common-eval-args.cc
@@ -0,0 +1,57 @@
+#include "common-eval-args.hh"
+#include "shared.hh"
+#include "download.hh"
+#include "util.hh"
+#include "eval.hh"
+
+namespace nix {
+
+MixEvalArgs::MixEvalArgs()
+{
+    mkFlag()
+        .longName("arg")
+        .description("argument to be passed to Nix functions")
+        .labels({"name", "expr"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
+
+    mkFlag()
+        .longName("argstr")
+        .description("string-valued argument to be passed to Nix functions")
+        .labels({"name", "string"})
+        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
+
+    mkFlag()
+        .shortName('I')
+        .longName("include")
+        .description("add a path to the list of locations used to look up <...> file names")
+        .label("path")
+        .handler([&](std::string s) { searchPath.push_back(s); });
+}
+
+Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
+{
+    Bindings * res = state.allocBindings(autoArgs.size());
+    for (auto & i : autoArgs) {
+        Value * v = state.allocValue();
+        if (i.second[0] == 'E')
+            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+        else
+            mkString(*v, string(i.second, 1));
+        res->push_back(Attr(state.symbols.create(i.first), v));
+    }
+    res->sort();
+    return res;
+}
+
+Path lookupFileArg(EvalState & state, string s)
+{
+    if (isUri(s))
+        return getDownloader()->downloadCached(state.store, s, true);
+    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+        Path p = s.substr(1, s.size() - 2);
+        return state.findFile(p);
+    } else
+        return absPath(s);
+}
+
+}
diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh
new file mode 100644
index 00000000000..09fa406b2cd
--- /dev/null
+++ b/src/libexpr/common-eval-args.hh
@@ -0,0 +1,26 @@
+#pragma once
+
+#include "args.hh"
+
+namespace nix {
+
+class Store;
+class EvalState;
+struct Bindings;
+
+struct MixEvalArgs : virtual Args
+{
+    MixEvalArgs();
+
+    Bindings * getAutoArgs(EvalState & state);
+
+    Strings searchPath;
+
+private:
+
+    std::map<std::string, std::string> autoArgs;
+};
+
+Path lookupFileArg(EvalState & state, string s);
+
+}
diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc
deleted file mode 100644
index 6b31961d345..00000000000
--- a/src/libexpr/common-opts.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-#include "common-opts.hh"
-#include "shared.hh"
-#include "download.hh"
-#include "util.hh"
-
-
-namespace nix {
-
-
-bool parseAutoArgs(Strings::iterator & i,
-    const Strings::iterator & argsEnd, std::map<string, string> & res)
-{
-    string arg = *i;
-    if (arg != "--arg" && arg != "--argstr") return false;
-
-    UsageError error(format("'%1%' requires two arguments") % arg);
-
-    if (++i == argsEnd) throw error;
-    string name = *i;
-    if (++i == argsEnd) throw error;
-    string value = *i;
-
-    res[name] = (arg == "--arg" ? 'E' : 'S') + value;
-
-    return true;
-}
-
-
-Bindings * evalAutoArgs(EvalState & state, std::map<string, string> & in)
-{
-    Bindings * res = state.allocBindings(in.size());
-    for (auto & i : in) {
-        Value * v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
-        else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
-    }
-    res->sort();
-    return res;
-}
-
-
-bool parseSearchPathArg(Strings::iterator & i,
-    const Strings::iterator & argsEnd, Strings & searchPath)
-{
-    if (*i != "-I") return false;
-    if (++i == argsEnd) throw UsageError("'-I' requires an argument");
-    searchPath.push_back(*i);
-    return true;
-}
-
-
-Path lookupFileArg(EvalState & state, string s)
-{
-    if (isUri(s))
-        return getDownloader()->downloadCached(state.store, s, true);
-    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
-        Path p = s.substr(1, s.size() - 2);
-        return state.findFile(p);
-    } else
-        return absPath(s);
-}
-
-
-}
diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh
deleted file mode 100644
index cb2732d6fe7..00000000000
--- a/src/libexpr/common-opts.hh
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include "eval.hh"
-
-namespace nix {
-
-class Store;
-
-/* Some common option parsing between nix-env and nix-instantiate. */
-bool parseAutoArgs(Strings::iterator & i,
-    const Strings::iterator & argsEnd, std::map<string, string> & res);
-
-Bindings * evalAutoArgs(EvalState & state, std::map<string, string> & in);
-
-bool parseSearchPathArg(Strings::iterator & i,
-    const Strings::iterator & argsEnd, Strings & searchPath);
-
-Path lookupFileArg(EvalState & state, string s);
-
-}
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 3fa42c2aafa..ea27aaa35e0 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -6,28 +6,30 @@ namespace nix {
 MixCommonArgs::MixCommonArgs(const string & programName)
     : programName(programName)
 {
-    mkFlag('v', "verbose", "increase verbosity level", []() {
-        verbosity = (Verbosity) (verbosity + 1);
-    });
+    mkFlag()
+        .longName("verbose")
+        .shortName('v')
+        .description("increase verbosity level")
+        .handler([]() { verbosity = (Verbosity) (verbosity + 1); });
 
-    mkFlag(0, "quiet", "decrease verbosity level", []() {
-        verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError;
-    });
+    mkFlag()
+        .longName("quiet")
+        .description("decrease verbosity level")
+        .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
 
-    mkFlag(0, "debug", "enable debug output", []() {
-        verbosity = lvlDebug;
-    });
+    mkFlag()
+        .longName("debug")
+        .description("enable debug output")
+        .handler([]() { verbosity = lvlDebug; });
 
     mkFlag()
         .longName("option")
         .labels({"name", "value"})
         .description("set a Nix configuration option (overriding nix.conf)")
         .arity(2)
-        .handler([](Strings ss) {
-            auto name = ss.front(); ss.pop_front();
-            auto value = ss.front();
+        .handler([](std::vector<std::string> ss) {
             try {
-                settings.set(name, value);
+                settings.set(ss[0], ss[1]);
             } catch (UsageError & e) {
                 warn(e.what());
             }
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index aef5d88bf46..9123a3d1ae0 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -1,4 +1,3 @@
-#include "common-args.hh"
 #include "globals.hh"
 #include "shared.hh"
 #include "store-api.hh"
@@ -149,71 +148,78 @@ void initNix()
 }
 
 
-struct LegacyArgs : public MixCommonArgs
+LegacyArgs::LegacyArgs(const std::string & programName,
+    std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
+    : MixCommonArgs(programName), parseArg(parseArg)
 {
-    std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;
-
-    LegacyArgs(const std::string & programName,
-        std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
-        : MixCommonArgs(programName), parseArg(parseArg)
-    {
-        mkFlag('Q', "no-build-output", "do not show build output",
-            &settings.verboseBuild, false);
-
-        mkFlag('K', "keep-failed", "keep temporary directories of failed builds",
-            &(bool&) settings.keepFailed);
-
-        mkFlag('k', "keep-going", "keep going after a build fails",
-            &(bool&) settings.keepGoing);
+    mkFlag()
+        .longName("no-build-output")
+        .shortName('Q')
+        .description("do not show build output")
+        .set(&settings.verboseBuild, false);
+
+    mkFlag()
+        .longName("keep-failed")
+        .shortName('K')
+        .description("keep temporary directories of failed builds")
+        .set(&(bool&) settings.keepFailed, true);
+
+    mkFlag()
+        .longName("keep-going")
+        .shortName('k')
+        .description("keep going after a build fails")
+        .set(&(bool&) settings.keepGoing, true);
+
+    mkFlag()
+        .longName("fallback")
+        .description("build from source if substitution fails")
+        .set(&(bool&) settings.tryFallback, true);
+
+    mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
+        settings.set("max-jobs", s);
+    });
 
-        mkFlag(0, "fallback", "build from source if substitution fails", []() {
-            settings.tryFallback = true;
+    auto intSettingAlias = [&](char shortName, const std::string & longName,
+        const std::string & description, const std::string & dest) {
+        mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
+            settings.set(dest, std::to_string(n));
         });
+    };
 
-        mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
-            settings.set("max-jobs", s);
-        });
+    intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
+    intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
+    intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
 
-        auto intSettingAlias = [&](char shortName, const std::string & longName,
-            const std::string & description, const std::string & dest) {
-            mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
-                settings.set(dest, std::to_string(n));
-            });
-        };
+    mkFlag(0, "readonly-mode", "do not write to the Nix store",
+        &settings.readOnlyMode);
 
-        intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
-        intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
-        intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
+    mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
+        &settings.showTrace);
 
-        mkFlag(0, "readonly-mode", "do not write to the Nix store",
-            &settings.readOnlyMode);
+    mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
+        &gcWarning, false);
+}
 
-        mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
-            &settings.showTrace);
 
-        mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
-            &gcWarning, false);
-    }
+bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end)
+{
+    if (MixCommonArgs::processFlag(pos, end)) return true;
+    bool res = parseArg(pos, end);
+    if (res) ++pos;
+    return res;
+}
 
-    bool processFlag(Strings::iterator & pos, Strings::iterator end) override
-    {
-        if (MixCommonArgs::processFlag(pos, end)) return true;
-        bool res = parseArg(pos, end);
-        if (res) ++pos;
-        return res;
-    }
 
-    bool processArgs(const Strings & args, bool finish) override
-    {
-        if (args.empty()) return true;
-        assert(args.size() == 1);
-        Strings ss(args);
-        auto pos = ss.begin();
-        if (!parseArg(pos, ss.end()))
-            throw UsageError(format("unexpected argument '%1%'") % args.front());
-        return true;
-    }
-};
+bool LegacyArgs::processArgs(const Strings & args, bool finish)
+{
+    if (args.empty()) return true;
+    assert(args.size() == 1);
+    Strings ss(args);
+    auto pos = ss.begin();
+    if (!parseArg(pos, ss.end()))
+        throw UsageError(format("unexpected argument '%1%'") % args.front());
+    return true;
+}
 
 
 void parseCmdLine(int argc, char * * argv,
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 2a1e42dd977..9219dbed832 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -2,6 +2,7 @@
 
 #include "util.hh"
 #include "args.hh"
+#include "common-args.hh"
 
 #include <signal.h>
 
@@ -69,6 +70,19 @@ template<class N> N getIntArg(const string & opt,
 }
 
 
+struct LegacyArgs : public MixCommonArgs
+{
+    std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;
+
+    LegacyArgs(const std::string & programName,
+        std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg);
+
+    bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
+
+    bool processArgs(const Strings & args, bool finish) override;
+};
+
+
 /* Show the manual page for the specified program. */
 void showManPage(const string & name);
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index bef2510d949..4fa02f92085 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -116,17 +116,17 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
     args.mkFlag()
         .longName(name)
         .description("Enable sandboxing.")
-        .handler([=](Strings ss) { value = smEnabled; })
+        .handler([=](std::vector<std::string> ss) { value = smEnabled; })
         .category(category);
     args.mkFlag()
         .longName("no-" + name)
         .description("Disable sandboxing.")
-        .handler([=](Strings ss) { value = smDisabled; })
+        .handler([=](std::vector<std::string> ss) { value = smDisabled; })
         .category(category);
     args.mkFlag()
         .longName("relaxed-" + name)
         .description("Enable sandboxing, but allow builds to disable it.")
-        .handler([=](Strings ss) { value = smRelaxed; })
+        .handler([=](std::vector<std::string> ss) { value = smRelaxed; })
         .category(category);
 }
 
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index d17a1e7a9ab..7af2a1bf731 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -100,7 +100,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
-        Strings args;
+        std::vector<std::string> args;
         for (size_t n = 0 ; n < flag.arity; ++n) {
             if (pos == end) {
                 if (flag.arity == ArityAny) break;
@@ -109,7 +109,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
             }
             args.push_back(*pos++);
         }
-        flag.handler(args);
+        flag.handler(std::move(args));
         return true;
     };
 
@@ -144,7 +144,9 @@ bool Args::processArgs(const Strings & args, bool finish)
     if ((exp.arity == 0 && finish) ||
         (exp.arity > 0 && args.size() == exp.arity))
     {
-        exp.handler(args);
+        std::vector<std::string> ss;
+        for (auto & s : args) ss.push_back(s);
+        exp.handler(std::move(ss));
         expectedArgs.pop_front();
         res = true;
     }
@@ -155,13 +157,17 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
-void Args::mkHashTypeFlag(const std::string & name, HashType * ht)
+Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
 {
-    mkFlag1(0, name, "TYPE", "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", [=](std::string s) {
+    arity(1);
+    label("type");
+    description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
+    handler([ht](std::string s) {
         *ht = parseHashType(s);
         if (*ht == htUnknown)
-            throw UsageError(format("unknown hash type '%1%'") % s);
+            throw UsageError("unknown hash type '%1%'", s);
     });
+    return *this;
 }
 
 Strings argvToStrings(int argc, char * * argv)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 37e31825ab3..ad5fcca3941 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -37,7 +37,7 @@ protected:
         std::string description;
         Strings labels;
         size_t arity = 0;
-        std::function<void(Strings)> handler;
+        std::function<void(std::vector<std::string>)> handler;
         std::string category;
     };
 
@@ -54,7 +54,7 @@ protected:
         std::string label;
         size_t arity; // 0 = any
         bool optional;
-        std::function<void(Strings)> handler;
+        std::function<void(std::vector<std::string>)> handler;
     };
 
     std::list<ExpectedArg> expectedArgs;
@@ -76,24 +76,35 @@ public:
         FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
         FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
         FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
-        FlagMaker & labels(const Strings & ls) { flag->labels = ls; return *this; };
+        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
+        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
         FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
-        FlagMaker & handler(std::function<void(Strings)> handler) { flag->handler = handler; return *this; };
+        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
+        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
+        FlagMaker & handler(std::function<void(std::string)> handler) {
+            flag->arity = 1;
+            flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
+            return *this;
+        };
         FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
 
         template<class T>
-        FlagMaker & dest(T * dest) {
+        FlagMaker & dest(T * dest)
+        {
             flag->arity = 1;
-            flag->handler = [=](Strings ss) { *dest = ss.front(); };
+            flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
             return *this;
         };
 
         template<class T>
-        FlagMaker & set(T * dest, const T & val) {
+        FlagMaker & set(T * dest, const T & val)
+        {
             flag->arity = 0;
-            flag->handler = [=](Strings ss) { *dest = val; };
+            flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
             return *this;
         };
+
+        FlagMaker & mkHashTypeFlag(HashType * ht);
     };
 
     FlagMaker mkFlag();
@@ -101,16 +112,6 @@ public:
     /* Helper functions for constructing flags / positional
        arguments. */
 
-    void mkFlag(char shortName, const std::string & longName,
-        const std::string & description, std::function<void()> fun)
-    {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .description(description)
-            .handler(std::bind(fun));
-    }
-
     void mkFlag1(char shortName, const std::string & longName,
         const std::string & label, const std::string & description,
         std::function<void(std::string)> fun)
@@ -121,7 +122,7 @@ public:
             .labels({label})
             .description(description)
             .arity(1)
-            .handler([=](Strings ss) { fun(ss.front()); });
+            .handler([=](std::vector<std::string> ss) { fun(ss[0]); });
     }
 
     void mkFlag(char shortName, const std::string & name,
@@ -130,17 +131,6 @@ public:
         mkFlag(shortName, name, description, dest, true);
     }
 
-    void mkFlag(char shortName, const std::string & longName,
-        const std::string & label, const std::string & description,
-        string * dest)
-    {
-        mkFlag1(shortName, longName, label, description, [=](std::string s) {
-            *dest = s;
-        });
-    }
-
-    void mkHashTypeFlag(const std::string & name, HashType * ht);
-
     template<class T>
     void mkFlag(char shortName, const std::string & longName, const std::string & description,
         T * dest, const T & value)
@@ -149,7 +139,7 @@ public:
             .shortName(shortName)
             .longName(longName)
             .description(description)
-            .handler([=](Strings ss) { *dest = value; });
+            .handler([=](std::vector<std::string> ss) { *dest = value; });
     }
 
     template<class I>
@@ -171,10 +161,10 @@ public:
             .labels({"N"})
             .description(description)
             .arity(1)
-            .handler([=](Strings ss) {
+            .handler([=](std::vector<std::string> ss) {
                 I n;
-                if (!string2Int(ss.front(), n))
-                    throw UsageError(format("flag '--%1%' requires a integer argument") % longName);
+                if (!string2Int(ss[0], n))
+                    throw UsageError("flag '--%s' requires a integer argument", longName);
                 fun(n);
             });
     }
@@ -182,16 +172,16 @@ public:
     /* Expect a string argument. */
     void expectArg(const std::string & label, string * dest, bool optional = false)
     {
-        expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](Strings ss) {
-            *dest = ss.front();
+        expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
+            *dest = ss[0];
         }});
     }
 
     /* Expect 0 or more arguments. */
-    void expectArgs(const std::string & label, Strings * dest)
+    void expectArgs(const std::string & label, std::vector<std::string> * dest)
     {
-        expectedArgs.push_back(ExpectedArg{label, 0, false, [=](Strings ss) {
-            *dest = ss;
+        expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
+            *dest = std::move(ss);
         }});
     }
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 27157a83178..14c4cca031b 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -152,7 +152,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .longName(name)
         .description(description)
         .arity(1)
-        .handler([=](Strings ss) { set(*ss.begin()); })
+        .handler([=](std::vector<std::string> ss) { set(ss[0]); })
         .category(category);
 }
 
@@ -201,12 +201,12 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
     args.mkFlag()
         .longName(name)
         .description(description)
-        .handler([=](Strings ss) { value = true; })
+        .handler([=](std::vector<std::string> ss) { value = true; })
         .category(category);
     args.mkFlag()
         .longName("no-" + name)
         .description(description)
-        .handler([=](Strings ss) { value = false; })
+        .handler([=](std::vector<std::string> ss) { value = false; })
         .category(category);
 }
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 4541c89a700..85137206fa7 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -14,7 +14,7 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "get-drvs.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 #include "attr-path.hh"
 
 using namespace nix;
@@ -80,8 +80,6 @@ void mainWrapped(int argc, char * * argv)
     auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO);
     Strings attrPaths;
     Strings left;
-    Strings searchPath;
-    std::map<string, string> autoArgs_;
     RepairFlag repair = NoRepair;
     Path gcRoot;
     BuildMode buildMode = bmNormal;
@@ -129,7 +127,12 @@ void mainWrapped(int argc, char * * argv)
         } catch (SysError &) { }
     }
 
-    parseCmdLine(myName, args, [&](Strings::iterator & arg, const Strings::iterator & end) {
+    struct MyArgs : LegacyArgs, MixEvalArgs
+    {
+        using LegacyArgs::LegacyArgs;
+    };
+
+    MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) {
         if (*arg == "--help") {
             deletePath(tmpDir);
             showManPage(myName);
@@ -153,12 +156,6 @@ void mainWrapped(int argc, char * * argv)
         else if (*arg == "--out-link" || *arg == "-o")
             outLink = getArg(*arg, arg, end);
 
-        else if (parseAutoArgs(arg, end, autoArgs_))
-            ;
-
-        else if (parseSearchPathArg(arg, end, searchPath))
-            ;
-
         else if (*arg == "--add-root")
             gcRoot = getArg(*arg, arg, end);
 
@@ -237,15 +234,17 @@ void mainWrapped(int argc, char * * argv)
         return true;
     });
 
+    myArgs.parseCmdline(args);
+
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
     auto store = openStore();
 
-    EvalState state(searchPath, store);
+    EvalState state(myArgs.searchPath, store);
     state.repair = repair;
 
-    Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+    Bindings & autoArgs = *myArgs.getAutoArgs(state);
 
     if (packages) {
         std::ostringstream joined;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 94fbc09f664..016caf6d234 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1,5 +1,5 @@
 #include "attr-path.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 #include "derivations.hh"
 #include "eval.hh"
 #include "get-drvs.hh"
@@ -1309,8 +1309,7 @@ int main(int argc, char * * argv)
         initNix();
         initGC();
 
-        Strings opFlags, opArgs, searchPath;
-        std::map<string, string> autoArgs_;
+        Strings opFlags, opArgs;
         Operation op = 0;
         RepairFlag repair = NoRepair;
         string file;
@@ -1326,7 +1325,12 @@ int main(int argc, char * * argv)
         globals.removeAll = false;
         globals.prebuiltOnly = false;
 
-        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+        struct MyArgs : LegacyArgs, MixEvalArgs
+        {
+            using LegacyArgs::LegacyArgs;
+        };
+
+        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
             Operation oldOp = op;
 
             if (*arg == "--help")
@@ -1335,10 +1339,6 @@ int main(int argc, char * * argv)
                 op = opVersion;
             else if (*arg == "--install" || *arg == "-i")
                 op = opInstall;
-            else if (parseAutoArgs(arg, end, autoArgs_))
-                ;
-            else if (parseSearchPathArg(arg, end, searchPath))
-                ;
             else if (*arg == "--force-name") // undocumented flag for nix-install-package
                 globals.forceName = getArg(*arg, arg, end);
             else if (*arg == "--uninstall" || *arg == "-e")
@@ -1391,17 +1391,19 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        myArgs.parseCmdline(argvToStrings(argc, argv));
+
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();
 
-        globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath, store));
+        globals.state = std::shared_ptr<EvalState>(new EvalState(myArgs.searchPath, store));
         globals.state->repair = repair;
 
         if (file != "")
             globals.instSource.nixExprPath = lookupFileArg(*globals.state, file);
 
-        globals.instSource.autoArgs = evalAutoArgs(*globals.state, autoArgs_);
+        globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state);
 
         if (globals.profile == "")
             globals.profile = getEnv("NIX_PROFILE", "");
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 2498df0f0d7..55ac007e868 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -8,7 +8,7 @@
 #include "value-to-json.hh"
 #include "util.hh"
 #include "store-api.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 
 #include <map>
 #include <iostream>
@@ -89,7 +89,7 @@ int main(int argc, char * * argv)
         initNix();
         initGC();
 
-        Strings files, searchPath;
+        Strings files;
         bool readStdin = false;
         bool fromArgs = false;
         bool findFile = false;
@@ -100,10 +100,14 @@ int main(int argc, char * * argv)
         bool strict = false;
         Strings attrPaths;
         bool wantsReadWrite = false;
-        std::map<string, string> autoArgs_;
         RepairFlag repair = NoRepair;
 
-        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+        struct MyArgs : LegacyArgs, MixEvalArgs
+        {
+            using LegacyArgs::LegacyArgs;
+        };
+
+        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
                 showManPage("nix-instantiate");
             else if (*arg == "--version")
@@ -122,10 +126,6 @@ int main(int argc, char * * argv)
                 findFile = true;
             else if (*arg == "--attr" || *arg == "-A")
                 attrPaths.push_back(getArg(*arg, arg, end));
-            else if (parseAutoArgs(arg, end, autoArgs_))
-                ;
-            else if (parseSearchPathArg(arg, end, searchPath))
-                ;
             else if (*arg == "--add-root")
                 gcRoot = getArg(*arg, arg, end);
             else if (*arg == "--indirect")
@@ -149,15 +149,17 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        myArgs.parseCmdline(argvToStrings(argc, argv));
+
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 
         auto store = openStore();
 
-        EvalState state(searchPath, store);
+        EvalState state(myArgs.searchPath, store);
         state.repair = repair;
 
-        Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+        Bindings & autoArgs = *myArgs.getAutoArgs(state);
 
         if (attrPaths.empty()) attrPaths = {""};
 
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 7e62a033b45..fef3eaa4553 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -4,7 +4,7 @@
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 #include "attr-path.hh"
 
 #include <iostream>
@@ -48,15 +48,18 @@ int main(int argc, char * * argv)
 
         HashType ht = htSHA256;
         std::vector<string> args;
-        Strings searchPath;
         bool printPath = getEnv("PRINT_PATH") != "";
         bool fromExpr = false;
         string attrPath;
-        std::map<string, string> autoArgs_;
         bool unpack = false;
         string name;
 
-        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+        struct MyArgs : LegacyArgs, MixEvalArgs
+        {
+            using LegacyArgs::LegacyArgs;
+        };
+
+        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
                 showManPage("nix-prefetch-url");
             else if (*arg == "--version")
@@ -77,10 +80,6 @@ int main(int argc, char * * argv)
                 unpack = true;
             else if (*arg == "--name")
                 name = getArg(*arg, arg, end);
-            else if (parseAutoArgs(arg, end, autoArgs_))
-                ;
-            else if (parseSearchPathArg(arg, end, searchPath))
-                ;
             else if (*arg != "" && arg->at(0) == '-')
                 return false;
             else
@@ -88,13 +87,15 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        myArgs.parseCmdline(argvToStrings(argc, argv));
+
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
         auto store = openStore();
-        EvalState state(searchPath, store);
+        EvalState state(myArgs.searchPath, store);
 
-        Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+        Bindings & autoArgs = *myArgs.getAutoArgs(state);
 
         /* If -A is given, get the URI from the specified Nix
            expression. */
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 70d642605d8..1e6f0d2bb75 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -24,11 +24,11 @@ void Command::printHelp(const string & programName, std::ostream & out)
 MultiCommand::MultiCommand(const Commands & _commands)
     : commands(_commands)
 {
-    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](Strings ss) {
+    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
         assert(!command);
-        auto i = commands.find(ss.front());
+        auto i = commands.find(ss[0]);
         if (i == commands.end())
-            throw UsageError(format("'%1%' is not a recognised command") % ss.front());
+            throw UsageError("'%s' is not a recognised command", ss[0]);
         command = i->second;
     }});
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 77ca8cfb64b..daa3b3fa703 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -1,10 +1,12 @@
 #pragma once
 
 #include "args.hh"
+#include "common-eval-args.hh"
 
 namespace nix {
 
 struct Value;
+struct Bindings;
 class EvalState;
 
 /* A command is an argument parser that can be executed by calling its
@@ -68,14 +70,11 @@ struct Installable
     }
 };
 
-struct SourceExprCommand : virtual Args, StoreCommand
+struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs
 {
     Path file;
 
-    SourceExprCommand()
-    {
-        mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file);
-    }
+    SourceExprCommand();
 
     /* Return a value representing the Nix expression from which we
        are installing. This is either the file specified by ‘--file’,
@@ -111,7 +110,7 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
 
 private:
 
-    Strings _installables;
+    std::vector<std::string> _installables;
 };
 
 struct InstallableCommand : virtual Args, SourceExprCommand
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 071ac3890aa..2ddea9e70a6 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -19,8 +19,16 @@ struct CmdCopy : StorePathsCommand
     CmdCopy()
         : StorePathsCommand(true)
     {
-        mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri);
-        mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri);
+        mkFlag()
+            .longName("from")
+            .labels({"store-uri"})
+            .description("URI of the source Nix store")
+            .dest(&srcUri);
+        mkFlag()
+            .longName("to")
+            .labels({"store-uri"})
+            .description("URI of the destination Nix store")
+            .dest(&dstUri);
 
         mkFlag()
             .longName("no-check-sigs")
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 923dabb103d..64062fb9795 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -12,14 +12,16 @@ struct CmdHash : Command
     Base base = Base16;
     bool truncate = false;
     HashType ht = htSHA512;
-    Strings paths;
+    std::vector<std::string> paths;
 
     CmdHash(Mode mode) : mode(mode)
     {
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base16);
-        mkHashTypeFlag("type", &ht);
+        mkFlag()
+            .longName("type")
+            .mkHashTypeFlag(&ht);
         expectArgs("paths", &paths);
     }
 
@@ -53,11 +55,13 @@ struct CmdToBase : Command
 {
     Base base;
     HashType ht = htSHA512;
-    Strings args;
+    std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
     {
-        mkHashTypeFlag("type", &ht);
+        mkFlag()
+            .longName("type")
+            .mkHashTypeFlag(&ht);
         expectArgs("strings", &args);
     }
 
@@ -95,7 +99,7 @@ static int compatNixHash(int argc, char * * argv)
     bool base32 = false;
     bool truncate = false;
     enum { opHash, opTo32, opTo16 } op = opHash;
-    Strings ss;
+    std::vector<std::string> ss;
 
     parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
         if (*arg == "--help")
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index c83d6316d3f..ae93c4ef649 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -1,6 +1,6 @@
 #include "command.hh"
 #include "attr-path.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 #include "derivations.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
@@ -12,6 +12,16 @@
 
 namespace nix {
 
+SourceExprCommand::SourceExprCommand()
+{
+    mkFlag()
+        .shortName('f')
+        .longName("file")
+        .label("file")
+        .description("evaluate FILE rather than the default")
+        .dest(&file);
+}
+
 Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
     if (vSourceExpr) return vSourceExpr;
@@ -66,7 +76,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 ref<EvalState> SourceExprCommand::getEvalState()
 {
     if (!evalState)
-        evalState = std::make_shared<EvalState>(Strings{}, getStore());
+        evalState = std::make_shared<EvalState>(searchPath, getStore());
     return ref<EvalState>(evalState);
 }
 
@@ -120,9 +130,7 @@ struct InstallableValue : Installable
 
         auto v = toValue(*state);
 
-        // FIXME
-        std::map<string, string> autoArgs_;
-        Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_));
+        Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
         DrvInfos drvs;
         getDerivations(*state, *v, "", autoArgs, drvs, false);
@@ -187,9 +195,7 @@ struct InstallableAttrPath : InstallableValue
     {
         auto source = cmd.getSourceExpr(state);
 
-        // FIXME
-        std::map<string, string> autoArgs_;
-        Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
+        Bindings & autoArgs = *cmd.getAutoArgs(state);
 
         Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source);
         state.forceValue(*v);
@@ -203,14 +209,14 @@ std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
 static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
 
 static std::vector<std::shared_ptr<Installable>> parseInstallables(
-    SourceExprCommand & cmd, ref<Store> store, Strings ss, bool useDefaultInstallables)
+    SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables)
 {
     std::vector<std::shared_ptr<Installable>> result;
 
     if (ss.empty() && useDefaultInstallables) {
         if (cmd.file == "")
             cmd.file = ".";
-        ss = Strings{""};
+        ss = {""};
     }
 
     for (auto & s : ss) {
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ec9b58b20fe..060402cd08d 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -20,19 +20,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
-        mkFlag('h', "help", "show usage information", [&]() { showHelpAndExit(); });
-
-        mkFlag(0, "help-config", "show configuration options", [=]() {
-            std::cout << "The following configuration options are available:\n\n";
-            Table2 tbl;
-            for (const auto & s : settings._getSettings())
-                if (!s.second.isAlias)
-                    tbl.emplace_back(s.first, s.second.setting->description);
-            printTable(std::cout, tbl);
-            throw Exit();
-        });
-
-        mkFlag(0, "version", "show version information", std::bind(printVersion, programName));
+        mkFlag()
+            .longName("help")
+            .shortName('h')
+            .description("show usage information")
+            .handler([&]() { showHelpAndExit(); });
+
+        mkFlag()
+            .longName("help-config")
+            .description("show configuration options")
+            .handler([&]() {
+                std::cout << "The following configuration options are available:\n\n";
+                Table2 tbl;
+                for (const auto & s : settings._getSettings())
+                    if (!s.second.isAlias)
+                        tbl.emplace_back(s.first, s.second.setting->description);
+                printTable(std::cout, tbl);
+                throw Exit();
+            });
+
+        mkFlag()
+            .longName("version")
+            .description("show version information")
+            .handler([&]() { printVersion(programName); });
 
         std::string cat = "config";
         settings.convertToArgs(*this, cat);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 781b4463e54..28a8ebc8c49 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -7,7 +7,7 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "store-api.hh"
-#include "common-opts.hh"
+#include "common-eval-args.hh"
 #include "get-drvs.hh"
 #include "derivations.hh"
 #include "affinity.hh"
@@ -44,7 +44,7 @@ struct NixRepl
 
     NixRepl(const Strings & searchPath, nix::ref<Store> store);
     ~NixRepl();
-    void mainLoop(const Strings & files);
+    void mainLoop(const std::vector<std::string> & files);
     StringSet completePrefix(string prefix);
     bool getLine(string & input, const std::string &prompt);
     Path getDerivationPath(Value & v);
@@ -131,7 +131,7 @@ static void completionCallback(const char * s, linenoiseCompletions *lc)
 }
 
 
-void NixRepl::mainLoop(const Strings & files)
+void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl;
@@ -664,9 +664,9 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     return str;
 }
 
-struct CmdRepl : StoreCommand
+struct CmdRepl : StoreCommand, MixEvalArgs
 {
-    Strings files;
+    std::vector<std::string> files;
 
     CmdRepl()
     {
@@ -682,8 +682,7 @@ struct CmdRepl : StoreCommand
 
     void run(ref<Store> store) override
     {
-        // FIXME: pass searchPath
-        NixRepl repl({}, openStore());
+        NixRepl repl(searchPath, openStore());
         repl.mainLoop(files);
     }
 };
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 2f93ca35150..6657a86314b 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -20,7 +20,7 @@ extern char * * environ;
 
 struct CmdRun : InstallablesCommand
 {
-    Strings command = { "bash" };
+    std::vector<std::string> command = { "bash" };
     StringSet keep, unset;
     bool ignoreEnvironment = false;
 
@@ -32,7 +32,7 @@ struct CmdRun : InstallablesCommand
             .description("command and arguments to be executed; defaults to 'bash'")
             .arity(ArityAny)
             .labels({"command", "args"})
-            .handler([&](Strings ss) {
+            .handler([&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
             });
@@ -49,7 +49,7 @@ struct CmdRun : InstallablesCommand
             .description("keep specified environment variable")
             .arity(1)
             .labels({"name"})
-            .handler([&](Strings ss) { keep.insert(ss.front()); });
+            .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
 
         mkFlag()
             .longName("unset")
@@ -57,7 +57,7 @@ struct CmdRun : InstallablesCommand
             .description("unset specified environment variable")
             .arity(1)
             .labels({"name"})
-            .handler([&](Strings ss) { unset.insert(ss.front()); });
+            .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
     }
 
     std::string name() override
@@ -126,7 +126,8 @@ struct CmdRun : InstallablesCommand
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
         std::string cmd = *command.begin();
-        Strings args = command;
+        Strings args;
+        for (auto & arg : command) args.push_back(arg);
 
         stopProgressBar();
 
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 9476b79fbc1..f458367dcb5 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -38,12 +38,12 @@ struct CmdSearch : SourceExprCommand, MixJSON
             .longName("update-cache")
             .shortName('u')
             .description("update the package search cache")
-            .handler([&](Strings ss) { writeCache = true; useCache = false; });
+            .handler([&]() { writeCache = true; useCache = false; });
 
         mkFlag()
             .longName("no-cache")
             .description("do not use or update the package search cache")
-            .handler([&](Strings ss) { writeCache = false; useCache = false; });
+            .handler([&]() { writeCache = false; useCache = false; });
     }
 
     std::string name() override
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 992ff742835..b1825c412c2 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -19,7 +19,7 @@ struct CmdCopySigs : StorePathsCommand
             .labels({"store-uri"})
             .description("use signatures from specified store")
             .arity(1)
-            .handler([&](Strings ss) { substituterUris.push_back(ss.front()); });
+            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
     }
 
     std::string name() override
@@ -101,7 +101,12 @@ struct CmdSignPaths : StorePathsCommand
 
     CmdSignPaths()
     {
-        mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile);
+        mkFlag()
+            .shortName('k')
+            .longName("key-file")
+            .label("file")
+            .description("file containing the secret signing key")
+            .dest(&secretKeyFile);
     }
 
     std::string name() override
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 4913d990097..6540208a8a2 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -25,7 +25,7 @@ struct CmdVerify : StorePathsCommand
             .labels({"store-uri"})
             .description("use signatures from specified store")
             .arity(1)
-            .handler([&](Strings ss) { substituterUris.push_back(ss.front()); });
+            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
         mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
     }
 

From fe9d2f974d41a943b5245dbadbff0d0adab42046 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 13:41:52 +0200
Subject: [PATCH 0608/2196] Handle log messages from build-remote

This makes the progress indicator show statuses like "connecting to
'root@machine'".
---
 src/build-remote/build-remote.cc |  28 ++++---
 src/libstore/build.cc            | 136 +++++--------------------------
 src/libutil/logging.cc           | 130 +++++++++++++++++++++++++++++
 src/libutil/logging.hh           |   5 ++
 4 files changed, 175 insertions(+), 124 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index f023aedb190..dc784f9477e 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -42,6 +42,8 @@ int main (int argc, char * * argv)
     return handleExceptions(argv[0], [&]() {
         initNix();
 
+        logger = makeJSONLogger(*logger);
+
         /* Ensure we don't get any SSH passphrase or host key popups. */
         unsetenv("DISPLAY");
         unsetenv("SSH_ASKPASS");
@@ -172,6 +174,8 @@ int main (int argc, char * * argv)
 
                 try {
 
+                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
+
                     Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}};
                     if (bestMachine->sshKey != "")
                         storeParams["ssh-key"] = bestMachine->sshKey;
@@ -197,21 +201,25 @@ int main (int argc, char * * argv)
         auto inputs = readStrings<PathSet>(source);
         auto outputs = readStrings<PathSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
 
-        auto old = signal(SIGALRM, handleAlarm);
-        alarm(15 * 60);
-        if (!lockFile(uploadLock.get(), ltWrite, true))
-            printError("somebody is hogging the upload lock for '%s', continuing...");
-        alarm(0);
-        signal(SIGALRM, old);
-        copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
-        uploadLock = -1;
+            AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+
+            auto old = signal(SIGALRM, handleAlarm);
+            alarm(15 * 60);
+            if (!lockFile(uploadLock.get(), ltWrite, true))
+                printError("somebody is hogging the upload lock for '%s', continuing...");
+            alarm(0);
+            signal(SIGALRM, old);
+            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
+            uploadLock = -1;
+        }
 
         BasicDerivation drv(readDerivation(drvPath));
         drv.inputSrcs = inputs;
 
-        printError("building '%s' on '%s'", drvPath, storeUri);
+        printInfo("building '%s' on '%s'", drvPath, storeUri);
         auto result = sshStore->buildDerivation(drvPath, drv);
 
         if (!result.success())
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index fc800f430c9..6e3d0a3ebb2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -55,8 +55,6 @@
 #include <sys/statvfs.h>
 #endif
 
-#include <nlohmann/json.hpp>
-
 
 namespace nix {
 
@@ -612,6 +610,8 @@ struct HookInstance
 
     FdSink sink;
 
+    std::map<ActivityId, Activity> activities;
+
     HookInstance();
 
     ~HookInstance();
@@ -769,6 +769,8 @@ class DerivationGoal : public Goal
     std::string currentLogLine;
     size_t currentLogLinePos = 0; // to handle carriage return
 
+    std::string currentHookLine;
+
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
 
@@ -1653,12 +1655,16 @@ HookReply DerivationGoal::tryBuildHook()
         string reply;
         while (true) {
             string s = readLine(worker.hook->fromHook.readSide.get());
-            if (string(s, 0, 2) == "# ") {
+            if (handleJSONLogMessage(s, worker.act, worker.hook->activities))
+                ;
+            else if (string(s, 0, 2) == "# ") {
                 reply = string(s, 2);
                 break;
             }
-            s += "\n";
-            writeToStderr(s);
+            else {
+                s += "\n";
+                writeToStderr(s);
+            }
         }
 
         debug(format("hook reply is '%1%'") % reply);
@@ -2414,64 +2420,6 @@ void setupSeccomp()
 }
 
 
-struct BuilderLogger : Logger
-{
-    Logger & prevLogger;
-
-    BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
-
-    void addFields(nlohmann::json & json, const Fields & fields)
-    {
-        if (fields.empty()) return;
-        auto & arr = json["fields"] = nlohmann::json::array();
-        for (auto & f : fields)
-            if (f.type == Logger::Field::tInt)
-                arr.push_back(f.i);
-            else if (f.type == Logger::Field::tString)
-                arr.push_back(f.s);
-            else
-                abort();
-    }
-
-    void log(Verbosity lvl, const FormatOrString & fs) override
-    {
-        prevLogger.log(lvl, fs);
-    }
-
-    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
-        const std::string & s, const Fields & fields, ActivityId parent) override
-    {
-        nlohmann::json json;
-        json["action"] = "start";
-        json["id"] = act;
-        json["level"] = lvl;
-        json["type"] = type;
-        json["text"] = s;
-        addFields(json, fields);
-        // FIXME: handle parent
-        log(lvlError, "@nix " + json.dump());
-    }
-
-    void stopActivity(ActivityId act) override
-    {
-        nlohmann::json json;
-        json["action"] = "stop";
-        json["id"] = act;
-        log(lvlError, "@nix " + json.dump());
-    }
-
-    void result(ActivityId act, ResultType type, const Fields & fields) override
-    {
-        nlohmann::json json;
-        json["action"] = "result";
-        json["id"] = act;
-        json["type"] = type;
-        addFields(json, fields);
-        log(lvlError, "@nix " + json.dump());
-    }
-};
-
-
 void DerivationGoal::runChild()
 {
     /* Warning: in the child we should absolutely not make any SQLite
@@ -2874,7 +2822,7 @@ void DerivationGoal::runChild()
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
             try {
-                logger = new BuilderLogger(*logger);
+                logger = makeJSONLogger(*logger);
                 if (drv->builder == "builtin:fetchurl")
                     builtinFetchurl(*drv, netrcData);
                 else
@@ -3315,8 +3263,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
         if (logSink) (*logSink)(data);
     }
 
-    if (hook && fd == hook->fromHook.readSide.get())
-        printError(chomp(data));
+    if (hook && fd == hook->fromHook.readSide.get()) {
+        for (auto c : data)
+            if (c == '\n') {
+                handleJSONLogMessage(currentHookLine, worker.act, hook->activities);
+                currentHookLine.clear();
+            } else
+                currentHookLine += c;
+    }
 }
 
 
@@ -3327,56 +3281,10 @@ void DerivationGoal::handleEOF(int fd)
 }
 
 
-static Logger::Fields getFields(nlohmann::json & json)
-{
-    Logger::Fields fields;
-    for (auto & f : json) {
-        if (f.type() == nlohmann::json::value_t::number_unsigned)
-            fields.emplace_back(Logger::Field(f.get<uint64_t>()));
-        else if (f.type() == nlohmann::json::value_t::string)
-            fields.emplace_back(Logger::Field(f.get<std::string>()));
-        else throw Error("unsupported JSON type %d", (int) f.type());
-    }
-    return fields;
-}
-
-
 void DerivationGoal::flushLine()
 {
-    if (hasPrefix(currentLogLine, "@nix ")) {
-
-        try {
-            auto json = nlohmann::json::parse(std::string(currentLogLine, 5));
-
-            std::string action = json["action"];
-
-            if (action == "start") {
-                auto type = (ActivityType) json["type"];
-                if (type == actDownload)
-                    builderActivities.emplace(std::piecewise_construct,
-                        std::forward_as_tuple(json["id"]),
-                        std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
-                            json["text"], getFields(json["fields"]), act->id));
-            }
-
-            else if (action == "stop")
-                builderActivities.erase((ActivityId) json["id"]);
-
-            else if (action == "result") {
-                auto i = builderActivities.find((ActivityId) json["id"]);
-                if (i != builderActivities.end())
-                    i->second.result((ResultType) json["type"], getFields(json["fields"]));
-            }
-
-            else if (action == "setPhase") {
-                std::string phase = json["phase"];
-                act->result(resSetPhase, phase);
-            }
-
-        } catch (std::exception & e) {
-            printError("bad log message from builder: %s", e.what());
-        }
-    }
+    if (handleJSONLogMessage(currentLogLine, *act, builderActivities))
+        ;
 
     else {
         if (settings.verboseBuild &&
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index e38a460537e..1a2684a43a5 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 
 #include <atomic>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -90,4 +91,133 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
+struct JSONLogger : Logger
+{
+    Logger & prevLogger;
+
+    JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
+
+    void addFields(nlohmann::json & json, const Fields & fields)
+    {
+        if (fields.empty()) return;
+        auto & arr = json["fields"] = nlohmann::json::array();
+        for (auto & f : fields)
+            if (f.type == Logger::Field::tInt)
+                arr.push_back(f.i);
+            else if (f.type == Logger::Field::tString)
+                arr.push_back(f.s);
+            else
+                abort();
+    }
+
+    void write(const nlohmann::json & json)
+    {
+        prevLogger.log(lvlError, "@nix " + json.dump());
+    }
+
+    void log(Verbosity lvl, const FormatOrString & fs) override
+    {
+        nlohmann::json json;
+        json["action"] = "msg";
+        json["level"] = lvl;
+        json["msg"] = fs.s;
+        write(json);
+    }
+
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
+        const std::string & s, const Fields & fields, ActivityId parent) override
+    {
+        nlohmann::json json;
+        json["action"] = "start";
+        json["id"] = act;
+        json["level"] = lvl;
+        json["type"] = type;
+        json["text"] = s;
+        addFields(json, fields);
+        // FIXME: handle parent
+        write(json);
+    }
+
+    void stopActivity(ActivityId act) override
+    {
+        nlohmann::json json;
+        json["action"] = "stop";
+        json["id"] = act;
+        write(json);
+    }
+
+    void result(ActivityId act, ResultType type, const Fields & fields) override
+    {
+        nlohmann::json json;
+        json["action"] = "result";
+        json["id"] = act;
+        json["type"] = type;
+        addFields(json, fields);
+        write(json);
+    }
+};
+
+Logger * makeJSONLogger(Logger & prevLogger)
+{
+    return new JSONLogger(prevLogger);
+}
+
+static Logger::Fields getFields(nlohmann::json & json)
+{
+    Logger::Fields fields;
+    for (auto & f : json) {
+        if (f.type() == nlohmann::json::value_t::number_unsigned)
+            fields.emplace_back(Logger::Field(f.get<uint64_t>()));
+        else if (f.type() == nlohmann::json::value_t::string)
+            fields.emplace_back(Logger::Field(f.get<std::string>()));
+        else throw Error("unsupported JSON type %d", (int) f.type());
+    }
+    return fields;
+}
+
+bool handleJSONLogMessage(const std::string & msg,
+    const Activity & act, std::map<ActivityId, Activity> & activities)
+{
+    if (!hasPrefix(msg, "@nix ")) return false;
+
+    try {
+        auto json = nlohmann::json::parse(std::string(msg, 5));
+
+        std::string action = json["action"];
+
+        if (action == "start") {
+            auto type = (ActivityType) json["type"];
+            if (type == actDownload || type == actUnknown)
+                activities.emplace(std::piecewise_construct,
+                    std::forward_as_tuple(json["id"]),
+                    std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
+                        json["text"], getFields(json["fields"]), act.id));
+        }
+
+        else if (action == "stop")
+            activities.erase((ActivityId) json["id"]);
+
+        else if (action == "result") {
+            auto i = activities.find((ActivityId) json["id"]);
+            if (i != activities.end())
+                i->second.result((ResultType) json["type"], getFields(json["fields"]));
+        }
+
+        else if (action == "setPhase") {
+            std::string phase = json["phase"];
+            act.result(resSetPhase, phase);
+        }
+
+        else if (action == "msg") {
+            std::string msg = json["msg"];
+            logger->log((Verbosity) json["level"], msg);
+        }
+
+    } catch (std::exception & e) {
+        printError("bad log message from builder: %s", e.what());
+    }
+
+    return true;
+}
+
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 21898c03a27..70a9f5d8624 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -130,6 +130,11 @@ extern Logger * logger;
 
 Logger * makeDefaultLogger();
 
+Logger * makeJSONLogger(Logger & prevLogger);
+
+bool handleJSONLogMessage(const std::string & msg,
+    const Activity & act, std::map<ActivityId, Activity> & activities);
+
 extern Verbosity verbosity; /* suppress msgs > this */
 
 /* Print a message if the current log level is at least the specified

From f8624762aca7df05cfb405f5e79065d61b1a8d4b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 14:14:28 +0200
Subject: [PATCH 0609/2196] build-remote: Work properly on a chrooted store

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index dc784f9477e..92ae99e168b 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -216,7 +216,7 @@ int main (int argc, char * * argv)
             uploadLock = -1;
         }
 
-        BasicDerivation drv(readDerivation(drvPath));
+        BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
         drv.inputSrcs = inputs;
 
         printInfo("building '%s' on '%s'", drvPath, storeUri);

From be220702a75427b80258a92d1ed8c4573c1478fa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 14:24:57 +0200
Subject: [PATCH 0610/2196] Progress indicator: Show on what machine we're
 building

E.g.

  $ nix build nixpkgs.hello --builders 'root@wendy'
  [1/0/1 built] building hello-2.10 on ssh://root@wendy: checking for minix/config.h... no
---
 src/build-remote/build-remote.cc |  2 +-
 src/libstore/build.cc            | 10 +++++++---
 src/nix/progress-bar.cc          |  3 +++
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 92ae99e168b..d2fac09f843 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -196,7 +196,7 @@ int main (int argc, char * * argv)
         }
 
 connected:
-        std::cerr << "# accept\n";
+        std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
         auto outputs = readStrings<PathSet>(source);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6e3d0a3ebb2..dcc48bef770 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -852,6 +852,9 @@ class DerivationGoal : public Goal
 
     std::map<ActivityId, Activity> builderActivities;
 
+    /* The remote machine on which we're building. */
+    std::string machineName;
+
 public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
@@ -1400,7 +1403,8 @@ void DerivationGoal::tryToBuild()
 
     auto started = [&]() {
         act = std::make_unique<Activity>(*logger, lvlInfo, actBuild,
-            fmt("building '%s'", drvPath), Logger::Fields{drvPath});
+            fmt("building '%s'", drvPath),
+            Logger::Fields{drvPath, hook ? machineName : ""});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
     };
@@ -1691,10 +1695,10 @@ HookReply DerivationGoal::tryBuildHook()
             throw;
     }
 
-    printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths));
-
     hook = std::move(worker.hook);
 
+    machineName = readLine(hook->fromHook.readSide.get());
+
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
     hook->sink << inputPaths;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 76138b2cce2..a33ea342383 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -156,6 +156,9 @@ class ProgressBar : public Logger
             if (hasSuffix(name, ".drv"))
                 name.resize(name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
+            auto machineName = getS(fields, 1);
+            if (machineName != "")
+                i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName);
         }
 
         if (type == actSubstitute) {

From 96051dd057d46d5953b2fbe3de67269d175306dc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 14:47:23 +0200
Subject: [PATCH 0611/2196] More progress indicator improvements

Fixes #1599.
---
 src/build-remote/build-remote.cc | 13 +++++++++----
 src/libstore/build.cc            |  8 ++++----
 src/libstore/store-api.cc        | 12 ++++++++++--
 src/libutil/logging.cc           |  4 ++--
 src/libutil/logging.hh           |  3 ++-
 5 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index d2fac09f843..419e7205372 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -201,25 +201,29 @@ int main (int argc, char * * argv)
         auto inputs = readStrings<PathSet>(source);
         auto outputs = readStrings<PathSet>(source);
 
+        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
 
-            AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
-
             auto old = signal(SIGALRM, handleAlarm);
             alarm(15 * 60);
             if (!lockFile(uploadLock.get(), ltWrite, true))
                 printError("somebody is hogging the upload lock for '%s', continuing...");
             alarm(0);
             signal(SIGALRM, old);
+        }
+
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
             copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
-            uploadLock = -1;
         }
 
+        uploadLock = -1;
+
         BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
         drv.inputSrcs = inputs;
 
-        printInfo("building '%s' on '%s'", drvPath, storeUri);
         auto result = sshStore->buildDerivation(drvPath, drv);
 
         if (!result.success())
@@ -230,6 +234,7 @@ int main (int argc, char * * argv)
             if (!store->isValidPath(path)) missing.insert(path);
 
         if (!missing.empty()) {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs);
         }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dcc48bef770..681879c6aa5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1403,7 +1403,7 @@ void DerivationGoal::tryToBuild()
 
     auto started = [&]() {
         act = std::make_unique<Activity>(*logger, lvlInfo, actBuild,
-            fmt("building '%s'", drvPath),
+            hook ? fmt("building '%s' on '%s'", drvPath, machineName) : fmt("building '%s'", drvPath),
             Logger::Fields{drvPath, hook ? machineName : ""});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
@@ -1659,7 +1659,7 @@ HookReply DerivationGoal::tryBuildHook()
         string reply;
         while (true) {
             string s = readLine(worker.hook->fromHook.readSide.get());
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities))
+            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
                 ;
             else if (string(s, 0, 2) == "# ") {
                 reply = string(s, 2);
@@ -3270,7 +3270,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     if (hook && fd == hook->fromHook.readSide.get()) {
         for (auto c : data)
             if (c == '\n') {
-                handleJSONLogMessage(currentHookLine, worker.act, hook->activities);
+                handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
                 currentHookLine.clear();
             } else
                 currentHookLine += c;
@@ -3287,7 +3287,7 @@ void DerivationGoal::handleEOF(int fd)
 
 void DerivationGoal::flushLine()
 {
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities))
+    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
         ;
 
     else {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 3bc5e4c57c9..17237a03541 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -565,8 +565,16 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    Activity act(*logger, lvlInfo, actCopyPath, fmt("copying path '%s'", storePath),
-        {storePath, srcStore->getUri(), dstStore->getUri()});
+    auto srcUri = srcStore->getUri();
+    auto dstUri = dstStore->getUri();
+
+    Activity act(*logger, lvlInfo, actCopyPath,
+        srcUri == "local"
+          ? fmt("copying path '%s' to '%s'", storePath, dstUri)
+          : dstUri == "local"
+            ? fmt("copying path '%s' from '%s'", storePath, srcUri)
+            : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
+        {storePath, srcUri, dstUri});
     PushActivity pact(act.id);
 
     auto info = srcStore->queryPathInfo(storePath);
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 1a2684a43a5..01115587112 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -176,7 +176,7 @@ static Logger::Fields getFields(nlohmann::json & json)
 }
 
 bool handleJSONLogMessage(const std::string & msg,
-    const Activity & act, std::map<ActivityId, Activity> & activities)
+    const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted)
 {
     if (!hasPrefix(msg, "@nix ")) return false;
 
@@ -187,7 +187,7 @@ bool handleJSONLogMessage(const std::string & msg,
 
         if (action == "start") {
             auto type = (ActivityType) json["type"];
-            if (type == actDownload || type == actUnknown)
+            if (trusted || type == actDownload)
                 activities.emplace(std::piecewise_construct,
                     std::forward_as_tuple(json["id"]),
                     std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 70a9f5d8624..677aa4daec4 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -133,7 +133,8 @@ Logger * makeDefaultLogger();
 Logger * makeJSONLogger(Logger & prevLogger);
 
 bool handleJSONLogMessage(const std::string & msg,
-    const Activity & act, std::map<ActivityId, Activity> & activities);
+    const Activity & act, std::map<ActivityId, Activity> & activities,
+    bool trusted);
 
 extern Verbosity verbosity; /* suppress msgs > this */
 

From 3460e4cf007017e89096b4fad083692666b87a20 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 15:05:21 +0200
Subject: [PATCH 0612/2196] More progress indicator improvements

In particular, don't show superfluous "fetching path" and "building
path(s)" messages, and show the current round (with --repeat).
---
 src/libstore/build.cc   | 22 +++++++++-------------
 src/nix/progress-bar.cc |  4 ++++
 tests/binary-cache.sh   |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 681879c6aa5..856d516d2bb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1402,9 +1402,15 @@ void DerivationGoal::tryToBuild()
     bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
 
     auto started = [&]() {
-        act = std::make_unique<Activity>(*logger, lvlInfo, actBuild,
-            hook ? fmt("building '%s' on '%s'", drvPath, machineName) : fmt("building '%s'", drvPath),
-            Logger::Fields{drvPath, hook ? machineName : ""});
+        auto msg = fmt(
+            buildMode == bmRepair ? "repairing outputs of '%s'" :
+            buildMode == bmCheck ? "checking outputs of '%s'" :
+            nrRounds > 1 ? "building '%s' (round %d/%d)" :
+            "building '%s'", drvPath, curRound, nrRounds);
+        fmt("building '%s'", drvPath);
+        if (hook) msg += fmt(" on '%s'", machineName);
+        act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
+            Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
     };
@@ -1738,14 +1744,6 @@ int childEntry(void * arg)
 
 void DerivationGoal::startBuilder()
 {
-    auto f = format(
-        buildMode == bmRepair ? "repairing path(s) %1%" :
-        buildMode == bmCheck ? "checking path(s) %1%" :
-        nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" :
-        "building path(s) %1%");
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    printInfo(f % showPaths(missingPaths) % curRound % nrRounds);
-
     /* Right platform? */
     if (!drv->canBuildLocally()) {
         throw Error(
@@ -3601,8 +3599,6 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
-    printInfo(format("fetching path '%1%'...") % storePath);
-
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index a33ea342383..fb9955190b4 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -159,6 +159,10 @@ class ProgressBar : public Logger
             auto machineName = getS(fields, 1);
             if (machineName != "")
                 i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName);
+            auto curRound = getI(fields, 2);
+            auto nrRounds = getI(fields, 3);
+            if (nrRounds != 1)
+                i->s += fmt(" (round %d/%d)", curRound, nrRounds);
         }
 
         if (type == actSubstitute) {
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 30d7cc6ba88..2a044d2edc5 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -100,7 +100,7 @@ clearStore
 rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
 nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
-grep -q "fetching path" $TEST_ROOT/log
+grep -q "copying path" $TEST_ROOT/log
 
 
 if [ -n "$HAVE_SODIUM" ]; then

From d16fd2497374671c92cb877f9570d65783a77d99 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 15:16:18 +0200
Subject: [PATCH 0613/2196] Allow shorter syntax for chroot stores

You can now say '--store /tmp/nix' instead of '--store local?root=/tmp/nix'.
---
 src/build-remote/build-remote.cc | 10 +++++++---
 src/libstore/machines.cc         |  6 +++++-
 src/libstore/store-api.cc        | 10 +++++++---
 tests/build-remote.sh            |  6 +++---
 tests/linux-sandbox.sh           |  2 +-
 5 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 419e7205372..6e05e165545 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -176,9 +176,13 @@ int main (int argc, char * * argv)
 
                     Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
 
-                    Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}};
-                    if (bestMachine->sshKey != "")
-                        storeParams["ssh-key"] = bestMachine->sshKey;
+                    Store::Params storeParams;
+                    if (hasPrefix(storeUri, "ssh://")) {
+                        storeParams["max-connections"] ="1";
+                        storeParams["log-fd"] = "4";
+                        if (bestMachine->sshKey != "")
+                            storeParams["ssh-key"] = bestMachine->sshKey;
+                    }
 
                     sshStore = openStore(bestMachine->storeUri, storeParams);
                     sshStore->connect();
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index d2faf45357e..edd03d14783 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -17,7 +17,11 @@ Machine::Machine(decltype(storeUri) storeUri,
     storeUri(
         // Backwards compatibility: if the URI is a hostname,
         // prepend ssh://.
-        storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto")
+        storeUri.find("://") != std::string::npos
+        || hasPrefix(storeUri, "local")
+        || hasPrefix(storeUri, "remote")
+        || hasPrefix(storeUri, "auto")
+        || hasPrefix(storeUri, "/")
         ? storeUri
         : "ssh://" + storeUri),
     systemTypes(systemTypes),
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 17237a03541..c57e42fec00 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -843,7 +843,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
 {
     if (uri == "daemon") {
         return tDaemon;
-    } else if (uri == "local") {
+    } else if (uri == "local" || hasPrefix(uri, "/")) {
         return tLocal;
     } else if (uri == "" || uri == "auto") {
         if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@@ -865,8 +865,12 @@ static RegisterStoreImplementation regStore([](
     switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
         case tDaemon:
             return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
-        case tLocal:
-            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
+        case tLocal: {
+            Store::Params params2 = params;
+            if (hasPrefix(uri, "/"))
+                params2["root"] = uri;
+            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
+        }
         default:
             return nullptr;
     }
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 603b57c59ed..cf3bb463318 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -11,7 +11,7 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 
 nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
   --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
-  --builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo"
+  --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo"
 
 outPath=$TEST_ROOT/result
 
@@ -19,5 +19,5 @@ cat $outPath/foobar | grep FOOBAR
 
 # Ensure that input1 was built on store1 due to the required feature.
 p=$(readlink -f $outPath/input-2)
-(! nix path-info --store local?root=$TEST_ROOT/store0 --all | grep dependencies.builder1.sh)
-nix path-info --store local?root=$TEST_ROOT/store1 --all | grep dependencies.builder1.sh
+(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh)
+nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 7f4e83b560b..0691c30be1c 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -14,7 +14,7 @@ chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0
 
 export NIX_STORE_DIR=/my/store
-export NIX_REMOTE="local?root=$TEST_ROOT/store0"
+export NIX_REMOTE=$TEST_ROOT/store0
 
 outPath=$(nix-build dependencies.nix --no-out-link --option sandbox-paths /nix/store)
 

From 86aab6e9a04874f368ca0534068ed9ca1d46ae59 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 15:41:11 +0200
Subject: [PATCH 0614/2196] Add --store argument to legacy commands

---
 src/libmain/shared.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 9123a3d1ae0..0f599f38858 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -198,6 +198,12 @@ LegacyArgs::LegacyArgs(const std::string & programName,
 
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
+
+    mkFlag()
+        .longName("store")
+        .label("store-uri")
+        .description("URI of the Nix store to use")
+        .dest(&(std::string&) settings.storeUri);
 }
 
 

From ac12517f3e969db9fb4834fdfd2da63101ffd3de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2017 15:44:29 +0200
Subject: [PATCH 0615/2196] Fix RPM build

---
 nix.spec.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix.spec.in b/nix.spec.in
index 241e28f3678..4e7dd90ea68 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -140,6 +140,7 @@ systemctl start  nix-daemon.socket
 %{_mandir}/man5/*.5*
 %{_mandir}/man8/*.8*
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
+%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
 
 %files devel

From 2d5b1b24bf70a498e4c0b378704cfdb6471cc699 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2017 13:01:50 +0200
Subject: [PATCH 0616/2196] Pass lists/attrsets to bash as (associative) arrays

---
 src/libexpr/primops.cc     |  6 +--
 src/libstore/build.cc      | 96 ++++++++++++++++++++++++++++++++++++--
 src/libutil/util.cc        | 10 ++++
 src/libutil/util.hh        |  6 +--
 src/nix-build/nix-build.cc |  4 --
 src/nix-store/nix-store.cc | 11 +----
 tests/config.nix           |  2 +-
 tests/local.mk             |  3 +-
 tests/structured-attrs.nix | 47 +++++++++++++++++++
 tests/structured-attrs.sh  |  7 +++
 10 files changed, 166 insertions(+), 26 deletions(-)
 create mode 100644 tests/structured-attrs.nix
 create mode 100644 tests/structured-attrs.sh

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index fcd3f8efee3..afd34428660 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
-        drv.env["out"] = outPath;
+        if (!jsonObject) drv.env["out"] = outPath;
         drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
     }
 
@@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            an empty value.  This ensures that changes in the set of
            output names do get reflected in the hash. */
         for (auto & i : outputs) {
-            drv.env[i] = "";
+            if (!jsonObject) drv.env[i] = "";
             drv.outputs[i] = DerivationOutput("", "", "");
         }
 
@@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : drv.outputs)
             if (i.second.path == "") {
                 Path outPath = state.store->makeOutputPath(i.first, h, drvName);
-                drv.env[i.first] = outPath;
+                if (!jsonObject) drv.env[i.first] = outPath;
                 i.second.path = outPath;
             }
     }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 856d516d2bb..a7d418404e8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -18,6 +18,7 @@
 #include <thread>
 #include <future>
 #include <chrono>
+#include <regex>
 
 #include <limits.h>
 #include <sys/time.h>
@@ -55,6 +56,8 @@
 #include <sys/statvfs.h>
 #endif
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -2286,12 +2289,99 @@ void DerivationGoal::initEnv()
 }
 
 
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
+
+
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto json = drv->env.find("__json");
-    if (json == drv->env.end()) return;
+    auto jsonAttr = drv->env.find("__json");
+    if (jsonAttr == drv->env.end()) return;
+
+    try {
+
+        auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
+
+        auto json = nlohmann::json::parse(jsonStr);
+
+        /* Add an "outputs" object containing the output paths. */
+        nlohmann::json outputs;
+        for (auto & i : drv->outputs)
+            outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+        json["outputs"] = outputs;
+
+        writeFile(tmpDir + "/.attrs.json", json.dump());
+
+        /* As a convenience to bash scripts, write a shell file that
+           maps all attributes that are representable in bash -
+           namely, strings, integers, nulls, Booleans, and arrays and
+           objects consisting entirely of those values. (So nested
+           arrays or objects are not supported.) */
+
+        auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+            if (value.is_string())
+                return shellEscape(value);
+
+            if (value.is_number()) {
+                auto f = value.get<float>();
+                if (std::ceil(f) == f)
+                    return std::to_string(value.get<int>());
+            }
+
+            if (value.is_null())
+                return "''";
+
+            if (value.is_boolean())
+                return value.get<bool>() ? "1" : "";
+
+            return {};
+        };
+
+        std::string jsonSh;
 
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites));
+        for (auto i = json.begin(); i != json.end(); ++i) {
+
+            if (!std::regex_match(i.key(), shVarName)) continue;
+
+            auto & value = i.value();
+
+            auto s = handleSimpleType(value);
+            if (s)
+                jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+
+            else if (value.is_array()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += *s3; s2 += ' ';
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+            }
+
+            else if (value.is_object()) {
+                std::string s2;
+                bool good = true;
+
+                for (auto i = value.begin(); i != value.end(); ++i) {
+                    auto s3 = handleSimpleType(i.value());
+                    if (!s3) { good = false; break; }
+                    s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+                }
+
+                if (good)
+                    jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+            }
+        }
+
+        writeFile(tmpDir + "/.attrs.sh", jsonSh);
+
+    } catch (std::exception & e) {
+        throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+    }
 }
 
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 3c98a61f9e5..9346d5dc4cf 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s)
 }
 
 
+std::string shellEscape(const std::string & s)
+{
+    std::string r = "'";
+    for (auto & i : s)
+        if (i == '\'') r += "'\\''"; else r += i;
+    r += '\'';
+    return r;
+}
+
+
 void ignoreException()
 {
     try {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 6a66576e96c..fccf5d85480 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix);
 std::string toLower(const std::string & s);
 
 
-/* Escape a string that contains octal-encoded escape codes such as
-   used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to
-   "foo bar"). */
-string decodeOctalEscaped(const string & s);
+/* Escape a string as a shell word. */
+std::string shellEscape(const std::string & s);
 
 
 /* Exception handling in destructors: print an error message, then
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 85137206fa7..8e56e5a46b1 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -196,10 +196,6 @@ void mainWrapped(int argc, char * * argv)
             interactive = false;
             auto execArgs = "";
 
-            auto shellEscape = [](const string & s) {
-                return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
-            };
-
             // Überhack to support Perl. Perl examines the shebang and
             // executes it unless it contains the string "perl" or "indir",
             // or (undocumented) argv[0] does not contain "perl". Exploit
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 85bcbc22e9d..f6f276dd179 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
 }
 
 
-static string shellEscape(const string & s)
-{
-    string r;
-    for (auto & i : s)
-        if (i == '\'') r += "'\\''"; else r += i;
-    return r;
-}
-
-
 static void opPrintEnv(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
     /* Print each environment variable in the derivation in a format
        that can be sourced by the shell. */
     for (auto & i : drv.env)
-        cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second);
+        cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second);
 
     /* Also output the arguments.  This doesn't preserve whitespace in
        arguments. */
diff --git a/tests/config.nix b/tests/config.nix
index 76388fdd5b9..6ba91065b83 100644
--- a/tests/config.nix
+++ b/tests/config.nix
@@ -13,7 +13,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
diff --git a/tests/local.mk b/tests/local.mk
index 19d6f1893d2..6160b04c259 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -14,7 +14,8 @@ nix_tests = \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
   build-remote.sh \
-  nar-index.sh
+  nar-index.sh \
+  structured-attrs.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix
new file mode 100644
index 00000000000..2adc6b6c34f
--- /dev/null
+++ b/tests/structured-attrs.nix
@@ -0,0 +1,47 @@
+with import ./config.nix;
+
+mkDerivation {
+  name = "structured";
+
+  __structuredAttrs = true;
+
+  buildCommand = ''
+    set -x
+
+    [[ $int = 123456789 ]]
+    [[ -z $float ]]
+    [[ -n $boolTrue ]]
+    [[ -z $boolFalse ]]
+    [[ -n ''${hardening[format]} ]]
+    [[ -z ''${hardening[fortify]} ]]
+    [[ ''${#buildInputs[@]} = 7 ]]
+    [[ ''${buildInputs[2]} = c ]]
+    [[ -v nothing ]]
+    [[ -z $nothing ]]
+
+    mkdir ''${outputs[out]}
+    echo bar > $dest
+  '';
+
+  buildInputs = [ "a" "b" "c" 123 "'" "\"" null ];
+
+  hardening.format = true;
+  hardening.fortify = false;
+
+  outer.inner = [ 1 2 3 ];
+
+  int = 123456789;
+
+  float = 123.456;
+
+  boolTrue = true;
+  boolFalse = false;
+
+  nothing = null;
+
+  dest = "${placeholder "out"}/foo";
+
+  "foo bar" = "BAD";
+  "1foobar" = "BAD";
+  "foo$" = "BAD";
+}
diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh
new file mode 100644
index 00000000000..9ba2672b683
--- /dev/null
+++ b/tests/structured-attrs.sh
@@ -0,0 +1,7 @@
+source common.sh
+
+clearStore
+
+outPath=$(nix-build structured-attrs.nix --no-out-link)
+
+[[ $(cat $outPath/foo) = bar ]]

From 3395e3bbc4bc88a102600155ea996777669364ab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2017 14:08:29 +0200
Subject: [PATCH 0617/2196] Fix exportReferencesGraph in the structured attrs
 case

---
 src/libstore/build.cc      | 137 ++++++++++++++++++-------------------
 tests/structured-attrs.nix |  19 +++++
 2 files changed, 87 insertions(+), 69 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a7d418404e8..2bca7e1d02a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -914,9 +914,6 @@ class DerivationGoal : public Goal
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
 
-    /* Handle the exportReferencesGraph attribute. */
-    void doExportReferencesGraph();
-
     /* Run the builder's process. */
     void runChild();
 
@@ -1745,6 +1742,37 @@ int childEntry(void * arg)
 }
 
 
+PathSet exportReferences(Store & store, Path storePath)
+{
+    /* Check that the store path is valid. */
+    if (!store.isInStore(storePath))
+        throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
+            % storePath);
+    storePath = store.toStorePath(storePath);
+    if (!store.isValidPath(storePath))
+        throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
+            % storePath);
+
+    /* If there are derivations in the graph, then include their
+       outputs as well.  This is useful if you want to do things
+       like passing all build-time dependencies of some path to a
+       derivation that builds a NixOS DVD image. */
+    PathSet paths, paths2;
+    store.computeFSClosure(storePath, paths);
+    paths2 = paths;
+
+    for (auto & j : paths2) {
+        if (isDerivation(j)) {
+            Derivation drv = store.derivationFromPath(j);
+            for (auto & k : drv.outputs)
+                store.computeFSClosure(k.second.path, paths);
+        }
+    }
+
+    return paths;
+}
+
+
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
@@ -1820,7 +1848,29 @@ void DerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    doExportReferencesGraph();
+    if (!drv->env.count("__json")) {
+        /* The `exportReferencesGraph' feature allows the references graph
+           to be passed to a builder.  This attribute should be a list of
+           pairs [name1 path1 name2 path2 ...].  The references graph of
+           each `pathN' will be stored in a text file `nameN' in the
+           temporary build directory.  The text files have the format used
+           by `nix-store --register-validity'.  However, the deriver
+           fields are left empty. */
+        string s = get(drv->env, "exportReferencesGraph");
+        Strings ss = tokenizeString<Strings>(s);
+        if (ss.size() % 2 != 0)
+            throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
+        for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
+            string fileName = *i++;
+            checkStoreName(fileName); /* !!! abuse of this function */
+            Path storePath = *i++;
+
+            /* Write closure info to <fileName>. */
+            writeFile(tmpDir + "/" + fileName,
+                worker.store.makeValidityRegistration(
+                    exportReferences(worker.store, storePath), false, false));
+        }
+    }
 
     if (useChroot) {
 
@@ -2309,6 +2359,20 @@ void DerivationGoal::writeStructuredAttrs()
             outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
         json["outputs"] = outputs;
 
+        /* Handle exportReferencesGraph. */
+        auto e = json.find("exportReferencesGraph");
+        if (e != json.end() && e->is_object()) {
+            for (auto i = e->begin(); i != e->end(); ++i) {
+                std::ostringstream str;
+                {
+                    JSONPlaceholder jsonRoot(str, true);
+                    worker.store.pathInfoToJSON(jsonRoot,
+                        exportReferences(worker.store, i->get<std::string>()), false, true);
+                }
+                json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+            }
+        }
+
         writeFile(tmpDir + "/.attrs.json", json.dump());
 
         /* As a convenience to bash scripts, write a shell file that
@@ -2393,71 +2457,6 @@ void DerivationGoal::chownToBuilder(const Path & path)
 }
 
 
-void DerivationGoal::doExportReferencesGraph()
-{
-    /* The `exportReferencesGraph' feature allows the references graph
-       to be passed to a builder.  This attribute should be a list of
-       pairs [name1 path1 name2 path2 ...].  The references graph of
-       each `pathN' will be stored in a text file `nameN' in the
-       temporary build directory.  The text files have the format used
-       by `nix-store --register-validity'.  However, the deriver
-       fields are left empty. */
-    string s = get(drv->env, "exportReferencesGraph");
-    Strings ss = tokenizeString<Strings>(s);
-    if (ss.size() % 2 != 0)
-        throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
-    for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
-        string fileName = *i++;
-        checkStoreName(fileName); /* !!! abuse of this function */
-
-        /* Check that the store path is valid. */
-        Path storePath = *i++;
-        if (!worker.store.isInStore(storePath))
-            throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
-                % storePath);
-        storePath = worker.store.toStorePath(storePath);
-        if (!worker.store.isValidPath(storePath))
-            throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
-                % storePath);
-
-        /* If there are derivations in the graph, then include their
-           outputs as well.  This is useful if you want to do things
-           like passing all build-time dependencies of some path to a
-           derivation that builds a NixOS DVD image. */
-        PathSet paths, paths2;
-        worker.store.computeFSClosure(storePath, paths);
-        paths2 = paths;
-
-        for (auto & j : paths2) {
-            if (isDerivation(j)) {
-                Derivation drv = worker.store.derivationFromPath(j);
-                for (auto & k : drv.outputs)
-                    worker.store.computeFSClosure(k.second.path, paths);
-            }
-        }
-
-        if (!drv->env.count("__json")) {
-
-            /* Write closure info to <fileName>. */
-            writeFile(tmpDir + "/" + fileName,
-                worker.store.makeValidityRegistration(paths, false, false));
-
-        } else {
-
-            /* Write a more comprehensive JSON serialisation to
-               <fileName>. */
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                worker.store.pathInfoToJSON(jsonRoot, paths, false, true);
-            }
-            writeFile(tmpDir + "/" + fileName, str.str());
-
-        }
-    }
-}
-
-
 void setupSeccomp()
 {
 #if __linux__
diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix
index 2adc6b6c34f..72e9c674750 100644
--- a/tests/structured-attrs.nix
+++ b/tests/structured-attrs.nix
@@ -1,5 +1,16 @@
 with import ./config.nix;
 
+let
+
+  dep = mkDerivation {
+    name = "dep";
+    buildCommand = ''
+      mkdir $out; echo bla > $out/bla
+    '';
+  };
+
+in
+
 mkDerivation {
   name = "structured";
 
@@ -21,6 +32,12 @@ mkDerivation {
 
     mkdir ''${outputs[out]}
     echo bar > $dest
+
+    json=$(cat .attrs.json)
+    [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]]
+    [[ $json =~ '"narSize":288' ]]
+    [[ $json =~ '"closureSize":288' ]]
+    [[ $json =~ '"references":[]' ]]
   '';
 
   buildInputs = [ "a" "b" "c" 123 "'" "\"" null ];
@@ -44,4 +61,6 @@ mkDerivation {
   "foo bar" = "BAD";
   "1foobar" = "BAD";
   "foo$" = "BAD";
+
+  exportReferencesGraph.refs = dep;
 }

From 8191992c83bf4387b03c5fdaba818dc2b520462d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2017 15:18:37 +0200
Subject: [PATCH 0618/2196] Bump language version

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index afd34428660..6a50c99345e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2008,7 +2008,7 @@ void EvalState::createBaseEnv()
        language feature gets added.  It's not necessary to increase it
        when primops get added, because you can just use `builtins ?
        primOp' to check. */
-    mkInt(v, 4);
+    mkInt(v, 5);
     addConstant("__langVersion", v);
 
     // Miscellaneous

From 82327e3cc474be3c72a22480ad6e219f072e27e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2017 15:18:49 +0200
Subject: [PATCH 0619/2196] exportReferencesGraph: Allow exporting a list of
 store paths

---
 src/libstore/build.cc      | 36 ++++++++++++++++++++++--------------
 tests/structured-attrs.nix |  2 +-
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2bca7e1d02a..51afbac7fc7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1742,24 +1742,29 @@ int childEntry(void * arg)
 }
 
 
-PathSet exportReferences(Store & store, Path storePath)
+PathSet exportReferences(Store & store, PathSet storePaths)
 {
-    /* Check that the store path is valid. */
-    if (!store.isInStore(storePath))
-        throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
-            % storePath);
-    storePath = store.toStorePath(storePath);
-    if (!store.isValidPath(storePath))
-        throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
-            % storePath);
+    PathSet paths;
+
+    for (auto storePath : storePaths) {
+
+        /* Check that the store path is valid. */
+        if (!store.isInStore(storePath))
+            throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
+                % storePath);
+        storePath = store.toStorePath(storePath);
+        if (!store.isValidPath(storePath))
+            throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
+                % storePath);
+
+        store.computeFSClosure(storePath, paths);
+    }
 
     /* If there are derivations in the graph, then include their
        outputs as well.  This is useful if you want to do things
        like passing all build-time dependencies of some path to a
        derivation that builds a NixOS DVD image. */
-    PathSet paths, paths2;
-    store.computeFSClosure(storePath, paths);
-    paths2 = paths;
+    PathSet paths2(paths);
 
     for (auto & j : paths2) {
         if (isDerivation(j)) {
@@ -1868,7 +1873,7 @@ void DerivationGoal::startBuilder()
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences(worker.store, storePath), false, false));
+                    exportReferences(worker.store, {storePath}), false, false));
         }
     }
 
@@ -2366,8 +2371,11 @@ void DerivationGoal::writeStructuredAttrs()
                 std::ostringstream str;
                 {
                     JSONPlaceholder jsonRoot(str, true);
+                    PathSet storePaths;
+                    for (auto & p : *i)
+                        storePaths.insert(p.get<std::string>());
                     worker.store.pathInfoToJSON(jsonRoot,
-                        exportReferences(worker.store, i->get<std::string>()), false, true);
+                        exportReferences(worker.store, storePaths), false, true);
                 }
                 json[i.key()] = nlohmann::json::parse(str.str()); // urgh
             }
diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix
index 72e9c674750..6c77a43913a 100644
--- a/tests/structured-attrs.nix
+++ b/tests/structured-attrs.nix
@@ -62,5 +62,5 @@ mkDerivation {
   "1foobar" = "BAD";
   "foo$" = "BAD";
 
-  exportReferencesGraph.refs = dep;
+  exportReferencesGraph.refs = [ dep ];
 }

From 9f01a3f0a87679ac3ba2c71ab6d31478bcbef481 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Wed, 25 Oct 2017 16:13:49 +0100
Subject: [PATCH 0620/2196] attempt to fix #1630: make the queries of store
 paths run in parallel using a thread pool

---
 src/libstore/store-api.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c57e42fec00..82e0bda5468 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -389,8 +389,9 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
     Sync<State> state_(State{paths.size(), PathSet()});
 
     std::condition_variable wakeup;
+    ThreadPool pool;
 
-    for (auto & path : paths)
+    auto doQuery = [&](const Path & path ) {
         queryPathInfo(path,
             [path, &state_, &wakeup](ref<ValidPathInfo> info) {
                 auto state(state_.lock());
@@ -411,6 +412,12 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
                 if (!--state->left)
                     wakeup.notify_one();
             });
+    };
+
+    for (auto & path : paths)
+        pool.enqueue(std::bind(doQuery, path));
+
+    pool.process();
 
     while (true) {
         auto state(state_.lock());

From 9971d875a462bc393724220f468825eca9e5c38a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2017 17:20:47 +0200
Subject: [PATCH 0621/2196] Fix building on clang

https://hydra.nixos.org/build/62945761
---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 51afbac7fc7..9f8edc826a0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2400,10 +2400,10 @@ void DerivationGoal::writeStructuredAttrs()
             }
 
             if (value.is_null())
-                return "''";
+                return std::string("''");
 
             if (value.is_boolean())
-                return value.get<bool>() ? "1" : "";
+                return value.get<bool>() ? std::string("1") : std::string("");
 
             return {};
         };

From 0f9a7225ab47fe313db41f4019177be8fb84a1dd Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Wed, 25 Oct 2017 16:51:45 +0100
Subject: [PATCH 0622/2196] respect SIGINT in nix copy during the paths queries
 #1629

---
 src/libstore/store-api.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 82e0bda5468..3631e1b3003 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -392,6 +392,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
     ThreadPool pool;
 
     auto doQuery = [&](const Path & path ) {
+        checkInterrupt();
         queryPathInfo(path,
             [path, &state_, &wakeup](ref<ValidPathInfo> info) {
                 auto state(state_.lock());

From e5c499b833a329876649f7efc1cbdc7b9a64092c Mon Sep 17 00:00:00 2001
From: Orivej Desh <orivej@gmx.fr>
Date: Thu, 26 Oct 2017 18:49:26 +0000
Subject: [PATCH 0623/2196] Describe arithmetic operators

---
 doc/manual/expressions/language-operators.xml | 40 ++++++++++++++-----
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml
index a3323ced4c5..f1f75093461 100644
--- a/doc/manual/expressions/language-operators.xml
+++ b/doc/manual/expressions/language-operators.xml
@@ -40,6 +40,11 @@ weakest binding).</para>
         <entry>Call function <replaceable>e1</replaceable> with
         argument <replaceable>e2</replaceable>.</entry>
       </row>
+      <row>
+        <entry><literal>-</literal> <replaceable>e</replaceable></entry>
+        <entry>none</entry>
+        <entry>Arithmetic negation.</entry>
+      </row>
       <row>
         <entry><replaceable>e</replaceable> <literal>?</literal>
         <replaceable>attrpath</replaceable></entry>
@@ -55,13 +60,24 @@ weakest binding).</para>
         <entry>List concatenation.</entry>
       </row>
       <row>
-        <entry><replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable></entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable>
+        </entry>
         <entry>left</entry>
-        <entry>String or path concatenation.</entry>
+        <entry>Arithmetic multiplication and division.</entry>
       </row>
       <row>
-        <entry><literal>!</literal> <replaceable>e</replaceable></entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable>
+        </entry>
         <entry>left</entry>
+        <entry>Arithmetic addition and subtraction.  String or path concatenation (only by <literal>+</literal>).</entry>
+      </row>
+      <row>
+        <entry><literal>!</literal> <replaceable>e</replaceable></entry>
+        <entry>none</entry>
         <entry>Boolean negation.</entry>
       </row>
       <row>
@@ -75,16 +91,22 @@ weakest binding).</para>
         attributes).</entry>
       </row>
       <row>
-        <entry><replaceable>e1</replaceable> <literal>==</literal>
-        <replaceable>e2</replaceable></entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>&lt;</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>&gt;=</literal> <replaceable>e2</replaceable>
+        </entry>
         <entry>none</entry>
-        <entry>Equality.</entry>
+        <entry>Arithmetic comparison.</entry>
       </row>
       <row>
-        <entry><replaceable>e1</replaceable> <literal>!=</literal>
-        <replaceable>e2</replaceable></entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>,
+          <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable>
+        </entry>
         <entry>none</entry>
-        <entry>Inequality.</entry>
+        <entry>Equality and inequality.</entry>
       </row>
       <row>
         <entry><replaceable>e1</replaceable> <literal>&amp;&amp;</literal>

From 15457c56731537e0d424d5feb3f3fcacba39b99b Mon Sep 17 00:00:00 2001
From: Orivej Desh <orivej@gmx.fr>
Date: Thu, 26 Oct 2017 18:50:30 +0000
Subject: [PATCH 0624/2196] Describe "with" scoping precedence

---
 doc/manual/expressions/language-constructs.xml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index fe69dba837a..2f0027d479c 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -333,7 +333,20 @@ with (import ./definitions.nix); ...</programlisting>
 
 makes all attributes defined in the file
 <filename>definitions.nix</filename> available as if they were defined
-locally in a <literal>rec</literal>-expression.</para>
+locally in a <literal>let</literal>-expression.</para>
+
+<para>The bindings introduced by <literal>with</literal> do not shadow bindings
+introduced by other means, e.g.
+
+<programlisting>
+let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...</programlisting>
+
+establishes the same scope as
+
+<programlisting>
+let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...</programlisting>
+
+</para>
 
 </simplesect>
 

From 842ce8bafd89a896680d8614ffb2b08b6b38452a Mon Sep 17 00:00:00 2001
From: Orivej Desh <orivej@gmx.fr>
Date: Thu, 26 Oct 2017 18:50:59 +0000
Subject: [PATCH 0625/2196] Clarify indented string escaping rules

---
 doc/manual/expressions/language-values.xml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml
index 67da688a4fc..bb2090c881f 100644
--- a/doc/manual/expressions/language-values.xml
+++ b/doc/manual/expressions/language-values.xml
@@ -98,13 +98,17 @@ configureFlags = "
 
     <para>Since <literal>${</literal> and <literal>''</literal> have
     special meaning in indented strings, you need a way to quote them.
-    <literal>${</literal> can be escaped by prefixing it with
+    <literal>$</literal> can be escaped by prefixing it with
     <literal>''</literal> (that is, two single quotes), i.e.,
-    <literal>''${</literal>.  <literal>''</literal> can be escaped by
+    <literal>''$</literal>. <literal>''</literal> can be escaped by
     prefixing it with <literal>'</literal>, i.e.,
-    <literal>'''</literal>.  Finally, linefeed, carriage-return and
-    tab characters can be written as <literal>''\n</literal>,
-    <literal>''\r</literal>, <literal>''\t</literal>.</para>
+    <literal>'''</literal>. <literal>$</literal> removes any special meaning
+    from the following <literal>$</literal>. Linefeed, carriage-return and tab
+    characters can be written as <literal>''\n</literal>,
+    <literal>''\r</literal>, <literal>''\t</literal>, and <literal>''\</literal>
+    escapes any other character.
+
+    </para>
 
     <para>Indented strings are primarily useful in that they allow
     multi-line string literals to follow the indentation of the

From c1ae18941a222d51a47d4971f4bc4cbc5dea449a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 29 Oct 2017 17:11:22 +0100
Subject: [PATCH 0626/2196] nix repl :t: Indicate strings that have a context

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 78f6b001052..548537b72ce 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -149,7 +149,7 @@ string showType(const Value & v)
     switch (v.type) {
         case tInt: return "an integer";
         case tBool: return "a boolean";
-        case tString: return "a string";
+        case tString: return v.string.context ? "a string with context" : "a string";
         case tPath: return "a path";
         case tNull: return "null";
         case tAttrs: return "a set";

From 65b5f177b5fbb1b0778ede047a13a3cee9c59cfe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 09:57:38 +0100
Subject: [PATCH 0627/2196] builtins.fetchgit: Support a "name" attribute

The "name" attribute defaults to "source", which we should use for all
similar functions (e.g. fetchTarball and in Hydra) to ensure that we
get a consistent store path regardless of how the tree is fetched.

"source" is not necessarily a correct label, but using an empty name
is problematic: you get an ugly store path ending in a dash, and it's
impossible to have a fixed-output derivation that produces that path
because ".drv" is not a valid store name.

Fixes #904.
---
 src/libexpr/primops/fetchgit.cc | 26 ++++++++++++++++----------
 src/libexpr/primops/fetchgit.hh |  3 ++-
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 9db4adbc9da..adc58770233 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -8,10 +8,13 @@
 
 #include <regex>
 
+using namespace std::string_literals;
+
 namespace nix {
 
 Path exportGit(ref<Store> store, const std::string & uri,
-    const std::string & ref, const std::string & rev)
+    const std::string & ref, const std::string & rev,
+    const std::string & name)
 {
     if (rev != "") {
         std::regex revRegex("^[0-9a-fA-F]{40}$");
@@ -51,12 +54,12 @@ Path exportGit(ref<Store> store, const std::string & uri,
     }
 
     // FIXME: check whether rev is an ancestor of ref.
-    std::string commitHash =
-        rev != "" ? rev : chomp(readFile(localRefFile));
+    std::string commitHash = rev != "" ? rev : chomp(readFile(localRefFile));
 
     printTalkative("using revision %s of repo '%s'", uri, commitHash);
 
-    Path storeLink = cacheDir + "/" + commitHash + ".link";
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + commitHash).to_string(Base32, false);
+    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
     PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink));
 
     if (pathExists(storeLink)) {
@@ -76,7 +79,7 @@ Path exportGit(ref<Store> store, const std::string & uri,
 
     runProgram("tar", true, { "x", "-C", tmpDir }, tar);
 
-    auto storePath = store->addToStore("git-export", tmpDir);
+    auto storePath = store->addToStore(name, tmpDir);
 
     replaceSymlink(storePath, storeLink);
 
@@ -91,6 +94,7 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
     std::string url;
     std::string ref = "master";
     std::string rev;
+    std::string name = "source";
 
     state.forceValue(*args[0]);
 
@@ -99,16 +103,18 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
         state.forceAttrs(*args[0], pos);
 
         for (auto & attr : *args[0]->attrs) {
-            string name(attr.name);
-            if (name == "url") {
+            string n(attr.name);
+            if (n == "url") {
                 PathSet context;
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
                 if (hasPrefix(url, "/")) url = "file://" + url;
             }
-            else if (name == "ref")
+            else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (name == "rev")
+            else if (n == "rev")
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchgit', at %s", attr.name, *attr.pos);
         }
@@ -119,7 +125,7 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    Path storePath = exportGit(state.store, url, ref, rev);
+    Path storePath = exportGit(state.store, url, ref, rev, name);
 
     mkString(v, storePath, PathSet({storePath}));
 }
diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh
index ff228f3b3c6..4bf34b76f8d 100644
--- a/src/libexpr/primops/fetchgit.hh
+++ b/src/libexpr/primops/fetchgit.hh
@@ -9,6 +9,7 @@ namespace nix {
 class Store;
 
 Path exportGit(ref<Store> store, const std::string & uri,
-    const std::string & ref, const std::string & rev = "");
+    const std::string & ref, const std::string & rev = "",
+    const std::string & name = "");
 
 }

From f25791c196e7bcf76aa76e143a679fe731f77513 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 10:05:43 +0100
Subject: [PATCH 0628/2196] builtins.fetchgit: Don't mess up the progress
 indicator

---
 src/libexpr/primops/fetchgit.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index adc58770233..4c5799bfed5 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -29,8 +29,6 @@ Path exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    //Activity act(*logger, lvlInfo, format("fetching Git repository '%s'") % uri);
-
     std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, ref)).to_string(Base32, false);
 
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
@@ -42,7 +40,11 @@ Path exportGit(ref<Store> store, const std::string & uri,
     if (stat(localRefFile.c_str(), &st) != 0 ||
         st.st_mtime < now - settings.tarballTtl)
     {
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--force", "--", uri, ref + ":" + localRef });
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, ref + ":" + localRef });
 
         struct timeval times[2];
         times[0].tv_sec = now;

From 66ddbef7547a2b5430c6d87e2d8d52977f7bf6f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 10:16:19 +0100
Subject: [PATCH 0629/2196] fetchurl/fetchTarball: Respect name changes

The computation of urlHash didn't take the name into account, so
subsequent fetchurl calls with the same URL but a different name would
resolve to the same cached store path.
---
 src/libexpr/primops.cc   | 2 +-
 src/libstore/download.cc | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6a50c99345e..43dc2008019 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1911,7 +1911,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 {
     string url;
     Hash expectedHash;
-    string name;
+    string name = "";
 
     state.forceValue(*args[0]);
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 608b8fd399b..579a5e8c1b5 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -23,6 +23,8 @@
 #include <cmath>
 #include <random>
 
+using namespace std::string_literals;
+
 namespace nix {
 
 double getTime()
@@ -604,7 +606,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path cacheDir = getCacheDir() + "/nix/tarballs";
     createDirs(cacheDir);
 
-    string urlHash = hashString(htSHA256, url).to_string(Base32, false);
+    string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
 
     Path dataFile = cacheDir + "/" + urlHash + ".info";
     Path fileLink = cacheDir + "/" + urlHash + "-file";

From 23ce4b33932260ad1f1bca4978ea274a1da1e0ad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 10:19:33 +0100
Subject: [PATCH 0630/2196] fetchTarball: Use "source" as the default name

This ensures that it produces the same output as fetchgit:

  $ nix eval --raw '(builtins.fetchgit https://github.com/NixOS/patchelf.git)'
  /nix/store/ghigrkw02l440g8vfxa9wj4c3zpfmw99-source

  $ nix eval --raw '(fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)'
  /nix/store/ghigrkw02l440g8vfxa9wj4c3zpfmw99-source
---
 src/libexpr/primops.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 43dc2008019..a01af05e3c7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1907,11 +1907,11 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
 
 
 void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack)
+    const string & who, bool unpack, const std::string & defaultName)
 {
     string url;
     Hash expectedHash;
-    string name = "";
+    string name = defaultName;
 
     state.forceValue(*args[0]);
 
@@ -1947,13 +1947,13 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
 static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    fetch(state, pos, args, v, "fetchurl", false);
+    fetch(state, pos, args, v, "fetchurl", false, "");
 }
 
 
 static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    fetch(state, pos, args, v, "fetchTarball", true);
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
 }
 
 

From 049322702b5629c1058249370d38ae8d23f47bdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 10:25:08 +0100
Subject: [PATCH 0631/2196] fetchgit -> fetchGit

Almost all other primops are camelCase so no reason not to use that
here.
---
 doc/manual/release-notes/rl-1.12.xml | 2 +-
 src/libexpr/primops/fetchgit.cc      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index adfbaa157f2..d8fc2bcf89b 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -184,7 +184,7 @@
   </listitem>
 
   <listitem>
-    <para><function>builtins.fetchgit</function>.
+    <para><function>builtins.fetchGit</function>.
     (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para>
   </listitem>
 
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 4c5799bfed5..18107f5cd41 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -88,10 +88,10 @@ Path exportGit(ref<Store> store, const std::string & uri,
     return storePath;
 }
 
-static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     // FIXME: cut&paste from fetch().
-    if (state.restricted) throw Error("'fetchgit' is not allowed in restricted mode");
+    if (state.restricted) throw Error("'fetchGit' is not allowed in restricted mode");
 
     std::string url;
     std::string ref = "master";
@@ -118,7 +118,7 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchgit', at %s", attr.name, *attr.pos);
+                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())
@@ -132,6 +132,6 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(v, storePath, PathSet({storePath}));
 }
 
-static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit);
+static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
 
 }

From f9686885be54a9b0f8947713a414de4ad3182037 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 11:00:59 +0100
Subject: [PATCH 0632/2196] enable-http2 -> http2

---
 src/libexpr/primops.cc  | 2 +-
 src/libstore/globals.hh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a01af05e3c7..22925ba4de1 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1937,7 +1937,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    if (state.restricted && !expectedHash)
+    if (state.restricted)
         throw Error(format("'%1%' is not allowed in restricted mode") % who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 88052732279..538273b546b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -271,7 +271,7 @@ public:
         "Number of parallel HTTP connections.",
         {"binary-caches-parallel-connections"}};
 
-    Setting<bool> enableHttp2{this, true, "enable-http2",
+    Setting<bool> enableHttp2{this, true, "http2",
         "Whether to enable HTTP/2 support."};
 
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",

From e38382895deef4c136b7425f480884d122f3c5ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 11:49:03 +0100
Subject: [PATCH 0633/2196] builtins.fetchGit: Return an attrset with revision
 info

This adds rev, shortRev and revCount attributes, equal to what Hydra
provides. E.g.

  $ nix eval '(fetchGit https://github.com/NixOS/patchelf.git)'
  { outPath = "/nix/store/ghigrkw02l440g8vfxa9wj4c3zpfmw99-source"; rev = "29c085fd9d3fc972f75b3961905d6b4ecce7eb2b"; revCount = 303; shortRev = "29c085f"; }
---
 src/libexpr/parser.y            |  2 +-
 src/libexpr/primops/fetchgit.cc | 58 ++++++++++++++++++++++++---------
 src/libexpr/primops/fetchgit.hh | 12 +++++--
 3 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 669312bb7cf..eee31522830 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -667,7 +667,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         try {
             if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git"))
                 // FIXME: support specifying revision/branch
-                res = { true, exportGit(store, elem.second, "master") };
+                res = { true, exportGit(store, elem.second, "master").storePath };
             else
                 res = { true, getDownloader()->downloadCached(store, elem.second, true) };
         } catch (DownloadError & e) {
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 18107f5cd41..38bffd8dbdb 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -1,3 +1,4 @@
+#include "fetchgit.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "download.hh"
@@ -8,11 +9,13 @@
 
 #include <regex>
 
+#include <nlohmann/json.hpp>
+
 using namespace std::string_literals;
 
 namespace nix {
 
-Path exportGit(ref<Store> store, const std::string & uri,
+GitInfo exportGit(ref<Store> store, const std::string & uri,
     const std::string & ref, const std::string & rev,
     const std::string & name)
 {
@@ -56,36 +59,56 @@ Path exportGit(ref<Store> store, const std::string & uri,
     }
 
     // FIXME: check whether rev is an ancestor of ref.
-    std::string commitHash = rev != "" ? rev : chomp(readFile(localRefFile));
+    GitInfo gitInfo;
+    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
+    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
-    printTalkative("using revision %s of repo '%s'", uri, commitHash);
+    printTalkative("using revision %s of repo '%s'", uri, gitInfo.rev);
 
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + commitHash).to_string(Base32, false);
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";
     PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink));
 
-    if (pathExists(storeLink)) {
-        auto storePath = readLink(storeLink);
-        store->addTempRoot(storePath);
-        if (store->isValidPath(storePath)) {
-            return storePath;
+    try {
+        // FIXME: doesn't handle empty lines
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["uri"] == uri && json["name"] == name && json["rev"] == gitInfo.rev);
+
+        gitInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(gitInfo.storePath)) {
+            gitInfo.revCount = json["revCount"];
+            return gitInfo;
         }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
     }
 
     // FIXME: should pipe this, or find some better way to extract a
     // revision.
-    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash });
+    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev });
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
     runProgram("tar", true, { "x", "-C", tmpDir }, tar);
 
-    auto storePath = store->addToStore(name, tmpDir);
+    gitInfo.storePath = store->addToStore(name, tmpDir);
+
+    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
+
+    nlohmann::json json;
+    json["storePath"] = gitInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["rev"] = gitInfo.rev;
+    json["revCount"] = gitInfo.revCount;
 
-    replaceSymlink(storePath, storeLink);
+    writeFile(storeLink, json.dump());
 
-    return storePath;
+    return gitInfo;
 }
 
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -127,9 +150,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    Path storePath = exportGit(state.store, url, ref, rev, name);
+    auto gitInfo = exportGit(state.store, url, ref, rev, name);
 
-    mkString(v, storePath, PathSet({storePath}));
+    state.mkAttrs(v, 8);
+    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    v.attrs->sort();
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh
index 4bf34b76f8d..056b6fcbe78 100644
--- a/src/libexpr/primops/fetchgit.hh
+++ b/src/libexpr/primops/fetchgit.hh
@@ -2,13 +2,21 @@
 
 #include <string>
 
-#include "ref.hh"
+#include "util.hh"
 
 namespace nix {
 
 class Store;
 
-Path exportGit(ref<Store> store, const std::string & uri,
+struct GitInfo
+{
+    Path storePath;
+    std::string rev;
+    std::string shortRev;
+    uint64_t revCount = 0;
+};
+
+GitInfo exportGit(ref<Store> store, const std::string & uri,
     const std::string & ref, const std::string & rev = "",
     const std::string & name = "");
 

From f1c555cef870654cdaf232b5d08fdbba0bf06add Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 11:58:14 +0100
Subject: [PATCH 0634/2196] fetchurl/fetchTarball are *not* allowed in
 restricted mode

Accidentally committed this change as part of
f9686885be54a9b0f8947713a414de4ad3182037.

Restricted mode != pure mode.
---
 doc/manual/release-notes/rl-1.12.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index d8fc2bcf89b..609dcef6b49 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -223,8 +223,7 @@
     <para><function>builtins.fetchurl</function> and
     <function>builtins.fetchTarball</function> now support
     <varname>sha256</varname> and <varname>name</varname>
-    attributes. Using these functions is now allowed in restricted
-    mode if a hash is supplied.</para>
+    attributes.</para>
   </listitem>
 
   <listitem>

From 812e027e1d5a4f83394069edd67bdf8404ffa2bb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 12:39:59 +0100
Subject: [PATCH 0635/2196] Add option allowed-uris

This allows network access in restricted eval mode.
---
 doc/manual/command-ref/conf-file.xml | 18 +++++++++++++++++-
 doc/manual/release-notes/rl-1.12.xml |  8 ++++++++
 src/libexpr/eval.cc                  | 20 ++++++++++++++++++++
 src/libexpr/eval.hh                  |  2 ++
 src/libexpr/primops.cc               |  3 +--
 src/libexpr/primops/fetchgit.cc      |  7 ++++---
 src/libstore/globals.hh              |  4 +++-
 tests/restricted.sh                  | 12 ++++++++++++
 8 files changed, 67 insertions(+), 7 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 6b90083f087..fb4d8cefc4d 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -563,7 +563,8 @@ password <replaceable>my-password</replaceable>
       <para>If set to <literal>true</literal>, the Nix evaluator will
       not allow access to any files outside of the Nix search path (as
       set via the <envar>NIX_PATH</envar> environment variable or the
-      <option>-I</option> option). The default is
+      <option>-I</option> option), or to URIs outside of
+      <option>allowed-uri</option>. The default is
       <literal>false</literal>.</para>
 
     </listitem>
@@ -571,6 +572,21 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term>
+
+    <listitem>
+
+      <para>A list of URI prefixes to which access is allowed in
+      restricted evaluation mode. For example, when set to
+      <literal>https://github.com/NixOS</literal>, builtin functions
+      such as <function>fetchGit</function> are allowed to access
+      <literal>https://github.com/NixOS/patchelf.git</literal>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
 
     <listitem>
diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 609dcef6b49..7c9a8b75ece 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -418,6 +418,14 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     through the MELPA package repository.</para>
   </listitem>
 
+  <listitem>
+    <para>In restricted evaluation mode
+    (<option>--restrict-eval</option>), builtin functions that
+    download from the network (such as <function>fetchGit</function>)
+    are permitted to fetch underneath the list of URI prefixes
+    specified in the option <option>allowed-uris</option>.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 548537b72ce..63de2d60a14 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -355,6 +355,26 @@ Path EvalState::checkSourcePath(const Path & path_)
 }
 
 
+void EvalState::checkURI(const std::string & uri)
+{
+    if (!restricted) return;
+
+    /* 'uri' should be equal to a prefix, or in a subdirectory of a
+       prefix. Thus, the prefix https://github.co does not permit
+       access to https://github.com. Note: this allows 'http://' and
+       'https://' as prefixes for any http/https URI. */
+    for (auto & prefix : settings.allowedUris.get())
+        if (uri == prefix ||
+            (uri.size() > prefix.size()
+            && prefix.size() > 0
+            && hasPrefix(uri, prefix)
+            && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
+            return;
+
+    throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri);
+}
+
+
 void EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 04a36b14cef..f0ab1435bff 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -110,6 +110,8 @@ public:
 
     Path checkSourcePath(const Path & path);
 
+    void checkURI(const std::string & uri);
+
     /* Parse a Nix expression from the specified file. */
     Expr * parseExprFromFile(const Path & path);
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 22925ba4de1..cd0dfbc03e9 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1937,8 +1937,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    if (state.restricted)
-        throw Error(format("'%1%' is not allowed in restricted mode") % who);
+    state.checkURI(url);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
     mkString(v, res, PathSet({res}));
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 38bffd8dbdb..81b64190059 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -113,9 +113,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    // FIXME: cut&paste from fetch().
-    if (state.restricted) throw Error("'fetchGit' is not allowed in restricted mode");
-
     std::string url;
     std::string ref = "master";
     std::string rev;
@@ -150,6 +147,10 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
+    // FIXME: git externals probably can be used to bypass the URI
+    // whitelist. Ah well.
+    state.checkURI(url);
+
     auto gitInfo = exportGit(state.store, url, ref, rev, name);
 
     state.mkAttrs(v, 8);
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 538273b546b..a4aa842d70f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -225,7 +225,7 @@ public:
 
     Setting<bool> restrictEval{this, false, "restrict-eval",
         "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and to disallow fetching files from the network."};
+        "and network access to the URI prefixes listed in 'allowed-uris'."};
 
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
@@ -353,6 +353,8 @@ public:
     Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
         "Stop deleting garbage when free disk space is above the specified amount."};
 
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 };
 
 
diff --git a/tests/restricted.sh b/tests/restricted.sh
index 19096a9f8dd..a297847cc8e 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -16,3 +16,15 @@ nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/b
 (! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
 nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
 
+p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)")
+cmp $p restricted.sh
+
+(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval)
+
+(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/")
+
+nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh"
+
+(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
+(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
+(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval)

From a5c392a80e2965f0bf4855309a1ae326ff625ba2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 12:55:17 +0100
Subject: [PATCH 0636/2196] fetchGit: Fix broken assertion

Different URIs can map to the same cache entry if they have the same
revision.
---
 src/libexpr/primops/fetchgit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 81b64190059..2b1d366ff4a 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -73,7 +73,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         // FIXME: doesn't handle empty lines
         auto json = nlohmann::json::parse(readFile(storeLink));
 
-        assert(json["uri"] == uri && json["name"] == name && json["rev"] == gitInfo.rev);
+        assert(json["name"] == name && json["rev"] == gitInfo.rev);
 
         gitInfo.storePath = json["storePath"];
 

From 63c80ae26fcde8cc2e18a01e325ef214de084517 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 13:18:28 +0100
Subject: [PATCH 0637/2196] Make "fetchGit /path" work

---
 src/libexpr/primops/fetchgit.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 2b1d366ff4a..4af5301247b 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -117,6 +117,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     std::string ref = "master";
     std::string rev;
     std::string name = "source";
+    PathSet context;
 
     state.forceValue(*args[0]);
 
@@ -126,11 +127,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
 
         for (auto & attr : *args[0]->attrs) {
             string n(attr.name);
-            if (n == "url") {
-                PathSet context;
+            if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-                if (hasPrefix(url, "/")) url = "file://" + url;
-            }
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
@@ -145,7 +143,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
-        url = state.forceStringNoCtx(*args[0], pos);
+        url = state.coerceToString(pos, *args[0], context, false, false);
+
+    if (hasPrefix(url, "/")) url = "file://" + url;
 
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.

From 12991152be43f63634e48a7d118040e485125df4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 13:31:05 +0100
Subject: [PATCH 0638/2196] nix-build: Fix --hash

---
 src/nix-build/nix-build.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 8e56e5a46b1..21b0a18dd88 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -167,6 +167,9 @@ void mainWrapped(int argc, char * * argv)
             buildMode = bmRepair;
         }
 
+        else if (*arg == "--hash")
+            buildMode = bmHash;
+
         else if (*arg == "--run-env") // obsolete
             runEnv = true;
 

From 07d2c6d213660e178fcdf128ea6f1f36d9d8a85d Mon Sep 17 00:00:00 2001
From: Tyson Whitehead <twhitehead@gmail.com>
Date: Fri, 27 Oct 2017 12:15:31 -0400
Subject: [PATCH 0639/2196] Fix (highly unlikely) race condition in readLink

Used to determine symlink size with stat and value with readlink.
This could technically result in garbage if symlink changed between
calls.  Also gets around the broken stat implementation in our
network filesystem (returns size + 1 giving a byte of garbage).
---
 src/libutil/util.cc | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 9346d5dc4cf..16c2c97aac9 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -216,18 +216,17 @@ bool pathExists(const Path & path)
 Path readLink(const Path & path)
 {
     checkInterrupt();
-    struct stat st = lstat(path);
-    if (!S_ISLNK(st.st_mode))
-        throw Error(format("'%1%' is not a symlink") % path);
-    auto bufSize = std::max(st.st_size, (off_t) PATH_MAX + 1);
-    char buf[bufSize];
-    ssize_t rlsize = readlink(path.c_str(), buf, bufSize);
-    if (rlsize == -1)
-        throw SysError(format("reading symbolic link '%1%'") % path);
-    else if (rlsize > bufSize)
-        throw Error(format("symbolic link '%1%' size overflow %2% > %3%")
-            % path % rlsize % bufSize);
-    return string(buf, rlsize);
+    for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
+        char buf[bufSize];
+        ssize_t rlSize = readlink(path.c_str(), buf, bufSize);
+        if (rlSize == -1)
+            if (errno == EINVAL)
+                throw Error(format("'%1%' is not a symlink") % path);
+            else
+                throw SysError(format("reading symbolic link '%1%'") % path);
+        else if (rlSize < bufSize)
+            return string(buf, rlSize);
+    }
 }
 
 

From f90f660b243866b8860eeb24cc4a345d32cc7ce7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 17:56:44 +0100
Subject: [PATCH 0640/2196] Don't parse "x:x" as a URI

URIs now have to contain "://" or start with "channel:".
---
 src/libexpr/lexer.l           | 2 +-
 tests/lang/lib.nix            | 2 +-
 tests/lang/parse-okay-url.nix | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 28a0a6a8789..828356bbf44 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -90,7 +90,7 @@ FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
 PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
 HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
 SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
+URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:\/\/[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+|channel\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
 
 %%
diff --git a/tests/lang/lib.nix b/tests/lang/lib.nix
index 028a538314b..1c63b2f31d6 100644
--- a/tests/lang/lib.nix
+++ b/tests/lang/lib.nix
@@ -49,7 +49,7 @@ rec {
     if comp (head list2) (head list1) then [(head list2)] ++ mergeLists comp list1 (tail list2) else
     [(head list1)] ++ mergeLists comp (tail list1) list2;
 
-  id = x: x;
+  id = x:x; # sic
 
   const = x: y: x;
 
diff --git a/tests/lang/parse-okay-url.nix b/tests/lang/parse-okay-url.nix
index fce3b13ee64..fb74d66f092 100644
--- a/tests/lang/parse-okay-url.nix
+++ b/tests/lang/parse-okay-url.nix
@@ -1,7 +1,8 @@
-[ x:x
+[
   https://svn.cs.uu.nl:12443/repos/trace/trunk
   http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2
   http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz
   http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz
   ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz
+  channel:nixos-17.09
 ]

From 4a4a009f78d7267d58a1dbd95f70f12ee3fe89f2 Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Mon, 30 Oct 2017 17:25:41 +0100
Subject: [PATCH 0641/2196] Allow optional localhost network access to
 sandboxed derivations

This will allow bind and connect to 127.0.0.1, which can reduce purity/
security (if you're running a vulnerable service on localhost) but is
also needed for a ton of test suites, so I'm leaving it turned off by
default but allowing certain derivations to turn it on as needed.

It also allows DNS resolution of arbitrary hostnames but I haven't found
a way to avoid that. In principle I'd just want to allow resolving
localhost but that doesn't seem to be possible.

I don't think this belongs under `build-use-sandbox = relaxed` because we
want it on Hydra and I don't think it's the end of the world.
---
 src/libstore/build.cc            | 12 +++++++++---
 src/libstore/sandbox-defaults.sb | 26 ++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 88c51654614..9069d9b06e0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2774,10 +2774,10 @@ void DerivationGoal::runChild()
                     sandboxProfile += "(deny default (with no-log))\n";
                 }
 
-                sandboxProfile += "(import \"sandbox-defaults.sb\")";
+                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
 
                 if (fixedOutput)
-                    sandboxProfile += "(import \"sandbox-network.sb\")";
+                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
                 /* Our rwx outputs */
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
@@ -2820,7 +2820,7 @@ void DerivationGoal::runChild()
 
                 sandboxProfile += additionalSandboxProfile;
             } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")";
+                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
 
             debug("Generated sandbox profile:");
             debug(sandboxProfile);
@@ -2829,6 +2829,8 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
+            bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1";
+
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
             Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
@@ -2844,6 +2846,10 @@ void DerivationGoal::runChild()
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
             args.push_back("-D");
             args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
+            if (allowLocalNetworking) {
+                args.push_back("-D");
+                args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+            }
             args.push_back(drv->builder);
         }
 #endif
diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index d63c8f813c9..cf700c62c6f 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -30,6 +30,29 @@
 ; Without this line clang cannot write to /dev/null, breaking some configure tests.
 (allow file-read-metadata (literal "/dev"))
 
+; Many packages like to do local networking in their test suites, but let's only
+; allow it if the package explicitly asks for it.
+(if (param "_ALLOW_LOCAL_NETWORKING")
+    (begin
+      (allow network* (local ip) (local tcp) (local udp))
+
+      ; Allow access to /etc/resolv.conf (which is a symlink to
+      ; /private/var/run/resolv.conf).
+      ; TODO: deduplicate with sandbox-network.sb
+      (allow file-read-metadata
+             (literal "/var")
+             (literal "/etc")
+             (literal "/etc/resolv.conf")
+             (literal "/private/etc/resolv.conf"))
+
+      (allow file-read*
+             (literal "/private/var/run/resolv.conf"))
+
+      ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on
+      (allow file-read-metadata (literal "/etc/hosts"))
+      (allow file-read*         (literal "/private/etc/hosts"))
+      (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))))
+
 ; Standard devices.
 (allow file*
        (literal "/dev/null")
@@ -54,5 +77,4 @@
 (allow file-read-metadata
        (literal "/etc")
        (literal "/var")
-       (literal "/private/var/tmp")
-       )
+       (literal "/private/var/tmp"))

From 72cd52c3cdd1fc465fade6d553b3823aca9f8b6e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Oct 2017 19:57:40 +0100
Subject: [PATCH 0642/2196] builtins.fetchgit: Support importing a working tree

For example, you can write

  src = fetchgit ./.;

and if ./. refers to an unclean working tree, that tree will be copied
to the Nix store. This removes the need for "cleanSource".
---
 src/libexpr/parser.y            |  2 +-
 src/libexpr/primops/fetchgit.cc | 49 +++++++++++++++++++++++++++++----
 src/libexpr/primops/fetchgit.hh |  3 +-
 src/libutil/archive.cc          |  2 +-
 src/libutil/archive.hh          |  7 -----
 src/libutil/hash.hh             |  2 --
 src/libutil/util.hh             |  6 ++++
 7 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index eee31522830..7e63dc89f60 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -667,7 +667,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         try {
             if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git"))
                 // FIXME: support specifying revision/branch
-                res = { true, exportGit(store, elem.second, "master").storePath };
+                res = { true, exportGit(store, elem.second).storePath };
             else
                 res = { true, getDownloader()->downloadCached(store, elem.second, true) };
         } catch (DownloadError & e) {
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 4af5301247b..dc526485fa3 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -16,9 +16,48 @@ using namespace std::string_literals;
 namespace nix {
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    const std::string & ref, const std::string & rev,
+    std::experimental::optional<std::string> ref, const std::string & rev,
     const std::string & name)
 {
+    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
+
+        bool clean = true;
+
+        try {
+            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
+        } catch (ExecError e) {
+            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            clean = false;
+        }
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked
+               files. */
+
+            GitInfo gitInfo;
+            gitInfo.rev = "0000000000000000000000000000000000000000";
+            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                auto st = lstat(p);
+                if (S_ISDIR(st.st_mode)) return true;
+                std::string file(p, uri.size() + 1);
+                return files.count(file);
+            };
+
+            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+
+            return gitInfo;
+        }
+    }
+
+    if (!ref) ref = "master";
+
     if (rev != "") {
         std::regex revRegex("^[0-9a-fA-F]{40}$");
         if (!std::regex_match(rev, revRegex))
@@ -32,7 +71,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, ref)).to_string(Base32, false);
+    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
 
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
@@ -47,7 +86,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
 
         struct timeval times[2];
         times[0].tv_sec = now;
@@ -114,7 +153,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::string ref = "master";
+    std::experimental::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
@@ -145,7 +184,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (hasPrefix(url, "/")) url = "file://" + url;
+    if (!isUri(url)) url = absPath(url);
 
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh
index 056b6fcbe78..818ab7102e9 100644
--- a/src/libexpr/primops/fetchgit.hh
+++ b/src/libexpr/primops/fetchgit.hh
@@ -17,7 +17,8 @@ struct GitInfo
 };
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    const std::string & ref, const std::string & rev = "",
+    std::experimental::optional<std::string> ref = {},
+    const std::string & rev = "",
     const std::string & name = "");
 
 }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index ea1deb924e6..f71229d8fdd 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -29,7 +29,7 @@ const std::string narVersionMagic1 = "nix-archive-1";
 
 static string caseHackSuffix = "~nix~case~hack~";
 
-PathFilter defaultPathFilter;
+PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
 static void dumpContents(const Path & path, size_t size,
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 607ebf8b28f..8a15e849c7b 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -44,13 +44,6 @@ namespace nix {
 
      `+' denotes string concatenation. */
 
-struct PathFilter
-{
-    virtual ~PathFilter() { }
-    virtual bool operator () (const Path & path) { return true; }
-};
-
-extern PathFilter defaultPathFilter;
 
 void dumpPath(const Path & path, Sink & sink,
     PathFilter & filter = defaultPathFilter);
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index d83049b0236..fd7a61df8e4 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -93,8 +93,6 @@ Hash hashFile(HashType ht, const Path & path);
 
 /* Compute the hash of the given path.  The hash is defined as
    (essentially) hashString(ht, dumpPath(path)). */
-struct PathFilter;
-extern PathFilter defaultPathFilter;
 typedef std::pair<Hash, unsigned long long> HashResult;
 HashResult hashPath(HashType ht, const Path & path,
     PathFilter & filter = defaultPathFilter);
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fccf5d85480..63a93f2ca6a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -481,4 +481,10 @@ struct MaintainCount
 std::pair<unsigned short, unsigned short> getWindowSize();
 
 
+/* Used in various places. */
+typedef std::function<bool(const Path & path)> PathFilter;
+
+extern PathFilter defaultPathFilter;
+
+
 }

From bc6b3f7e8fa46c183e20a9f28a5e0a7a6a19429d Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Tue, 31 Oct 2017 13:16:51 +0100
Subject: [PATCH 0643/2196] Always allow builds to use unix domain sockets in
 Darwin sandbox

---
 src/libstore/sandbox-defaults.sb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index cf700c62c6f..b4e29c94381 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -22,7 +22,14 @@
 (allow signal (target same-sandbox))
 
 ; Access to /tmp.
-(allow file* process-exec (literal "/tmp") (subpath TMPDIR))
+; The network-outbound/network-inbound ones are for unix domain sockets, which
+; we allow access to in TMPDIR (but if we allow them more broadly, you could in
+; theory escape the sandbox)
+(allow file* process-exec network-outbound network-inbound
+       (literal "/tmp") (subpath TMPDIR))
+
+; Always allow unix domain sockets, since they can't hurt purity or security
+
 
 ; Some packages like to read the system version.
 (allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist"))

From cd532a92510fdc895d45bb768b036921b4a26282 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Oct 2017 16:14:25 +0100
Subject: [PATCH 0644/2196] Fix filterSource

---
 src/libexpr/primops.cc | 46 +++++++++++++-----------------------------
 1 file changed, 14 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index cd0dfbc03e9..e3b5dfb420b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1009,22 +1009,21 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 }
 
 
-struct FilterFromExpr : PathFilter
+static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    EvalState & state;
-    Value & filter;
-    Pos pos;
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    if (!context.empty())
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
 
-    FilterFromExpr(EvalState & state, Value & filter, const Pos & pos)
-        : state(state), filter(filter), pos(pos)
-    {
-    }
+    state.forceValue(*args[0]);
+    if (args[0]->type != tLambda)
+        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
-    bool operator () (const Path & path)
-    {
-        struct stat st;
-        if (lstat(path.c_str(), &st))
-            throw SysError(format("getting attributes of path '%1%'") % path);
+    path = state.checkSourcePath(path);
+
+    PathFilter filter = [&](const Path & path) {
+        auto st = lstat(path);
 
         /* Call the filter function.  The first argument is the path,
            the second is a string indicating the type of the file. */
@@ -1032,7 +1031,7 @@ struct FilterFromExpr : PathFilter
         mkString(arg1, path);
 
         Value fun2;
-        state.callFunction(filter, arg1, fun2, noPos);
+        state.callFunction(*args[0], arg1, fun2, noPos);
 
         Value arg2;
         mkString(arg2,
@@ -1045,24 +1044,7 @@ struct FilterFromExpr : PathFilter
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
-    }
-};
-
-
-static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
-
-    state.forceValue(*args[0]);
-    if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
-
-    FilterFromExpr filter(state, *args[0], pos);
-
-    path = state.checkSourcePath(path);
+    };
 
     Path dstPath = settings.readOnlyMode
         ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first

From 1969f357b7f9abbff82b613b88ae53b84ff4e483 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2017 16:32:53 +0100
Subject: [PATCH 0645/2196] Add fetchMercurial primop

E.g.

  $ nix eval '(fetchMercurial https://www.mercurial-scm.org/repo/hello)'
  { branch = "default"; outPath = "/nix/store/alvb9y1kfz42bjishqmyy3pphnrh1pfa-source"; rev = "82e55d328c8ca4ee16520036c0aaace03a5beb65"; revCount = 1; shortRev = "82e55d328c8c"; }

  $ nix eval '(fetchMercurial { url = https://www.mercurial-scm.org/repo/hello; rev = "0a04b987be5ae354b710cefeba0e2d9de7ad41a9"; })'
  { branch = "default"; outPath = "/nix/store/alvb9y1kfz42bjishqmyy3pphnrh1pfa-source"; rev = "0a04b987be5ae354b710cefeba0e2d9de7ad41a9"; revCount = 0; shortRev = "0a04b987be5a"; }

  $ nix eval '(fetchMercurial /tmp/unclean-hg-tree)'
  { branch = "default"; outPath = "/nix/store/cm750cdw1x8wfpm3jq7mz09r30l9r024-source"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "000000000000"; }
---
 release.nix                           |   2 +-
 shell.nix                             |   3 +
 src/libexpr/primops/fetchMercurial.cc | 188 ++++++++++++++++++++++++++
 src/libexpr/primops/fetchgit.cc       |   3 +-
 src/libstore/download.cc              |   2 +-
 tests/fetchMercurial.sh               |  73 ++++++++++
 tests/local.mk                        |   3 +-
 7 files changed, 269 insertions(+), 5 deletions(-)
 create mode 100644 src/libexpr/primops/fetchMercurial.cc
 create mode 100644 tests/fetchMercurial.sh

diff --git a/release.nix b/release.nix
index a9819925884..f76e7cb9ea0 100644
--- a/release.nix
+++ b/release.nix
@@ -76,7 +76,7 @@ let
           [ curl
             bzip2 xz brotli
             openssl pkgconfig sqlite boehmgc
-
+            mercurial
           ]
           ++ lib.optional stdenv.isLinux libseccomp
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
diff --git a/shell.nix b/shell.nix
index 2b4d0dd201e..02fbd59b12a 100644
--- a/shell.nix
+++ b/shell.nix
@@ -23,6 +23,9 @@ with import ./release-common.nix { inherit pkgs; };
       # For nix-perl
       perl
       perlPackages.DBDSQLite
+
+      # Tests
+      mercurial
     ]
     ++ lib.optional stdenv.isLinux libseccomp;
 
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
new file mode 100644
index 00000000000..391ce45a0a9
--- /dev/null
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -0,0 +1,188 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "download.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
+
+#include <sys/time.h>
+
+#include <regex>
+
+#include <nlohmann/json.hpp>
+
+using namespace std::string_literals;
+
+namespace nix {
+
+struct HgInfo
+{
+    Path storePath;
+    std::string branch;
+    std::string rev;
+    uint64_t revCount = 0;
+};
+
+HgInfo exportMercurial(ref<Store> store, const std::string & uri,
+    std::string rev, const std::string & name)
+{
+    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
+
+        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
+
+        if (!clean) {
+
+            /* This is an unclean working tree. So copy all tracked
+               files. */
+
+            printTalkative("copying unclean Mercurial working tree '%s'", uri);
+
+            HgInfo hgInfo;
+            hgInfo.rev = "0000000000000000000000000000000000000000";
+            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
+
+            auto files = tokenizeString<std::set<std::string>>(
+                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+            PathFilter filter = [&](const Path & p) -> bool {
+                assert(hasPrefix(p, uri));
+                auto st = lstat(p);
+                std::string file(p, uri.size() + 1);
+                if (file == ".hg") return false;
+                // FIXME: filter out directories with no tracked files.
+                if (S_ISDIR(st.st_mode)) return true;
+                return files.count(file);
+            };
+
+            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+
+            return hgInfo;
+        }
+    }
+
+    if (rev == "") rev = "default";
+
+    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
+
+    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
+
+    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
+       do so now. FIXME: don't do this if "rev" is a hash and we
+       fetched it previously */
+    time_t now = time(0);
+    struct stat st;
+    if (stat(stampFile.c_str(), &st) != 0 ||
+        st.st_mtime < now - settings.tarballTtl)
+    {
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
+
+        if (pathExists(cacheDir)) {
+            runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+        } else {
+            createDirs(dirOf(cacheDir));
+            runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
+        }
+
+        writeFile(stampFile, "");
+    }
+
+    auto tokens = tokenizeString<std::vector<std::string>>(
+        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
+    assert(tokens.size() == 3);
+
+    HgInfo hgInfo;
+    hgInfo.rev = tokens[0];
+    hgInfo.revCount = std::stoull(tokens[1]);
+    hgInfo.branch = tokens[2];
+
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
+    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
+
+    try {
+        auto json = nlohmann::json::parse(readFile(storeLink));
+
+        assert(json["name"] == name && json["rev"] == hgInfo.rev);
+
+        hgInfo.storePath = json["storePath"];
+
+        if (store->isValidPath(hgInfo.storePath)) {
+            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
+            return hgInfo;
+        }
+
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT) throw;
+    }
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir, true);
+
+    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
+
+    deletePath(tmpDir + "/.hg_archival.txt");
+
+    hgInfo.storePath = store->addToStore(name, tmpDir);
+
+    nlohmann::json json;
+    json["storePath"] = hgInfo.storePath;
+    json["uri"] = uri;
+    json["name"] = name;
+    json["branch"] = hgInfo.branch;
+    json["rev"] = hgInfo.rev;
+    json["revCount"] = hgInfo.revCount;
+
+    writeFile(storeLink, json.dump());
+
+    return hgInfo;
+}
+
+static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    std::string url;
+    std::string rev;
+    std::string name = "source";
+    PathSet context;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+            else if (n == "rev")
+                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+        }
+
+        if (url.empty())
+            throw EvalError(format("'url' argument required, at %1%") % pos);
+
+    } else
+        url = state.coerceToString(pos, *args[0], context, false, false);
+
+    if (!isUri(url)) url = absPath(url);
+
+    // FIXME: git externals probably can be used to bypass the URI
+    // whitelist. Ah well.
+    state.checkURI(url);
+
+    auto hgInfo = exportMercurial(state.store, url, rev, name);
+
+    state.mkAttrs(v, 8);
+    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
+    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
+
+}
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index dc526485fa3..879e3656a25 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -106,10 +106,9 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink));
+    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
 
     try {
-        // FIXME: doesn't handle empty lines
         auto json = nlohmann::json::parse(readFile(storeLink));
 
         assert(json["name"] == name && json["rev"] == gitInfo.rev);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 579a5e8c1b5..70f9b1f5eac 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -707,7 +707,7 @@ bool isUri(const string & s)
     size_t pos = s.find("://");
     if (pos == string::npos) return false;
     string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3";
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
 }
 
 
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
new file mode 100644
index 00000000000..5e4782658a8
--- /dev/null
+++ b/tests/fetchMercurial.sh
@@ -0,0 +1,73 @@
+source common.sh
+
+if [[ -z $(type -p hg) ]]; then
+    echo "Mercurial not installed; skipping Mercurial tests"
+    exit 0
+fi
+
+clearStore
+
+repo=$TEST_ROOT/hg
+
+rm -rfv $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
+
+hg init $repo
+echo '[ui]' >> $repo/.hg/hgrc
+echo 'username = Foobar <foobar@example.org>' >> $repo/.hg/hgrc
+
+echo utrecht > $repo/hello
+hg add --cwd $repo hello
+hg commit --cwd $repo -m 'Bla1'
+rev1=$(hg log --cwd $repo -r tip --template '{node}')
+
+echo world > $repo/hello
+hg commit --cwd $repo -m 'Bla2'
+rev2=$(hg log --cwd $repo -r tip --template '{node}')
+
+hg log --cwd $repo
+
+hg log --cwd $repo -r tip --template '{node}\n'
+
+path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
+[[ $(cat $path/hello) = world ]]
+
+# Fetch again. This should be cached.
+mv $repo ${repo}-tmp
+path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
+[[ $path = $path2 ]]
+
+[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]]
+[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]]
+[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
+
+# But with TTL 0, it should fail.
+(! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)")
+
+mv ${repo}-tmp $repo
+
+# Using a clean working tree should produce the same result.
+path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
+[[ $path = $path2 ]]
+
+# Using an unclean tree should yield the tracked but uncommitted changes.
+echo foo > $repo/foo
+echo bar > $repo/bar
+hg add --cwd $repo foo
+hg rm --cwd $repo hello
+
+path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path2/hello ]
+[ ! -e $path2/bar ]
+[[ $(cat $path2/foo) = foo ]]
+
+[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
+
+# ... unless we're using an explicit rev.
+path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath")
+[[ $path = $path3 ]]
+
+# Committing should not affect the store path.
+hg commit --cwd $repo -m 'Bla3'
+
+path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath")
+[[ $path2 = $path4 ]]
diff --git a/tests/local.mk b/tests/local.mk
index 6160b04c259..fced281ee5b 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -15,7 +15,8 @@ nix_tests = \
   linux-sandbox.sh \
   build-remote.sh \
   nar-index.sh \
-  structured-attrs.sh
+  structured-attrs.sh \
+  fetchMercurial.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From e026bc3b05ca45c2d855d0a38820034ab4ef3c4c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2017 18:43:11 +0100
Subject: [PATCH 0646/2196] fetchMercurial: Don't fetch hashes we already have

---
 src/libexpr/primops/fetchMercurial.cc | 28 ++++++++++++------
 src/libutil/util.cc                   | 41 +++++++++++++++++----------
 src/libutil/util.hh                   | 17 +++++++++++
 tests/fetchMercurial.sh               | 18 ++++++++----
 4 files changed, 75 insertions(+), 29 deletions(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 391ce45a0a9..e68d1a2a553 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -22,6 +22,8 @@ struct HgInfo
     uint64_t revCount = 0;
 };
 
+std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
+
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
@@ -66,20 +68,28 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
 
     /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-       do so now. FIXME: don't do this if "rev" is a hash and we
-       fetched it previously */
+       do so now. */
     time_t now = time(0);
     struct stat st;
     if (stat(stampFile.c_str(), &st) != 0 ||
         st.st_mtime < now - settings.tarballTtl)
     {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
-
-        if (pathExists(cacheDir)) {
-            runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-        } else {
-            createDirs(dirOf(cacheDir));
-            runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
+        /* Except that if this is a commit hash that we already have,
+           we don't have to pull again. */
+        if (!(std::regex_match(rev, commitHashRegex)
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
+
+            if (pathExists(cacheDir)) {
+                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
+            }
         }
 
         writeFile(stampFile, "");
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 9346d5dc4cf..f56153cd4a8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -895,32 +895,46 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 
 string runProgram(Path program, bool searchPath, const Strings & args,
     const std::experimental::optional<std::string> & input)
+{
+    RunOptions opts(program, args);
+    opts.searchPath = searchPath;
+    opts.input = input;
+
+    auto res = runProgram(opts);
+
+    if (!statusOk(res.first))
+        throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
+
+    return res.second;
+}
+
+std::pair<int, std::string> runProgram(const RunOptions & options)
 {
     checkInterrupt();
 
     /* Create a pipe. */
     Pipe out, in;
     out.create();
-    if (input) in.create();
+    if (options.input) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
-        if (input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+        if (options.input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
-        Strings args_(args);
-        args_.push_front(program);
+        Strings args_(options.args);
+        args_.push_front(options.program);
 
         restoreSignals();
 
-        if (searchPath)
-            execvp(program.c_str(), stringsToCharPtrs(args_).data());
+        if (options.searchPath)
+            execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
         else
-            execv(program.c_str(), stringsToCharPtrs(args_).data());
+            execv(options.program.c_str(), stringsToCharPtrs(args_).data());
 
-        throw SysError(format("executing '%1%'") % program);
+        throw SysError("executing '%1%'", options.program);
     });
 
     out.writeSide = -1;
@@ -935,11 +949,11 @@ string runProgram(Path program, bool searchPath, const Strings & args,
     });
 
 
-    if (input) {
+    if (options.input) {
         in.readSide = -1;
         writerThread = std::thread([&]() {
             try {
-                writeFull(in.writeSide.get(), *input);
+                writeFull(in.writeSide.get(), *options.input);
                 promise.set_value();
             } catch (...) {
                 promise.set_exception(std::current_exception());
@@ -952,14 +966,11 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 
     /* Wait for the child to finish. */
     int status = pid.wait();
-    if (!statusOk(status))
-        throw ExecError(status, format("program '%1%' %2%")
-            % program % statusToString(status));
 
     /* Wait for the writer thread to finish. */
-    if (input) promise.get_future().get();
+    if (options.input) promise.get_future().get();
 
-    return result;
+    return {status, result};
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 63a93f2ca6a..a3494e09b09 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -245,6 +245,23 @@ string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
     const std::experimental::optional<std::string> & input = {});
 
+struct RunOptions
+{
+    Path program;
+    bool searchPath = true;
+    Strings args;
+    std::experimental::optional<std::string> input;
+    bool _killStderr = false;
+
+    RunOptions(const Path & program, const Strings & args)
+        : program(program), args(args) { };
+
+    RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
+};
+
+std::pair<int, std::string> runProgram(const RunOptions & options);
+
+
 class ExecError : public Error
 {
 public:
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 5e4782658a8..83bf1331a40 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -9,7 +9,7 @@ clearStore
 
 repo=$TEST_ROOT/hg
 
-rm -rfv $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
 
 hg init $repo
 echo '[ui]' >> $repo/.hg/hgrc
@@ -24,13 +24,14 @@ echo world > $repo/hello
 hg commit --cwd $repo -m 'Bla2'
 rev2=$(hg log --cwd $repo -r tip --template '{node}')
 
-hg log --cwd $repo
-
-hg log --cwd $repo -r tip --template '{node}\n'
-
+# Fetch the default branch.
 path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
 
+# Fetch using an explicit revision hash.
+path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
+[[ $path = $path2 ]]
+
 # Fetch again. This should be cached.
 mv $repo ${repo}-tmp
 path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
@@ -43,6 +44,13 @@ path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
 # But with TTL 0, it should fail.
 (! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)")
 
+# Fetching with a explicit hash should succeed.
+path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
+[[ $path = $path2 ]]
+
+path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
+[[ $(cat $path2/hello) = utrecht ]]
+
 mv ${repo}-tmp $repo
 
 # Using a clean working tree should produce the same result.

From 212e72c60913c78eb88e347660144b441ea9f72b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 1 Nov 2017 21:32:30 +0100
Subject: [PATCH 0647/2196] Fix build

https://hydra.nixos.org/build/63172338
---
 src/libexpr/common-eval-args.hh | 2 +-
 src/libexpr/primops/fetchgit.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh
index 09fa406b2cd..be7fda78378 100644
--- a/src/libexpr/common-eval-args.hh
+++ b/src/libexpr/common-eval-args.hh
@@ -6,7 +6,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct Bindings;
+class Bindings;
 
 struct MixEvalArgs : virtual Args
 {
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 879e3656a25..4b5ead320c8 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -56,7 +56,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         }
     }
 
-    if (!ref) ref = "master";
+    if (!ref) ref = "master"s;
 
     if (rev != "") {
         std::regex revRegex("^[0-9a-fA-F]{40}$");

From 30503958109f3282ca557fbf1fd62dec1fefe0ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zoran=20Plesiv=C4=8Dak?= <z@plesiv.com>
Date: Wed, 1 Nov 2017 21:31:20 +0000
Subject: [PATCH 0648/2196] Mention "float" type in builtins.typeOf section of
 the docs

 + remove trailing whitespace from the file
---
 doc/manual/expressions/builtins.xml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 615314880ab..14da1c63e67 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -264,13 +264,13 @@ stdenv.mkDerivation { … }
 
     Note that when obtaining the hash with <varname>nix-prefetch-url
     </varname> the option <varname>--unpack</varname> is required.
-    </para> 
-      
-    <para>This function can also verify the contents against a hash. 
-    In that case, the function takes a set instead of a URL. The set 
+    </para>
+
+    <para>This function can also verify the contents against a hash.
+    In that case, the function takes a set instead of a URL. The set
     requires the attribute <varname>url</varname> and the attribute
     <varname>sha256</varname>, e.g.
-      
+
 <programlisting>
 with import (fetchTarball {
   url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
@@ -279,7 +279,7 @@ with import (fetchTarball {
 
 stdenv.mkDerivation { … }
 </programlisting>
-      
+
     </para>
 
     <para>This function is not available if <link
@@ -1245,8 +1245,9 @@ stdenv.mkDerivation (rec {
     <replaceable>e</replaceable>, namely <literal>"int"</literal>,
     <literal>"bool"</literal>, <literal>"string"</literal>,
     <literal>"path"</literal>, <literal>"null"</literal>,
-    <literal>"set"</literal>, <literal>"list"</literal> or
-    <literal>"lambda"</literal>.</para></listitem>
+    <literal>"set"</literal>, <literal>"list"</literal>,
+    <literal>"lambda"</literal> or
+    <literal>"float"</literal>.</para></listitem>
 
   </varlistentry>
 

From ae506c1ea420ed2411493d5178a3356219e1c252 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zoran=20Plesiv=C4=8Dak?= <z@plesiv.com>
Date: Wed, 1 Nov 2017 23:00:08 +0000
Subject: [PATCH 0649/2196] Mention isFloat function in "Built-in Functions"
 section

---
 doc/manual/expressions/builtins.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 14da1c63e67..5e88b88561f 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -600,6 +600,16 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.isFloat</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a float, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><function>builtins.isBool</function>
   <replaceable>e</replaceable></term>
 

From 5ab37f0e44d6e5a4c7fb55d9974bc7e225eed200 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Thu, 2 Nov 2017 23:37:42 +0100
Subject: [PATCH 0650/2196] fetchMercurial: fix error message

---
 src/libexpr/primops/fetchMercurial.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index e68d1a2a553..f48d003fcd1 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -167,7 +167,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
         }
 
         if (url.empty())

From 310567922666472c01c69912c269706fb6bb1921 Mon Sep 17 00:00:00 2001
From: Daniel Peebles <copumpkin@users.noreply.github.com>
Date: Wed, 1 Nov 2017 18:33:31 +0100
Subject: [PATCH 0651/2196] Don't freak out if we get a 403 from S3

As far as we're concerned, not being able to access a file just means
the file is missing. Plus, AWS explicitly goes out of its way to
return a 403 if the file is missing and the requester doesn't have
permission to list the bucket.

Also getting rid of an old hack that Eelco said was only relevant
to an older AWS SDK.
---
 src/libstore/s3-binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 5fc7371a519..6a0f19238ad 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -241,8 +241,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             auto & error = res.GetError();
             if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
                 || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
-                || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
-                    && error.GetMessage().find("404") != std::string::npos))
+                // If bucket listing is disabled, 404s turn into 403s
+                || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                 return false;
             throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
         }

From 453f6758107dd51dd649fa6f1e9e61c21b90c0a3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 3 Nov 2017 10:50:49 +0100
Subject: [PATCH 0652/2196] Allow getpwuid in the darwin sandbox.

---
 src/libstore/sandbox-defaults.sb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index cf700c62c6f..c8436d9866c 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -21,6 +21,9 @@
 ; Allow sending signals within the sandbox.
 (allow signal (target same-sandbox))
 
+; Allow getpwuid.
+(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo"))
+
 ; Access to /tmp.
 (allow file* process-exec (literal "/tmp") (subpath TMPDIR))
 

From 0e77aa398240d04e4169208d14ea620b8507b732 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2017 13:33:15 +0100
Subject: [PATCH 0653/2196] fetchGit: Don't do a remote fetch if we already
 have the rev

---
 src/libexpr/primops/fetchgit.cc | 38 ++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 4b5ead320c8..1d8f55bcd51 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -45,8 +45,10 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
             PathFilter filter = [&](const Path & p) -> bool {
                 assert(hasPrefix(p, uri));
                 auto st = lstat(p);
-                if (S_ISDIR(st.st_mode)) return true;
                 std::string file(p, uri.size() + 1);
+                if (file == ".git") return false;
+                // FIXME: filter out directories with no tracked files.
+                if (S_ISDIR(st.st_mode)) return true;
                 return files.count(file);
             };
 
@@ -80,21 +82,27 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     time_t now = time(0);
     struct stat st;
     if (stat(localRefFile.c_str(), &st) != 0 ||
-        st.st_mtime < now - settings.tarballTtl)
+        st.st_mtime <= now - settings.tarballTtl)
     {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
-
-        struct timeval times[2];
-        times[0].tv_sec = now;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = now;
-        times[1].tv_usec = 0;
-
-        utimes(localRefFile.c_str(), times);
+        if (rev == "" ||
+            chomp(runProgram(
+                RunOptions("git", { "-C", cacheDir, "cat-file", "-t", rev })
+                .killStderr(true)).second) != "commit")
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
+
+            // FIXME: git stderr messes up our progress indicator, so
+            // we're using --quiet for now. Should process its stderr.
+            runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+
+            struct timeval times[2];
+            times[0].tv_sec = now;
+            times[0].tv_usec = 0;
+            times[1].tv_sec = now;
+            times[1].tv_usec = 0;
+
+            utimes(localRefFile.c_str(), times);
+        }
     }
 
     // FIXME: check whether rev is an ancestor of ref.

From 4dee01da7c7bc5d67dc916f159bfee8a873ac19b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2017 13:33:50 +0100
Subject: [PATCH 0654/2196] fetchGit: Add a test

---
 release.nix                           |  3 +
 shell.nix                             |  1 +
 src/libexpr/primops/fetchMercurial.cc |  2 +-
 tests/fetchGit.sh                     | 83 +++++++++++++++++++++++++++
 tests/fetchMercurial.sh               |  2 +-
 tests/local.mk                        |  1 +
 6 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 tests/fetchGit.sh

diff --git a/release.nix b/release.nix
index f76e7cb9ea0..aa4d63777eb 100644
--- a/release.nix
+++ b/release.nix
@@ -76,6 +76,9 @@ let
           [ curl
             bzip2 xz brotli
             openssl pkgconfig sqlite boehmgc
+
+            # Tests
+            git
             mercurial
           ]
           ++ lib.optional stdenv.isLinux libseccomp
diff --git a/shell.nix b/shell.nix
index 02fbd59b12a..f47952582fc 100644
--- a/shell.nix
+++ b/shell.nix
@@ -25,6 +25,7 @@ with import ./release-common.nix { inherit pkgs; };
       perlPackages.DBDSQLite
 
       # Tests
+      git
       mercurial
     ]
     ++ lib.optional stdenv.isLinux libseccomp;
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index f48d003fcd1..2a423f61bb8 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -72,7 +72,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     time_t now = time(0);
     struct stat st;
     if (stat(stampFile.c_str(), &st) != 0 ||
-        st.st_mtime < now - settings.tarballTtl)
+        st.st_mtime <= now - settings.tarballTtl)
     {
         /* Except that if this is a commit hash that we already have,
            we don't have to pull again. */
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
new file mode 100644
index 00000000000..e2bb7b68c4f
--- /dev/null
+++ b/tests/fetchGit.sh
@@ -0,0 +1,83 @@
+source common.sh
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git tests"
+    exit 0
+fi
+
+clearStore
+
+repo=$TEST_ROOT/git
+
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git
+
+git init $repo
+git -C $repo config user.email "foobar@example.com"
+git -C $repo config user.name "Foobar"
+
+echo utrecht > $repo/hello
+git -C $repo add hello
+git -C $repo commit -m 'Bla1'
+rev1=$(git -C $repo rev-parse HEAD)
+
+echo world > $repo/hello
+git -C $repo commit -m 'Bla2' -a
+rev2=$(git -C $repo rev-parse HEAD)
+
+# Fetch the default branch.
+path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+[[ $(cat $path/hello) = world ]]
+
+# Fetch using an explicit revision hash.
+path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
+[[ $path = $path2 ]]
+
+# Fetch again. This should be cached.
+mv $repo ${repo}-tmp
+path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+[[ $path = $path2 ]]
+
+[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
+[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
+
+# But with TTL 0, it should fail.
+(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv)
+
+# Fetching with a explicit hash should succeed.
+path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
+[[ $path = $path2 ]]
+
+path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
+[[ $(cat $path2/hello) = utrecht ]]
+
+mv ${repo}-tmp $repo
+
+# Using a clean working tree should produce the same result.
+path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+[[ $path = $path2 ]]
+
+# Using an unclean tree should yield the tracked but uncommitted changes.
+echo foo > $repo/foo
+echo bar > $repo/bar
+git -C $repo add foo
+git -C $repo rm hello
+
+path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+[ ! -e $path2/hello ]
+[ ! -e $path2/bar ]
+[[ $(cat $path2/foo) = foo ]]
+
+[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+
+# ... unless we're using an explicit ref or rev.
+path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
+[[ $path = $path3 ]]
+
+path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath")
+[[ $path = $path3 ]]
+
+# Committing should not affect the store path.
+git -C $repo commit -m 'Bla3' -a
+
+path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath")
+[[ $path2 = $path4 ]]
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 83bf1331a40..9c897ada96a 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -42,7 +42,7 @@ path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
 [[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
 
 # But with TTL 0, it should fail.
-(! nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo)")
+(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)")
 
 # Fetching with a explicit hash should succeed.
 path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
diff --git a/tests/local.mk b/tests/local.mk
index fced281ee5b..67b378839a2 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -16,6 +16,7 @@ nix_tests = \
   build-remote.sh \
   nar-index.sh \
   structured-attrs.sh \
+  fetchGit.sh \
   fetchMercurial.sh
   # parallel.sh
 

From ee6ac38848277d80c62fa5c2734ada1b5edc8680 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2017 13:48:02 +0100
Subject: [PATCH 0655/2196] fetchGit/fetchMercurial: Filter out directories
 with untracked files

---
 src/libexpr/primops/fetchMercurial.cc | 12 ++++++++----
 src/libexpr/primops/fetchgit.cc       | 12 ++++++++----
 tests/fetchGit.sh                     |  9 ++++++---
 tests/fetchMercurial.sh               |  9 ++++++---
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 2a423f61bb8..7def7103bf3 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -47,11 +47,15 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
             PathFilter filter = [&](const Path & p) -> bool {
                 assert(hasPrefix(p, uri));
-                auto st = lstat(p);
                 std::string file(p, uri.size() + 1);
-                if (file == ".hg") return false;
-                // FIXME: filter out directories with no tracked files.
-                if (S_ISDIR(st.st_mode)) return true;
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto i = files.lower_bound(file);
+                    return i != files.end() && hasPrefix(*i, file);
+                }
+
                 return files.count(file);
             };
 
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index 1d8f55bcd51..baa48f6422b 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -44,11 +44,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
             PathFilter filter = [&](const Path & p) -> bool {
                 assert(hasPrefix(p, uri));
-                auto st = lstat(p);
                 std::string file(p, uri.size() + 1);
-                if (file == ".git") return false;
-                // FIXME: filter out directories with no tracked files.
-                if (S_ISDIR(st.st_mode)) return true;
+
+                auto st = lstat(p);
+
+                if (S_ISDIR(st.st_mode)) {
+                    auto i = files.lower_bound(file);
+                    return i != files.end() && hasPrefix(*i, file);
+                }
+
                 return files.count(file);
             };
 
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index e2bb7b68c4f..a967380cd5b 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -57,15 +57,18 @@ path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [[ $path = $path2 ]]
 
 # Using an unclean tree should yield the tracked but uncommitted changes.
-echo foo > $repo/foo
+mkdir $repo/dir1 $repo/dir2
+echo foo > $repo/dir1/foo
 echo bar > $repo/bar
-git -C $repo add foo
+echo bar > $repo/dir2/bar
+git -C $repo add dir1/foo
 git -C $repo rm hello
 
 path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
-[[ $(cat $path2/foo) = foo ]]
+[ ! -e $path2/dir2/bar ]
+[[ $(cat $path2/dir1/foo) = foo ]]
 
 [[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
 
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 9c897ada96a..0c9f4bdbbc8 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -58,15 +58,18 @@ path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
 [[ $path = $path2 ]]
 
 # Using an unclean tree should yield the tracked but uncommitted changes.
-echo foo > $repo/foo
+mkdir $repo/dir1 $repo/dir2
+echo foo > $repo/dir1/foo
 echo bar > $repo/bar
-hg add --cwd $repo foo
+echo bar > $repo/dir2/bar
+hg add --cwd $repo dir1/foo
 hg rm --cwd $repo hello
 
 path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
-[[ $(cat $path2/foo) = foo ]]
+[ ! -e $path2/dir2/bar ]
+[[ $(cat $path2/dir1/foo) = foo ]]
 
 [[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
 

From 6cf7c6a6b0d06d88f2f27daae4febbbdc757841b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2017 13:52:32 +0100
Subject: [PATCH 0656/2196] Remove git:// support in NIX_PATH

This didn't support specifying a revision/branch, and was restricted
to git:// URIs (since https:// or ssh:// would be ambiguous).
---
 doc/manual/release-notes/rl-1.12.xml |  7 -------
 src/libexpr/parser.y                 |  7 +------
 src/libexpr/primops/fetchgit.cc      |  9 ++++++++-
 src/libexpr/primops/fetchgit.hh      | 24 ------------------------
 4 files changed, 9 insertions(+), 38 deletions(-)
 delete mode 100644 src/libexpr/primops/fetchgit.hh

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 7c9a8b75ece..29943e3e6e9 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -188,13 +188,6 @@
     (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para>
   </listitem>
 
-  <listitem>
-    <para>Git repositories can now be specified in the Nix search
-    path,
-    e.g. <literal>nixpkgs=git://github.com/NixOS/nixpkgs</literal>.
-    (d8bf0d4859e28ddd23401fbe89f4e528aa09ddb3)</para>
-  </listitem>
-
   <listitem>
     <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
     content-addressable tarball cache at
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 7e63dc89f60..ef11dd60921 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -523,7 +523,6 @@ formal
 #include "eval.hh"
 #include "download.hh"
 #include "store-api.hh"
-#include "primops/fetchgit.hh"
 
 
 namespace nix {
@@ -665,11 +664,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git"))
-                // FIXME: support specifying revision/branch
-                res = { true, exportGit(store, elem.second).storePath };
-            else
-                res = { true, getDownloader()->downloadCached(store, elem.second, true) };
+            res = { true, getDownloader()->downloadCached(store, elem.second, true) };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc
index baa48f6422b..bca68ed7209 100644
--- a/src/libexpr/primops/fetchgit.cc
+++ b/src/libexpr/primops/fetchgit.cc
@@ -1,4 +1,3 @@
-#include "fetchgit.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "download.hh"
@@ -15,6 +14,14 @@ using namespace std::string_literals;
 
 namespace nix {
 
+struct GitInfo
+{
+    Path storePath;
+    std::string rev;
+    std::string shortRev;
+    uint64_t revCount = 0;
+};
+
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::experimental::optional<std::string> ref, const std::string & rev,
     const std::string & name)
diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh
deleted file mode 100644
index 818ab7102e9..00000000000
--- a/src/libexpr/primops/fetchgit.hh
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-
-#include <string>
-
-#include "util.hh"
-
-namespace nix {
-
-class Store;
-
-struct GitInfo
-{
-    Path storePath;
-    std::string rev;
-    std::string shortRev;
-    uint64_t revCount = 0;
-};
-
-GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref = {},
-    const std::string & rev = "",
-    const std::string & name = "");
-
-}

From 7a4d9574d9275426e31bb2b3fbb8515600d233c4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Nov 2017 13:54:45 +0100
Subject: [PATCH 0657/2196] fetchgit.cc -> fetchGit.cc

---
 src/libexpr/primops/{fetchgit.cc => fetchGit.cc} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/libexpr/primops/{fetchgit.cc => fetchGit.cc} (100%)

diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchGit.cc
similarity index 100%
rename from src/libexpr/primops/fetchgit.cc
rename to src/libexpr/primops/fetchGit.cc

From fd10f6f2414521947ca60b9d1508d909f50e9faa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Nov 2017 12:06:58 +0100
Subject: [PATCH 0658/2196] Show when tests are skipped

Also, don't depend on tput (ncurses). It's really not needed since
ANSI escape sequences have been standardized for 35 years or so.
---
 mk/tests.mk             | 16 +++++++++-------
 tests/fetchGit.sh       |  2 +-
 tests/fetchMercurial.sh |  2 +-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/mk/tests.mk b/mk/tests.mk
index 1138857c3c1..e353d46a0d0 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -12,21 +12,23 @@ installcheck:
 	@total=0; failed=0; \
 	red=""; \
 	green=""; \
+	yellow=""; \
 	normal=""; \
 	if [ -t 1 ]; then \
-		ncolors="$$(tput colors)"; \
-		if [ -n "$$ncolors" ] && [ "$$ncolors" -ge 8 ]; then \
-			red="$$(tput setaf 1)"; \
-			green="$$(tput setaf 2)"; \
-			normal="$$(tput sgr0)"; \
-		fi; \
+		red=""; \
+		green=""; \
+		yellow=""; \
+		normal=""; \
 	fi; \
 	for i in $(_installcheck-list); do \
 	  total=$$((total + 1)); \
 	  printf "running test $$i..."; \
 	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  if [ $$? -eq 0 ]; then \
+	  status=$$?; \
+	  if [ $$status -eq 0 ]; then \
 	    echo " [$${green}PASS$$normal]"; \
+	  elif [ $$status -eq 99 ]; then \
+	    echo " [$${yellow}SKIP$$normal]"; \
 	  else \
 	    echo " [$${red}FAIL$$normal]"; \
 	    echo "$$log" | sed 's/^/    /'; \
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index a967380cd5b..f13de89f7ff 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -2,7 +2,7 @@ source common.sh
 
 if [[ -z $(type -p git) ]]; then
     echo "Git not installed; skipping Git tests"
-    exit 0
+    exit 99
 fi
 
 clearStore
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 0c9f4bdbbc8..6cfc0a708e8 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -2,7 +2,7 @@ source common.sh
 
 if [[ -z $(type -p hg) ]]; then
     echo "Mercurial not installed; skipping Mercurial tests"
-    exit 0
+    exit 99
 fi
 
 clearStore

From 9d87d03331132b1da71a62ae3aa4636726a3e860 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 8 Nov 2017 16:17:32 +0100
Subject: [PATCH 0659/2196] Remove extraneous comment

---
 src/libstore/sandbox-defaults.sb | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index f556a26a01f..0299d1ee45d 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -31,9 +31,6 @@
 (allow file* process-exec network-outbound network-inbound
        (literal "/tmp") (subpath TMPDIR))
 
-; Always allow unix domain sockets, since they can't hurt purity or security
-
-
 ; Some packages like to read the system version.
 (allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist"))
 

From c9c3fc710ba6995b20e078523aa616323dc5759a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 13:27:40 +0100
Subject: [PATCH 0660/2196] Rename tests/nar-index -> tests/nar-access

---
 tests/local.mk                          | 2 +-
 tests/{nar-index.nix => nar-access.nix} | 0
 tests/{nar-index.sh => nar-access.sh}   | 4 ++--
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename tests/{nar-index.nix => nar-access.nix} (100%)
 rename tests/{nar-index.sh => nar-access.sh} (84%)

diff --git a/tests/local.mk b/tests/local.mk
index 67b378839a2..47956c23246 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -14,7 +14,7 @@ nix_tests = \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
   build-remote.sh \
-  nar-index.sh \
+  nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
   fetchMercurial.sh
diff --git a/tests/nar-index.nix b/tests/nar-access.nix
similarity index 100%
rename from tests/nar-index.nix
rename to tests/nar-access.nix
diff --git a/tests/nar-index.sh b/tests/nar-access.sh
similarity index 84%
rename from tests/nar-index.sh
rename to tests/nar-access.sh
index 51369346c88..5c2d3058ccf 100644
--- a/tests/nar-index.sh
+++ b/tests/nar-access.sh
@@ -1,7 +1,7 @@
 source common.sh
 
 echo "building test path"
-storePath="$(nix-build nar-index.nix -A a --no-out-link)"
+storePath="$(nix-build nar-access.nix -A a --no-out-link)"
 
 cd "$TEST_ROOT"
 
@@ -20,4 +20,4 @@ diff -u data.cat-nar $storePath/foo/data
 
 echo "check that file contents of baz match"
 nix cat-nar $narFile /foo/baz > baz.cat-nar
-diff -u baz.cat-nar $storePath/foo/baz
\ No newline at end of file
+diff -u baz.cat-nar $storePath/foo/baz

From 5ef5d70b5e3dec6abf72b22e531f527fff82b375 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:04:09 +0100
Subject: [PATCH 0661/2196] Make config options available to legacy commands

---
 src/libmain/common-args.cc | 4 ++++
 src/nix/main.cc            | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index ea27aaa35e0..d3aac6aba1f 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -34,6 +34,10 @@ MixCommonArgs::MixCommonArgs(const string & programName)
                 warn(e.what());
             }
         });
+
+    std::string cat = "config";
+    settings.convertToArgs(*this, cat);
+    hiddenCategories.insert(cat);
 }
 
 }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 060402cd08d..06bb8a1c304 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -43,10 +43,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName("version")
             .description("show version information")
             .handler([&]() { printVersion(programName); });
-
-        std::string cat = "config";
-        settings.convertToArgs(*this, cat);
-        hiddenCategories.insert(cat);
     }
 
     void printFlags(std::ostream & out) override

From c0d93a01ee996427a1ec6b7602e655c8405624d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:16:16 +0100
Subject: [PATCH 0662/2196] Remove ncurses-bin

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index aa4d63777eb..0f9468dc75a 100644
--- a/release.nix
+++ b/release.nix
@@ -335,7 +335,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "ncurses-bin" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
             ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;

From bac8055652964d9ad5202011befb6b199463ddef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:23:53 +0100
Subject: [PATCH 0663/2196] nix ls-{store,nar}: Add --json flag

---
 src/libstore/binary-cache-store.cc | 35 +-----------------------------
 src/libstore/nar-accessor.cc       | 33 ++++++++++++++++++++++++++++
 src/libstore/nar-accessor.hh       |  4 ++++
 src/nix/ls.cc                      | 21 +++++++++++++-----
 4 files changed, 53 insertions(+), 40 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 67607ab3d43..9c6424a496e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -119,42 +119,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 accessor_->addToCache(info.path, *nar);
             }
 
-            std::function<void(const Path &, JSONPlaceholder &)> recurse;
-
-            recurse = [&](const Path & path, JSONPlaceholder & res) {
-                auto st = narAccessor->stat(path);
-
-                auto obj = res.object();
-
-                switch (st.type) {
-                case FSAccessor::Type::tRegular:
-                    obj.attr("type", "regular");
-                    obj.attr("size", st.fileSize);
-                    if (st.isExecutable)
-                        obj.attr("executable", true);
-                    break;
-                case FSAccessor::Type::tDirectory:
-                    obj.attr("type", "directory");
-                    {
-                        auto res2 = obj.object("entries");
-                        for (auto & name : narAccessor->readDirectory(path)) {
-                            auto res3 = res2.placeholder(name);
-                            recurse(path + "/" + name, res3);
-                        }
-                    }
-                    break;
-                case FSAccessor::Type::tSymlink:
-                    obj.attr("type", "symlink");
-                    obj.attr("target", narAccessor->readLink(path));
-                    break;
-                default:
-                    abort();
-                }
-            };
-
             {
                 auto res = jsonRoot.placeholder("root");
-                recurse("", res);
+                listNar(res, narAccessor, "");
             }
         }
 
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 2afdeb021a9..b2d15c395f1 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -1,5 +1,6 @@
 #include "nar-accessor.hh"
 #include "archive.hh"
+#include "json.hh"
 
 #include <map>
 #include <stack>
@@ -181,4 +182,36 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, const Path & path)
+{
+    auto st = accessor->stat(path);
+
+    auto obj = res.object();
+
+    switch (st.type) {
+    case FSAccessor::Type::tRegular:
+        obj.attr("type", "regular");
+        obj.attr("size", st.fileSize);
+        if (st.isExecutable)
+            obj.attr("executable", true);
+        break;
+    case FSAccessor::Type::tDirectory:
+        obj.attr("type", "directory");
+        {
+            auto res2 = obj.object("entries");
+            for (auto & name : accessor->readDirectory(path)) {
+                auto res3 = res2.placeholder(name);
+                listNar(res3, accessor, path + "/" + name);
+            }
+        }
+        break;
+    case FSAccessor::Type::tSymlink:
+        obj.attr("type", "symlink");
+        obj.attr("target", accessor->readLink(path));
+        break;
+    default:
+        abort();
+    }
+}
+
 }
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 83c570be4c7..7699cbbb587 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -8,4 +8,8 @@ namespace nix {
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+class JSONPlaceholder;
+
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, const Path & path);
+
 }
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 5a5fa8f62d9..8566b8b0682 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -2,10 +2,12 @@
 #include "store-api.hh"
 #include "fs-accessor.hh"
 #include "nar-accessor.hh"
+#include "common-args.hh"
+#include "json.hh"
 
 using namespace nix;
 
-struct MixLs : virtual Args
+struct MixLs : virtual Args, MixJSON
 {
     std::string path;
 
@@ -20,7 +22,7 @@ struct MixLs : virtual Args
         mkFlag('d', "directory", "show directories rather than their contents", &showDirectory);
     }
 
-    void list(ref<FSAccessor> accessor)
+    void listText(ref<FSAccessor> accessor)
     {
         std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath;
 
@@ -61,10 +63,6 @@ struct MixLs : virtual Args
                 showFile(curPath, relPath);
         };
 
-        if (path == "/") {
-            path = "";
-        }
-
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
             throw Error(format("path '%1%' does not exist") % path);
@@ -72,6 +70,17 @@ struct MixLs : virtual Args
             st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path),
             showDirectory);
     }
+
+    void list(ref<FSAccessor> accessor)
+    {
+        if (path == "/") path = "";
+
+        if (json) {
+            JSONPlaceholder jsonRoot(std::cout, true);
+            listNar(jsonRoot, accessor, path);
+        } else
+            listText(accessor);
+    }
 };
 
 struct CmdLsStore : StoreCommand, MixLs

From c8155e9f5fba14c4edc8c5763fb7d7e9d4d123a9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:27:01 +0100
Subject: [PATCH 0664/2196] Don't indent JSON output

---
 src/nix/ls.cc          | 2 +-
 src/nix/path-info.cc   | 2 +-
 src/nix/search.cc      | 2 +-
 src/nix/show-config.cc | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 8566b8b0682..5408c092992 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -76,7 +76,7 @@ struct MixLs : virtual Args, MixJSON
         if (path == "/") path = "";
 
         if (json) {
-            JSONPlaceholder jsonRoot(std::cout, true);
+            JSONPlaceholder jsonRoot(std::cout);
             listNar(jsonRoot, accessor, path);
         } else
             listText(accessor);
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index ca02a4c929b..47caa401d3c 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -65,7 +65,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             pathLen = std::max(pathLen, storePath.size());
 
         if (json) {
-            JSONPlaceholder jsonRoot(std::cout, true);
+            JSONPlaceholder jsonRoot(std::cout);
             store->pathInfoToJSON(jsonRoot,
                 // FIXME: preserve order?
                 PathSet(storePaths.begin(), storePaths.end()),
diff --git a/src/nix/search.cc b/src/nix/search.cc
index f458367dcb5..a9dc2d6b924 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -84,7 +84,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         bool first = true;
 
-        auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr;
+        auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
 
         auto sToplevel = state->symbols.create("_toplevel");
         auto sRecurse = state->symbols.create("recurseForDerivations");
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index c628c2898d7..c64b12c8dd6 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -26,7 +26,7 @@ struct CmdShowConfig : Command, MixJSON
     {
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
-            JSONObject jsonObj(std::cout, true);
+            JSONObject jsonObj(std::cout);
             settings.toJSON(jsonObj);
         } else {
             for (auto & s : settings.getSettings())

From 4db0a9555e3b39600847084e1b40e0bb935c13de Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:31:28 +0100
Subject: [PATCH 0665/2196] nix ls-{nar,store} --json: Respect -R

---
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/nar-accessor.cc       | 10 +++++++---
 src/libstore/nar-accessor.hh       |  5 ++++-
 src/nix/ls.cc                      |  2 +-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9c6424a496e..93caba67ea9 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -121,7 +121,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, narAccessor, "");
+                listNar(res, narAccessor, "", true);
             }
         }
 
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index b2d15c395f1..e5042208e58 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -182,7 +182,8 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
-void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, const Path & path)
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
+    const Path & path, bool recurse)
 {
     auto st = accessor->stat(path);
 
@@ -200,8 +201,11 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, const Path & path)
         {
             auto res2 = obj.object("entries");
             for (auto & name : accessor->readDirectory(path)) {
-                auto res3 = res2.placeholder(name);
-                listNar(res3, accessor, path + "/" + name);
+                if (recurse) {
+                    auto res3 = res2.placeholder(name);
+                    listNar(res3, accessor, path + "/" + name, true);
+                } else
+                    res2.object(name);
             }
         }
         break;
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 7699cbbb587..ed8fe15cad2 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -10,6 +10,9 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
 class JSONPlaceholder;
 
-void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor, const Path & path);
+/* Write a JSON representation of the contents of a NAR (except file
+   contents). */
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
+    const Path & path, bool recurse);
 
 }
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 5408c092992..69620595d8c 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -77,7 +77,7 @@ struct MixLs : virtual Args, MixJSON
 
         if (json) {
             JSONPlaceholder jsonRoot(std::cout);
-            listNar(jsonRoot, accessor, path);
+            listNar(jsonRoot, accessor, path, recursive);
         } else
             listText(accessor);
     }

From b8d446829eb9a0f65e314039e0f910a3aceaa143 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:37:39 +0100
Subject: [PATCH 0666/2196] Add some tests

---
 tests/nar-access.sh | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index 5c2d3058ccf..63f97f36cbf 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -5,19 +5,30 @@ storePath="$(nix-build nar-access.nix -A a --no-out-link)"
 
 cd "$TEST_ROOT"
 
-echo "dumping path to nar"
+# Dump path to nar.
 narFile="$TEST_ROOT/path.nar"
 nix-store --dump $storePath > $narFile
 
-echo "check that find and ls-nar match"
+# Check that find and ls-nar match.
 ( cd $storePath; find . | sort ) > files.find
 nix ls-nar -R -d $narFile "" | sort > files.ls-nar
 diff -u files.find files.ls-nar
 
-echo "check that file contents of data match"
+# Check that file contents of data match.
 nix cat-nar $narFile /foo/data > data.cat-nar
 diff -u data.cat-nar $storePath/foo/data
 
-echo "check that file contents of baz match"
+# Check that file contents of baz match.
 nix cat-nar $narFile /foo/baz > baz.cat-nar
 diff -u baz.cat-nar $storePath/foo/baz
+
+nix cat-store $storePath/foo/baz > baz.cat-nar
+diff -u baz.cat-nar $storePath/foo/baz
+
+# Test --json.
+[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
+[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
+[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0}' ]]
+[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
+[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
+[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]

From 5773d667ee26efb4b977b9109da509d46d8378b5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 14:49:06 +0100
Subject: [PATCH 0667/2196] nix ls-{nar,store}: Don't abort on missing files

---
 src/libstore/nar-accessor.cc | 2 +-
 tests/nar-access.sh          | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index e5042208e58..839a7991c89 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -214,7 +214,7 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
         obj.attr("target", accessor->readLink(path));
         break;
     default:
-        abort();
+        throw Error("path '%s' does not exist in NAR", path);
     }
 }
 
diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index 63f97f36cbf..e29e4689b63 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -32,3 +32,7 @@ diff -u baz.cat-nar $storePath/foo/baz
 [[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
 [[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
 [[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]
+
+# Test missing files.
+nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
+nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist'

From 2c39e4eca000037f704ad81d589f60964f6f13a7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 15:10:52 +0100
Subject: [PATCH 0668/2196] Revert "Don't parse "x:x" as a URI"

This reverts commit f90f660b243866b8860eeb24cc4a345d32cc7ce7.

This broke Hydra's release.nix, which contained

  preCheck = ''export LOGNAME=${LOGNAME:-foo}'';
---
 src/libexpr/lexer.l           | 2 +-
 tests/lang/lib.nix            | 2 +-
 tests/lang/parse-okay-url.nix | 3 +--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 828356bbf44..28a0a6a8789 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -90,7 +90,7 @@ FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
 PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
 HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
 SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:\/\/[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+|channel\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
+URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
 
 %%
diff --git a/tests/lang/lib.nix b/tests/lang/lib.nix
index 1c63b2f31d6..028a538314b 100644
--- a/tests/lang/lib.nix
+++ b/tests/lang/lib.nix
@@ -49,7 +49,7 @@ rec {
     if comp (head list2) (head list1) then [(head list2)] ++ mergeLists comp list1 (tail list2) else
     [(head list1)] ++ mergeLists comp (tail list1) list2;
 
-  id = x:x; # sic
+  id = x: x;
 
   const = x: y: x;
 
diff --git a/tests/lang/parse-okay-url.nix b/tests/lang/parse-okay-url.nix
index fb74d66f092..fce3b13ee64 100644
--- a/tests/lang/parse-okay-url.nix
+++ b/tests/lang/parse-okay-url.nix
@@ -1,8 +1,7 @@
-[
+[ x:x
   https://svn.cs.uu.nl:12443/repos/trace/trunk
   http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2
   http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz
   http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz
   ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz
-  channel:nixos-17.09
 ]

From d6dbda7004fda4af1fc89c5c947b4d1595a8c436 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 18:24:20 +0100
Subject: [PATCH 0669/2196] Add tests for "nix verify", "nix sign-paths" etc.

---
 tests/local.mk   |  3 ++-
 tests/signing.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 tests/signing.sh

diff --git a/tests/local.mk b/tests/local.mk
index 47956c23246..2b92ec4e5a8 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -17,7 +17,8 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
-  fetchMercurial.sh
+  fetchMercurial.sh \
+  signing.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/signing.sh b/tests/signing.sh
new file mode 100644
index 00000000000..c3638012e10
--- /dev/null
+++ b/tests/signing.sh
@@ -0,0 +1,62 @@
+source common.sh
+
+clearStore
+clearCache
+
+nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1
+pk1=$(cat $TEST_ROOT/pk1)
+nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2
+pk2=$(cat $TEST_ROOT/pk2)
+
+# Build a path.
+outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2")
+
+# Verify that the path got signed.
+info=$(nix path-info --json $outPath)
+[[ $info =~ '"ultimate":true' ]]
+[[ $info =~ 'cache1.example.org' ]]
+[[ $info =~ 'cache2.example.org' ]]
+
+# Test "nix verify".
+nix verify -r $outPath
+
+expect 2 nix verify -r $outPath --sigs-needed 1
+
+nix verify -r $outPath --sigs-needed 1 --binary-cache-public-keys $pk1
+
+expect 2 nix verify -r $outPath --sigs-needed 2 --binary-cache-public-keys $pk1
+
+nix verify -r $outPath --sigs-needed 2 --binary-cache-public-keys "$pk1 $pk2"
+
+nix verify --all --sigs-needed 2 --binary-cache-public-keys "$pk1 $pk2"
+
+# Build something unsigned.
+outPath2=$(nix-build simple.nix --no-out-link)
+
+nix verify -r $outPath
+
+# Verify that the path did not get signed but does have the ultimate bit.
+info=$(nix path-info --json $outPath2)
+[[ $info =~ '"ultimate":true' ]]
+(! [[ $info =~ 'signatures' ]])
+
+# Test "nix verify".
+nix verify -r $outPath2
+
+expect 2 nix verify -r $outPath2 --sigs-needed 1
+
+expect 2 nix verify -r $outPath2 --sigs-needed 1 --binary-cache-public-keys $pk1
+
+# Test "nix sign-paths".
+nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2
+
+nix verify -r $outPath2 --sigs-needed 1 --binary-cache-public-keys $pk1
+
+# Copy to a binary cache.
+nix copy --to file://$cacheDir $outPath2
+
+# Verify that signatures got copied.
+info=$(nix path-info --store file://$cacheDir --json $outPath2)
+(! [[ $info =~ '"ultimate":true' ]])
+[[ $info =~ 'cache1.example.org' ]]
+(! [[ $info =~ 'cache2.example.org' ]])

From ec5b04862ba48e6d7e61c8bf730ae37d48b6f70a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 18:44:05 +0100
Subject: [PATCH 0670/2196] nix sign-paths: Support binary caches

---
 src/libstore/binary-cache-store.cc | 47 +++++++++++++++++++++---------
 src/libstore/binary-cache-store.hh |  5 ++--
 tests/signing.sh                   |  6 ++++
 3 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 93caba67ea9..68af85bf16d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -73,6 +73,23 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
+void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
+{
+    auto narInfoFile = narInfoFileFor(narInfo->path);
+
+    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
+
+    auto hashPart = storePathToHash(narInfo->path);
+
+    {
+        auto state_(state.lock());
+        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
+    }
+
+    if (diskCache)
+        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+}
+
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
@@ -89,8 +106,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 % info.path % ref);
         }
 
-    auto narInfoFile = narInfoFileFor(info.path);
-
     assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
@@ -168,17 +183,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo->sign(*secretKey);
 
-    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
-
-    auto hashPart = storePathToHash(narInfo->path);
-
-    {
-        auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
-    }
-
-    if (diskCache)
-        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+    writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
 }
@@ -293,6 +298,22 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
 }
 
+void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
+{
+    /* Note: this is inherently racy since there is no locking on
+       binary caches. In particular, with S3 this unreliable, even
+       when addSignatures() is called sequentially on a path, because
+       S3 might return an outdated cached version. */
+
+    auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath));
+
+    narInfo->sigs.insert(sigs.begin(), sigs.end());
+
+    auto narInfoFile = narInfoFileFor(narInfo->path);
+
+    writeNarInfo(narInfo);
+}
+
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
 {
     Path drvPath;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index d3b0e0bd933..8492ff600eb 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -59,6 +59,8 @@ private:
 
     std::string narInfoFileFor(const Path & storePath);
 
+    void writeNarInfo(ref<NarInfo> narInfo);
+
 public:
 
     bool isValidPathUncached(const Path & path) override;
@@ -119,8 +121,7 @@ public:
 
     ref<FSAccessor> getFSAccessor() override;
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { unsupported(); }
+    void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
 
diff --git a/tests/signing.sh b/tests/signing.sh
index c3638012e10..bef27ac7a58 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -60,3 +60,9 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2)
 (! [[ $info =~ '"ultimate":true' ]])
 [[ $info =~ 'cache1.example.org' ]]
 (! [[ $info =~ 'cache2.example.org' ]])
+
+# Verify that adding a signature to a path in a binary cache works.
+nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
+info=$(nix path-info --store file://$cacheDir --json $outPath2)
+[[ $info =~ 'cache1.example.org' ]]
+[[ $info =~ 'cache2.example.org' ]]

From 4b45d8c95a037a9962842836be3073fda7329105 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 18:47:37 +0100
Subject: [PATCH 0671/2196] Update lcov filter

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 0f9468dc75a..56b88488a3f 100644
--- a/release.nix
+++ b/release.nix
@@ -196,7 +196,7 @@ let
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*/boost/*" "*-tab.*" ];
+        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some

From 2f5789c5d6e779ae246659b2974560f14e24ac31 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2017 18:47:44 +0100
Subject: [PATCH 0672/2196] Add dependencies for coverage test

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 56b88488a3f..c76966a1a44 100644
--- a/release.nix
+++ b/release.nix
@@ -185,7 +185,7 @@ let
         buildInputs =
           [ curl bzip2 openssl pkgconfig sqlite xz libsodium libseccomp
             # These are for "make check" only:
-            graphviz libxml2 libxslt
+            graphviz libxml2 libxslt git mercurial
           ];
 
         configureFlags = ''

From 9d7ce0bf45fe4b77f5483929609faad25a66ab8e Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 7 Nov 2017 14:42:34 -0600
Subject: [PATCH 0673/2196] anchor nix::Exit exception

---
 src/libmain/shared.cc | 1 +
 src/libmain/shared.hh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 0f599f38858..85d3c077ba5 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -369,5 +369,6 @@ PrintFreed::~PrintFreed()
             % showBytes(results.bytesFreed);
 }
 
+Exit::~Exit() { }
 
 }
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 9219dbed832..1dcc4f0ac94 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -17,6 +17,7 @@ public:
     int status;
     Exit() : status(0) { }
     Exit(int status) : status(status) { }
+    virtual ~Exit();
 };
 
 int handleExceptions(const string & programName, std::function<void()> fun);

From 27131866c929d2fc7690bc7f177375df9af8e213 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2017 12:23:31 +0100
Subject: [PATCH 0674/2196] Doh

---
 tests/common.sh.in | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/common.sh.in b/tests/common.sh.in
index 04d605e3443..ca6df25362d 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -90,4 +90,15 @@ fail() {
     exit 1
 }
 
+expect() {
+    local expected res
+    expected="$1"
+    shift
+    set +e
+    "$@"
+    res="$?"
+    set -e
+    [[ $res -eq $expected ]]
+}
+
 set -x

From 8956ae19879e0367250002b94e399d8c771f37a1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2017 14:16:04 +0100
Subject: [PATCH 0675/2196] Add a "profile" option to S3BinaryCacheStore

This allows specifying the AWS configuration profile to use. E.g.

  nix copy --from s3://my-cache?profile=aws-dev-account /nix/store/cf3isrlqavvd5w7rpky1fa8j9lcnlggm-...
---
 src/libstore/download.cc              |  2 +-
 src/libstore/s3-binary-cache-store.cc | 15 ++++++++++++---
 src/libstore/s3.hh                    |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 70f9b1f5eac..da31029b4f8 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -533,7 +533,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
 #ifdef ENABLE_S3
-                S3Helper s3Helper(Aws::Region::US_EAST_1); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 6a0f19238ad..0079da1becf 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -10,6 +10,8 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/FormattedLogSystem.h>
@@ -77,9 +79,15 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const string & region)
+S3Helper::S3Helper(const std::string & profile, const std::string & region)
     : config(makeConfig(region))
-    , client(make_ref<Aws::S3::S3Client>(*config, true, false))
+    , client(make_ref<Aws::S3::S3Client>(
+            profile == ""
+            ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
+                std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
+            : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
+                std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
+            *config, true, false))
 {
 }
 
@@ -148,6 +156,7 @@ S3Helper::DownloadResult S3Helper::getObject(
 
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
+    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
@@ -163,7 +172,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(region)
+        , s3Helper(profile, region)
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 08a7fbf96e9..4f996400343 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,7 +14,7 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & region);
+    S3Helper(const std::string & profile, const std::string & region);
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
 

From 7474ac871b73bd4e739a547b4870b33d625d4c70 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 15:17:11 +0100
Subject: [PATCH 0676/2196] nix copy: Abbreviate "daemon"

---
 src/libstore/store-api.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 3631e1b3003..8146513a4b6 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -577,9 +577,9 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     auto dstUri = dstStore->getUri();
 
     Activity act(*logger, lvlInfo, actCopyPath,
-        srcUri == "local"
+        srcUri == "local" || srcUri == "daemon"
           ? fmt("copying path '%s' to '%s'", storePath, dstUri)
-          : dstUri == "local"
+          : dstUri == "local" || dstUri == "daemon"
             ? fmt("copying path '%s' from '%s'", storePath, srcUri)
             : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
         {storePath, srcUri, dstUri});

From ea94a87493df0a60cadba291e31bbe7c6847c0c5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:13:45 +0100
Subject: [PATCH 0677/2196] install-darwin-multi-user.sh: Remove superfluous
 nix.conf settings

---
 scripts/install-darwin-multi-user.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index cea25eb8adf..6369b68ab91 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -747,14 +747,6 @@ build-users-group = $NIX_BUILD_GROUP_NAME
 max-jobs = $NIX_USER_COUNT
 cores = 1
 sandbox = false
-
-binary-caches = https://cache.nixos.org/
-trusted-binary-caches =
-binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-signed-binary-caches = *
-
-trusted-users = root
-allowed-users = *
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \
           install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf

From 7a2b64e55c7d57707f4d1ed54ee21bf69d0d0d16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:29:54 +0100
Subject: [PATCH 0678/2196] binary-cache-public-keys -> trusted-public-keys

The name had become a misnomer since it's not only for substitution
from binary caches, but when adding/copying any
(non-content-addressed) path to a store.
---
 doc/manual/command-ref/conf-file.xml | 10 +++++-----
 src/libstore/crypto.cc               |  2 +-
 src/libstore/globals.hh              |  7 ++++---
 tests/binary-cache.sh                |  8 ++++----
 tests/signing.sh                     | 12 ++++++------
 5 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index fb4d8cefc4d..a28f7089914 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -406,17 +406,17 @@ false</literal>.</para>
 
     <listitem><para>If set to <literal>*</literal> (the default), Nix
     will only download binaries if they are signed using one of the
-    keys listed in <option>binary-cache-public-keys</option>. Set to
+    keys listed in <option>trusted-public-keys</option>. Set to
     the empty string to disable signature checking.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>binary-cache-public-keys</literal></term>
+  <varlistentry><term><literal>trusted-public-keys</literal></term>
 
-    <listitem><para>A whitespace-separated list of public keys
-    corresponding to the secret keys trusted to sign binary
-    caches. For example:
+    <listitem><para>A whitespace-separated list of public keys. When
+    paths are copied from another Nix store (such as a binary cache),
+    they must be signed with one of these keys. For example:
     <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
     hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
 
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index f56a6adab9c..9ec8abd228e 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -105,7 +105,7 @@ PublicKeys getDefaultPublicKeys()
 
     // FIXME: filter duplicates
 
-    for (auto s : settings.binaryCachePublicKeys.get()) {
+    for (auto s : settings.trustedPublicKeys.get()) {
         PublicKey key(s);
         publicKeys.emplace(key.name, key);
     }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index a4aa842d70f..70c01bb3266 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -259,10 +259,11 @@ public:
     Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
         "Whether to fail if repeated builds produce different output."};
 
-    Setting<Strings> binaryCachePublicKeys{this,
+    Setting<Strings> trustedPublicKeys{this,
         {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
-        "binary-cache-public-keys",
-        "Trusted public keys for secure substitution."};
+        "trusted-public-keys",
+        "Trusted public keys for secure substitution.",
+        {"binary-cache-public-keys"}};
 
     Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
         "Secret keys with which to sign local builds."};
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 2a044d2edc5..f7c0b2f7891 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -131,11 +131,11 @@ clearCacheCache
 clearStore
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$badKey")
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$badKey")
 
 
 # It should succeed if we provide the correct key.
-nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$otherKey $publicKey"
+nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$otherKey $publicKey"
 
 
 # It should fail if we corrupt the .narinfo.
@@ -152,10 +152,10 @@ done
 
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey")
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option signed-binary-caches '*' --option trusted-public-keys "$publicKey")
 
 # If we provide a bad and a good binary cache, it should succeed.
 
-nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey"
+nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$publicKey"
 
 fi # HAVE_LIBSODIUM
diff --git a/tests/signing.sh b/tests/signing.sh
index bef27ac7a58..39aaa1e765b 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -22,13 +22,13 @@ nix verify -r $outPath
 
 expect 2 nix verify -r $outPath --sigs-needed 1
 
-nix verify -r $outPath --sigs-needed 1 --binary-cache-public-keys $pk1
+nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1
 
-expect 2 nix verify -r $outPath --sigs-needed 2 --binary-cache-public-keys $pk1
+expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1
 
-nix verify -r $outPath --sigs-needed 2 --binary-cache-public-keys "$pk1 $pk2"
+nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
-nix verify --all --sigs-needed 2 --binary-cache-public-keys "$pk1 $pk2"
+nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
 
 # Build something unsigned.
 outPath2=$(nix-build simple.nix --no-out-link)
@@ -45,12 +45,12 @@ nix verify -r $outPath2
 
 expect 2 nix verify -r $outPath2 --sigs-needed 1
 
-expect 2 nix verify -r $outPath2 --sigs-needed 1 --binary-cache-public-keys $pk1
+expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
 # Test "nix sign-paths".
 nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2
 
-nix verify -r $outPath2 --sigs-needed 1 --binary-cache-public-keys $pk1
+nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
 # Copy to a binary cache.
 nix copy --to file://$cacheDir $outPath2

From d0b88db44138c6348bb8ed587286f6016ea11f4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:32:58 +0100
Subject: [PATCH 0679/2196] Cleanup

---
 src/libutil/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 96c0cd78383..197df0c44aa 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -221,9 +221,9 @@ Path readLink(const Path & path)
         ssize_t rlSize = readlink(path.c_str(), buf, bufSize);
         if (rlSize == -1)
             if (errno == EINVAL)
-                throw Error(format("'%1%' is not a symlink") % path);
+                throw Error("'%1%' is not a symlink", path);
             else
-                throw SysError(format("reading symbolic link '%1%'") % path);
+                throw SysError("reading symbolic link '%1%'", path);
         else if (rlSize < bufSize)
             return string(buf, rlSize);
     }

From 91a19876073a2ed8fef2139fba906cfac1e96f83 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:44:07 +0100
Subject: [PATCH 0680/2196] signed-binary-caches -> require-sigs

Unlike signed-binary-caches (which could only be '*' or ''),
require-sigs is a proper Boolean option. The default is true.
---
 doc/manual/command-ref/conf-file.xml | 13 ++++++++-----
 src/libstore/globals.hh              |  5 +++++
 src/libstore/local-store.hh          |  2 +-
 tests/binary-cache.sh                | 18 +++++++++---------
 tests/repair.sh                      |  4 ++--
 5 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index a28f7089914..c3a9cc56063 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -402,12 +402,15 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>signed-binary-caches</literal></term>
+  <varlistentry><term><literal>require-sigs</literal></term>
 
-    <listitem><para>If set to <literal>*</literal> (the default), Nix
-    will only download binaries if they are signed using one of the
-    keys listed in <option>trusted-public-keys</option>. Set to
-    the empty string to disable signature checking.</para></listitem>
+    <listitem><para>If set to <literal>true</literal> (the default),
+    any non-content-addressed path added or copied to the Nix store
+    (e.g. when substituting from a binary cache) must have a valid
+    signature, that is, be signed using one of the keys listed in
+    <option>trusted-public-keys</option>. Set to
+    <literal>false</literal> to disable signature
+    checking.</para></listitem>
 
   </varlistentry>
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 70c01bb3266..5c857cbb6a9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -281,6 +281,11 @@ public:
     Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches",
         "Obsolete."};
 
+    Setting<bool> requireSigs{this, signedBinaryCaches == "*", "require-sigs",
+        "Whether to check that any non-content-addressed path added to the "
+        "Nix store has a valid signature (that is, one signed using a key "
+        "listed in 'trusted-public-keys'."};
+
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 0a3841133e5..d35cd1a949e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -97,7 +97,7 @@ public:
 private:
 
     Setting<bool> requireSigs{(Store*) this,
-        settings.signedBinaryCaches != "", // FIXME
+        settings.requireSigs,
         "require-sigs", "whether store paths should have a trusted signature on import"};
 
     PublicKeys publicKeys;
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index f7c0b2f7891..dd8dae687e9 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -18,7 +18,7 @@ basicTests() {
 
     nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
 
-    nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '' -r $outPath
+    nix-store --option binary-caches "file://$cacheDir" --no-require-sigs -r $outPath
 
     [ -x $outPath/program ]
 
@@ -34,7 +34,7 @@ basicTests() {
     x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
     [ -z "$x" ]
 
-    nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '' -r $outPath
+    nix-store --option binary-caches "file://$cacheDir" --no-require-sigs -r $outPath
 
     nix-store --check-validity $outPath
     nix-store -qR $outPath | grep input-2
@@ -63,7 +63,7 @@ mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar
 
-nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --option binary-caches "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "hash mismatch" $TEST_ROOT/log
 
 mv $nar.good $nar
@@ -99,7 +99,7 @@ clearStore
 
 rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
-nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --option binary-caches "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "copying path" $TEST_ROOT/log
 
 
@@ -124,18 +124,18 @@ nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 clearStore
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' )
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir")
 
 
 # And it should fail if we provide an incorrect key.
 clearStore
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$badKey")
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option trusted-public-keys "$badKey")
 
 
 # It should succeed if we provide the correct key.
-nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$otherKey $publicKey"
+nix-store -r $outPath --option binary-caches "file://$cacheDir" --option trusted-public-keys "$otherKey $publicKey"
 
 
 # It should fail if we corrupt the .narinfo.
@@ -152,10 +152,10 @@ done
 
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option signed-binary-caches '*' --option trusted-public-keys "$publicKey")
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option trusted-public-keys "$publicKey")
 
 # If we provide a bad and a good binary cache, it should succeed.
 
-nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option signed-binary-caches '*' --option trusted-public-keys "$publicKey"
+nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option trusted-public-keys "$publicKey"
 
 fi # HAVE_LIBSODIUM
diff --git a/tests/repair.sh b/tests/repair.sh
index 7c928e3be73..7f9f97fd06e 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -51,7 +51,7 @@ nix copy --to file://$cacheDir $path
 chmod u+w $path2
 rm -rf $path2
 
-nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir" --option signed-binary-caches ''
+nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir" --no-require-sigs
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2
@@ -69,7 +69,7 @@ if nix-store --verify-path $path2; then
     exit 1
 fi
 
-nix-store --repair-path $path2 --option binary-caches "file://$cacheDir" --option signed-binary-caches ''
+nix-store --repair-path $path2 --option binary-caches "file://$cacheDir" --no-require-sigs
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2

From 193330d321d3e394d1ce01c7e1dbea28ace68323 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:50:49 +0100
Subject: [PATCH 0681/2196] Test: Replace --option with the corresponding flag

---
 tests/binary-cache.sh     | 30 +++++++++++++++---------------
 tests/fetchurl.sh         |  6 +++---
 tests/linux-sandbox.sh    |  2 +-
 tests/logging.sh          |  2 +-
 tests/multiple-outputs.sh |  2 +-
 tests/optimise-store.sh   |  4 ++--
 tests/repair.sh           |  4 ++--
 tests/restricted.sh       | 20 ++++++++++----------
 tests/setuid.nix          | 16 ++++++++--------
 tests/shell.shebang.sh    |  2 +-
 tests/timeout.sh          |  2 +-
 11 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index dd8dae687e9..7365a550e57 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -16,9 +16,9 @@ basicTests() {
     clearStore
     clearCacheCache
 
-    nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
+    nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
 
-    nix-store --option binary-caches "file://$cacheDir" --no-require-sigs -r $outPath
+    nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath
 
     [ -x $outPath/program ]
 
@@ -28,13 +28,13 @@ basicTests() {
     clearCacheCache
     echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
 
-    nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
-    nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
+    nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
+    nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
 
     x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
     [ -z "$x" ]
 
-    nix-store --option binary-caches "file://$cacheDir" --no-require-sigs -r $outPath
+    nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath
 
     nix-store --check-validity $outPath
     nix-store -qR $outPath | grep input-2
@@ -63,7 +63,7 @@ mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar
 
-nix-build --option binary-caches "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "hash mismatch" $TEST_ROOT/log
 
 mv $nar.good $nar
@@ -73,7 +73,7 @@ mv $nar.good $nar
 clearStore
 clearCacheCache
 
-if nix-store --option binary-caches "file://$cacheDir" -r $outPath; then
+if nix-store --substituters "file://$cacheDir" -r $outPath; then
     echo "unsigned binary cache incorrectly accepted"
     exit 1
 fi
@@ -83,12 +83,12 @@ fi
 # corresponding NAR has disappeared.
 clearStore
 
-nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
+nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info
 
 mkdir $cacheDir/tmp
 mv $cacheDir/*.nar* $cacheDir/tmp/
 
-NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
+NIX_DEBUG_SUBST=1 nix-build --substituters "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
 
 mv $cacheDir/tmp/* $cacheDir/
 
@@ -99,7 +99,7 @@ clearStore
 
 rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
-nix-build --option binary-caches "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "copying path" $TEST_ROOT/log
 
 
@@ -124,18 +124,18 @@ nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 clearStore
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir")
+(! nix-store -r $outPath --substituters "file://$cacheDir")
 
 
 # And it should fail if we provide an incorrect key.
 clearStore
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option trusted-public-keys "$badKey")
+(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey")
 
 
 # It should succeed if we provide the correct key.
-nix-store -r $outPath --option binary-caches "file://$cacheDir" --option trusted-public-keys "$otherKey $publicKey"
+nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey"
 
 
 # It should fail if we corrupt the .narinfo.
@@ -152,10 +152,10 @@ done
 
 clearCacheCache
 
-(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option trusted-public-keys "$publicKey")
+(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey")
 
 # If we provide a bad and a good binary cache, it should succeed.
 
-nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option trusted-public-keys "$publicKey"
+nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
 
 fi # HAVE_LIBSODIUM
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 7f2de907049..9bbf044f732 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
 # Test fetching a flat file.
 hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --option hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
 
 cmp $outPath fetchurl.sh
 
@@ -14,7 +14,7 @@ clearStore
 
 hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --option hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
 
 cmp $outPath fetchurl.sh
 
@@ -29,7 +29,7 @@ rm -rf $mirror
 mkdir -p $mirror/sha512
 ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
 
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --option hashed-mirrors "file://$mirror")
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
 
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 0691c30be1c..4a686bb59a3 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -16,7 +16,7 @@ rm -rf $TEST_ROOT/store0
 export NIX_STORE_DIR=/my/store
 export NIX_REMOTE=$TEST_ROOT/store0
 
-outPath=$(nix-build dependencies.nix --no-out-link --option sandbox-paths /nix/store)
+outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store)
 
 [[ $outPath =~ /my/store/.*-dependencies ]]
 
diff --git a/tests/logging.sh b/tests/logging.sh
index d38136531ba..c894ad3ff07 100644
--- a/tests/logging.sh
+++ b/tests/logging.sh
@@ -11,5 +11,5 @@ path=$(nix-build dependencies.nix --no-out-link)
 clearStore
 rm -rf $NIX_LOG_DIR
 (! nix-store -l $path)
-nix-build dependencies.nix --no-out-link --option compress-build-log true
+nix-build dependencies.nix --no-out-link --compress-build-log
 [ "$(nix-store -l $path)" = FOO ]
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index 47698b10381..ced6d758f4a 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -59,5 +59,5 @@ fi
 
 echo "collecting garbage..."
 rm $TEST_ROOT/result*
-nix-store --gc --option keep-derivations true --option keep-outputs true
+nix-store --gc --keep-derivations --keep-outputs
 nix-store --gc --print-roots
diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh
index bd88662bc37..61e3df2f9f7 100644
--- a/tests/optimise-store.sh
+++ b/tests/optimise-store.sh
@@ -2,8 +2,8 @@ source common.sh
 
 clearStore
 
-outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true)
-outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --option auto-optimise-store true)
+outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store)
+outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store)
 
 inode1="$(stat --format=%i $outPath1/foo)"
 inode2="$(stat --format=%i $outPath2/foo)"
diff --git a/tests/repair.sh b/tests/repair.sh
index 7f9f97fd06e..ec7ad5dcaff 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -51,7 +51,7 @@ nix copy --to file://$cacheDir $path
 chmod u+w $path2
 rm -rf $path2
 
-nix-store --verify --check-contents --repair --option binary-caches "file://$cacheDir" --no-require-sigs
+nix-store --verify --check-contents --repair --substituters "file://$cacheDir" --no-require-sigs
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2
@@ -69,7 +69,7 @@ if nix-store --verify-path $path2; then
     exit 1
 fi
 
-nix-store --repair-path $path2 --option binary-caches "file://$cacheDir" --no-require-sigs
+nix-store --repair-path $path2 --substituters "file://$cacheDir" --no-require-sigs
 
 if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then
     echo "path not repaired properly" >&2
diff --git a/tests/restricted.sh b/tests/restricted.sh
index a297847cc8e..c063c8693d5 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -2,19 +2,19 @@ source common.sh
 
 clearStore
 
-nix-instantiate --option restrict-eval true --eval -E '1 + 2'
-(! nix-instantiate --option restrict-eval true ./simple.nix)
-nix-instantiate --option restrict-eval true ./simple.nix -I src=.
-nix-instantiate --option restrict-eval true ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh
+nix-instantiate --restrict-eval --eval -E '1 + 2'
+(! nix-instantiate --restrict-eval ./simple.nix)
+nix-instantiate --restrict-eval ./simple.nix -I src=.
+nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh
 
-(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix')
-nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix' -I src=..
+(! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix')
+nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=..
 
-(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost')
-nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost' -I src=../src
+(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost')
+nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost' -I src=../src
 
-(! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
-nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
+(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
+nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
 
 p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)")
 cmp $p restricted.sh
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 7a12b4fb316..c982d9cf036 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -20,7 +20,7 @@ makeTest {
       startAll;
 
       # Copying to /tmp should succeed.
-      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
       ")\' ');
@@ -30,7 +30,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # Creating a setuid binary should fail.
-      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 4755 /tmp/id
@@ -41,7 +41,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # Creating a setgid binary should fail.
-      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
+      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 2755 /tmp/id
@@ -52,7 +52,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # The checks should also work on 32-bit binaries.
-      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
+      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         chmod 2755 /tmp/id
@@ -63,7 +63,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # The tests above use fchmodat(). Test chmod() as well.
-      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"chmod 0666, qw(/tmp/id) or die\"
@@ -73,7 +73,7 @@ makeTest {
 
       $machine->succeed("rm /tmp/id");
 
-      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"chmod 04755, qw(/tmp/id) or die\"
@@ -84,7 +84,7 @@ makeTest {
       $machine->succeed("rm /tmp/id");
 
       # And test fchmod().
-      $machine->succeed('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\"
@@ -94,7 +94,7 @@ makeTest {
 
       $machine->succeed("rm /tmp/id");
 
-      $machine->fail('nix-build --option sandbox false -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
+      $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } "
         mkdir -p $out
         cp ${pkgs.coreutils}/bin/id /tmp/id
         perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\"
diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh
index a6c4bc94592..c8e55ca9b90 100755
--- a/tests/shell.shebang.sh
+++ b/tests/shell.shebang.sh
@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --option use-substitutes false
+#! nix-shell -I nixpkgs=shell.nix --no-use-substitutes
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"
diff --git a/tests/timeout.sh b/tests/timeout.sh
index d3d85200fa5..39ecf0a1a30 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -15,7 +15,7 @@ if ! echo "$messages" | grep -q "timed out"; then
     exit 1
 fi
 
-if nix-build -Q timeout.nix -A infiniteLoop --option max-build-log-size 100; then
+if nix-build -Q timeout.nix -A infiniteLoop --max-build-log-size 100; then
     echo "build should have failed"
     exit 1
 fi

From a3aa850f0f71644ac4559f958e180c3e543c74e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:58:23 +0100
Subject: [PATCH 0682/2196] nix run: Ignore non-directories while setting up
 the chroot

Fixes #1686.
---
 src/nix/run.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 6657a86314b..11acbc3db67 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -195,12 +195,15 @@ void chrootHelper(int argc, char * * argv)
             throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
 
         for (auto entry : readDirectory("/")) {
+            auto src = "/" + entry.name;
+            auto st = lstat(src);
+            if (!S_ISDIR(st.st_mode)) continue;
             Path dst = tmpDir + "/" + entry.name;
             if (pathExists(dst)) continue;
             if (mkdir(dst.c_str(), 0700) == -1)
-                throw SysError(format("creating directory '%s'") % dst);
-            if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-                throw SysError(format("mounting '%s' on '%s'") %  ("/" + entry.name) % dst);
+                throw SysError("creating directory '%s'", dst);
+            if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+                throw SysError("mounting '%s' on '%s'", src, dst);
         }
 
         char * cwd = getcwd(0, 0);

From 4eb9e20028c4e52c23ce6a53fe02cac87171fda6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 17:59:32 +0100
Subject: [PATCH 0683/2196] nix run: Fix accidental removal of /nix/store
 existence check

Parenthetical to #1686, we don't need to create a new root if we can
just bind-mount on top of the existing /nix/store.
---
 src/nix/run.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 11acbc3db67..66f416efcec 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -184,7 +184,7 @@ void chrootHelper(int argc, char * * argv)
        but that doesn't work in a user namespace yet (Ubuntu has a
        patch for this:
        https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */
-    if (true /* !pathExists(storeDir) */) {
+    if (!pathExists(storeDir)) {
         // FIXME: Use overlayfs?
 
         Path tmpDir = createTempDir();

From 1ff01187e230ddab210b173335e82f286626b16e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 18:07:58 +0100
Subject: [PATCH 0684/2196] nix run: Fix "flag '--command' requires 2
 argument(s)"

---
 src/nix/run.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 66f416efcec..ade87e63a49 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -30,8 +30,8 @@ struct CmdRun : InstallablesCommand
             .longName("command")
             .shortName('c')
             .description("command and arguments to be executed; defaults to 'bash'")
-            .arity(ArityAny)
             .labels({"command", "args"})
+            .arity(ArityAny)
             .handler([&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;

From f5f29dc2b7c266875fb9cbb0c8d5cdde3ac4a73c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 18:27:29 +0100
Subject: [PATCH 0685/2196] nix eval --raw: Use coerceToString()

Thus,

  $ nix eval --raw '("foo")'
  foo

  $ nix eval --raw nixpkgs.hello
  /nix/store/1y6ckg6khrdsvll54s5spcmf3w6ka9k4-hello-2.10

  $ nix eval --raw '(/etc/resolv.conf)'
  /nix/store/vml92ama92i8mz013nny461mlvg8mvap-resolv.conf
---
 src/nix/eval.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index e2212869263..0fbeca1c121 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -42,6 +42,10 @@ struct CmdEval : MixJSON, InstallablesCommand
                 "To get the current version of Nixpkgs:",
                 "nix eval --raw nixpkgs.lib.nixpkgsVersion"
             },
+            Example{
+                "To print the store path of the Hello package:",
+                "nix eval --raw nixpkgs.hello"
+            },
         };
     }
 
@@ -56,10 +60,10 @@ struct CmdEval : MixJSON, InstallablesCommand
 
         for (auto & i : installables) {
             auto v = i->toValue(*state);
+            PathSet context;
             if (raw) {
-                std::cout << state->forceString(*v);
+                std::cout << state->coerceToString(noPos, *v, context);
             } else if (json) {
-                PathSet context;
                 auto jsonElem = jsonOut->placeholder();
                 printValueAsJSON(*state, true, *v, jsonElem, context);
             } else {

From 4cde04f476fa0f61680f8e53fe81cc70923689fc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 18:36:36 +0100
Subject: [PATCH 0686/2196] Add tests for "nix run"

---
 tests/local.mk |  3 ++-
 tests/run.nix  | 17 +++++++++++++++++
 tests/run.sh   | 27 +++++++++++++++++++++++++++
 3 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 tests/run.nix
 create mode 100644 tests/run.sh

diff --git a/tests/local.mk b/tests/local.mk
index 2b92ec4e5a8..baf74224bb1 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -18,7 +18,8 @@ nix_tests = \
   structured-attrs.sh \
   fetchGit.sh \
   fetchMercurial.sh \
-  signing.sh
+  signing.sh \
+  run.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/run.nix b/tests/run.nix
new file mode 100644
index 00000000000..77dcbd2a9df
--- /dev/null
+++ b/tests/run.nix
@@ -0,0 +1,17 @@
+with import ./config.nix;
+
+{
+  hello = mkDerivation {
+    name = "hello";
+    buildCommand =
+      ''
+        mkdir -p $out/bin
+        cat > $out/bin/hello <<EOF
+        #! ${shell}
+        who=\$1
+        echo "Hello \''${who:-World} from $out/bin/hello"
+        EOF
+        chmod +x $out/bin/hello
+      '';
+  };
+}
diff --git a/tests/run.sh b/tests/run.sh
new file mode 100644
index 00000000000..1ba181c9cd5
--- /dev/null
+++ b/tests/run.sh
@@ -0,0 +1,27 @@
+source common.sh
+
+clearStore
+clearCache
+
+nix run -f run.nix hello -c hello | grep 'Hello World'
+nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS'
+
+if [[ $(uname) = Linux ]]; then
+
+    chmod -R u+w $TEST_ROOT/store0 || true
+    rm -rf $TEST_ROOT/store0
+
+    clearStore
+
+    path=$(nix eval --raw -f run.nix hello)
+
+    # Note: we need the sandbox paths to ensure that the shell is
+    # visible in the sandbox.
+    nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
+
+    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
+
+    [[ $path/bin/hello = $path2 ]]
+
+    [[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]
+fi

From 8df60b4ea8fab5fd3571e55de95a4b956f0f6105 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 18:51:04 +0100
Subject: [PATCH 0687/2196] Document secret-key-files

---
 doc/manual/command-ref/conf-file.xml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c3a9cc56063..868cca1da40 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -408,9 +408,9 @@ false</literal>.</para>
     any non-content-addressed path added or copied to the Nix store
     (e.g. when substituting from a binary cache) must have a valid
     signature, that is, be signed using one of the keys listed in
-    <option>trusted-public-keys</option>. Set to
-    <literal>false</literal> to disable signature
-    checking.</para></listitem>
+    <option>trusted-public-keys</option> or
+    <option>secret-key-files</option>. Set to <literal>false</literal>
+    to disable signature checking.</para></listitem>
 
   </varlistentry>
 
@@ -426,6 +426,19 @@ false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>secret-key-files</literal></term>
+
+    <listitem><para>A whitespace-separated list of files containing
+    secret (private) keys. These are used to sign locally-built
+    paths. They can be generated using <command>nix-store
+    --generate-binary-cache-key</command>. The corresponding public
+    key can be distributed to other users, who can add it to
+    <option>trusted-public-keys</option> in their
+    <filename>nix.conf</filename>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections

From 0c9718aabc2dda4647e29213a26708eacd71e963 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 19:02:57 +0100
Subject: [PATCH 0688/2196] Add tests for signature checking when copying
 between local stores

---
 tests/signing.sh | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/signing.sh b/tests/signing.sh
index 39aaa1e765b..221dfa35ec0 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -66,3 +66,23 @@ nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2
 info=$(nix path-info --store file://$cacheDir --json $outPath2)
 [[ $info =~ 'cache1.example.org' ]]
 [[ $info =~ 'cache2.example.org' ]]
+
+# Copying to a diverted store should fail due to a lack of valid signatures.
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
+(! nix copy --to $TEST_ROOT/store0 $outPath)
+
+# But succeed if we supply the public keys.
+(nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1)
+
+expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath
+
+nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1
+nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2"
+
+# It should also succeed if we disable signature checking.
+(! nix copy --to $TEST_ROOT/store0 $outPath2)
+nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2
+
+# But signatures should still get copied.
+nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1

From 4fcf44825fbcfbc46fd6dfe48ea09164aa003647 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2017 19:11:02 +0100
Subject: [PATCH 0689/2196] Add tests for verifying/copying content-addressed
 paths

These don't require signatures.
---
 tests/signing.sh | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tests/signing.sh b/tests/signing.sh
index 221dfa35ec0..8e8215618f8 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -52,6 +52,16 @@ nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2
 
 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
+# Build something content-addressed.
+outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0)
+
+[[ $(nix path-info --json $outPathCA) =~ '"ca":"fixed:md5:' ]]
+
+# Content-addressed paths don't need signatures, so they verify
+# regardless of --sigs-needed.
+nix verify $outPathCA
+nix verify $outPathCA --sigs-needed 1000
+
 # Copy to a binary cache.
 nix copy --to file://$cacheDir $outPath2
 
@@ -73,7 +83,7 @@ rm -rf $TEST_ROOT/store0
 (! nix copy --to $TEST_ROOT/store0 $outPath)
 
 # But succeed if we supply the public keys.
-(nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1)
+nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1
 
 expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath
 
@@ -86,3 +96,6 @@ nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2
 
 # But signatures should still get copied.
 nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1
+
+# Content-addressed stuff can be copied without signatures.
+nix copy --to $TEST_ROOT/store0 $outPathCA

From 7536fe31dd8c162026d517521dc49b5d9286bfb1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Nov 2017 18:49:52 +0100
Subject: [PATCH 0690/2196] Add a warning about the 'trusted-users' option

---
 doc/manual/command-ref/conf-file.xml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 868cca1da40..e52cbcd535e 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -543,11 +543,12 @@ password <replaceable>my-password</replaceable>
       <literal>wheel</literal> group. The default is
       <literal>root</literal>.</para>
 
-      <warning><para>The users listed here have the ability to
-      compromise the security of a multi-user Nix store. For instance,
-      they could install Trojan horses subsequently executed by other
-      users. So you should consider carefully whether to add users to
-      this list.</para></warning>
+      <warning><para>Adding a user to <option>trusted-users</option>
+      is essentially equivalent to giving that user root access to the
+      system. For example, the user can set
+      <option>sandbox-paths</option> and thereby obtain read access to
+      directories that are otherwise inacessible to
+      them.</para></warning>
 
     </listitem>
 

From 6cdaa858d03b9c24f54f8157026cebaa02d9ac8a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Nov 2017 18:50:56 +0100
Subject: [PATCH 0691/2196] Propagate flags like --sandbox to the daemon
 properly

---
 src/libstore/globals.cc | 6 +++---
 src/libutil/config.cc   | 6 +++---
 src/libutil/config.hh   | 6 ++++++
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4fa02f92085..d3c96ddd6e6 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -116,17 +116,17 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
     args.mkFlag()
         .longName(name)
         .description("Enable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { value = smEnabled; })
+        .handler([=](std::vector<std::string> ss) { override(smEnabled); })
         .category(category);
     args.mkFlag()
         .longName("no-" + name)
         .description("Disable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { value = smDisabled; })
+        .handler([=](std::vector<std::string> ss) { override(smDisabled); })
         .category(category);
     args.mkFlag()
         .longName("relaxed-" + name)
         .description("Enable sandboxing, but allow builds to disable it.")
-        .handler([=](std::vector<std::string> ss) { value = smRelaxed; })
+        .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
         .category(category);
 }
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 14c4cca031b..d46ca65a386 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -152,7 +152,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .longName(name)
         .description(description)
         .arity(1)
-        .handler([=](std::vector<std::string> ss) { set(ss[0]); })
+        .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); })
         .category(category);
 }
 
@@ -201,12 +201,12 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
     args.mkFlag()
         .longName(name)
         .description(description)
-        .handler([=](std::vector<std::string> ss) { value = true; })
+        .handler([=](std::vector<std::string> ss) { override(true); })
         .category(category);
     args.mkFlag()
         .longName("no-" + name)
         .description(description)
-        .handler([=](std::vector<std::string> ss) { value = false; })
+        .handler([=](std::vector<std::string> ss) { override(false); })
         .category(category);
 }
 
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 99850c1cdfd..9a32af528ec 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -142,6 +142,12 @@ public:
 
     void set(const std::string & str) override;
 
+    virtual void override(const T & v)
+    {
+        overriden = true;
+        value = v;
+    }
+
     std::string to_string() override;
 
     void convertToArg(Args & args, const std::string & category) override;

From d7da6c9ea9459a2fa17c3e409120c5cdfde6afdc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Nov 2017 19:12:47 +0100
Subject: [PATCH 0692/2196] fetchGit/fetchMercurial: Fix directory inclusion
 check

E.g. the existence of .gitignore would cause .git to be included.
---
 src/libexpr/primops/fetchGit.cc       | 5 +++--
 src/libexpr/primops/fetchMercurial.cc | 5 +++--
 tests/fetchGit.sh                     | 4 +++-
 tests/fetchMercurial.sh               | 4 +++-
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index bca68ed7209..fd3e84c292c 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -56,8 +56,9 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 auto st = lstat(p);
 
                 if (S_ISDIR(st.st_mode)) {
-                    auto i = files.lower_bound(file);
-                    return i != files.end() && hasPrefix(*i, file);
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
                 }
 
                 return files.count(file);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 7def7103bf3..a317476c582 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -52,8 +52,9 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 auto st = lstat(p);
 
                 if (S_ISDIR(st.st_mode)) {
-                    auto i = files.lower_bound(file);
-                    return i != files.end() && hasPrefix(*i, file);
+                    auto prefix = file + "/";
+                    auto i = files.lower_bound(prefix);
+                    return i != files.end() && hasPrefix(*i, prefix);
                 }
 
                 return files.count(file);
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index f13de89f7ff..7b13b587def 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -16,7 +16,8 @@ git -C $repo config user.email "foobar@example.com"
 git -C $repo config user.name "Foobar"
 
 echo utrecht > $repo/hello
-git -C $repo add hello
+touch $repo/.gitignore
+git -C $repo add hello .gitignore
 git -C $repo commit -m 'Bla1'
 rev1=$(git -C $repo rev-parse HEAD)
 
@@ -68,6 +69,7 @@ path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
 [ ! -e $path2/dir2/bar ]
+[ ! -e $path2/.git ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
 [[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 6cfc0a708e8..271350ecd17 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -16,7 +16,8 @@ echo '[ui]' >> $repo/.hg/hgrc
 echo 'username = Foobar <foobar@example.org>' >> $repo/.hg/hgrc
 
 echo utrecht > $repo/hello
-hg add --cwd $repo hello
+touch $repo/.hgignore
+hg add --cwd $repo hello .hgignore
 hg commit --cwd $repo -m 'Bla1'
 rev1=$(hg log --cwd $repo -r tip --template '{node}')
 
@@ -69,6 +70,7 @@ path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
 [ ! -e $path2/dir2/bar ]
+[ ! -e $path2/.hg ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
 [[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]

From 0fc3e581e0585e377d4b42e343b0487606add547 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Nov 2017 19:43:12 +0100
Subject: [PATCH 0693/2196] tests/run.sh: Fix /build conflict when building in
 a sandbox

https://hydra.nixos.org/build/64519355
---
 tests/run.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/run.sh b/tests/run.sh
index 1ba181c9cd5..784d29183cf 100644
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -17,7 +17,9 @@ if [[ $(uname) = Linux ]]; then
 
     # Note: we need the sandbox paths to ensure that the shell is
     # visible in the sandbox.
-    nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
+    nix run --sandbox-build-dir /build-tmp \
+        --sandbox-paths '/nix? /bin? /lib? /usr?' \
+        --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
 
     path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
 

From eedbc4e06c017d84814b4c1fad8c6b6db958f3da Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 23 Nov 2017 21:50:01 -0500
Subject: [PATCH 0694/2196] fetchGit: Ignore tarballTtl if rev is set and not
 in the repo.

Fixes #1697.
---
 src/libexpr/primops/fetchGit.cc | 59 +++++++++++++++++++--------------
 tests/common.sh.in              |  1 +
 tests/fetchGit.sh               |  7 ++++
 3 files changed, 43 insertions(+), 24 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index fd3e84c292c..e92e0638031 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -89,32 +89,43 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     Path localRefFile = cacheDir + "/refs/heads/" + localRef;
 
-    /* If the local ref is older than ‘tarball-ttl’ seconds, do a git
-       fetch to update the local ref to the remote ref. */
+    bool doFetch;
     time_t now = time(0);
-    struct stat st;
-    if (stat(localRefFile.c_str(), &st) != 0 ||
-        st.st_mtime <= now - settings.tarballTtl)
-    {
-        if (rev == "" ||
-            chomp(runProgram(
-                RunOptions("git", { "-C", cacheDir, "cat-file", "-t", rev })
-                .killStderr(true)).second) != "commit")
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-            // FIXME: git stderr messes up our progress indicator, so
-            // we're using --quiet for now. Should process its stderr.
-            runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
-
-            struct timeval times[2];
-            times[0].tv_sec = now;
-            times[0].tv_usec = 0;
-            times[1].tv_sec = now;
-            times[1].tv_usec = 0;
-
-            utimes(localRefFile.c_str(), times);
+    /* If a rev was specified, we need to fetch if it's not in the
+       repo. */
+    if (rev != "") {
+        try {
+            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
+            doFetch = false;
+        } catch (ExecError & e) {
+            if (WIFEXITED(e.status)) {
+                doFetch = true;
+            } else {
+                throw;
+            }
         }
+    } else {
+        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+           git fetch to update the local ref to the remote ref. */
+        struct stat st;
+        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+            st.st_mtime <= now - settings.tarballTtl;
+    }
+    if (doFetch)
+    {
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+
+        struct timeval times[2];
+        times[0].tv_sec = now;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = now;
+        times[1].tv_usec = 0;
+
+        utimes(localRefFile.c_str(), times);
     }
 
     // FIXME: check whether rev is an ancestor of ref.
diff --git a/tests/common.sh.in b/tests/common.sh.in
index ca6df25362d..09f2949141a 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -21,6 +21,7 @@ export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
+unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 
 export PATH=@bindir@:$PATH
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 7b13b587def..09e4f742668 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -86,3 +86,10 @@ git -C $repo commit -m 'Bla3' -a
 
 path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $path2 = $path4 ]]
+
+# tarball-ttl should be ignored if we specify a rev
+echo delft > $repo/hello
+git -C $repo add hello
+git -C $repo commit -m 'Bla4'
+rev3=$(git -C $repo rev-parse HEAD)
+nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null

From 90948a4e3a64492b7d117d93657221fa7b598e6e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Nov 2017 18:07:29 +0100
Subject: [PATCH 0695/2196] nix-shell/nix-build: Support .drv files again

Fixes #1663.

Also handle '!<output-name>' (#1694).
---
 src/libexpr/get-drvs.cc    | 28 ++++++++++++++++++++++++++++
 src/libexpr/get-drvs.hh    |  1 +
 src/nix-build/nix-build.cc |  6 ++++--
 tests/multiple-outputs.sh  | 15 ++++++++++++++-
 tests/nix-build.sh         |  8 +++++++-
 tests/nix-shell.sh         |  7 +++++++
 6 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index d5bc42352a2..d38ed2df3b1 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -1,6 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
+#include "derivations.hh"
 
 #include <cstring>
 #include <regex>
@@ -15,6 +16,33 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 }
 
 
+DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
+    : state(&state), attrs(nullptr), attrPath("")
+{
+    auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
+
+    drvPath = spec.first;
+
+    auto drv = store->derivationFromPath(drvPath);
+
+    name = storePathToName(drvPath);
+
+    if (spec.second.size() > 1)
+        throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
+
+    outputName =
+        spec.second.empty()
+        ? get(drv.env, "outputName", "out")
+        : *spec.second.begin();
+
+    auto i = drv.outputs.find(outputName);
+    if (i == drv.outputs.end())
+        throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
+
+    outPath = i->second.path;
+}
+
+
 string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 32294e45875..4d9128e3f44 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -37,6 +37,7 @@ public:
 
     DrvInfo(EvalState & state) : state(&state) { };
     DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
+    DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     string queryName() const;
     string querySystem() const;
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 21b0a18dd88..58366daa6e8 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -263,6 +263,8 @@ void mainWrapped(int argc, char * * argv)
     if (runEnv)
         setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1);
 
+    DrvInfos drvs;
+
     /* Parse the expressions. */
     std::vector<Expr *> exprs;
 
@@ -272,6 +274,8 @@ void mainWrapped(int argc, char * * argv)
         for (auto i : left) {
             if (fromArgs)
                 exprs.push_back(state.parseExprFromString(i, absPath(".")));
+            else if (store->isStorePath(i) && std::regex_match(i, std::regex(".*\\.drv(!.*)?")))
+                drvs.push_back(DrvInfo(state, store, i));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
@@ -280,8 +284,6 @@ void mainWrapped(int argc, char * * argv)
         }
 
     /* Evaluate them into derivations. */
-    DrvInfos drvs;
-
     if (attrPaths.empty()) attrPaths = {""};
 
     for (auto e : exprs) {
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index ced6d758f4a..bedbc39a4eb 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -2,6 +2,8 @@ source common.sh
 
 clearStore
 
+rm -f $TEST_ROOT/result*
+
 # Test whether read-only evaluation works when referring to the
 # ‘drvPath’ attribute.
 echo "evaluating c..."
@@ -28,7 +30,7 @@ echo "output path is $outPath"
 [ "$(cat "$outPath"/file)" = "success" ]
 
 # Test nix-build on a derivation with multiple outputs.
-nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result
+outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result)
 [ -e $TEST_ROOT/result-first ]
 (! [ -e $TEST_ROOT/result-second ])
 nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
@@ -37,6 +39,17 @@ nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
 [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ]
 hash1=$(nix-store -q --hash $TEST_ROOT/result-second)
 
+outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a) --no-out-link)
+[[ $outPath1 = $outPath2 ]]
+
+outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.first) --no-out-link)
+[[ $outPath1 = $outPath2 ]]
+
+outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-out-link)
+[[ $(cat $outPath2/file) = second ]]
+
+[[ $(nix-build $(nix-instantiate multiple-outputs.nix -A a.all) --no-out-link | wc -l) -eq 2 ]]
+
 # Delete one of the outputs and rebuild it.  This will cause a hash
 # rewrite.
 nix-store --delete $TEST_ROOT/result-second --ignore-liveness
diff --git a/tests/nix-build.sh b/tests/nix-build.sh
index dc0e99c7362..39526486319 100644
--- a/tests/nix-build.sh
+++ b/tests/nix-build.sh
@@ -2,7 +2,7 @@ source common.sh
 
 clearStore
 
-nix-build dependencies.nix -o $TEST_ROOT/result
+outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result)
 test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR
 
 # The result should be retained by a GC.
@@ -17,3 +17,9 @@ test -e $target/foobar
 rm $TEST_ROOT/result
 nix-store --gc
 if test -e $target/foobar; then false; fi
+
+outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
+[[ $outPath = $outPath2 ]]
+
+outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
+[[ $outPath = $outPath2 ]]
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index f0f34a5f870..f7cac6ecf8e 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -10,6 +10,13 @@ output=$(nix-shell --pure shell.nix -A shellDrv --run \
 
 [ "$output" = " - foo - bar" ]
 
+# Test nix-shell on a .drv
+[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+
+[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+
 # Test nix-shell -p
 output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"')
 [ "$output" = "foo bar" ]

From 549c3706a5d63e10d908b43da8479331a844018f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 24 Nov 2017 18:08:50 +0100
Subject: [PATCH 0696/2196] nix path-info: Show URL of NARs in binary caches

---
 src/libstore/store-api.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8146513a4b6..77ab87ef728 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -516,6 +516,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
                     std::shared_ptr<const ValidPathInfo>(info));
 
                 if (narInfo) {
+                    if (!narInfo->url.empty())
+                        jsonPath.attr("url", narInfo->url);
                     if (narInfo->fileHash)
                         jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
                     if (narInfo->fileSize)

From 6567ab95a0b5cdca3f8b22dbbcd98ec5d18b63b0 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Sat, 25 Nov 2017 01:03:23 +0100
Subject: [PATCH 0697/2196] build-remote: Fix missing log output

The storeUri variable in the build-remote hook is declared very much to
the start of the main function and a bunch of lines later, the same
variable gets checked via hasPrefix() but it gets assigned *after* that
check when the most suitable machine for the build was choosen.

So I guess this was just a typo in d16fd2497374671c92cb877f9570d65783a7
and what we really want is to either checkd the prefix *after* assigning
storeUri or use bestMachine->storeUri directly.

I choose the latter, because the former could introduce even more
regressions if the try block where the variable gets assigned terminates
early.

Nevertheless, the reason why the log output didn't work is because
hasPrefix() checked for "ssh://" in front of storeUri, but if the
storeUri isn't set correctly (or at all), we don't get the log file
descriptor set up properly, leading to no log output.

I've adjusted the remote-builds test to include a regression test for
this, so that we can make sure we get a build output when using remote
builds.

In addition to that I've tested this with two of my build farms and the
build logs are emitted correctly again.

Signed-off-by: aszlig <aszlig@nix.build>
---
 src/build-remote/build-remote.cc | 2 +-
 tests/remote-builds.nix          | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 6e05e165545..445006b327f 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -177,7 +177,7 @@ int main (int argc, char * * argv)
                     Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
 
                     Store::Params storeParams;
-                    if (hasPrefix(storeUri, "ssh://")) {
+                    if (hasPrefix(bestMachine->storeUri, "ssh://")) {
                         storeParams["max-connections"] ="1";
                         storeParams["log-fd"] = "4";
                         if (bestMachine->sshKey != "")
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 39bd090e43e..58a26d8b618 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -85,7 +85,10 @@ in
       }
 
       # Perform a build and check that it was performed on the slave.
-      my $out = $client->succeed("nix-build ${expr nodes.client.config 1}");
+      my $out = $client->succeed(
+        "nix-build ${expr nodes.client.config 1} 2> build-output",
+        "grep -q Hello build-output"
+      );
       $slave1->succeed("test -e $out");
 
       # And a parallel build.

From 21f515724c74d802bb89fdeb9e6dd24ad8497fb7 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 27 Nov 2017 18:27:28 -0600
Subject: [PATCH 0698/2196] replace vendored linenoise with linenoise-ng

---
 src/linenoise/ConvertUTF.cpp |  542 ++++++
 src/linenoise/ConvertUTF.h   |  162 ++
 src/linenoise/LICENSE        |   79 +-
 src/linenoise/linenoise.c    | 1199 ------------
 src/linenoise/linenoise.cpp  | 3457 ++++++++++++++++++++++++++++++++++
 src/linenoise/linenoise.h    |   82 +-
 src/linenoise/wcwidth.cpp    |  315 ++++
 src/nix/local.mk             |    2 +-
 8 files changed, 4578 insertions(+), 1260 deletions(-)
 create mode 100644 src/linenoise/ConvertUTF.cpp
 create mode 100755 src/linenoise/ConvertUTF.h
 delete mode 100644 src/linenoise/linenoise.c
 create mode 100644 src/linenoise/linenoise.cpp
 create mode 100644 src/linenoise/wcwidth.cpp

diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp
new file mode 100644
index 00000000000..f7e5915d5e8
--- /dev/null
+++ b/src/linenoise/ConvertUTF.cpp
@@ -0,0 +1,542 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+    Sept 2001: fixed const & error conditions per
+        mods suggested by S. Parent & A. Lillich.
+    June 2002: Tim Dodd added detection and handling of incomplete
+        source sequences, enhanced error detection, added casts
+        to eliminate compiler warnings.
+    July 2003: slight mods to back out aggressive FFFE detection.
+    Jan 2004: updated switches in from-UTF8 conversions.
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
+
+    See the header file "ConvertUTF.h" for complete documentation.
+
+------------------------------------------------------------------------ */
+
+#include "ConvertUTF.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+
+namespace linenoise_ng {
+
+static const int halfShift  = 10; /* used for shifting by 10 bits */
+
+static const UTF32 halfBase = 0x0010000UL;
+static const UTF32 halfMask = 0x3FFUL;
+
+#define UNI_SUR_HIGH_START  (UTF32)0xD800
+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
+#define UNI_SUR_LOW_START   (UTF32)0xDC00
+#define UNI_SUR_LOW_END     (UTF32)0xDFFF
+#define false      0
+#define true        1
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF16 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    char16_t* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        if (target >= targetEnd) {
+            result = targetExhausted; break;
+        }
+        ch = *source++;
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_LEGAL_UTF32) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                --source; /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF32 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF32* target = *targetStart;
+    UTF32 ch, ch2;
+    while (source < sourceEnd) {
+        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        if (target >= targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        *target++ = ch;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+#ifdef CVTUTF_DEBUG
+if (result == sourceIllegal) {
+    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
+    fflush(stderr);
+}
+#endif
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
+ * left as-is for anyone who may want to do such conversion, which was
+ * allowed in earlier algorithms.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/*
+ * Magic values subtracted from a buffer value during UTF8 conversion.
+ * This table contains as many values as there might be trailing bytes
+ * in a UTF-8 sequence.
+ */
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
+                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+
+/*
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
+ * into the first byte, depending on how many bytes follow.  There are
+ * as many entries in this table as there are UTF-8 sequence types.
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
+ */
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+/* --------------------------------------------------------------------- */
+
+/* The interface converts a whole buffer to avoid function-call overhead.
+ * Constants have been gathered. Loops & conditionals have been removed as
+ * much as possible for efficiency, in favor of drop-through switches.
+ * (See "Note A" at the bottom of the file for equivalent code.)
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
+ * into an inline function.
+ */
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF16toUTF8 (
+        const UTF16** sourceStart, const UTF16* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF16* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
+        ch = *source++;
+        /* If we have a surrogate pair, convert to UTF32 first. */
+        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
+            /* If the 16 bits following the high surrogate are in the source buffer... */
+            if (source < sourceEnd) {
+                UTF32 ch2 = *source;
+                /* If it's a low surrogate, convert to UTF32. */
+                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
+                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
+                    ++source;
+                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
+                    --source; /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                }
+            } else { /* We don't have the 16 bits following the high surrogate. */
+                --source; /* return to the high surrogate */
+                result = sourceExhausted;
+                break;
+            }
+        } else if (flags == strictConversion) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /* Figure out how many bytes the result will require */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+        }
+
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            source = oldSource; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns false.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+
+static Boolean isLegalUTF8(const UTF8 *source, int length) {
+    UTF8 a;
+    const UTF8 *srcptr = source+length;
+    switch (length) {
+    default: return false;
+        /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+
+        switch (*source) {
+            /* no fall-through in this inner switch */
+            case 0xE0: if (a < 0xA0) return false; break;
+            case 0xED: if (a > 0x9F) return false; break;
+            case 0xF0: if (a < 0x90) return false; break;
+            case 0xF4: if (a > 0x8F) return false; break;
+            default:   if (a < 0x80) return false;
+        }
+
+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+    }
+    if (*source > 0xF4) return false;
+    return true;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Exported function to return whether a UTF-8 sequence is legal or not.
+ * This is not used here; it's just exported.
+ */
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
+    int length = trailingBytesForUTF8[*source]+1;
+    if (source+length > sourceEnd) {
+        return false;
+    }
+    return isLegalUTF8(source, length);
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF16 (
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF16* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (source + extraBytesToRead >= sourceEnd) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (! isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = (UTF16)ch; /* normal case */
+            }
+        } else if (ch > UNI_MAX_UTF16) {
+            if (flags == strictConversion) {
+                result = sourceIllegal;
+                source -= (extraBytesToRead+1); /* return to the start */
+                break; /* Bail out; shouldn't continue */
+            } else {
+                *target++ = UNI_REPLACEMENT_CHAR;
+            }
+        } else {
+            /* target is a character in range 0xFFFF - 0x10FFFF. */
+            if (target + 1 >= targetEnd) {
+                source -= (extraBytesToRead+1); /* Back up source pointer! */
+                result = targetExhausted; break;
+            }
+            ch -= halfBase;
+            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
+            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF32toUTF8 (
+        const UTF32** sourceStart, const UTF32* sourceEnd, 
+        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF32* source = *sourceStart;
+    UTF8* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch;
+        unsigned short bytesToWrite = 0;
+        const UTF32 byteMask = 0xBF;
+        const UTF32 byteMark = 0x80; 
+        ch = *source++;
+        if (flags == strictConversion ) {
+            /* UTF-16 surrogate values are illegal in UTF-32 */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                --source; /* return to the illegal value itself */
+                result = sourceIllegal;
+                break;
+            }
+        }
+        /*
+         * Figure out how many bytes the result will require. Turn any
+         * illegally large UTF32 things (> Plane 17) into replacement chars.
+         */
+        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
+        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
+        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
+        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
+        } else {                            bytesToWrite = 3;
+                                            ch = UNI_REPLACEMENT_CHAR;
+                                            result = sourceIllegal;
+        }
+        
+        target += bytesToWrite;
+        if (target > targetEnd) {
+            --source; /* Back up source pointer! */
+            target -= bytesToWrite; result = targetExhausted; break;
+        }
+        switch (bytesToWrite) { /* note: everything falls through. */
+            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
+            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
+        }
+        target += bytesToWrite;
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+/* --------------------------------------------------------------------- */
+
+ConversionResult ConvertUTF8toUTF32 (
+        const UTF8** sourceStart, const UTF8* sourceEnd, 
+        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
+    ConversionResult result = conversionOK;
+    const UTF8* source = *sourceStart;
+    UTF32* target = *targetStart;
+    while (source < sourceEnd) {
+        UTF32 ch = 0;
+        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
+        if (source + extraBytesToRead >= sourceEnd) {
+            result = sourceExhausted; break;
+        }
+        /* Do this check whether lenient or strict */
+        if (! isLegalUTF8(source, extraBytesToRead+1)) {
+            result = sourceIllegal;
+            break;
+        }
+        /*
+         * The cases all fall through. See "Note A" below.
+         */
+        switch (extraBytesToRead) {
+            case 5: ch += *source++; ch <<= 6;
+            case 4: ch += *source++; ch <<= 6;
+            case 3: ch += *source++; ch <<= 6;
+            case 2: ch += *source++; ch <<= 6;
+            case 1: ch += *source++; ch <<= 6;
+            case 0: ch += *source++;
+        }
+        ch -= offsetsFromUTF8[extraBytesToRead];
+
+        if (target >= targetEnd) {
+            source -= (extraBytesToRead+1); /* Back up the source pointer! */
+            result = targetExhausted; break;
+        }
+        if (ch <= UNI_MAX_LEGAL_UTF32) {
+            /*
+             * UTF-16 surrogate values are illegal in UTF-32, and anything
+             * over Plane 17 (> 0x10FFFF) is illegal.
+             */
+            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
+                if (flags == strictConversion) {
+                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
+                    result = sourceIllegal;
+                    break;
+                } else {
+                    *target++ = UNI_REPLACEMENT_CHAR;
+                }
+            } else {
+                *target++ = ch;
+            }
+        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
+            result = sourceIllegal;
+            *target++ = UNI_REPLACEMENT_CHAR;
+        }
+    }
+    *sourceStart = source;
+    *targetStart = target;
+    return result;
+}
+
+}
+
+/* ---------------------------------------------------------------------
+
+    Note A.
+    The fall-through switches in UTF-8 reading code save a
+    temp variable, some decrements & conditionals.  The switches
+    are equivalent to the following loop:
+        {
+            int tmpBytesToRead = extraBytesToRead+1;
+            do {
+                ch += *source++;
+                --tmpBytesToRead;
+                if (tmpBytesToRead) ch <<= 6;
+            } while (tmpBytesToRead > 0);
+        }
+    In UTF-8 writing code, the switches on "bytesToWrite" are
+    similarly unrolled loops.
+
+   --------------------------------------------------------------------- */
diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h
new file mode 100755
index 00000000000..8a296235dcd
--- /dev/null
+++ b/src/linenoise/ConvertUTF.h
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ * 
+ * Disclaimer
+ * 
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ * 
+ * Limitations on Rights to Redistribute This Code
+ * 
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/* ---------------------------------------------------------------------
+
+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
+
+    Several funtions are included here, forming a complete set of
+    conversions between the three formats.  UTF-7 is not included
+    here, but is handled in a separate source file.
+
+    Each of these routines takes pointers to input buffers and output
+    buffers.  The input buffers are const.
+
+    Each routine converts the text between *sourceStart and sourceEnd,
+    putting the result into the buffer between *targetStart and
+    targetEnd. Note: the end pointers are *after* the last item: e.g. 
+    *(sourceEnd - 1) is the last item.
+
+    The return result indicates whether the conversion was successful,
+    and if not, whether the problem was in the source or target buffers.
+    (Only the first encountered problem is indicated.)
+
+    After the conversion, *sourceStart and *targetStart are both
+    updated to point to the end of last text successfully converted in
+    the respective buffers.
+
+    Input parameters:
+        sourceStart - pointer to a pointer to the source buffer.
+                The contents of this are modified on return so that
+                it points at the next thing to be converted.
+        targetStart - similarly, pointer to pointer to the target buffer.
+        sourceEnd, targetEnd - respectively pointers to the ends of the
+                two buffers, for overflow checking only.
+
+    These conversion functions take a ConversionFlags argument. When this
+    flag is set to strict, both irregular sequences and isolated surrogates
+    will cause an error.  When the flag is set to lenient, both irregular
+    sequences and isolated surrogates are converted.
+
+    Whether the flag is strict or lenient, all illegal sequences will cause
+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
+    must check for illegal sequences.
+
+    When the flag is set to lenient, characters over 0x10FFFF are converted
+    to the replacement character; otherwise (when the flag is set to strict)
+    they constitute an error.
+
+    Output parameters:
+        The value "sourceIllegal" is returned from some routines if the input
+        sequence is malformed.  When "sourceIllegal" is returned, the source
+        value will point to the illegal value that caused the problem. E.g.,
+        in UTF-8 when a sequence is malformed, it points to the start of the
+        malformed sequence.  
+
+    Author: Mark E. Davis, 1994.
+    Rev History: Rick McGowan, fixes & updates May 2001.
+                 Fixes & updates, Sept 2001.
+
+------------------------------------------------------------------------ */
+
+/* ---------------------------------------------------------------------
+    The following 4 definitions are compiler-specific.
+    The C standard does not guarantee that wchar_t has at least
+    16 bits, so wchar_t is no less portable than unsigned short!
+    All should be unsigned values to avoid sign extension during
+    bit mask & shift operations.
+------------------------------------------------------------------------ */
+
+#if 0
+typedef unsigned long  UTF32; /* at least 32 bits */
+typedef unsigned short UTF16; /* at least 16 bits */
+typedef unsigned char  UTF8;  /* typically 8 bits */
+#endif
+
+#include <stdint.h>
+#include <string>
+
+namespace linenoise_ng {
+
+typedef uint32_t         UTF32;
+typedef uint16_t         UTF16;
+typedef uint8_t          UTF8;
+typedef unsigned char    Boolean; /* 0 or 1 */
+
+/* Some fundamental constants */
+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
+#define UNI_MAX_BMP (UTF32)0x0000FFFF
+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
+
+typedef enum {
+        conversionOK,           /* conversion successful */
+        sourceExhausted,        /* partial character in source, but hit end */
+        targetExhausted,        /* insuff. room in target for conversion */
+        sourceIllegal           /* source sequence is illegal/malformed */
+} ConversionResult;
+
+typedef enum {
+        strictConversion = 0,
+        lenientConversion
+} ConversionFlags;
+
+// /* This is for C++ and does no harm in C */
+// #ifdef __cplusplus
+// extern "C" {
+// #endif
+
+ConversionResult ConvertUTF8toUTF16 (
+                const UTF8** sourceStart, const UTF8* sourceEnd, 
+                UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF16toUTF8 (
+                const UTF16** sourceStart, const UTF16* sourceEnd, 
+                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+                
+ConversionResult ConvertUTF8toUTF32 (
+                const UTF8** sourceStart, const UTF8* sourceEnd, 
+                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF8 (
+                const UTF32** sourceStart, const UTF32* sourceEnd, 
+                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
+                
+ConversionResult ConvertUTF16toUTF32 (
+                const UTF16** sourceStart, const UTF16* sourceEnd, 
+                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
+
+ConversionResult ConvertUTF32toUTF16 (
+                const UTF32** sourceStart, const UTF32* sourceEnd, 
+                char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags);
+
+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
+
+// #ifdef __cplusplus
+// }
+// #endif
+
+}
+
+/* --------------------------------------------------------------------- */
diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE
index 18e814865a5..b7c58c44586 100644
--- a/src/linenoise/LICENSE
+++ b/src/linenoise/LICENSE
@@ -1,25 +1,66 @@
-Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
-Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+linenoise.cpp
+=============
+
+Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
 
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 
-* Redistributions of source code must retain the above copyright notice,
-  this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+  * Neither the name of Redis nor the names of its contributors may be used
+    to endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+wcwidth.cpp
+===========
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.
+
+
+
+ConvertUTF.cpp
+==============
+
+Copyright 2001-2004 Unicode, Inc.
+ 
+Disclaimer
+ 
+This source code is provided as is by Unicode, Inc. No claims are
+made as to fitness for any particular purpose. No warranties of any
+kind are expressed or implied. The recipient agrees to determine
+applicability of information provided. If this file has been
+purchased on magnetic or optical media from Unicode, Inc., the
+sole remedy for any claim will be exchange of defective media
+within 90 days of receipt.
+ 
+Limitations on Rights to Redistribute This Code
+ 
+Unicode, Inc. hereby grants the right to freely use the information
+supplied in this file in the creation of products supporting the
+Unicode Standard, and to make copies of this file in any form
+for internal or external distribution as long as this notice
+remains attached.
diff --git a/src/linenoise/linenoise.c b/src/linenoise/linenoise.c
deleted file mode 100644
index fce14a7c53a..00000000000
--- a/src/linenoise/linenoise.c
+++ /dev/null
@@ -1,1199 +0,0 @@
-/* linenoise.c -- guerrilla line editing library against the idea that a
- * line editing lib needs to be 20,000 lines of C code.
- *
- * You can find the latest source code at:
- *
- *   http://github.com/antirez/linenoise
- *
- * Does a number of crazy assumptions that happen to be true in 99.9999% of
- * the 2010 UNIX computers around.
- *
- * ------------------------------------------------------------------------
- *
- * Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *  *  Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *
- *  *  Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ------------------------------------------------------------------------
- *
- * References:
- * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
- * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
- *
- * Todo list:
- * - Filter bogus Ctrl+<char> combinations.
- * - Win32 support
- *
- * Bloat:
- * - History search like Ctrl+r in readline?
- *
- * List of escape sequences used by this program, we do everything just
- * with three sequences. In order to be so cheap we may have some
- * flickering effect with some slow terminal, but the lesser sequences
- * the more compatible.
- *
- * EL (Erase Line)
- *    Sequence: ESC [ n K
- *    Effect: if n is 0 or missing, clear from cursor to end of line
- *    Effect: if n is 1, clear from beginning of line to cursor
- *    Effect: if n is 2, clear entire line
- *
- * CUF (CUrsor Forward)
- *    Sequence: ESC [ n C
- *    Effect: moves cursor forward n chars
- *
- * CUB (CUrsor Backward)
- *    Sequence: ESC [ n D
- *    Effect: moves cursor backward n chars
- *
- * The following is used to get the terminal width if getting
- * the width with the TIOCGWINSZ ioctl fails
- *
- * DSR (Device Status Report)
- *    Sequence: ESC [ 6 n
- *    Effect: reports the current cusor position as ESC [ n ; m R
- *            where n is the row and m is the column
- *
- * When multi line mode is enabled, we also use an additional escape
- * sequence. However multi line editing is disabled by default.
- *
- * CUU (Cursor Up)
- *    Sequence: ESC [ n A
- *    Effect: moves cursor up of n chars.
- *
- * CUD (Cursor Down)
- *    Sequence: ESC [ n B
- *    Effect: moves cursor down of n chars.
- *
- * When linenoiseClearScreen() is called, two additional escape sequences
- * are used in order to clear the screen and position the cursor at home
- * position.
- *
- * CUP (Cursor position)
- *    Sequence: ESC [ H
- *    Effect: moves the cursor to upper left corner
- *
- * ED (Erase display)
- *    Sequence: ESC [ 2 J
- *    Effect: clear the whole screen
- *
- */
-
-#include <termios.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/ioctl.h>
-#include <unistd.h>
-#include "linenoise.h"
-
-#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
-#define LINENOISE_MAX_LINE 4096
-static char *unsupported_term[] = {"dumb","cons25","emacs",NULL};
-static linenoiseCompletionCallback *completionCallback = NULL;
-static linenoiseHintsCallback *hintsCallback = NULL;
-static linenoiseFreeHintsCallback *freeHintsCallback = NULL;
-
-static struct termios orig_termios; /* In order to restore at exit.*/
-static int rawmode = 0; /* For atexit() function to check if restore is needed*/
-static int mlmode = 0;  /* Multi line mode. Default is single line. */
-static int atexit_registered = 0; /* Register atexit just 1 time. */
-static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
-static int history_len = 0;
-static char **history = NULL;
-
-/* The linenoiseState structure represents the state during line editing.
- * We pass this state to functions implementing specific editing
- * functionalities. */
-struct linenoiseState {
-    int ifd;            /* Terminal stdin file descriptor. */
-    int ofd;            /* Terminal stdout file descriptor. */
-    char *buf;          /* Edited line buffer. */
-    size_t buflen;      /* Edited line buffer size. */
-    const char *prompt; /* Prompt to display. */
-    size_t plen;        /* Prompt length. */
-    size_t pos;         /* Current cursor position. */
-    size_t oldpos;      /* Previous refresh cursor position. */
-    size_t len;         /* Current edited line length. */
-    size_t cols;        /* Number of columns in terminal. */
-    size_t maxrows;     /* Maximum num of rows used so far (multiline mode) */
-    int history_index;  /* The history index we are currently editing. */
-};
-
-enum KEY_ACTION{
-	KEY_NULL = 0,	    /* NULL */
-	CTRL_A = 1,         /* Ctrl+a */
-	CTRL_B = 2,         /* Ctrl-b */
-	CTRL_C = 3,         /* Ctrl-c */
-	CTRL_D = 4,         /* Ctrl-d */
-	CTRL_E = 5,         /* Ctrl-e */
-	CTRL_F = 6,         /* Ctrl-f */
-	CTRL_H = 8,         /* Ctrl-h */
-	TAB = 9,            /* Tab */
-	CTRL_K = 11,        /* Ctrl+k */
-	CTRL_L = 12,        /* Ctrl+l */
-	ENTER = 13,         /* Enter */
-	CTRL_N = 14,        /* Ctrl-n */
-	CTRL_P = 16,        /* Ctrl-p */
-	CTRL_T = 20,        /* Ctrl-t */
-	CTRL_U = 21,        /* Ctrl+u */
-	CTRL_W = 23,        /* Ctrl+w */
-	ESC = 27,           /* Escape */
-	BACKSPACE =  127    /* Backspace */
-};
-
-static void linenoiseAtExit(void);
-int linenoiseHistoryAdd(const char *line);
-static void refreshLine(struct linenoiseState *l);
-
-/* Debugging macro. */
-#if 0
-FILE *lndebug_fp = NULL;
-#define lndebug(...) \
-    do { \
-        if (lndebug_fp == NULL) { \
-            lndebug_fp = fopen("/tmp/lndebug.txt","a"); \
-            fprintf(lndebug_fp, \
-            "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \
-            (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \
-            (int)l->maxrows,old_rows); \
-        } \
-        fprintf(lndebug_fp, ", " __VA_ARGS__); \
-        fflush(lndebug_fp); \
-    } while (0)
-#else
-#define lndebug(fmt, ...)
-#endif
-
-/* ======================= Low level terminal handling ====================== */
-
-/* Set if to use or not the multi line mode. */
-void linenoiseSetMultiLine(int ml) {
-    mlmode = ml;
-}
-
-/* Return true if the terminal name is in the list of terminals we know are
- * not able to understand basic escape sequences. */
-static int isUnsupportedTerm(void) {
-    char *term = getenv("TERM");
-    int j;
-
-    if (term == NULL) return 0;
-    for (j = 0; unsupported_term[j]; j++)
-        if (!strcasecmp(term,unsupported_term[j])) return 1;
-    return 0;
-}
-
-/* Raw mode: 1960 magic shit. */
-static int enableRawMode(int fd) {
-    struct termios raw;
-
-    if (!isatty(STDIN_FILENO)) goto fatal;
-    if (!atexit_registered) {
-        atexit(linenoiseAtExit);
-        atexit_registered = 1;
-    }
-    if (tcgetattr(fd,&orig_termios) == -1) goto fatal;
-
-    raw = orig_termios;  /* modify the original mode */
-    /* input modes: no break, no CR to NL, no parity check, no strip char,
-     * no start/stop output control. */
-    raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
-    /* output modes - disable post processing */
-    raw.c_oflag &= ~(OPOST);
-    /* control modes - set 8 bit chars */
-    raw.c_cflag |= (CS8);
-    /* local modes - choing off, canonical off, no extended functions,
-     * no signal chars (^Z,^C) */
-    raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
-    /* control chars - set return condition: min number of bytes and timer.
-     * We want read to return every single byte, without timeout. */
-    raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
-
-    /* put terminal in raw mode after flushing */
-    if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal;
-    rawmode = 1;
-    return 0;
-
-fatal:
-    errno = ENOTTY;
-    return -1;
-}
-
-static void disableRawMode(int fd) {
-    /* Don't even check the return value as it's too late. */
-    if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1)
-        rawmode = 0;
-}
-
-/* Use the ESC [6n escape sequence to query the horizontal cursor position
- * and return it. On error -1 is returned, on success the position of the
- * cursor. */
-static int getCursorPosition(int ifd, int ofd) {
-    char buf[32];
-    int cols, rows;
-    unsigned int i = 0;
-
-    /* Report cursor location */
-    if (write(ofd, "\x1b[6n", 4) != 4) return -1;
-
-    /* Read the response: ESC [ rows ; cols R */
-    while (i < sizeof(buf)-1) {
-        if (read(ifd,buf+i,1) != 1) break;
-        if (buf[i] == 'R') break;
-        i++;
-    }
-    buf[i] = '\0';
-
-    /* Parse it. */
-    if (buf[0] != ESC || buf[1] != '[') return -1;
-    if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1;
-    return cols;
-}
-
-/* Try to get the number of columns in the current terminal, or assume 80
- * if it fails. */
-static int getColumns(int ifd, int ofd) {
-    struct winsize ws;
-
-    if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) {
-        /* ioctl() failed. Try to query the terminal itself. */
-        int start, cols;
-
-        /* Get the initial position so we can restore it later. */
-        start = getCursorPosition(ifd,ofd);
-        if (start == -1) goto failed;
-
-        /* Go to right margin and get position. */
-        if (write(ofd,"\x1b[999C",6) != 6) goto failed;
-        cols = getCursorPosition(ifd,ofd);
-        if (cols == -1) goto failed;
-
-        /* Restore position. */
-        if (cols > start) {
-            char seq[32];
-            snprintf(seq,32,"\x1b[%dD",cols-start);
-            if (write(ofd,seq,strlen(seq)) == -1) {
-                /* Can't recover... */
-            }
-        }
-        return cols;
-    } else {
-        return ws.ws_col;
-    }
-
-failed:
-    return 80;
-}
-
-/* Clear the screen. Used to handle ctrl+l */
-void linenoiseClearScreen(void) {
-    if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) {
-        /* nothing to do, just to avoid warning. */
-    }
-}
-
-/* Beep, used for completion when there is nothing to complete or when all
- * the choices were already shown. */
-static void linenoiseBeep(void) {
-    fprintf(stderr, "\x7");
-    fflush(stderr);
-}
-
-/* ============================== Completion ================================ */
-
-/* Free a list of completion option populated by linenoiseAddCompletion(). */
-static void freeCompletions(linenoiseCompletions *lc) {
-    size_t i;
-    for (i = 0; i < lc->len; i++)
-        free(lc->cvec[i]);
-    if (lc->cvec != NULL)
-        free(lc->cvec);
-}
-
-/* This is an helper function for linenoiseEdit() and is called when the
- * user types the <tab> key in order to complete the string currently in the
- * input.
- *
- * The state of the editing is encapsulated into the pointed linenoiseState
- * structure as described in the structure definition. */
-static int completeLine(struct linenoiseState *ls) {
-    linenoiseCompletions lc = { 0, NULL };
-    int nread, nwritten;
-    char c = 0;
-
-    completionCallback(ls->buf,&lc);
-    if (lc.len == 0) {
-        linenoiseBeep();
-    } else {
-        size_t stop = 0, i = 0;
-
-        while(!stop) {
-            /* Show completion or original buffer */
-            if (i < lc.len) {
-                struct linenoiseState saved = *ls;
-
-                ls->len = ls->pos = strlen(lc.cvec[i]);
-                ls->buf = lc.cvec[i];
-                refreshLine(ls);
-                ls->len = saved.len;
-                ls->pos = saved.pos;
-                ls->buf = saved.buf;
-            } else {
-                refreshLine(ls);
-            }
-
-            nread = read(ls->ifd,&c,1);
-            if (nread <= 0) {
-                freeCompletions(&lc);
-                return -1;
-            }
-
-            switch(c) {
-                case 9: /* tab */
-                    i = (i+1) % (lc.len+1);
-                    if (i == lc.len) linenoiseBeep();
-                    break;
-                case 27: /* escape */
-                    /* Re-show original buffer */
-                    if (i < lc.len) refreshLine(ls);
-                    stop = 1;
-                    break;
-                default:
-                    /* Update buffer and return */
-                    if (i < lc.len) {
-                        nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]);
-                        ls->len = ls->pos = nwritten;
-                    }
-                    stop = 1;
-                    break;
-            }
-        }
-    }
-
-    freeCompletions(&lc);
-    return c; /* Return last read character */
-}
-
-/* Register a callback function to be called for tab-completion. */
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) {
-    completionCallback = fn;
-}
-
-/* Register a hits function to be called to show hits to the user at the
- * right of the prompt. */
-void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) {
-    hintsCallback = fn;
-}
-
-/* Register a function to free the hints returned by the hints callback
- * registered with linenoiseSetHintsCallback(). */
-void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) {
-    freeHintsCallback = fn;
-}
-
-/* This function is used by the callback function registered by the user
- * in order to add completion options given the input string when the
- * user typed <tab>. See the example.c source code for a very easy to
- * understand example. */
-void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
-    size_t len = strlen(str);
-    char *copy, **cvec;
-
-    copy = malloc(len+1);
-    if (copy == NULL) return;
-    memcpy(copy,str,len+1);
-    cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1));
-    if (cvec == NULL) {
-        free(copy);
-        return;
-    }
-    lc->cvec = cvec;
-    lc->cvec[lc->len++] = copy;
-}
-
-/* =========================== Line editing ================================= */
-
-/* We define a very simple "append buffer" structure, that is an heap
- * allocated string where we can append to. This is useful in order to
- * write all the escape sequences in a buffer and flush them to the standard
- * output in a single call, to avoid flickering effects. */
-struct abuf {
-    char *b;
-    int len;
-};
-
-static void abInit(struct abuf *ab) {
-    ab->b = NULL;
-    ab->len = 0;
-}
-
-static void abAppend(struct abuf *ab, const char *s, int len) {
-    char *new = realloc(ab->b,ab->len+len);
-
-    if (new == NULL) return;
-    memcpy(new+ab->len,s,len);
-    ab->b = new;
-    ab->len += len;
-}
-
-static void abFree(struct abuf *ab) {
-    free(ab->b);
-}
-
-/* Helper of refreshSingleLine() and refreshMultiLine() to show hints
- * to the right of the prompt. */
-void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
-    char seq[64];
-    if (hintsCallback && plen+l->len < l->cols) {
-        int color = -1, bold = 0;
-        char *hint = hintsCallback(l->buf,&color,&bold);
-        if (hint) {
-            int hintlen = strlen(hint);
-            int hintmaxlen = l->cols-(plen+l->len);
-            if (hintlen > hintmaxlen) hintlen = hintmaxlen;
-            if (bold == 1 && color == -1) color = 37;
-            if (color != -1 || bold != 0)
-                snprintf(seq,64,"\033[%d;%d;49m",bold,color);
-            abAppend(ab,seq,strlen(seq));
-            abAppend(ab,hint,hintlen);
-            if (color != -1 || bold != 0)
-                abAppend(ab,"\033[0m",4);
-            /* Call the function to free the hint returned. */
-            if (freeHintsCallback) freeHintsCallback(hint);
-        }
-    }
-}
-
-/* Single line low level line refresh.
- *
- * Rewrite the currently edited line accordingly to the buffer content,
- * cursor position, and number of columns of the terminal. */
-static void refreshSingleLine(struct linenoiseState *l) {
-    char seq[64];
-    size_t plen = strlen(l->prompt);
-    int fd = l->ofd;
-    char *buf = l->buf;
-    size_t len = l->len;
-    size_t pos = l->pos;
-    struct abuf ab;
-
-    while((plen+pos) >= l->cols) {
-        buf++;
-        len--;
-        pos--;
-    }
-    while (plen+len > l->cols) {
-        len--;
-    }
-
-    abInit(&ab);
-    /* Cursor to left edge */
-    snprintf(seq,64,"\r");
-    abAppend(&ab,seq,strlen(seq));
-    /* Write the prompt and the current buffer content */
-    abAppend(&ab,l->prompt,strlen(l->prompt));
-    abAppend(&ab,buf,len);
-    /* Show hits if any. */
-    refreshShowHints(&ab,l,plen);
-    /* Erase to right */
-    snprintf(seq,64,"\x1b[0K");
-    abAppend(&ab,seq,strlen(seq));
-    /* Move cursor to original position. */
-    snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen));
-    abAppend(&ab,seq,strlen(seq));
-    if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
-    abFree(&ab);
-}
-
-/* Multi line low level line refresh.
- *
- * Rewrite the currently edited line accordingly to the buffer content,
- * cursor position, and number of columns of the terminal. */
-static void refreshMultiLine(struct linenoiseState *l) {
-    char seq[64];
-    int plen = strlen(l->prompt);
-    int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */
-    int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */
-    int rpos2; /* rpos after refresh. */
-    int col; /* colum position, zero-based. */
-    int old_rows = l->maxrows;
-    int fd = l->ofd, j;
-    struct abuf ab;
-
-    /* Update maxrows if needed. */
-    if (rows > (int)l->maxrows) l->maxrows = rows;
-
-    /* First step: clear all the lines used before. To do so start by
-     * going to the last row. */
-    abInit(&ab);
-    if (old_rows-rpos > 0) {
-        lndebug("go down %d", old_rows-rpos);
-        snprintf(seq,64,"\x1b[%dB", old_rows-rpos);
-        abAppend(&ab,seq,strlen(seq));
-    }
-
-    /* Now for every row clear it, go up. */
-    for (j = 0; j < old_rows-1; j++) {
-        lndebug("clear+up");
-        snprintf(seq,64,"\r\x1b[0K\x1b[1A");
-        abAppend(&ab,seq,strlen(seq));
-    }
-
-    /* Clean the top line. */
-    lndebug("clear");
-    snprintf(seq,64,"\r\x1b[0K");
-    abAppend(&ab,seq,strlen(seq));
-
-    /* Write the prompt and the current buffer content */
-    abAppend(&ab,l->prompt,strlen(l->prompt));
-    abAppend(&ab,l->buf,l->len);
-
-    /* Show hits if any. */
-    refreshShowHints(&ab,l,plen);
-
-    /* If we are at the very end of the screen with our prompt, we need to
-     * emit a newline and move the prompt to the first column. */
-    if (l->pos &&
-        l->pos == l->len &&
-        (l->pos+plen) % l->cols == 0)
-    {
-        lndebug("<newline>");
-        abAppend(&ab,"\n",1);
-        snprintf(seq,64,"\r");
-        abAppend(&ab,seq,strlen(seq));
-        rows++;
-        if (rows > (int)l->maxrows) l->maxrows = rows;
-    }
-
-    /* Move cursor to right position. */
-    rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */
-    lndebug("rpos2 %d", rpos2);
-
-    /* Go up till we reach the expected positon. */
-    if (rows-rpos2 > 0) {
-        lndebug("go-up %d", rows-rpos2);
-        snprintf(seq,64,"\x1b[%dA", rows-rpos2);
-        abAppend(&ab,seq,strlen(seq));
-    }
-
-    /* Set column. */
-    col = (plen+(int)l->pos) % (int)l->cols;
-    lndebug("set col %d", 1+col);
-    if (col)
-        snprintf(seq,64,"\r\x1b[%dC", col);
-    else
-        snprintf(seq,64,"\r");
-    abAppend(&ab,seq,strlen(seq));
-
-    lndebug("\n");
-    l->oldpos = l->pos;
-
-    if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */
-    abFree(&ab);
-}
-
-/* Calls the two low level functions refreshSingleLine() or
- * refreshMultiLine() according to the selected mode. */
-static void refreshLine(struct linenoiseState *l) {
-    if (mlmode)
-        refreshMultiLine(l);
-    else
-        refreshSingleLine(l);
-}
-
-/* Insert the character 'c' at cursor current position.
- *
- * On error writing to the terminal -1 is returned, otherwise 0. */
-int linenoiseEditInsert(struct linenoiseState *l, char c) {
-    if (l->len < l->buflen) {
-        if (l->len == l->pos) {
-            l->buf[l->pos] = c;
-            l->pos++;
-            l->len++;
-            l->buf[l->len] = '\0';
-            if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) {
-                /* Avoid a full update of the line in the
-                 * trivial case. */
-                if (write(l->ofd,&c,1) == -1) return -1;
-            } else {
-                refreshLine(l);
-            }
-        } else {
-            memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos);
-            l->buf[l->pos] = c;
-            l->len++;
-            l->pos++;
-            l->buf[l->len] = '\0';
-            refreshLine(l);
-        }
-    }
-    return 0;
-}
-
-/* Move cursor on the left. */
-void linenoiseEditMoveLeft(struct linenoiseState *l) {
-    if (l->pos > 0) {
-        l->pos--;
-        refreshLine(l);
-    }
-}
-
-/* Move cursor on the right. */
-void linenoiseEditMoveRight(struct linenoiseState *l) {
-    if (l->pos != l->len) {
-        l->pos++;
-        refreshLine(l);
-    }
-}
-
-/* Move cursor to the start of the line. */
-void linenoiseEditMoveHome(struct linenoiseState *l) {
-    if (l->pos != 0) {
-        l->pos = 0;
-        refreshLine(l);
-    }
-}
-
-/* Move cursor to the end of the line. */
-void linenoiseEditMoveEnd(struct linenoiseState *l) {
-    if (l->pos != l->len) {
-        l->pos = l->len;
-        refreshLine(l);
-    }
-}
-
-/* Substitute the currently edited line with the next or previous history
- * entry as specified by 'dir'. */
-#define LINENOISE_HISTORY_NEXT 0
-#define LINENOISE_HISTORY_PREV 1
-void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) {
-    if (history_len > 1) {
-        /* Update the current history entry before to
-         * overwrite it with the next one. */
-        free(history[history_len - 1 - l->history_index]);
-        history[history_len - 1 - l->history_index] = strdup(l->buf);
-        /* Show the new entry */
-        l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1;
-        if (l->history_index < 0) {
-            l->history_index = 0;
-            return;
-        } else if (l->history_index >= history_len) {
-            l->history_index = history_len-1;
-            return;
-        }
-        strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen);
-        l->buf[l->buflen-1] = '\0';
-        l->len = l->pos = strlen(l->buf);
-        refreshLine(l);
-    }
-}
-
-/* Delete the character at the right of the cursor without altering the cursor
- * position. Basically this is what happens with the "Delete" keyboard key. */
-void linenoiseEditDelete(struct linenoiseState *l) {
-    if (l->len > 0 && l->pos < l->len) {
-        memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1);
-        l->len--;
-        l->buf[l->len] = '\0';
-        refreshLine(l);
-    }
-}
-
-/* Backspace implementation. */
-void linenoiseEditBackspace(struct linenoiseState *l) {
-    if (l->pos > 0 && l->len > 0) {
-        memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos);
-        l->pos--;
-        l->len--;
-        l->buf[l->len] = '\0';
-        refreshLine(l);
-    }
-}
-
-/* Delete the previosu word, maintaining the cursor at the start of the
- * current word. */
-void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
-    size_t old_pos = l->pos;
-    size_t diff;
-
-    while (l->pos > 0 && l->buf[l->pos-1] == ' ')
-        l->pos--;
-    while (l->pos > 0 && l->buf[l->pos-1] != ' ')
-        l->pos--;
-    diff = old_pos - l->pos;
-    memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1);
-    l->len -= diff;
-    refreshLine(l);
-}
-
-/* This function is the core of the line editing capability of linenoise.
- * It expects 'fd' to be already in "raw mode" so that every key pressed
- * will be returned ASAP to read().
- *
- * The resulting string is put into 'buf' when the user type enter, or
- * when ctrl+d is typed.
- *
- * The function returns the length of the current buffer. */
-static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt)
-{
-    struct linenoiseState l;
-
-    /* Populate the linenoise state that we pass to functions implementing
-     * specific editing functionalities. */
-    l.ifd = stdin_fd;
-    l.ofd = stdout_fd;
-    l.buf = buf;
-    l.buflen = buflen;
-    l.prompt = prompt;
-    l.plen = strlen(prompt);
-    l.oldpos = l.pos = 0;
-    l.len = 0;
-    l.cols = getColumns(stdin_fd, stdout_fd);
-    l.maxrows = 0;
-    l.history_index = 0;
-
-    /* Buffer starts empty. */
-    l.buf[0] = '\0';
-    l.buflen--; /* Make sure there is always space for the nulterm */
-
-    /* The latest history entry is always our current buffer, that
-     * initially is just an empty string. */
-    linenoiseHistoryAdd("");
-
-    if (write(l.ofd,prompt,l.plen) == -1) return -1;
-    while(1) {
-        char c;
-        int nread;
-        char seq[3];
-
-        nread = read(l.ifd,&c,1);
-        if (nread <= 0) return l.len;
-
-        /* Only autocomplete when the callback is set. It returns < 0 when
-         * there was an error reading from fd. Otherwise it will return the
-         * character that should be handled next. */
-        if (c == 9 && completionCallback != NULL) {
-            c = completeLine(&l);
-            /* Return on errors */
-            if (c < 0) return l.len;
-            /* Read next character when 0 */
-            if (c == 0) continue;
-        }
-
-        switch(c) {
-        case ENTER:    /* enter */
-            history_len--;
-            free(history[history_len]);
-            if (mlmode) linenoiseEditMoveEnd(&l);
-            if (hintsCallback) {
-                /* Force a refresh without hints to leave the previous
-                 * line as the user typed it after a newline. */
-                linenoiseHintsCallback *hc = hintsCallback;
-                hintsCallback = NULL;
-                refreshLine(&l);
-                hintsCallback = hc;
-            }
-            return (int)l.len;
-        case CTRL_C:     /* ctrl-c */
-            errno = EAGAIN;
-            return -1;
-        case BACKSPACE:   /* backspace */
-        case 8:     /* ctrl-h */
-            linenoiseEditBackspace(&l);
-            break;
-        case CTRL_D:     /* ctrl-d, remove char at right of cursor, or if the
-                            line is empty, act as end-of-file. */
-            if (l.len > 0) {
-                linenoiseEditDelete(&l);
-            } else {
-                history_len--;
-                free(history[history_len]);
-                return -1;
-            }
-            break;
-        case CTRL_T:    /* ctrl-t, swaps current character with previous. */
-            if (l.pos > 0 && l.pos < l.len) {
-                int aux = buf[l.pos-1];
-                buf[l.pos-1] = buf[l.pos];
-                buf[l.pos] = aux;
-                if (l.pos != l.len-1) l.pos++;
-                refreshLine(&l);
-            }
-            break;
-        case CTRL_B:     /* ctrl-b */
-            linenoiseEditMoveLeft(&l);
-            break;
-        case CTRL_F:     /* ctrl-f */
-            linenoiseEditMoveRight(&l);
-            break;
-        case CTRL_P:    /* ctrl-p */
-            linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV);
-            break;
-        case CTRL_N:    /* ctrl-n */
-            linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT);
-            break;
-        case ESC:    /* escape sequence */
-            /* Read the next two bytes representing the escape sequence.
-             * Use two calls to handle slow terminals returning the two
-             * chars at different times. */
-            if (read(l.ifd,seq,1) == -1) break;
-            if (read(l.ifd,seq+1,1) == -1) break;
-
-            /* ESC [ sequences. */
-            if (seq[0] == '[') {
-                if (seq[1] >= '0' && seq[1] <= '9') {
-                    /* Extended escape, read additional byte. */
-                    if (read(l.ifd,seq+2,1) == -1) break;
-                    if (seq[2] == '~') {
-                        switch(seq[1]) {
-                        case '3': /* Delete key. */
-                            linenoiseEditDelete(&l);
-                            break;
-                        }
-                    }
-                } else {
-                    switch(seq[1]) {
-                    case 'A': /* Up */
-                        linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV);
-                        break;
-                    case 'B': /* Down */
-                        linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT);
-                        break;
-                    case 'C': /* Right */
-                        linenoiseEditMoveRight(&l);
-                        break;
-                    case 'D': /* Left */
-                        linenoiseEditMoveLeft(&l);
-                        break;
-                    case 'H': /* Home */
-                        linenoiseEditMoveHome(&l);
-                        break;
-                    case 'F': /* End*/
-                        linenoiseEditMoveEnd(&l);
-                        break;
-                    }
-                }
-            }
-
-            /* ESC O sequences. */
-            else if (seq[0] == 'O') {
-                switch(seq[1]) {
-                case 'H': /* Home */
-                    linenoiseEditMoveHome(&l);
-                    break;
-                case 'F': /* End*/
-                    linenoiseEditMoveEnd(&l);
-                    break;
-                }
-            }
-            break;
-        default:
-            if (linenoiseEditInsert(&l,c)) return -1;
-            break;
-        case CTRL_U: /* Ctrl+u, delete the whole line. */
-            buf[0] = '\0';
-            l.pos = l.len = 0;
-            refreshLine(&l);
-            break;
-        case CTRL_K: /* Ctrl+k, delete from current to end of line. */
-            buf[l.pos] = '\0';
-            l.len = l.pos;
-            refreshLine(&l);
-            break;
-        case CTRL_A: /* Ctrl+a, go to the start of the line */
-            linenoiseEditMoveHome(&l);
-            break;
-        case CTRL_E: /* ctrl+e, go to the end of the line */
-            linenoiseEditMoveEnd(&l);
-            break;
-        case CTRL_L: /* ctrl+l, clear screen */
-            linenoiseClearScreen();
-            refreshLine(&l);
-            break;
-        case CTRL_W: /* ctrl+w, delete previous word */
-            linenoiseEditDeletePrevWord(&l);
-            break;
-        }
-    }
-    return l.len;
-}
-
-/* This special mode is used by linenoise in order to print scan codes
- * on screen for debugging / development purposes. It is implemented
- * by the linenoise_example program using the --keycodes option. */
-void linenoisePrintKeyCodes(void) {
-    char quit[4];
-
-    printf("Linenoise key codes debugging mode.\n"
-            "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
-    if (enableRawMode(STDIN_FILENO) == -1) return;
-    memset(quit,' ',4);
-    while(1) {
-        char c;
-        int nread;
-
-        nread = read(STDIN_FILENO,&c,1);
-        if (nread <= 0) continue;
-        memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */
-        quit[sizeof(quit)-1] = c; /* Insert current char on the right. */
-        if (memcmp(quit,"quit",sizeof(quit)) == 0) break;
-
-        printf("'%c' %02x (%d) (type quit to exit)\n",
-            isprint(c) ? c : '?', (int)c, (int)c);
-        printf("\r"); /* Go left edge manually, we are in raw mode. */
-        fflush(stdout);
-    }
-    disableRawMode(STDIN_FILENO);
-}
-
-/* This function calls the line editing function linenoiseEdit() using
- * the STDIN file descriptor set in raw mode. */
-static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
-    int count;
-
-    if (buflen == 0) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (enableRawMode(STDIN_FILENO) == -1) return -1;
-    count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt);
-    disableRawMode(STDIN_FILENO);
-    printf("\n");
-    return count;
-}
-
-/* This function is called when linenoise() is called with the standard
- * input file descriptor not attached to a TTY. So for example when the
- * program using linenoise is called in pipe or with a file redirected
- * to its standard input. In this case, we want to be able to return the
- * line regardless of its length (by default we are limited to 4k). */
-static char *linenoiseNoTTY(void) {
-    char *line = NULL;
-    size_t len = 0, maxlen = 0;
-
-    while(1) {
-        if (len == maxlen) {
-            if (maxlen == 0) maxlen = 16;
-            maxlen *= 2;
-            char *oldval = line;
-            line = realloc(line,maxlen);
-            if (line == NULL) {
-                if (oldval) free(oldval);
-                return NULL;
-            }
-        }
-        int c = fgetc(stdin);
-        if (c == EOF || c == '\n') {
-            if (c == EOF && len == 0) {
-                free(line);
-                return NULL;
-            } else {
-                line[len] = '\0';
-                return line;
-            }
-        } else {
-            line[len] = c;
-            len++;
-        }
-    }
-}
-
-/* The high level function that is the main API of the linenoise library.
- * This function checks if the terminal has basic capabilities, just checking
- * for a blacklist of stupid terminals, and later either calls the line
- * editing function or uses dummy fgets() so that you will be able to type
- * something even in the most desperate of the conditions. */
-char *linenoise(const char *prompt) {
-    char buf[LINENOISE_MAX_LINE];
-    int count;
-
-    if (!isatty(STDIN_FILENO)) {
-        /* Not a tty: read from file / pipe. In this mode we don't want any
-         * limit to the line size, so we call a function to handle that. */
-        return linenoiseNoTTY();
-    } else if (isUnsupportedTerm()) {
-        size_t len;
-
-        printf("%s",prompt);
-        fflush(stdout);
-        if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL;
-        len = strlen(buf);
-        while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) {
-            len--;
-            buf[len] = '\0';
-        }
-        return strdup(buf);
-    } else {
-        count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt);
-        if (count == -1) return NULL;
-        return strdup(buf);
-    }
-}
-
-/* This is just a wrapper the user may want to call in order to make sure
- * the linenoise returned buffer is freed with the same allocator it was
- * created with. Useful when the main program is using an alternative
- * allocator. */
-void linenoiseFree(void *ptr) {
-    free(ptr);
-}
-
-/* ================================ History ================================= */
-
-/* Free the history, but does not reset it. Only used when we have to
- * exit() to avoid memory leaks are reported by valgrind & co. */
-static void freeHistory(void) {
-    if (history) {
-        int j;
-
-        for (j = 0; j < history_len; j++)
-            free(history[j]);
-        free(history);
-    }
-}
-
-/* At exit we'll try to fix the terminal to the initial conditions. */
-static void linenoiseAtExit(void) {
-    disableRawMode(STDIN_FILENO);
-    freeHistory();
-}
-
-/* This is the API call to add a new entry in the linenoise history.
- * It uses a fixed array of char pointers that are shifted (memmoved)
- * when the history max length is reached in order to remove the older
- * entry and make room for the new one, so it is not exactly suitable for huge
- * histories, but will work well for a few hundred of entries.
- *
- * Using a circular buffer is smarter, but a bit more complex to handle. */
-int linenoiseHistoryAdd(const char *line) {
-    char *linecopy;
-
-    if (history_max_len == 0) return 0;
-
-    /* Initialization on first call. */
-    if (history == NULL) {
-        history = malloc(sizeof(char*)*history_max_len);
-        if (history == NULL) return 0;
-        memset(history,0,(sizeof(char*)*history_max_len));
-    }
-
-    /* Don't add duplicated lines. */
-    if (history_len && !strcmp(history[history_len-1], line)) return 0;
-
-    /* Add an heap allocated copy of the line in the history.
-     * If we reached the max length, remove the older line. */
-    linecopy = strdup(line);
-    if (!linecopy) return 0;
-    if (history_len == history_max_len) {
-        free(history[0]);
-        memmove(history,history+1,sizeof(char*)*(history_max_len-1));
-        history_len--;
-    }
-    history[history_len] = linecopy;
-    history_len++;
-    return 1;
-}
-
-/* Set the maximum length for the history. This function can be called even
- * if there is already some history, the function will make sure to retain
- * just the latest 'len' elements if the new history length value is smaller
- * than the amount of items already inside the history. */
-int linenoiseHistorySetMaxLen(int len) {
-    char **new;
-
-    if (len < 1) return 0;
-    if (history) {
-        int tocopy = history_len;
-
-        new = malloc(sizeof(char*)*len);
-        if (new == NULL) return 0;
-
-        /* If we can't copy everything, free the elements we'll not use. */
-        if (len < tocopy) {
-            int j;
-
-            for (j = 0; j < tocopy-len; j++) free(history[j]);
-            tocopy = len;
-        }
-        memset(new,0,sizeof(char*)*len);
-        memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy);
-        free(history);
-        history = new;
-    }
-    history_max_len = len;
-    if (history_len > history_max_len)
-        history_len = history_max_len;
-    return 1;
-}
-
-/* Save the history in the specified file. On success 0 is returned
- * otherwise -1 is returned. */
-int linenoiseHistorySave(const char *filename) {
-    mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO);
-    FILE *fp;
-    int j;
-
-    fp = fopen(filename,"w");
-    umask(old_umask);
-    if (fp == NULL) return -1;
-    chmod(filename,S_IRUSR|S_IWUSR);
-    for (j = 0; j < history_len; j++)
-        fprintf(fp,"%s\n",history[j]);
-    fclose(fp);
-    return 0;
-}
-
-/* Load the history from the specified file. If the file does not exist
- * zero is returned and no operation is performed.
- *
- * If the file exists and the operation succeeded 0 is returned, otherwise
- * on error -1 is returned. */
-int linenoiseHistoryLoad(const char *filename) {
-    FILE *fp = fopen(filename,"r");
-    char buf[LINENOISE_MAX_LINE];
-
-    if (fp == NULL) return -1;
-
-    while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) {
-        char *p;
-
-        p = strchr(buf,'\r');
-        if (!p) p = strchr(buf,'\n');
-        if (p) *p = '\0';
-        linenoiseHistoryAdd(buf);
-    }
-    fclose(fp);
-    return 0;
-}
diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp
new file mode 100644
index 00000000000..8ee8984d694
--- /dev/null
+++ b/src/linenoise/linenoise.cpp
@@ -0,0 +1,3457 @@
+/* linenoise.c -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * You can find the latest source code at:
+ *
+ *   http://github.com/antirez/linenoise
+ *
+ * Does a number of crazy assumptions that happen to be true in 99.9999% of
+ * the 2010 UNIX computers around.
+ *
+ * References:
+ * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+ * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
+ *
+ * Todo list:
+ * - Switch to gets() if $TERM is something we can't support.
+ * - Filter bogus Ctrl+<char> combinations.
+ * - Win32 support
+ *
+ * Bloat:
+ * - Completion?
+ * - History search like Ctrl+r in readline?
+ *
+ * List of escape sequences used by this program, we do everything just
+ * with three sequences. In order to be so cheap we may have some
+ * flickering effect with some slow terminal, but the lesser sequences
+ * the more compatible.
+ *
+ * CHA (Cursor Horizontal Absolute)
+ *    Sequence: ESC [ n G
+ *    Effect: moves cursor to column n (1 based)
+ *
+ * EL (Erase Line)
+ *    Sequence: ESC [ n K
+ *    Effect: if n is 0 or missing, clear from cursor to end of line
+ *    Effect: if n is 1, clear from beginning of line to cursor
+ *    Effect: if n is 2, clear entire line
+ *
+ * CUF (Cursor Forward)
+ *    Sequence: ESC [ n C
+ *    Effect: moves cursor forward of n chars
+ *
+ * The following are used to clear the screen: ESC [ H ESC [ 2 J
+ * This is actually composed of two sequences:
+ *
+ * cursorhome
+ *    Sequence: ESC [ H
+ *    Effect: moves the cursor to upper left corner
+ *
+ * ED2 (Clear entire screen)
+ *    Sequence: ESC [ 2 J
+ *    Effect: clear the whole screen
+ *
+ */
+
+#ifdef _WIN32
+
+#include <conio.h>
+#include <windows.h>
+#include <io.h>
+
+#if defined(_MSC_VER) && _MSC_VER < 1900
+#define snprintf _snprintf  // Microsoft headers use underscores in some names
+#endif
+
+#if !defined GNUC
+#define strcasecmp _stricmp
+#endif
+
+#define strdup _strdup
+#define isatty _isatty
+#define write _write
+#define STDIN_FILENO 0
+
+#else /* _WIN32 */
+
+#include <signal.h>
+#include <termios.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <cctype>
+#include <wctype.h>
+
+#endif /* _WIN32 */
+
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include "linenoise.h"
+#include "ConvertUTF.h"
+
+#include <string>
+#include <vector>
+#include <memory>
+
+using std::string;
+using std::vector;
+using std::unique_ptr;
+using namespace linenoise_ng;
+
+typedef unsigned char char8_t;
+
+static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
+                                        size_t& dstCount, const char* src) {
+  const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src);
+  const UTF8* sourceEnd = sourceStart + strlen(src);
+  UTF32* targetStart = reinterpret_cast<UTF32*>(dst);
+  UTF32* targetEnd = targetStart + dstSize;
+
+  ConversionResult res = ConvertUTF8toUTF32(
+      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
+
+  if (res == conversionOK) {
+    dstCount = targetStart - reinterpret_cast<UTF32*>(dst);
+
+    if (dstCount < dstSize) {
+      *targetStart = 0;
+    }
+  }
+
+  return res;
+}
+
+static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
+                                        size_t& dstCount, const char8_t* src) {
+  return copyString8to32(dst, dstSize, dstCount,
+                         reinterpret_cast<const char*>(src));
+}
+
+static size_t strlen32(const char32_t* str) {
+  const char32_t* ptr = str;
+
+  while (*ptr) {
+    ++ptr;
+  }
+
+  return ptr - str;
+}
+
+static size_t strlen8(const char8_t* str) {
+  return strlen(reinterpret_cast<const char*>(str));
+}
+
+static char8_t* strdup8(const char* src) {
+  return reinterpret_cast<char8_t*>(strdup(src));
+}
+
+#ifdef _WIN32
+static const int FOREGROUND_WHITE =
+    FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
+static const int BACKGROUND_WHITE =
+    BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE;
+static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY;
+
+class WinAttributes {
+ public:
+  WinAttributes() {
+    CONSOLE_SCREEN_BUFFER_INFO info;
+    GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
+    _defaultAttribute = info.wAttributes & INTENSITY;
+    _defaultColor = info.wAttributes & FOREGROUND_WHITE;
+    _defaultBackground = info.wAttributes & BACKGROUND_WHITE;
+
+    _consoleAttribute = _defaultAttribute;
+    _consoleColor = _defaultColor | _defaultBackground;
+  }
+
+ public:
+  int _defaultAttribute;
+  int _defaultColor;
+  int _defaultBackground;
+
+  int _consoleAttribute;
+  int _consoleColor;
+};
+
+static WinAttributes WIN_ATTR;
+
+static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount,
+                             const char32_t* src, size_t srcSize) {
+  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
+  const UTF32* sourceEnd = sourceStart + srcSize;
+  char16_t* targetStart = reinterpret_cast<char16_t*>(dst);
+  char16_t* targetEnd = targetStart + dstSize;
+
+  ConversionResult res = ConvertUTF32toUTF16(
+      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
+
+  if (res == conversionOK) {
+    *dstCount = targetStart - reinterpret_cast<char16_t*>(dst);
+
+    if (*dstCount < dstSize) {
+      *targetStart = 0;
+    }
+  }
+}
+#endif
+
+static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount,
+                            const char32_t* src, size_t srcSize) {
+  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
+  const UTF32* sourceEnd = sourceStart + srcSize;
+  UTF8* targetStart = reinterpret_cast<UTF8*>(dst);
+  UTF8* targetEnd = targetStart + dstSize;
+
+  ConversionResult res = ConvertUTF32toUTF8(
+      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
+
+  if (res == conversionOK) {
+    *dstCount = targetStart - reinterpret_cast<UTF8*>(dst);
+
+    if (*dstCount < dstSize) {
+      *targetStart = 0;
+    }
+  }
+}
+
+static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) {
+  size_t dstCount = 0;
+  copyString32to8(dst, dstLen, &dstCount, src, strlen32(src));
+}
+
+static void copyString32(char32_t* dst, const char32_t* src, size_t len) {
+  while (0 < len && *src) {
+    *dst++ = *src++;
+    --len;
+  }
+
+  *dst = 0;
+}
+
+static int strncmp32(const char32_t* left, const char32_t* right, size_t len) {
+  while (0 < len && *left) {
+    if (*left != *right) {
+      return *left - *right;
+    }
+
+    ++left;
+    ++right;
+    --len;
+  }
+
+  return 0;
+}
+
+#ifdef _WIN32
+#include <iostream>
+
+static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) {
+  size_t count16 = 0;
+
+  copyString32to16(text16, len32, &count16, text32, len32);
+  WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16,
+                static_cast<DWORD>(count16), nullptr, nullptr);
+
+  return count16;
+}
+
+static char32_t* HandleEsc(char32_t* p, char32_t* end) {
+  if (*p == '[') {
+    int code = 0;
+
+    for (++p; p < end; ++p) {
+      char32_t c = *p;
+
+      if ('0' <= c && c <= '9') {
+        code = code * 10 + (c - '0');
+      } else if (c == 'm' || c == ';') {
+        switch (code) {
+          case 0:
+            WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
+            WIN_ATTR._consoleColor =
+                WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground;
+            break;
+
+          case 1:  // BOLD
+          case 5:  // BLINK
+            WIN_ATTR._consoleAttribute =
+                (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
+            break;
+
+          case 30:
+            WIN_ATTR._consoleColor = BACKGROUND_WHITE;
+            break;
+
+          case 31:
+            WIN_ATTR._consoleColor =
+                FOREGROUND_RED | WIN_ATTR._defaultBackground;
+            break;
+
+          case 32:
+            WIN_ATTR._consoleColor =
+                FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
+            break;
+
+          case 33:
+            WIN_ATTR._consoleColor =
+                FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
+            break;
+
+          case 34:
+            WIN_ATTR._consoleColor =
+                FOREGROUND_BLUE | WIN_ATTR._defaultBackground;
+            break;
+
+          case 35:
+            WIN_ATTR._consoleColor =
+                FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground;
+            break;
+
+          case 36:
+            WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN |
+                                     WIN_ATTR._defaultBackground;
+            break;
+
+          case 37:
+            WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED |
+                                     FOREGROUND_BLUE |
+                                     WIN_ATTR._defaultBackground;
+            break;
+        }
+
+        code = 0;
+      }
+
+      if (*p == 'm') {
+        ++p;
+        break;
+      }
+    }
+  } else {
+    ++p;
+  }
+
+  auto handle = GetStdHandle(STD_OUTPUT_HANDLE);
+  SetConsoleTextAttribute(handle,
+                          WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor);
+
+  return p;
+}
+
+static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) {
+  char32_t* p = text32;
+  char32_t* q = p;
+  char32_t* e = text32 + len32;
+  size_t count16 = 0;
+
+  while (p < e) {
+    if (*p == 27) {
+      if (q < p) {
+        count16 += OutputWin(text16, q, p - q);
+      }
+
+      q = p = HandleEsc(p + 1, e);
+    } else {
+      ++p;
+    }
+  }
+
+  if (q < p) {
+    count16 += OutputWin(text16, q, p - q);
+  }
+
+  return count16;
+}
+#endif
+
+static int write32(int fd, char32_t* text32, int len32) {
+#ifdef _WIN32
+  if (isatty(fd)) {
+    size_t len16 = 2 * len32 + 1;
+    unique_ptr<char16_t[]> text16(new char16_t[len16]);
+    size_t count16 = WinWrite32(text16.get(), text32, len32);
+
+    return static_cast<int>(count16);
+  } else {
+    size_t len8 = 4 * len32 + 1;
+    unique_ptr<char[]> text8(new char[len8]);
+    size_t count8 = 0;
+
+    copyString32to8(text8.get(), len8, &count8, text32, len32);
+
+    return write(fd, text8.get(), static_cast<unsigned int>(count8));
+  }
+#else
+  size_t len8 = 4 * len32 + 1;
+  unique_ptr<char[]> text8(new char[len8]);
+  size_t count8 = 0;
+
+  copyString32to8(text8.get(), len8, &count8, text32, len32);
+
+  return write(fd, text8.get(), count8);
+#endif
+}
+
+class Utf32String {
+ public:
+  Utf32String() : _length(0), _data(nullptr) { 
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[1](); 
+  }
+
+  explicit Utf32String(const char* src) : _length(0), _data(nullptr) {
+    size_t len = strlen(src);
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[len + 1]();
+    copyString8to32(_data, len + 1, _length, src);
+  }
+
+  explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) {
+    size_t len = strlen(reinterpret_cast<const char*>(src));
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[len + 1]();
+    copyString8to32(_data, len + 1, _length, src);
+  }
+
+  explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) {
+    for (_length = 0; src[_length] != 0; ++_length) {
+    }
+
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[_length + 1]();
+    memcpy(_data, src, _length * sizeof(char32_t));
+  }
+
+  explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) {
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[len + 1]();
+    memcpy(_data, src, len * sizeof(char32_t));
+  }
+
+  explicit Utf32String(int len) : _length(0), _data(nullptr) { 
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[len](); 
+  }
+
+  explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) {
+    // note: parens intentional, _data must be properly initialized
+    _data = new char32_t[_length + 1]();
+    memcpy(_data, that._data, sizeof(char32_t) * _length);
+  }
+
+  Utf32String& operator=(const Utf32String& that) {
+    if (this != &that) {
+      delete[] _data;
+      _data = new char32_t[that._length]();
+      _length = that._length;
+      memcpy(_data, that._data, sizeof(char32_t) * _length);
+    }
+
+    return *this;
+  }
+
+  ~Utf32String() { delete[] _data; }
+
+ public:
+  char32_t* get() const { return _data; }
+
+  size_t length() const { return _length; }
+
+  size_t chars() const { return _length; }
+
+  void initFromBuffer() {
+    for (_length = 0; _data[_length] != 0; ++_length) {
+    }
+  }
+
+  const char32_t& operator[](size_t pos) const { return _data[pos]; }
+
+  char32_t& operator[](size_t pos) { return _data[pos]; }
+
+ private:
+  size_t _length;
+  char32_t* _data;
+};
+
+class Utf8String {
+  Utf8String(const Utf8String&) = delete;
+  Utf8String& operator=(const Utf8String&) = delete;
+
+ public:
+  explicit Utf8String(const Utf32String& src) {
+    size_t len = src.length() * 4 + 1;
+    _data = new char[len];
+    copyString32to8(_data, len, src.get());
+  }
+
+  ~Utf8String() { delete[] _data; }
+
+ public:
+  char* get() const { return _data; }
+
+ private:
+  char* _data;
+};
+
+struct linenoiseCompletions {
+  vector<Utf32String> completionStrings;
+};
+
+#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
+#define LINENOISE_MAX_LINE 4096
+
+// make control-characters more readable
+#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40)
+
+/**
+ * Recompute widths of all characters in a char32_t buffer
+ * @param text          input buffer of Unicode characters
+ * @param widths        output buffer of character widths
+ * @param charCount     number of characters in buffer
+ */
+namespace linenoise_ng {
+int mk_wcwidth(char32_t ucs);
+}
+
+static void recomputeCharacterWidths(const char32_t* text, char* widths,
+                                     int charCount) {
+  for (int i = 0; i < charCount; ++i) {
+    widths[i] = mk_wcwidth(text[i]);
+  }
+}
+
+/**
+ * Calculate a new screen position given a starting position, screen width and
+ * character count
+ * @param x             initial x position (zero-based)
+ * @param y             initial y position (zero-based)
+ * @param screenColumns screen column count
+ * @param charCount     character positions to advance
+ * @param xOut          returned x position (zero-based)
+ * @param yOut          returned y position (zero-based)
+ */
+static void calculateScreenPosition(int x, int y, int screenColumns,
+                                    int charCount, int& xOut, int& yOut) {
+  xOut = x;
+  yOut = y;
+  int charsRemaining = charCount;
+  while (charsRemaining > 0) {
+    int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining
+                                                            : screenColumns - x;
+    xOut = x + charsThisRow;
+    yOut = y;
+    charsRemaining -= charsThisRow;
+    x = 0;
+    ++y;
+  }
+  if (xOut == screenColumns) {  // we have to special-case line wrap
+    xOut = 0;
+    ++yOut;
+  }
+}
+
+/**
+ * Calculate a column width using mk_wcswidth()
+ * @param buf32  text to calculate
+ * @param len    length of text to calculate
+ */
+namespace linenoise_ng {
+int mk_wcswidth(const char32_t* pwcs, size_t n);
+}
+
+static int calculateColumnPosition(char32_t* buf32, int len) {
+  int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len);
+  if (width == -1)
+    return len;
+  else
+    return width;
+}
+
+static bool isControlChar(char32_t testChar) {
+  return (testChar < ' ') ||                      // C0 controls
+         (testChar >= 0x7F && testChar <= 0x9F);  // DEL and C1 controls
+}
+
+struct PromptBase {            // a convenience struct for grouping prompt info
+  Utf32String promptText;      // our copy of the prompt text, edited
+  char* promptCharWidths;      // character widths from mk_wcwidth()
+  int promptChars;             // chars in promptText
+  int promptBytes;             // bytes in promptText
+  int promptExtraLines;        // extra lines (beyond 1) occupied by prompt
+  int promptIndentation;       // column offset to end of prompt
+  int promptLastLinePosition;  // index into promptText where last line begins
+  int promptPreviousInputLen;  // promptChars of previous input line, for
+                               // clearing
+  int promptCursorRowOffset;   // where the cursor is relative to the start of
+                               // the prompt
+  int promptScreenColumns;     // width of screen in columns
+  int promptPreviousLen;       // help erasing
+  int promptErrorCode;         // error code (invalid UTF-8) or zero
+
+  PromptBase() : promptPreviousInputLen(0) {}
+
+  bool write() {
+    if (write32(1, promptText.get(), promptBytes) == -1) return false;
+
+    return true;
+  }
+};
+
+struct PromptInfo : public PromptBase {
+  PromptInfo(const char* textPtr, int columns) {
+    promptExtraLines = 0;
+    promptLastLinePosition = 0;
+    promptPreviousLen = 0;
+    promptScreenColumns = columns;
+    Utf32String tempUnicode(textPtr);
+
+    // strip control characters from the prompt -- we do allow newline
+    char32_t* pIn = tempUnicode.get();
+    char32_t* pOut = pIn;
+
+    int len = 0;
+    int x = 0;
+
+    bool const strip = (isatty(1) == 0);
+
+    while (*pIn) {
+      char32_t c = *pIn;
+      if ('\n' == c || !isControlChar(c)) {
+        *pOut = c;
+        ++pOut;
+        ++pIn;
+        ++len;
+        if ('\n' == c || ++x >= promptScreenColumns) {
+          x = 0;
+          ++promptExtraLines;
+          promptLastLinePosition = len;
+        }
+      } else if (c == '\x1b') {
+        if (strip) {
+          // jump over control chars
+          ++pIn;
+          if (*pIn == '[') {
+            ++pIn;
+            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
+              ++pIn;
+            }
+            if (*pIn == 'm') {
+              ++pIn;
+            }
+          }
+        } else {
+          // copy control chars
+          *pOut = *pIn;
+          ++pOut;
+          ++pIn;
+          if (*pIn == '[') {
+            *pOut = *pIn;
+            ++pOut;
+            ++pIn;
+            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
+              *pOut = *pIn;
+              ++pOut;
+              ++pIn;
+            }
+            if (*pIn == 'm') {
+              *pOut = *pIn;
+              ++pOut;
+              ++pIn;
+            }
+          }
+        }
+      } else {
+        ++pIn;
+      }
+    }
+    *pOut = 0;
+    promptChars = len;
+    promptBytes = static_cast<int>(pOut - tempUnicode.get());
+    promptText = tempUnicode;
+
+    promptIndentation = len - promptLastLinePosition;
+    promptCursorRowOffset = promptExtraLines;
+  }
+};
+
+// Used with DynamicPrompt (history search)
+//
+static const Utf32String forwardSearchBasePrompt("(i-search)`");
+static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`");
+static const Utf32String endSearchBasePrompt("': ");
+static Utf32String
+    previousSearchText;  // remembered across invocations of linenoise()
+
+// changing prompt for "(reverse-i-search)`text':" etc.
+//
+struct DynamicPrompt : public PromptBase {
+  Utf32String searchText;  // text we are searching for
+  char* searchCharWidths;  // character widths from mk_wcwidth()
+  int searchTextLen;       // chars in searchText
+  int direction;           // current search direction, 1=forward, -1=reverse
+
+  DynamicPrompt(PromptBase& pi, int initialDirection)
+      : searchTextLen(0), direction(initialDirection) {
+    promptScreenColumns = pi.promptScreenColumns;
+    promptCursorRowOffset = 0;
+    Utf32String emptyString(1);
+    searchText = emptyString;
+    const Utf32String* basePrompt =
+        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
+    size_t promptStartLength = basePrompt->length();
+    promptChars =
+        static_cast<int>(promptStartLength + endSearchBasePrompt.length());
+    promptBytes = promptChars;
+    promptLastLinePosition = promptChars;  // TODO fix this, we are asssuming
+                                           // that the history prompt won't wrap
+                                           // (!)
+    promptPreviousLen = promptChars;
+    Utf32String tempUnicode(promptChars + 1);
+    memcpy(tempUnicode.get(), basePrompt->get(),
+           sizeof(char32_t) * promptStartLength);
+    memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(),
+           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
+    tempUnicode.initFromBuffer();
+    promptText = tempUnicode;
+    calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars,
+                            promptIndentation, promptExtraLines);
+  }
+
+  void updateSearchPrompt(void) {
+    const Utf32String* basePrompt =
+        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
+    size_t promptStartLength = basePrompt->length();
+    promptChars = static_cast<int>(promptStartLength + searchTextLen +
+                                   endSearchBasePrompt.length());
+    promptBytes = promptChars;
+    Utf32String tempUnicode(promptChars + 1);
+    memcpy(tempUnicode.get(), basePrompt->get(),
+           sizeof(char32_t) * promptStartLength);
+    memcpy(&tempUnicode[promptStartLength], searchText.get(),
+           sizeof(char32_t) * searchTextLen);
+    size_t endIndex = promptStartLength + searchTextLen;
+    memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(),
+           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
+    tempUnicode.initFromBuffer();
+    promptText = tempUnicode;
+  }
+
+  void updateSearchText(const char32_t* textPtr) {
+    Utf32String tempUnicode(textPtr);
+    searchTextLen = static_cast<int>(tempUnicode.chars());
+    searchText = tempUnicode;
+    updateSearchPrompt();
+  }
+};
+
+class KillRing {
+  static const int capacity = 10;
+  int size;
+  int index;
+  char indexToSlot[10];
+  vector<Utf32String> theRing;
+
+ public:
+  enum action { actionOther, actionKill, actionYank };
+  action lastAction;
+  size_t lastYankSize;
+
+  KillRing() : size(0), index(0), lastAction(actionOther) {
+    theRing.reserve(capacity);
+  }
+
+  void kill(const char32_t* text, int textLen, bool forward) {
+    if (textLen == 0) {
+      return;
+    }
+    Utf32String killedText(text, textLen);
+    if (lastAction == actionKill && size > 0) {
+      int slot = indexToSlot[0];
+      int currentLen = static_cast<int>(theRing[slot].length());
+      int resultLen = currentLen + textLen;
+      Utf32String temp(resultLen + 1);
+      if (forward) {
+        memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t));
+        memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t));
+      } else {
+        memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t));
+        memcpy(&temp[textLen], theRing[slot].get(),
+               currentLen * sizeof(char32_t));
+      }
+      temp[resultLen] = 0;
+      temp.initFromBuffer();
+      theRing[slot] = temp;
+    } else {
+      if (size < capacity) {
+        if (size > 0) {
+          memmove(&indexToSlot[1], &indexToSlot[0], size);
+        }
+        indexToSlot[0] = size;
+        size++;
+        theRing.push_back(killedText);
+      } else {
+        int slot = indexToSlot[capacity - 1];
+        theRing[slot] = killedText;
+        memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1);
+        indexToSlot[0] = slot;
+      }
+      index = 0;
+    }
+  }
+
+  Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; }
+
+  Utf32String* yankPop() {
+    if (size == 0) {
+      return 0;
+    }
+    ++index;
+    if (index == size) {
+      index = 0;
+    }
+    return &theRing[indexToSlot[index]];
+  }
+};
+
+class InputBuffer {
+  char32_t* buf32;   // input buffer
+  char* charWidths;  // character widths from mk_wcwidth()
+  int buflen;        // buffer size in characters
+  int len;           // length of text in input buffer
+  int pos;           // character position in buffer ( 0 <= pos <= len )
+
+  void clearScreen(PromptBase& pi);
+  int incrementalHistorySearch(PromptBase& pi, int startChar);
+  int completeLine(PromptBase& pi);
+  void refreshLine(PromptBase& pi);
+
+ public:
+  InputBuffer(char32_t* buffer, char* widthArray, int bufferLen)
+      : buf32(buffer),
+        charWidths(widthArray),
+        buflen(bufferLen - 1),
+        len(0),
+        pos(0) {
+    buf32[0] = 0;
+  }
+  void preloadBuffer(const char* preloadText) {
+    size_t ucharCount = 0;
+    copyString8to32(buf32, buflen + 1, ucharCount, preloadText);
+    recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount));
+    len = static_cast<int>(ucharCount);
+    pos = static_cast<int>(ucharCount);
+  }
+  int getInputLine(PromptBase& pi);
+  int length(void) const { return len; }
+};
+
+// Special codes for keyboard input:
+//
+// Between Windows and the various Linux "terminal" programs, there is some
+// pretty diverse behavior in the "scan codes" and escape sequences we are
+// presented with.  So ... we'll translate them all into our own pidgin
+// pseudocode, trying to stay out of the way of UTF-8 and international
+// characters.  Here's the general plan.
+//
+// "User input keystrokes" (key chords, whatever) will be encoded as a single
+// value.
+// The low 21 bits are reserved for Unicode characters.  Popular function-type
+// keys
+// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000,
+// currently
+// just arrow keys, Home, End and Delete.  Keypresses with Ctrl get ORed with
+// 0x20000000, with Alt get ORed with 0x40000000.  So, Ctrl+Alt+Home is encoded
+// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000.  To keep things
+// complicated,
+// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC
+// followed by
+// D is treated the same as Alt + D ... we'll just use Emacs terminology and
+// call
+// this "Meta".  So, we will encode both ESC followed by D and Alt held down
+// while D
+// is pressed the same, as Meta-D, encoded as 0x40000064.
+//
+// Here are the definitions of our component constants:
+//
+// Maximum unsigned 32-bit value    = 0xFFFFFFFF;   // For reference, max 32-bit
+// value
+// Highest allocated Unicode char   = 0x001FFFFF;   // For reference, max
+// Unicode value
+static const int META = 0x40000000;  // Meta key combination
+static const int CTRL = 0x20000000;  // Ctrl key combination
+// static const int SPECIAL_KEY = 0x10000000;   // Common bit for all special
+// keys
+static const int UP_ARROW_KEY = 0x10200000;  // Special keys
+static const int DOWN_ARROW_KEY = 0x10400000;
+static const int RIGHT_ARROW_KEY = 0x10600000;
+static const int LEFT_ARROW_KEY = 0x10800000;
+static const int HOME_KEY = 0x10A00000;
+static const int END_KEY = 0x10C00000;
+static const int DELETE_KEY = 0x10E00000;
+static const int PAGE_UP_KEY = 0x11000000;
+static const int PAGE_DOWN_KEY = 0x11200000;
+
+static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
+static linenoiseCompletionCallback* completionCallback = NULL;
+
+#ifdef _WIN32
+static HANDLE console_in, console_out;
+static DWORD oldMode;
+static WORD oldDisplayAttribute;
+#else
+static struct termios orig_termios; /* in order to restore at exit */
+#endif
+
+static KillRing killRing;
+
+static int rawmode = 0; /* for atexit() function to check if restore is needed*/
+static int atexit_registered = 0; /* register atexit just 1 time */
+static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
+static int historyLen = 0;
+static int historyIndex = 0;
+static char8_t** history = NULL;
+
+// used to emulate Windows command prompt on down-arrow after a recall
+// we use -2 as our "not set" value because we add 1 to the previous index on
+// down-arrow,
+// and zero is a valid index (so -1 is a valid "previous index")
+static int historyPreviousIndex = -2;
+static bool historyRecallMostRecent = false;
+
+static void linenoiseAtExit(void);
+
+static bool isUnsupportedTerm(void) {
+  char* term = getenv("TERM");
+  if (term == NULL) return false;
+  for (int j = 0; unsupported_term[j]; ++j)
+    if (!strcasecmp(term, unsupported_term[j])) {
+      return true;
+    }
+  return false;
+}
+
+static void beep() {
+  fprintf(stderr, "\x7");  // ctrl-G == bell/beep
+  fflush(stderr);
+}
+
+void linenoiseHistoryFree(void) {
+  if (history) {
+    for (int j = 0; j < historyLen; ++j) free(history[j]);
+    historyLen = 0;
+    free(history);
+    history = 0;
+  }
+}
+
+static int enableRawMode(void) {
+#ifdef _WIN32
+  if (!console_in) {
+    console_in = GetStdHandle(STD_INPUT_HANDLE);
+    console_out = GetStdHandle(STD_OUTPUT_HANDLE);
+
+    GetConsoleMode(console_in, &oldMode);
+    SetConsoleMode(console_in, oldMode &
+                                   ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT |
+                                     ENABLE_PROCESSED_INPUT));
+  }
+  return 0;
+#else
+  struct termios raw;
+
+  if (!isatty(STDIN_FILENO)) goto fatal;
+  if (!atexit_registered) {
+    atexit(linenoiseAtExit);
+    atexit_registered = 1;
+  }
+  if (tcgetattr(0, &orig_termios) == -1) goto fatal;
+
+  raw = orig_termios; /* modify the original mode */
+  /* input modes: no break, no CR to NL, no parity check, no strip char,
+   * no start/stop output control. */
+  raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+  /* output modes - disable post processing */
+  // this is wrong, we don't want raw output, it turns newlines into straight
+  // linefeeds
+  // raw.c_oflag &= ~(OPOST);
+  /* control modes - set 8 bit chars */
+  raw.c_cflag |= (CS8);
+  /* local modes - echoing off, canonical off, no extended functions,
+   * no signal chars (^Z,^C) */
+  raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+  /* control chars - set return condition: min number of bytes and timer.
+   * We want read to return every single byte, without timeout. */
+  raw.c_cc[VMIN] = 1;
+  raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+  /* put terminal in raw mode after flushing */
+  if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal;
+  rawmode = 1;
+  return 0;
+
+fatal:
+  errno = ENOTTY;
+  return -1;
+#endif
+}
+
+static void disableRawMode(void) {
+#ifdef _WIN32
+  SetConsoleMode(console_in, oldMode);
+  console_in = 0;
+  console_out = 0;
+#else
+  if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0;
+#endif
+}
+
+// At exit we'll try to fix the terminal to the initial conditions
+static void linenoiseAtExit(void) { disableRawMode(); }
+
+static int getScreenColumns(void) {
+  int cols;
+#ifdef _WIN32
+  CONSOLE_SCREEN_BUFFER_INFO inf;
+  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
+  cols = inf.dwSize.X;
+#else
+  struct winsize ws;
+  cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col;
+#endif
+  // cols is 0 in certain circumstances like inside debugger, which creates
+  // further issues
+  return (cols > 0) ? cols : 80;
+}
+
+static int getScreenRows(void) {
+  int rows;
+#ifdef _WIN32
+  CONSOLE_SCREEN_BUFFER_INFO inf;
+  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
+  rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top;
+#else
+  struct winsize ws;
+  rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row;
+#endif
+  return (rows > 0) ? rows : 24;
+}
+
+static void setDisplayAttribute(bool enhancedDisplay, bool error) {
+#ifdef _WIN32
+  if (enhancedDisplay) {
+    CONSOLE_SCREEN_BUFFER_INFO inf;
+    GetConsoleScreenBufferInfo(console_out, &inf);
+    oldDisplayAttribute = inf.wAttributes;
+    BYTE oldLowByte = oldDisplayAttribute & 0xFF;
+    BYTE newLowByte;
+    switch (oldLowByte) {
+      case 0x07:
+        // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY;  // too dim
+        // newLowByte = FOREGROUND_BLUE;                         // even dimmer
+        newLowByte = FOREGROUND_BLUE |
+                     FOREGROUND_GREEN;  // most similar to xterm appearance
+        break;
+      case 0x70:
+        newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY;
+        break;
+      default:
+        newLowByte = oldLowByte ^ 0xFF;  // default to inverse video
+        break;
+    }
+    inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte;
+    SetConsoleTextAttribute(console_out, inf.wAttributes);
+  } else {
+    SetConsoleTextAttribute(console_out, oldDisplayAttribute);
+  }
+#else
+  if (enhancedDisplay) {
+    char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m");
+    if (write(1, p, 7) == -1)
+      return; /* bright blue (visible with both B&W bg) */
+  } else {
+    if (write(1, "\x1b[0m", 4) == -1) return; /* reset */
+  }
+#endif
+}
+
+/**
+ * Display the dynamic incremental search prompt and the current user input
+ * line.
+ * @param pi   PromptBase struct holding information about the prompt and our
+ * screen position
+ * @param buf32  input buffer to be displayed
+ * @param len  count of characters in the buffer
+ * @param pos  current cursor position within the buffer (0 <= pos <= len)
+ */
+static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) {
+  // calculate the position of the end of the prompt
+  int xEndOfPrompt, yEndOfPrompt;
+  calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars,
+                          xEndOfPrompt, yEndOfPrompt);
+  pi.promptIndentation = xEndOfPrompt;
+
+  // calculate the position of the end of the input line
+  int xEndOfInput, yEndOfInput;
+  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
+                          calculateColumnPosition(buf32, len), xEndOfInput,
+                          yEndOfInput);
+
+  // calculate the desired position of the cursor
+  int xCursorPos, yCursorPos;
+  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
+                          calculateColumnPosition(buf32, pos), xCursorPos,
+                          yCursorPos);
+
+#ifdef _WIN32
+  // position at the start of the prompt, clear to end of previous input
+  CONSOLE_SCREEN_BUFFER_INFO inf;
+  GetConsoleScreenBufferInfo(console_out, &inf);
+  inf.dwCursorPosition.X = 0;
+  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/;
+  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
+  DWORD count;
+  FillConsoleOutputCharacterA(console_out, ' ',
+                              pi.promptPreviousLen + pi.promptPreviousInputLen,
+                              inf.dwCursorPosition, &count);
+  pi.promptPreviousLen = pi.promptIndentation;
+  pi.promptPreviousInputLen = len;
+
+  // display the prompt
+  if (!pi.write()) return;
+
+  // display the input line
+  if (write32(1, buf32, len) == -1) return;
+
+  // position the cursor
+  GetConsoleScreenBufferInfo(console_out, &inf);
+  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
+  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
+  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
+#else  // _WIN32
+  char seq[64];
+  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
+  if (cursorRowMovement > 0) {  // move the cursor up as required
+    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
+    if (write(1, seq, strlen(seq)) == -1) return;
+  }
+  // position at the start of the prompt, clear to end of screen
+  snprintf(seq, sizeof seq, "\x1b[1G\x1b[J");  // 1-based on VT100
+  if (write(1, seq, strlen(seq)) == -1) return;
+
+  // display the prompt
+  if (!pi.write()) return;
+
+  // display the input line
+  if (write32(1, buf32, len) == -1) return;
+
+  // we have to generate our own newline on line wrap
+  if (xEndOfInput == 0 && yEndOfInput > 0)
+    if (write(1, "\n", 1) == -1) return;
+
+  // position the cursor
+  cursorRowMovement = yEndOfInput - yCursorPos;
+  if (cursorRowMovement > 0) {  // move the cursor up as required
+    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
+    if (write(1, seq, strlen(seq)) == -1) return;
+  }
+  // position the cursor within the line
+  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
+  if (write(1, seq, strlen(seq)) == -1) return;
+#endif
+
+  pi.promptCursorRowOffset =
+      pi.promptExtraLines + yCursorPos;  // remember row for next pass
+}
+
+/**
+ * Refresh the user's input line: the prompt is already onscreen and is not
+ * redrawn here
+ * @param pi   PromptBase struct holding information about the prompt and our
+ * screen position
+ */
+void InputBuffer::refreshLine(PromptBase& pi) {
+  // check for a matching brace/bracket/paren, remember its position if found
+  int highlight = -1;
+  bool indicateError = false;
+  if (pos < len) {
+    /* this scans for a brace matching buf32[pos] to highlight */
+    unsigned char part1, part2;
+    int scanDirection = 0;
+    if (strchr("}])", buf32[pos])) {
+      scanDirection = -1; /* backwards */
+      if (buf32[pos] == '}') {
+        part1 = '}'; part2 = '{';
+      } else if (buf32[pos] == ']') {
+        part1 = ']'; part2 = '[';
+      } else {
+        part1 = ')'; part2 = '(';
+      }
+    }
+    else if (strchr("{[(", buf32[pos])) {
+      scanDirection = 1; /* forwards */
+      if (buf32[pos] == '{') {
+        //part1 = '{'; part2 = '}';
+        part1 = '}'; part2 = '{';
+      } else if (buf32[pos] == '[') {
+        //part1 = '['; part2 = ']';
+        part1 = ']'; part2 = '[';
+      } else {
+        //part1 = '('; part2 = ')';
+        part1 = ')'; part2 = '(';
+      }
+    }
+
+    if (scanDirection) {
+      int unmatched = scanDirection;
+      int unmatchedOther = 0;
+      for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) {
+        /* TODO: the right thing when inside a string */
+        if (strchr("}])", buf32[i])) {
+          if (buf32[i] == part1) {
+            --unmatched;
+          } else {
+            --unmatchedOther;
+          }
+        } else if (strchr("{[(", buf32[i])) {
+          if (buf32[i] == part2) {
+            ++unmatched;
+          } else {
+            ++unmatchedOther;
+          }
+        }
+/*
+        if (strchr("}])", buf32[i]))
+          --unmatched;
+        else if (strchr("{[(", buf32[i]))
+          ++unmatched;
+*/
+        if (unmatched == 0) {
+          highlight = i;
+          indicateError = (unmatchedOther != 0);
+          break;
+        }
+      }
+    }
+  }
+
+  // calculate the position of the end of the input line
+  int xEndOfInput, yEndOfInput;
+  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
+                          calculateColumnPosition(buf32, len), xEndOfInput,
+                          yEndOfInput);
+
+  // calculate the desired position of the cursor
+  int xCursorPos, yCursorPos;
+  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
+                          calculateColumnPosition(buf32, pos), xCursorPos,
+                          yCursorPos);
+
+#ifdef _WIN32
+  // position at the end of the prompt, clear to end of previous input
+  CONSOLE_SCREEN_BUFFER_INFO inf;
+  GetConsoleScreenBufferInfo(console_out, &inf);
+  inf.dwCursorPosition.X = pi.promptIndentation;  // 0-based on Win32
+  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines;
+  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
+  DWORD count;
+  if (len < pi.promptPreviousInputLen)
+    FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen,
+                                inf.dwCursorPosition, &count);
+  pi.promptPreviousInputLen = len;
+
+  // display the input line
+  if (highlight == -1) {
+    if (write32(1, buf32, len) == -1) return;
+  } else {
+    if (write32(1, buf32, highlight) == -1) return;
+    setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */
+    if (write32(1, &buf32[highlight], 1) == -1) return;
+    setDisplayAttribute(false, indicateError);
+    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
+  }
+
+  // position the cursor
+  GetConsoleScreenBufferInfo(console_out, &inf);
+  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
+  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
+  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
+#else  // _WIN32
+  char seq[64];
+  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
+  if (cursorRowMovement > 0) {  // move the cursor up as required
+    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
+    if (write(1, seq, strlen(seq)) == -1) return;
+  }
+  // position at the end of the prompt, clear to end of screen
+  snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J",
+           pi.promptIndentation + 1);  // 1-based on VT100
+  if (write(1, seq, strlen(seq)) == -1) return;
+
+  if (highlight == -1) {  // write unhighlighted text
+    if (write32(1, buf32, len) == -1) return;
+  } else {  // highlight the matching brace/bracket/parenthesis
+    if (write32(1, buf32, highlight) == -1) return;
+    setDisplayAttribute(true, indicateError);
+    if (write32(1, &buf32[highlight], 1) == -1) return;
+    setDisplayAttribute(false, indicateError);
+    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
+  }
+
+  // we have to generate our own newline on line wrap
+  if (xEndOfInput == 0 && yEndOfInput > 0)
+    if (write(1, "\n", 1) == -1) return;
+
+  // position the cursor
+  cursorRowMovement = yEndOfInput - yCursorPos;
+  if (cursorRowMovement > 0) {  // move the cursor up as required
+    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
+    if (write(1, seq, strlen(seq)) == -1) return;
+  }
+  // position the cursor within the line
+  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
+  if (write(1, seq, strlen(seq)) == -1) return;
+#endif
+
+  pi.promptCursorRowOffset =
+      pi.promptExtraLines + yCursorPos;  // remember row for next pass
+}
+
+#ifndef _WIN32
+
+/**
+ * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode
+ * (char32_t) character it
+ * encodes
+ *
+ * @return  char32_t Unicode character
+ */
+static char32_t readUnicodeCharacter(void) {
+  static char8_t utf8String[5];
+  static size_t utf8Count = 0;
+  while (true) {
+    char8_t c;
+
+    /* Continue reading if interrupted by signal. */
+    ssize_t nread;
+    do {
+      nread = read(0, &c, 1);
+    } while ((nread == -1) && (errno == EINTR));
+
+    if (nread <= 0) return 0;
+    if (c <= 0x7F) {  // short circuit ASCII
+      utf8Count = 0;
+      return c;
+    } else if (utf8Count < sizeof(utf8String) - 1) {
+      utf8String[utf8Count++] = c;
+      utf8String[utf8Count] = 0;
+      char32_t unicodeChar[2];
+      size_t ucharCount;
+      ConversionResult res =
+          copyString8to32(unicodeChar, 2, ucharCount, utf8String);
+      if (res == conversionOK && ucharCount) {
+        utf8Count = 0;
+        return unicodeChar[0];
+      }
+    } else {
+      utf8Count =
+          0;  // this shouldn't happen: got four bytes but no UTF-8 character
+    }
+  }
+}
+
+namespace EscapeSequenceProcessing {  // move these out of global namespace
+
+// This chunk of code does parsing of the escape sequences sent by various Linux
+// terminals.
+//
+// It handles arrow keys, Home, End and Delete keys by interpreting the
+// sequences sent by
+// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and
+// Ctrl key
+// combinations that are understood by linenoise.
+//
+// The parsing uses tables, a bunch of intermediate dispatch routines and a
+// doDispatch
+// loop that reads the tables and sends control to "deeper" routines to continue
+// the
+// parsing.  The starting call to doDispatch( c, initialDispatch ) will
+// eventually return
+// either a character (with optional CTRL and META bits set), or -1 if parsing
+// fails, or
+// zero if an attempt to read from the keyboard fails.
+//
+// This is rather sloppy escape sequence processing, since we're not paying
+// attention to what the
+// actual TERM is set to and are processing all key sequences for all terminals,
+// but it works with
+// the most common keystrokes on the most common terminals.  It's intricate, but
+// the nested 'if'
+// statements required to do it directly would be worse.  This way has the
+// advantage of allowing
+// changes and extensions without having to touch a lot of code.
+
+// This is a typedef for the routine called by doDispatch().  It takes the
+// current character
+// as input, does any required processing including reading more characters and
+// calling other
+// dispatch routines, then eventually returns the final (possibly extended or
+// special) character.
+//
+typedef char32_t (*CharacterDispatchRoutine)(char32_t);
+
+// This structure is used by doDispatch() to hold a list of characters to test
+// for and
+// a list of routines to call if the character matches.  The dispatch routine
+// list is one
+// longer than the character list; the final entry is used if no character
+// matches.
+//
+struct CharacterDispatch {
+  unsigned int len;                    // length of the chars list
+  const char* chars;                   // chars to test
+  CharacterDispatchRoutine* dispatch;  // array of routines to call
+};
+
+// This dispatch routine is given a dispatch table and then farms work out to
+// routines
+// listed in the table based on the character it is called with.  The dispatch
+// routines can
+// read more input characters to decide what should eventually be returned.
+// Eventually,
+// a called routine returns either a character or -1 to indicate parsing
+// failure.
+//
+static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) {
+  for (unsigned int i = 0; i < dispatchTable.len; ++i) {
+    if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) {
+      return dispatchTable.dispatch[i](c);
+    }
+  }
+  return dispatchTable.dispatch[dispatchTable.len](c);
+}
+
+static char32_t thisKeyMetaCtrl =
+    0;  // holds pre-set Meta and/or Ctrl modifiers
+
+// Final dispatch routines -- return something
+//
+static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; }
+static char32_t upArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | UP_ARROW_KEY;
+}
+static char32_t downArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | DOWN_ARROW_KEY;
+}
+static char32_t rightArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | RIGHT_ARROW_KEY;
+}
+static char32_t leftArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | LEFT_ARROW_KEY;
+}
+static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; }
+static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; }
+static char32_t pageUpKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | PAGE_UP_KEY;
+}
+static char32_t pageDownKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | PAGE_DOWN_KEY;
+}
+static char32_t deleteCharRoutine(char32_t) {
+  return thisKeyMetaCtrl | ctrlChar('H');
+}  // key labeled Backspace
+static char32_t deleteKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | DELETE_KEY;
+}  // key labeled Delete
+static char32_t ctrlUpArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY;
+}
+static char32_t ctrlDownArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY;
+}
+static char32_t ctrlRightArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY;
+}
+static char32_t ctrlLeftArrowKeyRoutine(char32_t) {
+  return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY;
+}
+static char32_t escFailureRoutine(char32_t) {
+  beep();
+  return -1;
+}
+
+// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = {
+    upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
+    leftArrowKeyRoutine, escFailureRoutine};
+static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = {
+    4, "ABCD", escLeftBracket1Semicolon3or5Routines};
+
+// Handle ESC [ 1 ; <more stuff> escape sequences
+//
+static char32_t escLeftBracket1Semicolon3Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  thisKeyMetaCtrl |= META;
+  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
+}
+static char32_t escLeftBracket1Semicolon5Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  thisKeyMetaCtrl |= CTRL;
+  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
+}
+static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = {
+    escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine,
+    escFailureRoutine};
+static CharacterDispatch escLeftBracket1SemicolonDispatch = {
+    2, "35", escLeftBracket1SemicolonRoutines};
+
+// Handle ESC [ 1 <more stuff> escape sequences
+//
+static char32_t escLeftBracket1SemicolonRoutine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket1SemicolonDispatch);
+}
+static CharacterDispatchRoutine escLeftBracket1Routines[] = {
+    homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine};
+static CharacterDispatch escLeftBracket1Dispatch = {2, "~;",
+                                                    escLeftBracket1Routines};
+
+// Handle ESC [ 3 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket3Dispatch = {1, "~",
+                                                    escLeftBracket3Routines};
+
+// Handle ESC [ 4 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket4Dispatch = {1, "~",
+                                                    escLeftBracket4Routines};
+
+// Handle ESC [ 5 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket5Dispatch = {1, "~",
+                                                    escLeftBracket5Routines};
+
+// Handle ESC [ 6 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket6Dispatch = {1, "~",
+                                                    escLeftBracket6Routines};
+
+// Handle ESC [ 7 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket7Dispatch = {1, "~",
+                                                    escLeftBracket7Routines};
+
+// Handle ESC [ 8 <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine,
+                                                             escFailureRoutine};
+static CharacterDispatch escLeftBracket8Dispatch = {1, "~",
+                                                    escLeftBracket8Routines};
+
+// Handle ESC [ <digit> escape sequences
+//
+static char32_t escLeftBracket0Routine(char32_t c) {
+  return escFailureRoutine(c);
+}
+static char32_t escLeftBracket1Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket1Dispatch);
+}
+static char32_t escLeftBracket2Routine(char32_t c) {
+  return escFailureRoutine(c);  // Insert key, unused
+}
+static char32_t escLeftBracket3Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket3Dispatch);
+}
+static char32_t escLeftBracket4Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket4Dispatch);
+}
+static char32_t escLeftBracket5Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket5Dispatch);
+}
+static char32_t escLeftBracket6Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket6Dispatch);
+}
+static char32_t escLeftBracket7Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket7Dispatch);
+}
+static char32_t escLeftBracket8Routine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracket8Dispatch);
+}
+static char32_t escLeftBracket9Routine(char32_t c) {
+  return escFailureRoutine(c);
+}
+
+// Handle ESC [ <more stuff> escape sequences
+//
+static CharacterDispatchRoutine escLeftBracketRoutines[] = {
+    upArrowKeyRoutine,      downArrowKeyRoutine,    rightArrowKeyRoutine,
+    leftArrowKeyRoutine,    homeKeyRoutine,         endKeyRoutine,
+    escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine,
+    escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine,
+    escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine,
+    escLeftBracket9Routine, escFailureRoutine};
+static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789",
+                                                   escLeftBracketRoutines};
+
+// Handle ESC O <char> escape sequences
+//
+static CharacterDispatchRoutine escORoutines[] = {
+    upArrowKeyRoutine,       downArrowKeyRoutine,     rightArrowKeyRoutine,
+    leftArrowKeyRoutine,     homeKeyRoutine,          endKeyRoutine,
+    ctrlUpArrowKeyRoutine,   ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine,
+    ctrlLeftArrowKeyRoutine, escFailureRoutine};
+static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines};
+
+// Initial ESC dispatch -- could be a Meta prefix or the start of an escape
+// sequence
+//
+static char32_t escLeftBracketRoutine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escLeftBracketDispatch);
+}
+static char32_t escORoutine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escODispatch);
+}
+static char32_t setMetaRoutine(char32_t c);  // need forward reference
+static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine,
+                                                 escORoutine, setMetaRoutine};
+static CharacterDispatch escDispatch = {2, "[O", escRoutines};
+
+// Initial dispatch -- we are not in the middle of anything yet
+//
+static char32_t escRoutine(char32_t c) {
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+  return doDispatch(c, escDispatch);
+}
+static CharacterDispatchRoutine initialRoutines[] = {
+    escRoutine, deleteCharRoutine, normalKeyRoutine};
+static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines};
+
+// Special handling for the ESC key because it does double duty
+//
+static char32_t setMetaRoutine(char32_t c) {
+  thisKeyMetaCtrl = META;
+  if (c == 0x1B) {  // another ESC, stay in ESC processing mode
+    c = readUnicodeCharacter();
+    if (c == 0) return 0;
+    return doDispatch(c, escDispatch);
+  }
+  return doDispatch(c, initialDispatch);
+}
+
+}  // namespace EscapeSequenceProcessing // move these out of global namespace
+
+#endif  // #ifndef _WIN32
+
+// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and
+// translate it
+// into an encoded "keystroke".  When convenient, extended keys are translated
+// into their
+// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B.
+//
+// A return value of zero means "no input available", and a return value of -1
+// means "invalid key".
+//
+static char32_t linenoiseReadChar(void) {
+#ifdef _WIN32
+
+  INPUT_RECORD rec;
+  DWORD count;
+  int modifierKeys = 0;
+  bool escSeen = false;
+  while (true) {
+    ReadConsoleInputW(console_in, &rec, 1, &count);
+#if 0  // helper for debugging keystrokes, display info in the debug "Output"
+       // window in the debugger
+        {
+            if ( rec.EventType == KEY_EVENT ) {
+                //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) {
+                    char buf[1024];
+                    sprintf(
+                            buf,
+                            "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, "
+                            "virtual scancode 0x%04X, key %s%s%s%s%s\n",
+                            rec.Event.KeyEvent.uChar.UnicodeChar,
+                            rec.Event.KeyEvent.wRepeatCount,
+                            rec.Event.KeyEvent.wVirtualKeyCode,
+                            rec.Event.KeyEvent.wVirtualScanCode,
+                            rec.Event.KeyEvent.bKeyDown ? "down" : "up",
+                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED)  ?
+                                    " L-Ctrl" : "",
+                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ?
+                                    " R-Ctrl" : "",
+                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED)   ?
+                                    " L-Alt"  : "",
+                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED)  ?
+                                    " R-Alt"  : ""
+                           );
+                    OutputDebugStringA( buf );
+                //}
+            }
+        }
+#endif
+    if (rec.EventType != KEY_EVENT) {
+      continue;
+    }
+    // Windows provides for entry of characters that are not on your keyboard by
+    // sending the
+    // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU ==
+    // Alt key) ...
+    // accept these characters, otherwise only process characters on "key down"
+    if (!rec.Event.KeyEvent.bKeyDown &&
+        rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) {
+      continue;
+    }
+    modifierKeys = 0;
+    // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't
+    // treat this
+    // combination as either CTRL or META we just turn off those two bits, so it
+    // is still
+    // possible to combine CTRL and/or META with an AltGr key by using
+    // right-Ctrl and/or
+    // left-Alt
+    if ((rec.Event.KeyEvent.dwControlKeyState &
+         (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) ==
+        (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) {
+      rec.Event.KeyEvent.dwControlKeyState &=
+          ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED);
+    }
+    if (rec.Event.KeyEvent.dwControlKeyState &
+        (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) {
+      modifierKeys |= CTRL;
+    }
+    if (rec.Event.KeyEvent.dwControlKeyState &
+        (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) {
+      modifierKeys |= META;
+    }
+    if (escSeen) {
+      modifierKeys |= META;
+    }
+    if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) {
+      switch (rec.Event.KeyEvent.wVirtualKeyCode) {
+        case VK_LEFT:
+          return modifierKeys | LEFT_ARROW_KEY;
+        case VK_RIGHT:
+          return modifierKeys | RIGHT_ARROW_KEY;
+        case VK_UP:
+          return modifierKeys | UP_ARROW_KEY;
+        case VK_DOWN:
+          return modifierKeys | DOWN_ARROW_KEY;
+        case VK_DELETE:
+          return modifierKeys | DELETE_KEY;
+        case VK_HOME:
+          return modifierKeys | HOME_KEY;
+        case VK_END:
+          return modifierKeys | END_KEY;
+        case VK_PRIOR:
+          return modifierKeys | PAGE_UP_KEY;
+        case VK_NEXT:
+          return modifierKeys | PAGE_DOWN_KEY;
+        default:
+          continue;  // in raw mode, ReadConsoleInput shows shift, ctrl ...
+      }              //  ... ignore them
+    } else if (rec.Event.KeyEvent.uChar.UnicodeChar ==
+               ctrlChar('[')) {  // ESC, set flag for later
+      escSeen = true;
+      continue;
+    } else {
+      // we got a real character, return it
+      return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar;
+    }
+  }
+
+#else
+  char32_t c;
+  c = readUnicodeCharacter();
+  if (c == 0) return 0;
+
+// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard
+// debugging mode
+// where we print out decimal and decoded values for whatever the "terminal"
+// program
+// gives us on different keystrokes.  Hit ctrl-C to exit this mode.
+//
+#define _DEBUG_LINUX_KEYBOARD
+#if defined(_DEBUG_LINUX_KEYBOARD)
+  if (c == ctrlChar('^')) {  // ctrl-^, special debug mode, prints all keys hit,
+                             // ctrl-C to get out
+    printf(
+        "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit "
+        "this mode\n");
+    while (true) {
+      unsigned char keys[10];
+      int ret = read(0, keys, 10);
+
+      if (ret <= 0) {
+        printf("\nret: %d\n", ret);
+      }
+      for (int i = 0; i < ret; ++i) {
+        char32_t key = static_cast<char32_t>(keys[i]);
+        char* friendlyTextPtr;
+        char friendlyTextBuf[10];
+        const char* prefixText = (key < 0x80) ? "" : "0x80+";
+        char32_t keyCopy = (key < 0x80) ? key : key - 0x80;
+        if (keyCopy >= '!' && keyCopy <= '~') {  // printable
+          friendlyTextBuf[0] = '\'';
+          friendlyTextBuf[1] = keyCopy;
+          friendlyTextBuf[2] = '\'';
+          friendlyTextBuf[3] = 0;
+          friendlyTextPtr = friendlyTextBuf;
+        } else if (keyCopy == ' ') {
+          friendlyTextPtr = const_cast<char*>("space");
+        } else if (keyCopy == 27) {
+          friendlyTextPtr = const_cast<char*>("ESC");
+        } else if (keyCopy == 0) {
+          friendlyTextPtr = const_cast<char*>("NUL");
+        } else if (keyCopy == 127) {
+          friendlyTextPtr = const_cast<char*>("DEL");
+        } else {
+          friendlyTextBuf[0] = '^';
+          friendlyTextBuf[1] = keyCopy + 0x40;
+          friendlyTextBuf[2] = 0;
+          friendlyTextPtr = friendlyTextBuf;
+        }
+        printf("%d x%02X (%s%s)  ", key, key, prefixText, friendlyTextPtr);
+      }
+      printf("\x1b[1G\n");  // go to first column of new line
+
+      // drop out of this loop on ctrl-C
+      if (keys[0] == ctrlChar('C')) {
+        printf("Leaving keyboard debugging mode (on ctrl-C)\n");
+        fflush(stdout);
+        return -2;
+      }
+    }
+  }
+#endif  // _DEBUG_LINUX_KEYBOARD
+
+  EscapeSequenceProcessing::thisKeyMetaCtrl =
+      0;  // no modifiers yet at initialDispatch
+  return EscapeSequenceProcessing::doDispatch(
+      c, EscapeSequenceProcessing::initialDispatch);
+#endif  // #_WIN32
+}
+
+/**
+ * Free memory used in a recent command completion session
+ *
+ * @param lc pointer to a linenoiseCompletions struct
+ */
+static void freeCompletions(linenoiseCompletions* lc) {
+  lc->completionStrings.clear();
+}
+
+/**
+ * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into
+ * ctrlChar( 'A' )
+ * leave META alone
+ *
+ * @param c character to clean up
+ * @return cleaned-up character
+ */
+static int cleanupCtrl(int c) {
+  if (c & CTRL) {
+    int d = c & 0x1FF;
+    if (d >= 'a' && d <= 'z') {
+      c = (c + ('a' - ctrlChar('A'))) & ~CTRL;
+    }
+    if (d >= 'A' && d <= 'Z') {
+      c = (c + ('A' - ctrlChar('A'))) & ~CTRL;
+    }
+    if (d >= ctrlChar('A') && d <= ctrlChar('Z')) {
+      c = c & ~CTRL;
+    }
+  }
+  return c;
+}
+
+// break characters that may precede items to be completed
+static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}";
+
+// maximum number of completions to display without asking
+static const size_t completionCountCutoff = 100;
+
+/**
+ * Handle command completion, using a completionCallback() routine to provide
+ * possible substitutions
+ * This routine handles the mechanics of updating the user's input buffer with
+ * possible replacement
+ * of text as the user selects a proposed completion string, or cancels the
+ * completion attempt.
+ * @param pi     PromptBase struct holding information about the prompt and our
+ * screen position
+ */
+int InputBuffer::completeLine(PromptBase& pi) {
+  linenoiseCompletions lc;
+  char32_t c = 0;
+
+  // completionCallback() expects a parsable entity, so find the previous break
+  // character and
+  // extract a copy to parse.  we also handle the case where tab is hit while
+  // not at end-of-line.
+  int startIndex = pos;
+  while (--startIndex >= 0) {
+    if (strchr(breakChars, buf32[startIndex])) {
+      break;
+    }
+  }
+  ++startIndex;
+  int itemLength = pos - startIndex;
+  Utf32String unicodeCopy(&buf32[startIndex], itemLength);
+  Utf8String parseItem(unicodeCopy);
+
+  // get a list of completions
+  completionCallback(parseItem.get(), &lc);
+
+  // if no completions, we are done
+  if (lc.completionStrings.size() == 0) {
+    beep();
+    freeCompletions(&lc);
+    return 0;
+  }
+
+  // at least one completion
+  int longestCommonPrefix = 0;
+  int displayLength = 0;
+  if (lc.completionStrings.size() == 1) {
+    longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length());
+  } else {
+    bool keepGoing = true;
+    while (keepGoing) {
+      for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) {
+        char32_t c1 = lc.completionStrings[j][longestCommonPrefix];
+        char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix];
+        if ((0 == c1) || (0 == c2) || (c1 != c2)) {
+          keepGoing = false;
+          break;
+        }
+      }
+      if (keepGoing) {
+        ++longestCommonPrefix;
+      }
+    }
+  }
+  if (lc.completionStrings.size() != 1) {  // beep if ambiguous
+    beep();
+  }
+
+  // if we can extend the item, extend it and return to main loop
+  if (longestCommonPrefix > itemLength) {
+    displayLength = len + longestCommonPrefix - itemLength;
+    if (displayLength > buflen) {
+      longestCommonPrefix -= displayLength - buflen;  // don't overflow buffer
+      displayLength = buflen;                         // truncate the insertion
+      beep();                                         // and make a noise
+    }
+    Utf32String displayText(displayLength + 1);
+    memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex);
+    memcpy(&displayText[startIndex], &lc.completionStrings[0][0],
+           sizeof(char32_t) * longestCommonPrefix);
+    int tailIndex = startIndex + longestCommonPrefix;
+    memcpy(&displayText[tailIndex], &buf32[pos],
+           sizeof(char32_t) * (displayLength - tailIndex + 1));
+    copyString32(buf32, displayText.get(), displayLength);
+    pos = startIndex + longestCommonPrefix;
+    len = displayLength;
+    refreshLine(pi);
+    return 0;
+  }
+
+  // we can't complete any further, wait for second tab
+  do {
+    c = linenoiseReadChar();
+    c = cleanupCtrl(c);
+  } while (c == static_cast<char32_t>(-1));
+
+  // if any character other than tab, pass it to the main loop
+  if (c != ctrlChar('I')) {
+    freeCompletions(&lc);
+    return c;
+  }
+
+  // we got a second tab, maybe show list of possible completions
+  bool showCompletions = true;
+  bool onNewLine = false;
+  if (lc.completionStrings.size() > completionCountCutoff) {
+    int savePos =
+        pos;  // move cursor to EOL to avoid overwriting the command line
+    pos = len;
+    refreshLine(pi);
+    pos = savePos;
+    printf("\nDisplay all %u possibilities? (y or n)",
+           static_cast<unsigned int>(lc.completionStrings.size()));
+    fflush(stdout);
+    onNewLine = true;
+    while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) {
+      do {
+        c = linenoiseReadChar();
+        c = cleanupCtrl(c);
+      } while (c == static_cast<char32_t>(-1));
+    }
+    switch (c) {
+      case 'n':
+      case 'N':
+        showCompletions = false;
+        freeCompletions(&lc);
+        break;
+      case ctrlChar('C'):
+        showCompletions = false;
+        freeCompletions(&lc);
+        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
+        c = 0;
+        break;
+    }
+  }
+
+  // if showing the list, do it the way readline does it
+  bool stopList = false;
+  if (showCompletions) {
+    int longestCompletion = 0;
+    for (size_t j = 0; j < lc.completionStrings.size(); ++j) {
+      itemLength = static_cast<int>(lc.completionStrings[j].length());
+      if (itemLength > longestCompletion) {
+        longestCompletion = itemLength;
+      }
+    }
+    longestCompletion += 2;
+    int columnCount = pi.promptScreenColumns / longestCompletion;
+    if (columnCount < 1) {
+      columnCount = 1;
+    }
+    if (!onNewLine) {  // skip this if we showed "Display all %d possibilities?"
+      int savePos =
+          pos;  // move cursor to EOL to avoid overwriting the command line
+      pos = len;
+      refreshLine(pi);
+      pos = savePos;
+    }
+    size_t pauseRow = getScreenRows() - 1;
+    size_t rowCount =
+        (lc.completionStrings.size() + columnCount - 1) / columnCount;
+    for (size_t row = 0; row < rowCount; ++row) {
+      if (row == pauseRow) {
+        printf("\n--More--");
+        fflush(stdout);
+        c = 0;
+        bool doBeep = false;
+        while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' &&
+               c != 'n' && c != 'N' && c != 'q' && c != 'Q' &&
+               c != ctrlChar('C')) {
+          if (doBeep) {
+            beep();
+          }
+          doBeep = true;
+          do {
+            c = linenoiseReadChar();
+            c = cleanupCtrl(c);
+          } while (c == static_cast<char32_t>(-1));
+        }
+        switch (c) {
+          case ' ':
+          case 'y':
+          case 'Y':
+            printf("\r        \r");
+            pauseRow += getScreenRows() - 1;
+            break;
+          case '\r':
+          case '\n':
+            printf("\r        \r");
+            ++pauseRow;
+            break;
+          case 'n':
+          case 'N':
+          case 'q':
+          case 'Q':
+            printf("\r        \r");
+            stopList = true;
+            break;
+          case ctrlChar('C'):
+            if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
+            stopList = true;
+            break;
+        }
+      } else {
+        printf("\n");
+      }
+      if (stopList) {
+        break;
+      }
+      for (int column = 0; column < columnCount; ++column) {
+        size_t index = (column * rowCount) + row;
+        if (index < lc.completionStrings.size()) {
+          itemLength = static_cast<int>(lc.completionStrings[index].length());
+          fflush(stdout);
+          if (write32(1, lc.completionStrings[index].get(), itemLength) == -1)
+            return -1;
+          if (((column + 1) * rowCount) + row < lc.completionStrings.size()) {
+            for (int k = itemLength; k < longestCompletion; ++k) {
+              printf(" ");
+            }
+          }
+        }
+      }
+    }
+    fflush(stdout);
+    freeCompletions(&lc);
+  }
+
+  // display the prompt on a new line, then redisplay the input buffer
+  if (!stopList || c == ctrlChar('C')) {
+    if (write(1, "\n", 1) == -1) return 0;
+  }
+  if (!pi.write()) return 0;
+#ifndef _WIN32
+  // we have to generate our own newline on line wrap on Linux
+  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
+    if (write(1, "\n", 1) == -1) return 0;
+#endif
+  pi.promptCursorRowOffset = pi.promptExtraLines;
+  refreshLine(pi);
+  return 0;
+}
+
+/**
+ * Clear the screen ONLY (no redisplay of anything)
+ */
+void linenoiseClearScreen(void) {
+#ifdef _WIN32
+  COORD coord = {0, 0};
+  CONSOLE_SCREEN_BUFFER_INFO inf;
+  HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+  GetConsoleScreenBufferInfo(screenHandle, &inf);
+  SetConsoleCursorPosition(screenHandle, coord);
+  DWORD count;
+  FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y,
+                              coord, &count);
+#else
+  if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return;
+#endif
+}
+
+void InputBuffer::clearScreen(PromptBase& pi) {
+  linenoiseClearScreen();
+  if (!pi.write()) return;
+#ifndef _WIN32
+  // we have to generate our own newline on line wrap on Linux
+  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
+    if (write(1, "\n", 1) == -1) return;
+#endif
+  pi.promptCursorRowOffset = pi.promptExtraLines;
+  refreshLine(pi);
+}
+
+/**
+ * Incremental history search -- take over the prompt and keyboard as the user
+ * types a search
+ * string, deletes characters from it, changes direction, and either accepts the
+ * found line (for
+ * execution orediting) or cancels.
+ * @param pi        PromptBase struct holding information about the (old,
+ * static) prompt and our
+ *                  screen position
+ * @param startChar the character that began the search, used to set the initial
+ * direction
+ */
+int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) {
+  size_t bufferSize;
+  size_t ucharCount = 0;
+
+  // if not already recalling, add the current line to the history list so we
+  // don't have to
+  // special case it
+  if (historyIndex == historyLen - 1) {
+    free(history[historyLen - 1]);
+    bufferSize = sizeof(char32_t) * len + 1;
+    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
+    copyString32to8(tempBuffer.get(), bufferSize, buf32);
+    history[historyLen - 1] = strdup8(tempBuffer.get());
+  }
+  int historyLineLength = len;
+  int historyLinePosition = pos;
+  char32_t emptyBuffer[1];
+  char emptyWidths[1];
+  InputBuffer empty(emptyBuffer, emptyWidths, 1);
+  empty.refreshLine(pi);  // erase the old input first
+  DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1);
+
+  dp.promptPreviousLen = pi.promptPreviousLen;
+  dp.promptPreviousInputLen = pi.promptPreviousInputLen;
+  dynamicRefresh(dp, buf32, historyLineLength,
+                 historyLinePosition);  // draw user's text with our prompt
+
+  // loop until we get an exit character
+  int c = 0;
+  bool keepLooping = true;
+  bool useSearchedLine = true;
+  bool searchAgain = false;
+  char32_t* activeHistoryLine = 0;
+  while (keepLooping) {
+    c = linenoiseReadChar();
+    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
+
+    switch (c) {
+      // these characters keep the selected text but do not execute it
+      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
+      case HOME_KEY:
+      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
+      case LEFT_ARROW_KEY:
+      case META + 'b':  // meta-B, move cursor left by one word
+      case META + 'B':
+      case CTRL + LEFT_ARROW_KEY:
+      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
+      case ctrlChar('D'):
+      case META + 'd':  // meta-D, kill word to right of cursor
+      case META + 'D':
+      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
+      case END_KEY:
+      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
+      case RIGHT_ARROW_KEY:
+      case META + 'f':  // meta-F, move cursor right by one word
+      case META + 'F':
+      case CTRL + RIGHT_ARROW_KEY:
+      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
+      case META + ctrlChar('H'):
+      case ctrlChar('J'):
+      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
+      case ctrlChar('M'):
+      case ctrlChar('N'):  // ctrl-N, recall next line in history
+      case ctrlChar('P'):  // ctrl-P, recall previous line in history
+      case DOWN_ARROW_KEY:
+      case UP_ARROW_KEY:
+      case ctrlChar('T'):  // ctrl-T, transpose characters
+      case ctrlChar(
+          'U'):  // ctrl-U, kill all characters to the left of the cursor
+      case ctrlChar('W'):
+      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
+      case META + 'Y':
+      case 127:
+      case DELETE_KEY:
+      case META + '<':  // start of history
+      case PAGE_UP_KEY:
+      case META + '>':  // end of history
+      case PAGE_DOWN_KEY:
+        keepLooping = false;
+        break;
+
+      // these characters revert the input line to its previous state
+      case ctrlChar('C'):  // ctrl-C, abort this line
+      case ctrlChar('G'):
+      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
+        keepLooping = false;
+        useSearchedLine = false;
+        if (c != ctrlChar('L')) {
+          c = -1;  // ctrl-C and ctrl-G just abort the search and do nothing
+                   // else
+        }
+        break;
+
+      // these characters stay in search mode and update the display
+      case ctrlChar('S'):
+      case ctrlChar('R'):
+        if (dp.searchTextLen ==
+            0) {  // if no current search text, recall previous text
+          if (previousSearchText.length()) {
+            dp.updateSearchText(previousSearchText.get());
+          }
+        }
+        if ((dp.direction == 1 && c == ctrlChar('R')) ||
+            (dp.direction == -1 && c == ctrlChar('S'))) {
+          dp.direction = 0 - dp.direction;  // reverse direction
+          dp.updateSearchPrompt();          // change the prompt
+        } else {
+          searchAgain = true;  // same direction, search again
+        }
+        break;
+
+// job control is its own thing
+#ifndef _WIN32
+      case ctrlChar('Z'):  // ctrl-Z, job control
+        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
+                           // mode
+        raise(SIGSTOP);    // Break out in mid-line
+        enableRawMode();   // Back from Linux shell, re-enter raw mode
+        {
+          bufferSize = historyLineLength + 1;
+          unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]);
+          copyString8to32(tempUnicode.get(), bufferSize, ucharCount,
+                          history[historyIndex]);
+          dynamicRefresh(dp, tempUnicode.get(), historyLineLength,
+                         historyLinePosition);
+        }
+        continue;
+        break;
+#endif
+
+      // these characters update the search string, and hence the selected input
+      // line
+      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
+        if (dp.searchTextLen > 0) {
+          unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]);
+          --dp.searchTextLen;
+          dp.searchText[dp.searchTextLen] = 0;
+          copyString32(tempUnicode.get(), dp.searchText.get(),
+                       dp.searchTextLen);
+          dp.updateSearchText(tempUnicode.get());
+        } else {
+          beep();
+        }
+        break;
+
+      case ctrlChar('Y'):  // ctrl-Y, yank killed text
+        break;
+
+      default:
+        if (!isControlChar(c) && c <= 0x0010FFFF) {  // not an action character
+          unique_ptr<char32_t[]> tempUnicode(
+              new char32_t[dp.searchTextLen + 2]);
+          copyString32(tempUnicode.get(), dp.searchText.get(),
+                       dp.searchTextLen);
+          tempUnicode[dp.searchTextLen] = c;
+          tempUnicode[dp.searchTextLen + 1] = 0;
+          dp.updateSearchText(tempUnicode.get());
+        } else {
+          beep();
+        }
+    }  // switch
+
+    // if we are staying in search mode, search now
+    if (keepLooping) {
+      bufferSize = historyLineLength + 1;
+      if (activeHistoryLine) {
+        delete[] activeHistoryLine;
+        activeHistoryLine = nullptr;
+      }
+      activeHistoryLine = new char32_t[bufferSize];
+      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
+                      history[historyIndex]);
+      if (dp.searchTextLen > 0) {
+        bool found = false;
+        int historySearchIndex = historyIndex;
+        int lineLength = static_cast<int>(ucharCount);
+        int lineSearchPos = historyLinePosition;
+        if (searchAgain) {
+          lineSearchPos += dp.direction;
+        }
+        searchAgain = false;
+        while (true) {
+          while ((dp.direction > 0) ? (lineSearchPos < lineLength)
+                                    : (lineSearchPos >= 0)) {
+            if (strncmp32(dp.searchText.get(),
+                          &activeHistoryLine[lineSearchPos],
+                          dp.searchTextLen) == 0) {
+              found = true;
+              break;
+            }
+            lineSearchPos += dp.direction;
+          }
+          if (found) {
+            historyIndex = historySearchIndex;
+            historyLineLength = lineLength;
+            historyLinePosition = lineSearchPos;
+            break;
+          } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1)
+                                        : (historySearchIndex > 0)) {
+            historySearchIndex += dp.direction;
+            bufferSize = strlen8(history[historySearchIndex]) + 1;
+            delete[] activeHistoryLine;
+            activeHistoryLine = nullptr;
+            activeHistoryLine = new char32_t[bufferSize];
+            copyString8to32(activeHistoryLine, bufferSize, ucharCount,
+                            history[historySearchIndex]);
+            lineLength = static_cast<int>(ucharCount);
+            lineSearchPos =
+                (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen);
+          } else {
+            beep();
+            break;
+          }
+        };  // while
+      }
+      if (activeHistoryLine) {
+        delete[] activeHistoryLine;
+        activeHistoryLine = nullptr;
+      }
+      bufferSize = historyLineLength + 1;
+      activeHistoryLine = new char32_t[bufferSize];
+      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
+                      history[historyIndex]);
+      dynamicRefresh(dp, activeHistoryLine, historyLineLength,
+                     historyLinePosition);  // draw user's text with our prompt
+    }
+  }  // while
+
+  // leaving history search, restore previous prompt, maybe make searched line
+  // current
+  PromptBase pb;
+  pb.promptChars = pi.promptIndentation;
+  pb.promptBytes = pi.promptBytes;
+  Utf32String tempUnicode(pb.promptBytes + 1);
+
+  copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition],
+               pb.promptBytes - pi.promptLastLinePosition);
+  tempUnicode.initFromBuffer();
+  pb.promptText = tempUnicode;
+  pb.promptExtraLines = 0;
+  pb.promptIndentation = pi.promptIndentation;
+  pb.promptLastLinePosition = 0;
+  pb.promptPreviousInputLen = historyLineLength;
+  pb.promptCursorRowOffset = dp.promptCursorRowOffset;
+  pb.promptScreenColumns = pi.promptScreenColumns;
+  pb.promptPreviousLen = dp.promptChars;
+  if (useSearchedLine && activeHistoryLine) {
+    historyRecallMostRecent = true;
+    copyString32(buf32, activeHistoryLine, buflen + 1);
+    len = historyLineLength;
+    pos = historyLinePosition;
+  }
+  if (activeHistoryLine) {
+    delete[] activeHistoryLine;
+    activeHistoryLine = nullptr;
+  }
+  dynamicRefresh(pb, buf32, len,
+                 pos);  // redraw the original prompt with current input
+  pi.promptPreviousInputLen = len;
+  pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset;
+  previousSearchText =
+      dp.searchText;  // save search text for possible reuse on ctrl-R ctrl-R
+  return c;           // pass a character or -1 back to main loop
+}
+
+static bool isCharacterAlphanumeric(char32_t testChar) {
+#ifdef _WIN32
+  return (iswalnum((wint_t)testChar) != 0 ? true : false);
+#else
+  return (iswalnum(testChar) != 0 ? true : false);
+#endif
+}
+
+#ifndef _WIN32
+static bool gotResize = false;
+#endif
+static int keyType = 0;
+
+int InputBuffer::getInputLine(PromptBase& pi) {
+  keyType = 0;
+
+  // The latest history entry is always our current buffer
+  if (len > 0) {
+    size_t bufferSize = sizeof(char32_t) * len + 1;
+    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
+    copyString32to8(tempBuffer.get(), bufferSize, buf32);
+    linenoiseHistoryAdd(tempBuffer.get());
+  } else {
+    linenoiseHistoryAdd("");
+  }
+  historyIndex = historyLen - 1;
+  historyRecallMostRecent = false;
+
+  // display the prompt
+  if (!pi.write()) return -1;
+
+#ifndef _WIN32
+  // we have to generate our own newline on line wrap on Linux
+  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
+    if (write(1, "\n", 1) == -1) return -1;
+#endif
+
+  // the cursor starts out at the end of the prompt
+  pi.promptCursorRowOffset = pi.promptExtraLines;
+
+  // kill and yank start in "other" mode
+  killRing.lastAction = KillRing::actionOther;
+
+  // when history search returns control to us, we execute its terminating
+  // keystroke
+  int terminatingKeystroke = -1;
+
+  // if there is already text in the buffer, display it first
+  if (len > 0) {
+    refreshLine(pi);
+  }
+
+  // loop collecting characters, respond to line editing characters
+  while (true) {
+    int c;
+    if (terminatingKeystroke == -1) {
+      c = linenoiseReadChar();  // get a new keystroke
+
+      keyType = 0; 
+      if (c != 0) {
+        // set flag that we got some input
+        if (c == ctrlChar('C')) {
+          keyType = 1;
+        } else if (c == ctrlChar('D')) {
+          keyType = 2;
+        }
+      }
+
+#ifndef _WIN32
+      if (c == 0 && gotResize) {
+        // caught a window resize event
+        // now redraw the prompt and line
+        gotResize = false;
+        pi.promptScreenColumns = getScreenColumns();
+        dynamicRefresh(pi, buf32, len,
+                       pos);  // redraw the original prompt with current input
+        continue;
+      }
+#endif
+    } else {
+      c = terminatingKeystroke;   // use the terminating keystroke from search
+      terminatingKeystroke = -1;  // clear it once we've used it
+    }
+
+    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
+
+    if (c == 0) {
+      return len;
+    }
+
+    if (c == -1) {
+      refreshLine(pi);
+      continue;
+    }
+
+    if (c == -2) {
+      if (!pi.write()) return -1;
+      refreshLine(pi);
+      continue;
+    }
+
+    // ctrl-I/tab, command completion, needs to be before switch statement
+    if (c == ctrlChar('I') && completionCallback) {
+      if (pos == 0)  // SERVER-4967 -- in earlier versions, you could paste
+                     // previous output
+        continue;    //  back into the shell ... this output may have leading
+                     //  tabs.
+      // This hack (i.e. what the old code did) prevents command completion
+      //  on an empty line but lets users paste text with leading tabs.
+
+      killRing.lastAction = KillRing::actionOther;
+      historyRecallMostRecent = false;
+
+      // completeLine does the actual completion and replacement
+      c = completeLine(pi);
+
+      if (c < 0)  // return on error
+        return len;
+
+      if (c == 0)  // read next character when 0
+        continue;
+
+      // deliberate fall-through here, so we use the terminating character
+    }
+
+    switch (c) {
+      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
+      case HOME_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        pos = 0;
+        refreshLine(pi);
+        break;
+
+      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
+      case LEFT_ARROW_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        if (pos > 0) {
+          --pos;
+          refreshLine(pi);
+        }
+        break;
+
+      case META + 'b':  // meta-B, move cursor left by one word
+      case META + 'B':
+      case CTRL + LEFT_ARROW_KEY:
+      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
+        killRing.lastAction = KillRing::actionOther;
+        if (pos > 0) {
+          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
+            --pos;
+          }
+          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
+            --pos;
+          }
+          refreshLine(pi);
+        }
+        break;
+
+      case ctrlChar('C'):  // ctrl-C, abort this line
+        killRing.lastAction = KillRing::actionOther;
+        historyRecallMostRecent = false;
+        errno = EAGAIN;
+        --historyLen;
+        free(history[historyLen]);
+        // we need one last refresh with the cursor at the end of the line
+        // so we don't display the next prompt over the previous input line
+        pos = len;  // pass len as pos for EOL
+        refreshLine(pi);
+        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
+        return -1;
+
+      case META + 'c':  // meta-C, give word initial Cap
+      case META + 'C':
+        killRing.lastAction = KillRing::actionOther;
+        historyRecallMostRecent = false;
+        if (pos < len) {
+          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
+            ++pos;
+          }
+          if (pos < len && isCharacterAlphanumeric(buf32[pos])) {
+            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
+              buf32[pos] += 'A' - 'a';
+            }
+            ++pos;
+          }
+          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
+            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
+              buf32[pos] += 'a' - 'A';
+            }
+            ++pos;
+          }
+          refreshLine(pi);
+        }
+        break;
+
+      // ctrl-D, delete the character under the cursor
+      // on an empty line, exit the shell
+      case ctrlChar('D'):
+        killRing.lastAction = KillRing::actionOther;
+        if (len > 0 && pos < len) {
+          historyRecallMostRecent = false;
+          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
+          --len;
+          refreshLine(pi);
+        } else if (len == 0) {
+          --historyLen;
+          free(history[historyLen]);
+          return -1;
+        }
+        break;
+
+      case META + 'd':  // meta-D, kill word to right of cursor
+      case META + 'D':
+        if (pos < len) {
+          historyRecallMostRecent = false;
+          int endingPos = pos;
+          while (endingPos < len &&
+                 !isCharacterAlphanumeric(buf32[endingPos])) {
+            ++endingPos;
+          }
+          while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) {
+            ++endingPos;
+          }
+          killRing.kill(&buf32[pos], endingPos - pos, true);
+          memmove(buf32 + pos, buf32 + endingPos,
+                  sizeof(char32_t) * (len - endingPos + 1));
+          len -= endingPos - pos;
+          refreshLine(pi);
+        }
+        killRing.lastAction = KillRing::actionKill;
+        break;
+
+      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
+      case END_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        pos = len;
+        refreshLine(pi);
+        break;
+
+      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
+      case RIGHT_ARROW_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        if (pos < len) {
+          ++pos;
+          refreshLine(pi);
+        }
+        break;
+
+      case META + 'f':  // meta-F, move cursor right by one word
+      case META + 'F':
+      case CTRL + RIGHT_ARROW_KEY:
+      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
+        killRing.lastAction = KillRing::actionOther;
+        if (pos < len) {
+          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
+            ++pos;
+          }
+          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
+            ++pos;
+          }
+          refreshLine(pi);
+        }
+        break;
+
+      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
+        killRing.lastAction = KillRing::actionOther;
+        if (pos > 0) {
+          historyRecallMostRecent = false;
+          memmove(buf32 + pos - 1, buf32 + pos,
+                  sizeof(char32_t) * (1 + len - pos));
+          --pos;
+          --len;
+          refreshLine(pi);
+        }
+        break;
+
+      // meta-Backspace, kill word to left of cursor
+      case META + ctrlChar('H'):
+        if (pos > 0) {
+          historyRecallMostRecent = false;
+          int startingPos = pos;
+          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
+            --pos;
+          }
+          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
+            --pos;
+          }
+          killRing.kill(&buf32[pos], startingPos - pos, false);
+          memmove(buf32 + pos, buf32 + startingPos,
+                  sizeof(char32_t) * (len - startingPos + 1));
+          len -= startingPos - pos;
+          refreshLine(pi);
+        }
+        killRing.lastAction = KillRing::actionKill;
+        break;
+
+      case ctrlChar('J'):  // ctrl-J/linefeed/newline, accept line
+      case ctrlChar('M'):  // ctrl-M/return/enter
+        killRing.lastAction = KillRing::actionOther;
+        // we need one last refresh with the cursor at the end of the line
+        // so we don't display the next prompt over the previous input line
+        pos = len;  // pass len as pos for EOL
+        refreshLine(pi);
+        historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2;
+        --historyLen;
+        free(history[historyLen]);
+        return len;
+
+      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
+        killRing.kill(&buf32[pos], len - pos, true);
+        buf32[pos] = '\0';
+        len = pos;
+        refreshLine(pi);
+        killRing.lastAction = KillRing::actionKill;
+        historyRecallMostRecent = false;
+        break;
+
+      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
+        clearScreen(pi);
+        break;
+
+      case META + 'l':  // meta-L, lowercase word
+      case META + 'L':
+        killRing.lastAction = KillRing::actionOther;
+        if (pos < len) {
+          historyRecallMostRecent = false;
+          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
+            ++pos;
+          }
+          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
+            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
+              buf32[pos] += 'a' - 'A';
+            }
+            ++pos;
+          }
+          refreshLine(pi);
+        }
+        break;
+
+      case ctrlChar('N'):  // ctrl-N, recall next line in history
+      case ctrlChar('P'):  // ctrl-P, recall previous line in history
+      case DOWN_ARROW_KEY:
+      case UP_ARROW_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        // if not already recalling, add the current line to the history list so
+        // we don't
+        // have to special case it
+        if (historyIndex == historyLen - 1) {
+          free(history[historyLen - 1]);
+          size_t tempBufferSize = sizeof(char32_t) * len + 1;
+          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
+          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
+          history[historyLen - 1] = strdup8(tempBuffer.get());
+        }
+        if (historyLen > 1) {
+          if (c == UP_ARROW_KEY) {
+            c = ctrlChar('P');
+          }
+          if (historyPreviousIndex != -2 && c != ctrlChar('P')) {
+            historyIndex =
+                1 + historyPreviousIndex;  // emulate Windows down-arrow
+          } else {
+            historyIndex += (c == ctrlChar('P')) ? -1 : 1;
+          }
+          historyPreviousIndex = -2;
+          if (historyIndex < 0) {
+            historyIndex = 0;
+            break;
+          } else if (historyIndex >= historyLen) {
+            historyIndex = historyLen - 1;
+            break;
+          }
+          historyRecallMostRecent = true;
+          size_t ucharCount = 0;
+          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
+          len = pos = static_cast<int>(ucharCount);
+          refreshLine(pi);
+        }
+        break;
+
+      case ctrlChar('R'):  // ctrl-R, reverse history search
+      case ctrlChar('S'):  // ctrl-S, forward history search
+        terminatingKeystroke = incrementalHistorySearch(pi, c);
+        break;
+
+      case ctrlChar('T'):  // ctrl-T, transpose characters
+        killRing.lastAction = KillRing::actionOther;
+        if (pos > 0 && len > 1) {
+          historyRecallMostRecent = false;
+          size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1;
+          char32_t aux = buf32[leftCharPos];
+          buf32[leftCharPos] = buf32[leftCharPos + 1];
+          buf32[leftCharPos + 1] = aux;
+          if (pos != len) ++pos;
+          refreshLine(pi);
+        }
+        break;
+
+      case ctrlChar(
+          'U'):  // ctrl-U, kill all characters to the left of the cursor
+        if (pos > 0) {
+          historyRecallMostRecent = false;
+          killRing.kill(&buf32[0], pos, false);
+          len -= pos;
+          memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1));
+          pos = 0;
+          refreshLine(pi);
+        }
+        killRing.lastAction = KillRing::actionKill;
+        break;
+
+      case META + 'u':  // meta-U, uppercase word
+      case META + 'U':
+        killRing.lastAction = KillRing::actionOther;
+        if (pos < len) {
+          historyRecallMostRecent = false;
+          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
+            ++pos;
+          }
+          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
+            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
+              buf32[pos] += 'A' - 'a';
+            }
+            ++pos;
+          }
+          refreshLine(pi);
+        }
+        break;
+
+      // ctrl-W, kill to whitespace (not word) to left of cursor
+      case ctrlChar('W'):
+        if (pos > 0) {
+          historyRecallMostRecent = false;
+          int startingPos = pos;
+          while (pos > 0 && buf32[pos - 1] == ' ') {
+            --pos;
+          }
+          while (pos > 0 && buf32[pos - 1] != ' ') {
+            --pos;
+          }
+          killRing.kill(&buf32[pos], startingPos - pos, false);
+          memmove(buf32 + pos, buf32 + startingPos,
+                  sizeof(char32_t) * (len - startingPos + 1));
+          len -= startingPos - pos;
+          refreshLine(pi);
+        }
+        killRing.lastAction = KillRing::actionKill;
+        break;
+
+      case ctrlChar('Y'):  // ctrl-Y, yank killed text
+        historyRecallMostRecent = false;
+        {
+          Utf32String* restoredText = killRing.yank();
+          if (restoredText) {
+            bool truncated = false;
+            size_t ucharCount = restoredText->length();
+            if (ucharCount > static_cast<size_t>(buflen - len)) {
+              ucharCount = buflen - len;
+              truncated = true;
+            }
+            memmove(buf32 + pos + ucharCount, buf32 + pos,
+                    sizeof(char32_t) * (len - pos + 1));
+            memmove(buf32 + pos, restoredText->get(),
+                    sizeof(char32_t) * ucharCount);
+            pos += static_cast<int>(ucharCount);
+            len += static_cast<int>(ucharCount);
+            refreshLine(pi);
+            killRing.lastAction = KillRing::actionYank;
+            killRing.lastYankSize = ucharCount;
+            if (truncated) {
+              beep();
+            }
+          } else {
+            beep();
+          }
+        }
+        break;
+
+      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
+      case META + 'Y':
+        if (killRing.lastAction == KillRing::actionYank) {
+          historyRecallMostRecent = false;
+          Utf32String* restoredText = killRing.yankPop();
+          if (restoredText) {
+            bool truncated = false;
+            size_t ucharCount = restoredText->length();
+            if (ucharCount >
+                static_cast<size_t>(killRing.lastYankSize + buflen - len)) {
+              ucharCount = killRing.lastYankSize + buflen - len;
+              truncated = true;
+            }
+            if (ucharCount > killRing.lastYankSize) {
+              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
+                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
+              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
+                      sizeof(char32_t) * ucharCount);
+            } else {
+              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
+                      sizeof(char32_t) * ucharCount);
+              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
+                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
+            }
+            pos += static_cast<int>(ucharCount - killRing.lastYankSize);
+            len += static_cast<int>(ucharCount - killRing.lastYankSize);
+            killRing.lastYankSize = ucharCount;
+            refreshLine(pi);
+            if (truncated) {
+              beep();
+            }
+            break;
+          }
+        }
+        beep();
+        break;
+
+#ifndef _WIN32
+      case ctrlChar('Z'):  // ctrl-Z, job control
+        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
+                           // mode
+        raise(SIGSTOP);    // Break out in mid-line
+        enableRawMode();   // Back from Linux shell, re-enter raw mode
+        if (!pi.write()) break;  // Redraw prompt
+        refreshLine(pi);         // Refresh the line
+        break;
+#endif
+
+      // DEL, delete the character under the cursor
+      case 127:
+      case DELETE_KEY:
+        killRing.lastAction = KillRing::actionOther;
+        if (len > 0 && pos < len) {
+          historyRecallMostRecent = false;
+          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
+          --len;
+          refreshLine(pi);
+        }
+        break;
+
+      case META + '<':     // meta-<, beginning of history
+      case PAGE_UP_KEY:    // Page Up, beginning of history
+      case META + '>':     // meta->, end of history
+      case PAGE_DOWN_KEY:  // Page Down, end of history
+        killRing.lastAction = KillRing::actionOther;
+        // if not already recalling, add the current line to the history list so
+        // we don't
+        // have to special case it
+        if (historyIndex == historyLen - 1) {
+          free(history[historyLen - 1]);
+          size_t tempBufferSize = sizeof(char32_t) * len + 1;
+          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
+          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
+          history[historyLen - 1] = strdup8(tempBuffer.get());
+        }
+        if (historyLen > 1) {
+          historyIndex =
+              (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1;
+          historyPreviousIndex = -2;
+          historyRecallMostRecent = true;
+          size_t ucharCount = 0;
+          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
+          len = pos = static_cast<int>(ucharCount);
+          refreshLine(pi);
+        }
+        break;
+
+      // not one of our special characters, maybe insert it in the buffer
+      default:
+        killRing.lastAction = KillRing::actionOther;
+        historyRecallMostRecent = false;
+        if (c & (META | CTRL)) {  // beep on unknown Ctrl and/or Meta keys
+          beep();
+          break;
+        }
+        if (len < buflen) {
+          if (isControlChar(c)) {  // don't insert control characters
+            beep();
+            break;
+          }
+          if (len == pos) {  // at end of buffer
+            buf32[pos] = c;
+            ++pos;
+            ++len;
+            buf32[len] = '\0';
+            int inputLen = calculateColumnPosition(buf32, len);
+            if (pi.promptIndentation + inputLen < pi.promptScreenColumns) {
+              if (inputLen > pi.promptPreviousInputLen)
+                pi.promptPreviousInputLen = inputLen;
+              /* Avoid a full update of the line in the
+               * trivial case. */
+              if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1)
+                return -1;
+            } else {
+              refreshLine(pi);
+            }
+          } else {  // not at end of buffer, have to move characters to our
+                    // right
+            memmove(buf32 + pos + 1, buf32 + pos,
+                    sizeof(char32_t) * (len - pos));
+            buf32[pos] = c;
+            ++len;
+            ++pos;
+            buf32[len] = '\0';
+            refreshLine(pi);
+          }
+        } else {
+          beep();  // buffer is full, beep on new characters
+        }
+        break;
+    }
+  }
+  return len;
+}
+
+static string preloadedBufferContents;  // used with linenoisePreloadBuffer
+static string preloadErrorMessage;
+
+/**
+ * linenoisePreloadBuffer provides text to be inserted into the command buffer
+ *
+ * the provided text will be processed to be usable and will be used to preload
+ * the input buffer on the next call to linenoise()
+ *
+ * @param preloadText text to begin with on the next call to linenoise()
+ */
+void linenoisePreloadBuffer(const char* preloadText) {
+  if (!preloadText) {
+    return;
+  }
+  int bufferSize = static_cast<int>(strlen(preloadText) + 1);
+  unique_ptr<char[]> tempBuffer(new char[bufferSize]);
+  strncpy(&tempBuffer[0], preloadText, bufferSize);
+
+  // remove characters that won't display correctly
+  char* pIn = &tempBuffer[0];
+  char* pOut = pIn;
+  bool controlsStripped = false;
+  bool whitespaceSeen = false;
+  while (*pIn) {
+    unsigned char c =
+        *pIn++;       // we need unsigned so chars 0x80 and above are allowed
+    if ('\r' == c) {  // silently skip CR
+      continue;
+    }
+    if ('\n' == c || '\t' == c) {  // note newline or tab
+      whitespaceSeen = true;
+      continue;
+    }
+    if (isControlChar(
+            c)) {  // remove other control characters, flag for message
+      controlsStripped = true;
+      *pOut++ = ' ';
+      continue;
+    }
+    if (whitespaceSeen) {  // convert whitespace to a single space
+      *pOut++ = ' ';
+      whitespaceSeen = false;
+    }
+    *pOut++ = c;
+  }
+  *pOut = 0;
+  int processedLength = static_cast<int>(pOut - tempBuffer.get());
+  bool lineTruncated = false;
+  if (processedLength > (LINENOISE_MAX_LINE - 1)) {
+    lineTruncated = true;
+    tempBuffer[LINENOISE_MAX_LINE - 1] = 0;
+  }
+  preloadedBufferContents = tempBuffer.get();
+  if (controlsStripped) {
+    preloadErrorMessage +=
+        " [Edited line: control characters were converted to spaces]\n";
+  }
+  if (lineTruncated) {
+    preloadErrorMessage += " [Edited line: the line length was reduced from ";
+    char buf[128];
+    snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength,
+             (LINENOISE_MAX_LINE - 1));
+    preloadErrorMessage += buf;
+  }
+}
+
+/**
+ * linenoise is a readline replacement.
+ *
+ * call it with a prompt to display and it will return a line of input from the
+ * user
+ *
+ * @param prompt text of prompt to display to the user
+ * @return       the returned string belongs to the caller on return and must be
+ * freed to prevent
+ *               memory leaks
+ */
+char* linenoise(const char* prompt) {
+#ifndef _WIN32
+  gotResize = false;
+#endif
+  if (isatty(STDIN_FILENO)) {  // input is from a terminal
+    char32_t buf32[LINENOISE_MAX_LINE];
+    char charWidths[LINENOISE_MAX_LINE];
+    if (!preloadErrorMessage.empty()) {
+      printf("%s", preloadErrorMessage.c_str());
+      fflush(stdout);
+      preloadErrorMessage.clear();
+    }
+    PromptInfo pi(prompt, getScreenColumns());
+    if (isUnsupportedTerm()) {
+      if (!pi.write()) return 0;
+      fflush(stdout);
+      if (preloadedBufferContents.empty()) {
+        unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
+        if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
+          return NULL;
+        }
+        size_t len = strlen(buf8.get());
+        while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) {
+          --len;
+          buf8[len] = '\0';
+        }
+        return strdup(buf8.get());  // caller must free buffer
+      } else {
+        char* buf8 = strdup(preloadedBufferContents.c_str());
+        preloadedBufferContents.clear();
+        return buf8;  // caller must free buffer
+      }
+    } else {
+      if (enableRawMode() == -1) {
+        return NULL;
+      }
+      InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE);
+      if (!preloadedBufferContents.empty()) {
+        ib.preloadBuffer(preloadedBufferContents.c_str());
+        preloadedBufferContents.clear();
+      }
+      int count = ib.getInputLine(pi);
+      disableRawMode();
+      printf("\n");
+      if (count == -1) {
+        return NULL;
+      }
+      size_t bufferSize = sizeof(char32_t) * ib.length() + 1;
+      unique_ptr<char[]> buf8(new char[bufferSize]);
+      copyString32to8(buf8.get(), bufferSize, buf32);
+      return strdup(buf8.get());  // caller must free buffer
+    }
+  } else {  // input not from a terminal, we should work with piped input, i.e.
+            // redirected stdin
+    unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
+    if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
+      return NULL;
+    }
+
+    // if fgets() gave us the newline, remove it
+    int count = static_cast<int>(strlen(buf8.get()));
+    if (count > 0 && buf8[count - 1] == '\n') {
+      --count;
+      buf8[count] = '\0';
+    }
+    return strdup(buf8.get());  // caller must free buffer
+  }
+}
+
+/* Register a callback function to be called for tab-completion. */
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) {
+  completionCallback = fn;
+}
+
+void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) {
+  lc->completionStrings.push_back(Utf32String(str));
+}
+
+int linenoiseHistoryAdd(const char* line) {
+  if (historyMaxLen == 0) {
+    return 0;
+  }
+  if (history == NULL) {
+    history =
+        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen));
+    if (history == NULL) {
+      return 0;
+    }
+    memset(history, 0, (sizeof(char*) * historyMaxLen));
+  }
+  char8_t* linecopy = strdup8(line);
+  if (!linecopy) {
+    return 0;
+  }
+
+  // convert newlines in multi-line code to spaces before storing
+  char8_t* p = linecopy;
+  while (*p) {
+    if (*p == '\n') {
+      *p = ' ';
+    }
+    ++p;
+  }
+
+  // prevent duplicate history entries
+  if (historyLen > 0 && history[historyLen - 1] != nullptr &&
+      strcmp(reinterpret_cast<char const*>(history[historyLen - 1]),
+             reinterpret_cast<char const*>(linecopy)) == 0) {
+    free(linecopy);
+    return 0;
+  }
+
+  if (historyLen == historyMaxLen) {
+    free(history[0]);
+    memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1));
+    --historyLen;
+    if (--historyPreviousIndex < -1) {
+      historyPreviousIndex = -2;
+    }
+  }
+
+  history[historyLen] = linecopy;
+  ++historyLen;
+  return 1;
+}
+
+int linenoiseHistorySetMaxLen(int len) {
+  if (len < 1) {
+    return 0;
+  }
+  if (history) {
+    int tocopy = historyLen;
+    char8_t** newHistory =
+        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len));
+    if (newHistory == NULL) {
+      return 0;
+    }
+    if (len < tocopy) {
+      tocopy = len;
+    }
+    memcpy(newHistory, history + historyMaxLen - tocopy,
+           sizeof(char8_t*) * tocopy);
+    free(history);
+    history = newHistory;
+  }
+  historyMaxLen = len;
+  if (historyLen > historyMaxLen) {
+    historyLen = historyMaxLen;
+  }
+  return 1;
+}
+
+/* Fetch a line of the history by (zero-based) index.  If the requested
+ * line does not exist, NULL is returned.  The return value is a heap-allocated
+ * copy of the line, and the caller is responsible for de-allocating it. */
+char* linenoiseHistoryLine(int index) {
+  if (index < 0 || index >= historyLen) return NULL;
+
+  return strdup(reinterpret_cast<char const*>(history[index]));
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int linenoiseHistorySave(const char* filename) {
+#if _WIN32
+  FILE* fp = fopen(filename, "wt");
+#else
+  int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR);
+
+  if (fd < 0) {
+    return -1;
+  }
+
+  FILE* fp = fdopen(fd, "wt");
+#endif
+
+  if (fp == NULL) {
+    return -1;
+  }
+
+  for (int j = 0; j < historyLen; ++j) {
+    if (history[j][0] != '\0') {
+      fprintf(fp, "%s\n", history[j]);
+    }
+  }
+
+  fclose(fp);
+
+  return 0;
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int linenoiseHistoryLoad(const char* filename) {
+  FILE* fp = fopen(filename, "rt");
+  if (fp == NULL) {
+    return -1;
+  }
+
+  char buf[LINENOISE_MAX_LINE];
+  while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) {
+    char* p = strchr(buf, '\r');
+    if (!p) {
+      p = strchr(buf, '\n');
+    }
+    if (p) {
+      *p = '\0';
+    }
+    if (p != buf) {
+      linenoiseHistoryAdd(buf);
+    }
+  }
+  fclose(fp);
+  return 0;
+}
+
+/* Set if to use or not the multi line mode. */
+/* note that this is a stub only, as linenoise-ng always multi-line */
+void linenoiseSetMultiLine(int) {}
+
+/* This special mode is used by linenoise in order to print scan codes
+ * on screen for debugging / development purposes. It is implemented
+ * by the linenoise_example program using the --keycodes option. */
+void linenoisePrintKeyCodes(void) {
+  char quit[4];
+
+  printf(
+      "Linenoise key codes debugging mode.\n"
+      "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
+  if (enableRawMode() == -1) return;
+  memset(quit, ' ', 4);
+  while (1) {
+    char c;
+    int nread;
+
+#if _WIN32
+    nread = _read(STDIN_FILENO, &c, 1);
+#else
+    nread = read(STDIN_FILENO, &c, 1);
+#endif
+    if (nread <= 0) continue;
+    memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
+    quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
+    if (memcmp(quit, "quit", sizeof(quit)) == 0) break;
+
+    printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c,
+           (int)c);
+    printf("\r"); /* Go left edge manually, we are in raw mode. */
+    fflush(stdout);
+  }
+  disableRawMode();
+}
+
+#ifndef _WIN32
+static void WindowSizeChanged(int) {
+  // do nothing here but setting this flag
+  gotResize = true;
+}
+#endif
+
+int linenoiseInstallWindowChangeHandler(void) {
+#ifndef _WIN32
+  struct sigaction sa;
+  sigemptyset(&sa.sa_mask);
+  sa.sa_flags = 0;
+  sa.sa_handler = &WindowSizeChanged;
+
+  if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
+    return errno;
+  }
+#endif
+  return 0;
+}
+
+int linenoiseKeyType(void) {
+  return keyType;
+}
diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h
index ed20232c576..3a8eb9f7ee6 100644
--- a/src/linenoise/linenoise.h
+++ b/src/linenoise/linenoise.h
@@ -1,70 +1,70 @@
-/* linenoise.h -- VERSION 1.0
- *
- * Guerrilla line editing library against the idea that a line editing lib
- * needs to be 20,000 lines of C code.
+/* linenoise.h -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
  *
  * See linenoise.c for more information.
  *
- * ------------------------------------------------------------------------
- *
- * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
  *
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
+ * modification, are permitted provided that the following conditions are met:
  *
- *  *  Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *
- *  *  Redistributions in binary form must reproduce the above copyright
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in the
  *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __LINENOISE_H
 #define __LINENOISE_H
 
+#define LINENOISE_VERSION "1.0.0"
+#define LINENOISE_VERSION_MAJOR 1
+#define LINENOISE_VERSION_MINOR 1
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct linenoiseCompletions {
-  size_t len;
-  char **cvec;
-} linenoiseCompletions;
+typedef struct linenoiseCompletions linenoiseCompletions;
 
-typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
-typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold);
-typedef void(linenoiseFreeHintsCallback)(void *);
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
-void linenoiseSetHintsCallback(linenoiseHintsCallback *);
-void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
-void linenoiseAddCompletion(linenoiseCompletions *, const char *);
+typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
+void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn);
+void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str);
 
-char *linenoise(const char *prompt);
-void linenoiseFree(void *ptr);
-int linenoiseHistoryAdd(const char *line);
+char* linenoise(const char* prompt);
+void linenoisePreloadBuffer(const char* preloadText);
+int linenoiseHistoryAdd(const char* line);
 int linenoiseHistorySetMaxLen(int len);
-int linenoiseHistorySave(const char *filename);
-int linenoiseHistoryLoad(const char *filename);
+char* linenoiseHistoryLine(int index);
+int linenoiseHistorySave(const char* filename);
+int linenoiseHistoryLoad(const char* filename);
+void linenoiseHistoryFree(void);
 void linenoiseClearScreen(void);
 void linenoiseSetMultiLine(int ml);
 void linenoisePrintKeyCodes(void);
+/* the following are extensions to the original linenoise API */
+int linenoiseInstallWindowChangeHandler(void);
+/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ 
+int linenoiseKeyType(void);
 
 #ifdef __cplusplus
 }
diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp
new file mode 100644
index 00000000000..deec0ba6b57
--- /dev/null
+++ b/src/linenoise/wcwidth.cpp
@@ -0,0 +1,315 @@
+/*
+ * This is an implementation of wcwidth() and wcswidth() (defined in
+ * IEEE Std 1002.1-2001) for Unicode.
+ *
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+ * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+ *
+ * In fixed-width output devices, Latin characters all occupy a single
+ * "cell" position of equal width, whereas ideographic CJK characters
+ * occupy two such cells. Interoperability between terminal-line
+ * applications and (teletype-style) character terminals using the
+ * UTF-8 encoding requires agreement on which character should advance
+ * the cursor by how many cell positions. No established formal
+ * standards exist at present on which Unicode character shall occupy
+ * how many cell positions on character terminals. These routines are
+ * a first attempt of defining such behavior based on simple rules
+ * applied to data provided by the Unicode Consortium.
+ *
+ * For some graphical characters, the Unicode standard explicitly
+ * defines a character-cell width via the definition of the East Asian
+ * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+ * In all these cases, there is no ambiguity about which width a
+ * terminal shall use. For characters in the East Asian Ambiguous (A)
+ * class, the width choice depends purely on a preference of backward
+ * compatibility with either historic CJK or Western practice.
+ * Choosing single-width for these characters is easy to justify as
+ * the appropriate long-term solution, as the CJK practice of
+ * displaying these characters as double-width comes from historic
+ * implementation simplicity (8-bit encoded characters were displayed
+ * single-width and 16-bit ones double-width, even for Greek,
+ * Cyrillic, etc.) and not any typographic considerations.
+ *
+ * Much less clear is the choice of width for the Not East Asian
+ * (Neutral) class. Existing practice does not dictate a width for any
+ * of these characters. It would nevertheless make sense
+ * typographically to allocate two character cells to characters such
+ * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+ * represented adequately with a single-width glyph. The following
+ * routines at present merely assign a single-cell width to all
+ * neutral characters, in the interest of simplicity. This is not
+ * entirely satisfactory and should be reconsidered before
+ * establishing a formal standard in this area. At the moment, the
+ * decision which Not East Asian (Neutral) characters should be
+ * represented by double-width glyphs cannot yet be answered by
+ * applying a simple rule from the Unicode database content. Setting
+ * up a proper standard for the behavior of UTF-8 character terminals
+ * will require a careful analysis not only of each Unicode character,
+ * but also of each presentation form, something the author of these
+ * routines has avoided to do so far.
+ *
+ * http://www.unicode.org/unicode/reports/tr11/
+ *
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+ *
+ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#include <wchar.h>
+#include <string>
+#include <memory>
+
+namespace linenoise_ng {
+
+struct interval {
+  char32_t first;
+  char32_t last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(char32_t ucs, const struct interval *table, int max) {
+  int min = 0;
+  int mid;
+
+  if (ucs < table[0].first || ucs > table[max].last)
+    return 0;
+  while (max >= min) {
+    mid = (min + max) / 2;
+    if (ucs > table[mid].last)
+      min = mid + 1;
+    else if (ucs < table[mid].first)
+      max = mid - 1;
+    else
+      return 1;
+  }
+
+  return 0;
+}
+
+
+/* The following two functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ *    - The null character (U+0000) has a column width of 0.
+ *
+ *    - Other C0/C1 control characters and DEL will lead to a return
+ *      value of -1.
+ *
+ *    - Non-spacing and enclosing combining characters (general
+ *      category code Mn or Me in the Unicode database) have a
+ *      column width of 0.
+ *
+ *    - SOFT HYPHEN (U+00AD) has a column width of 1.
+ *
+ *    - Other format characters (general category code Cf in the Unicode
+ *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ *      have a column width of 0.
+ *
+ *    - Spacing characters in the East Asian Wide (W) or East Asian
+ *      Full-width (F) category as defined in Unicode Technical
+ *      Report #11 have a column width of 2.
+ *
+ *    - All remaining characters (including all printable
+ *      ISO 8859-1 and WGL4 characters, Unicode control characters,
+ *      etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+int mk_wcwidth(char32_t ucs)
+{
+  /* sorted list of non-overlapping intervals of non-spacing characters */
+  /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+  static const struct interval combining[] = {
+    { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
+    { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
+    { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
+    { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
+    { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
+    { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
+    { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
+    { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
+    { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
+    { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
+    { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
+    { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
+    { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
+    { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
+    { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
+    { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
+    { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
+    { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
+    { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
+    { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
+    { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
+    { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
+    { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
+    { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
+    { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
+    { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
+    { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
+    { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
+    { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
+    { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
+    { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
+    { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
+    { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
+    { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
+    { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
+    { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
+    { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
+    { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
+    { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
+    { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
+    { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
+    { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
+    { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
+    { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
+    { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
+    { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
+    { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
+    { 0xE0100, 0xE01EF }
+  };
+
+  /* test for 8-bit control characters */
+  if (ucs == 0)
+    return 0;
+  if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
+    return -1;
+
+  /* binary search in table of non-spacing characters */
+  if (bisearch(ucs, combining,
+               sizeof(combining) / sizeof(struct interval) - 1))
+    return 0;
+
+  /* if we arrive here, ucs is not a combining or C0/C1 control character */
+
+  return 1 + 
+    (ucs >= 0x1100 &&
+     (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
+      ucs == 0x2329 || ucs == 0x232a ||
+      (ucs >= 0x2e80 && ucs <= 0xa4cf &&
+       ucs != 0x303f) ||                  /* CJK ... Yi */
+      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
+      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
+      (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
+      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
+      (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
+      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
+      (ucs >= 0x20000 && ucs <= 0x2fffd) ||
+      (ucs >= 0x30000 && ucs <= 0x3fffd)));
+}
+
+
+int mk_wcswidth(const char32_t* pwcs, size_t n)
+{
+  int w, width = 0;
+
+  for (;*pwcs && n-- > 0; pwcs++)
+    if ((w = mk_wcwidth(*pwcs)) < 0)
+      return -1;
+    else
+      width += w;
+
+  return width;
+}
+
+
+/*
+ * The following functions are the same as mk_wcwidth() and
+ * mk_wcswidth(), except that spacing characters in the East Asian
+ * Ambiguous (A) category as defined in Unicode Technical Report #11
+ * have a column width of 2. This variant might be useful for users of
+ * CJK legacy encodings who want to migrate to UCS without changing
+ * the traditional terminal character-width behaviour. It is not
+ * otherwise recommended for general use.
+ */
+int mk_wcwidth_cjk(wchar_t ucs)
+{
+  /* sorted list of non-overlapping intervals of East Asian Ambiguous
+   * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
+  static const struct interval ambiguous[] = {
+    { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 },
+    { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 },
+    { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 },
+    { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 },
+    { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED },
+    { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA },
+    { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 },
+    { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B },
+    { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 },
+    { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 },
+    { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 },
+    { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE },
+    { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 },
+    { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA },
+    { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 },
+    { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB },
+    { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB },
+    { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 },
+    { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 },
+    { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 },
+    { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 },
+    { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 },
+    { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 },
+    { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 },
+    { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC },
+    { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 },
+    { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 },
+    { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 },
+    { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 },
+    { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 },
+    { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 },
+    { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B },
+    { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 },
+    { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 },
+    { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E },
+    { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 },
+    { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 },
+    { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F },
+    { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 },
+    { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF },
+    { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B },
+    { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 },
+    { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 },
+    { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 },
+    { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 },
+    { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 },
+    { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 },
+    { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 },
+    { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 },
+    { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F },
+    { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
+    { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
+  };
+
+  /* binary search in table of non-spacing characters */
+  if (bisearch(ucs, ambiguous,
+               sizeof(ambiguous) / sizeof(struct interval) - 1))
+    return 2;
+
+  return mk_wcwidth(ucs);
+}
+
+
+int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n)
+{
+  int w, width = 0;
+
+  for (;*pwcs && n-- > 0; pwcs++)
+    if ((w = mk_wcwidth_cjk(*pwcs)) < 0)
+      return -1;
+    else
+      width += w;
+
+  return width;
+}
+
+}
diff --git a/src/nix/local.mk b/src/nix/local.mk
index c7d2d328aab..bddd53b168d 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -2,7 +2,7 @@ programs += nix
 
 nix_DIR := $(d)
 
-nix_SOURCES := $(wildcard $(d)/*.cc) src/linenoise/linenoise.c
+nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 

From 964349e44d9859068eeb199c47336b06aae5450a Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 27 Nov 2017 18:30:05 -0600
Subject: [PATCH 0699/2196] no "linenoiseFree" in linenoise-ng

---
 src/nix/repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 28a8ebc8c49..1adb816c5bf 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -185,7 +185,7 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 bool NixRepl::getLine(string & input, const std::string &prompt)
 {
     char * s = linenoise(prompt.c_str());
-    Finally doFree([&]() { linenoiseFree(s); });
+    Finally doFree([&]() { free(s); });
     if (!s) return false;
     input += s;
     return true;

From e50d7335bf82f03c5a427892060819afcb32d086 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 27 Nov 2017 18:36:20 -0600
Subject: [PATCH 0700/2196] doc: linenoise -> linenoise-ng

---
 doc/manual/introduction/about-nix.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index 7cb124033f3..83a2b6786ac 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -262,7 +262,7 @@ xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
 <para>Nix uses the <link
-xlink:href="https://github.com/antirez/linenoise">linenoise
+xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng
 library</link>, which has the following license:</para>
 
 <programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting>

From b52846ab5b5d1317a6f7a5967e35b360a731b9c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Nov 2017 15:36:28 +0100
Subject: [PATCH 0701/2196] Show log tail when a remote build fails

---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 06168237725..866964a4c46 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1608,8 +1608,8 @@ void DerivationGoal::buildDone()
         outputLocks.unlock();
 
     } catch (BuildError & e) {
-        if (!hook)
-            printError(e.msg());
+        printError(e.msg());
+
         outputLocks.unlock();
 
         BuildResult::Status st = BuildResult::MiscFailure;

From f986a44980742ce74b6fd3321696f6b4937f696b Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Wed, 29 Nov 2017 11:36:28 +0000
Subject: [PATCH 0702/2196] Fix "Unexpected EOF reading a line" error

propagated-user-env-packages files in nixpkgs aren't all terminated by
newlines, as buildenv expected. Now it does not require a terminating
newline; note that this introduces a behaviour change: propagated user
env packages may now be spread across multiple lines. However, nix
1.11.x still expects them to be on a single line so this shouldn't be
used in nixpkgs for now.
---
 src/buildenv/buildenv.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc
index f05aa7bf2fb..eddb9fdaa8d 100644
--- a/src/buildenv/buildenv.cc
+++ b/src/buildenv/buildenv.cc
@@ -115,9 +115,9 @@ static void addPkg(const Path & pkgDir, int priority)
                 return;
             throw SysError(format("opening '%1%'") % propagatedFN);
         }
-        propagated = readLine(fd.get());
+        propagated = readFile(fd.get());
     }
-    for (const auto & p : tokenizeString<std::vector<string>>(propagated, " "))
+    for (const auto & p : tokenizeString<std::vector<string>>(propagated, " \n"))
         if (done.find(p) == done.end())
             postponed.insert(p);
 }

From 79f5c296c05bd8754d90272468c985f34b824b4b Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 30 Nov 2017 11:38:12 -0600
Subject: [PATCH 0703/2196] ignore "interrupted" exception in progress callback

Context/discusson:

https://github.com/NixOS/nix/issues/1692#issuecomment-348282301
---
 src/libstore/download.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index da31029b4f8..56560203a09 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -169,7 +169,11 @@ struct CurlDownloader : public Downloader
 
         int progressCallback(double dltotal, double dlnow)
         {
-            act.progress(dlnow, dltotal);
+            try {
+              act.progress(dlnow, dltotal);
+            } catch (nix::Interrupted &) {
+              assert(_isInterrupted);
+            }
             return _isInterrupted;
         }
 

From f1efb97075d301c05f5fe974b9234f2920e83c27 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Dec 2017 13:18:25 +0100
Subject: [PATCH 0704/2196] Fix #1635.

---
 src/nix/search.cc | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index a9dc2d6b924..87cdb2d7ed8 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -214,13 +214,28 @@ struct CmdSearch : SourceExprCommand, MixJSON
         }
 
         else {
+            createDirs(dirOf(jsonCacheFileName));
+
             Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
 
-            std::ofstream jsonCacheFile(tmpFile);
+            std::ofstream jsonCacheFile;
+
+            try {
+                // iostream considered harmful
+                jsonCacheFile.exceptions(std::ofstream::failbit);
+                jsonCacheFile.open(tmpFile);
 
-            auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
+                auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
 
-            doExpr(getSourceExpr(*state), "", true, cache.get());
+                doExpr(getSourceExpr(*state), "", true, cache.get());
+
+            } catch (std::exception &) {
+                /* Fun fact: catching std::ios::failure does not work
+                   due to C++11 ABI shenanigans.
+                   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */
+                if (!jsonCacheFile)
+                    throw Error("error writing to %s", tmpFile);
+            }
 
             if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);

From 7f2c324ed18cba4004ff89dfd84cf2df979b2571 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Dec 2017 17:07:26 +0100
Subject: [PATCH 0705/2196] Simplify build by including nlohmann/json.hpp

---
 local.mk              |     2 +-
 release.nix           |     2 -
 shell.nix             |     1 -
 src/nlohmann/json.hpp | 14723 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 14724 insertions(+), 4 deletions(-)
 create mode 100644 src/nlohmann/json.hpp

diff --git a/local.mk b/local.mk
index 5c80451bd57..0a225423741 100644
--- a/local.mk
+++ b/local.mk
@@ -3,7 +3,7 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
 endif
 
-dist-files += configure config.h.in nix.spec perl/configure src/nlohmann/json.hpp
+dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
diff --git a/release.nix b/release.nix
index c76966a1a44..c5c2170f78d 100644
--- a/release.nix
+++ b/release.nix
@@ -43,8 +43,6 @@ let
           (cd perl ; autoreconf --install --force --verbose)
           # TeX needs a writable font cache.
           export VARTEXFONTS=$TMPDIR/texfonts
-
-          cp -rv ${nlohmann_json}/include/nlohmann src/nlohmann
         '';
 
         distPhase =
diff --git a/shell.nix b/shell.nix
index f47952582fc..d6ae5a55226 100644
--- a/shell.nix
+++ b/shell.nix
@@ -18,7 +18,6 @@ with import ./release-common.nix { inherit pkgs; };
         customMemoryManagement = false;
       })
       autoreconfHook
-      nlohmann_json
 
       # For nix-perl
       perl
diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
new file mode 100644
index 00000000000..9754e464c76
--- /dev/null
+++ b/src/nlohmann/json.hpp
@@ -0,0 +1,14723 @@
+/*
+    __ _____ _____ _____
+ __|  |   __|     |   | |  JSON for Modern C++
+|  |  |__   |  |  | | | |  version 2.1.1
+|_____|_____|_____|_|___|  https://github.com/nlohmann/json
+
+Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>.
+
+Permission is hereby  granted, free of charge, to any  person obtaining a copy
+of this software and associated  documentation files (the "Software"), to deal
+in the Software  without restriction, including without  limitation the rights
+to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
+copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
+IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
+FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
+AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
+LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef NLOHMANN_JSON_HPP
+#define NLOHMANN_JSON_HPP
+
+#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform
+#include <array> // array
+#include <cassert> // assert
+#include <ciso646> // and, not, or
+#include <clocale> // lconv, localeconv
+#include <cmath> // isfinite, labs, ldexp, signbit
+#include <cstddef> // nullptr_t, ptrdiff_t, size_t
+#include <cstdint> // int64_t, uint64_t
+#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull
+#include <cstring> // memcpy, strlen
+#include <forward_list> // forward_list
+#include <functional> // function, hash, less
+#include <initializer_list> // initializer_list
+#include <iomanip> // hex
+#include <iosfwd>   // istream, ostream
+#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
+#include <limits> // numeric_limits
+#include <locale> // locale
+#include <map> // map
+#include <memory> // addressof, allocator, allocator_traits, unique_ptr
+#include <numeric> // accumulate
+#include <sstream> // stringstream
+#include <string> // getline, stoi, string, to_string
+#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
+#include <utility> // declval, forward, make_pair, move, pair, swap
+#include <valarray> // valarray
+#include <vector> // vector
+
+// exclude unsupported compilers
+#if defined(__clang__)
+    #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
+        #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
+    #endif
+#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
+    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
+        #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+    #endif
+#endif
+
+// disable float-equal warnings on GCC/clang
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+
+// disable documentation warnings on clang
+#if defined(__clang__)
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wdocumentation"
+#endif
+
+// allow for portable deprecation warnings
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #define JSON_DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+    #define JSON_DEPRECATED __declspec(deprecated)
+#else
+    #define JSON_DEPRECATED
+#endif
+
+// allow to disable exceptions
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION)
+    #define JSON_THROW(exception) throw exception
+    #define JSON_TRY try
+    #define JSON_CATCH(exception) catch(exception)
+#else
+    #define JSON_THROW(exception) std::abort()
+    #define JSON_TRY if(true)
+    #define JSON_CATCH(exception) if(false)
+#endif
+
+// manual branch prediction
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #define JSON_LIKELY(x)      __builtin_expect(!!(x), 1)
+    #define JSON_UNLIKELY(x)    __builtin_expect(!!(x), 0)
+#else
+    #define JSON_LIKELY(x)      x
+    #define JSON_UNLIKELY(x)    x
+#endif
+
+// cpp language standard detection
+#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
+    #define JSON_HAS_CPP_17
+    #define JSON_HAS_CPP_14
+#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1)
+    #define JSON_HAS_CPP_14
+#endif
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+template<typename = void, typename = void>
+struct adl_serializer;
+
+// forward declaration of basic_json (required to split the class)
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer>
+class basic_json;
+
+// Ugly macros to avoid uglier copy-paste when specializing basic_json
+// This is only temporary and will be removed in 3.0
+
+#define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \
+    template<template<typename, typename, typename...> class ObjectType,   \
+             template<typename, typename...> class ArrayType,              \
+             class StringType, class BooleanType, class NumberIntegerType, \
+             class NumberUnsignedType, class NumberFloatType,              \
+             template<typename> class AllocatorType,                       \
+             template<typename, typename = void> class JSONSerializer>
+
+#define NLOHMANN_BASIC_JSON_TPL                                            \
+    basic_json<ObjectType, ArrayType, StringType, BooleanType,             \
+    NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
+    AllocatorType, JSONSerializer>
+
+
+/*!
+@brief unnamed namespace with internal helper functions
+
+This namespace collects some functions that could not be defined inside the
+@ref basic_json class.
+
+@since version 2.1.0
+*/
+namespace detail
+{
+////////////////
+// exceptions //
+////////////////
+
+/*!
+@brief general exception of the @ref basic_json class
+
+This class is an extension of `std::exception` objects with a member @a id for
+exception ids. It is used as the base class for all exceptions thrown by the
+@ref basic_json class. This class can hence be used as "wildcard" to catch
+exceptions.
+
+Subclasses:
+- @ref parse_error for exceptions indicating a parse error
+- @ref invalid_iterator for exceptions indicating errors with iterators
+- @ref type_error for exceptions indicating executing a member function with
+                  a wrong type
+- @ref out_of_range for exceptions indicating access out of the defined range
+- @ref other_error for exceptions indicating other library errors
+
+@internal
+@note To have nothrow-copy-constructible exceptions, we internally use
+      `std::runtime_error` which can cope with arbitrary-length error messages.
+      Intermediate strings are built with static functions and then passed to
+      the actual constructor.
+@endinternal
+
+@liveexample{The following code shows how arbitrary library exceptions can be
+caught.,exception}
+
+@since version 3.0.0
+*/
+class exception : public std::exception
+{
+  public:
+    /// returns the explanatory string
+    const char* what() const noexcept override
+    {
+        return m.what();
+    }
+
+    /// the id of the exception
+    const int id;
+
+  protected:
+    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
+
+    static std::string name(const std::string& ename, int id_)
+    {
+        return "[json.exception." + ename + "." + std::to_string(id_) + "] ";
+    }
+
+  private:
+    /// an exception object as storage for error messages
+    std::runtime_error m;
+};
+
+/*!
+@brief exception indicating a parse error
+
+This excpetion is thrown by the library when a parse error occurs. Parse errors
+can occur during the deserialization of JSON text, CBOR, MessagePack, as well
+as when using JSON Patch.
+
+Member @a byte holds the byte index of the last read character in the input
+file.
+
+Exceptions have ids 1xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
+json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
+json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
+json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
+json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
+json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`.
+json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
+json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
+json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
+json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
+json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
+json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+
+@note For an input with n bytes, 1 is the index of the first character and n+1
+      is the index of the terminating null byte or the end of file. This also
+      holds true when reading a byte vector (CBOR or MessagePack).
+
+@liveexample{The following code shows how a `parse_error` exception can be
+caught.,parse_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class parse_error : public exception
+{
+  public:
+    /*!
+    @brief create a parse error exception
+    @param[in] id_       the id of the exception
+    @param[in] byte_     the byte index where the error occurred (or 0 if the
+                         position cannot be determined)
+    @param[in] what_arg  the explanatory string
+    @return parse_error object
+    */
+    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") +
+                        ": " + what_arg;
+        return parse_error(id_, byte_, w.c_str());
+    }
+
+    /*!
+    @brief byte index of the parse error
+
+    The byte index of the last read character in the input file.
+
+    @note For an input with n bytes, 1 is the index of the first character and
+          n+1 is the index of the terminating null byte or the end of file.
+          This also holds true when reading a byte vector (CBOR or MessagePack).
+    */
+    const std::size_t byte;
+
+  private:
+    parse_error(int id_, std::size_t byte_, const char* what_arg)
+        : exception(id_, what_arg), byte(byte_) {}
+};
+
+/*!
+@brief exception indicating errors with iterators
+
+This exception is thrown if iterators passed to a library function do not match
+the expected semantics.
+
+Exceptions have ids 2xx.
+
+name / id                           | example message | description
+----------------------------------- | --------------- | -------------------------
+json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
+json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
+json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
+json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
+json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
+json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
+json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
+json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
+json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
+json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
+
+@liveexample{The following code shows how an `invalid_iterator` exception can be
+caught.,invalid_iterator}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class invalid_iterator : public exception
+{
+  public:
+    static invalid_iterator create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("invalid_iterator", id_) + what_arg;
+        return invalid_iterator(id_, w.c_str());
+    }
+
+  private:
+    invalid_iterator(int id_, const char* what_arg)
+        : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating executing a member function with a wrong type
+
+This exception is thrown in case of a type error; that is, a library function is
+executed on a JSON value whose type does not match the expected semantics.
+
+Exceptions have ids 3xx.
+
+name / id                     | example message | description
+----------------------------- | --------------- | -------------------------
+json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
+json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
+json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
+json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
+json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
+json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
+json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
+json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
+json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
+json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
+json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
+json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
+json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
+json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
+json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
+
+@liveexample{The following code shows how a `type_error` exception can be
+caught.,type_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class type_error : public exception
+{
+  public:
+    static type_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("type_error", id_) + what_arg;
+        return type_error(id_, w.c_str());
+    }
+
+  private:
+    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating access out of the defined range
+
+This exception is thrown in case a library function is called on an input
+parameter that exceeds the expected range, for instance in case of array
+indices or nonexisting object keys.
+
+Exceptions have ids 4xx.
+
+name / id                       | example message | description
+------------------------------- | --------------- | -------------------------
+json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
+json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
+json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
+json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
+json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
+json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+
+@liveexample{The following code shows how an `out_of_range` exception can be
+caught.,out_of_range}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class out_of_range : public exception
+{
+  public:
+    static out_of_range create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("out_of_range", id_) + what_arg;
+        return out_of_range(id_, w.c_str());
+    }
+
+  private:
+    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating other library errors
+
+This exception is thrown in case of errors that cannot be classified with the
+other exception types.
+
+Exceptions have ids 5xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
+json.exception.other_error.502 | invalid object size for conversion | Some conversions to user-defined types impose constraints on the object size (e.g. std::pair)
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+
+@liveexample{The following code shows how an `other_error` exception can be
+caught.,other_error}
+
+@since version 3.0.0
+*/
+class other_error : public exception
+{
+  public:
+    static other_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("other_error", id_) + what_arg;
+        return other_error(id_, w.c_str());
+    }
+
+  private:
+    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+
+
+///////////////////////////
+// JSON type enumeration //
+///////////////////////////
+
+/*!
+@brief the JSON type enumeration
+
+This enumeration collects the different JSON types. It is internally used to
+distinguish the stored values, and the functions @ref basic_json::is_null(),
+@ref basic_json::is_object(), @ref basic_json::is_array(),
+@ref basic_json::is_string(), @ref basic_json::is_boolean(),
+@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
+@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
+@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
+@ref basic_json::is_structured() rely on it.
+
+@note There are three enumeration entries (number_integer, number_unsigned, and
+number_float), because the library distinguishes these three types for numbers:
+@ref basic_json::number_unsigned_t is used for unsigned integers,
+@ref basic_json::number_integer_t is used for signed integers, and
+@ref basic_json::number_float_t is used for floating-point numbers or to
+approximate integers which do not fit in the limits of their respective type.
+
+@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
+value with the default value for a given type
+
+@since version 1.0.0
+*/
+enum class value_t : uint8_t
+{
+    null,             ///< null value
+    object,           ///< object (unordered set of name/value pairs)
+    array,            ///< array (ordered collection of values)
+    string,           ///< string value
+    boolean,          ///< boolean value
+    number_integer,   ///< number value (signed integer)
+    number_unsigned,  ///< number value (unsigned integer)
+    number_float,     ///< number value (floating-point)
+    discarded         ///< discarded by the the parser callback function
+};
+
+/*!
+@brief comparison operator for JSON types
+
+Returns an ordering that is similar to Python:
+- order: null < boolean < number < object < array < string
+- furthermore, each type is not smaller than itself
+- discarded values are not comparable
+
+@since version 1.0.0
+*/
+inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+{
+    static constexpr std::array<uint8_t, 8> order = {{
+            0, // null
+            3, // object
+            4, // array
+            5, // string
+            1, // boolean
+            2, // integer
+            2, // unsigned
+            2, // float
+        }
+    };
+
+    const auto l_index = static_cast<std::size_t>(lhs);
+    const auto r_index = static_cast<std::size_t>(rhs);
+    return (l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]);
+}
+
+
+/////////////
+// helpers //
+/////////////
+
+template<typename> struct is_basic_json : std::false_type {};
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+// implementation of C++14 index_sequence and affiliates
+// source: https://stackoverflow.com/a/32223343
+template<std::size_t... Ints>
+struct index_sequence
+{
+    using type = index_sequence;
+    using value_type = std::size_t;
+    static constexpr std::size_t size() noexcept
+    {
+        return sizeof...(Ints);
+    }
+};
+
+template<class Sequence1, class Sequence2>
+struct merge_and_renumber;
+
+template<std::size_t... I1, std::size_t... I2>
+struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
+        : index_sequence < I1..., (sizeof...(I1) + I2)... >
+          {};
+
+template<std::size_t N>
+struct make_index_sequence
+    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
+      typename make_index_sequence < N - N / 2 >::type >
+{};
+
+template<> struct make_index_sequence<0> : index_sequence<> { };
+template<> struct make_index_sequence<1> : index_sequence<0> { };
+
+template<typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+/*
+Implementation of two C++17 constructs: conjunction, negation. This is needed
+to avoid evaluating all the traits in a condition
+
+For example: not std::is_same<void, T>::value and has_value_type<T>::value
+will not compile when T = void (on MSVC at least). Whereas
+conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will
+stop evaluating if negation<...>::value == false
+
+Please note that those constructs must be used with caution, since symbols can
+become very long quickly (which can slow down compilation and cause MSVC
+internal compiler errors). Only use it when you have to (see example ahead).
+*/
+template<class...> struct conjunction : std::true_type {};
+template<class B1> struct conjunction<B1> : B1 {};
+template<class B1, class... Bn>
+struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
+
+template<class B> struct negation : std::integral_constant < bool, !B::value > {};
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+
+//////////////////
+// constructors //
+//////////////////
+
+template<value_t> struct external_constructor;
+
+template<>
+struct external_constructor<value_t::boolean>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
+    {
+        j.m_type = value_t::boolean;
+        j.m_value = b;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::string>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = s;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = std::move(s);
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_float>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
+    {
+        j.m_type = value_t::number_float;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_unsigned>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
+    {
+        j.m_type = value_t::number_unsigned;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_integer>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
+    {
+        j.m_type = value_t::number_integer;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::array>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = arr;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = std::move(arr);
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleArrayType,
+             enable_if_t<not std::is_same<CompatibleArrayType,
+                                          typename BasicJsonType::array_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
+    {
+        using std::begin;
+        using std::end;
+        j.m_type = value_t::array;
+        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->reserve(arr.size());
+        for (bool x : arr)
+        {
+            j.m_value.array->push_back(x);
+        }
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename T,
+             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->resize(arr.size());
+        std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::object>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = obj;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = std::move(obj);
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleObjectType,
+             enable_if_t<not std::is_same<CompatibleObjectType,
+                                          typename BasicJsonType::object_t>::value, int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
+    {
+        using std::begin;
+        using std::end;
+
+        j.m_type = value_t::object;
+        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
+        j.assert_invariant();
+    }
+};
+
+
+////////////////////////
+// has_/is_ functions //
+////////////////////////
+
+/*!
+@brief Helper to determine whether there's a key_type for T.
+
+This helper is used to tell associative containers apart from other containers
+such as sequence containers. For instance, `std::map` passes the test as it
+contains a `mapped_type`, whereas `std::vector` fails the test.
+
+@sa http://stackoverflow.com/a/7728728/266378
+@since version 1.0.0, overworked in version 2.0.6
+*/
+#define NLOHMANN_JSON_HAS_HELPER(type)                                        \
+    template<typename T> struct has_##type {                                  \
+    private:                                                                  \
+        template<typename U, typename = typename U::type>                     \
+        static int detect(U &&);                                              \
+        static void detect(...);                                              \
+    public:                                                                   \
+        static constexpr bool value =                                         \
+                std::is_integral<decltype(detect(std::declval<T>()))>::value; \
+    }
+
+NLOHMANN_JSON_HAS_HELPER(mapped_type);
+NLOHMANN_JSON_HAS_HELPER(key_type);
+NLOHMANN_JSON_HAS_HELPER(value_type);
+NLOHMANN_JSON_HAS_HELPER(iterator);
+
+#undef NLOHMANN_JSON_HAS_HELPER
+
+
+template<bool B, class RealType, class CompatibleObjectType>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template<class RealType, class CompatibleObjectType>
+struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType>
+{
+    static constexpr auto value =
+        std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value;
+};
+
+template<class BasicJsonType, class CompatibleObjectType>
+struct is_compatible_object_type
+{
+    static auto constexpr value = is_compatible_object_type_impl <
+                                  conjunction<negation<std::is_same<void, CompatibleObjectType>>,
+                                  has_mapped_type<CompatibleObjectType>,
+                                  has_key_type<CompatibleObjectType>>::value,
+                                  typename BasicJsonType::object_t, CompatibleObjectType >::value;
+};
+
+template<typename BasicJsonType, typename T>
+struct is_basic_json_nested_type
+{
+    static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::const_iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::reverse_iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value;
+};
+
+template<class BasicJsonType, class CompatibleArrayType>
+struct is_compatible_array_type
+{
+    static auto constexpr value =
+        conjunction<negation<std::is_same<void, CompatibleArrayType>>,
+        negation<is_compatible_object_type<
+        BasicJsonType, CompatibleArrayType>>,
+        negation<std::is_constructible<typename BasicJsonType::string_t,
+        CompatibleArrayType>>,
+        negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>,
+        has_value_type<CompatibleArrayType>,
+        has_iterator<CompatibleArrayType>>::value;
+};
+
+template<bool, typename, typename>
+struct is_compatible_integer_type_impl : std::false_type {};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType>
+{
+    // is there an assert somewhere on overflows?
+    using RealLimits = std::numeric_limits<RealIntegerType>;
+    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+
+    static constexpr auto value =
+        std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and
+        CompatibleLimits::is_integer and
+        RealLimits::is_signed == CompatibleLimits::is_signed;
+};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+{
+    static constexpr auto value =
+        is_compatible_integer_type_impl <
+        std::is_integral<CompatibleNumberIntegerType>::value and
+        not std::is_same<bool, CompatibleNumberIntegerType>::value,
+        RealIntegerType, CompatibleNumberIntegerType > ::value;
+};
+
+
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template<typename BasicJsonType, typename T>
+struct has_from_json
+{
+  private:
+    // also check the return type of from_json
+    template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json(
+                 std::declval<BasicJsonType>(), std::declval<T&>()))>::value>>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(
+                                      detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json
+{
+  private:
+    template <
+        typename U,
+        typename = enable_if_t<std::is_same<
+                                   T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value >>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(detect(
+                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+template<typename BasicJsonType, typename T>
+struct has_to_json
+{
+  private:
+    template<typename U, typename = decltype(uncvref_t<U>::to_json(
+                 std::declval<BasicJsonType&>(), std::declval<T>()))>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(detect(
+                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+
+/////////////
+// to_json //
+/////////////
+
+template<typename BasicJsonType, typename T, enable_if_t<
+             std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+void to_json(BasicJsonType& j, T b) noexcept
+{
+    external_constructor<value_t::boolean>::construct(j, b);
+}
+
+template<typename BasicJsonType, typename CompatibleString,
+         enable_if_t<std::is_constructible<typename BasicJsonType::string_t,
+                     CompatibleString>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleString& s)
+{
+    external_constructor<value_t::string>::construct(j, s);
+}
+
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+{
+    external_constructor<value_t::string>::construct(j, std::move(s));
+}
+
+template<typename BasicJsonType, typename FloatType,
+         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
+void to_json(BasicJsonType& j, FloatType val) noexcept
+{
+    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
+}
+
+template <
+    typename BasicJsonType, typename CompatibleNumberUnsignedType,
+    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t,
+                CompatibleNumberUnsignedType>::value, int> = 0 >
+void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+{
+    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
+}
+
+template <
+    typename BasicJsonType, typename CompatibleNumberIntegerType,
+    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t,
+                CompatibleNumberIntegerType>::value, int> = 0 >
+void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
+{
+    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
+}
+
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void to_json(BasicJsonType& j, EnumType e) noexcept
+{
+    using underlying_type = typename std::underlying_type<EnumType>::type;
+    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, const std::vector<bool>& e)
+{
+    external_constructor<value_t::array>::construct(j, e);
+}
+
+template <
+    typename BasicJsonType, typename CompatibleArrayType,
+    enable_if_t <
+        is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
+        std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
+        int > = 0 >
+void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template <typename BasicJsonType, typename T,
+          enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+void to_json(BasicJsonType& j, std::valarray<T> arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template <
+    typename BasicJsonType, typename CompatibleObjectType,
+    enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value,
+                int> = 0 >
+void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
+{
+    external_constructor<value_t::object>::construct(j, obj);
+}
+
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+{
+    external_constructor<value_t::object>::construct(j, std::move(obj));
+}
+
+template<typename BasicJsonType, typename T, std::size_t N,
+         enable_if_t<not std::is_constructible<
+                         typename BasicJsonType::string_t, T (&)[N]>::value,
+                     int> = 0>
+void to_json(BasicJsonType& j, T (&arr)[N])
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::pair<Args...>& p)
+{
+    j = {p.first, p.second};
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>)
+{
+    j = {std::get<Idx>(t)...};
+}
+
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
+{
+    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
+///////////////
+// from_json //
+///////////////
+
+// overloads for basic_json template parameters
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
+                     not std::is_same<ArithmeticType,
+                                      typename BasicJsonType::boolean_t>::value,
+                     int> = 0>
+void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
+{
+    if (JSON_UNLIKELY(not j.is_boolean()))
+    {
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
+    }
+    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void from_json(const BasicJsonType& j, EnumType& e)
+{
+    typename std::underlying_type<EnumType>::type val;
+    get_arithmetic_value(j, val);
+    e = static_cast<EnumType>(val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
+
+// forward_list doesn't have an insert method
+template<typename BasicJsonType, typename T, typename Allocator,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    std::transform(j.rbegin(), j.rend(),
+                   std::front_inserter(l), [](const BasicJsonType & i)
+    {
+        return i.template get<T>();
+    });
+}
+
+// valarray doesn't have an insert method
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::valarray<T>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    l.resize(j.size());
+    std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/)
+{
+    using std::end;
+
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename CompatibleArrayType::value_type>();
+    });
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/)
+-> decltype(
+    arr.reserve(std::declval<typename CompatibleArrayType::size_type>()),
+    void())
+{
+    using std::end;
+
+    arr.reserve(j.size());
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename CompatibleArrayType::value_type>();
+    });
+}
+
+template<typename BasicJsonType, typename T, std::size_t N>
+void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/)
+{
+    for (std::size_t i = 0; i < N; ++i)
+    {
+        arr[i] = j.at(i).template get<T>();
+    }
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType,
+         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and
+                     std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and
+                     not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
+void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+
+    from_json_array_impl(j, arr, priority_tag<2> {});
+}
+
+template<typename BasicJsonType, typename CompatibleObjectType,
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
+void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
+{
+    if (JSON_UNLIKELY(not j.is_object()))
+    {
+        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
+    }
+
+    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    using value_type = typename CompatibleObjectType::value_type;
+    std::transform(
+        inner_object->begin(), inner_object->end(),
+        std::inserter(obj, obj.begin()),
+        [](typename BasicJsonType::object_t::value_type const & p)
+    {
+        return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>());
+    });
+}
+
+// overload for arithmetic types, not chosen for basic_json template arguments
+// (BooleanType, etc..); note: Is it really necessary to provide explicit
+// overloads for boolean_t etc. in case of a custom BooleanType which is not
+// an arithmetic type?
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t <
+             std::is_arithmetic<ArithmeticType>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+             int> = 0>
+void from_json(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+        case value_t::boolean:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
+            break;
+        }
+
+        default:
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
+    }
+}
+
+template<typename BasicJsonType, typename A1, typename A2>
+void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
+{
+    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
+}
+
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>)
+{
+    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
+}
+
+template<typename BasicJsonType, typename... Args>
+void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
+{
+    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
+struct to_json_fn
+{
+  private:
+    template<typename BasicJsonType, typename T>
+    auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    -> decltype(to_json(j, std::forward<T>(val)), void())
+    {
+        return to_json(j, std::forward<T>(val));
+    }
+
+    template<typename BasicJsonType, typename T>
+    void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept
+    {
+        static_assert(sizeof(BasicJsonType) == 0,
+                      "could not find to_json() method in T's namespace");
+    }
+
+  public:
+    template<typename BasicJsonType, typename T>
+    void operator()(BasicJsonType& j, T&& val) const
+    noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {})))
+    {
+        return call(j, std::forward<T>(val), priority_tag<1> {});
+    }
+};
+
+struct from_json_fn
+{
+  private:
+    template<typename BasicJsonType, typename T>
+    auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const
+    noexcept(noexcept(from_json(j, val)))
+    -> decltype(from_json(j, val), void())
+    {
+        return from_json(j, val);
+    }
+
+    template<typename BasicJsonType, typename T>
+    void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept
+    {
+        static_assert(sizeof(BasicJsonType) == 0,
+                      "could not find from_json() method in T's namespace");
+    }
+
+  public:
+    template<typename BasicJsonType, typename T>
+    void operator()(const BasicJsonType& j, T& val) const
+    noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {})))
+    {
+        return call(j, val, priority_tag<1> {});
+    }
+};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+    static constexpr T value{};
+};
+
+template<typename T>
+constexpr T static_const<T>::value;
+
+////////////////////
+// input adapters //
+////////////////////
+
+/*!
+@brief abstract input adapter interface
+
+Produces a stream of std::char_traits<char>::int_type characters from a
+std::istream, a buffer, or some other input type.  Accepts the return of exactly
+one non-EOF character for future input.  The int_type characters returned
+consist of all valid char values as positive values (typically unsigned char),
+plus an EOF value outside that range, specified by the value of the function
+std::char_traits<char>::eof().  This value is typically -1, but could be any
+arbitrary value which is not a valid char value.
+*/
+struct input_adapter_protocol
+{
+    /// get a character [0,255] or std::char_traits<char>::eof().
+    virtual std::char_traits<char>::int_type get_character() = 0;
+    /// restore the last non-eof() character to input
+    virtual void unget_character() = 0;
+    virtual ~input_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
+
+/*!
+Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
+beginning of input. Does not support changing the underlying std::streambuf
+in mid-input. Maintains underlying std::istream and std::streambuf to support
+subsequent use of standard std::istream operations to process any input
+characters following those used in parsing the JSON input.  Clears the
+std::istream flags; any input errors (e.g., EOF) will be detected by the first
+subsequent call for input from the std::istream.
+*/
+class input_stream_adapter : public input_adapter_protocol
+{
+  public:
+    ~input_stream_adapter() override
+    {
+        // clear stream flags; we use underlying streambuf I/O, do not
+        // maintain ifstream flags
+        is.clear();
+    }
+
+    explicit input_stream_adapter(std::istream& i)
+        : is(i), sb(*i.rdbuf())
+    {
+        // ignore Byte Order Mark at start of input
+        std::char_traits<char>::int_type c;
+        if ((c = get_character()) == 0xEF)
+        {
+            if ((c = get_character()) == 0xBB)
+            {
+                if ((c = get_character()) == 0xBF)
+                {
+                    return; // Ignore BOM
+                }
+                else if (c != std::char_traits<char>::eof())
+                {
+                    is.unget();
+                }
+                is.putback('\xBB');
+            }
+            else if (c != std::char_traits<char>::eof())
+            {
+                is.unget();
+            }
+            is.putback('\xEF');
+        }
+        else if (c != std::char_traits<char>::eof())
+        {
+            is.unget(); // Not BOM. Process as usual.
+        }
+    }
+
+    // delete because of pointer members
+    input_stream_adapter(const input_stream_adapter&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&) = delete;
+
+    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
+    // ensure that std::char_traits<char>::eof() and the character 0xff do not
+    // end up as the same value, eg. 0xffffffff.
+    std::char_traits<char>::int_type get_character() override
+    {
+        return sb.sbumpc();
+    }
+
+    void unget_character() override
+    {
+        sb.sungetc();  // is.unget() avoided for performance
+    }
+
+  private:
+    /// the associated input stream
+    std::istream& is;
+    std::streambuf& sb;
+};
+
+/// input adapter for buffer input
+class input_buffer_adapter : public input_adapter_protocol
+{
+  public:
+    input_buffer_adapter(const char* b, const std::size_t l)
+        : cursor(b), limit(b + l), start(b)
+    {
+        // skip byte order mark
+        if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
+        {
+            cursor += 3;
+        }
+    }
+
+    // delete because of pointer members
+    input_buffer_adapter(const input_buffer_adapter&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        if (JSON_LIKELY(cursor < limit))
+        {
+            return std::char_traits<char>::to_int_type(*(cursor++));
+        }
+
+        return std::char_traits<char>::eof();
+    }
+
+    void unget_character() noexcept override
+    {
+        if (JSON_LIKELY(cursor > start))
+        {
+            --cursor;
+        }
+    }
+
+  private:
+    /// pointer to the current character
+    const char* cursor;
+    /// pointer past the last character
+    const char* limit;
+    /// pointer to the first character
+    const char* start;
+};
+
+class input_adapter
+{
+  public:
+    // native support
+
+    /// input adapter for input stream
+    input_adapter(std::istream& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    /// input adapter for input stream
+    input_adapter(std::istream&& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    /// input adapter for buffer
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<
+                     typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b, std::size_t l)
+        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
+
+    // derived support
+
+    /// input adapter for string literal
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<
+                     typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b)
+        : input_adapter(reinterpret_cast<const char*>(b),
+                        std::strlen(reinterpret_cast<const char*>(b))) {}
+
+    /// input adapter for iterator range with contiguous storage
+    template<class IteratorType,
+             typename std::enable_if<
+                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category,
+                              std::random_access_iterator_tag>::value,
+                 int>::type = 0>
+    input_adapter(IteratorType first, IteratorType last)
+    {
+        // assertion to check that the iterator range is indeed contiguous,
+        // see http://stackoverflow.com/a/35008842/266378 for more discussion
+        assert(std::accumulate(
+                   first, last, std::pair<bool, int>(true, 0),
+                   [&first](std::pair<bool, int> res, decltype(*first) val)
+        {
+            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
+            return res;
+        }).first);
+
+        // assertion to check that each element is 1 byte long
+        static_assert(
+            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+            "each element in the iterator range must have the size of 1 byte");
+
+        const auto len = static_cast<size_t>(std::distance(first, last));
+        if (JSON_LIKELY(len > 0))
+        {
+            // there is at least one element: use the address of first
+            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
+        }
+        else
+        {
+            // the address of first cannot be used: use nullptr
+            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
+        }
+    }
+
+    /// input adapter for array
+    template<class T, std::size_t N>
+    input_adapter(T (&array)[N])
+        : input_adapter(std::begin(array), std::end(array)) {}
+
+    /// input adapter for contiguous container
+    template <
+        class ContiguousContainer,
+        typename std::enable_if <
+            not std::is_pointer<ContiguousContainer>::value and
+            std::is_base_of<std::random_access_iterator_tag,
+                            typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+            int >::type = 0 >
+    input_adapter(const ContiguousContainer& c)
+        : input_adapter(std::begin(c), std::end(c)) {}
+
+    operator input_adapter_t()
+    {
+        return ia;
+    }
+
+  private:
+    /// the actual adapter
+    input_adapter_t ia = nullptr;
+};
+
+//////////////////////
+// lexer and parser //
+//////////////////////
+
+/*!
+@brief lexical analysis
+
+This class organizes the lexical analysis during JSON deserialization.
+*/
+template<typename BasicJsonType>
+class lexer
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+
+  public:
+    /// token types for the parser
+    enum class token_type
+    {
+        uninitialized,    ///< indicating the scanner is uninitialized
+        literal_true,     ///< the `true` literal
+        literal_false,    ///< the `false` literal
+        literal_null,     ///< the `null` literal
+        value_string,     ///< a string -- use get_string() for actual value
+        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
+        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
+        value_float,      ///< an floating point number -- use get_number_float() for actual value
+        begin_array,      ///< the character for array begin `[`
+        begin_object,     ///< the character for object begin `{`
+        end_array,        ///< the character for array end `]`
+        end_object,       ///< the character for object end `}`
+        name_separator,   ///< the name separator `:`
+        value_separator,  ///< the value separator `,`
+        parse_error,      ///< indicating a parse error
+        end_of_input,     ///< indicating the end of the input buffer
+        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
+    };
+
+    /// return name of values of type token_type (only used for errors)
+    static const char* token_type_name(const token_type t) noexcept
+    {
+        switch (t)
+        {
+            case token_type::uninitialized:
+                return "<uninitialized>";
+            case token_type::literal_true:
+                return "true literal";
+            case token_type::literal_false:
+                return "false literal";
+            case token_type::literal_null:
+                return "null literal";
+            case token_type::value_string:
+                return "string literal";
+            case lexer::token_type::value_unsigned:
+            case lexer::token_type::value_integer:
+            case lexer::token_type::value_float:
+                return "number literal";
+            case token_type::begin_array:
+                return "'['";
+            case token_type::begin_object:
+                return "'{'";
+            case token_type::end_array:
+                return "']'";
+            case token_type::end_object:
+                return "'}'";
+            case token_type::name_separator:
+                return "':'";
+            case token_type::value_separator:
+                return "','";
+            case token_type::parse_error:
+                return "<parse error>";
+            case token_type::end_of_input:
+                return "end of input";
+            case token_type::literal_or_value:
+                return "'[', '{', or a literal";
+            default: // catch non-enum values
+                return "unknown token"; // LCOV_EXCL_LINE
+        }
+    }
+
+    explicit lexer(detail::input_adapter_t adapter)
+        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
+
+    // delete because of pointer members
+    lexer(const lexer&) = delete;
+    lexer& operator=(lexer&) = delete;
+
+  private:
+    /////////////////////
+    // locales
+    /////////////////////
+
+    /// return the locale-dependent decimal point
+    static char get_decimal_point() noexcept
+    {
+        const auto loc = localeconv();
+        assert(loc != nullptr);
+        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+    }
+
+    /////////////////////
+    // scan functions
+    /////////////////////
+
+    /*!
+    @brief get codepoint from 4 hex characters following `\u`
+
+    For input "\u c1 c2 c3 c4" the codepoint is:
+      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
+    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+
+    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
+    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
+    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
+    between the ASCII value of the character and the desired integer value.
+
+    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
+            non-hex character)
+    */
+    int get_codepoint()
+    {
+        // this function only makes sense after reading `\u`
+        assert(current == 'u');
+        int codepoint = 0;
+
+        const auto factors = { 12, 8, 4, 0 };
+        for (const auto factor : factors)
+        {
+            get();
+
+            if (current >= '0' and current <= '9')
+            {
+                codepoint += ((current - 0x30) << factor);
+            }
+            else if (current >= 'A' and current <= 'F')
+            {
+                codepoint += ((current - 0x37) << factor);
+            }
+            else if (current >= 'a' and current <= 'f')
+            {
+                codepoint += ((current - 0x57) << factor);
+            }
+            else
+            {
+                return -1;
+            }
+        }
+
+        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
+        return codepoint;
+    }
+
+    /*!
+    @brief check if the next byte(s) are inside a given range
+
+    Adds the current byte and, for each passed range, reads a new byte and
+    checks if it is inside the range. If a violation was detected, set up an
+    error message and return false. Otherwise, return true.
+
+    @return true if and only if no range violation was detected
+    */
+    bool next_byte_in_range(std::initializer_list<int> ranges)
+    {
+        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
+        add(current);
+
+        for (auto range = ranges.begin(); range != ranges.end(); ++range)
+        {
+            get();
+            if (JSON_LIKELY(*range <= current and current <= *(++range)))
+            {
+                add(current);
+            }
+            else
+            {
+                error_message = "invalid string: ill-formed UTF-8 byte";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief scan a string literal
+
+    This function scans a string according to Sect. 7 of RFC 7159. While
+    scanning, bytes are escaped and copied into buffer yytext. Then the function
+    returns successfully, yytext is *not* null-terminated (as it may contain \0
+    bytes), and yytext.size() is the number of bytes in the string.
+
+    @return token_type::value_string if string could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note In case of errors, variable error_message contains a textual
+          description.
+    */
+    token_type scan_string()
+    {
+        // reset yytext (ignore opening quote)
+        reset();
+
+        // we entered the function by reading an open quote
+        assert(current == '\"');
+
+        while (true)
+        {
+            // get next character
+            switch (get())
+            {
+                // end of file while parsing string
+                case std::char_traits<char>::eof():
+                {
+                    error_message = "invalid string: missing closing quote";
+                    return token_type::parse_error;
+                }
+
+                // closing quote
+                case '\"':
+                {
+                    return token_type::value_string;
+                }
+
+                // escapes
+                case '\\':
+                {
+                    switch (get())
+                    {
+                        // quotation mark
+                        case '\"':
+                            add('\"');
+                            break;
+                        // reverse solidus
+                        case '\\':
+                            add('\\');
+                            break;
+                        // solidus
+                        case '/':
+                            add('/');
+                            break;
+                        // backspace
+                        case 'b':
+                            add('\b');
+                            break;
+                        // form feed
+                        case 'f':
+                            add('\f');
+                            break;
+                        // line feed
+                        case 'n':
+                            add('\n');
+                            break;
+                        // carriage return
+                        case 'r':
+                            add('\r');
+                            break;
+                        // tab
+                        case 't':
+                            add('\t');
+                            break;
+
+                        // unicode escapes
+                        case 'u':
+                        {
+                            const int codepoint1 = get_codepoint();
+                            int codepoint = codepoint1; // start with codepoint1
+
+                            if (JSON_UNLIKELY(codepoint1 == -1))
+                            {
+                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                return token_type::parse_error;
+                            }
+
+                            // check if code point is a high surrogate
+                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
+                            {
+                                // expect next \uxxxx entry
+                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
+                                {
+                                    const int codepoint2 = get_codepoint();
+
+                                    if (JSON_UNLIKELY(codepoint2 == -1))
+                                    {
+                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                        return token_type::parse_error;
+                                    }
+
+                                    // check if codepoint2 is a low surrogate
+                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
+                                    {
+                                        // overwrite codepoint
+                                        codepoint =
+                                            // high surrogate occupies the most significant 22 bits
+                                            (codepoint1 << 10)
+                                            // low surrogate occupies the least significant 15 bits
+                                            + codepoint2
+                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
+                                            // in the result so we have to subtract with:
+                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+                                            - 0x35FDC00;
+                                    }
+                                    else
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                        return token_type::parse_error;
+                                    }
+                                }
+                                else
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+                            else
+                            {
+                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+
+                            // result of the above calculation yields a proper codepoint
+                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+                            // translate code point to bytes
+                            if (codepoint < 0x80)
+                            {
+                                // 1-byte characters: 0xxxxxxx (ASCII)
+                                add(codepoint);
+                            }
+                            else if (codepoint <= 0x7ff)
+                            {
+                                // 2-byte characters: 110xxxxx 10xxxxxx
+                                add(0xC0 | (codepoint >> 6));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else if (codepoint <= 0xffff)
+                            {
+                                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
+                                add(0xE0 | (codepoint >> 12));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else
+                            {
+                                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                                add(0xF0 | (codepoint >> 18));
+                                add(0x80 | ((codepoint >> 12) & 0x3F));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+
+                            break;
+                        }
+
+                        // other characters after escape
+                        default:
+                            error_message = "invalid string: forbidden character after backslash";
+                            return token_type::parse_error;
+                    }
+
+                    break;
+                }
+
+                // invalid control characters
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x08:
+                case 0x09:
+                case 0x0a:
+                case 0x0b:
+                case 0x0c:
+                case 0x0d:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                case 0x18:
+                case 0x19:
+                case 0x1a:
+                case 0x1b:
+                case 0x1c:
+                case 0x1d:
+                case 0x1e:
+                case 0x1f:
+                {
+                    error_message = "invalid string: control character must be escaped";
+                    return token_type::parse_error;
+                }
+
+                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+                case 0x20:
+                case 0x21:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2a:
+                case 0x2b:
+                case 0x2c:
+                case 0x2d:
+                case 0x2e:
+                case 0x2f:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                case 0x38:
+                case 0x39:
+                case 0x3a:
+                case 0x3b:
+                case 0x3c:
+                case 0x3d:
+                case 0x3e:
+                case 0x3f:
+                case 0x40:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x44:
+                case 0x45:
+                case 0x46:
+                case 0x47:
+                case 0x48:
+                case 0x49:
+                case 0x4a:
+                case 0x4b:
+                case 0x4c:
+                case 0x4d:
+                case 0x4e:
+                case 0x4f:
+                case 0x50:
+                case 0x51:
+                case 0x52:
+                case 0x53:
+                case 0x54:
+                case 0x55:
+                case 0x56:
+                case 0x57:
+                case 0x58:
+                case 0x59:
+                case 0x5a:
+                case 0x5b:
+                case 0x5d:
+                case 0x5e:
+                case 0x5f:
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6a:
+                case 0x6b:
+                case 0x6c:
+                case 0x6d:
+                case 0x6e:
+                case 0x6f:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78:
+                case 0x79:
+                case 0x7a:
+                case 0x7b:
+                case 0x7c:
+                case 0x7d:
+                case 0x7e:
+                case 0x7f:
+                {
+                    add(current);
+                    break;
+                }
+
+                // U+0080..U+07FF: bytes C2..DF 80..BF
+                case 0xc2:
+                case 0xc3:
+                case 0xc4:
+                case 0xc5:
+                case 0xc6:
+                case 0xc7:
+                case 0xc8:
+                case 0xc9:
+                case 0xca:
+                case 0xcb:
+                case 0xcc:
+                case 0xcd:
+                case 0xce:
+                case 0xcf:
+                case 0xd0:
+                case 0xd1:
+                case 0xd2:
+                case 0xd3:
+                case 0xd4:
+                case 0xd5:
+                case 0xd6:
+                case 0xd7:
+                case 0xd8:
+                case 0xd9:
+                case 0xda:
+                case 0xdb:
+                case 0xdc:
+                case 0xdd:
+                case 0xde:
+                case 0xdf:
+                {
+                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+                case 0xe0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+                case 0xe1:
+                case 0xe2:
+                case 0xe3:
+                case 0xe4:
+                case 0xe5:
+                case 0xe6:
+                case 0xe7:
+                case 0xe8:
+                case 0xe9:
+                case 0xea:
+                case 0xeb:
+                case 0xec:
+                case 0xee:
+                case 0xef:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+                case 0xed:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+                case 0xf0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+                case 0xf1:
+                case 0xf2:
+                case 0xf3:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+                case 0xf4:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // remaining bytes (80..C1 and F5..FF) are ill-formed
+                default:
+                {
+                    error_message = "invalid string: ill-formed UTF-8 byte";
+                    return token_type::parse_error;
+                }
+            }
+        }
+    }
+
+    static void strtof(float& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtof(str, endptr);
+    }
+
+    static void strtof(double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtod(str, endptr);
+    }
+
+    static void strtof(long double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtold(str, endptr);
+    }
+
+    /*!
+    @brief scan a number literal
+
+    This function scans a string according to Sect. 6 of RFC 7159.
+
+    The function is realized with a deterministic finite state machine derived
+    from the grammar described in RFC 7159. Starting in state "init", the
+    input is read and used to determined the next state. Only state "done"
+    accepts the number. State "error" is a trap state to model errors. In the
+    table below, "anything" means any character but the ones listed before.
+
+    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
+    ---------|----------|----------|----------|---------|---------|----------|-----------
+    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
+    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
+    zero     | done     | done     | exponent | done    | done    | decimal1 | done
+    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
+    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
+    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
+    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
+    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
+    any2     | any2     | any2     | done     | done    | done    | done     | done
+
+    The state machine is realized with one label per state (prefixed with
+    "scan_number_") and `goto` statements between them. The state machine
+    contains cycles, but any cycle can be left when EOF is read. Therefore,
+    the function is guaranteed to terminate.
+
+    During scanning, the read bytes are stored in yytext. This string is
+    then converted to a signed integer, an unsigned integer, or a
+    floating-point number.
+
+    @return token_type::value_unsigned, token_type::value_integer, or
+            token_type::value_float if number could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note The scanner is independent of the current locale. Internally, the
+          locale's decimal point is used instead of `.` to work with the
+          locale-dependent converters.
+    */
+    token_type scan_number()
+    {
+        // reset yytext to store the number's bytes
+        reset();
+
+        // the type of the parsed number; initially set to unsigned; will be
+        // changed if minus sign, decimal point or exponent is read
+        token_type number_type = token_type::value_unsigned;
+
+        // state (init): we just found out we need to scan a number
+        switch (current)
+        {
+            case '-':
+            {
+                add(current);
+                goto scan_number_minus;
+            }
+
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                // all other characters are rejected outside scan_number()
+                assert(false); // LCOV_EXCL_LINE
+            }
+        }
+
+scan_number_minus:
+        // state: we just parsed a leading minus sign
+        number_type = token_type::value_integer;
+        switch (get())
+        {
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '-'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_zero:
+        // state: we just parse a zero (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_any1:
+        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_decimal1:
+        // state: we just parsed a decimal point
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '.'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_decimal2:
+        // we just parsed at least one number after a decimal point
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_exponent:
+        // we just parsed an exponent
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '+':
+            case '-':
+            {
+                add(current);
+                goto scan_number_sign;
+            }
+
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message =
+                    "invalid number; expected '+', '-', or digit after exponent";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_sign:
+        // we just parsed an exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after exponent sign";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_any2:
+        // we just parsed a number after the exponent or exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_done:
+        // unget the character after the number (we only read it to know that
+        // we are done scanning a number)
+        unget();
+
+        char* endptr = nullptr;
+        errno = 0;
+
+        // try to parse integers first and fall back to floats
+        if (number_type == token_type::value_unsigned)
+        {
+            const auto x = std::strtoull(yytext.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == yytext.data() + yytext.size());
+
+            if (errno == 0)
+            {
+                value_unsigned = static_cast<number_unsigned_t>(x);
+                if (value_unsigned == x)
+                {
+                    return token_type::value_unsigned;
+                }
+            }
+        }
+        else if (number_type == token_type::value_integer)
+        {
+            const auto x = std::strtoll(yytext.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == yytext.data() + yytext.size());
+
+            if (errno == 0)
+            {
+                value_integer = static_cast<number_integer_t>(x);
+                if (value_integer == x)
+                {
+                    return token_type::value_integer;
+                }
+            }
+        }
+
+        // this code is reached if we parse a floating-point number or if an
+        // integer conversion above failed
+        strtof(value_float, yytext.data(), &endptr);
+
+        // we checked the number format before
+        assert(endptr == yytext.data() + yytext.size());
+
+        return token_type::value_float;
+    }
+
+    /*!
+    @param[in] literal_text  the literal text to expect
+    @param[in] length        the length of the passed literal text
+    @param[in] return_type   the token type to return on success
+    */
+    token_type scan_literal(const char* literal_text, const std::size_t length,
+                            token_type return_type)
+    {
+        assert(current == literal_text[0]);
+        for (std::size_t i = 1; i < length; ++i)
+        {
+            if (JSON_UNLIKELY(get() != literal_text[i]))
+            {
+                error_message = "invalid literal";
+                return token_type::parse_error;
+            }
+        }
+        return return_type;
+    }
+
+    /////////////////////
+    // input management
+    /////////////////////
+
+    /// reset yytext; current character is beginning of token
+    void reset() noexcept
+    {
+        yytext.clear();
+        token_string.clear();
+        token_string.push_back(std::char_traits<char>::to_char_type(current));
+    }
+
+    /*
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a
+    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters
+    for use in error messages.
+
+    @return character read from the input
+    */
+    std::char_traits<char>::int_type get()
+    {
+        ++chars_read;
+        current = ia->get_character();
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            token_string.push_back(std::char_traits<char>::to_char_type(current));
+        }
+        return current;
+    }
+
+    /// unget current character (return it again on next get)
+    void unget()
+    {
+        --chars_read;
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            ia->unget_character();
+            assert(token_string.size() != 0);
+            token_string.pop_back();
+        }
+    }
+
+    /// add a character to yytext
+    void add(int c)
+    {
+        yytext.push_back(std::char_traits<char>::to_char_type(c));
+    }
+
+  public:
+    /////////////////////
+    // value getters
+    /////////////////////
+
+    /// return integer value
+    constexpr number_integer_t get_number_integer() const noexcept
+    {
+        return value_integer;
+    }
+
+    /// return unsigned integer value
+    constexpr number_unsigned_t get_number_unsigned() const noexcept
+    {
+        return value_unsigned;
+    }
+
+    /// return floating-point value
+    constexpr number_float_t get_number_float() const noexcept
+    {
+        return value_float;
+    }
+
+    /// return current string value (implicitly resets the token; useful only once)
+    std::string move_string()
+    {
+        return std::move(yytext);
+    }
+
+    /////////////////////
+    // diagnostics
+    /////////////////////
+
+    /// return position of last read token
+    constexpr std::size_t get_position() const noexcept
+    {
+        return chars_read;
+    }
+
+    /// return the last read token (for errors only).  Will never contain EOF
+    /// (an arbitrary value that is not a valid char value, often -1), because
+    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
+    std::string get_token_string() const
+    {
+        // escape control characters
+        std::string result;
+        for (auto c : token_string)
+        {
+            if ('\x00' <= c and c <= '\x1f')
+            {
+                // escape control characters
+                std::stringstream ss;
+                ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0')
+                   << std::hex << static_cast<int>(c) << ">";
+                result += ss.str();
+            }
+            else
+            {
+                // add character as is
+                result.push_back(c);
+            }
+        }
+
+        return result;
+    }
+
+    /// return syntax error message
+    constexpr const char* get_error_message() const noexcept
+    {
+        return error_message;
+    }
+
+    /////////////////////
+    // actual scanner
+    /////////////////////
+
+    token_type scan()
+    {
+        // read next character and ignore whitespace
+        do
+        {
+            get();
+        }
+        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
+
+        switch (current)
+        {
+            // structural characters
+            case '[':
+                return token_type::begin_array;
+            case ']':
+                return token_type::end_array;
+            case '{':
+                return token_type::begin_object;
+            case '}':
+                return token_type::end_object;
+            case ':':
+                return token_type::name_separator;
+            case ',':
+                return token_type::value_separator;
+
+            // literals
+            case 't':
+                return scan_literal("true", 4, token_type::literal_true);
+            case 'f':
+                return scan_literal("false", 5, token_type::literal_false);
+            case 'n':
+                return scan_literal("null", 4, token_type::literal_null);
+
+            // string
+            case '\"':
+                return scan_string();
+
+            // number
+            case '-':
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+                return scan_number();
+
+            // end of input (the null byte is needed when parsing from
+            // string literals)
+            case '\0':
+            case std::char_traits<char>::eof():
+                return token_type::end_of_input;
+
+            // error
+            default:
+                error_message = "invalid literal";
+                return token_type::parse_error;
+        }
+    }
+
+  private:
+    /// input adapter
+    detail::input_adapter_t ia = nullptr;
+
+    /// the current character
+    std::char_traits<char>::int_type current = std::char_traits<char>::eof();
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+
+    /// raw input token string (for error messages)
+    std::vector<char> token_string { };
+
+    /// buffer for variable-length tokens (numbers, strings)
+    std::string yytext { };
+
+    /// a description of occurred lexer errors
+    const char* error_message = "";
+
+    // number values
+    number_integer_t value_integer = 0;
+    number_unsigned_t value_unsigned = 0;
+    number_float_t value_float = 0;
+
+    /// the decimal point
+    const char decimal_point_char = '.';
+};
+
+/*!
+@brief syntax analysis
+
+This class implements a recursive decent parser.
+*/
+template<typename BasicJsonType>
+class parser
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using lexer_t = lexer<BasicJsonType>;
+    using token_type = typename lexer_t::token_type;
+
+  public:
+    enum class parse_event_t : uint8_t
+    {
+        /// the parser read `{` and started to process a JSON object
+        object_start,
+        /// the parser read `}` and finished processing a JSON object
+        object_end,
+        /// the parser read `[` and started to process a JSON array
+        array_start,
+        /// the parser read `]` and finished processing a JSON array
+        array_end,
+        /// the parser read a key of a value in an object
+        key,
+        /// the parser finished reading a JSON value
+        value
+    };
+
+    using parser_callback_t =
+        std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
+
+    /// a parser reading from an input adapter
+    explicit parser(detail::input_adapter_t adapter,
+                    const parser_callback_t cb = nullptr,
+                    const bool allow_exceptions_ = true)
+        : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_)
+    {}
+
+    /*!
+    @brief public parser interface
+
+    @param[in] strict      whether to expect the last token to be EOF
+    @param[in,out] result  parsed JSON value
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    */
+    void parse(const bool strict, BasicJsonType& result)
+    {
+        // read first token
+        get_token();
+
+        parse_internal(true, result);
+        result.assert_invariant();
+
+        // in strict mode, input must be completely read
+        if (strict)
+        {
+            get_token();
+            expect(token_type::end_of_input);
+        }
+
+        // in case of an error, return discarded value
+        if (errored)
+        {
+            result = value_t::discarded;
+            return;
+        }
+
+        // set top-level value to null if it was discarded by the callback
+        // function
+        if (result.is_discarded())
+        {
+            result = nullptr;
+        }
+    }
+
+    /*!
+    @brief public accept interface
+
+    @param[in] strict  whether to expect the last token to be EOF
+    @return whether the input is a proper JSON text
+    */
+    bool accept(const bool strict = true)
+    {
+        // read first token
+        get_token();
+
+        if (not accept_internal())
+        {
+            return false;
+        }
+
+        // strict => last token must be EOF
+        return not strict or (get_token() == token_type::end_of_input);
+    }
+
+  private:
+    /*!
+    @brief the actual parser
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    */
+    void parse_internal(bool keep, BasicJsonType& result)
+    {
+        // never parse after a parse error was detected
+        assert(not errored);
+
+        // start with a discarded value
+        if (not result.is_discarded())
+        {
+            result.m_value.destroy(result.m_type);
+            result.m_type = value_t::discarded;
+        }
+
+        switch (last_token)
+        {
+            case token_type::begin_object:
+            {
+                if (keep)
+                {
+                    if (callback)
+                    {
+                        keep = callback(depth++, parse_event_t::object_start, result);
+                    }
+
+                    if (not callback or keep)
+                    {
+                        // explicitly set result to object to cope with {}
+                        result.m_type = value_t::object;
+                        result.m_value = value_t::object;
+                    }
+                }
+
+                // read next token
+                get_token();
+
+                // closing } -> we are done
+                if (last_token == token_type::end_object)
+                {
+                    if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
+                    {
+                        result.m_value.destroy(result.m_type);
+                        result.m_type = value_t::discarded;
+                    }
+                    break;
+                }
+
+                // parse values
+                std::string key;
+                BasicJsonType value;
+                while (true)
+                {
+                    // store key
+                    if (not expect(token_type::value_string))
+                    {
+                        return;
+                    }
+                    key = m_lexer.move_string();
+
+                    bool keep_tag = false;
+                    if (keep)
+                    {
+                        if (callback)
+                        {
+                            BasicJsonType k(key);
+                            keep_tag = callback(depth, parse_event_t::key, k);
+                        }
+                        else
+                        {
+                            keep_tag = true;
+                        }
+                    }
+
+                    // parse separator (:)
+                    get_token();
+                    if (not expect(token_type::name_separator))
+                    {
+                        return;
+                    }
+
+                    // parse and add value
+                    get_token();
+                    value.m_value.destroy(value.m_type);
+                    value.m_type = value_t::discarded;
+                    parse_internal(keep, value);
+
+                    if (JSON_UNLIKELY(errored))
+                    {
+                        return;
+                    }
+
+                    if (keep and keep_tag and not value.is_discarded())
+                    {
+                        result.m_value.object->emplace(std::move(key), std::move(value));
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing }
+                    if (not expect(token_type::end_object))
+                    {
+                        return;
+                    }
+                    break;
+                }
+
+                if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
+                {
+                    result.m_value.destroy(result.m_type);
+                    result.m_type = value_t::discarded;
+                }
+                break;
+            }
+
+            case token_type::begin_array:
+            {
+                if (keep)
+                {
+                    if (callback)
+                    {
+                        keep = callback(depth++, parse_event_t::array_start, result);
+                    }
+
+                    if (not callback or keep)
+                    {
+                        // explicitly set result to array to cope with []
+                        result.m_type = value_t::array;
+                        result.m_value = value_t::array;
+                    }
+                }
+
+                // read next token
+                get_token();
+
+                // closing ] -> we are done
+                if (last_token == token_type::end_array)
+                {
+                    if (callback and not callback(--depth, parse_event_t::array_end, result))
+                    {
+                        result.m_value.destroy(result.m_type);
+                        result.m_type = value_t::discarded;
+                    }
+                    break;
+                }
+
+                // parse values
+                BasicJsonType value;
+                while (true)
+                {
+                    // parse value
+                    value.m_value.destroy(value.m_type);
+                    value.m_type = value_t::discarded;
+                    parse_internal(keep, value);
+
+                    if (JSON_UNLIKELY(errored))
+                    {
+                        return;
+                    }
+
+                    if (keep and not value.is_discarded())
+                    {
+                        result.m_value.array->push_back(std::move(value));
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing ]
+                    if (not expect(token_type::end_array))
+                    {
+                        return;
+                    }
+                    break;
+                }
+
+                if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
+                {
+                    result.m_value.destroy(result.m_type);
+                    result.m_type = value_t::discarded;
+                }
+                break;
+            }
+
+            case token_type::literal_null:
+            {
+                result.m_type = value_t::null;
+                break;
+            }
+
+            case token_type::value_string:
+            {
+                result.m_type = value_t::string;
+                result.m_value = m_lexer.move_string();
+                break;
+            }
+
+            case token_type::literal_true:
+            {
+                result.m_type = value_t::boolean;
+                result.m_value = true;
+                break;
+            }
+
+            case token_type::literal_false:
+            {
+                result.m_type = value_t::boolean;
+                result.m_value = false;
+                break;
+            }
+
+            case token_type::value_unsigned:
+            {
+                result.m_type = value_t::number_unsigned;
+                result.m_value = m_lexer.get_number_unsigned();
+                break;
+            }
+
+            case token_type::value_integer:
+            {
+                result.m_type = value_t::number_integer;
+                result.m_value = m_lexer.get_number_integer();
+                break;
+            }
+
+            case token_type::value_float:
+            {
+                result.m_type = value_t::number_float;
+                result.m_value = m_lexer.get_number_float();
+
+                // throw in case of infinity or NAN
+                if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
+                {
+                    if (allow_exceptions)
+                    {
+                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" +
+                                                        m_lexer.get_token_string() + "'"));
+                    }
+                    expect(token_type::uninitialized);
+                }
+                break;
+            }
+
+            case token_type::parse_error:
+            {
+                // using "uninitialized" to avoid "expected" message
+                if (not expect(token_type::uninitialized))
+                {
+                    return;
+                }
+                break; // LCOV_EXCL_LINE
+            }
+
+            default:
+            {
+                // the last token was unexpected; we expected a value
+                if (not expect(token_type::literal_or_value))
+                {
+                    return;
+                }
+                break; // LCOV_EXCL_LINE
+            }
+        }
+
+        if (keep and callback and not callback(depth, parse_event_t::value, result))
+        {
+            result.m_type = value_t::discarded;
+        }
+    }
+
+    /*!
+    @brief the acutal acceptor
+
+    @invariant 1. The last token is not yet processed. Therefore, the caller
+                  of this function must make sure a token has been read.
+               2. When this function returns, the last token is processed.
+                  That is, the last read character was already considered.
+
+    This invariant makes sure that no token needs to be "unput".
+    */
+    bool accept_internal()
+    {
+        switch (last_token)
+        {
+            case token_type::begin_object:
+            {
+                // read next token
+                get_token();
+
+                // closing } -> we are done
+                if (last_token == token_type::end_object)
+                {
+                    return true;
+                }
+
+                // parse values
+                while (true)
+                {
+                    // parse key
+                    if (last_token != token_type::value_string)
+                    {
+                        return false;
+                    }
+
+                    // parse separator (:)
+                    get_token();
+                    if (last_token != token_type::name_separator)
+                    {
+                        return false;
+                    }
+
+                    // parse value
+                    get_token();
+                    if (not accept_internal())
+                    {
+                        return false;
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing }
+                    return (last_token == token_type::end_object);
+                }
+            }
+
+            case token_type::begin_array:
+            {
+                // read next token
+                get_token();
+
+                // closing ] -> we are done
+                if (last_token == token_type::end_array)
+                {
+                    return true;
+                }
+
+                // parse values
+                while (true)
+                {
+                    // parse value
+                    if (not accept_internal())
+                    {
+                        return false;
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing ]
+                    return (last_token == token_type::end_array);
+                }
+            }
+
+            case token_type::value_float:
+            {
+                // reject infinity or NAN
+                return std::isfinite(m_lexer.get_number_float());
+            }
+
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::literal_true:
+            case token_type::value_integer:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+                return true;
+
+            default: // the last token was unexpected
+                return false;
+        }
+    }
+
+    /// get next token from lexer
+    token_type get_token()
+    {
+        return (last_token = m_lexer.scan());
+    }
+
+    /*!
+    @throw parse_error.101 if expected token did not occur
+    */
+    bool expect(token_type t)
+    {
+        if (JSON_UNLIKELY(t != last_token))
+        {
+            errored = true;
+            expected = t;
+            if (allow_exceptions)
+            {
+                throw_exception();
+            }
+            else
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    [[noreturn]] void throw_exception() const
+    {
+        std::string error_msg = "syntax error - ";
+        if (last_token == token_type::parse_error)
+        {
+            error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
+                         m_lexer.get_token_string() + "'";
+        }
+        else
+        {
+            error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token));
+        }
+
+        if (expected != token_type::uninitialized)
+        {
+            error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
+        }
+
+        JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
+    }
+
+  private:
+    /// current level of recursion
+    int depth = 0;
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// the type of the last read token
+    token_type last_token = token_type::uninitialized;
+    /// the lexer
+    lexer_t m_lexer;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// possible reason for the syntax error
+    token_type expected = token_type::uninitialized;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+
+///////////////
+// iterators //
+///////////////
+
+/*!
+@brief an iterator for primitive JSON types
+
+This class models an iterator for primitive JSON types (boolean, number,
+string). It's only purpose is to allow the iterator/const_iterator classes
+to "iterate" over primitive values. Internally, the iterator is modeled by
+a `difference_type` variable. Value begin_value (`0`) models the begin,
+end_value (`1`) models past the end.
+*/
+class primitive_iterator_t
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+
+    constexpr difference_type get_value() const noexcept
+    {
+        return m_it;
+    }
+
+    /// set iterator to a defined beginning
+    void set_begin() noexcept
+    {
+        m_it = begin_value;
+    }
+
+    /// set iterator to a defined past the end
+    void set_end() noexcept
+    {
+        m_it = end_value;
+    }
+
+    /// return whether the iterator can be dereferenced
+    constexpr bool is_begin() const noexcept
+    {
+        return m_it == begin_value;
+    }
+
+    /// return whether the iterator is at end
+    constexpr bool is_end() const noexcept
+    {
+        return m_it == end_value;
+    }
+
+    friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it == rhs.m_it;
+    }
+
+    friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it < rhs.m_it;
+    }
+
+    primitive_iterator_t operator+(difference_type i)
+    {
+        auto result = *this;
+        result += i;
+        return result;
+    }
+
+    friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it - rhs.m_it;
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
+    {
+        return os << it.m_it;
+    }
+
+    primitive_iterator_t& operator++()
+    {
+        ++m_it;
+        return *this;
+    }
+
+    primitive_iterator_t operator++(int)
+    {
+        auto result = *this;
+        m_it++;
+        return result;
+    }
+
+    primitive_iterator_t& operator--()
+    {
+        --m_it;
+        return *this;
+    }
+
+    primitive_iterator_t operator--(int)
+    {
+        auto result = *this;
+        m_it--;
+        return result;
+    }
+
+    primitive_iterator_t& operator+=(difference_type n)
+    {
+        m_it += n;
+        return *this;
+    }
+
+    primitive_iterator_t& operator-=(difference_type n)
+    {
+        m_it -= n;
+        return *this;
+    }
+
+  private:
+    static constexpr difference_type begin_value = 0;
+    static constexpr difference_type end_value = begin_value + 1;
+
+    /// iterator as signed integer type
+    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
+};
+
+/*!
+@brief an iterator value
+
+@note This structure could easily be a union, but MSVC currently does not allow
+unions members with complex constructors, see https://github.com/nlohmann/json/pull/105.
+*/
+template<typename BasicJsonType> struct internal_iterator
+{
+    /// iterator for JSON objects
+    typename BasicJsonType::object_t::iterator object_iterator {};
+    /// iterator for JSON arrays
+    typename BasicJsonType::array_t::iterator array_iterator {};
+    /// generic iterator for all other types
+    primitive_iterator_t primitive_iterator {};
+};
+
+template<typename IteratorType> class iteration_proxy;
+
+/*!
+@brief a template for a bidirectional iterator for the @ref basic_json class
+
+This class implements a both iterators (iterator and const_iterator) for the
+@ref basic_json class.
+
+@note An iterator is called *initialized* when a pointer to a JSON value has
+      been set (e.g., by a constructor or a copy assignment). If the iterator is
+      default-constructed, it is *uninitialized* and most methods are undefined.
+      **The library uses assertions to detect calls on uninitialized iterators.**
+
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+  The iterator that can be moved can be moved in both directions (i.e.
+  incremented and decremented).
+
+@since version 1.0.0, simplified in version 2.0.9, change to bidirectional
+       iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
+*/
+template<typename BasicJsonType>
+class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJsonType>
+{
+    /// allow basic_json to access private members
+    friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
+    friend BasicJsonType;
+    friend iteration_proxy<iter_impl>;
+
+    using object_t = typename BasicJsonType::object_t;
+    using array_t = typename BasicJsonType::array_t;
+    // make sure BasicJsonType is basic_json or const basic_json
+    static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
+                  "iter_impl only accepts (const) basic_json");
+
+  public:
+    /// the type of the values when the iterator is dereferenced
+    using value_type = typename BasicJsonType::value_type;
+    /// a type to represent differences between iterators
+    using difference_type = typename BasicJsonType::difference_type;
+    /// defines a pointer to the type iterated over (value_type)
+    using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,
+          typename BasicJsonType::const_pointer,
+          typename BasicJsonType::pointer>::type;
+    /// defines a reference to the type iterated over (value_type)
+    using reference =
+        typename std::conditional<std::is_const<BasicJsonType>::value,
+        typename BasicJsonType::const_reference,
+        typename BasicJsonType::reference>::type;
+
+    /// default constructor
+    iter_impl() = default;
+
+    /*!
+    @brief constructor for a given JSON instance
+    @param[in] object  pointer to a JSON object for this iterator
+    @pre object != nullptr
+    @post The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    explicit iter_impl(pointer object) noexcept : m_object(object)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = typename object_t::iterator();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = typename array_t::iterator();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator = primitive_iterator_t();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @note The conventional copy constructor and copy assignment are implicitly
+          defined. Combined with the following converting constructor and
+          assignment, they support: (1) copy from iterator to iterator, (2)
+          copy from const iterator to const iterator, and (3) conversion from
+          iterator to const iterator. However conversion from const iterator
+          to iterator is not defined.
+    */
+
+    /*!
+    @brief converting constructor
+    @param[in] other  non-const iterator to copy from
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+        : m_object(other.m_object), m_it(other.m_it) {}
+
+    /*!
+    @brief converting assignment
+    @param[in,out] other  non-const iterator to copy from
+    @return const/non-const iterator
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+    {
+        m_object = other.m_object;
+        m_it = other.m_it;
+        return *this;
+    }
+
+  private:
+    /*!
+    @brief set the iterator to the first value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_begin() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->begin();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->begin();
+                break;
+            }
+
+            case value_t::null:
+            {
+                // set to end so begin()==end() is true: null is empty
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_begin();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @brief set the iterator past the last value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_end() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->end();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+        }
+    }
+
+  public:
+    /*!
+    @brief return a reference to the value pointed to by the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator*() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return m_it.object_iterator->second;
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return *m_it.array_iterator;
+            }
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief dereference the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    pointer operator->() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return &(m_it.object_iterator->second);
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return &*m_it.array_iterator;
+            }
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief post-increment (it++)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator++(int)
+    {
+        auto result = *this;
+        ++(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-increment (++it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator++()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, 1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, 1);
+                break;
+            }
+
+            default:
+            {
+                ++m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief post-decrement (it--)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator--(int)
+    {
+        auto result = *this;
+        --(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-decrement (--it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator--()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, -1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, -1);
+                break;
+            }
+
+            default:
+            {
+                --m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  comparison: equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator==(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                return (m_it.object_iterator == other.m_it.object_iterator);
+
+            case value_t::array:
+                return (m_it.array_iterator == other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator == other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: not equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator!=(const iter_impl& other) const
+    {
+        return not operator==(other);
+    }
+
+    /*!
+    @brief  comparison: smaller
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
+
+            case value_t::array:
+                return (m_it.array_iterator < other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator < other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: less than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<=(const iter_impl& other) const
+    {
+        return not other.operator < (*this);
+    }
+
+    /*!
+    @brief  comparison: greater than
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>(const iter_impl& other) const
+    {
+        return not operator<=(other);
+    }
+
+    /*!
+    @brief  comparison: greater than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>=(const iter_impl& other) const
+    {
+        return not operator<(other);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator+=(difference_type i)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, i);
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator += i;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator-=(difference_type i)
+    {
+        return operator+=(-i);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator+(difference_type i) const
+    {
+        auto result = *this;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  addition of distance and iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    friend iter_impl operator+(difference_type i, const iter_impl& it)
+    {
+        auto result = it;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator-(difference_type i) const
+    {
+        auto result = *this;
+        result -= i;
+        return result;
+    }
+
+    /*!
+    @brief  return difference
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    difference_type operator-(const iter_impl& other) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+                return m_it.array_iterator - other.m_it.array_iterator;
+
+            default:
+                return m_it.primitive_iterator - other.m_it.primitive_iterator;
+        }
+    }
+
+    /*!
+    @brief  access to successor
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator[](difference_type n) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
+
+            case value_t::array:
+                return *std::next(m_it.array_iterator, n);
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.get_value() == -n))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief  return the key of an object iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    typename object_t::key_type key() const
+    {
+        assert(m_object != nullptr);
+
+        if (JSON_LIKELY(m_object->is_object()))
+        {
+            return m_it.object_iterator->first;
+        }
+
+        JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
+    }
+
+    /*!
+    @brief  return the value of an iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference value() const
+    {
+        return operator*();
+    }
+
+  private:
+    /// associated JSON instance
+    pointer m_object = nullptr;
+    /// the actual iterator of the associated instance
+    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {};
+};
+
+/// proxy class for the iterator_wrapper functions
+template<typename IteratorType> class iteration_proxy
+{
+  private:
+    /// helper class for iteration
+    class iteration_proxy_internal
+    {
+      private:
+        /// the iterator
+        IteratorType anchor;
+        /// an index for arrays (used to create key names)
+        std::size_t array_index = 0;
+
+      public:
+        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
+
+        /// dereference operator (needed for range-based for)
+        iteration_proxy_internal& operator*()
+        {
+            return *this;
+        }
+
+        /// increment operator (needed for range-based for)
+        iteration_proxy_internal& operator++()
+        {
+            ++anchor;
+            ++array_index;
+
+            return *this;
+        }
+
+        /// inequality operator (needed for range-based for)
+        bool operator!=(const iteration_proxy_internal& o) const noexcept
+        {
+            return anchor != o.anchor;
+        }
+
+        /// return key of the iterator
+        std::string key() const
+        {
+            assert(anchor.m_object != nullptr);
+
+            switch (anchor.m_object->type())
+            {
+                // use integer array index as key
+                case value_t::array:
+                    return std::to_string(array_index);
+
+                // use key from the object
+                case value_t::object:
+                    return anchor.key();
+
+                // use an empty key for all primitive types
+                default:
+                    return "";
+            }
+        }
+
+        /// return value of the iterator
+        typename IteratorType::reference value() const
+        {
+            return anchor.value();
+        }
+    };
+
+    /// the container to iterate
+    typename IteratorType::reference container;
+
+  public:
+    /// construct iteration proxy from a container
+    explicit iteration_proxy(typename IteratorType::reference cont)
+        : container(cont) {}
+
+    /// return iterator begin (needed for range-based for)
+    iteration_proxy_internal begin() noexcept
+    {
+        return iteration_proxy_internal(container.begin());
+    }
+
+    /// return iterator end (needed for range-based for)
+    iteration_proxy_internal end() noexcept
+    {
+        return iteration_proxy_internal(container.end());
+    }
+};
+
+/*!
+@brief a template for a reverse iterator class
+
+@tparam Base the base iterator type to reverse. Valid types are @ref
+iterator (to create @ref reverse_iterator) and @ref const_iterator (to
+create @ref const_reverse_iterator).
+
+@requirement The class satisfies the following concept requirements:
+-
+[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+  The iterator that can be moved can be moved in both directions (i.e.
+  incremented and decremented).
+- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator):
+  It is possible to write to the pointed-to element (only if @a Base is
+  @ref iterator).
+
+@since version 1.0.0
+*/
+template<typename Base>
+class json_reverse_iterator : public std::reverse_iterator<Base>
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+    /// shortcut to the reverse iterator adaptor
+    using base_iterator = std::reverse_iterator<Base>;
+    /// the reference type for the pointed-to element
+    using reference = typename Base::reference;
+
+    /// create reverse iterator from iterator
+    json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+        : base_iterator(it) {}
+
+    /// create reverse iterator from base class
+    json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+
+    /// post-increment (it++)
+    json_reverse_iterator operator++(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
+    }
+
+    /// pre-increment (++it)
+    json_reverse_iterator& operator++()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator++());
+    }
+
+    /// post-decrement (it--)
+    json_reverse_iterator operator--(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
+    }
+
+    /// pre-decrement (--it)
+    json_reverse_iterator& operator--()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator--());
+    }
+
+    /// add to iterator
+    json_reverse_iterator& operator+=(difference_type i)
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
+    }
+
+    /// add to iterator
+    json_reverse_iterator operator+(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
+    }
+
+    /// subtract from iterator
+    json_reverse_iterator operator-(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
+    }
+
+    /// return difference
+    difference_type operator-(const json_reverse_iterator& other) const
+    {
+        return base_iterator(*this) - base_iterator(other);
+    }
+
+    /// access to successor
+    reference operator[](difference_type n) const
+    {
+        return *(this->operator+(n));
+    }
+
+    /// return the key of an object iterator
+    auto key() const -> decltype(std::declval<Base>().key())
+    {
+        auto it = --this->base();
+        return it.key();
+    }
+
+    /// return the value of an iterator
+    reference value() const
+    {
+        auto it = --this->base();
+        return it.operator * ();
+    }
+};
+
+/////////////////////
+// output adapters //
+/////////////////////
+
+/// abstract output adapter interface
+template<typename CharType> struct output_adapter_protocol
+{
+    virtual void write_character(CharType c) = 0;
+    virtual void write_characters(const CharType* s, std::size_t length) = 0;
+    virtual ~output_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+template<typename CharType>
+using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
+
+/// output adapter for byte vectors
+template<typename CharType>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {}
+
+    void write_character(CharType c) override
+    {
+        v.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        std::copy(s, s + length, std::back_inserter(v));
+    }
+
+  private:
+    std::vector<CharType>& v;
+};
+
+/// output adapter for output streams
+template<typename CharType>
+class output_stream_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {}
+
+    void write_character(CharType c) override
+    {
+        stream.put(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        stream.write(s, static_cast<std::streamsize>(length));
+    }
+
+  private:
+    std::basic_ostream<CharType>& stream;
+};
+
+/// output adapter for basic_string
+template<typename CharType>
+class output_string_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {}
+
+    void write_character(CharType c) override
+    {
+        str.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        str.append(s, length);
+    }
+
+  private:
+    std::basic_string<CharType>& str;
+};
+
+template<typename CharType>
+class output_adapter
+{
+  public:
+    output_adapter(std::vector<CharType>& vec)
+        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
+
+    output_adapter(std::basic_ostream<CharType>& s)
+        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+
+    output_adapter(std::basic_string<CharType>& s)
+        : oa(std::make_shared<output_string_adapter<CharType>>(s)) {}
+
+    operator output_adapter_t<CharType>()
+    {
+        return oa;
+    }
+
+  private:
+    output_adapter_t<CharType> oa = nullptr;
+};
+
+//////////////////////////////
+// binary reader and writer //
+//////////////////////////////
+
+/*!
+@brief deserialization of CBOR and MessagePack values
+*/
+template<typename BasicJsonType>
+class binary_reader
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+
+  public:
+    /*!
+    @brief create a binary reader
+
+    @param[in] adapter  input adapter to read from
+    */
+    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
+    {
+        assert(ia);
+    }
+
+    /*!
+    @brief create a JSON value from CBOR input
+
+    @param[in] strict  whether to expect the input to be consumed completed
+    @return JSON value created from CBOR input
+
+    @throw parse_error.110 if input ended unexpectedly or the end of file was
+                           not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported byte was read
+    */
+    BasicJsonType parse_cbor(const bool strict)
+    {
+        const auto res = parse_cbor_internal();
+        if (strict)
+        {
+            get();
+            check_eof(true);
+        }
+        return res;
+    }
+
+    /*!
+    @brief create a JSON value from MessagePack input
+
+    @param[in] strict  whether to expect the input to be consumed completed
+    @return JSON value created from MessagePack input
+
+    @throw parse_error.110 if input ended unexpectedly or the end of file was
+                           not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported byte was read
+    */
+    BasicJsonType parse_msgpack(const bool strict)
+    {
+        const auto res = parse_msgpack_internal();
+        if (strict)
+        {
+            get();
+            check_eof(true);
+        }
+        return res;
+    }
+
+    /*!
+    @brief determine system byte order
+
+    @return true if and only if system's byte order is little endian
+
+    @note from http://stackoverflow.com/a/1001328/266378
+    */
+    static constexpr bool little_endianess(int num = 1) noexcept
+    {
+        return (*reinterpret_cast<char*>(&num) == 1);
+    }
+
+  private:
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+    */
+    BasicJsonType parse_cbor_internal(const bool get_char = true)
+    {
+        switch (get_char ? get() : current)
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+
+            // Integer 0x00..0x17 (0..23)
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0a:
+            case 0x0b:
+            case 0x0c:
+            case 0x0d:
+            case 0x0e:
+            case 0x0f:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+                return static_cast<number_unsigned_t>(current);
+
+            case 0x18: // Unsigned integer (one-byte uint8_t follows)
+                return get_number<uint8_t>();
+
+            case 0x19: // Unsigned integer (two-byte uint16_t follows)
+                return get_number<uint16_t>();
+
+            case 0x1a: // Unsigned integer (four-byte uint32_t follows)
+                return get_number<uint32_t>();
+
+            case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
+                return get_number<uint64_t>();
+
+            // Negative integer -1-0x00..-1-0x17 (-1..-24)
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2a:
+            case 0x2b:
+            case 0x2c:
+            case 0x2d:
+            case 0x2e:
+            case 0x2f:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+                return static_cast<int8_t>(0x20 - 1 - current);
+
+            case 0x38: // Negative integer (one-byte uint8_t follows)
+            {
+                // must be uint8_t !
+                return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
+            }
+
+            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
+            {
+                return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
+            }
+
+            case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
+            {
+                return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
+            }
+
+            case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
+            {
+                return static_cast<number_integer_t>(-1) -
+                       static_cast<number_integer_t>(get_number<uint64_t>());
+            }
+
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 0x7f: // UTF-8 string (indefinite length)
+            {
+                return get_cbor_string();
+            }
+
+            // array (0x00..0x17 data items follow)
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8a:
+            case 0x8b:
+            case 0x8c:
+            case 0x8d:
+            case 0x8e:
+            case 0x8f:
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            {
+                return get_cbor_array(current & 0x1f);
+            }
+
+            case 0x98: // array (one-byte uint8_t for n follows)
+            {
+                return get_cbor_array(get_number<uint8_t>());
+            }
+
+            case 0x99: // array (two-byte uint16_t for n follow)
+            {
+                return get_cbor_array(get_number<uint16_t>());
+            }
+
+            case 0x9a: // array (four-byte uint32_t for n follow)
+            {
+                return get_cbor_array(get_number<uint32_t>());
+            }
+
+            case 0x9b: // array (eight-byte uint64_t for n follow)
+            {
+                return get_cbor_array(get_number<uint64_t>());
+            }
+
+            case 0x9f: // array (indefinite length)
+            {
+                BasicJsonType result = value_t::array;
+                while (get() != 0xff)
+                {
+                    result.push_back(parse_cbor_internal(false));
+                }
+                return result;
+            }
+
+            // map (0x00..0x17 pairs of data items follow)
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            {
+                return get_cbor_object(current & 0x1f);
+            }
+
+            case 0xb8: // map (one-byte uint8_t for n follows)
+            {
+                return get_cbor_object(get_number<uint8_t>());
+            }
+
+            case 0xb9: // map (two-byte uint16_t for n follow)
+            {
+                return get_cbor_object(get_number<uint16_t>());
+            }
+
+            case 0xba: // map (four-byte uint32_t for n follow)
+            {
+                return get_cbor_object(get_number<uint32_t>());
+            }
+
+            case 0xbb: // map (eight-byte uint64_t for n follow)
+            {
+                return get_cbor_object(get_number<uint64_t>());
+            }
+
+            case 0xbf: // map (indefinite length)
+            {
+                BasicJsonType result = value_t::object;
+                while (get() != 0xff)
+                {
+                    auto key = get_cbor_string();
+                    result[key] = parse_cbor_internal();
+                }
+                return result;
+            }
+
+            case 0xf4: // false
+            {
+                return false;
+            }
+
+            case 0xf5: // true
+            {
+                return true;
+            }
+
+            case 0xf6: // null
+            {
+                return value_t::null;
+            }
+
+            case 0xf9: // Half-Precision Float (two-byte IEEE 754)
+            {
+                const int byte1 = get();
+                check_eof();
+                const int byte2 = get();
+                check_eof();
+
+                // code from RFC 7049, Appendix D, Figure 3:
+                // As half-precision floating-point numbers were only added
+                // to IEEE 754 in 2008, today's programming platforms often
+                // still only have limited support for them. It is very
+                // easy to include at least decoding support for them even
+                // without such support. An example of a small decoder for
+                // half-precision floating-point numbers in the C language
+                // is shown in Fig. 3.
+                const int half = (byte1 << 8) + byte2;
+                const int exp = (half >> 10) & 0x1f;
+                const int mant = half & 0x3ff;
+                double val;
+                if (exp == 0)
+                {
+                    val = std::ldexp(mant, -24);
+                }
+                else if (exp != 31)
+                {
+                    val = std::ldexp(mant + 1024, exp - 25);
+                }
+                else
+                {
+                    val = (mant == 0) ? std::numeric_limits<double>::infinity()
+                          : std::numeric_limits<double>::quiet_NaN();
+                }
+                return (half & 0x8000) != 0 ? -val : val;
+            }
+
+            case 0xfa: // Single-Precision Float (four-byte IEEE 754)
+            {
+                return get_number<float>();
+            }
+
+            case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
+            {
+                return get_number<double>();
+            }
+
+            default: // anything else (0xFF is handled inside the other types)
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    BasicJsonType parse_msgpack_internal()
+    {
+        switch (get())
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+
+            // positive fixint
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0a:
+            case 0x0b:
+            case 0x0c:
+            case 0x0d:
+            case 0x0e:
+            case 0x0f:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+            case 0x18:
+            case 0x19:
+            case 0x1a:
+            case 0x1b:
+            case 0x1c:
+            case 0x1d:
+            case 0x1e:
+            case 0x1f:
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2a:
+            case 0x2b:
+            case 0x2c:
+            case 0x2d:
+            case 0x2e:
+            case 0x2f:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+            case 0x38:
+            case 0x39:
+            case 0x3a:
+            case 0x3b:
+            case 0x3c:
+            case 0x3d:
+            case 0x3e:
+            case 0x3f:
+            case 0x40:
+            case 0x41:
+            case 0x42:
+            case 0x43:
+            case 0x44:
+            case 0x45:
+            case 0x46:
+            case 0x47:
+            case 0x48:
+            case 0x49:
+            case 0x4a:
+            case 0x4b:
+            case 0x4c:
+            case 0x4d:
+            case 0x4e:
+            case 0x4f:
+            case 0x50:
+            case 0x51:
+            case 0x52:
+            case 0x53:
+            case 0x54:
+            case 0x55:
+            case 0x56:
+            case 0x57:
+            case 0x58:
+            case 0x59:
+            case 0x5a:
+            case 0x5b:
+            case 0x5c:
+            case 0x5d:
+            case 0x5e:
+            case 0x5f:
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78:
+            case 0x79:
+            case 0x7a:
+            case 0x7b:
+            case 0x7c:
+            case 0x7d:
+            case 0x7e:
+            case 0x7f:
+                return static_cast<number_unsigned_t>(current);
+
+            // fixmap
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8a:
+            case 0x8b:
+            case 0x8c:
+            case 0x8d:
+            case 0x8e:
+            case 0x8f:
+            {
+                return get_msgpack_object(current & 0x0f);
+            }
+
+            // fixarray
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            case 0x98:
+            case 0x99:
+            case 0x9a:
+            case 0x9b:
+            case 0x9c:
+            case 0x9d:
+            case 0x9e:
+            case 0x9f:
+            {
+                return get_msgpack_array(current & 0x0f);
+            }
+
+            // fixstr
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            case 0xb8:
+            case 0xb9:
+            case 0xba:
+            case 0xbb:
+            case 0xbc:
+            case 0xbd:
+            case 0xbe:
+            case 0xbf:
+                return get_msgpack_string();
+
+            case 0xc0: // nil
+                return value_t::null;
+
+            case 0xc2: // false
+                return false;
+
+            case 0xc3: // true
+                return true;
+
+            case 0xca: // float 32
+                return get_number<float>();
+
+            case 0xcb: // float 64
+                return get_number<double>();
+
+            case 0xcc: // uint 8
+                return get_number<uint8_t>();
+
+            case 0xcd: // uint 16
+                return get_number<uint16_t>();
+
+            case 0xce: // uint 32
+                return get_number<uint32_t>();
+
+            case 0xcf: // uint 64
+                return get_number<uint64_t>();
+
+            case 0xd0: // int 8
+                return get_number<int8_t>();
+
+            case 0xd1: // int 16
+                return get_number<int16_t>();
+
+            case 0xd2: // int 32
+                return get_number<int32_t>();
+
+            case 0xd3: // int 64
+                return get_number<int64_t>();
+
+            case 0xd9: // str 8
+            case 0xda: // str 16
+            case 0xdb: // str 32
+                return get_msgpack_string();
+
+            case 0xdc: // array 16
+            {
+                return get_msgpack_array(get_number<uint16_t>());
+            }
+
+            case 0xdd: // array 32
+            {
+                return get_msgpack_array(get_number<uint32_t>());
+            }
+
+            case 0xde: // map 16
+            {
+                return get_msgpack_object(get_number<uint16_t>());
+            }
+
+            case 0xdf: // map 32
+            {
+                return get_msgpack_object(get_number<uint32_t>());
+            }
+
+            // positive fixint
+            case 0xe0:
+            case 0xe1:
+            case 0xe2:
+            case 0xe3:
+            case 0xe4:
+            case 0xe5:
+            case 0xe6:
+            case 0xe7:
+            case 0xe8:
+            case 0xe9:
+            case 0xea:
+            case 0xeb:
+            case 0xec:
+            case 0xed:
+            case 0xee:
+            case 0xef:
+            case 0xf0:
+            case 0xf1:
+            case 0xf2:
+            case 0xf3:
+            case 0xf4:
+            case 0xf5:
+            case 0xf6:
+            case 0xf7:
+            case 0xf8:
+            case 0xf9:
+            case 0xfa:
+            case 0xfb:
+            case 0xfc:
+            case 0xfd:
+            case 0xfe:
+            case 0xff:
+                return static_cast<int8_t>(current);
+
+            default: // anything else
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(112, chars_read,
+                                               "error reading MessagePack; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    /*!
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a -'ve valued
+    `std::char_traits<char>::eof()` in that case.
+
+    @return character read from the input
+    */
+    int get()
+    {
+        ++chars_read;
+        return (current = ia->get_character());
+    }
+
+    /*
+    @brief read a number from the input
+
+    @tparam NumberType the type of the number
+
+    @return number of type @a NumberType
+
+    @note This function needs to respect the system's endianess, because
+          bytes in CBOR and MessagePack are stored in network order (big
+          endian) and therefore need reordering on little endian systems.
+
+    @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes
+    */
+    template<typename NumberType> NumberType get_number()
+    {
+        // step 1: read input into array with system's byte order
+        std::array<uint8_t, sizeof(NumberType)> vec;
+        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
+        {
+            get();
+            check_eof();
+
+            // reverse byte order prior to conversion if necessary
+            if (is_little_endian)
+            {
+                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
+            }
+            else
+            {
+                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
+            }
+        }
+
+        // step 2: convert array into number of type T and return
+        NumberType result;
+        std::memcpy(&result, vec.data(), sizeof(NumberType));
+        return result;
+    }
+
+    /*!
+    @brief create a string by reading characters from the input
+
+    @param[in] len number of bytes to read
+
+    @note We can not reserve @a len bytes for the result, because @a len
+          may be too large. Usually, @ref check_eof() detects the end of
+          the input before we run out of string memory.
+
+    @return string created by reading @a len bytes
+
+    @throw parse_error.110 if input has less than @a len bytes
+    */
+    template<typename NumberType>
+    std::string get_string(const NumberType len)
+    {
+        std::string result;
+        std::generate_n(std::back_inserter(result), len, [this]()
+        {
+            get();
+            check_eof();
+            return static_cast<char>(current);
+        });
+        return result;
+    }
+
+    /*!
+    @brief reads a CBOR string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+    Additionally, CBOR's strings with indefinite lengths are supported.
+
+    @return string
+
+    @throw parse_error.110 if input ended
+    @throw parse_error.113 if an unexpected byte is read
+    */
+    std::string get_cbor_string()
+    {
+        check_eof();
+
+        switch (current)
+        {
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            {
+                return get_string(current & 0x1f);
+            }
+
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            {
+                return get_string(get_number<uint8_t>());
+            }
+
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            {
+                return get_string(get_number<uint16_t>());
+            }
+
+            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+            {
+                return get_string(get_number<uint32_t>());
+            }
+
+            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+            {
+                return get_string(get_number<uint64_t>());
+            }
+
+            case 0x7f: // UTF-8 string (indefinite length)
+            {
+                std::string result;
+                while (get() != 0xff)
+                {
+                    check_eof();
+                    result.push_back(static_cast<char>(current));
+                }
+                return result;
+            }
+
+            default:
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_cbor_array(const NumberType len)
+    {
+        BasicJsonType result = value_t::array;
+        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
+        {
+            return parse_cbor_internal();
+        });
+        return result;
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_cbor_object(const NumberType len)
+    {
+        BasicJsonType result = value_t::object;
+        std::generate_n(std::inserter(*result.m_value.object,
+                                      result.m_value.object->end()),
+                        len, [this]()
+        {
+            get();
+            auto key = get_cbor_string();
+            auto val = parse_cbor_internal();
+            return std::make_pair(std::move(key), std::move(val));
+        });
+        return result;
+    }
+
+    /*!
+    @brief reads a MessagePack string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+
+    @return string
+
+    @throw parse_error.110 if input ended
+    @throw parse_error.113 if an unexpected byte is read
+    */
+    std::string get_msgpack_string()
+    {
+        check_eof();
+
+        switch (current)
+        {
+            // fixstr
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            case 0xb8:
+            case 0xb9:
+            case 0xba:
+            case 0xbb:
+            case 0xbc:
+            case 0xbd:
+            case 0xbe:
+            case 0xbf:
+            {
+                return get_string(current & 0x1f);
+            }
+
+            case 0xd9: // str 8
+            {
+                return get_string(get_number<uint8_t>());
+            }
+
+            case 0xda: // str 16
+            {
+                return get_string(get_number<uint16_t>());
+            }
+
+            case 0xdb: // str 32
+            {
+                return get_string(get_number<uint32_t>());
+            }
+
+            default:
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(113, chars_read,
+                                               "expected a MessagePack string; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_msgpack_array(const NumberType len)
+    {
+        BasicJsonType result = value_t::array;
+        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
+        {
+            return parse_msgpack_internal();
+        });
+        return result;
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_msgpack_object(const NumberType len)
+    {
+        BasicJsonType result = value_t::object;
+        std::generate_n(std::inserter(*result.m_value.object,
+                                      result.m_value.object->end()),
+                        len, [this]()
+        {
+            get();
+            auto key = get_msgpack_string();
+            auto val = parse_msgpack_internal();
+            return std::make_pair(std::move(key), std::move(val));
+        });
+        return result;
+    }
+
+    /*!
+    @brief check if input ended
+    @throw parse_error.110 if input ended
+    */
+    void check_eof(const bool expect_eof = false) const
+    {
+        if (expect_eof)
+        {
+            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            {
+                JSON_THROW(parse_error::create(110, chars_read, "expected end of input"));
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+            {
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+            }
+        }
+    }
+
+  private:
+    /// input adapter
+    input_adapter_t ia = nullptr;
+
+    /// the current character
+    int current = std::char_traits<char>::eof();
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+
+    /// whether we can assume little endianess
+    const bool is_little_endian = little_endianess();
+};
+
+/*!
+@brief serialization to CBOR and MessagePack values
+*/
+template<typename BasicJsonType, typename CharType>
+class binary_writer
+{
+  public:
+    /*!
+    @brief create a binary writer
+
+    @param[in] adapter  output adapter to write to
+    */
+    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
+    {
+        assert(oa);
+    }
+
+    /*!
+    @brief[in] j  JSON value to serialize
+    */
+    void write_cbor(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+            {
+                oa->write_character(static_cast<CharType>(0xf6));
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? static_cast<CharType>(0xf5)
+                                    : static_cast<CharType>(0xf4));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // CBOR does not differentiate between positive signed
+                    // integers and unsigned integers. Therefore, we used the
+                    // code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_integer <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x18));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x19));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x1a));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else
+                    {
+                        oa->write_character(static_cast<CharType>(0x1b));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    // The conversions below encode the sign in the first
+                    // byte, and the value is converted to a positive number.
+                    const auto positive_number = -1 - j.m_value.number_integer;
+                    if (j.m_value.number_integer >= -24)
+                    {
+                        write_number(static_cast<uint8_t>(0x20 + positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x38));
+                        write_number(static_cast<uint8_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x39));
+                        write_number(static_cast<uint16_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x3a));
+                        write_number(static_cast<uint32_t>(positive_number));
+                    }
+                    else
+                    {
+                        oa->write_character(static_cast<CharType>(0x3b));
+                        write_number(static_cast<uint64_t>(positive_number));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x18));
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x19));
+                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x1a));
+                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                }
+                else
+                {
+                    oa->write_character(static_cast<CharType>(0x1b));
+                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
+                }
+                break;
+            }
+
+            case value_t::number_float: // Double-Precision Float
+            {
+                oa->write_character(static_cast<CharType>(0xfb));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x60 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0x78));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x79));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x7a));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x7b));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x80 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0x98));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x99));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x9a));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x9b));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_cbor(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0xa0 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0xb8));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xb9));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xba));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xbb));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_cbor(el.first);
+                    write_cbor(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @brief[in] j  JSON value to serialize
+    */
+    void write_msgpack(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null: // nil
+            {
+                oa->write_character(static_cast<CharType>(0xc0));
+                break;
+            }
+
+            case value_t::boolean: // true and false
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? static_cast<CharType>(0xc3)
+                                    : static_cast<CharType>(0xc2));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // MessagePack does not differentiate between positive
+                    // signed integers and unsigned integers. Therefore, we used
+                    // the code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_unsigned < 128)
+                    {
+                        // positive fixnum
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        // uint 8
+                        oa->write_character(static_cast<CharType>(0xcc));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        // uint 16
+                        oa->write_character(static_cast<CharType>(0xcd));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        // uint 32
+                        oa->write_character(static_cast<CharType>(0xce));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                    {
+                        // uint 64
+                        oa->write_character(static_cast<CharType>(0xcf));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    if (j.m_value.number_integer >= -32)
+                    {
+                        // negative fixnum
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                    {
+                        // int 8
+                        oa->write_character(static_cast<CharType>(0xd0));
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                    {
+                        // int 16
+                        oa->write_character(static_cast<CharType>(0xd1));
+                        write_number(static_cast<int16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                    {
+                        // int 32
+                        oa->write_character(static_cast<CharType>(0xd2));
+                        write_number(static_cast<int32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
+                    {
+                        // int 64
+                        oa->write_character(static_cast<CharType>(0xd3));
+                        write_number(static_cast<int64_t>(j.m_value.number_integer));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned < 128)
+                {
+                    // positive fixnum
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // uint 8
+                    oa->write_character(static_cast<CharType>(0xcc));
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // uint 16
+                    oa->write_character(static_cast<CharType>(0xcd));
+                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // uint 32
+                    oa->write_character(static_cast<CharType>(0xce));
+                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    // uint 64
+                    oa->write_character(static_cast<CharType>(0xcf));
+                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                }
+                break;
+            }
+
+            case value_t::number_float: // float 64
+            {
+                oa->write_character(static_cast<CharType>(0xcb));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 31)
+                {
+                    // fixstr
+                    write_number(static_cast<uint8_t>(0xa0 | N));
+                }
+                else if (N <= 255)
+                {
+                    // str 8
+                    oa->write_character(static_cast<CharType>(0xd9));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 65535)
+                {
+                    // str 16
+                    oa->write_character(static_cast<CharType>(0xda));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 4294967295)
+                {
+                    // str 32
+                    oa->write_character(static_cast<CharType>(0xdb));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 15)
+                {
+                    // fixarray
+                    write_number(static_cast<uint8_t>(0x90 | N));
+                }
+                else if (N <= 0xffff)
+                {
+                    // array 16
+                    oa->write_character(static_cast<CharType>(0xdc));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    // array 32
+                    oa->write_character(static_cast<CharType>(0xdd));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_msgpack(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 15)
+                {
+                    // fixmap
+                    write_number(static_cast<uint8_t>(0x80 | (N & 0xf)));
+                }
+                else if (N <= 65535)
+                {
+                    // map 16
+                    oa->write_character(static_cast<CharType>(0xde));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 4294967295)
+                {
+                    // map 32
+                    oa->write_character(static_cast<CharType>(0xdf));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_msgpack(el.first);
+                    write_msgpack(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+  private:
+    /*
+    @brief write a number to output input
+
+    @param[in] n number of type @a NumberType
+    @tparam NumberType the type of the number
+
+    @note This function needs to respect the system's endianess, because bytes
+          in CBOR and MessagePack are stored in network order (big endian) and
+          therefore need reordering on little endian systems.
+    */
+    template<typename NumberType> void write_number(NumberType n)
+    {
+        // step 1: write number to array of length NumberType
+        std::array<CharType, sizeof(NumberType)> vec;
+        std::memcpy(vec.data(), &n, sizeof(NumberType));
+
+        // step 2: write array to output (with possible reordering)
+        if (is_little_endian)
+        {
+            // reverse byte order prior to conversion if necessary
+            std::reverse(vec.begin(), vec.end());
+        }
+
+        oa->write_characters(vec.data(), sizeof(NumberType));
+    }
+
+  private:
+    /// whether we can assume little endianess
+    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
+
+    /// the output
+    output_adapter_t<CharType> oa = nullptr;
+};
+
+///////////////////
+// serialization //
+///////////////////
+
+template<typename BasicJsonType>
+class serializer
+{
+    using string_t = typename BasicJsonType::string_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+  public:
+    /*!
+    @param[in] s  output stream to serialize to
+    @param[in] ichar  indentation character to use
+    */
+    serializer(output_adapter_t<char> s, const char ichar)
+        : o(std::move(s)), loc(std::localeconv()),
+          thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)),
+          decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)),
+          indent_char(ichar), indent_string(512, indent_char) {}
+
+    // delete because of pointer members
+    serializer(const serializer&) = delete;
+    serializer& operator=(const serializer&) = delete;
+
+    /*!
+    @brief internal implementation of the serialization function
+
+    This function is called by the public member function dump and organizes
+    the serialization internally. The indentation level is propagated as
+    additional parameter. In case of arrays and objects, the function is
+    called recursively.
+
+    - strings and object keys are escaped using `escape_string()`
+    - integer numbers are converted implicitly via `operator<<`
+    - floating-point numbers are converted to a string using `"%g"` format
+
+    @param[in] val             value to serialize
+    @param[in] pretty_print    whether the output shall be pretty-printed
+    @param[in] indent_step     the indent level
+    @param[in] current_indent  the current indent level (only used internally)
+    */
+    void dump(const BasicJsonType& val, const bool pretty_print,
+              const bool ensure_ascii,
+              const unsigned int indent_step,
+              const unsigned int current_indent = 0)
+    {
+        switch (val.m_type)
+        {
+            case value_t::object:
+            {
+                if (val.m_value.object->empty())
+                {
+                    o->write_characters("{}", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("{\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\": ", 3);
+                        dump(i->second, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\": ", 3);
+                    dump(i->second, true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character('}');
+                }
+                else
+                {
+                    o->write_character('{');
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\":", 2);
+                        dump(i->second, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\":", 2);
+                    dump(i->second, false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character('}');
+                }
+
+                return;
+            }
+
+            case value_t::array:
+            {
+                if (val.m_value.array->empty())
+                {
+                    o->write_characters("[]", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("[\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        dump(*i, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character(']');
+                }
+                else
+                {
+                    o->write_character('[');
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        dump(*i, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character(']');
+                }
+
+                return;
+            }
+
+            case value_t::string:
+            {
+                o->write_character('\"');
+                dump_escaped(*val.m_value.string, ensure_ascii);
+                o->write_character('\"');
+                return;
+            }
+
+            case value_t::boolean:
+            {
+                if (val.m_value.boolean)
+                {
+                    o->write_characters("true", 4);
+                }
+                else
+                {
+                    o->write_characters("false", 5);
+                }
+                return;
+            }
+
+            case value_t::number_integer:
+            {
+                dump_integer(val.m_value.number_integer);
+                return;
+            }
+
+            case value_t::number_unsigned:
+            {
+                dump_integer(val.m_value.number_unsigned);
+                return;
+            }
+
+            case value_t::number_float:
+            {
+                dump_float(val.m_value.number_float);
+                return;
+            }
+
+            case value_t::discarded:
+            {
+                o->write_characters("<discarded>", 11);
+                return;
+            }
+
+            case value_t::null:
+            {
+                o->write_characters("null", 4);
+                return;
+            }
+        }
+    }
+
+  private:
+    /*!
+    @brief returns the number of expected bytes following in UTF-8 string
+
+    @param[in]  u  the first byte of a UTF-8 string
+    @return  the number of expected bytes following
+    */
+    static constexpr std::size_t bytes_following(const uint8_t u)
+    {
+        return ((u <= 127) ? 0
+                : ((192 <= u and u <= 223) ? 1
+                   : ((224 <= u and u <= 239) ? 2
+                      : ((240 <= u and u <= 247) ? 3 : std::string::npos))));
+    }
+
+    /*!
+    @brief calculates the extra space to escape a JSON string
+
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+    @return the number of characters required to escape string @a s
+
+    @complexity Linear in the length of string @a s.
+    */
+    static std::size_t extra_space(const string_t& s,
+                                   const bool ensure_ascii) noexcept
+    {
+        std::size_t res = 0;
+
+        for (std::size_t i = 0; i < s.size(); ++i)
+        {
+            switch (s[i])
+            {
+                // control characters that can be escaped with a backslash
+                case '"':
+                case '\\':
+                case '\b':
+                case '\f':
+                case '\n':
+                case '\r':
+                case '\t':
+                {
+                    // from c (1 byte) to \x (2 bytes)
+                    res += 1;
+                    break;
+                }
+
+                // control characters that need \uxxxx escaping
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x0b:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                case 0x18:
+                case 0x19:
+                case 0x1a:
+                case 0x1b:
+                case 0x1c:
+                case 0x1d:
+                case 0x1e:
+                case 0x1f:
+                {
+                    // from c (1 byte) to \uxxxx (6 bytes)
+                    res += 5;
+                    break;
+                }
+
+                default:
+                {
+                    if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))
+                    {
+                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
+                        if (bytes == std::string::npos)
+                        {
+                            // invalid characters are treated as is, so no
+                            // additional space will be used
+                            break;
+                        }
+
+                        if (bytes == 3)
+                        {
+                            // codepoints that need 4 bytes (i.e., 3 additional
+                            // bytes) in UTF-8 need a surrogate pair when \u
+                            // escaping is used: from 4 bytes to \uxxxx\uxxxx
+                            // (12 bytes)
+                            res += (12 - bytes - 1);
+                        }
+                        else
+                        {
+                            // from x bytes to \uxxxx (6 bytes)
+                            res += (6 - bytes - 1);
+                        }
+
+                        // skip the additional bytes
+                        i += bytes;
+                    }
+                    break;
+                }
+            }
+        }
+
+        return res;
+    }
+
+    static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos)
+    {
+        // expecting a proper codepoint
+        assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+        // the last written character was the backslash before the 'u'
+        assert(result[pos] == '\\');
+
+        // write the 'u'
+        result[++pos] = 'u';
+
+        // convert a number 0..15 to its hex representation (0..f)
+        static const std::array<char, 16> hexify =
+        {
+            {
+                '0', '1', '2', '3', '4', '5', '6', '7',
+                '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+            }
+        };
+
+        if (codepoint < 0x10000)
+        {
+            // codepoints U+0000..U+FFFF can be represented as \uxxxx.
+            result[++pos] = hexify[(codepoint >> 12) & 0x0F];
+            result[++pos] = hexify[(codepoint >> 8) & 0x0F];
+            result[++pos] = hexify[(codepoint >> 4) & 0x0F];
+            result[++pos] = hexify[codepoint & 0x0F];
+        }
+        else
+        {
+            // codepoints U+10000..U+10FFFF need a surrogate pair to be
+            // represented as \uxxxx\uxxxx.
+            // http://www.unicode.org/faq/utf_bom.html#utf16-4
+            codepoint -= 0x10000;
+            const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF);
+            const int low_surrogate = 0xDC00 | (codepoint & 0x3FF);
+            result[++pos] = hexify[(high_surrogate >> 12) & 0x0F];
+            result[++pos] = hexify[(high_surrogate >> 8) & 0x0F];
+            result[++pos] = hexify[(high_surrogate >> 4) & 0x0F];
+            result[++pos] = hexify[high_surrogate & 0x0F];
+            ++pos;  // backslash is already in output
+            result[++pos] = 'u';
+            result[++pos] = hexify[(low_surrogate >> 12) & 0x0F];
+            result[++pos] = hexify[(low_surrogate >> 8) & 0x0F];
+            result[++pos] = hexify[(low_surrogate >> 4) & 0x0F];
+            result[++pos] = hexify[low_surrogate & 0x0F];
+        }
+
+        ++pos;
+    }
+
+    /*!
+    @brief dump escaped string
+
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
+
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+
+    @complexity Linear in the length of string @a s.
+    */
+    void dump_escaped(const string_t& s, const bool ensure_ascii) const
+    {
+        const auto space = extra_space(s, ensure_ascii);
+        if (space == 0)
+        {
+            o->write_characters(s.c_str(), s.size());
+            return;
+        }
+
+        // create a result string of necessary size
+        string_t result(s.size() + space, '\\');
+        std::size_t pos = 0;
+
+        for (std::size_t i = 0; i < s.size(); ++i)
+        {
+            switch (s[i])
+            {
+                case '"': // quotation mark (0x22)
+                {
+                    result[pos + 1] = '"';
+                    pos += 2;
+                    break;
+                }
+
+                case '\\': // reverse solidus (0x5c)
+                {
+                    // nothing to change
+                    pos += 2;
+                    break;
+                }
+
+                case '\b': // backspace (0x08)
+                {
+                    result[pos + 1] = 'b';
+                    pos += 2;
+                    break;
+                }
+
+                case '\f': // formfeed (0x0c)
+                {
+                    result[pos + 1] = 'f';
+                    pos += 2;
+                    break;
+                }
+
+                case '\n': // newline (0x0a)
+                {
+                    result[pos + 1] = 'n';
+                    pos += 2;
+                    break;
+                }
+
+                case '\r': // carriage return (0x0d)
+                {
+                    result[pos + 1] = 'r';
+                    pos += 2;
+                    break;
+                }
+
+                case '\t': // horizontal tab (0x09)
+                {
+                    result[pos + 1] = 't';
+                    pos += 2;
+                    break;
+                }
+
+                default:
+                {
+                    // escape control characters (0x00..0x1F) or, if
+                    // ensure_ascii parameter is used, non-ASCII characters
+                    if ((0x00 <= s[i] and s[i] <= 0x1F) or
+                            (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)))
+                    {
+                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
+                        if (bytes == std::string::npos)
+                        {
+                            // copy invalid character as is
+                            result[pos++] = s[i];
+                            break;
+                        }
+
+                        // check that the additional bytes are present
+                        assert(i + bytes < s.size());
+
+                        // to use \uxxxx escaping, we first need to caluclate
+                        // the codepoint from the UTF-8 bytes
+                        int codepoint = 0;
+
+                        assert(0 <= bytes and bytes <= 3);
+                        switch (bytes)
+                        {
+                            case 0:
+                            {
+                                codepoint = s[i] & 0xFF;
+                                break;
+                            }
+
+                            case 1:
+                            {
+                                codepoint = ((s[i] & 0x3F) << 6)
+                                            + (s[i + 1] & 0x7F);
+                                break;
+                            }
+
+                            case 2:
+                            {
+                                codepoint = ((s[i] & 0x1F) << 12)
+                                            + ((s[i + 1] & 0x7F) << 6)
+                                            + (s[i + 2] & 0x7F);
+                                break;
+                            }
+
+                            case 3:
+                            {
+                                codepoint = ((s[i] & 0xF) << 18)
+                                            + ((s[i + 1] & 0x7F) << 12)
+                                            + ((s[i + 2] & 0x7F) << 6)
+                                            + (s[i + 3] & 0x7F);
+                                break;
+                            }
+
+                            default:
+                                break;  // LCOV_EXCL_LINE
+                        }
+
+                        escape_codepoint(codepoint, result, pos);
+                        i += bytes;
+                    }
+                    else
+                    {
+                        // all other characters are added as-is
+                        result[pos++] = s[i];
+                    }
+                    break;
+                }
+            }
+        }
+
+        assert(pos == result.size());
+        o->write_characters(result.c_str(), result.size());
+    }
+
+    /*!
+    @brief dump an integer
+
+    Dump a given integer to output stream @a o. Works internally with
+    @a number_buffer.
+
+    @param[in] x  integer number (signed or unsigned) to dump
+    @tparam NumberType either @a number_integer_t or @a number_unsigned_t
+    */
+    template <
+        typename NumberType,
+        detail::enable_if_t<std::is_same<NumberType, number_unsigned_t>::value or
+                            std::is_same<NumberType, number_integer_t>::value,
+                            int> = 0 >
+    void dump_integer(NumberType x)
+    {
+        // special case for "0"
+        if (x == 0)
+        {
+            o->write_character('0');
+            return;
+        }
+
+        const bool is_negative = (x <= 0) and (x != 0);  // see issue #755
+        std::size_t i = 0;
+
+        while (x != 0)
+        {
+            // spare 1 byte for '\0'
+            assert(i < number_buffer.size() - 1);
+
+            const auto digit = std::labs(static_cast<long>(x % 10));
+            number_buffer[i++] = static_cast<char>('0' + digit);
+            x /= 10;
+        }
+
+        if (is_negative)
+        {
+            // make sure there is capacity for the '-'
+            assert(i < number_buffer.size() - 2);
+            number_buffer[i++] = '-';
+        }
+
+        std::reverse(number_buffer.begin(), number_buffer.begin() + i);
+        o->write_characters(number_buffer.data(), i);
+    }
+
+    /*!
+    @brief dump a floating-point number
+
+    Dump a given floating-point number to output stream @a o. Works internally
+    with @a number_buffer.
+
+    @param[in] x  floating-point number to dump
+    */
+    void dump_float(number_float_t x)
+    {
+        // NaN / inf
+        if (not std::isfinite(x) or std::isnan(x))
+        {
+            o->write_characters("null", 4);
+            return;
+        }
+
+        // get number of digits for a text -> float -> text round-trip
+        static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
+
+        // the actual conversion
+        std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+
+        // negative value indicates an error
+        assert(len > 0);
+        // check if buffer was large enough
+        assert(static_cast<std::size_t>(len) < number_buffer.size());
+
+        // erase thousands separator
+        if (thousands_sep != '\0')
+        {
+            const auto end = std::remove(number_buffer.begin(),
+                                         number_buffer.begin() + len, thousands_sep);
+            std::fill(end, number_buffer.end(), '\0');
+            assert((end - number_buffer.begin()) <= len);
+            len = (end - number_buffer.begin());
+        }
+
+        // convert decimal point to '.'
+        if (decimal_point != '\0' and decimal_point != '.')
+        {
+            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            if (dec_pos != number_buffer.end())
+            {
+                *dec_pos = '.';
+            }
+        }
+
+        o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));
+
+        // determine if need to append ".0"
+        const bool value_is_int_like =
+            std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,
+                         [](char c)
+        {
+            return (c == '.' or c == 'e');
+        });
+
+        if (value_is_int_like)
+        {
+            o->write_characters(".0", 2);
+        }
+    }
+
+  private:
+    /// the output of the serializer
+    output_adapter_t<char> o = nullptr;
+
+    /// a (hopefully) large enough character buffer
+    std::array<char, 64> number_buffer{{}};
+
+    /// the locale
+    const std::lconv* loc = nullptr;
+    /// the locale's thousand separator character
+    const char thousands_sep = '\0';
+    /// the locale's decimal point character
+    const char decimal_point = '\0';
+
+    /// the indentation character
+    const char indent_char;
+
+    /// the indentation string
+    string_t indent_string;
+};
+
+template<typename BasicJsonType>
+class json_ref
+{
+  public:
+    using value_type = BasicJsonType;
+
+    json_ref(value_type&& value)
+        : owned_value(std::move(value)),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    json_ref(const value_type& value)
+        : value_ref(const_cast<value_type*>(&value)),
+          is_rvalue(false)
+    {}
+
+    json_ref(std::initializer_list<json_ref> init)
+        : owned_value(init),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    template <class... Args>
+    json_ref(Args&& ... args)
+        : owned_value(std::forward<Args>(args)...),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    // class should be movable only
+    json_ref(json_ref&&) = default;
+    json_ref(const json_ref&) = delete;
+    json_ref& operator=(const json_ref&) = delete;
+
+    value_type moved_or_copied() const
+    {
+        if (is_rvalue)
+        {
+            return std::move(*value_ref);
+        }
+        return *value_ref;
+    }
+
+    value_type const& operator*() const
+    {
+        return *static_cast<value_type const*>(value_ref);
+    }
+
+    value_type const* operator->() const
+    {
+        return static_cast<value_type const*>(value_ref);
+    }
+
+  private:
+    mutable value_type owned_value = nullptr;
+    value_type* value_ref = nullptr;
+    const bool is_rvalue;
+};
+
+} // namespace detail
+
+/// namespace to hold default `to_json` / `from_json` functions
+namespace
+{
+constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
+constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
+}
+
+
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename, typename>
+struct adl_serializer
+{
+    /*!
+    @brief convert a JSON value to any value type
+
+    This function is usually called by the `get()` function of the
+    @ref basic_json class (either explicit or via conversion operators).
+
+    @param[in] j         JSON value to read from
+    @param[in,out] val  value to write to
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static void from_json(BasicJsonType&& j, ValueType& val) noexcept(
+        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+    {
+        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+    }
+
+    /*!
+    @brief convert any value type to a JSON value
+
+    This function is usually called by the constructors of the @ref basic_json
+    class.
+
+    @param[in,out] j  JSON value to write to
+    @param[in] val     value to read from
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static void to_json(BasicJsonType& j, ValueType&& val) noexcept(
+        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
+    {
+        ::nlohmann::to_json(j, std::forward<ValueType>(val));
+    }
+};
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+class json_pointer
+{
+    /// allow basic_json to access private members
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    friend class basic_json;
+
+  public:
+    /*!
+    @brief create JSON pointer
+
+    Create a JSON pointer according to the syntax described in
+    [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3).
+
+    @param[in] s  string representing the JSON pointer; if omitted, the empty
+                  string is assumed which references the whole JSON value
+
+    @throw parse_error.107 if the given JSON pointer @a s is nonempty and
+    does not begin with a slash (`/`); see example below
+
+    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
+    is not followed by `0` (representing `~`) or `1` (representing `/`);
+    see example below
+
+    @liveexample{The example shows the construction several valid JSON
+    pointers as well as the exceptional behavior.,json_pointer}
+
+    @since version 2.0.0
+    */
+    explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {}
+
+    /*!
+    @brief return a string representation of the JSON pointer
+
+    @invariant For each JSON pointer `ptr`, it holds:
+    @code {.cpp}
+    ptr == json_pointer(ptr.to_string());
+    @endcode
+
+    @return a string representation of the JSON pointer
+
+    @liveexample{The example shows the result of `to_string`.,
+    json_pointer__to_string}
+
+    @since version 2.0.0
+    */
+    std::string to_string() const noexcept
+    {
+        return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
+                               std::string{},
+                               [](const std::string & a, const std::string & b)
+        {
+            return a + "/" + escape(b);
+        });
+    }
+
+    /// @copydoc to_string()
+    operator std::string() const
+    {
+        return to_string();
+    }
+
+  private:
+    /*!
+    @brief remove and return last reference pointer
+    @throw out_of_range.405 if JSON pointer has no parent
+    */
+    std::string pop_back()
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        auto last = reference_tokens.back();
+        reference_tokens.pop_back();
+        return last;
+    }
+
+    /// return whether pointer points to the root document
+    bool is_root() const
+    {
+        return reference_tokens.empty();
+    }
+
+    json_pointer top() const
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        json_pointer result = *this;
+        result.reference_tokens = {reference_tokens[0]};
+        return result;
+    }
+
+
+    /*!
+    @brief create and return a reference to the pointed to value
+
+    @complexity Linear in the number of reference tokens.
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.313 if value cannot be unflattened
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const;
+
+    /*!
+    @brief return a reference to the pointed to value
+
+    @note This version does not throw if a value is not present, but tries to
+          create nested values instead. For instance, calling this function
+          with pointer `"/this/that"` on a null value is equivalent to calling
+          `operator[]("this").operator[]("that")` on that value, effectively
+          changing the null value to an object.
+
+    @param[in] ptr  a JSON value
+
+    @return reference to the JSON value pointed to by the JSON pointer
+
+    @complexity Linear in the length of the JSON pointer.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @brief return a const reference to the pointed to value
+
+    @param[in] ptr  a JSON value
+
+    @return const reference to the JSON value pointed to by the JSON
+    pointer
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @brief split the string input to reference tokens
+
+    @note This function is only called by the json_pointer constructor.
+          All exceptions below are documented there.
+
+    @throw parse_error.107  if the pointer is not empty or begins with '/'
+    @throw parse_error.108  if character '~' is not followed by '0' or '1'
+    */
+    static std::vector<std::string> split(const std::string& reference_string)
+    {
+        std::vector<std::string> result;
+
+        // special case: empty reference string -> no reference tokens
+        if (reference_string.empty())
+        {
+            return result;
+        }
+
+        // check if nonempty reference string begins with slash
+        if (JSON_UNLIKELY(reference_string[0] != '/'))
+        {
+            JSON_THROW(detail::parse_error::create(107, 1,
+                                                   "JSON pointer must be empty or begin with '/' - was: '" +
+                                                   reference_string + "'"));
+        }
+
+        // extract the reference tokens:
+        // - slash: position of the last read slash (or end of string)
+        // - start: position after the previous slash
+        for (
+            // search for the first slash after the first character
+            std::size_t slash = reference_string.find_first_of('/', 1),
+            // set the beginning of the first reference token
+            start = 1;
+            // we can stop if start == string::npos+1 = 0
+            start != 0;
+            // set the beginning of the next reference token
+            // (will eventually be 0 if slash == std::string::npos)
+            start = slash + 1,
+            // find next slash
+            slash = reference_string.find_first_of('/', start))
+        {
+            // use the text between the beginning of the reference token
+            // (start) and the last slash (slash).
+            auto reference_token = reference_string.substr(start, slash - start);
+
+            // check reference tokens are properly escaped
+            for (std::size_t pos = reference_token.find_first_of('~');
+                    pos != std::string::npos;
+                    pos = reference_token.find_first_of('~', pos + 1))
+            {
+                assert(reference_token[pos] == '~');
+
+                // ~ must be followed by 0 or 1
+                if (JSON_UNLIKELY(pos == reference_token.size() - 1 or
+                                  (reference_token[pos + 1] != '0' and
+                                   reference_token[pos + 1] != '1')))
+                {
+                    JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
+                }
+            }
+
+            // finally, store the reference token
+            unescape(reference_token);
+            result.push_back(reference_token);
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief replace all occurrences of a substring by another string
+
+    @param[in,out] s  the string to manipulate; changed so that all
+                   occurrences of @a f are replaced with @a t
+    @param[in]     f  the substring to replace with @a t
+    @param[in]     t  the string to replace @a f
+
+    @pre The search string @a f must not be empty. **This precondition is
+    enforced with an assertion.**
+
+    @since version 2.0.0
+    */
+    static void replace_substring(std::string& s, const std::string& f,
+                                  const std::string& t)
+    {
+        assert(not f.empty());
+        for (auto pos = s.find(f);                // find first occurrence of f
+                pos != std::string::npos;         // make sure f was found
+                s.replace(pos, f.size(), t),      // replace with t, and
+                pos = s.find(f, pos + t.size()))  // find next occurrence of f
+        {}
+    }
+
+    /// escape "~"" to "~0" and "/" to "~1"
+    static std::string escape(std::string s)
+    {
+        replace_substring(s, "~", "~0");
+        replace_substring(s, "/", "~1");
+        return s;
+    }
+
+    /// unescape "~1" to tilde and "~0" to slash (order is important!)
+    static void unescape(std::string& s)
+    {
+        replace_substring(s, "~1", "/");
+        replace_substring(s, "~0", "~");
+    }
+
+    /*!
+    @param[in] reference_string  the reference string to the current value
+    @param[in] value             the value to consider
+    @param[in,out] result        the result object to insert values to
+
+    @note Empty objects or arrays are flattened to `null`.
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    static void flatten(const std::string& reference_string,
+                        const NLOHMANN_BASIC_JSON_TPL& value,
+                        NLOHMANN_BASIC_JSON_TPL& result);
+
+    /*!
+    @param[in] value  flattened JSON
+
+    @return unflattened JSON
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+    @throw type_error.313  if value cannot be unflattened
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    static NLOHMANN_BASIC_JSON_TPL
+    unflatten(const NLOHMANN_BASIC_JSON_TPL& value);
+
+    friend bool operator==(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept;
+
+    friend bool operator!=(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept;
+
+    /// the reference tokens
+    std::vector<std::string> reference_tokens;
+};
+
+/*!
+@brief a class to store JSON values
+
+@tparam ObjectType type for JSON objects (`std::map` by default; will be used
+in @ref object_t)
+@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used
+in @ref array_t)
+@tparam StringType type for JSON strings and object keys (`std::string` by
+default; will be used in @ref string_t)
+@tparam BooleanType type for JSON booleans (`bool` by default; will be used
+in @ref boolean_t)
+@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by
+default; will be used in @ref number_integer_t)
+@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c
+`uint64_t` by default; will be used in @ref number_unsigned_t)
+@tparam NumberFloatType type for JSON floating-point numbers (`double` by
+default; will be used in @ref number_float_t)
+@tparam AllocatorType type of the allocator to use (`std::allocator` by
+default)
+@tparam JSONSerializer the serializer to resolve internal calls to `to_json()`
+and `from_json()` (@ref adl_serializer by default)
+
+@requirement The class satisfies the following concept requirements:
+- Basic
+ - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible):
+   JSON values can be default constructed. The result will be a JSON null
+   value.
+ - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible):
+   A JSON value can be constructed from an rvalue argument.
+ - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible):
+   A JSON value can be copy-constructed from an lvalue expression.
+ - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable):
+   A JSON value van be assigned from an rvalue argument.
+ - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable):
+   A JSON value can be copy-assigned from an lvalue expression.
+ - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible):
+   JSON values can be destructed.
+- Layout
+ - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType):
+   JSON values have
+   [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
+   All non-static data members are private and standard layout types, the
+   class has no virtual functions or (virtual) base classes.
+- Library-wide
+ - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable):
+   JSON values can be compared with `==`, see @ref
+   operator==(const_reference,const_reference).
+ - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable):
+   JSON values can be compared with `<`, see @ref
+   operator<(const_reference,const_reference).
+ - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable):
+   Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of
+   other compatible types, using unqualified function call @ref swap().
+ - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer):
+   JSON values can be compared against `std::nullptr_t` objects which are used
+   to model the `null` value.
+- Container
+ - [Container](http://en.cppreference.com/w/cpp/concept/Container):
+   JSON values can be used like STL containers and provide iterator access.
+ - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer);
+   JSON values can be used like STL containers and provide reverse iterator
+   access.
+
+@invariant The member variables @a m_value and @a m_type have the following
+relationship:
+- If `m_type == value_t::object`, then `m_value.object != nullptr`.
+- If `m_type == value_t::array`, then `m_value.array != nullptr`.
+- If `m_type == value_t::string`, then `m_value.string != nullptr`.
+The invariants are checked by member function assert_invariant().
+
+@internal
+@note ObjectType trick from http://stackoverflow.com/a/9860911
+@endinternal
+
+@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange
+Format](http://rfc7159.net/rfc7159)
+
+@since version 1.0.0
+
+@nosubgrouping
+*/
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+class basic_json
+{
+  private:
+    template<detail::value_t> friend struct detail::external_constructor;
+    friend ::nlohmann::json_pointer;
+    friend ::nlohmann::detail::parser<basic_json>;
+    friend ::nlohmann::detail::serializer<basic_json>;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::iter_impl;
+    template<typename BasicJsonType, typename CharType>
+    friend class ::nlohmann::detail::binary_writer;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::binary_reader;
+
+    /// workaround type for MSVC
+    using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
+
+    // convenience aliases for types residing in namespace detail;
+    using lexer = ::nlohmann::detail::lexer<basic_json>;
+    using parser = ::nlohmann::detail::parser<basic_json>;
+
+    using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;
+    template<typename BasicJsonType>
+    using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;
+    template<typename BasicJsonType>
+    using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;
+    template<typename Iterator>
+    using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;
+    template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;
+
+    template<typename CharType>
+    using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;
+
+    using binary_reader = ::nlohmann::detail::binary_reader<basic_json>;
+    template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;
+
+    using serializer = ::nlohmann::detail::serializer<basic_json>;
+
+  public:
+    using value_t = detail::value_t;
+    // forward declarations
+    using json_pointer = ::nlohmann::json_pointer;
+    template<typename T, typename SFINAE>
+    using json_serializer = JSONSerializer<T, SFINAE>;
+
+    using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
+
+    ////////////////
+    // exceptions //
+    ////////////////
+
+    /// @name exceptions
+    /// Classes to implement user-defined exceptions.
+    /// @{
+
+    /// @copydoc detail::exception
+    using exception = detail::exception;
+    /// @copydoc detail::parse_error
+    using parse_error = detail::parse_error;
+    /// @copydoc detail::invalid_iterator
+    using invalid_iterator = detail::invalid_iterator;
+    /// @copydoc detail::type_error
+    using type_error = detail::type_error;
+    /// @copydoc detail::out_of_range
+    using out_of_range = detail::out_of_range;
+    /// @copydoc detail::other_error
+    using other_error = detail::other_error;
+
+    /// @}
+
+
+    /////////////////////
+    // container types //
+    /////////////////////
+
+    /// @name container types
+    /// The canonic container types to use @ref basic_json like any other STL
+    /// container.
+    /// @{
+
+    /// the type of elements in a basic_json container
+    using value_type = basic_json;
+
+    /// the type of an element reference
+    using reference = value_type&;
+    /// the type of an element const reference
+    using const_reference = const value_type&;
+
+    /// a type to represent differences between iterators
+    using difference_type = std::ptrdiff_t;
+    /// a type to represent container sizes
+    using size_type = std::size_t;
+
+    /// the allocator type
+    using allocator_type = AllocatorType<basic_json>;
+
+    /// the type of an element pointer
+    using pointer = typename std::allocator_traits<allocator_type>::pointer;
+    /// the type of an element const pointer
+    using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+
+    /// an iterator for a basic_json container
+    using iterator = iter_impl<basic_json>;
+    /// a const iterator for a basic_json container
+    using const_iterator = iter_impl<const basic_json>;
+    /// a reverse iterator for a basic_json container
+    using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>;
+    /// a const reverse iterator for a basic_json container
+    using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>;
+
+    /// @}
+
+
+    /*!
+    @brief returns the allocator associated with the container
+    */
+    static allocator_type get_allocator()
+    {
+        return allocator_type();
+    }
+
+    /*!
+    @brief returns version information on the library
+
+    This function returns a JSON object with information about the library,
+    including the version number and information on the platform and compiler.
+
+    @return JSON object holding version information
+    key         | description
+    ----------- | ---------------
+    `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version).
+    `copyright` | The copyright line for the library as string.
+    `name`      | The name of the library as string.
+    `platform`  | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`.
+    `url`       | The URL of the project as string.
+    `version`   | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string).
+
+    @liveexample{The following code shows an example output of the `meta()`
+    function.,meta}
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @complexity Constant.
+
+    @since 2.1.0
+    */
+    static basic_json meta()
+    {
+        basic_json result;
+
+        result["copyright"] = "(C) 2013-2017 Niels Lohmann";
+        result["name"] = "JSON for Modern C++";
+        result["url"] = "https://github.com/nlohmann/json";
+        result["version"] =
+        {
+            {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1}
+        };
+
+#ifdef _WIN32
+        result["platform"] = "win32";
+#elif defined __linux__
+        result["platform"] = "linux";
+#elif defined __APPLE__
+        result["platform"] = "apple";
+#elif defined __unix__
+        result["platform"] = "unix";
+#else
+        result["platform"] = "unknown";
+#endif
+
+#if defined(__ICC) || defined(__INTEL_COMPILER)
+        result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
+#elif defined(__clang__)
+        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
+#elif defined(__GNUC__) || defined(__GNUG__)
+        result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}};
+#elif defined(__HP_cc) || defined(__HP_aCC)
+        result["compiler"] = "hp"
+#elif defined(__IBMCPP__)
+        result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}};
+#elif defined(_MSC_VER)
+        result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}};
+#elif defined(__PGI)
+        result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}};
+#elif defined(__SUNPRO_CC)
+        result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}};
+#else
+        result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};
+#endif
+
+#ifdef __cplusplus
+        result["compiler"]["c++"] = std::to_string(__cplusplus);
+#else
+        result["compiler"]["c++"] = "unknown";
+#endif
+        return result;
+    }
+
+
+    ///////////////////////////
+    // JSON value data types //
+    ///////////////////////////
+
+    /// @name JSON value data types
+    /// The data types to store a JSON value. These types are derived from
+    /// the template arguments passed to class @ref basic_json.
+    /// @{
+
+    /*!
+    @brief a type for an object
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows:
+    > An object is an unordered collection of zero or more name/value pairs,
+    > where a name is a string and a value is a string, number, boolean, null,
+    > object, or array.
+
+    To store objects in C++, a type is defined by the template parameters
+    described below.
+
+    @tparam ObjectType  the container to store objects (e.g., `std::map` or
+    `std::unordered_map`)
+    @tparam StringType the type of the keys or names (e.g., `std::string`).
+    The comparison function `std::less<StringType>` is used to order elements
+    inside the container.
+    @tparam AllocatorType the allocator to use for objects (e.g.,
+    `std::allocator`)
+
+    #### Default type
+
+    With the default values for @a ObjectType (`std::map`), @a StringType
+    (`std::string`), and @a AllocatorType (`std::allocator`), the default
+    value for @a object_t is:
+
+    @code {.cpp}
+    std::map<
+      std::string, // key_type
+      basic_json, // value_type
+      std::less<std::string>, // key_compare
+      std::allocator<std::pair<const std::string, basic_json>> // allocator_type
+    >
+    @endcode
+
+    #### Behavior
+
+    The choice of @a object_t influences the behavior of the JSON class. With
+    the default type, objects have the following behavior:
+
+    - When all names are unique, objects will be interoperable in the sense
+      that all software implementations receiving that object will agree on
+      the name-value mappings.
+    - When the names within an object are not unique, later stored name/value
+      pairs overwrite previously stored name/value pairs, leaving the used
+      names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will
+      be treated as equal and both stored as `{"key": 1}`.
+    - Internally, name/value pairs are stored in lexicographical order of the
+      names. Objects will also be serialized (see @ref dump) in this order.
+      For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored
+      and serialized as `{"a": 2, "b": 1}`.
+    - When comparing objects, the order of the name/value pairs is irrelevant.
+      This makes objects interoperable in the sense that they will not be
+      affected by these differences. For instance, `{"b": 1, "a": 2}` and
+      `{"a": 2, "b": 1}` will be treated as equal.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the maximum depth of nesting.
+
+    In this class, the object's limit of nesting is not explicitly constrained.
+    However, a maximum depth of nesting may be introduced by the compiler or
+    runtime environment. A theoretical limit can be queried by calling the
+    @ref max_size function of a JSON object.
+
+    #### Storage
+
+    Objects are stored as pointers in a @ref basic_json type. That is, for any
+    access to object values, a pointer of type `object_t*` must be
+    dereferenced.
+
+    @sa @ref array_t -- type for an array value
+
+    @since version 1.0.0
+
+    @note The order name/value pairs are added to the object is *not*
+    preserved by the library. Therefore, iterating an object may return
+    name/value pairs in a different order than they were originally stored. In
+    fact, keys will be traversed in alphabetical order as `std::map` with
+    `std::less` is used by default. Please note this behavior conforms to [RFC
+    7159](http://rfc7159.net/rfc7159), because any order implements the
+    specified "unordered" nature of JSON objects.
+    */
+
+#if defined(JSON_HAS_CPP_14)
+    // Use transparent comparator if possible, combined with perfect forwarding
+    // on find() and count() calls prevents unnecessary string construction.
+    using object_comparator_t = std::less<>;
+#else
+    using object_comparator_t = std::less<StringType>;
+#endif
+    using object_t = ObjectType<StringType,
+          basic_json,
+          object_comparator_t,
+          AllocatorType<std::pair<const StringType,
+          basic_json>>>;
+
+    /*!
+    @brief a type for an array
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows:
+    > An array is an ordered sequence of zero or more values.
+
+    To store objects in C++, a type is defined by the template parameters
+    explained below.
+
+    @tparam ArrayType  container type to store arrays (e.g., `std::vector` or
+    `std::list`)
+    @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`)
+
+    #### Default type
+
+    With the default values for @a ArrayType (`std::vector`) and @a
+    AllocatorType (`std::allocator`), the default value for @a array_t is:
+
+    @code {.cpp}
+    std::vector<
+      basic_json, // value_type
+      std::allocator<basic_json> // allocator_type
+    >
+    @endcode
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the maximum depth of nesting.
+
+    In this class, the array's limit of nesting is not explicitly constrained.
+    However, a maximum depth of nesting may be introduced by the compiler or
+    runtime environment. A theoretical limit can be queried by calling the
+    @ref max_size function of a JSON array.
+
+    #### Storage
+
+    Arrays are stored as pointers in a @ref basic_json type. That is, for any
+    access to array values, a pointer of type `array_t*` must be dereferenced.
+
+    @sa @ref object_t -- type for an object value
+
+    @since version 1.0.0
+    */
+    using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
+
+    /*!
+    @brief a type for a string
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows:
+    > A string is a sequence of zero or more Unicode characters.
+
+    To store objects in C++, a type is defined by the template parameter
+    described below. Unicode values are split by the JSON class into
+    byte-sized characters during deserialization.
+
+    @tparam StringType  the container to store strings (e.g., `std::string`).
+    Note this container is used for keys/names in objects, see @ref object_t.
+
+    #### Default type
+
+    With the default values for @a StringType (`std::string`), the default
+    value for @a string_t is:
+
+    @code {.cpp}
+    std::string
+    @endcode
+
+    #### Encoding
+
+    Strings are stored in UTF-8 encoding. Therefore, functions like
+    `std::string::size()` or `std::string::length()` return the number of
+    bytes in the string rather than the number of characters or glyphs.
+
+    #### String comparison
+
+    [RFC 7159](http://rfc7159.net/rfc7159) states:
+    > Software implementations are typically required to test names of object
+    > members for equality. Implementations that transform the textual
+    > representation into sequences of Unicode code units and then perform the
+    > comparison numerically, code unit by code unit, are interoperable in the
+    > sense that implementations will agree in all cases on equality or
+    > inequality of two strings. For example, implementations that compare
+    > strings with escaped characters unconverted may incorrectly find that
+    > `"a\\b"` and `"a\u005Cb"` are not equal.
+
+    This implementation is interoperable as it does compare strings code unit
+    by code unit.
+
+    #### Storage
+
+    String values are stored as pointers in a @ref basic_json type. That is,
+    for any access to string values, a pointer of type `string_t*` must be
+    dereferenced.
+
+    @since version 1.0.0
+    */
+    using string_t = StringType;
+
+    /*!
+    @brief a type for a boolean
+
+    [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a
+    type which differentiates the two literals `true` and `false`.
+
+    To store objects in C++, a type is defined by the template parameter @a
+    BooleanType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a BooleanType (`bool`), the default value for
+    @a boolean_t is:
+
+    @code {.cpp}
+    bool
+    @endcode
+
+    #### Storage
+
+    Boolean values are stored directly inside a @ref basic_json type.
+
+    @since version 1.0.0
+    */
+    using boolean_t = BooleanType;
+
+    /*!
+    @brief a type for a number (integer)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store integer numbers in C++, a type is defined by the template
+    parameter @a NumberIntegerType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberIntegerType (`int64_t`), the default
+    value for @a number_integer_t is:
+
+    @code {.cpp}
+    int64_t
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in integer literals lead to an interpretation as octal
+      number. Internally, the value will be stored as decimal number. For
+      instance, the C++ integer literal `010` will be serialized to `8`.
+      During deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the range and precision of numbers.
+
+    When the default type is used, the maximal integer number that can be
+    stored is `9223372036854775807` (INT64_MAX) and the minimal integer number
+    that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers
+    that are out of range will yield over/underflow when used in a
+    constructor. During deserialization, too large or small integer numbers
+    will be automatically be stored as @ref number_unsigned_t or @ref
+    number_float_t.
+
+    [RFC 7159](http://rfc7159.net/rfc7159) further states:
+    > Note that when such software is used, numbers that are integers and are
+    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
+    > that implementations will agree exactly on their numeric values.
+
+    As this range is a subrange of the exactly supported range [INT64_MIN,
+    INT64_MAX], this class's integer type is interoperable.
+
+    #### Storage
+
+    Integer number values are stored directly inside a @ref basic_json type.
+
+    @sa @ref number_float_t -- type for number values (floating-point)
+
+    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
+
+    @since version 1.0.0
+    */
+    using number_integer_t = NumberIntegerType;
+
+    /*!
+    @brief a type for a number (unsigned)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store unsigned integer numbers in C++, a type is defined by the
+    template parameter @a NumberUnsignedType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberUnsignedType (`uint64_t`), the
+    default value for @a number_unsigned_t is:
+
+    @code {.cpp}
+    uint64_t
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in integer literals lead to an interpretation as octal
+      number. Internally, the value will be stored as decimal number. For
+      instance, the C++ integer literal `010` will be serialized to `8`.
+      During deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
+    > An implementation may set limits on the range and precision of numbers.
+
+    When the default type is used, the maximal integer number that can be
+    stored is `18446744073709551615` (UINT64_MAX) and the minimal integer
+    number that can be stored is `0`. Integer numbers that are out of range
+    will yield over/underflow when used in a constructor. During
+    deserialization, too large or small integer numbers will be automatically
+    be stored as @ref number_integer_t or @ref number_float_t.
+
+    [RFC 7159](http://rfc7159.net/rfc7159) further states:
+    > Note that when such software is used, numbers that are integers and are
+    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
+    > that implementations will agree exactly on their numeric values.
+
+    As this range is a subrange (when considered in conjunction with the
+    number_integer_t type) of the exactly supported range [0, UINT64_MAX],
+    this class's integer type is interoperable.
+
+    #### Storage
+
+    Integer number values are stored directly inside a @ref basic_json type.
+
+    @sa @ref number_float_t -- type for number values (floating-point)
+    @sa @ref number_integer_t -- type for number values (integer)
+
+    @since version 2.0.0
+    */
+    using number_unsigned_t = NumberUnsignedType;
+
+    /*!
+    @brief a type for a number (floating-point)
+
+    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
+    > The representation of numbers is similar to that used in most
+    > programming languages. A number is represented in base 10 using decimal
+    > digits. It contains an integer component that may be prefixed with an
+    > optional minus sign, which may be followed by a fraction part and/or an
+    > exponent part. Leading zeros are not allowed. (...) Numeric values that
+    > cannot be represented in the grammar below (such as Infinity and NaN)
+    > are not permitted.
+
+    This description includes both integer and floating-point numbers.
+    However, C++ allows more precise storage if it is known whether the number
+    is a signed integer, an unsigned integer or a floating-point number.
+    Therefore, three different types, @ref number_integer_t, @ref
+    number_unsigned_t and @ref number_float_t are used.
+
+    To store floating-point numbers in C++, a type is defined by the template
+    parameter @a NumberFloatType which chooses the type to use.
+
+    #### Default type
+
+    With the default values for @a NumberFloatType (`double`), the default
+    value for @a number_float_t is:
+
+    @code {.cpp}
+    double
+    @endcode
+
+    #### Default behavior
+
+    - The restrictions about leading zeros is not enforced in C++. Instead,
+      leading zeros in floating-point literals will be ignored. Internally,
+      the value will be stored as decimal number. For instance, the C++
+      floating-point literal `01.2` will be serialized to `1.2`. During
+      deserialization, leading zeros yield an error.
+    - Not-a-number (NaN) values will be serialized to `null`.
+
+    #### Limits
+
+    [RFC 7159](http://rfc7159.net/rfc7159) states:
+    > This specification allows implementations to set limits on the range and
+    > precision of numbers accepted. Since software that implements IEEE
+    > 754-2008 binary64 (double precision) numbers is generally available and
+    > widely used, good interoperability can be achieved by implementations
+    > that expect no more precision or range than these provide, in the sense
+    > that implementations will approximate JSON numbers within the expected
+    > precision.
+
+    This implementation does exactly follow this approach, as it uses double
+    precision floating-point numbers. Note values smaller than
+    `-1.79769313486232e+308` and values greater than `1.79769313486232e+308`
+    will be stored as NaN internally and be serialized to `null`.
+
+    #### Storage
+
+    Floating-point number values are stored directly inside a @ref basic_json
+    type.
+
+    @sa @ref number_integer_t -- type for number values (integer)
+
+    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
+
+    @since version 1.0.0
+    */
+    using number_float_t = NumberFloatType;
+
+    /// @}
+
+  private:
+
+    /// helper for exception-safe object creation
+    template<typename T, typename... Args>
+    static T* create(Args&& ... args)
+    {
+        AllocatorType<T> alloc;
+        auto deleter = [&](T * object)
+        {
+            alloc.deallocate(object, 1);
+        };
+        std::unique_ptr<T, decltype(deleter)> object(alloc.allocate(1), deleter);
+        alloc.construct(object.get(), std::forward<Args>(args)...);
+        assert(object != nullptr);
+        return object.release();
+    }
+
+    ////////////////////////
+    // JSON value storage //
+    ////////////////////////
+
+    /*!
+    @brief a JSON value
+
+    The actual storage for a JSON value of the @ref basic_json class. This
+    union combines the different storage types for the JSON value types
+    defined in @ref value_t.
+
+    JSON type | value_t type    | used type
+    --------- | --------------- | ------------------------
+    object    | object          | pointer to @ref object_t
+    array     | array           | pointer to @ref array_t
+    string    | string          | pointer to @ref string_t
+    boolean   | boolean         | @ref boolean_t
+    number    | number_integer  | @ref number_integer_t
+    number    | number_unsigned | @ref number_unsigned_t
+    number    | number_float    | @ref number_float_t
+    null      | null            | *no value is stored*
+
+    @note Variable-length types (objects, arrays, and strings) are stored as
+    pointers. The size of the union should not exceed 64 bits if the default
+    value types are used.
+
+    @since version 1.0.0
+    */
+    union json_value
+    {
+        /// object (stored with pointer to save storage)
+        object_t* object;
+        /// array (stored with pointer to save storage)
+        array_t* array;
+        /// string (stored with pointer to save storage)
+        string_t* string;
+        /// boolean
+        boolean_t boolean;
+        /// number (integer)
+        number_integer_t number_integer;
+        /// number (unsigned integer)
+        number_unsigned_t number_unsigned;
+        /// number (floating-point)
+        number_float_t number_float;
+
+        /// default constructor (for null values)
+        json_value() = default;
+        /// constructor for booleans
+        json_value(boolean_t v) noexcept : boolean(v) {}
+        /// constructor for numbers (integer)
+        json_value(number_integer_t v) noexcept : number_integer(v) {}
+        /// constructor for numbers (unsigned)
+        json_value(number_unsigned_t v) noexcept : number_unsigned(v) {}
+        /// constructor for numbers (floating-point)
+        json_value(number_float_t v) noexcept : number_float(v) {}
+        /// constructor for empty values of a given type
+        json_value(value_t t)
+        {
+            switch (t)
+            {
+                case value_t::object:
+                {
+                    object = create<object_t>();
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    array = create<array_t>();
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    string = create<string_t>("");
+                    break;
+                }
+
+                case value_t::boolean:
+                {
+                    boolean = boolean_t(false);
+                    break;
+                }
+
+                case value_t::number_integer:
+                {
+                    number_integer = number_integer_t(0);
+                    break;
+                }
+
+                case value_t::number_unsigned:
+                {
+                    number_unsigned = number_unsigned_t(0);
+                    break;
+                }
+
+                case value_t::number_float:
+                {
+                    number_float = number_float_t(0.0);
+                    break;
+                }
+
+                case value_t::null:
+                {
+                    object = nullptr;  // silence warning, see #821
+                    break;
+                }
+
+                default:
+                {
+                    object = nullptr;  // silence warning, see #821
+                    if (JSON_UNLIKELY(t == value_t::null))
+                    {
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE
+                    }
+                    break;
+                }
+            }
+        }
+
+        /// constructor for strings
+        json_value(const string_t& value)
+        {
+            string = create<string_t>(value);
+        }
+
+        /// constructor for rvalue strings
+        json_value(string_t&& value)
+        {
+            string = create<string_t>(std::move(value));
+        }
+
+        /// constructor for objects
+        json_value(const object_t& value)
+        {
+            object = create<object_t>(value);
+        }
+
+        /// constructor for rvalue objects
+        json_value(object_t&& value)
+        {
+            object = create<object_t>(std::move(value));
+        }
+
+        /// constructor for arrays
+        json_value(const array_t& value)
+        {
+            array = create<array_t>(value);
+        }
+
+        /// constructor for rvalue arrays
+        json_value(array_t&& value)
+        {
+            array = create<array_t>(std::move(value));
+        }
+
+        void destroy(value_t t)
+        {
+            switch (t)
+            {
+                case value_t::object:
+                {
+                    AllocatorType<object_t> alloc;
+                    alloc.destroy(object);
+                    alloc.deallocate(object, 1);
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    AllocatorType<array_t> alloc;
+                    alloc.destroy(array);
+                    alloc.deallocate(array, 1);
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    AllocatorType<string_t> alloc;
+                    alloc.destroy(string);
+                    alloc.deallocate(string, 1);
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
+    };
+
+    /*!
+    @brief checks the class invariants
+
+    This function asserts the class invariants. It needs to be called at the
+    end of every constructor to make sure that created objects respect the
+    invariant. Furthermore, it has to be called each time the type of a JSON
+    value is changed, because the invariant expresses a relationship between
+    @a m_type and @a m_value.
+    */
+    void assert_invariant() const
+    {
+        assert(m_type != value_t::object or m_value.object != nullptr);
+        assert(m_type != value_t::array or m_value.array != nullptr);
+        assert(m_type != value_t::string or m_value.string != nullptr);
+    }
+
+  public:
+    //////////////////////////
+    // JSON parser callback //
+    //////////////////////////
+
+    using parse_event_t = typename parser::parse_event_t;
+
+    /*!
+    @brief per-element parser callback type
+
+    With a parser callback function, the result of parsing a JSON text can be
+    influenced. When passed to @ref parse, it is called on certain events
+    (passed as @ref parse_event_t via parameter @a event) with a set recursion
+    depth @a depth and context JSON value @a parsed. The return value of the
+    callback function is a boolean indicating whether the element that emitted
+    the callback shall be kept or not.
+
+    We distinguish six scenarios (determined by the event type) in which the
+    callback function can be called. The following table describes the values
+    of the parameters @a depth, @a event, and @a parsed.
+
+    parameter @a event | description | parameter @a depth | parameter @a parsed
+    ------------------ | ----------- | ------------------ | -------------------
+    parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded
+    parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key
+    parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object
+    parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded
+    parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array
+    parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value
+
+    @image html callback_events.png "Example when certain parse events are triggered"
+
+    Discarding a value (i.e., returning `false`) has different effects
+    depending on the context in which function was called:
+
+    - Discarded values in structured types are skipped. That is, the parser
+      will behave as if the discarded value was never read.
+    - In case a value outside a structured type is skipped, it is replaced
+      with `null`. This case happens if the top-level element is skipped.
+
+    @param[in] depth  the depth of the recursion during parsing
+
+    @param[in] event  an event of type parse_event_t indicating the context in
+    the callback function has been called
+
+    @param[in,out] parsed  the current intermediate parse result; note that
+    writing to this value has no effect for parse_event_t::key events
+
+    @return Whether the JSON value which called the function during parsing
+    should be kept (`true`) or not (`false`). In the latter case, it is either
+    skipped completely or replaced by an empty discarded object.
+
+    @sa @ref parse for examples
+
+    @since version 1.0.0
+    */
+    using parser_callback_t = typename parser::parser_callback_t;
+
+
+    //////////////////
+    // constructors //
+    //////////////////
+
+    /// @name constructors and destructors
+    /// Constructors of class @ref basic_json, copy/move constructor, copy
+    /// assignment, static functions creating objects, and the destructor.
+    /// @{
+
+    /*!
+    @brief create an empty value with a given type
+
+    Create an empty JSON value with a given type. The value will be default
+    initialized with an empty value which depends on the type:
+
+    Value type  | initial value
+    ----------- | -------------
+    null        | `null`
+    boolean     | `false`
+    string      | `""`
+    number      | `0`
+    object      | `{}`
+    array       | `[]`
+
+    @param[in] v  the type of the value to create
+
+    @complexity Constant.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows the constructor for different @ref
+    value_t values,basic_json__value_t}
+
+    @sa @ref clear() -- restores the postcondition of this constructor
+
+    @since version 1.0.0
+    */
+    basic_json(const value_t v)
+        : m_type(v), m_value(v)
+    {
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a null object
+
+    Create a `null` JSON value. It either takes a null pointer as parameter
+    (explicitly creating `null`) or no parameter (implicitly creating `null`).
+    The passed null pointer itself is not read -- it is only used to choose
+    the right constructor.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this constructor never throws
+    exceptions.
+
+    @liveexample{The following code shows the constructor with and without a
+    null pointer parameter.,basic_json__nullptr_t}
+
+    @since version 1.0.0
+    */
+    basic_json(std::nullptr_t = nullptr) noexcept
+        : basic_json(value_t::null)
+    {
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a JSON value
+
+    This is a "catch all" constructor for all compatible JSON types; that is,
+    types for which a `to_json()` method exsits. The constructor forwards the
+    parameter @a val to that method (to `json_serializer<U>::to_json` method
+    with `U = uncvref_t<CompatibleType>`, to be exact).
+
+    Template type @a CompatibleType includes, but is not limited to, the
+    following types:
+    - **arrays**: @ref array_t and all kinds of compatible containers such as
+      `std::vector`, `std::deque`, `std::list`, `std::forward_list`,
+      `std::array`, `std::valarray`, `std::set`, `std::unordered_set`,
+      `std::multiset`, and `std::unordered_multiset` with a `value_type` from
+      which a @ref basic_json value can be constructed.
+    - **objects**: @ref object_t and all kinds of compatible associative
+      containers such as `std::map`, `std::unordered_map`, `std::multimap`,
+      and `std::unordered_multimap` with a `key_type` compatible to
+      @ref string_t and a `value_type` from which a @ref basic_json value can
+      be constructed.
+    - **strings**: @ref string_t, string literals, and all compatible string
+      containers can be used.
+    - **numbers**: @ref number_integer_t, @ref number_unsigned_t,
+      @ref number_float_t, and all convertible number types such as `int`,
+      `size_t`, `int64_t`, `float` or `double` can be used.
+    - **boolean**: @ref boolean_t / `bool` can be used.
+
+    See the examples below.
+
+    @tparam CompatibleType a type such that:
+    - @a CompatibleType is not derived from `std::istream`,
+    - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
+         constructors),
+    - @a CompatibleType is not a @ref basic_json nested type (e.g.,
+         @ref json_pointer, @ref iterator, etc ...)
+    - @ref @ref json_serializer<U> has a
+         `to_json(basic_json_t&, CompatibleType&&)` method
+
+    @tparam U = `uncvref_t<CompatibleType>`
+
+    @param[in] val the value to be forwarded to the respective constructor
+
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
+
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
+
+    @liveexample{The following code shows the constructor with several
+    compatible types.,basic_json__CompatibleType}
+
+    @since version 2.1.0
+    */
+    template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>,
+             detail::enable_if_t<not std::is_base_of<std::istream, U>::value and
+                                 not std::is_same<U, basic_json_t>::value and
+                                 not detail::is_basic_json_nested_type<
+                                     basic_json_t, U>::value and
+                                 detail::has_to_json<basic_json, U>::value,
+                                 int> = 0>
+    basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json(
+                std::declval<basic_json_t&>(), std::forward<CompatibleType>(val))))
+    {
+        JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
+        assert_invariant();
+    }
+
+    /*!
+    @brief create a container (array or object) from an initializer list
+
+    Creates a JSON value of type array or object from the passed initializer
+    list @a init. In case @a type_deduction is `true` (default), the type of
+    the JSON value to be created is deducted from the initializer list @a init
+    according to the following rules:
+
+    1. If the list is empty, an empty JSON object value `{}` is created.
+    2. If the list consists of pairs whose first element is a string, a JSON
+       object value is created where the first elements of the pairs are
+       treated as keys and the second elements are as values.
+    3. In all other cases, an array is created.
+
+    The rules aim to create the best fit between a C++ initializer list and
+    JSON values. The rationale is as follows:
+
+    1. The empty initializer list is written as `{}` which is exactly an empty
+       JSON object.
+    2. C++ has no way of describing mapped types other than to list a list of
+       pairs. As JSON requires that keys must be of type string, rule 2 is the
+       weakest constraint one can pose on initializer lists to interpret them
+       as an object.
+    3. In all other cases, the initializer list could not be interpreted as
+       JSON object type, so interpreting it as JSON array type is safe.
+
+    With the rules described above, the following JSON values cannot be
+    expressed by an initializer list:
+
+    - the empty array (`[]`): use @ref array(initializer_list_t)
+      with an empty initializer list in this case
+    - arrays whose elements satisfy rule 2: use @ref
+      array(initializer_list_t) with the same initializer list
+      in this case
+
+    @note When used without parentheses around an empty initializer list, @ref
+    basic_json() is called instead of this function, yielding the JSON null
+    value.
+
+    @param[in] init  initializer list with JSON values
+
+    @param[in] type_deduction internal parameter; when set to `true`, the type
+    of the JSON value is deducted from the initializer list @a init; when set
+    to `false`, the type provided via @a manual_type is forced. This mode is
+    used by the functions @ref array(initializer_list_t) and
+    @ref object(initializer_list_t).
+
+    @param[in] manual_type internal parameter; when @a type_deduction is set
+    to `false`, the created JSON value will use the provided type (only @ref
+    value_t::array and @ref value_t::object are valid); when @a type_deduction
+    is set to `true`, this parameter has no effect
+
+    @throw type_error.301 if @a type_deduction is `false`, @a manual_type is
+    `value_t::object`, but @a init contains an element which is not a pair
+    whose first element is a string. In this case, the constructor could not
+    create an object. If @a type_deduction would have be `true`, an array
+    would have been created. See @ref object(initializer_list_t)
+    for an example.
+
+    @complexity Linear in the size of the initializer list @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The example below shows how JSON values are created from
+    initializer lists.,basic_json__list_init_t}
+
+    @sa @ref array(initializer_list_t) -- create a JSON array
+    value from an initializer list
+    @sa @ref object(initializer_list_t) -- create a JSON object
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    basic_json(initializer_list_t init,
+               bool type_deduction = true,
+               value_t manual_type = value_t::array)
+    {
+        // check if each element is an array with two elements whose first
+        // element is a string
+        bool is_an_object = std::all_of(init.begin(), init.end(),
+                                        [](const detail::json_ref<basic_json>& element_ref)
+        {
+            return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string());
+        });
+
+        // adjust type if type deduction is not wanted
+        if (not type_deduction)
+        {
+            // if array is wanted, do not create an object though possible
+            if (manual_type == value_t::array)
+            {
+                is_an_object = false;
+            }
+
+            // if object is wanted but impossible, throw an exception
+            if (JSON_UNLIKELY(manual_type == value_t::object and not is_an_object))
+            {
+                JSON_THROW(type_error::create(301, "cannot create object from initializer list"));
+            }
+        }
+
+        if (is_an_object)
+        {
+            // the initializer list is a list of pairs -> create object
+            m_type = value_t::object;
+            m_value = value_t::object;
+
+            std::for_each(init.begin(), init.end(), [this](const detail::json_ref<basic_json>& element_ref)
+            {
+                auto element = element_ref.moved_or_copied();
+                m_value.object->emplace(
+                    std::move(*((*element.m_value.array)[0].m_value.string)),
+                    std::move((*element.m_value.array)[1]));
+            });
+        }
+        else
+        {
+            // the initializer list describes an array -> create array
+            m_type = value_t::array;
+            m_value.array = create<array_t>(init.begin(), init.end());
+        }
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief explicitly create an array from an initializer list
+
+    Creates a JSON array value from a given initializer list. That is, given a
+    list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the
+    initializer list is empty, the empty array `[]` is created.
+
+    @note This function is only needed to express two edge cases that cannot
+    be realized with the initializer list constructor (@ref
+    basic_json(initializer_list_t, bool, value_t)). These cases
+    are:
+    1. creating an array whose elements are all pairs whose first element is a
+    string -- in this case, the initializer list constructor would create an
+    object, taking the first elements as keys
+    2. creating an empty array -- passing the empty initializer list to the
+    initializer list constructor yields an empty object
+
+    @param[in] init  initializer list with JSON values to create an array from
+    (optional)
+
+    @return JSON array value
+
+    @complexity Linear in the size of @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows an example for the `array`
+    function.,array}
+
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
+    create a JSON value from an initializer list
+    @sa @ref object(initializer_list_t) -- create a JSON object
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    static basic_json array(initializer_list_t init = {})
+    {
+        return basic_json(init, false, value_t::array);
+    }
+
+    /*!
+    @brief explicitly create an object from an initializer list
+
+    Creates a JSON object value from a given initializer list. The initializer
+    lists elements must be pairs, and their first elements must be strings. If
+    the initializer list is empty, the empty object `{}` is created.
+
+    @note This function is only added for symmetry reasons. In contrast to the
+    related function @ref array(initializer_list_t), there are
+    no cases which can only be expressed by this function. That is, any
+    initializer list @a init can also be passed to the initializer list
+    constructor @ref basic_json(initializer_list_t, bool, value_t).
+
+    @param[in] init  initializer list to create an object from (optional)
+
+    @return JSON object value
+
+    @throw type_error.301 if @a init is not a list of pairs whose first
+    elements are strings. In this case, no object can be created. When such a
+    value is passed to @ref basic_json(initializer_list_t, bool, value_t),
+    an array would have been created from the passed initializer list @a init.
+    See example below.
+
+    @complexity Linear in the size of @a init.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows an example for the `object`
+    function.,object}
+
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
+    create a JSON value from an initializer list
+    @sa @ref array(initializer_list_t) -- create a JSON array
+    value from an initializer list
+
+    @since version 1.0.0
+    */
+    static basic_json object(initializer_list_t init = {})
+    {
+        return basic_json(init, false, value_t::object);
+    }
+
+    /*!
+    @brief construct an array with count copies of given value
+
+    Constructs a JSON array value by creating @a cnt copies of a passed value.
+    In case @a cnt is `0`, an empty array is created.
+
+    @param[in] cnt  the number of JSON copies of @a val to create
+    @param[in] val  the JSON value to copy
+
+    @post `std::distance(begin(),end()) == cnt` holds.
+
+    @complexity Linear in @a cnt.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The following code shows examples for the @ref
+    basic_json(size_type\, const basic_json&)
+    constructor.,basic_json__size_type_basic_json}
+
+    @since version 1.0.0
+    */
+    basic_json(size_type cnt, const basic_json& val)
+        : m_type(value_t::array)
+    {
+        m_value.array = create<array_t>(cnt, val);
+        assert_invariant();
+    }
+
+    /*!
+    @brief construct a JSON container given an iterator range
+
+    Constructs the JSON value with the contents of the range `[first, last)`.
+    The semantics depends on the different types a JSON value can have:
+    - In case of a null type, invalid_iterator.206 is thrown.
+    - In case of other primitive types (number, boolean, or string), @a first
+      must be `begin()` and @a last must be `end()`. In this case, the value is
+      copied. Otherwise, invalid_iterator.204 is thrown.
+    - In case of structured types (array, object), the constructor behaves as
+      similar versions for `std::vector` or `std::map`; that is, a JSON array
+      or object is constructed from the values in the range.
+
+    @tparam InputIT an input iterator type (@ref iterator or @ref
+    const_iterator)
+
+    @param[in] first begin of the range to copy from (included)
+    @param[in] last end of the range to copy from (excluded)
+
+    @pre Iterators @a first and @a last must be initialized. **This
+         precondition is enforced with an assertion (see warning).** If
+         assertions are switched off, a violation of this precondition yields
+         undefined behavior.
+
+    @pre Range `[first, last)` is valid. Usually, this precondition cannot be
+         checked efficiently. Only certain edge cases are detected; see the
+         description of the exceptions below. A violation of this precondition
+         yields undefined behavior.
+
+    @warning A precondition is enforced with a runtime assertion that will
+             result in calling `std::abort` if this precondition is not met.
+             Assertions can be disabled by defining `NDEBUG` at compile time.
+             See http://en.cppreference.com/w/cpp/error/assert for more
+             information.
+
+    @throw invalid_iterator.201 if iterators @a first and @a last are not
+    compatible (i.e., do not belong to the same JSON value). In this case,
+    the range `[first, last)` is undefined.
+    @throw invalid_iterator.204 if iterators @a first and @a last belong to a
+    primitive type (number, boolean, or string), but @a first does not point
+    to the first element any more. In this case, the range `[first, last)` is
+    undefined. See example code below.
+    @throw invalid_iterator.206 if iterators @a first and @a last belong to a
+    null value. In this case, the range `[first, last)` is undefined.
+
+    @complexity Linear in distance between @a first and @a last.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @liveexample{The example below shows several ways to create JSON values by
+    specifying a subrange with iterators.,basic_json__InputIt_InputIt}
+
+    @since version 1.0.0
+    */
+    template<class InputIT, typename std::enable_if<
+                 std::is_same<InputIT, typename basic_json_t::iterator>::value or
+                 std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int>::type = 0>
+    basic_json(InputIT first, InputIT last)
+    {
+        assert(first.m_object != nullptr);
+        assert(last.m_object != nullptr);
+
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(201, "iterators are not compatible"));
+        }
+
+        // copy type from first iterator
+        m_type = first.m_object->m_type;
+
+        // check if iterator range is complete for primitive values
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_UNLIKELY(not first.m_it.primitive_iterator.is_begin()
+                                  or not last.m_it.primitive_iterator.is_end()))
+                {
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+
+        switch (m_type)
+        {
+            case value_t::number_integer:
+            {
+                m_value.number_integer = first.m_object->m_value.number_integer;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value.number_unsigned = first.m_object->m_value.number_unsigned;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value.number_float = first.m_object->m_value.number_float;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value.boolean = first.m_object->m_value.boolean;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value = *first.m_object->m_value.string;
+                break;
+            }
+
+            case value_t::object:
+            {
+                m_value.object = create<object_t>(first.m_it.object_iterator,
+                                                  last.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value.array = create<array_t>(first.m_it.array_iterator,
+                                                last.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " +
+                                                    std::string(first.m_object->type_name())));
+        }
+
+        assert_invariant();
+    }
+
+
+    ///////////////////////////////////////
+    // other constructors and destructor //
+    ///////////////////////////////////////
+
+    /// @private
+    basic_json(const detail::json_ref<basic_json>& ref)
+        : basic_json(ref.moved_or_copied())
+    {}
+
+    /*!
+    @brief copy constructor
+
+    Creates a copy of a given JSON value.
+
+    @param[in] other  the JSON value to copy
+
+    @post `*this == other`
+
+    @complexity Linear in the size of @a other.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is linear.
+    - As postcondition, it holds: `other == basic_json(other)`.
+
+    @liveexample{The following code shows an example for the copy
+    constructor.,basic_json__basic_json}
+
+    @since version 1.0.0
+    */
+    basic_json(const basic_json& other)
+        : m_type(other.m_type)
+    {
+        // check of passed value is valid
+        other.assert_invariant();
+
+        switch (m_type)
+        {
+            case value_t::object:
+            {
+                m_value = *other.m_value.object;
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value = *other.m_value.array;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value = *other.m_value.string;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value = other.m_value.boolean;
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                m_value = other.m_value.number_integer;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value = other.m_value.number_unsigned;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value = other.m_value.number_float;
+                break;
+            }
+
+            default:
+                break;
+        }
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief move constructor
+
+    Move constructor. Constructs a JSON value with the contents of the given
+    value @a other using move semantics. It "steals" the resources from @a
+    other and leaves it as JSON null value.
+
+    @param[in,out] other  value to move to this object
+
+    @post `*this` has the same value as @a other before the call.
+    @post @a other is a JSON null value.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this constructor never throws
+    exceptions.
+
+    @requirement This function helps `basic_json` satisfying the
+    [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible)
+    requirements.
+
+    @liveexample{The code below shows the move constructor explicitly called
+    via std::move.,basic_json__moveconstructor}
+
+    @since version 1.0.0
+    */
+    basic_json(basic_json&& other) noexcept
+        : m_type(std::move(other.m_type)),
+          m_value(std::move(other.m_value))
+    {
+        // check that passed value is valid
+        other.assert_invariant();
+
+        // invalidate payload
+        other.m_type = value_t::null;
+        other.m_value = {};
+
+        assert_invariant();
+    }
+
+    /*!
+    @brief copy assignment
+
+    Copy assignment operator. Copies a JSON value via the "copy and swap"
+    strategy: It is expressed in terms of the copy constructor, destructor,
+    and the `swap()` member function.
+
+    @param[in] other  value to copy from
+
+    @complexity Linear.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is linear.
+
+    @liveexample{The code below shows and example for the copy assignment. It
+    creates a copy of value `a` which is then swapped with `b`. Finally\, the
+    copy of `a` (which is the null value after the swap) is
+    destroyed.,basic_json__copyassignment}
+
+    @since version 1.0.0
+    */
+    reference& operator=(basic_json other) noexcept (
+        std::is_nothrow_move_constructible<value_t>::value and
+        std::is_nothrow_move_assignable<value_t>::value and
+        std::is_nothrow_move_constructible<json_value>::value and
+        std::is_nothrow_move_assignable<json_value>::value
+    )
+    {
+        // check that passed value is valid
+        other.assert_invariant();
+
+        using std::swap;
+        swap(m_type, other.m_type);
+        swap(m_value, other.m_value);
+
+        assert_invariant();
+        return *this;
+    }
+
+    /*!
+    @brief destructor
+
+    Destroys the JSON value and frees all allocated memory.
+
+    @complexity Linear.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is linear.
+    - All stored elements are destroyed and all memory is freed.
+
+    @since version 1.0.0
+    */
+    ~basic_json()
+    {
+        assert_invariant();
+        m_value.destroy(m_type);
+    }
+
+    /// @}
+
+  public:
+    ///////////////////////
+    // object inspection //
+    ///////////////////////
+
+    /// @name object inspection
+    /// Functions to inspect the type of a JSON value.
+    /// @{
+
+    /*!
+    @brief serialization
+
+    Serialization function for JSON values. The function tries to mimic
+    Python's `json.dumps()` function, and currently supports its @a indent
+    and @a ensure_ascii parameters.
+
+    @param[in] indent If indent is nonnegative, then array elements and object
+    members will be pretty-printed with that indent level. An indent level of
+    `0` will only insert newlines. `-1` (the default) selects the most compact
+    representation.
+    @param[in] indent_char The character to use for indentation if @a indent is
+    greater than `0`. The default is ` ` (space).
+    @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
+    in the output are escaped with \uXXXX sequences, and the result consists
+    of ASCII characters only.
+
+    @return string containing the serialization of the JSON value
+
+    @complexity Linear.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @liveexample{The following example shows the effect of different @a indent\,
+    @a indent_char\, and @a ensure_ascii parameters to the result of the
+    serialization.,dump}
+
+    @see https://docs.python.org/2/library/json.html#json.dump
+
+    @since version 1.0.0; indentation character @a indent_char and option
+           @a ensure_ascii added in version 3.0.0
+    */
+    string_t dump(const int indent = -1, const char indent_char = ' ',
+                  const bool ensure_ascii = false) const
+    {
+        string_t result;
+        serializer s(detail::output_adapter<char>(result), indent_char);
+
+        if (indent >= 0)
+        {
+            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
+        }
+        else
+        {
+            s.dump(*this, false, ensure_ascii, 0);
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief return the type of the JSON value (explicit)
+
+    Return the type of the JSON value as a value from the @ref value_t
+    enumeration.
+
+    @return the type of the JSON value
+            Value type                | return value
+            ------------------------- | -------------------------
+            null                      | value_t::null
+            boolean                   | value_t::boolean
+            string                    | value_t::string
+            number (integer)          | value_t::number_integer
+            number (unsigned integer) | value_t::number_unsigned
+            number (foating-point)    | value_t::number_float
+            object                    | value_t::object
+            array                     | value_t::array
+            discarded                 | value_t::discarded
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `type()` for all JSON
+    types.,type}
+
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
+    @sa @ref type_name() -- return the type as string
+
+    @since version 1.0.0
+    */
+    constexpr value_t type() const noexcept
+    {
+        return m_type;
+    }
+
+    /*!
+    @brief return whether type is primitive
+
+    This function returns true if and only if the JSON type is primitive
+    (string, number, boolean, or null).
+
+    @return `true` if type is primitive (string, number, boolean, or null),
+    `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_primitive()` for all JSON
+    types.,is_primitive}
+
+    @sa @ref is_structured() -- returns whether JSON value is structured
+    @sa @ref is_null() -- returns whether JSON value is `null`
+    @sa @ref is_string() -- returns whether JSON value is a string
+    @sa @ref is_boolean() -- returns whether JSON value is a boolean
+    @sa @ref is_number() -- returns whether JSON value is a number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_primitive() const noexcept
+    {
+        return is_null() or is_string() or is_boolean() or is_number();
+    }
+
+    /*!
+    @brief return whether type is structured
+
+    This function returns true if and only if the JSON type is structured
+    (array or object).
+
+    @return `true` if type is structured (array or object), `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_structured()` for all JSON
+    types.,is_structured}
+
+    @sa @ref is_primitive() -- returns whether value is primitive
+    @sa @ref is_array() -- returns whether value is an array
+    @sa @ref is_object() -- returns whether value is an object
+
+    @since version 1.0.0
+    */
+    constexpr bool is_structured() const noexcept
+    {
+        return is_array() or is_object();
+    }
+
+    /*!
+    @brief return whether value is null
+
+    This function returns true if and only if the JSON value is null.
+
+    @return `true` if type is null, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_null()` for all JSON
+    types.,is_null}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_null() const noexcept
+    {
+        return (m_type == value_t::null);
+    }
+
+    /*!
+    @brief return whether value is a boolean
+
+    This function returns true if and only if the JSON value is a boolean.
+
+    @return `true` if type is boolean, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_boolean()` for all JSON
+    types.,is_boolean}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_boolean() const noexcept
+    {
+        return (m_type == value_t::boolean);
+    }
+
+    /*!
+    @brief return whether value is a number
+
+    This function returns true if and only if the JSON value is a number. This
+    includes both integer (signed and unsigned) and floating-point values.
+
+    @return `true` if type is number (regardless whether integer, unsigned
+    integer or floating-type), `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number()` for all JSON
+    types.,is_number}
+
+    @sa @ref is_number_integer() -- check if value is an integer or unsigned
+    integer number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number() const noexcept
+    {
+        return is_number_integer() or is_number_float();
+    }
+
+    /*!
+    @brief return whether value is an integer number
+
+    This function returns true if and only if the JSON value is a signed or
+    unsigned integer number. This excludes floating-point values.
+
+    @return `true` if type is an integer or unsigned integer number, `false`
+    otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_integer()` for all
+    JSON types.,is_number_integer}
+
+    @sa @ref is_number() -- check if value is a number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number_integer() const noexcept
+    {
+        return (m_type == value_t::number_integer or m_type == value_t::number_unsigned);
+    }
+
+    /*!
+    @brief return whether value is an unsigned integer number
+
+    This function returns true if and only if the JSON value is an unsigned
+    integer number. This excludes floating-point and signed integer values.
+
+    @return `true` if type is an unsigned integer number, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_unsigned()` for all
+    JSON types.,is_number_unsigned}
+
+    @sa @ref is_number() -- check if value is a number
+    @sa @ref is_number_integer() -- check if value is an integer or unsigned
+    integer number
+    @sa @ref is_number_float() -- check if value is a floating-point number
+
+    @since version 2.0.0
+    */
+    constexpr bool is_number_unsigned() const noexcept
+    {
+        return (m_type == value_t::number_unsigned);
+    }
+
+    /*!
+    @brief return whether value is a floating-point number
+
+    This function returns true if and only if the JSON value is a
+    floating-point number. This excludes signed and unsigned integer values.
+
+    @return `true` if type is a floating-point number, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_number_float()` for all
+    JSON types.,is_number_float}
+
+    @sa @ref is_number() -- check if value is number
+    @sa @ref is_number_integer() -- check if value is an integer number
+    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
+    number
+
+    @since version 1.0.0
+    */
+    constexpr bool is_number_float() const noexcept
+    {
+        return (m_type == value_t::number_float);
+    }
+
+    /*!
+    @brief return whether value is an object
+
+    This function returns true if and only if the JSON value is an object.
+
+    @return `true` if type is object, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_object()` for all JSON
+    types.,is_object}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_object() const noexcept
+    {
+        return (m_type == value_t::object);
+    }
+
+    /*!
+    @brief return whether value is an array
+
+    This function returns true if and only if the JSON value is an array.
+
+    @return `true` if type is array, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_array()` for all JSON
+    types.,is_array}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_array() const noexcept
+    {
+        return (m_type == value_t::array);
+    }
+
+    /*!
+    @brief return whether value is a string
+
+    This function returns true if and only if the JSON value is a string.
+
+    @return `true` if type is string, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_string()` for all JSON
+    types.,is_string}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_string() const noexcept
+    {
+        return (m_type == value_t::string);
+    }
+
+    /*!
+    @brief return whether value is discarded
+
+    This function returns true if and only if the JSON value was discarded
+    during parsing with a callback function (see @ref parser_callback_t).
+
+    @note This function will always be `false` for JSON values after parsing.
+    That is, discarded values can only occur during parsing, but will be
+    removed when inside a structured value or replaced by null in other cases.
+
+    @return `true` if type is discarded, `false` otherwise.
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies `is_discarded()` for all JSON
+    types.,is_discarded}
+
+    @since version 1.0.0
+    */
+    constexpr bool is_discarded() const noexcept
+    {
+        return (m_type == value_t::discarded);
+    }
+
+    /*!
+    @brief return the type of the JSON value (implicit)
+
+    Implicitly return the type of the JSON value as a value from the @ref
+    value_t enumeration.
+
+    @return the type of the JSON value
+
+    @complexity Constant.
+
+    @exceptionsafety No-throw guarantee: this member function never throws
+    exceptions.
+
+    @liveexample{The following code exemplifies the @ref value_t operator for
+    all JSON types.,operator__value_t}
+
+    @sa @ref type() -- return the type of the JSON value (explicit)
+    @sa @ref type_name() -- return the type as string
+
+    @since version 1.0.0
+    */
+    constexpr operator value_t() const noexcept
+    {
+        return m_type;
+    }
+
+    /// @}
+
+  private:
+    //////////////////
+    // value access //
+    //////////////////
+
+    /// get a boolean (explicit)
+    boolean_t get_impl(boolean_t* /*unused*/) const
+    {
+        if (JSON_LIKELY(is_boolean()))
+        {
+            return m_value.boolean;
+        }
+
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name())));
+    }
+
+    /// get a pointer to the value (object)
+    object_t* get_impl_ptr(object_t* /*unused*/) noexcept
+    {
+        return is_object() ? m_value.object : nullptr;
+    }
+
+    /// get a pointer to the value (object)
+    constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept
+    {
+        return is_object() ? m_value.object : nullptr;
+    }
+
+    /// get a pointer to the value (array)
+    array_t* get_impl_ptr(array_t* /*unused*/) noexcept
+    {
+        return is_array() ? m_value.array : nullptr;
+    }
+
+    /// get a pointer to the value (array)
+    constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept
+    {
+        return is_array() ? m_value.array : nullptr;
+    }
+
+    /// get a pointer to the value (string)
+    string_t* get_impl_ptr(string_t* /*unused*/) noexcept
+    {
+        return is_string() ? m_value.string : nullptr;
+    }
+
+    /// get a pointer to the value (string)
+    constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept
+    {
+        return is_string() ? m_value.string : nullptr;
+    }
+
+    /// get a pointer to the value (boolean)
+    boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept
+    {
+        return is_boolean() ? &m_value.boolean : nullptr;
+    }
+
+    /// get a pointer to the value (boolean)
+    constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept
+    {
+        return is_boolean() ? &m_value.boolean : nullptr;
+    }
+
+    /// get a pointer to the value (integer number)
+    number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept
+    {
+        return is_number_integer() ? &m_value.number_integer : nullptr;
+    }
+
+    /// get a pointer to the value (integer number)
+    constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept
+    {
+        return is_number_integer() ? &m_value.number_integer : nullptr;
+    }
+
+    /// get a pointer to the value (unsigned number)
+    number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept
+    {
+        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
+    }
+
+    /// get a pointer to the value (unsigned number)
+    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept
+    {
+        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
+    }
+
+    /// get a pointer to the value (floating-point number)
+    number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept
+    {
+        return is_number_float() ? &m_value.number_float : nullptr;
+    }
+
+    /// get a pointer to the value (floating-point number)
+    constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept
+    {
+        return is_number_float() ? &m_value.number_float : nullptr;
+    }
+
+    /*!
+    @brief helper function to implement get_ref()
+
+    This function helps to implement get_ref() without code duplication for
+    const and non-const overloads
+
+    @tparam ThisType will be deduced as `basic_json` or `const basic_json`
+
+    @throw type_error.303 if ReferenceType does not match underlying value
+    type of the current JSON
+    */
+    template<typename ReferenceType, typename ThisType>
+    static ReferenceType get_ref_impl(ThisType& obj)
+    {
+        // delegate the call to get_ptr<>()
+        auto ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();
+
+        if (JSON_LIKELY(ptr != nullptr))
+        {
+            return *ptr;
+        }
+
+        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name())));
+    }
+
+  public:
+    /// @name value access
+    /// Direct access to the stored value of a JSON value.
+    /// @{
+
+    /*!
+    @brief get special-case overload
+
+    This overloads avoids a lot of template boilerplate, it can be seen as the
+    identity method
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this
+
+    @complexity Constant.
+
+    @since version 2.1.0
+    */
+    template <
+        typename BasicJsonType,
+        detail::enable_if_t<std::is_same<typename std::remove_const<BasicJsonType>::type,
+                                         basic_json_t>::value,
+                            int> = 0 >
+    basic_json get() const
+    {
+        return *this;
+    }
+
+    /*!
+    @brief get a value (explicit)
+
+    Explicit type conversion between the JSON value and a compatible value
+    which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
+    and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType ret;
+    JSONSerializer<ValueType>::from_json(*this, ret);
+    return ret;
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const basic_json&, ValueType&)`, and
+    - @ref json_serializer<ValueType> does not have a `from_json()` method of
+      the form `ValueType from_json(const basic_json&)`
+
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
+
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,get__ValueType_const}
+
+    @since version 2.1.0
+    */
+    template <
+        typename ValueTypeCV,
+        typename ValueType = detail::uncvref_t<ValueTypeCV>,
+        detail::enable_if_t <
+            not std::is_same<basic_json_t, ValueType>::value and
+            detail::has_from_json<basic_json_t, ValueType>::value and
+            not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+            int > = 0 >
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
+    {
+        // we cannot static_assert on ValueTypeCV being non-const, because
+        // there is support for get<const basic_json_t>(), which is why we
+        // still need the uncvref
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        static_assert(std::is_default_constructible<ValueType>::value,
+                      "types must be DefaultConstructible when used with get()");
+
+        ValueType ret;
+        JSONSerializer<ValueType>::from_json(*this, ret);
+        return ret;
+    }
+
+    /*!
+    @brief get a value (explicit); special case
+
+    Explicit type conversion between the JSON value and a compatible value
+    which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
+    and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    return JSONSerializer<ValueTypeCV>::from_json(*this);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json and
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `ValueType from_json(const basic_json&)`
+
+    @note If @ref json_serializer<ValueType> has both overloads of
+    `from_json()`, this one is chosen.
+
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
+
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @since version 2.1.0
+    */
+    template <
+        typename ValueTypeCV,
+        typename ValueType = detail::uncvref_t<ValueTypeCV>,
+        detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
+                            detail::has_non_default_from_json<basic_json_t,
+                                    ValueType>::value, int> = 0 >
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>())))
+    {
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        return JSONSerializer<ValueTypeCV>::from_json(*this);
+    }
+
+    /*!
+    @brief get a pointer value (explicit)
+
+    Explicit pointer access to the internally stored JSON value. No copies are
+    made.
+
+    @warning The pointer becomes invalid if the underlying JSON object
+    changes.
+
+    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
+    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
+    @ref number_unsigned_t, or @ref number_float_t.
+
+    @return pointer to the internally stored JSON value if the requested
+    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how pointers to internal values of a
+    JSON value can be requested. Note that no type conversions are made and a
+    `nullptr` is returned if the value and the requested pointer type does not
+    match.,get__PointerType}
+
+    @sa @ref get_ptr() for explicit pointer-member access
+
+    @since version 1.0.0
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    PointerType get() noexcept
+    {
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
+    }
+
+    /*!
+    @brief get a pointer value (explicit)
+    @copydoc get()
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    constexpr const PointerType get() const noexcept
+    {
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
+    }
+
+    /*!
+    @brief get a pointer value (implicit)
+
+    Implicit pointer access to the internally stored JSON value. No copies are
+    made.
+
+    @warning Writing data to the pointee of the result yields an undefined
+    state.
+
+    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
+    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
+    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
+    assertion.
+
+    @return pointer to the internally stored JSON value if the requested
+    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how pointers to internal values of a
+    JSON value can be requested. Note that no type conversions are made and a
+    `nullptr` is returned if the value and the requested pointer type does not
+    match.,get_ptr}
+
+    @since version 1.0.0
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    PointerType get_ptr() noexcept
+    {
+        // get the type of the PointerType (remove pointer and const)
+        using pointee_t = typename std::remove_const<typename
+                          std::remove_pointer<typename
+                          std::remove_const<PointerType>::type>::type>::type;
+        // make sure the type matches the allowed types
+        static_assert(
+            std::is_same<object_t, pointee_t>::value
+            or std::is_same<array_t, pointee_t>::value
+            or std::is_same<string_t, pointee_t>::value
+            or std::is_same<boolean_t, pointee_t>::value
+            or std::is_same<number_integer_t, pointee_t>::value
+            or std::is_same<number_unsigned_t, pointee_t>::value
+            or std::is_same<number_float_t, pointee_t>::value
+            , "incompatible pointer type");
+
+        // delegate the call to get_impl_ptr<>()
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
+    }
+
+    /*!
+    @brief get a pointer value (implicit)
+    @copydoc get_ptr()
+    */
+    template<typename PointerType, typename std::enable_if<
+                 std::is_pointer<PointerType>::value and
+                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
+    constexpr const PointerType get_ptr() const noexcept
+    {
+        // get the type of the PointerType (remove pointer and const)
+        using pointee_t = typename std::remove_const<typename
+                          std::remove_pointer<typename
+                          std::remove_const<PointerType>::type>::type>::type;
+        // make sure the type matches the allowed types
+        static_assert(
+            std::is_same<object_t, pointee_t>::value
+            or std::is_same<array_t, pointee_t>::value
+            or std::is_same<string_t, pointee_t>::value
+            or std::is_same<boolean_t, pointee_t>::value
+            or std::is_same<number_integer_t, pointee_t>::value
+            or std::is_same<number_unsigned_t, pointee_t>::value
+            or std::is_same<number_float_t, pointee_t>::value
+            , "incompatible pointer type");
+
+        // delegate the call to get_impl_ptr<>() const
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
+    }
+
+    /*!
+    @brief get a reference value (implicit)
+
+    Implicit reference access to the internally stored JSON value. No copies
+    are made.
+
+    @warning Writing data to the referee of the result yields an undefined
+    state.
+
+    @tparam ReferenceType reference type; must be a reference to @ref array_t,
+    @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or
+    @ref number_float_t. Enforced by static assertion.
+
+    @return reference to the internally stored JSON value if the requested
+    reference type @a ReferenceType fits to the JSON value; throws
+    type_error.303 otherwise
+
+    @throw type_error.303 in case passed type @a ReferenceType is incompatible
+    with the stored JSON value; see example below
+
+    @complexity Constant.
+
+    @liveexample{The example shows several calls to `get_ref()`.,get_ref}
+
+    @since version 1.1.0
+    */
+    template<typename ReferenceType, typename std::enable_if<
+                 std::is_reference<ReferenceType>::value, int>::type = 0>
+    ReferenceType get_ref()
+    {
+        // delegate call to get_ref_impl
+        return get_ref_impl<ReferenceType>(*this);
+    }
+
+    /*!
+    @brief get a reference value (implicit)
+    @copydoc get_ref()
+    */
+    template<typename ReferenceType, typename std::enable_if<
+                 std::is_reference<ReferenceType>::value and
+                 std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int>::type = 0>
+    ReferenceType get_ref() const
+    {
+        // delegate call to get_ref_impl
+        return get_ref_impl<ReferenceType>(*this);
+    }
+
+    /*!
+    @brief get a value (implicit)
+
+    Implicit type conversion between the JSON value and a compatible value.
+    The call is realized by calling @ref get() const.
+
+    @tparam ValueType non-pointer type compatible to the JSON value, for
+    instance `int` for JSON integer numbers, `bool` for JSON booleans, or
+    `std::vector` types for JSON arrays. The character type of @ref string_t
+    as well as an initializer list of this type is excluded to avoid
+    ambiguities as these types implicitly convert to `std::string`.
+
+    @return copy of the JSON value, converted to type @a ValueType
+
+    @throw type_error.302 in case passed type @a ValueType is incompatible
+    to the JSON value type (e.g., the JSON value is of type boolean, but a
+    string is requested); see example below
+
+    @complexity Linear in the size of the JSON value.
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,operator__ValueType}
+
+    @since version 1.0.0
+    */
+    template < typename ValueType, typename std::enable_if <
+                   not std::is_pointer<ValueType>::value and
+                   not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
+                   not std::is_same<ValueType, typename string_t::value_type>::value
+#ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
+                   and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
+#endif
+#if defined(JSON_HAS_CPP_17)
+                   and not std::is_same<ValueType, typename std::string_view>::value
+#endif
+                   , int >::type = 0 >
+    operator ValueType() const
+    {
+        // delegate the call to get<>() const
+        return get<ValueType>();
+    }
+
+    /// @}
+
+
+    ////////////////////
+    // element access //
+    ////////////////////
+
+    /// @name element access
+    /// Access to the JSON value.
+    /// @{
+
+    /*!
+    @brief access specified array element with bounds checking
+
+    Returns a reference to the element at specified location @a idx, with
+    bounds checking.
+
+    @param[in] idx  index of the element to access
+
+    @return reference to the element at index @a idx
+
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__size_type}
+    */
+    reference at(size_type idx)
+    {
+        // at only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            JSON_TRY
+            {
+                return m_value.array->at(idx);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified array element with bounds checking
+
+    Returns a const reference to the element at specified location @a idx,
+    with bounds checking.
+
+    @param[in] idx  index of the element to access
+
+    @return const reference to the element at index @a idx
+
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__size_type_const}
+    */
+    const_reference at(size_type idx) const
+    {
+        // at only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            JSON_TRY
+            {
+                return m_value.array->at(idx);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified object element with bounds checking
+
+    Returns a reference to the element at with specified key @a key, with
+    bounds checking.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Logarithmic in the size of the container.
+
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__object_t_key_type}
+    */
+    reference at(const typename object_t::key_type& key)
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            JSON_TRY
+            {
+                return m_value.object->at(key);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified object element with bounds checking
+
+    Returns a const reference to the element at with specified key @a key,
+    with bounds checking.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Logarithmic in the size of the container.
+
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__object_t_key_type_const}
+    */
+    const_reference at(const typename object_t::key_type& key) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            JSON_TRY
+            {
+                return m_value.object->at(key);
+            }
+            JSON_CATCH (std::out_of_range&)
+            {
+                // create better exception explanation
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
+            }
+        }
+        else
+        {
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief access specified array element
+
+    Returns a reference to the element at specified location @a idx.
+
+    @note If @a idx is beyond the range of the array (i.e., `idx >= size()`),
+    then the array is silently filled up with `null` values to make `idx` a
+    valid reference to the last stored element.
+
+    @param[in] idx  index of the element to access
+
+    @return reference to the element at index @a idx
+
+    @throw type_error.305 if the JSON value is not an array or null; in that
+    cases, using the [] operator with an index makes no sense.
+
+    @complexity Constant if @a idx is in the range of the array. Otherwise
+    linear in `idx - size()`.
+
+    @liveexample{The example below shows how array elements can be read and
+    written using `[]` operator. Note the addition of `null`
+    values.,operatorarray__size_type}
+
+    @since version 1.0.0
+    */
+    reference operator[](size_type idx)
+    {
+        // implicitly convert null value to an empty array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value.array = create<array_t>();
+            assert_invariant();
+        }
+
+        // operator[] only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // fill up array with null values if given idx is outside range
+            if (idx >= m_value.array->size())
+            {
+                m_value.array->insert(m_value.array->end(),
+                                      idx - m_value.array->size() + 1,
+                                      basic_json());
+            }
+
+            return m_value.array->operator[](idx);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified array element
+
+    Returns a const reference to the element at specified location @a idx.
+
+    @param[in] idx  index of the element to access
+
+    @return const reference to the element at index @a idx
+
+    @throw type_error.305 if the JSON value is not an array; in that cases,
+    using the [] operator with an index makes no sense.
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how array elements can be read using
+    the `[]` operator.,operatorarray__size_type_const}
+
+    @since version 1.0.0
+    */
+    const_reference operator[](size_type idx) const
+    {
+        // const operator[] only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            return m_value.array->operator[](idx);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element
+
+    Returns a reference to the element at with specified key @a key.
+
+    @note If @a key is not found in the object, then it is silently added to
+    the object and filled with a `null` value to make `key` a valid reference.
+    In case the value was `null` before, it is converted to an object.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read and
+    written using the `[]` operator.,operatorarray__key_type}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+    */
+    reference operator[](const typename object_t::key_type& key)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        // operator[] only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->operator[](key);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief read-only access specified object element
+
+    Returns a const reference to the element at with specified key @a key. No
+    bounds checking is performed.
+
+    @warning If the element with key @a key does not exist, the behavior is
+    undefined.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @pre The element with key @a key must exist. **This precondition is
+         enforced with an assertion.**
+
+    @throw type_error.305 if the JSON value is not an object; in that cases,
+    using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read using
+    the `[]` operator.,operatorarray__key_type_const}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.0.0
+    */
+    const_reference operator[](const typename object_t::key_type& key) const
+    {
+        // const operator[] only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            assert(m_value.object->find(key) != m_value.object->end());
+            return m_value.object->find(key)->second;
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element
+
+    Returns a reference to the element at with specified key @a key.
+
+    @note If @a key is not found in the object, then it is silently added to
+    the object and filled with a `null` value to make `key` a valid reference.
+    In case the value was `null` before, it is converted to an object.
+
+    @param[in] key  key of the element to access
+
+    @return reference to the element at key @a key
+
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read and
+    written using the `[]` operator.,operatorarray__key_type}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.1.0
+    */
+    template<typename T>
+    reference operator[](T* key)
+    {
+        // implicitly convert null to object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->operator[](key);
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief read-only access specified object element
+
+    Returns a const reference to the element at with specified key @a key. No
+    bounds checking is performed.
+
+    @warning If the element with key @a key does not exist, the behavior is
+    undefined.
+
+    @param[in] key  key of the element to access
+
+    @return const reference to the element at key @a key
+
+    @pre The element with key @a key must exist. **This precondition is
+         enforced with an assertion.**
+
+    @throw type_error.305 if the JSON value is not an object; in that cases,
+    using the [] operator with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be read using
+    the `[]` operator.,operatorarray__key_type_const}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref value() for access by value with a default value
+
+    @since version 1.1.0
+    */
+    template<typename T>
+    const_reference operator[](T* key) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            assert(m_value.object->find(key) != m_value.object->end());
+            return m_value.object->find(key)->second;
+        }
+
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief access specified object element with default value
+
+    Returns either a copy of an object's element at the specified key @a key
+    or a given default value if no element with key @a key exists.
+
+    The function is basically equivalent to executing
+    @code {.cpp}
+    try {
+        return at(key);
+    } catch(out_of_range) {
+        return default_value;
+    }
+    @endcode
+
+    @note Unlike @ref at(const typename object_t::key_type&), this function
+    does not throw if the given key @a key was not found.
+
+    @note Unlike @ref operator[](const typename object_t::key_type& key), this
+    function does not implicitly add an element to the position defined by @a
+    key. This function is furthermore also applicable to const objects.
+
+    @param[in] key  key of the element to access
+    @param[in] default_value  the value to return if @a key is not found
+
+    @tparam ValueType type compatible to JSON values, for instance `int` for
+    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
+    JSON arrays. Note the type of the expected value at @a key and the default
+    value @a default_value must be compatible.
+
+    @return copy of the element at key @a key or @a default_value if @a key
+    is not found
+
+    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    using `value()` with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be queried
+    with a default value.,basic_json__value}
+
+    @sa @ref at(const typename object_t::key_type&) for access by reference
+    with range checking
+    @sa @ref operator[](const typename object_t::key_type&) for unchecked
+    access by reference
+
+    @since version 1.0.0
+    */
+    template<class ValueType, typename std::enable_if<
+                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
+    ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            // if key is found, return value and given default value otherwise
+            const auto it = find(key);
+            if (it != end())
+            {
+                return *it;
+            }
+
+            return default_value;
+        }
+
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief overload for a default value of type const char*
+    @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const
+    */
+    string_t value(const typename object_t::key_type& key, const char* default_value) const
+    {
+        return value(key, string_t(default_value));
+    }
+
+    /*!
+    @brief access specified object element via JSON Pointer with default value
+
+    Returns either a copy of an object's element at the specified key @a key
+    or a given default value if no element with key @a key exists.
+
+    The function is basically equivalent to executing
+    @code {.cpp}
+    try {
+        return at(ptr);
+    } catch(out_of_range) {
+        return default_value;
+    }
+    @endcode
+
+    @note Unlike @ref at(const json_pointer&), this function does not throw
+    if the given key @a key was not found.
+
+    @param[in] ptr  a JSON pointer to the element to access
+    @param[in] default_value  the value to return if @a ptr found no value
+
+    @tparam ValueType type compatible to JSON values, for instance `int` for
+    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
+    JSON arrays. Note the type of the expected value at @a key and the default
+    value @a default_value must be compatible.
+
+    @return copy of the element at key @a key or @a default_value if @a key
+    is not found
+
+    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    using `value()` with a key makes no sense.
+
+    @complexity Logarithmic in the size of the container.
+
+    @liveexample{The example below shows how object elements can be queried
+    with a default value.,basic_json__value_ptr}
+
+    @sa @ref operator[](const json_pointer&) for unchecked access by reference
+
+    @since version 2.0.2
+    */
+    template<class ValueType, typename std::enable_if<
+                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
+    ValueType value(const json_pointer& ptr, const ValueType& default_value) const
+    {
+        // at only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            // if pointer resolves a value, return it or use default value
+            JSON_TRY
+            {
+                return ptr.get_checked(this);
+            }
+            JSON_CATCH (out_of_range&)
+            {
+                return default_value;
+            }
+        }
+
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief overload for a default value of type const char*
+    @copydoc basic_json::value(const json_pointer&, ValueType) const
+    */
+    string_t value(const json_pointer& ptr, const char* default_value) const
+    {
+        return value(ptr, string_t(default_value));
+    }
+
+    /*!
+    @brief access the first element
+
+    Returns a reference to the first element in the container. For a JSON
+    container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
+
+    @return In case of a structured type (array or object), a reference to the
+    first element is returned. In case of number, string, or boolean values, a
+    reference to the value is returned.
+
+    @complexity Constant.
+
+    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
+    or an empty array or object (undefined behavior, **guarded by
+    assertions**).
+    @post The JSON value remains unchanged.
+
+    @throw invalid_iterator.214 when called on `null` value
+
+    @liveexample{The following code shows an example for `front()`.,front}
+
+    @sa @ref back() -- access the last element
+
+    @since version 1.0.0
+    */
+    reference front()
+    {
+        return *begin();
+    }
+
+    /*!
+    @copydoc basic_json::front()
+    */
+    const_reference front() const
+    {
+        return *cbegin();
+    }
+
+    /*!
+    @brief access the last element
+
+    Returns a reference to the last element in the container. For a JSON
+    container `c`, the expression `c.back()` is equivalent to
+    @code {.cpp}
+    auto tmp = c.end();
+    --tmp;
+    return *tmp;
+    @endcode
+
+    @return In case of a structured type (array or object), a reference to the
+    last element is returned. In case of number, string, or boolean values, a
+    reference to the value is returned.
+
+    @complexity Constant.
+
+    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
+    or an empty array or object (undefined behavior, **guarded by
+    assertions**).
+    @post The JSON value remains unchanged.
+
+    @throw invalid_iterator.214 when called on a `null` value. See example
+    below.
+
+    @liveexample{The following code shows an example for `back()`.,back}
+
+    @sa @ref front() -- access the first element
+
+    @since version 1.0.0
+    */
+    reference back()
+    {
+        auto tmp = end();
+        --tmp;
+        return *tmp;
+    }
+
+    /*!
+    @copydoc basic_json::back()
+    */
+    const_reference back() const
+    {
+        auto tmp = cend();
+        --tmp;
+        return *tmp;
+    }
+
+    /*!
+    @brief remove element given an iterator
+
+    Removes the element specified by iterator @a pos. The iterator @a pos must
+    be valid and dereferenceable. Thus the `end()` iterator (which is valid,
+    but is not dereferenceable) cannot be used as a value for @a pos.
+
+    If called on a primitive type other than `null`, the resulting JSON value
+    will be `null`.
+
+    @param[in] pos iterator to the element to remove
+    @return Iterator following the last removed element. If the iterator @a
+    pos refers to the last element, the `end()` iterator is returned.
+
+    @tparam IteratorType an @ref iterator or @ref const_iterator
+
+    @post Invalidates iterators and references at or after the point of the
+    erase, including the `end()` iterator.
+
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.202 if called on an iterator which does not belong
+    to the current JSON value; example: `"iterator does not fit current
+    value"`
+    @throw invalid_iterator.205 if called on a primitive type with invalid
+    iterator (i.e., any iterator which is not `begin()`); example: `"iterator
+    out of range"`
+
+    @complexity The complexity depends on the type:
+    - objects: amortized constant
+    - arrays: linear in distance between @a pos and the end of the container
+    - strings: linear in the length of the string
+    - other types: constant
+
+    @liveexample{The example shows the result of `erase()` for different JSON
+    types.,erase__IteratorType}
+
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
+                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
+             = 0>
+    IteratorType erase(IteratorType pos)
+    {
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(this != pos.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        IteratorType result = end();
+
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_UNLIKELY(not pos.m_it.primitive_iterator.is_begin()))
+                {
+                    JSON_THROW(invalid_iterator::create(205, "iterator out of range"));
+                }
+
+                if (is_string())
+                {
+                    AllocatorType<string_t> alloc;
+                    alloc.destroy(m_value.string);
+                    alloc.deallocate(m_value.string, 1);
+                    m_value.string = nullptr;
+                }
+
+                m_type = value_t::null;
+                assert_invariant();
+                break;
+            }
+
+            case value_t::object:
+            {
+                result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief remove elements given an iterator range
+
+    Removes the element specified by the range `[first; last)`. The iterator
+    @a first does not need to be dereferenceable if `first == last`: erasing
+    an empty range is a no-op.
+
+    If called on a primitive type other than `null`, the resulting JSON value
+    will be `null`.
+
+    @param[in] first iterator to the beginning of the range to remove
+    @param[in] last iterator past the end of the range to remove
+    @return Iterator following the last removed element. If the iterator @a
+    second refers to the last element, the `end()` iterator is returned.
+
+    @tparam IteratorType an @ref iterator or @ref const_iterator
+
+    @post Invalidates iterators and references at or after the point of the
+    erase, including the `end()` iterator.
+
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.203 if called on iterators which does not belong
+    to the current JSON value; example: `"iterators do not fit current value"`
+    @throw invalid_iterator.204 if called on a primitive type with invalid
+    iterators (i.e., if `first != begin()` and `last != end()`); example:
+    `"iterators out of range"`
+
+    @complexity The complexity depends on the type:
+    - objects: `log(size()) + std::distance(first, last)`
+    - arrays: linear in the distance between @a first and @a last, plus linear
+      in the distance between @a last and end of the container
+    - strings: linear in the length of the string
+    - other types: constant
+
+    @liveexample{The example shows the result of `erase()` for different JSON
+    types.,erase__IteratorType_IteratorType}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
+                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
+             = 0>
+    IteratorType erase(IteratorType first, IteratorType last)
+    {
+        // make sure iterator fits the current value
+        if (JSON_UNLIKELY(this != first.m_object or this != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value"));
+        }
+
+        IteratorType result = end();
+
+        switch (m_type)
+        {
+            case value_t::boolean:
+            case value_t::number_float:
+            case value_t::number_integer:
+            case value_t::number_unsigned:
+            case value_t::string:
+            {
+                if (JSON_LIKELY(not first.m_it.primitive_iterator.is_begin()
+                                or not last.m_it.primitive_iterator.is_end()))
+                {
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
+                }
+
+                if (is_string())
+                {
+                    AllocatorType<string_t> alloc;
+                    alloc.destroy(m_value.string);
+                    alloc.deallocate(m_value.string, 1);
+                    m_value.string = nullptr;
+                }
+
+                m_type = value_t::null;
+                assert_invariant();
+                break;
+            }
+
+            case value_t::object:
+            {
+                result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator,
+                                              last.m_it.object_iterator);
+                break;
+            }
+
+            case value_t::array:
+            {
+                result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator,
+                                             last.m_it.array_iterator);
+                break;
+            }
+
+            default:
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief remove element from a JSON object given a key
+
+    Removes elements from a JSON object with the key value @a key.
+
+    @param[in] key value of the elements to remove
+
+    @return Number of elements removed. If @a ObjectType is the default
+    `std::map` type, the return value will always be `0` (@a key was not
+    found) or `1` (@a key was found).
+
+    @post References and iterators to the erased elements are invalidated.
+    Other references and iterators are not affected.
+
+    @throw type_error.307 when called on a type other than JSON object;
+    example: `"cannot use erase() with null"`
+
+    @complexity `log(size()) + count(key)`
+
+    @liveexample{The example shows the effect of `erase()`.,erase__key_type}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const size_type) -- removes the element from an array at
+    the given index
+
+    @since version 1.0.0
+    */
+    size_type erase(const typename object_t::key_type& key)
+    {
+        // this erase only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            return m_value.object->erase(key);
+        }
+
+        JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief remove element from a JSON array given an index
+
+    Removes element from a JSON array at the index @a idx.
+
+    @param[in] idx index of the element to remove
+
+    @throw type_error.307 when called on a type other than JSON object;
+    example: `"cannot use erase() with null"`
+    @throw out_of_range.401 when `idx >= size()`; example: `"array index 17
+    is out of range"`
+
+    @complexity Linear in distance between @a idx and the end of the container.
+
+    @liveexample{The example shows the effect of `erase()`.,erase__size_type}
+
+    @sa @ref erase(IteratorType) -- removes the element at a given position
+    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
+    the given range
+    @sa @ref erase(const typename object_t::key_type&) -- removes the element
+    from an object at the given key
+
+    @since version 1.0.0
+    */
+    void erase(const size_type idx)
+    {
+        // this erase only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            if (JSON_UNLIKELY(idx >= size()))
+            {
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+            }
+
+            m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));
+        }
+        else
+        {
+            JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
+        }
+    }
+
+    /// @}
+
+
+    ////////////
+    // lookup //
+    ////////////
+
+    /// @name lookup
+    /// @{
+
+    /*!
+    @brief find an element in a JSON object
+
+    Finds an element in a JSON object with key equivalent to @a key. If the
+    element is not found or the JSON value is not an object, end() is
+    returned.
+
+    @note This method always returns @ref end() when executed on a JSON type
+          that is not an object.
+
+    @param[in] key key value of the element to search for.
+
+    @return Iterator to an element with key equivalent to @a key. If no such
+    element is found or the JSON value is not an object, past-the-end (see
+    @ref end()) iterator is returned.
+
+    @complexity Logarithmic in the size of the JSON object.
+
+    @liveexample{The example shows how `find()` is used.,find__key_type}
+
+    @since version 1.0.0
+    */
+    template<typename KeyT>
+    iterator find(KeyT&& key)
+    {
+        auto result = end();
+
+        if (is_object())
+        {
+            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief find an element in a JSON object
+    @copydoc find(KeyT&&)
+    */
+    template<typename KeyT>
+    const_iterator find(KeyT&& key) const
+    {
+        auto result = cend();
+
+        if (is_object())
+        {
+            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief returns the number of occurrences of a key in a JSON object
+
+    Returns the number of elements with key @a key. If ObjectType is the
+    default `std::map` type, the return value will always be `0` (@a key was
+    not found) or `1` (@a key was found).
+
+    @note This method always returns `0` when executed on a JSON type that is
+          not an object.
+
+    @param[in] key key value of the element to count
+
+    @return Number of elements with key @a key. If the JSON value is not an
+    object, the return value will be `0`.
+
+    @complexity Logarithmic in the size of the JSON object.
+
+    @liveexample{The example shows how `count()` is used.,count}
+
+    @since version 1.0.0
+    */
+    template<typename KeyT>
+    size_type count(KeyT&& key) const
+    {
+        // return 0 for all nonobject types
+        return is_object() ? m_value.object->count(std::forward<KeyT>(key)) : 0;
+    }
+
+    /// @}
+
+
+    ///////////////
+    // iterators //
+    ///////////////
+
+    /// @name iterators
+    /// @{
+
+    /*!
+    @brief returns an iterator to the first element
+
+    Returns an iterator to the first element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return iterator to the first element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+
+    @liveexample{The following code shows an example for `begin()`.,begin}
+
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref cend() -- returns a const iterator to the end
+
+    @since version 1.0.0
+    */
+    iterator begin() noexcept
+    {
+        iterator result(this);
+        result.set_begin();
+        return result;
+    }
+
+    /*!
+    @copydoc basic_json::cbegin()
+    */
+    const_iterator begin() const noexcept
+    {
+        return cbegin();
+    }
+
+    /*!
+    @brief returns a const iterator to the first element
+
+    Returns a const iterator to the first element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return const iterator to the first element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).begin()`.
+
+    @liveexample{The following code shows an example for `cbegin()`.,cbegin}
+
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref cend() -- returns a const iterator to the end
+
+    @since version 1.0.0
+    */
+    const_iterator cbegin() const noexcept
+    {
+        const_iterator result(this);
+        result.set_begin();
+        return result;
+    }
+
+    /*!
+    @brief returns an iterator to one past the last element
+
+    Returns an iterator to one past the last element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return iterator one past the last element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+
+    @liveexample{The following code shows an example for `end()`.,end}
+
+    @sa @ref cend() -- returns a const iterator to the end
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+
+    @since version 1.0.0
+    */
+    iterator end() noexcept
+    {
+        iterator result(this);
+        result.set_end();
+        return result;
+    }
+
+    /*!
+    @copydoc basic_json::cend()
+    */
+    const_iterator end() const noexcept
+    {
+        return cend();
+    }
+
+    /*!
+    @brief returns a const iterator to one past the last element
+
+    Returns a const iterator to one past the last element.
+
+    @image html range-begin-end.svg "Illustration from cppreference.com"
+
+    @return const iterator one past the last element
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).end()`.
+
+    @liveexample{The following code shows an example for `cend()`.,cend}
+
+    @sa @ref end() -- returns an iterator to the end
+    @sa @ref begin() -- returns an iterator to the beginning
+    @sa @ref cbegin() -- returns a const iterator to the beginning
+
+    @since version 1.0.0
+    */
+    const_iterator cend() const noexcept
+    {
+        const_iterator result(this);
+        result.set_end();
+        return result;
+    }
+
+    /*!
+    @brief returns an iterator to the reverse-beginning
+
+    Returns an iterator to the reverse-beginning; that is, the last element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `reverse_iterator(end())`.
+
+    @liveexample{The following code shows an example for `rbegin()`.,rbegin}
+
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref crend() -- returns a const reverse iterator to the end
+
+    @since version 1.0.0
+    */
+    reverse_iterator rbegin() noexcept
+    {
+        return reverse_iterator(end());
+    }
+
+    /*!
+    @copydoc basic_json::crbegin()
+    */
+    const_reverse_iterator rbegin() const noexcept
+    {
+        return crbegin();
+    }
+
+    /*!
+    @brief returns an iterator to the reverse-end
+
+    Returns an iterator to the reverse-end; that is, one before the first
+    element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `reverse_iterator(begin())`.
+
+    @liveexample{The following code shows an example for `rend()`.,rend}
+
+    @sa @ref crend() -- returns a const reverse iterator to the end
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+
+    @since version 1.0.0
+    */
+    reverse_iterator rend() noexcept
+    {
+        return reverse_iterator(begin());
+    }
+
+    /*!
+    @copydoc basic_json::crend()
+    */
+    const_reverse_iterator rend() const noexcept
+    {
+        return crend();
+    }
+
+    /*!
+    @brief returns a const reverse iterator to the last element
+
+    Returns a const iterator to the reverse-beginning; that is, the last
+    element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`.
+
+    @liveexample{The following code shows an example for `crbegin()`.,crbegin}
+
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref crend() -- returns a const reverse iterator to the end
+
+    @since version 1.0.0
+    */
+    const_reverse_iterator crbegin() const noexcept
+    {
+        return const_reverse_iterator(cend());
+    }
+
+    /*!
+    @brief returns a const reverse iterator to one before the first
+
+    Returns a const reverse iterator to the reverse-end; that is, one before
+    the first element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
+
+    @liveexample{The following code shows an example for `crend()`.,crend}
+
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+
+    @since version 1.0.0
+    */
+    const_reverse_iterator crend() const noexcept
+    {
+        return const_reverse_iterator(cbegin());
+    }
+
+  public:
+    /*!
+    @brief wrapper to access iterator member functions in range-based for
+
+    This function allows to access @ref iterator::key() and @ref
+    iterator::value() during range-based for loops. In these loops, a
+    reference to the JSON values is returned, so there is no access to the
+    underlying iterator.
+
+    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
+
+    @note The name of this function is not yet final and may change in the
+    future.
+    */
+    static iteration_proxy<iterator> iterator_wrapper(reference cont)
+    {
+        return iteration_proxy<iterator>(cont);
+    }
+
+    /*!
+    @copydoc iterator_wrapper(reference)
+    */
+    static iteration_proxy<const_iterator> iterator_wrapper(const_reference cont)
+    {
+        return iteration_proxy<const_iterator>(cont);
+    }
+
+    /// @}
+
+
+    //////////////
+    // capacity //
+    //////////////
+
+    /// @name capacity
+    /// @{
+
+    /*!
+    @brief checks whether the container is empty.
+
+    Checks if a JSON value has no elements (i.e. whether its @ref size is `0`).
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `true`
+            boolean     | `false`
+            string      | `false`
+            number      | `false`
+            object      | result of function `object_t::empty()`
+            array       | result of function `array_t::empty()`
+
+    @liveexample{The following code uses `empty()` to check if a JSON
+    object contains any elements.,empty}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their `empty()` functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return whether a string stored as JSON value
+    is empty - it returns whether the JSON container itself is empty which is
+    false in the case of a string.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `begin() == end()`.
+
+    @sa @ref size() -- returns the number of elements
+
+    @since version 1.0.0
+    */
+    bool empty() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::null:
+            {
+                // null values are empty
+                return true;
+            }
+
+            case value_t::array:
+            {
+                // delegate call to array_t::empty()
+                return m_value.array->empty();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::empty()
+                return m_value.object->empty();
+            }
+
+            default:
+            {
+                // all other types are nonempty
+                return false;
+            }
+        }
+    }
+
+    /*!
+    @brief returns the number of elements
+
+    Returns the number of elements in a JSON value.
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `0`
+            boolean     | `1`
+            string      | `1`
+            number      | `1`
+            object      | result of function object_t::size()
+            array       | result of function array_t::size()
+
+    @liveexample{The following code calls `size()` on the different value
+    types.,size}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their size() functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return the length of a string stored as JSON
+    value - it returns the number of elements in the JSON value which is 1 in
+    the case of a string.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `std::distance(begin(), end())`.
+
+    @sa @ref empty() -- checks whether the container is empty
+    @sa @ref max_size() -- returns the maximal number of elements
+
+    @since version 1.0.0
+    */
+    size_type size() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::null:
+            {
+                // null values are empty
+                return 0;
+            }
+
+            case value_t::array:
+            {
+                // delegate call to array_t::size()
+                return m_value.array->size();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::size()
+                return m_value.object->size();
+            }
+
+            default:
+            {
+                // all other types have size 1
+                return 1;
+            }
+        }
+    }
+
+    /*!
+    @brief returns the maximum possible number of elements
+
+    Returns the maximum number of elements a JSON value is able to hold due to
+    system or library implementation limitations, i.e. `std::distance(begin(),
+    end())` for the JSON value.
+
+    @return The return value depends on the different types and is
+            defined as follows:
+            Value type  | return value
+            ----------- | -------------
+            null        | `0` (same as `size()`)
+            boolean     | `1` (same as `size()`)
+            string      | `1` (same as `size()`)
+            number      | `1` (same as `size()`)
+            object      | result of function `object_t::max_size()`
+            array       | result of function `array_t::max_size()`
+
+    @liveexample{The following code calls `max_size()` on the different value
+    types. Note the output is implementation specific.,max_size}
+
+    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
+    the Container concept; that is, their `max_size()` functions have constant
+    complexity.
+
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @requirement This function helps `basic_json` satisfying the
+    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of returning `b.size()` where `b` is the largest
+      possible JSON value.
+
+    @sa @ref size() -- returns the number of elements
+
+    @since version 1.0.0
+    */
+    size_type max_size() const noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::array:
+            {
+                // delegate call to array_t::max_size()
+                return m_value.array->max_size();
+            }
+
+            case value_t::object:
+            {
+                // delegate call to object_t::max_size()
+                return m_value.object->max_size();
+            }
+
+            default:
+            {
+                // all other types have max_size() == size()
+                return size();
+            }
+        }
+    }
+
+    /// @}
+
+
+    ///////////////
+    // modifiers //
+    ///////////////
+
+    /// @name modifiers
+    /// @{
+
+    /*!
+    @brief clears the contents
+
+    Clears the content of a JSON value and resets it to the default value as
+    if @ref basic_json(value_t) would have been called with the current value
+    type from @ref type():
+
+    Value type  | initial value
+    ----------- | -------------
+    null        | `null`
+    boolean     | `false`
+    string      | `""`
+    number      | `0`
+    object      | `{}`
+    array       | `[]`
+
+    @post Has the same effect as calling
+    @code {.cpp}
+    *this = basic_json(type());
+    @endcode
+
+    @liveexample{The example below shows the effect of `clear()` to different
+    JSON types.,clear}
+
+    @complexity Linear in the size of the JSON value.
+
+    @iterators All iterators, pointers and references related to this container
+               are invalidated.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @sa @ref basic_json(value_t) -- constructor that creates an object with the
+        same value than calling `clear()`
+
+    @since version 1.0.0
+    */
+    void clear() noexcept
+    {
+        switch (m_type)
+        {
+            case value_t::number_integer:
+            {
+                m_value.number_integer = 0;
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                m_value.number_unsigned = 0;
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                m_value.number_float = 0.0;
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                m_value.boolean = false;
+                break;
+            }
+
+            case value_t::string:
+            {
+                m_value.string->clear();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_value.array->clear();
+                break;
+            }
+
+            case value_t::object:
+            {
+                m_value.object->clear();
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @brief add an object to an array
+
+    Appends the given element @a val to the end of the JSON value. If the
+    function is called on a JSON null value, an empty array is created before
+    appending @a val.
+
+    @param[in] val the value to add to the JSON array
+
+    @throw type_error.308 when called on a type other than JSON array or
+    null; example: `"cannot use push_back() with number"`
+
+    @complexity Amortized constant.
+
+    @liveexample{The example shows how `push_back()` and `+=` can be used to
+    add elements to a JSON array. Note how the `null` value was silently
+    converted to a JSON array.,push_back}
+
+    @since version 1.0.0
+    */
+    void push_back(basic_json&& val)
+    {
+        // push_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array (move semantics)
+        m_value.array->push_back(std::move(val));
+        // invalidate object
+        val.m_type = value_t::null;
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    reference operator+=(basic_json&& val)
+    {
+        push_back(std::move(val));
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    void push_back(const basic_json& val)
+    {
+        // push_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array
+        m_value.array->push_back(val);
+    }
+
+    /*!
+    @brief add an object to an array
+    @copydoc push_back(basic_json&&)
+    */
+    reference operator+=(const basic_json& val)
+    {
+        push_back(val);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an object
+
+    Inserts the given element @a val to the JSON object. If the function is
+    called on a JSON null value, an empty object is created before inserting
+    @a val.
+
+    @param[in] val the value to add to the JSON object
+
+    @throw type_error.308 when called on a type other than JSON object or
+    null; example: `"cannot use push_back() with number"`
+
+    @complexity Logarithmic in the size of the container, O(log(`size()`)).
+
+    @liveexample{The example shows how `push_back()` and `+=` can be used to
+    add elements to a JSON object. Note how the `null` value was silently
+    converted to a JSON object.,push_back__object_t__value}
+
+    @since version 1.0.0
+    */
+    void push_back(const typename object_t::value_type& val)
+    {
+        // push_back only works for null objects or objects
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
+        {
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // add element to array
+        m_value.object->insert(val);
+    }
+
+    /*!
+    @brief add an object to an object
+    @copydoc push_back(const typename object_t::value_type&)
+    */
+    reference operator+=(const typename object_t::value_type& val)
+    {
+        push_back(val);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an object
+
+    This function allows to use `push_back` with an initializer list. In case
+
+    1. the current value is an object,
+    2. the initializer list @a init contains only two elements, and
+    3. the first element of @a init is a string,
+
+    @a init is converted into an object element and added using
+    @ref push_back(const typename object_t::value_type&). Otherwise, @a init
+    is converted to a JSON value and added using @ref push_back(basic_json&&).
+
+    @param[in] init  an initializer list
+
+    @complexity Linear in the size of the initializer list @a init.
+
+    @note This function is required to resolve an ambiguous overload error,
+          because pairs like `{"key", "value"}` can be both interpreted as
+          `object_t::value_type` or `std::initializer_list<basic_json>`, see
+          https://github.com/nlohmann/json/issues/235 for more information.
+
+    @liveexample{The example shows how initializer lists are treated as
+    objects when possible.,push_back__initializer_list}
+    */
+    void push_back(initializer_list_t init)
+    {
+        if (is_object() and init.size() == 2 and (*init.begin())->is_string())
+        {
+            basic_json&& key = init.begin()->moved_or_copied();
+            push_back(typename object_t::value_type(
+                          std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));
+        }
+        else
+        {
+            push_back(basic_json(init));
+        }
+    }
+
+    /*!
+    @brief add an object to an object
+    @copydoc push_back(initializer_list_t)
+    */
+    reference operator+=(initializer_list_t init)
+    {
+        push_back(init);
+        return *this;
+    }
+
+    /*!
+    @brief add an object to an array
+
+    Creates a JSON value from the passed parameters @a args to the end of the
+    JSON value. If the function is called on a JSON null value, an empty array
+    is created before appending the value created from @a args.
+
+    @param[in] args arguments to forward to a constructor of @ref basic_json
+    @tparam Args compatible types to create a @ref basic_json object
+
+    @throw type_error.311 when called on a type other than JSON array or
+    null; example: `"cannot use emplace_back() with number"`
+
+    @complexity Amortized constant.
+
+    @liveexample{The example shows how `push_back()` can be used to add
+    elements to a JSON array. Note how the `null` value was silently converted
+    to a JSON array.,emplace_back}
+
+    @since version 2.0.8
+    */
+    template<class... Args>
+    void emplace_back(Args&& ... args)
+    {
+        // emplace_back only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
+        {
+            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name())));
+        }
+
+        // transform null object into an array
+        if (is_null())
+        {
+            m_type = value_t::array;
+            m_value = value_t::array;
+            assert_invariant();
+        }
+
+        // add element to array (perfect forwarding)
+        m_value.array->emplace_back(std::forward<Args>(args)...);
+    }
+
+    /*!
+    @brief add an object to an object if key does not exist
+
+    Inserts a new element into a JSON object constructed in-place with the
+    given @a args if there is no element with the key in the container. If the
+    function is called on a JSON null value, an empty object is created before
+    appending the value created from @a args.
+
+    @param[in] args arguments to forward to a constructor of @ref basic_json
+    @tparam Args compatible types to create a @ref basic_json object
+
+    @return a pair consisting of an iterator to the inserted element, or the
+            already-existing element if no insertion happened, and a bool
+            denoting whether the insertion took place.
+
+    @throw type_error.311 when called on a type other than JSON object or
+    null; example: `"cannot use emplace() with number"`
+
+    @complexity Logarithmic in the size of the container, O(log(`size()`)).
+
+    @liveexample{The example shows how `emplace()` can be used to add elements
+    to a JSON object. Note how the `null` value was silently converted to a
+    JSON object. Further note how no value is added if there was already one
+    value stored with the same key.,emplace}
+
+    @since version 2.0.8
+    */
+    template<class... Args>
+    std::pair<iterator, bool> emplace(Args&& ... args)
+    {
+        // emplace only works for null objects or arrays
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
+        {
+            JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name())));
+        }
+
+        // transform null object into an object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value = value_t::object;
+            assert_invariant();
+        }
+
+        // add element to array (perfect forwarding)
+        auto res = m_value.object->emplace(std::forward<Args>(args)...);
+        // create result iterator and set iterator to the result of emplace
+        auto it = begin();
+        it.m_it.object_iterator = res.first;
+
+        // return pair of iterator and boolean
+        return {it, res.second};
+    }
+
+    /*!
+    @brief inserts element
+
+    Inserts element @a val before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] val element to insert
+    @return iterator pointing to the inserted @a val.
+
+    @throw type_error.309 if called on JSON values other than arrays;
+    example: `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @complexity Constant plus linear in the distance between @a pos and end of
+    the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, const basic_json& val)
+    {
+        // insert only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // check if iterator pos fits to this JSON value
+            if (JSON_UNLIKELY(pos.m_object != this))
+            {
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+            }
+
+            // insert to array and return iterator
+            iterator result(this);
+            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val);
+            return result;
+        }
+
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief inserts element
+    @copydoc insert(const_iterator, const basic_json&)
+    */
+    iterator insert(const_iterator pos, basic_json&& val)
+    {
+        return insert(pos, val);
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts @a cnt copies of @a val before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] cnt number of copies of @a val to insert
+    @param[in] val element to insert
+    @return iterator pointing to the first element inserted, or @a pos if
+    `cnt==0`
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @complexity Linear in @a cnt plus linear in the distance between @a pos
+    and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__count}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, size_type cnt, const basic_json& val)
+    {
+        // insert only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            // check if iterator pos fits to this JSON value
+            if (JSON_UNLIKELY(pos.m_object != this))
+            {
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+            }
+
+            // insert to array and return iterator
+            iterator result(this);
+            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
+            return result;
+        }
+
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from range `[first, last)` before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+    @throw invalid_iterator.211 if @a first or @a last are iterators into
+    container for which insert is called; example: `"passed iterators may not
+    belong to container"`
+
+    @return iterator pointing to the first element inserted, or @a pos if
+    `first==last`
+
+    @complexity Linear in `std::distance(first, last)` plus linear in the
+    distance between @a pos and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__range}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, const_iterator first, const_iterator last)
+    {
+        // insert only works for arrays
+        if (JSON_UNLIKELY(not is_array()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if iterator pos fits to this JSON value
+        if (JSON_UNLIKELY(pos.m_object != this))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        if (JSON_UNLIKELY(first.m_object == this))
+        {
+            JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container"));
+        }
+
+        // insert to array and return iterator
+        iterator result(this);
+        result.m_it.array_iterator = m_value.array->insert(
+                                         pos.m_it.array_iterator,
+                                         first.m_it.array_iterator,
+                                         last.m_it.array_iterator);
+        return result;
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from initializer list @a ilist before iterator @a pos.
+
+    @param[in] pos iterator before which the content will be inserted; may be
+    the end() iterator
+    @param[in] ilist initializer list to insert the values from
+
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+
+    @return iterator pointing to the first element inserted, or @a pos if
+    `ilist` is empty
+
+    @complexity Linear in `ilist.size()` plus linear in the distance between
+    @a pos and end of the container.
+
+    @liveexample{The example shows how `insert()` is used.,insert__ilist}
+
+    @since version 1.0.0
+    */
+    iterator insert(const_iterator pos, initializer_list_t ilist)
+    {
+        // insert only works for arrays
+        if (JSON_UNLIKELY(not is_array()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if iterator pos fits to this JSON value
+        if (JSON_UNLIKELY(pos.m_object != this))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
+        }
+
+        // insert to array and return iterator
+        iterator result(this);
+        result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end());
+        return result;
+    }
+
+    /*!
+    @brief inserts elements
+
+    Inserts elements from range `[first, last)`.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.309 if called on JSON values other than objects; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number
+    of elements to insert.
+
+    @liveexample{The example shows how `insert()` is used.,insert__range_object}
+
+    @since version 3.0.0
+    */
+    void insert(const_iterator first, const_iterator last)
+    {
+        // insert only works for objects
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (JSON_UNLIKELY(not first.m_object->is_object()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
+    }
+
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from JSON object @a j and overwrites existing keys.
+
+    @param[in] j  JSON object to read values from
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_reference j)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+        if (JSON_UNLIKELY(not j.is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name())));
+        }
+
+        for (auto it = j.begin(); it != j.end(); ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from from range `[first, last)` and overwrites existing
+    keys.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used__range.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_iterator first, const_iterator last)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (JSON_UNLIKELY(not first.m_object->is_object()
+                          or not first.m_object->is_object()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        for (auto it = first; it != last; ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of the JSON value with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other JSON value to exchange the contents with
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how JSON values can be swapped with
+    `swap()`.,swap__reference}
+
+    @since version 1.0.0
+    */
+    void swap(reference other) noexcept (
+        std::is_nothrow_move_constructible<value_t>::value and
+        std::is_nothrow_move_assignable<value_t>::value and
+        std::is_nothrow_move_constructible<json_value>::value and
+        std::is_nothrow_move_assignable<json_value>::value
+    )
+    {
+        std::swap(m_type, other.m_type);
+        std::swap(m_value, other.m_value);
+        assert_invariant();
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON array with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other array to exchange the contents with
+
+    @throw type_error.310 when JSON value is not an array; example: `"cannot
+    use swap() with string"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how arrays can be swapped with
+    `swap()`.,swap__array_t}
+
+    @since version 1.0.0
+    */
+    void swap(array_t& other)
+    {
+        // swap only works for arrays
+        if (JSON_LIKELY(is_array()))
+        {
+            std::swap(*(m_value.array), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON object with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other object to exchange the contents with
+
+    @throw type_error.310 when JSON value is not an object; example:
+    `"cannot use swap() with string"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how objects can be swapped with
+    `swap()`.,swap__object_t}
+
+    @since version 1.0.0
+    */
+    void swap(object_t& other)
+    {
+        // swap only works for objects
+        if (JSON_LIKELY(is_object()))
+        {
+            std::swap(*(m_value.object), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /*!
+    @brief exchanges the values
+
+    Exchanges the contents of a JSON string with those of @a other. Does not
+    invoke any move, copy, or swap operations on individual elements. All
+    iterators and references remain valid. The past-the-end iterator is
+    invalidated.
+
+    @param[in,out] other string to exchange the contents with
+
+    @throw type_error.310 when JSON value is not a string; example: `"cannot
+    use swap() with boolean"`
+
+    @complexity Constant.
+
+    @liveexample{The example below shows how strings can be swapped with
+    `swap()`.,swap__string_t}
+
+    @since version 1.0.0
+    */
+    void swap(string_t& other)
+    {
+        // swap only works for strings
+        if (JSON_LIKELY(is_string()))
+        {
+            std::swap(*(m_value.string), other);
+        }
+        else
+        {
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
+        }
+    }
+
+    /// @}
+
+  public:
+    //////////////////////////////////////////
+    // lexicographical comparison operators //
+    //////////////////////////////////////////
+
+    /// @name lexicographical comparison operators
+    /// @{
+
+    /*!
+    @brief comparison: equal
+
+    Compares two JSON values for equality according to the following rules:
+    - Two JSON values are equal if (1) they are from the same type and (2)
+      their stored values are the same according to their respective
+      `operator==`.
+    - Integer and floating-point numbers are automatically converted before
+      comparison. Note than two NaN values are always treated as unequal.
+    - Two JSON null values are equal.
+
+    @note Floating-point inside JSON values numbers are compared with
+    `json::number_float_t::operator==` which is `double::operator==` by
+    default. To compare floating-point while respecting an epsilon, an alternative
+    [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39)
+    could be used, for instance
+    @code {.cpp}
+    template <typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
+    inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept
+    {
+        return std::abs(a - b) <= epsilon;
+    }
+    @endcode
+
+    @note NaN values never compare equal to themselves or to other NaN values.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether the values @a lhs and @a rhs are equal
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @complexity Linear.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__equal}
+
+    @since version 1.0.0
+    */
+    friend bool operator==(const_reference lhs, const_reference rhs) noexcept
+    {
+        const auto lhs_type = lhs.type();
+        const auto rhs_type = rhs.type();
+
+        if (lhs_type == rhs_type)
+        {
+            switch (lhs_type)
+            {
+                case value_t::array:
+                    return (*lhs.m_value.array == *rhs.m_value.array);
+
+                case value_t::object:
+                    return (*lhs.m_value.object == *rhs.m_value.object);
+
+                case value_t::null:
+                    return true;
+
+                case value_t::string:
+                    return (*lhs.m_value.string == *rhs.m_value.string);
+
+                case value_t::boolean:
+                    return (lhs.m_value.boolean == rhs.m_value.boolean);
+
+                case value_t::number_integer:
+                    return (lhs.m_value.number_integer == rhs.m_value.number_integer);
+
+                case value_t::number_unsigned:
+                    return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned);
+
+                case value_t::number_float:
+                    return (lhs.m_value.number_float == rhs.m_value.number_float);
+
+                default:
+                    return false;
+            }
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
+        {
+            return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float);
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
+        {
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer));
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
+        {
+            return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float);
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
+        {
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned));
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
+        {
+            return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer);
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
+        {
+            return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned));
+        }
+
+        return false;
+    }
+
+    /*!
+    @brief comparison: equal
+    @copydoc operator==(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs == basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: equal
+    @copydoc operator==(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) == rhs);
+    }
+
+    /*!
+    @brief comparison: not equal
+
+    Compares two JSON values for inequality by calculating `not (lhs == rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether the values @a lhs and @a rhs are not equal
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__notequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator!=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs == rhs);
+    }
+
+    /*!
+    @brief comparison: not equal
+    @copydoc operator!=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs != basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: not equal
+    @copydoc operator!=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) != rhs);
+    }
+
+    /*!
+    @brief comparison: less than
+
+    Compares whether one JSON value @a lhs is less than another JSON value @a
+    rhs according to the following rules:
+    - If @a lhs and @a rhs have the same type, the values are compared using
+      the default `<` operator.
+    - Integer and floating-point numbers are automatically converted before
+      comparison
+    - In case @a lhs and @a rhs have different types, the values are ignored
+      and the order of the types is considered, see
+      @ref operator<(const value_t, const value_t).
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is less than @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__less}
+
+    @since version 1.0.0
+    */
+    friend bool operator<(const_reference lhs, const_reference rhs) noexcept
+    {
+        const auto lhs_type = lhs.type();
+        const auto rhs_type = rhs.type();
+
+        if (lhs_type == rhs_type)
+        {
+            switch (lhs_type)
+            {
+                case value_t::array:
+                    return (*lhs.m_value.array) < (*rhs.m_value.array);
+
+                case value_t::object:
+                    return *lhs.m_value.object < *rhs.m_value.object;
+
+                case value_t::null:
+                    return false;
+
+                case value_t::string:
+                    return *lhs.m_value.string < *rhs.m_value.string;
+
+                case value_t::boolean:
+                    return lhs.m_value.boolean < rhs.m_value.boolean;
+
+                case value_t::number_integer:
+                    return lhs.m_value.number_integer < rhs.m_value.number_integer;
+
+                case value_t::number_unsigned:
+                    return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned;
+
+                case value_t::number_float:
+                    return lhs.m_value.number_float < rhs.m_value.number_float;
+
+                default:
+                    return false;
+            }
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
+        {
+            return static_cast<number_float_t>(lhs.m_value.number_integer) < rhs.m_value.number_float;
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
+        {
+            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_integer);
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
+        {
+            return static_cast<number_float_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_float;
+        }
+        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
+        {
+            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_unsigned);
+        }
+        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
+        {
+            return lhs.m_value.number_integer < static_cast<number_integer_t>(rhs.m_value.number_unsigned);
+        }
+        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
+        {
+            return static_cast<number_integer_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_integer;
+        }
+
+        // We only reach this line if we cannot compare values. In that case,
+        // we compare types. Note we have to call the operator explicitly,
+        // because MSVC has problems otherwise.
+        return operator<(lhs_type, rhs_type);
+    }
+
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs < basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) < rhs);
+    }
+
+    /*!
+    @brief comparison: less than or equal
+
+    Compares whether one JSON value @a lhs is less than or equal to another
+    JSON value by calculating `not (rhs < lhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is less than or equal to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__greater}
+
+    @since version 1.0.0
+    */
+    friend bool operator<=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (rhs < lhs);
+    }
+
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs <= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) <= rhs);
+    }
+
+    /*!
+    @brief comparison: greater than
+
+    Compares whether one JSON value @a lhs is greater than another
+    JSON value by calculating `not (lhs <= rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is greater than to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__lessequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator>(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs <= rhs);
+    }
+
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs > basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) > rhs);
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+
+    Compares whether one JSON value @a lhs is greater than or equal to another
+    JSON value by calculating `not (lhs < rhs)`.
+
+    @param[in] lhs  first JSON value to consider
+    @param[in] rhs  second JSON value to consider
+    @return whether @a lhs is greater than or equal to @a rhs
+
+    @complexity Linear.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @liveexample{The example demonstrates comparing several JSON
+    types.,operator__greaterequal}
+
+    @since version 1.0.0
+    */
+    friend bool operator>=(const_reference lhs, const_reference rhs) noexcept
+    {
+        return not (lhs < rhs);
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs >= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) >= rhs);
+    }
+
+    /// @}
+
+    ///////////////////
+    // serialization //
+    ///////////////////
+
+    /// @name serialization
+    /// @{
+
+    /*!
+    @brief serialize to stream
+
+    Serialize the given JSON value @a j to the output stream @a o. The JSON
+    value will be serialized using the @ref dump member function.
+
+    - The indentation of the output can be controlled with the member variable
+      `width` of the output stream @a o. For instance, using the manipulator
+      `std::setw(4)` on @a o sets the indentation level to `4` and the
+      serialization result is the same as calling `dump(4)`.
+
+    - The indentation characrer can be controlled with the member variable
+      `fill` of the output stream @a o. For instance, the manipulator
+      `std::setfill('\\t')` sets indentation to use a tab character rather than
+      the default space character.
+
+    @param[in,out] o  stream to serialize to
+    @param[in] j  JSON value to serialize
+
+    @return the stream @a o
+
+    @complexity Linear.
+
+    @liveexample{The example below shows the serialization with different
+    parameters to `width` to adjust the indentation level.,operator_serialize}
+
+    @since version 1.0.0; indentaction character added in version 3.0.0
+    */
+    friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
+    {
+        // read width member and use it as indentation parameter if nonzero
+        const bool pretty_print = (o.width() > 0);
+        const auto indentation = (pretty_print ? o.width() : 0);
+
+        // reset width to 0 for subsequent calls to this stream
+        o.width(0);
+
+        // do the actual serialization
+        serializer s(detail::output_adapter<char>(o), o.fill());
+        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
+        return o;
+    }
+
+    /*!
+    @brief serialize to stream
+    @deprecated This stream operator is deprecated and will be removed in a
+                future version of the library. Please use
+                @ref operator<<(std::ostream&, const basic_json&)
+                instead; that is, replace calls like `j >> o;` with `o << j;`.
+    @since version 1.0.0; deprecated since version 3.0.0
+    */
+    JSON_DEPRECATED
+    friend std::ostream& operator>>(const basic_json& j, std::ostream& o)
+    {
+        return o << j;
+    }
+
+    /// @}
+
+
+    /////////////////////
+    // deserialization //
+    /////////////////////
+
+    /// @name deserialization
+    /// @{
+
+    /*!
+    @brief deserialize from a compatible input
+
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
+
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
+    @param[in] cb  a parser callback function of type @ref parser_callback_t
+    which is used to control the deserialization by filtering unwanted values
+    (optional)
+
+    @return result of the deserialization
+
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the parser callback function
+    @a cb has a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from an array.,parse__array__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function with
+    and without callback function.,parse__string__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function with
+    and without callback function.,parse__istream__parser_callback_t}
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from a contiguous container.,parse__contiguouscontainer__parser_callback_t}
+
+    @since version 2.0.3 (contiguous containers)
+    */
+    static basic_json parse(detail::input_adapter i,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
+    {
+        basic_json result;
+        parser(i, cb, allow_exceptions).parse(true, result);
+        return result;
+    }
+
+    /*!
+    @copydoc basic_json parse(detail::input_adapter, const parser_callback_t)
+    */
+    static basic_json parse(detail::input_adapter& i,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
+    {
+        basic_json result;
+        parser(i, cb, allow_exceptions).parse(true, result);
+        return result;
+    }
+
+    static bool accept(detail::input_adapter i)
+    {
+        return parser(i).accept(true);
+    }
+
+    static bool accept(detail::input_adapter& i)
+    {
+        return parser(i).accept(true);
+    }
+
+    /*!
+    @brief deserialize from an iterator range with contiguous storage
+
+    This function reads from an iterator range of a container with contiguous
+    storage of 1-byte values. Compatible container types include
+    `std::vector`, `std::string`, `std::array`, `std::valarray`, and
+    `std::initializer_list`. Furthermore, C-style arrays can be used with
+    `std::begin()`/`std::end()`. User-defined containers can be used as long
+    as they implement random-access iterators and a contiguous storage.
+
+    @pre The iterator range is contiguous. Violating this precondition yields
+    undefined behavior. **This precondition is enforced with an assertion.**
+    @pre Each element in the range has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with noncompliant iterators and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @tparam IteratorType iterator of container with contiguous storage
+    @param[in] first  begin of the range to parse (included)
+    @param[in] last  end of the range to parse (excluded)
+    @param[in] cb  a parser callback function of type @ref parser_callback_t
+    which is used to control the deserialization by filtering unwanted values
+    (optional)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return result of the deserialization
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the parser callback function
+    @a cb has a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from an iterator range.,parse__iteratortype__parser_callback_t}
+
+    @since version 2.0.3
+    */
+    template<class IteratorType, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static basic_json parse(IteratorType first, IteratorType last,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
+    {
+        basic_json result;
+        parser(detail::input_adapter(first, last), cb, allow_exceptions).parse(true, result);
+        return result;
+    }
+
+    template<class IteratorType, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool accept(IteratorType first, IteratorType last)
+    {
+        return parser(detail::input_adapter(first, last)).accept(true);
+    }
+
+    /*!
+    @brief deserialize from stream
+    @deprecated This stream operator is deprecated and will be removed in a
+                future version of the library. Please use
+                @ref operator>>(std::istream&, basic_json&)
+                instead; that is, replace calls like `j << i;` with `i >> j;`.
+    @since version 1.0.0; deprecated since version 3.0.0
+    */
+    JSON_DEPRECATED
+    friend std::istream& operator<<(basic_json& j, std::istream& i)
+    {
+        return operator>>(i, j);
+    }
+
+    /*!
+    @brief deserialize from stream
+
+    Deserializes an input stream to a JSON value.
+
+    @param[in,out] i  input stream to read a serialized JSON value from
+    @param[in,out] j  JSON value to write the deserialized input to
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below shows how a JSON value is constructed by
+    reading a serialization from a stream.,operator_deserialize}
+
+    @sa parse(std::istream&, const parser_callback_t) for a variant with a
+    parser callback function to filter values while parsing
+
+    @since version 1.0.0
+    */
+    friend std::istream& operator>>(std::istream& i, basic_json& j)
+    {
+        parser(detail::input_adapter(i)).parse(false, j);
+        return i;
+    }
+
+    /// @}
+
+    ///////////////////////////
+    // convenience functions //
+    ///////////////////////////
+
+    /*!
+    @brief return the type as string
+
+    Returns the type name as string to be used in error messages - usually to
+    indicate that a function was called on a wrong JSON type.
+
+    @return a string representation of a the @a m_type member:
+            Value type  | return value
+            ----------- | -------------
+            null        | `"null"`
+            boolean     | `"boolean"`
+            string      | `"string"`
+            number      | `"number"` (for all number types)
+            object      | `"object"`
+            array       | `"array"`
+            discarded   | `"discarded"`
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @complexity Constant.
+
+    @liveexample{The following code exemplifies `type_name()` for all JSON
+    types.,type_name}
+
+    @sa @ref type() -- return the type of the JSON value
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
+
+    @since version 1.0.0, public since 2.1.0, `const char*` and `noexcept`
+    since 3.0.0
+    */
+    const char* type_name() const noexcept
+    {
+        {
+            switch (m_type)
+            {
+                case value_t::null:
+                    return "null";
+                case value_t::object:
+                    return "object";
+                case value_t::array:
+                    return "array";
+                case value_t::string:
+                    return "string";
+                case value_t::boolean:
+                    return "boolean";
+                case value_t::discarded:
+                    return "discarded";
+                default:
+                    return "number";
+            }
+        }
+    }
+
+
+  private:
+    //////////////////////
+    // member variables //
+    //////////////////////
+
+    /// the type of the current element
+    value_t m_type = value_t::null;
+
+    /// the value of the current element
+    json_value m_value = {};
+
+    //////////////////////////////////////////
+    // binary serialization/deserialization //
+    //////////////////////////////////////////
+
+    /// @name binary serialization/deserialization support
+    /// @{
+
+  public:
+    /*!
+    @brief create a CBOR serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the CBOR (Concise
+    Binary Object Representation) serialization format. CBOR is a binary
+    serialization format which aims to be more compact than JSON itself, yet
+    more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    CBOR types according to the CBOR specification (RFC 7049):
+
+    JSON value type | value/range                                | CBOR type                          | first byte
+    --------------- | ------------------------------------------ | ---------------------------------- | ---------------
+    null            | `null`                                     | Null                               | 0xf6
+    boolean         | `true`                                     | True                               | 0xf5
+    boolean         | `false`                                    | False                              | 0xf4
+    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3b
+    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3a
+    number_integer  | -32768..-129                               | Negative integer (2 bytes follow)  | 0x39
+    number_integer  | -128..-25                                  | Negative integer (1 byte follow)   | 0x38
+    number_integer  | -24..-1                                    | Negative integer                   | 0x20..0x37
+    number_integer  | 0..23                                      | Integer                            | 0x00..0x17
+    number_integer  | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_integer  | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
+    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
+    number_unsigned | 0..23                                      | Integer                            | 0x00..0x17
+    number_unsigned | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_unsigned | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
+    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
+    number_float    | *any value*                                | Double-Precision Float             | 0xfb
+    string          | *length*: 0..23                            | UTF-8 string                       | 0x60..0x77
+    string          | *length*: 23..255                          | UTF-8 string (1 byte follow)       | 0x78
+    string          | *length*: 256..65535                       | UTF-8 string (2 bytes follow)      | 0x79
+    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7a
+    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7b
+    array           | *size*: 0..23                              | array                              | 0x80..0x97
+    array           | *size*: 23..255                            | array (1 byte follow)              | 0x98
+    array           | *size*: 256..65535                         | array (2 bytes follow)             | 0x99
+    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9a
+    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9b
+    object          | *size*: 0..23                              | map                                | 0xa0..0xb7
+    object          | *size*: 23..255                            | map (1 byte follow)                | 0xb8
+    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xb9
+    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xba
+    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xbb
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a CBOR value.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @note The following CBOR types are not used in the conversion:
+          - byte strings (0x40..0x5f)
+          - UTF-8 strings terminated by "break" (0x7f)
+          - arrays terminated by "break" (0x9f)
+          - maps terminated by "break" (0xbf)
+          - date/time (0xc0..0xc1)
+          - bignum (0xc2..0xc3)
+          - decimal fraction (0xc4)
+          - bigfloat (0xc5)
+          - tagged items (0xc6..0xd4, 0xd8..0xdb)
+          - expected conversions (0xd5..0xd7)
+          - simple values (0xe0..0xf3, 0xf8)
+          - undefined (0xf7)
+          - half and single-precision floats (0xf9-0xfa)
+          - break (0xff)
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in CBOR format.,to_cbor}
+
+    @sa http://cbor.io
+    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_cbor(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_cbor(j, result);
+        return result;
+    }
+
+    static void to_cbor(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_cbor(j);
+    }
+
+    static void to_cbor(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_cbor(j);
+    }
+
+    /*!
+    @brief create a MessagePack serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the MessagePack
+    serialization format. MessagePack is a binary serialization format which
+    aims to be more compact than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    MessagePack types according to the MessagePack specification:
+
+    JSON value type | value/range                       | MessagePack type | first byte
+    --------------- | --------------------------------- | ---------------- | ----------
+    null            | `null`                            | nil              | 0xc0
+    boolean         | `true`                            | true             | 0xc3
+    boolean         | `false`                           | false            | 0xc2
+    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xd3
+    number_integer  | -2147483648..-32769               | int32            | 0xd2
+    number_integer  | -32768..-129                      | int16            | 0xd1
+    number_integer  | -128..-33                         | int8             | 0xd0
+    number_integer  | -32..-1                           | negative fixint  | 0xe0..0xff
+    number_integer  | 0..127                            | positive fixint  | 0x00..0x7f
+    number_integer  | 128..255                          | uint 8           | 0xcc
+    number_integer  | 256..65535                        | uint 16          | 0xcd
+    number_integer  | 65536..4294967295                 | uint 32          | 0xce
+    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xcf
+    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7f
+    number_unsigned | 128..255                          | uint 8           | 0xcc
+    number_unsigned | 256..65535                        | uint 16          | 0xcd
+    number_unsigned | 65536..4294967295                 | uint 32          | 0xce
+    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xcf
+    number_float    | *any value*                       | float 64         | 0xcb
+    string          | *length*: 0..31                   | fixstr           | 0xa0..0xbf
+    string          | *length*: 32..255                 | str 8            | 0xd9
+    string          | *length*: 256..65535              | str 16           | 0xda
+    string          | *length*: 65536..4294967295       | str 32           | 0xdb
+    array           | *size*: 0..15                     | fixarray         | 0x90..0x9f
+    array           | *size*: 16..65535                 | array 16         | 0xdc
+    array           | *size*: 65536..4294967295         | array 32         | 0xdd
+    object          | *size*: 0..15                     | fix map          | 0x80..0x8f
+    object          | *size*: 16..65535                 | map 16           | 0xde
+    object          | *size*: 65536..4294967295         | map 32           | 0xdf
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a MessagePack value.
+
+    @note The following values can **not** be converted to a MessagePack value:
+          - strings with more than 4294967295 bytes
+          - arrays with more than 4294967295 elements
+          - objects with more than 4294967295 elements
+
+    @note The following MessagePack types are not used in the conversion:
+          - bin 8 - bin 32 (0xc4..0xc6)
+          - ext 8 - ext 32 (0xc7..0xc9)
+          - float 32 (0xca)
+          - fixext 1 - fixext 16 (0xd4..0xd8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in MessagePack format.,to_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_msgpack(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_msgpack(j, result);
+        return result;
+    }
+
+    static void to_msgpack(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_msgpack(j);
+    }
+
+    static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_msgpack(j);
+    }
+
+    /*!
+    @brief create a JSON value from an input in CBOR format
+
+    Deserializes a given input @a i to a JSON value using the CBOR (Concise
+    Binary Object Representation) serialization format.
+
+    The library maps CBOR types to JSON value types as follows:
+
+    CBOR type              | JSON value type | first byte
+    ---------------------- | --------------- | ----------
+    Integer                | number_unsigned | 0x00..0x17
+    Unsigned integer       | number_unsigned | 0x18
+    Unsigned integer       | number_unsigned | 0x19
+    Unsigned integer       | number_unsigned | 0x1a
+    Unsigned integer       | number_unsigned | 0x1b
+    Negative integer       | number_integer  | 0x20..0x37
+    Negative integer       | number_integer  | 0x38
+    Negative integer       | number_integer  | 0x39
+    Negative integer       | number_integer  | 0x3a
+    Negative integer       | number_integer  | 0x3b
+    Negative integer       | number_integer  | 0x40..0x57
+    UTF-8 string           | string          | 0x60..0x77
+    UTF-8 string           | string          | 0x78
+    UTF-8 string           | string          | 0x79
+    UTF-8 string           | string          | 0x7a
+    UTF-8 string           | string          | 0x7b
+    UTF-8 string           | string          | 0x7f
+    array                  | array           | 0x80..0x97
+    array                  | array           | 0x98
+    array                  | array           | 0x99
+    array                  | array           | 0x9a
+    array                  | array           | 0x9b
+    array                  | array           | 0x9f
+    map                    | object          | 0xa0..0xb7
+    map                    | object          | 0xb8
+    map                    | object          | 0xb9
+    map                    | object          | 0xba
+    map                    | object          | 0xbb
+    map                    | object          | 0xbf
+    False                  | `false`         | 0xf4
+    True                   | `true`          | 0xf5
+    Nill                   | `null`          | 0xf6
+    Half-Precision Float   | number_float    | 0xf9
+    Single-Precision Float | number_float    | 0xfa
+    Double-Precision Float | number_float    | 0xfb
+
+    @warning The mapping is **incomplete** in the sense that not all CBOR
+             types can be converted to a JSON value. The following CBOR types
+             are not supported and will yield parse errors (parse_error.112):
+             - byte strings (0x40..0x5f)
+             - date/time (0xc0..0xc1)
+             - bignum (0xc2..0xc3)
+             - decimal fraction (0xc4)
+             - bigfloat (0xc5)
+             - tagged items (0xc6..0xd4, 0xd8..0xdb)
+             - expected conversions (0xd5..0xd7)
+             - simple values (0xe0..0xf3, 0xf8)
+             - undefined (0xf7)
+
+    @warning CBOR allows map keys of any type, whereas JSON only allows
+             strings as keys in object values. Therefore, CBOR maps with keys
+             other than UTF-8 strings are rejected (parse_error.113).
+
+    @note Any CBOR output created @ref to_cbor can be successfully parsed by
+          @ref from_cbor.
+
+    @param[in] i  an input in CBOR format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported features from CBOR were
+    used in the given input @a v or if the input is not valid CBOR
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in CBOR
+    format to a JSON value.,from_cbor}
+
+    @sa http://cbor.io
+    @sa @ref to_cbor(const basic_json&) for the analogous serialization
+    @sa @ref from_msgpack(detail::input_adapter, const bool) for the
+        related MessagePack format
+
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0
+    */
+    static basic_json from_cbor(detail::input_adapter i,
+                                const bool strict = true)
+    {
+        return binary_reader(i).parse_cbor(strict);
+    }
+
+    /*!
+    @copydoc from_cbor(detail::input_adapter, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true)
+    {
+        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict);
+    }
+
+    /*!
+    @brief create a JSON value from an input in MessagePack format
+
+    Deserializes a given input @a i to a JSON value using the MessagePack
+    serialization format.
+
+    The library maps MessagePack types to JSON value types as follows:
+
+    MessagePack type | JSON value type | first byte
+    ---------------- | --------------- | ----------
+    positive fixint  | number_unsigned | 0x00..0x7f
+    fixmap           | object          | 0x80..0x8f
+    fixarray         | array           | 0x90..0x9f
+    fixstr           | string          | 0xa0..0xbf
+    nil              | `null`          | 0xc0
+    false            | `false`         | 0xc2
+    true             | `true`          | 0xc3
+    float 32         | number_float    | 0xca
+    float 64         | number_float    | 0xcb
+    uint 8           | number_unsigned | 0xcc
+    uint 16          | number_unsigned | 0xcd
+    uint 32          | number_unsigned | 0xce
+    uint 64          | number_unsigned | 0xcf
+    int 8            | number_integer  | 0xd0
+    int 16           | number_integer  | 0xd1
+    int 32           | number_integer  | 0xd2
+    int 64           | number_integer  | 0xd3
+    str 8            | string          | 0xd9
+    str 16           | string          | 0xda
+    str 32           | string          | 0xdb
+    array 16         | array           | 0xdc
+    array 32         | array           | 0xdd
+    map 16           | object          | 0xde
+    map 32           | object          | 0xdf
+    negative fixint  | number_integer  | 0xe0-0xff
+
+    @warning The mapping is **incomplete** in the sense that not all
+             MessagePack types can be converted to a JSON value. The following
+             MessagePack types are not supported and will yield parse errors:
+              - bin 8 - bin 32 (0xc4..0xc6)
+              - ext 8 - ext 32 (0xc7..0xc9)
+              - fixext 1 - fixext 16 (0xd4..0xd8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @param[in] i  an input in MessagePack format convertible to an input
+                  adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported features from MessagePack were
+    used in the given input @a i or if the input is not valid MessagePack
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    MessagePack format to a JSON value.,from_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref to_msgpack(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR
+        format
+
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0
+    */
+    static basic_json from_msgpack(detail::input_adapter i,
+                                   const bool strict = true)
+    {
+        return binary_reader(i).parse_msgpack(strict);
+    }
+
+    /*!
+    @copydoc from_msgpack(detail::input_adapter, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true)
+    {
+        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict);
+    }
+
+    /// @}
+
+    //////////////////////////
+    // JSON Pointer support //
+    //////////////////////////
+
+    /// @name JSON Pointer functions
+    /// @{
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Uses a JSON pointer to retrieve a reference to the respective JSON value.
+    No bound checking is performed. Similar to @ref operator[](const typename
+    object_t::key_type&), `null` values are created in arrays and objects if
+    necessary.
+
+    In particular:
+    - If the JSON pointer points to an object key that does not exist, it
+      is created an filled with a `null` value before a reference to it
+      is returned.
+    - If the JSON pointer points to an array index that does not exist, it
+      is created an filled with a `null` value before a reference to it
+      is returned. All indices between the current maximum and the given
+      index are also filled with `null`.
+    - The special value `-` is treated as a synonym for the index past the
+      end.
+
+    @param[in] ptr  a JSON pointer
+
+    @return reference to the element pointed to by @a ptr
+
+    @complexity Constant.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+
+    @liveexample{The behavior is shown in the example.,operatorjson_pointer}
+
+    @since version 2.0.0
+    */
+    reference operator[](const json_pointer& ptr)
+    {
+        return ptr.get_unchecked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Uses a JSON pointer to retrieve a reference to the respective JSON value.
+    No bound checking is performed. The function does not change the JSON
+    value; no `null` values are created. In particular, the the special value
+    `-` yields an exception.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return const reference to the element pointed to by @a ptr
+
+    @complexity Constant.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+
+    @liveexample{The behavior is shown in the example.,operatorjson_pointer_const}
+
+    @since version 2.0.0
+    */
+    const_reference operator[](const json_pointer& ptr) const
+    {
+        return ptr.get_unchecked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Returns a reference to the element at with specified JSON pointer @a ptr,
+    with bounds checking.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return reference to the element pointed to by @a ptr
+
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 2.0.0
+
+    @liveexample{The behavior is shown in the example.,at_json_pointer}
+    */
+    reference at(const json_pointer& ptr)
+    {
+        return ptr.get_checked(this);
+    }
+
+    /*!
+    @brief access specified element via JSON Pointer
+
+    Returns a const reference to the element at with specified JSON pointer @a
+    ptr, with bounds checking.
+
+    @param[in] ptr  JSON pointer to the desired element
+
+    @return reference to the element pointed to by @a ptr
+
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
+    @since version 2.0.0
+
+    @liveexample{The behavior is shown in the example.,at_json_pointer_const}
+    */
+    const_reference at(const json_pointer& ptr) const
+    {
+        return ptr.get_checked(this);
+    }
+
+    /*!
+    @brief return flattened JSON value
+
+    The function creates a JSON object whose keys are JSON pointers (see [RFC
+    6901](https://tools.ietf.org/html/rfc6901)) and whose values are all
+    primitive. The original JSON value can be restored using the @ref
+    unflatten() function.
+
+    @return an object that maps JSON pointers to primitive values
+
+    @note Empty objects and arrays are flattened to `null` and will not be
+          reconstructed correctly by the @ref unflatten() function.
+
+    @complexity Linear in the size the JSON value.
+
+    @liveexample{The following code shows how a JSON object is flattened to an
+    object whose keys consist of JSON pointers.,flatten}
+
+    @sa @ref unflatten() for the reverse function
+
+    @since version 2.0.0
+    */
+    basic_json flatten() const
+    {
+        basic_json result(value_t::object);
+        json_pointer::flatten("", *this, result);
+        return result;
+    }
+
+    /*!
+    @brief unflatten a previously flattened JSON value
+
+    The function restores the arbitrary nesting of a JSON value that has been
+    flattened before using the @ref flatten() function. The JSON value must
+    meet certain constraints:
+    1. The value must be an object.
+    2. The keys must be JSON pointers (see
+       [RFC 6901](https://tools.ietf.org/html/rfc6901))
+    3. The mapped values must be primitive JSON types.
+
+    @return the original JSON from a flattened version
+
+    @note Empty objects and arrays are flattened by @ref flatten() to `null`
+          values and can not unflattened to their original type. Apart from
+          this example, for a JSON value `j`, the following is always true:
+          `j == j.flatten().unflatten()`.
+
+    @complexity Linear in the size the JSON value.
+
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+
+    @liveexample{The following code shows how a flattened JSON object is
+    unflattened into the original nested JSON object.,unflatten}
+
+    @sa @ref flatten() for the reverse function
+
+    @since version 2.0.0
+    */
+    basic_json unflatten() const
+    {
+        return json_pointer::unflatten(*this);
+    }
+
+    /// @}
+
+    //////////////////////////
+    // JSON Patch functions //
+    //////////////////////////
+
+    /// @name JSON Patch functions
+    /// @{
+
+    /*!
+    @brief applies a JSON patch
+
+    [JSON Patch](http://jsonpatch.com) defines a JSON document structure for
+    expressing a sequence of operations to apply to a JSON) document. With
+    this function, a JSON Patch is applied to the current JSON value by
+    executing all operations from the patch.
+
+    @param[in] json_patch  JSON patch document
+    @return patched document
+
+    @note The application of a patch is atomic: Either all operations succeed
+          and the patched document is returned or an exception is thrown. In
+          any case, the original value is not changed: the patch is applied
+          to a copy of the value.
+
+    @throw parse_error.104 if the JSON patch does not consist of an array of
+    objects
+
+    @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory
+    attributes are missing); example: `"operation add must have member path"`
+
+    @throw out_of_range.401 if an array index is out of range.
+
+    @throw out_of_range.403 if a JSON pointer inside the patch could not be
+    resolved successfully in the current JSON value; example: `"key baz not
+    found"`
+
+    @throw out_of_range.405 if JSON pointer has no parent ("add", "remove",
+    "move")
+
+    @throw other_error.501 if "test" operation was unsuccessful
+
+    @complexity Linear in the size of the JSON value and the length of the
+    JSON patch. As usually only a fraction of the JSON value is affected by
+    the patch, the complexity can usually be neglected.
+
+    @liveexample{The following code shows how a JSON patch is applied to a
+    value.,patch}
+
+    @sa @ref diff -- create a JSON patch by comparing two JSON values
+
+    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
+    @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901)
+
+    @since version 2.0.0
+    */
+    basic_json patch(const basic_json& json_patch) const
+    {
+        // make a working copy to apply the patch to
+        basic_json result = *this;
+
+        // the valid JSON Patch operations
+        enum class patch_operations {add, remove, replace, move, copy, test, invalid};
+
+        const auto get_op = [](const std::string & op)
+        {
+            if (op == "add")
+            {
+                return patch_operations::add;
+            }
+            if (op == "remove")
+            {
+                return patch_operations::remove;
+            }
+            if (op == "replace")
+            {
+                return patch_operations::replace;
+            }
+            if (op == "move")
+            {
+                return patch_operations::move;
+            }
+            if (op == "copy")
+            {
+                return patch_operations::copy;
+            }
+            if (op == "test")
+            {
+                return patch_operations::test;
+            }
+
+            return patch_operations::invalid;
+        };
+
+        // wrapper for "add" operation; add value at ptr
+        const auto operation_add = [&result](json_pointer & ptr, basic_json val)
+        {
+            // adding to the root of the target document means replacing it
+            if (ptr.is_root())
+            {
+                result = val;
+            }
+            else
+            {
+                // make sure the top element of the pointer exists
+                json_pointer top_pointer = ptr.top();
+                if (top_pointer != ptr)
+                {
+                    result.at(top_pointer);
+                }
+
+                // get reference to parent of JSON pointer ptr
+                const auto last_path = ptr.pop_back();
+                basic_json& parent = result[ptr];
+
+                switch (parent.m_type)
+                {
+                    case value_t::null:
+                    case value_t::object:
+                    {
+                        // use operator[] to add value
+                        parent[last_path] = val;
+                        break;
+                    }
+
+                    case value_t::array:
+                    {
+                        if (last_path == "-")
+                        {
+                            // special case: append to back
+                            parent.push_back(val);
+                        }
+                        else
+                        {
+                            const auto idx = std::stoi(last_path);
+                            if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
+                            {
+                                // avoid undefined behavior
+                                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
+                            }
+                            else
+                            {
+                                // default case: insert add offset
+                                parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
+                            }
+                        }
+                        break;
+                    }
+
+                    default:
+                    {
+                        // if there exists a parent it cannot be primitive
+                        assert(false);  // LCOV_EXCL_LINE
+                    }
+                }
+            }
+        };
+
+        // wrapper for "remove" operation; remove value at ptr
+        const auto operation_remove = [&result](json_pointer & ptr)
+        {
+            // get reference to parent of JSON pointer ptr
+            const auto last_path = ptr.pop_back();
+            basic_json& parent = result.at(ptr);
+
+            // remove child
+            if (parent.is_object())
+            {
+                // perform range check
+                auto it = parent.find(last_path);
+                if (JSON_LIKELY(it != parent.end()))
+                {
+                    parent.erase(it);
+                }
+                else
+                {
+                    JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found"));
+                }
+            }
+            else if (parent.is_array())
+            {
+                // note erase performs range check
+                parent.erase(static_cast<size_type>(std::stoi(last_path)));
+            }
+        };
+
+        // type check: top level value must be an array
+        if (JSON_UNLIKELY(not json_patch.is_array()))
+        {
+            JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
+        }
+
+        // iterate and apply the operations
+        for (const auto& val : json_patch)
+        {
+            // wrapper to get a value for an operation
+            const auto get_value = [&val](const std::string & op,
+                                          const std::string & member,
+                                          bool string_type) -> basic_json&
+            {
+                // find value
+                auto it = val.m_value.object->find(member);
+
+                // context-sensitive error message
+                const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'";
+
+                // check if desired value is present
+                if (JSON_UNLIKELY(it == val.m_value.object->end()))
+                {
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'"));
+                }
+
+                // check if result is of type string
+                if (JSON_UNLIKELY(string_type and not it->second.is_string()))
+                {
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'"));
+                }
+
+                // no error: return value
+                return it->second;
+            };
+
+            // type check: every element of the array must be an object
+            if (JSON_UNLIKELY(not val.is_object()))
+            {
+                JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
+            }
+
+            // collect mandatory members
+            const std::string op = get_value("op", "op", true);
+            const std::string path = get_value(op, "path", true);
+            json_pointer ptr(path);
+
+            switch (get_op(op))
+            {
+                case patch_operations::add:
+                {
+                    operation_add(ptr, get_value("add", "value", false));
+                    break;
+                }
+
+                case patch_operations::remove:
+                {
+                    operation_remove(ptr);
+                    break;
+                }
+
+                case patch_operations::replace:
+                {
+                    // the "path" location must exist - use at()
+                    result.at(ptr) = get_value("replace", "value", false);
+                    break;
+                }
+
+                case patch_operations::move:
+                {
+                    const std::string from_path = get_value("move", "from", true);
+                    json_pointer from_ptr(from_path);
+
+                    // the "from" location must exist - use at()
+                    basic_json v = result.at(from_ptr);
+
+                    // The move operation is functionally identical to a
+                    // "remove" operation on the "from" location, followed
+                    // immediately by an "add" operation at the target
+                    // location with the value that was just removed.
+                    operation_remove(from_ptr);
+                    operation_add(ptr, v);
+                    break;
+                }
+
+                case patch_operations::copy:
+                {
+                    const std::string from_path = get_value("copy", "from", true);
+                    const json_pointer from_ptr(from_path);
+
+                    // the "from" location must exist - use at()
+                    result[ptr] = result.at(from_ptr);
+                    break;
+                }
+
+                case patch_operations::test:
+                {
+                    bool success = false;
+                    JSON_TRY
+                    {
+                        // check if "value" matches the one at "path"
+                        // the "path" location must exist - use at()
+                        success = (result.at(ptr) == get_value("test", "value", false));
+                    }
+                    JSON_CATCH (out_of_range&)
+                    {
+                        // ignore out of range errors: success remains false
+                    }
+
+                    // throw an exception if test fails
+                    if (JSON_UNLIKELY(not success))
+                    {
+                        JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump()));
+                    }
+
+                    break;
+                }
+
+                case patch_operations::invalid:
+                {
+                    // op must be "add", "remove", "replace", "move", "copy", or
+                    // "test"
+                    JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid"));
+                }
+            }
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief creates a diff as a JSON patch
+
+    Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can
+    be changed into the value @a target by calling @ref patch function.
+
+    @invariant For two JSON values @a source and @a target, the following code
+    yields always `true`:
+    @code {.cpp}
+    source.patch(diff(source, target)) == target;
+    @endcode
+
+    @note Currently, only `remove`, `add`, and `replace` operations are
+          generated.
+
+    @param[in] source  JSON value to compare from
+    @param[in] target  JSON value to compare against
+    @param[in] path    helper value to create JSON pointers
+
+    @return a JSON patch to convert the @a source to @a target
+
+    @complexity Linear in the lengths of @a source and @a target.
+
+    @liveexample{The following code shows how a JSON patch is created as a
+    diff for two JSON values.,diff}
+
+    @sa @ref patch -- apply a JSON patch
+
+    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
+
+    @since version 2.0.0
+    */
+    static basic_json diff(const basic_json& source, const basic_json& target,
+                           const std::string& path = "")
+    {
+        // the patch
+        basic_json result(value_t::array);
+
+        // if the values are the same, return empty patch
+        if (source == target)
+        {
+            return result;
+        }
+
+        if (source.type() != target.type())
+        {
+            // different types: replace value
+            result.push_back(
+            {
+                {"op", "replace"}, {"path", path}, {"value", target}
+            });
+        }
+        else
+        {
+            switch (source.type())
+            {
+                case value_t::array:
+                {
+                    // first pass: traverse common elements
+                    std::size_t i = 0;
+                    while (i < source.size() and i < target.size())
+                    {
+                        // recursive call to compare array values at index i
+                        auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i));
+                        result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+                        ++i;
+                    }
+
+                    // i now reached the end of at least one array
+                    // in a second pass, traverse the remaining elements
+
+                    // remove my remaining elements
+                    const auto end_index = static_cast<difference_type>(result.size());
+                    while (i < source.size())
+                    {
+                        // add operations in reverse order to avoid invalid
+                        // indices
+                        result.insert(result.begin() + end_index, object(
+                        {
+                            {"op", "remove"},
+                            {"path", path + "/" + std::to_string(i)}
+                        }));
+                        ++i;
+                    }
+
+                    // add other remaining elements
+                    while (i < target.size())
+                    {
+                        result.push_back(
+                        {
+                            {"op", "add"},
+                            {"path", path + "/" + std::to_string(i)},
+                            {"value", target[i]}
+                        });
+                        ++i;
+                    }
+
+                    break;
+                }
+
+                case value_t::object:
+                {
+                    // first pass: traverse this object's elements
+                    for (auto it = source.begin(); it != source.end(); ++it)
+                    {
+                        // escape the key name to be used in a JSON patch
+                        const auto key = json_pointer::escape(it.key());
+
+                        if (target.find(it.key()) != target.end())
+                        {
+                            // recursive call to compare object values at key it
+                            auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key);
+                            result.insert(result.end(), temp_diff.begin(), temp_diff.end());
+                        }
+                        else
+                        {
+                            // found a key that is not in o -> remove it
+                            result.push_back(object(
+                            {
+                                {"op", "remove"}, {"path", path + "/" + key}
+                            }));
+                        }
+                    }
+
+                    // second pass: traverse other object's elements
+                    for (auto it = target.begin(); it != target.end(); ++it)
+                    {
+                        if (source.find(it.key()) == source.end())
+                        {
+                            // found a key that is not in this -> add it
+                            const auto key = json_pointer::escape(it.key());
+                            result.push_back(
+                            {
+                                {"op", "add"}, {"path", path + "/" + key},
+                                {"value", it.value()}
+                            });
+                        }
+                    }
+
+                    break;
+                }
+
+                default:
+                {
+                    // both primitive type: replace value
+                    result.push_back(
+                    {
+                        {"op", "replace"}, {"path", path}, {"value", target}
+                    });
+                    break;
+                }
+            }
+        }
+
+        return result;
+    }
+
+    /// @}
+};
+
+/////////////
+// presets //
+/////////////
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+
+//////////////////
+// json_pointer //
+//////////////////
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    auto result = &j;
+
+    // in case no reference tokens exist, return a reference to the JSON value
+    // j which will be overwritten by a primitive value
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (result->m_type)
+        {
+            case detail::value_t::null:
+            {
+                if (reference_token == "0")
+                {
+                    // start a new array if reference token is 0
+                    result = &result->operator[](0);
+                }
+                else
+                {
+                    // start a new object otherwise
+                    result = &result->operator[](reference_token);
+                }
+                break;
+            }
+
+            case detail::value_t::object:
+            {
+                // create an entry in the object
+                result = &result->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                // create an entry in the array
+                JSON_TRY
+                {
+                    result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            /*
+            The following code is only reached if there exists a reference
+            token _and_ the current value is primitive. In this case, we have
+            an error situation, because primitive values may only occur as
+            single value; that is, with an empty list of reference tokens.
+            */
+            default:
+                JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
+        }
+    }
+
+    return *result;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        // convert null values to arrays or objects before continuing
+        if (ptr->m_type == detail::value_t::null)
+        {
+            // check if reference token is a number
+            const bool nums =
+                std::all_of(reference_token.begin(), reference_token.end(),
+                            [](const char x)
+            {
+                return (x >= '0' and x <= '9');
+            });
+
+            // change value to array for numbers or "-" or to object otherwise
+            *ptr = (nums or reference_token == "-")
+                   ? detail::value_t::array
+                   : detail::value_t::object;
+        }
+
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // use unchecked object access
+                ptr = &ptr->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                if (reference_token == "-")
+                {
+                    // explicitly treat "-" as index beyond the end
+                    ptr = &ptr->operator[](ptr->m_value.array->size());
+                }
+                else
+                {
+                    // convert array index to number; unchecked access
+                    JSON_TRY
+                    {
+                        ptr = &ptr->operator[](
+                            static_cast<size_type>(std::stoi(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // note: at performs range check
+                ptr = &ptr->at(reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" always fails the range check
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // note: at performs range check
+                JSON_TRY
+                {
+                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+const NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // use unchecked object access
+                ptr = &ptr->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" cannot be used for const access
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // use unchecked array access
+                JSON_TRY
+                {
+                    ptr = &ptr->operator[](
+                        static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+const NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // note: at performs range check
+                ptr = &ptr->at(reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" always fails the range check
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // note: at performs range check
+                JSON_TRY
+                {
+                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+void json_pointer::flatten(const std::string& reference_string,
+                           const NLOHMANN_BASIC_JSON_TPL& value,
+                           NLOHMANN_BASIC_JSON_TPL& result)
+{
+    switch (value.m_type)
+    {
+        case detail::value_t::array:
+        {
+            if (value.m_value.array->empty())
+            {
+                // flatten empty array as null
+                result[reference_string] = nullptr;
+            }
+            else
+            {
+                // iterate array and use index as reference string
+                for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
+                {
+                    flatten(reference_string + "/" + std::to_string(i),
+                            value.m_value.array->operator[](i), result);
+                }
+            }
+            break;
+        }
+
+        case detail::value_t::object:
+        {
+            if (value.m_value.object->empty())
+            {
+                // flatten empty object as null
+                result[reference_string] = nullptr;
+            }
+            else
+            {
+                // iterate object and use keys as reference string
+                for (const auto& element : *value.m_value.object)
+                {
+                    flatten(reference_string + "/" + escape(element.first), element.second, result);
+                }
+            }
+            break;
+        }
+
+        default:
+        {
+            // add primitive value with its reference string
+            result[reference_string] = value;
+            break;
+        }
+    }
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL
+json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value)
+{
+    if (JSON_UNLIKELY(not value.is_object()))
+    {
+        JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
+    }
+
+    NLOHMANN_BASIC_JSON_TPL result;
+
+    // iterate the JSON object values
+    for (const auto& element : *value.m_value.object)
+    {
+        if (JSON_UNLIKELY(not element.second.is_primitive()))
+        {
+            JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+        }
+
+        // assign value to reference pointed to by JSON pointer; Note that if
+        // the JSON pointer is "" (i.e., points to the whole value), function
+        // get_and_create returns a reference to result itself. An assignment
+        // will then create a primitive value.
+        json_pointer(element.first).get_and_create(result) = element.second;
+    }
+
+    return result;
+}
+
+inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept
+{
+    return (lhs.reference_tokens == rhs.reference_tokens);
+}
+
+inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept
+{
+    return not (lhs == rhs);
+}
+} // namespace nlohmann
+
+
+///////////////////////
+// nonmember support //
+///////////////////////
+
+// specialization of std::swap, and std::hash
+namespace std
+{
+/*!
+@brief exchanges the values of two JSON objects
+
+@since version 1.0.0
+*/
+template<>
+inline void swap(nlohmann::json& j1,
+                 nlohmann::json& j2) noexcept(
+                     is_nothrow_move_constructible<nlohmann::json>::value and
+                     is_nothrow_move_assignable<nlohmann::json>::value
+                 )
+{
+    j1.swap(j2);
+}
+
+/// hash value for JSON objects
+template<>
+struct hash<nlohmann::json>
+{
+    /*!
+    @brief return a hash value for a JSON object
+
+    @since version 1.0.0
+    */
+    std::size_t operator()(const nlohmann::json& j) const
+    {
+        // a naive hashing via the string representation
+        const auto& h = hash<nlohmann::json::string_t>();
+        return h(j.dump());
+    }
+};
+
+/// specialization for std::less<value_t>
+/// @note: do not remove the space after '<',
+///        see https://github.com/nlohmann/json/pull/679
+template<>
+struct less< ::nlohmann::detail::value_t>
+{
+    /*!
+    @brief compare two value_t enum values
+    @since version 3.0.0
+    */
+    bool operator()(nlohmann::detail::value_t lhs,
+                    nlohmann::detail::value_t rhs) const noexcept
+    {
+        return nlohmann::detail::operator<(lhs, rhs);
+    }
+};
+
+} // namespace std
+
+/*!
+@brief user-defined string literal for JSON values
+
+This operator implements a user-defined string literal for JSON objects. It
+can be used by adding `"_json"` to a string literal and returns a JSON object
+if no parse error occurred.
+
+@param[in] s  a string representation of a JSON object
+@param[in] n  the length of string @a s
+@return a JSON object
+
+@since version 1.0.0
+*/
+inline nlohmann::json operator "" _json(const char* s, std::size_t n)
+{
+    return nlohmann::json::parse(s, s + n);
+}
+
+/*!
+@brief user-defined string literal for JSON pointer
+
+This operator implements a user-defined string literal for JSON Pointers. It
+can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer
+object if no parse error occurred.
+
+@param[in] s  a string representation of a JSON Pointer
+@param[in] n  the length of string @a s
+@return a JSON pointer object
+
+@since version 2.0.0
+*/
+inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)
+{
+    return nlohmann::json::json_pointer(std::string(s, n));
+}
+
+// restore GCC/clang diagnostic settings
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #pragma GCC diagnostic pop
+#endif
+#if defined(__clang__)
+    #pragma GCC diagnostic pop
+#endif
+
+// clean up
+#undef JSON_CATCH
+#undef JSON_THROW
+#undef JSON_TRY
+#undef JSON_LIKELY
+#undef JSON_UNLIKELY
+#undef JSON_DEPRECATED
+#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
+#undef NLOHMANN_BASIC_JSON_TPL
+
+#endif

From 2cb8aaa6109b4f5a31a1091dc3f56c8059a9c2c1 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 6 Dec 2017 11:35:24 +0100
Subject: [PATCH 0706/2196] ssh-store: fix length when writing nar

This fixes nix copy and other things that use copyStorePath.
---
 src/libstore/ssh-store.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index bb536fadfd5..107c6e1ecb4 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -62,9 +62,9 @@ class ForwardSource : public Source
     ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {}
     size_t read(unsigned char * data, size_t len) override
     {
-        auto res = readSource.read(data, len);
-        writeSink(data, len);
-        return res;
+        auto n = readSource.read(data, len);
+        writeSink(data, n);
+        return n;
     }
 };
 

From 338f29dbd4ee04f2de4c747eadb8c106a98f885c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2017 22:44:08 +0100
Subject: [PATCH 0707/2196] nix ls-{nar,store}: Return offset of files in the
 NAR if known

E.g.

  $ nix ls-store --json --recursive --store https://cache.nixos.org /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79 \
    | jq .entries.bin.entries.blender.narOffset
  400
---
 src/libstore/fs-accessor.hh  |  7 ++++---
 src/libstore/nar-accessor.cc | 10 ++++++----
 tests/nar-access.sh          |  4 ++--
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index a67e0775b97..f703e1d1540 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -13,9 +13,10 @@ public:
 
     struct Stat
     {
-        Type type;
-        uint64_t fileSize; // regular files only
-        bool isExecutable; // regular files only
+        Type type = tMissing;
+        uint64_t fileSize = 0; // regular files only
+        bool isExecutable = false; // regular files only
+        uint64_t narOffset = 0; // regular files only
     };
 
     virtual Stat stat(const Path & path) = 0;
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 839a7991c89..d6e9757c2cb 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -10,13 +10,13 @@ namespace nix {
 
 struct NarMember
 {
-    FSAccessor::Type type;
+    FSAccessor::Type type = FSAccessor::Type::tMissing;
 
-    bool isExecutable;
+    bool isExecutable = false;
 
     /* If this is a regular file, position of the contents of this
        file in the NAR. */
-    size_t start, size;
+    size_t start = 0, size = 0;
 
     std::string target;
 
@@ -142,7 +142,7 @@ struct NarAccessor : public FSAccessor
         auto i = indexer.find(path);
         if (i == nullptr)
             return {FSAccessor::Type::tMissing, 0, false};
-        return {i->type, i->size, i->isExecutable};
+        return {i->type, i->size, i->isExecutable, i->start};
     }
 
     StringSet readDirectory(const Path & path) override
@@ -195,6 +195,8 @@ void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
         obj.attr("size", st.fileSize);
         if (st.isExecutable)
             obj.attr("executable", true);
+        if (st.narOffset)
+            obj.attr("narOffset", st.narOffset);
         break;
     case FSAccessor::Type::tDirectory:
         obj.attr("type", "directory");
diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index e29e4689b63..bd849cbfab1 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -27,8 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz
 
 # Test --json.
 [[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
-[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
-[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0}' ]]
+[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]]
+[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]]
 [[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
 [[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
 [[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]

From 2df9cbeb47177d6a21606b4b509ebcf6bd0437a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Dec 2017 00:50:46 +0100
Subject: [PATCH 0708/2196] Provide random access to cached NARs

E.g.

  $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \
    /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null
  real    0m4.139s

  $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \
    /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null
  real    0m0.024s

(Before, the second call took ~0.220s.)

This will use a NAR listing in
/tmp/nars/b0w2hafndl09h64fhb86kw6bmhbmnpm1.ls containing all metadata,
including the offsets of regular files inside the NAR. Thus, we don't
need to read the entire NAR. (We do read the entire listing, but
that's generally pretty small. We could use a SQLite DB by borrowing
some more code from nixos-channel-scripts/file-cache.hh.)

This is primarily useful when Hydra is serving files from an S3 binary
cache, in particular when you have giant NARs. E.g. we had some 12 GiB
NARs, so accessing individuals files was pretty slow.
---
 src/libstore/binary-cache-store.cc |  12 +-
 src/libstore/nar-accessor.cc       | 193 ++++++++++++++++++-----------
 src/libstore/nar-accessor.hh       |  10 ++
 src/libstore/remote-fs-accessor.cc |  82 +++++++++---
 src/libstore/remote-fs-accessor.hh |   5 +-
 5 files changed, 198 insertions(+), 104 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 68af85bf16d..ab971dd8b6d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -129,10 +129,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             auto narAccessor = makeNarAccessor(nar);
 
-            if (accessor_) {
-                accessor_->nars.emplace(info.path, narAccessor);
-                accessor_->addToCache(info.path, *nar);
-            }
+            if (accessor_)
+                accessor_->addToCache(info.path, *nar, narAccessor);
 
             {
                 auto res = jsonRoot.placeholder("root");
@@ -144,10 +142,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     }
 
     else {
-        if (accessor_) {
-            accessor_->nars.emplace(info.path, makeNarAccessor(nar));
-            accessor_->addToCache(info.path, *nar);
-        }
+        if (accessor_)
+            accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
     }
 
     /* Compress the NAR. */
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index d6e9757c2cb..b74480684f2 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -6,6 +6,8 @@
 #include <stack>
 #include <algorithm>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 struct NarMember
@@ -24,83 +26,127 @@ struct NarMember
     std::map<std::string, NarMember> children;
 };
 
-struct NarIndexer : ParseSink, StringSource
+struct NarAccessor : public FSAccessor
 {
-    NarMember root;
-    std::stack<NarMember*> parents;
+    std::shared_ptr<const std::string> nar;
 
-    std::string currentStart;
-    bool isExec = false;
+    GetNarBytes getNarBytes;
 
-    NarIndexer(const std::string & nar) : StringSource(nar)
+    NarMember root;
+
+    struct NarIndexer : ParseSink, StringSource
     {
-    }
+        NarAccessor & acc;
 
-    void createMember(const Path & path, NarMember member) {
-        size_t level = std::count(path.begin(), path.end(), '/');
-        while(parents.size() > level) {
-            parents.pop();
-        }
+        std::stack<NarMember *> parents;
 
-        if(parents.empty()) {
-            root = std::move(member);
-            parents.push(&root);
-        } else {
-            if(parents.top()->type != FSAccessor::Type::tDirectory) {
-                throw Error(format("NAR file missing parent directory of path '%1%'") % path);
+        std::string currentStart;
+        bool isExec = false;
+
+        NarIndexer(NarAccessor & acc, const std::string & nar)
+            : StringSource(nar), acc(acc)
+        { }
+
+        void createMember(const Path & path, NarMember member) {
+            size_t level = std::count(path.begin(), path.end(), '/');
+            while (parents.size() > level) parents.pop();
+
+            if (parents.empty()) {
+                acc.root = std::move(member);
+                parents.push(&acc.root);
+            } else {
+                if (parents.top()->type != FSAccessor::Type::tDirectory)
+                    throw Error("NAR file missing parent directory of path '%s'", path);
+                auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
+                parents.push(&result.first->second);
             }
-            auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
-            parents.push(&result.first->second);
         }
-    }
 
-    void createDirectory(const Path & path) override
-    {
-        createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 });
-    }
+        void createDirectory(const Path & path) override
+        {
+            createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
+        }
 
-    void createRegularFile(const Path & path) override
-    {
-        createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 });
-    }
+        void createRegularFile(const Path & path) override
+        {
+            createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
+        }
 
-    void isExecutable() override
-    {
-        parents.top()->isExecutable = true;
-    }
+        void isExecutable() override
+        {
+            parents.top()->isExecutable = true;
+        }
 
-    void preallocateContents(unsigned long long size) override
-    {
-        currentStart = string(s, pos, 16);
-        assert(size <= std::numeric_limits<size_t>::max());
-        parents.top()->size = (size_t)size;
-        parents.top()->start = pos;
-    }
+        void preallocateContents(unsigned long long size) override
+        {
+            currentStart = string(s, pos, 16);
+            assert(size <= std::numeric_limits<size_t>::max());
+            parents.top()->size = (size_t)size;
+            parents.top()->start = pos;
+        }
 
-    void receiveContents(unsigned char * data, unsigned int len) override
-    {
-        // Sanity check
-        if (!currentStart.empty()) {
-            assert(len < 16 || currentStart == string((char *) data, 16));
-            currentStart.clear();
+        void receiveContents(unsigned char * data, unsigned int len) override
+        {
+            // Sanity check
+            if (!currentStart.empty()) {
+                assert(len < 16 || currentStart == string((char *) data, 16));
+                currentStart.clear();
+            }
+        }
+
+        void createSymlink(const Path & path, const string & target) override
+        {
+            createMember(path,
+                NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
+    };
+
+    NarAccessor(ref<const std::string> nar) : nar(nar)
+    {
+        NarIndexer indexer(*this, *nar);
+        parseDump(indexer, indexer);
     }
 
-    void createSymlink(const Path & path, const string & target) override
+    NarAccessor(const std::string & listing, GetNarBytes getNarBytes)
+        : getNarBytes(getNarBytes)
     {
-        createMember(path,
-            NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
+        using json = nlohmann::json;
+
+        std::function<void(NarMember &, json &)> recurse;
+
+        recurse = [&](NarMember & member, json & v) {
+            std::string type = v["type"];
+
+            if (type == "directory") {
+                member.type = FSAccessor::Type::tDirectory;
+                for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
+                    std::string name = i.key();
+                    recurse(member.children[name], i.value());
+                }
+            } else if (type == "regular") {
+                member.type = FSAccessor::Type::tRegular;
+                member.size = v["size"];
+                member.isExecutable = v.value("executable", false);
+                member.start = v["narOffset"];
+            } else if (type == "symlink") {
+                member.type = FSAccessor::Type::tSymlink;
+                member.target = v.value("target", "");
+            } else return;
+        };
+
+        json v = json::parse(listing);
+        recurse(root, v);
     }
 
-    NarMember* find(const Path & path)
+    NarMember * find(const Path & path)
     {
         Path canon = path == "" ? "" : canonPath(path);
-        NarMember* current = &root;
+        NarMember * current = &root;
         auto end = path.end();
-        for(auto it = path.begin(); it != end; ) {
+        for (auto it = path.begin(); it != end; ) {
             // because it != end, the remaining component is non-empty so we need
             // a directory
-            if(current->type != FSAccessor::Type::tDirectory) return nullptr;
+            if (current->type != FSAccessor::Type::tDirectory) return nullptr;
 
             // skip slash (canonPath above ensures that this is always a slash)
             assert(*it == '/');
@@ -109,7 +155,7 @@ struct NarIndexer : ParseSink, StringSource
             // lookup current component
             auto next = std::find(it, end, '/');
             auto child = current->children.find(std::string(it, next));
-            if(child == current->children.end()) return nullptr;
+            if (child == current->children.end()) return nullptr;
             current = &child->second;
 
             it = next;
@@ -118,28 +164,16 @@ struct NarIndexer : ParseSink, StringSource
         return current;
     }
 
-    NarMember& at(const Path & path) {
+    NarMember & get(const Path & path) {
         auto result = find(path);
-        if(result == nullptr) {
-            throw Error(format("NAR file does not contain path '%1%'") % path);
-        }
+        if (result == nullptr)
+            throw Error("NAR file does not contain path '%1%'", path);
         return *result;
     }
-};
-
-struct NarAccessor : public FSAccessor
-{
-    ref<const std::string> nar;
-    NarIndexer indexer;
-
-    NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
-    {
-        parseDump(indexer, indexer);
-    }
 
     Stat stat(const Path & path) override
     {
-        auto i = indexer.find(path);
+        auto i = find(path);
         if (i == nullptr)
             return {FSAccessor::Type::tMissing, 0, false};
         return {i->type, i->size, i->isExecutable, i->start};
@@ -147,30 +181,33 @@ struct NarAccessor : public FSAccessor
 
     StringSet readDirectory(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
 
         if (i.type != FSAccessor::Type::tDirectory)
             throw Error(format("path '%1%' inside NAR file is not a directory") % path);
 
         StringSet res;
-        for(auto&& child : i.children) {
+        for (auto & child : i.children)
             res.insert(child.first);
 
-        }
         return res;
     }
 
     std::string readFile(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)
             throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
+
+        if (getNarBytes) return getNarBytes(i.start, i.size);
+
+        assert(nar);
         return std::string(*nar, i.start, i.size);
     }
 
     std::string readLink(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
         if (i.type != FSAccessor::Type::tSymlink)
             throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
         return i.target;
@@ -182,6 +219,12 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
+    GetNarBytes getNarBytes)
+{
+    return make_ref<NarAccessor>(listing, getNarBytes);
+}
+
 void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
     const Path & path, bool recurse)
 {
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index ed8fe15cad2..1903355a236 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -8,6 +8,16 @@ namespace nix {
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+/* Create a NAR accessor from a NAR listing (in the format produced by
+   listNar()). The callback getNarBytes(offset, length) is used by the
+   readFile() method of the accessor to get the contents of files
+   inside the NAR. */
+typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
+
+ref<FSAccessor> makeLazyNarAccessor(
+    const std::string & listing,
+    GetNarBytes getNarBytes);
+
 class JSONPlaceholder;
 
 /* Write a JSON representation of the contents of a NAR (except file
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index ba9620a175b..5233fb2c239 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -1,5 +1,10 @@
 #include "remote-fs-accessor.hh"
 #include "nar-accessor.hh"
+#include "json.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 namespace nix {
 
@@ -11,20 +16,30 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
-Path RemoteFSAccessor::makeCacheFile(const Path & storePath)
+Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath));
+    return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
 }
 
-void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar)
+void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
+    ref<FSAccessor> narAccessor)
 {
-    try {
-        if (cacheDir == "") return;
-        /* FIXME: do this asynchronously. */
-        writeFile(makeCacheFile(storePath), nar);
-    } catch (...) {
-        ignoreException();
+    nars.emplace(storePath, narAccessor);
+
+    if (cacheDir != "") {
+        try {
+            std::ostringstream str;
+            JSONPlaceholder jsonRoot(str);
+            listNar(jsonRoot, narAccessor, "", true);
+            writeFile(makeCacheFile(storePath, "ls"), str.str());
+
+            /* FIXME: do this asynchronously. */
+            writeFile(makeCacheFile(storePath, "nar"), nar);
+
+        } catch (...) {
+            ignoreException();
+        }
     }
 }
 
@@ -42,20 +57,49 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
+    std::string listing;
+    Path cacheFile;
+
+    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
+
+        try {
+            listing = nix::readFile(makeCacheFile(storePath, "ls"));
+
+            auto narAccessor = makeLazyNarAccessor(listing,
+                [cacheFile](uint64_t offset, uint64_t length) {
+
+                    AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
+                    if (!fd)
+                        throw SysError("opening NAR cache file '%s'", cacheFile);
+
+                    if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset)
+                        throw SysError("seeking in '%s'", cacheFile);
+
+                    std::string buf(length, 0);
+                    readFull(fd.get(), (unsigned char *) buf.data(), length);
+
+                    return buf;
+                });
+
+            nars.emplace(storePath, narAccessor);
+            return {narAccessor, restPath};
+
+        } catch (SysError &) { }
+
+        try {
+            *sink.s = nix::readFile(cacheFile);
 
-    try {
-        if (cacheDir != "")
-            *sink.s = nix::readFile(makeCacheFile(storePath));
-    } catch (SysError &) { }
+            auto narAccessor = makeNarAccessor(sink.s);
+            nars.emplace(storePath, narAccessor);
+            return {narAccessor, restPath};
 
-    if (sink.s->empty()) {
-        store->narFromPath(storePath, sink);
-        addToCache(storePath, *sink.s);
+        } catch (SysError &) { }
     }
 
-    auto accessor = makeNarAccessor(sink.s);
-    nars.emplace(storePath, accessor);
-    return {accessor, restPath};
+    store->narFromPath(storePath, sink);
+    auto narAccessor = makeNarAccessor(sink.s);
+    addToCache(storePath, *sink.s, narAccessor);
+    return {narAccessor, restPath};
 }
 
 FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 2a3fc01eff5..4afb3be9573 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -18,9 +18,10 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
-    Path makeCacheFile(const Path & storePath);
+    Path makeCacheFile(const Path & storePath, const std::string & ext);
 
-    void addToCache(const Path & storePath, const std::string & nar);
+    void addToCache(const Path & storePath, const std::string & nar,
+        ref<FSAccessor> narAccessor);
 
 public:
 

From e3013543d36926ecfe51e9eceab42c88cb40b138 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Dec 2017 01:06:56 +0100
Subject: [PATCH 0709/2196] Fix test

---
 tests/signing.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/signing.sh b/tests/signing.sh
index 8e8215618f8..46929639199 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -53,7 +53,7 @@ nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2
 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1
 
 # Build something content-addressed.
-outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0)
+outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link)
 
 [[ $(nix path-info --json $outPathCA) =~ '"ca":"fixed:md5:' ]]
 

From fbab9ed01e603d5af310f4d9b95e86e29c587f43 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 7 Dec 2017 19:02:35 -0500
Subject: [PATCH 0710/2196] upload-release.pl: support aarch64-linux

---
 maintainers/upload-release.pl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 743829e3f5f..313ff93d4c8 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -83,6 +83,7 @@ sub downloadFile {
 my ($tarball, $tarballHash) = downloadFile("tarball", "4"); # .tar.xz
 my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1");
 my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1");
+my ($tarball_aarch64_linux, $tarball_aarch64_linux_hash) = downloadFile("binaryTarball.aarch64-linux", "1");
 my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1");
 
 # Update Nixpkgs in a very hacky way.
@@ -111,6 +112,7 @@ sub getStorePath {
            "{\n" .
            "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
            "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+           "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
            "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
            "}\n");
 
@@ -144,6 +146,7 @@ sub getStorePath {
            "latestNixVersion = \"$version\"\n" .
            "nix_hash_i686_linux = \"$tarball_i686_linux_hash\"\n" .
            "nix_hash_x86_64_linux = \"$tarball_x86_64_linux_hash\"\n" .
+           "nix_hash_aarch64_linux = \"$tarball_aarch64_linux_hash\"\n" .
            "nix_hash_x86_64_darwin = \"$tarball_x86_64_darwin_hash\"\n" .
            "-%]\n");
 

From 0af668426d0916c9d09a06d7aacf2dc1e9590289 Mon Sep 17 00:00:00 2001
From: Markus Hauck <markus1189@gmail.com>
Date: Fri, 8 Dec 2017 10:27:34 +0100
Subject: [PATCH 0711/2196] nix-hash: Add sentence and example for
 nix-prefetch-url hash

---
 doc/manual/command-ref/nix-hash.xml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-hash.xml b/doc/manual/command-ref/nix-hash.xml
index b4b509773d3..8d8213c0bca 100644
--- a/doc/manual/command-ref/nix-hash.xml
+++ b/doc/manual/command-ref/nix-hash.xml
@@ -44,7 +44,9 @@
 cryptographic hash of the contents of each
 <replaceable>path</replaceable> and prints it on standard output.  By
 default, it computes an MD5 hash, but other hash algorithms are
-available as well.  The hash is printed in hexadecimal.</para>
+available as well.  The hash is printed in hexadecimal.  To generate
+the same hash as <command>nix-prefetch-url</command> you have to
+specify multiple arguments, see below for an example.</para>
 
 <para>The hash is computed over a <emphasis>serialisation</emphasis>
 of each path: a dump of the file system tree rooted at the path.  This
@@ -122,6 +124,15 @@ cryptographic hash as <literal>nix-store --dump
 
 <refsection><title>Examples</title>
 
+<para>Computing the same hash as <command>nix-prefetch-url</command>:
+<screen>
+$ nix-prefetch-url file://<(echo test)
+1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+$ nix-hash --type sha256 --flat --base32 <(echo test)
+1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
+</screen>
+</para>
+
 <para>Computing hashes:
 
 <screen>

From be79d1f1890c737dd04886120ef900d369d48a22 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 25 Nov 2017 20:54:40 -0500
Subject: [PATCH 0712/2196] darwin installer: fix on High Sierra

---
 scripts/install-darwin-multi-user.sh | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 6369b68ab91..968dd1b8454 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -473,10 +473,8 @@ create_build_user_for_core() {
 
     if ! /usr/bin/dscl . -read "$dsclpath" > /dev/null 2>&1; then
         _sudo "Creating the Nix build user, $username" \
-              /usr/sbin/sysadminctl -addUser -fullName "Nix build user $coreid" \
-	      -home /var/empty \
-	      -UID "${uid}" \
-              -addUser "${username}"
+              /usr/bin/dscl . create "$dsclpath" \
+              UniqueID "${uid}"
         row "           Created" "Yes"
     else
         actual_uid=$(dsclattr "$dsclpath" "UniqueID")
@@ -504,6 +502,22 @@ EOF
         row "          IsHidden" "Yes"
     fi
 
+    if [ "$(dsclattr "$dsclpath" "NFSHomeDirectory")" = "/var/empty" ]; then
+        row "          NFSHomeDirectory" "/var/empty"
+    else
+        _sudo "in order to give $username a safe home directory" \
+              /usr/bin/dscl . -create "$dsclpath" "NFSHomeDirectory" "/var/empty"
+        row "          NFSHomeDirectory" "/var/empty"
+    fi
+
+    if [ "$(dsclattr "$dsclpath" "RealName")" = "Nix build user $coreid" ]; then
+        row "          RealName" "Nix build user $coreid"
+    else
+        _sudo "in order to give $username a useful name" \
+              /usr/bin/dscl . -create "$dsclpath" "RealName" "Nix build user $coreid"
+        row "          RealName" "Nix build user $coreid"
+    fi
+
     if [ "$(dsclattr "$dsclpath" "UserShell")" = "/sbin/nologin" ]; then
         row "   Logins Disabled" "Yes"
     else
@@ -521,11 +535,11 @@ EOF
         row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
     fi
 
-    if [ "$(dsclattr "$dsclpath" "PrimaryGroupId")" = "$NIX_BUILD_GROUP_ID" ]; then
+    if [ "$(dsclattr "$dsclpath" "PrimaryGroupID")" = "$NIX_BUILD_GROUP_ID" ]; then
         row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
     else
         _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
-              /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupId" "$NIX_BUILD_GROUP_ID"
+              /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
         row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
 
     fi

From 1db034364a69acfe59c4e5a1792afccb99457a9b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 25 Nov 2017 21:05:33 -0500
Subject: [PATCH 0713/2196] replace lolcat with nix-info

---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 968dd1b8454..b6b3305bac7 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -264,7 +264,7 @@ them and open them again. Other than that, you should be ready to go.
 
 Try it! Open a new terminal, and type:
 
-  $ nix-shell -p figlet -p lolcat --run "echo 'nix rules' | figlet | lolcat"
+  $ nix-shell -p nix-info --run "nix-info -m"
 
 Thank you for using this installer. If you have any feedback, don't
 hesitate:

From 35a49f1d7fad32ab2668a9acd0dc18fb191dde8c Mon Sep 17 00:00:00 2001
From: Joe Hermaszewski <github@sub.monoid.al>
Date: Sat, 9 Dec 2017 15:31:03 +0000
Subject: [PATCH 0714/2196] Escape left angle brackets in XML documentation

---
 doc/manual/command-ref/nix-hash.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/nix-hash.xml b/doc/manual/command-ref/nix-hash.xml
index 8d8213c0bca..80263e18e33 100644
--- a/doc/manual/command-ref/nix-hash.xml
+++ b/doc/manual/command-ref/nix-hash.xml
@@ -126,9 +126,9 @@ cryptographic hash as <literal>nix-store --dump
 
 <para>Computing the same hash as <command>nix-prefetch-url</command>:
 <screen>
-$ nix-prefetch-url file://<(echo test)
+$ nix-prefetch-url file://&lt;(echo test)
 1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
-$ nix-hash --type sha256 --flat --base32 <(echo test)
+$ nix-hash --type sha256 --flat --base32 &lt;(echo test)
 1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj
 </screen>
 </para>

From d76c4fe77080a924dd870e5c5a370a4e4a30d0d4 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sat, 9 Dec 2017 20:02:21 -0500
Subject: [PATCH 0715/2196] Fix for builds with system libcurl < 7.30

CentOS 7.4 and RHEL 7.4 ship with libcurl-devel-7.29.0-42.el7.x86_64; this flag
was added in 7.30.0
https://curl.haxx.se/libcurl/c/CURLMOPT_MAX_TOTAL_CONNECTIONS.html
---
 src/libstore/download.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index da31029b4f8..4474dfd4b96 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -369,11 +369,13 @@ struct CurlDownloader : public Downloader
 
         curlm = curl_multi_init();
 
-        #if LIBCURL_VERSION_NUM >= 0x072b00 // correct?
+        #if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0
         curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
         #endif
+        #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
             settings.binaryCachesParallelConnections.get());
+        #endif
 
         enableHttp2 = settings.enableHttp2;
 

From c800f473950066c932cbb88fca17f9d31385e76c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 11 Dec 2017 11:51:55 -0600
Subject: [PATCH 0716/2196] Fix forward-decl of Bindings as "class", match
 definition.

(appease clang -Wmismatched-tags warning)
---
 src/nix/command.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/command.hh b/src/nix/command.hh
index daa3b3fa703..6b34e3881e7 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -6,7 +6,7 @@
 namespace nix {
 
 struct Value;
-struct Bindings;
+class Bindings;
 class EvalState;
 
 /* A command is an argument parser that can be executed by calling its

From e4d43f1f94f7488ae8dc1139c8485446c4209836 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 11 Dec 2017 11:58:58 -0600
Subject: [PATCH 0717/2196] logging.cc: add missing 'override'

---
 src/libutil/logging.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 01115587112..6924e008047 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -49,6 +49,7 @@ class SimpleLogger : public Logger
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent)
+        override
     {
         if (lvl <= verbosity && !s.empty())
             log(lvl, s + "...");

From 00e0c416ffccf6edb63fe7f4b01422915ff12f86 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 11 Dec 2017 12:05:14 -0600
Subject: [PATCH 0718/2196] Mark comparison call operator as const

---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 866964a4c46..d4bd650baf2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -79,7 +79,7 @@ typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
 
 struct CompareGoalPtrs {
-    bool operator() (const GoalPtr & a, const GoalPtr & b);
+    bool operator() (const GoalPtr & a, const GoalPtr & b) const;
 };
 
 /* Set of goals. */
@@ -178,7 +178,7 @@ class Goal : public std::enable_shared_from_this<Goal>
 };
 
 
-bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) {
+bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
     string s1 = a->key();
     string s2 = b->key();
     return s1 < s2;

From 2a8f09a8c12246d52a3a0443b4bc9d87a632e98e Mon Sep 17 00:00:00 2001
From: Chris Martin <ch.martin@gmail.com>
Date: Mon, 11 Dec 2017 01:20:17 -0500
Subject: [PATCH 0719/2196] Add builtins.concatStringSep to the manual

---
 doc/manual/expressions/builtins.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 5e88b88561f..5a3a8645c1d 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -134,6 +134,14 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
+  <varlistentry><term><function>builtins.concatStringsSep</function>
+  <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
+
+    <listitem><para>Concatenate a list of strings with a separator
+    between each element, e.g. <literal>concatStringsSep "/"
+    ["usr" "local" "bin"] == "usr/local/bin"</literal></para></listitem>
+
+  </varlistentry>
 
   <varlistentry
   xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term>

From e4ece83b1aad1bbbdc7cd39d9d0829f85330e505 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 12 Dec 2017 08:31:31 -0500
Subject: [PATCH 0720/2196] tests.setuid: only on i686 and x86_64 linuxs

---
 release.nix | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/release.nix b/release.nix
index c5c2170f78d..538211ca949 100644
--- a/release.nix
+++ b/release.nix
@@ -225,10 +225,12 @@ let
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
-    tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
-      import ./tests/setuid.nix rec {
-        nix = build.${system}; inherit system;
-      });
+    tests.setuid = pkgs.lib.genAttrs
+      (pkgs.lib.filter (system: system == "x86_64-linux" || system == "i686-linux") systems)
+      (system:
+        import ./tests/setuid.nix rec {
+          nix = build.${system}; inherit system;
+        });
 
     tests.binaryTarball =
       with import <nixpkgs> { system = "x86_64-linux"; };

From 0413aeb35d6ee869a98b6565781c1cf47dc80080 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 31 Oct 2017 11:57:53 +0100
Subject: [PATCH 0721/2196] search.cc: sort attribute names with `std::map`

---
 src/nix/search.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 87cdb2d7ed8..0fbd3b6c073 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -82,8 +82,6 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         auto state = getEvalState();
 
-        bool first = true;
-
         auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
 
         auto sToplevel = state->symbols.create("_toplevel");
@@ -91,6 +89,8 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
         bool fromCache = false;
 
+        std::map<std::string, std::string> results;
+
         std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
 
         doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
@@ -138,10 +138,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                             jsonElem.attr("description", description);
 
                         } else {
-                            if (!first) std::cout << "\n";
-                            first = false;
-
-                            std::cout << fmt(
+                            results[attrPath] = fmt(
                                 "Attribute name: %s\n"
                                 "Package name: %s\n"
                                 "Version: %s\n"
@@ -240,6 +237,9 @@ struct CmdSearch : SourceExprCommand, MixJSON
             if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
+
+        for (auto el : results) std::cout << el.second << "\n";
+
     }
 };
 

From 5d5b931fb178046ba286b8ef2b56a00b3a85c51c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Dec 2017 16:45:40 +0100
Subject: [PATCH 0722/2196] Fix build against current aws-sdk-cpp

---
 src/libstore/s3-binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 0079da1becf..f5b6a9125ce 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -87,7 +87,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
                 std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
             : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                 std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
-            *config, true, false))
+            *config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false))
 {
 }
 

From f9bcbddef260872878708317e9f5fa78cd0fe849 Mon Sep 17 00:00:00 2001
From: Ben Gamari <ben@smart-cactus.org>
Date: Thu, 14 Dec 2017 19:09:45 -0500
Subject: [PATCH 0723/2196] json-to-value: Throw sensible error message on
 invalid numbers

---
 src/libexpr/json-to-value.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 9380de3a66b..8b140459554 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -106,10 +106,16 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             tmp_number += *s++;
         }
 
-        if (number_type == tFloat)
-            mkFloat(v, stod(tmp_number));
-        else
-            mkInt(v, stoi(tmp_number));
+        try {
+            if (number_type == tFloat)
+                mkFloat(v, stod(tmp_number));
+            else
+                mkInt(v, stoi(tmp_number));
+        } catch (std::invalid_argument e) {
+            throw JSONParseError("invalid JSON number");
+        } catch (std::out_of_range e) {
+            throw JSONParseError("out-of-range JSON number");
+        }
     }
 
     else if (strncmp(s, "true", 4) == 0) {

From 626a94d70ea7a5c5b87f4d6061633c019ca8ea13 Mon Sep 17 00:00:00 2001
From: Ben Gamari <ben@smart-cactus.org>
Date: Thu, 14 Dec 2017 19:11:56 -0500
Subject: [PATCH 0724/2196] edit: Catch stoi exceptions from line number
 parsing

---
 src/nix/edit.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 127be321eee..7eaa86e2f91 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -52,7 +52,12 @@ struct CmdEdit : InstallableCommand
             throw Error("cannot parse meta.position attribute '%s'", pos);
 
         std::string filename(pos, 0, colon);
-        int lineno = std::stoi(std::string(pos, colon + 1));
+        int lineno;
+        try {
+            lineno = std::stoi(std::string(pos, colon + 1));
+        } catch (std::invalid_argument e) {
+            throw Error("cannot parse line number '%s'", pos);
+        }
 
         auto editor = getEnv("EDITOR", "cat");
 

From 8efe937a350eca57cbc3cb9837f28d985e0d281f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 16 Dec 2017 00:48:09 +0100
Subject: [PATCH 0725/2196] Fix build on gcc 7

Fixes #1738.
---
 src/libstore/nar-accessor.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 1903355a236..2871199de16 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <functional>
+
 #include "fs-accessor.hh"
 
 namespace nix {

From ca40fbdc504b05e73e92d19fdff95e397dc3873a Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Sun, 17 Dec 2017 11:03:44 -0500
Subject: [PATCH 0726/2196] Add an issue template to cut down on the confusion

---
 .github/ISSUE_TEMPLATE.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 00000000000..cdf642e6abd
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,27 @@
+<!--
+
+# Filing a Nix issue
+
+*WAIT* Are you sure you're filing your issue in the right repository?
+
+We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
+
+This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
+
+If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
+
+Examples of _Nix_ issues:
+
+- Nix segfaults when I run `nix-build -A blahblah`
+- The Nix language needs a new builtin: `builtins.foobar`
+- Regression in the behavior of `nix-env` in Nix 1.12
+
+Examples of _nixpkgs_ issues:
+
+- glibc is b0rked on aarch64
+- chromium in NixOS doesn't support U2F but google-chrome does!
+- The OpenJDK package on macOS is missing a key component
+
+Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
+
+-->
\ No newline at end of file

From 71987b18d472c1c32214b8e4b36c09a972c82c0c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 19 Dec 2017 12:48:34 -0600
Subject: [PATCH 0727/2196] linenoise.cpp: allow completions from empty input

Fixes #1742.
---
 src/linenoise/linenoise.cpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp
index 8ee8984d694..c57505d2fa9 100644
--- a/src/linenoise/linenoise.cpp
+++ b/src/linenoise/linenoise.cpp
@@ -2587,13 +2587,6 @@ int InputBuffer::getInputLine(PromptBase& pi) {
 
     // ctrl-I/tab, command completion, needs to be before switch statement
     if (c == ctrlChar('I') && completionCallback) {
-      if (pos == 0)  // SERVER-4967 -- in earlier versions, you could paste
-                     // previous output
-        continue;    //  back into the shell ... this output may have leading
-                     //  tabs.
-      // This hack (i.e. what the old code did) prevents command completion
-      //  on an empty line but lets users paste text with leading tabs.
-
       killRing.lastAction = KillRing::actionOther;
       historyRecallMostRecent = false;
 

From af1e2ffca11c311333f6be13add8a93e7d5428db Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Wed, 20 Dec 2017 13:05:56 +0100
Subject: [PATCH 0728/2196] Fix escaping, fixes build

---
 scripts/install-darwin-multi-user.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index b6b3305bac7..7bd00e42b97 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -153,7 +153,7 @@ subheader() {
 }
 
 row() {
-    printf "$BOLD%s$ESC:\t%s\n" "$1" "$2"
+    printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2"
 }
 
 task() {
@@ -218,7 +218,7 @@ __sudo() {
 
     echo "I am executing:"
     echo ""
-    printf "    $ sudo %s\n" "$cmd"
+    printf "    $ sudo %s\\n" "$cmd"
     echo ""
     echo "$expl"
     echo ""

From 6d8087083278b078d0db6238fb16929163388acd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Dec 2017 11:33:34 +0100
Subject: [PATCH 0729/2196] release.nix: Use fetchTarball and fetchGit

In particular, using fetchGit means we don't need hackery to clean the
source tree when building from an unclean tree.
---
 local.mk                   |  3 +--
 release.nix                | 33 +++++++++++++++++----------------
 tests/nix-copy-closure.nix |  4 ++--
 tests/remote-builds.nix    |  4 ++--
 tests/setuid.nix           |  4 ++--
 5 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/local.mk b/local.mk
index 0a225423741..40a910991a4 100644
--- a/local.mk
+++ b/local.mk
@@ -1,6 +1,5 @@
 ifeq ($(MAKECMDGOALS), dist)
-  # Make sure we are in repo root with `--git-dir`
-  dist-files += $(shell git --git-dir=.git ls-files || find * -type f)
+  dist-files += $(shell cat .dist-files)
 endif
 
 dist-files += configure config.h.in nix.spec perl/configure
diff --git a/release.nix b/release.nix
index c5c2170f78d..04fbcd563b8 100644
--- a/release.nix
+++ b/release.nix
@@ -1,12 +1,12 @@
-{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
-, nixpkgs ? { outPath = <nixpkgs>; revCount = 1234; shortRev = "abcdef"; }
+{ nix ? fetchGit ./.
+, nixpkgs ? fetchTarball channel:nixos-17.09
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
 
 let
 
-  pkgs = import <nixpkgs> {};
+  pkgs = import nixpkgs {};
 
   jobs = rec {
 
@@ -27,16 +27,14 @@ let
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
-            git
           ] ++ lib.optional stdenv.isLinux libseccomp;
 
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          # Clean up when building from a working tree.
-          if [[ -d $sourceRoot/.git ]]; then
-            git -C $sourceRoot clean -fd
-          fi
+          ls -la source
+          (cd source && find . -type f) | cut -c3- > source/.dist-files
+          cat source/.dist-files
         '';
 
         preConfigure = ''
@@ -62,7 +60,7 @@ let
 
     build = pkgs.lib.genAttrs systems (system:
 
-      with import <nixpkgs> { inherit system; };
+      with import nixpkgs { inherit system; };
 
       with import ./release-common.nix { inherit pkgs; };
 
@@ -105,7 +103,7 @@ let
 
     perlBindings = pkgs.lib.genAttrs systems (system:
 
-      let pkgs = import <nixpkgs> { inherit system; }; in with pkgs;
+      let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
         name = "nix-perl";
@@ -131,7 +129,7 @@ let
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
       # FIXME: temporarily use a different branch for the Darwin build.
-      with import <nixpkgs> { inherit system; };
+      with import nixpkgs { inherit system; };
 
       let
         toplevel = builtins.getAttr system jobs.build;
@@ -174,7 +172,7 @@ let
 
 
     coverage =
-      with import <nixpkgs> { system = "x86_64-linux"; };
+      with import nixpkgs { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
@@ -218,20 +216,23 @@ let
 
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
+      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
+      inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
     tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
       import ./tests/setuid.nix rec {
+        inherit nixpkgs;
         nix = build.${system}; inherit system;
       });
 
     tests.binaryTarball =
-      with import <nixpkgs> { system = "x86_64-linux"; };
+      with import nixpkgs { system = "x86_64-linux"; };
       vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
         { diskImage = vmTools.diskImages.ubuntu1204x86_64;
         }
@@ -250,7 +251,7 @@ let
         ''); # */
 
     tests.evalNixpkgs =
-      import <nixpkgs/pkgs/top-level/make-tarball.nix> {
+      import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
         inherit nixpkgs;
         inherit pkgs;
         nix = build.x86_64-linux;
@@ -304,7 +305,7 @@ let
   makeRPM =
     system: diskImageFun: extraPackages:
 
-    with import <nixpkgs> { inherit system; };
+    with import nixpkgs { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm";
@@ -326,7 +327,7 @@ let
   makeDeb =
     system: diskImageFun: extraPackages: extraDebPackages:
 
-    with import <nixpkgs> { inherit system; };
+    with import nixpkgs { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index 44126dd64e4..0bf5b42d84a 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -1,8 +1,8 @@
 # Test ‘nix-copy-closure’.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + /nixos/lib/testing.nix) { inherit system; };
 
 makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 58a26d8b618..75704ace2db 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -1,8 +1,8 @@
 # Test Nix's remote build feature.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
 
 makeTest (
 
diff --git a/tests/setuid.nix b/tests/setuid.nix
index c982d9cf036..77e83c8d6c2 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -1,8 +1,8 @@
 # Verify that Linux builds cannot create setuid or setgid binaries.
 
-{ system, nix }:
+{ nixpkgs, system, nix }:
 
-with import <nixpkgs/nixos/lib/testing.nix> { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
 
 makeTest {
 

From aa43cbb7646e880f871df4280f8a1909520136f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Dec 2017 12:05:13 +0100
Subject: [PATCH 0730/2196] Check aws-sdk-cpp version

---
 configure.ac                          |  8 +++++++-
 src/libstore/s3-binary-cache-store.cc | 10 +++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 9d8a81d0427..c395b8713f2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -186,11 +186,17 @@ fi
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-cpp-sdk-s3.])
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.])
   enable_s3=1], [enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
+if test -n "$enable_s3"; then
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | cpp -E | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
+fi
+
 
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f5b6a9125ce..23af452094c 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -10,6 +10,7 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
+#include <aws/core/VersionConfig.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
@@ -87,7 +88,14 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
                 std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
             : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                 std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
-            *config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false))
+            *config,
+            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
+#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
+            false,
+#else
+            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+#endif
+            false))
 {
 }
 

From 2e6f06c37e26a5ac5be35fe18f283a1b26de64bf Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 22 Dec 2017 14:18:29 -0600
Subject: [PATCH 0731/2196] fetchGit: Fix handling of local repo when not using
 'master' branch

Add tests checking this behavior.
---
 src/libexpr/primops/fetchGit.cc |  6 +++++-
 tests/fetchGit.sh               | 26 ++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index e92e0638031..0d0b11958a4 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -23,7 +23,7 @@ struct GitInfo
 };
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, const std::string & rev,
+    std::experimental::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
@@ -68,6 +68,10 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
             return gitInfo;
         }
+
+        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
+        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
+        ref = "HEAD"s;
     }
 
     if (!ref) ref = "master"s;
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 09e4f742668..65d673c0885 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -93,3 +93,29 @@ git -C $repo add hello
 git -C $repo commit -m 'Bla4'
 rev3=$(git -C $repo rev-parse HEAD)
 nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null
+
+# Update 'path' to reflect latest master
+path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+
+# Check behavior when non-master branch is used
+git -C $repo checkout $rev2 -b dev
+echo dev > $repo/hello
+
+# File URI uses 'master' unless specified otherwise
+path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+[[ $path = $path2 ]]
+
+# Using local path with branch other than 'master' should work when clean or dirty
+path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+# (check dirty-tree handling was used)
+[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+
+# Committing shouldn't change store path, or switch to using 'master'
+git -C $repo commit -m 'Bla5' -a
+path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+[[ $(cat $path4/hello) = dev ]]
+[[ $path3 = $path4 ]]
+
+# Confirm same as 'dev' branch
+path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+[[ $path3 = $path5 ]]

From 4801420893766a3845f6ce6dc8c07bab1a2f15e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Dec 2017 14:53:15 +0100
Subject: [PATCH 0732/2196] Remove debug line

---
 release.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/release.nix b/release.nix
index 04fbcd563b8..68b58623289 100644
--- a/release.nix
+++ b/release.nix
@@ -32,7 +32,6 @@ let
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          ls -la source
           (cd source && find . -type f) | cut -c3- > source/.dist-files
           cat source/.dist-files
         '';

From bd17ccf1d822ba76cdd58e9547bc18db35189c55 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 26 Dec 2017 19:22:28 -0600
Subject: [PATCH 0733/2196] nix repl: use linenoiseKeyType to differentiate ^C
 and ^D

Fixes #1757.
---
 src/nix/repl.cc | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 1adb816c5bf..9216209173d 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -186,7 +186,16 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
 {
     char * s = linenoise(prompt.c_str());
     Finally doFree([&]() { free(s); });
-    if (!s) return false;
+    if (!s) {
+      switch (auto type = linenoiseKeyType()) {
+        case 1: // ctrl-C
+          return true;
+        case 2: // ctrl-D
+          return false;
+        default:
+          throw Error(format("Unexpected linenoise keytype: %1%") % type);
+      }
+    }
     input += s;
     return true;
 }

From ab8ba712054e170dfbef30f53b4a5a88b93aad6f Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Fri, 29 Dec 2017 13:45:54 +0100
Subject: [PATCH 0734/2196] Do not export ASPELL_CONF

This does not belong in Nix. Setting this env var is already done by the aspell derivation found in Nixpkgs.
---
 scripts/nix-profile.sh.in | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index ab95c09c830..450d683c7f4 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -60,12 +60,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR"
 
-    for i in $NIX_PROFILES; do
-        if [ -d "$i/lib/aspell" ]; then
-            export ASPELL_CONF="dict-dir $i/lib/aspell"
-        fi
-    done
-
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

From 9dd2b8ac7b8d82df8c1f3f36efb683175fd6ecee Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 29 Dec 2017 14:42:14 -0600
Subject: [PATCH 0735/2196] use libbrotli directly when available

* Look for both 'brotli' and 'bro' as external command,
  since upstream has renamed it in newer versions.
  If neither are found, current runtime behavior
  is preserved: try to find 'bro' on PATH.
* Limit amount handed to BrotliEncoderCompressStream
  to ensure interrupts are processed in a timely manner.
  Testing shows negligible performance impact.
  (Other compression sinks don't seem to require this)
---
 Makefile.config.in         |   4 +-
 configure.ac               |   9 +-
 src/libutil/compression.cc | 171 +++++++++++++++++++++++++++++++++++--
 src/libutil/local.mk       |   4 +-
 tests/brotli.sh            |  28 ++++++
 tests/common.sh.in         |   1 +
 tests/local.mk             |   3 +-
 7 files changed, 207 insertions(+), 13 deletions(-)
 create mode 100644 tests/brotli.sh

diff --git a/Makefile.config.in b/Makefile.config.in
index 45a70cd6dd1..fab82194656 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -6,6 +6,7 @@ CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
+HAVE_BROTLI = @HAVE_BROTLI@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
@@ -13,9 +14,10 @@ PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 bash = @bash@
 bindir = @bindir@
-bro = @bro@
+brotli = @brotli@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
diff --git a/configure.ac b/configure.ac
index c395b8713f2..9db92ce9140 100644
--- a/configure.ac
+++ b/configure.ac
@@ -127,7 +127,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
-AC_PATH_PROG(bro, bro, bro)
+AC_PATH_PROGS(brotli, brotli bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
@@ -176,6 +176,13 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 
 
+# Look for libbrotli{enc,dec}, optional dependencies
+PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec],
+  [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.])
+   CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]
+   have_brotli=1], [have_brotli=])
+AC_SUBST(HAVE_BROTLI, [$have_brotli])
+
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
   PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 2b3dff3a5ea..5e2631ba340 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -7,6 +7,11 @@
 #include <cstdio>
 #include <cstring>
 
+#if HAVE_BROTLI
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+#endif // HAVE_BROTLI
+
 #include <iostream>
 
 namespace nix {
@@ -94,8 +99,56 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
 static ref<std::string> decompressBrotli(const std::string & in)
 {
-    // FIXME: use libbrotli
-    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in}));
+#if !HAVE_BROTLI
+    return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in}));
+#else
+    auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+    if (!s)
+        throw CompressionError("unable to initialize brotli decoder");
+
+    Finally free([s]() { BrotliDecoderDestroyInstance(s); });
+
+    uint8_t outbuf[BUFSIZ];
+    ref<std::string> res = make_ref<std::string>();
+    const uint8_t *next_in = (uint8_t *)in.c_str();
+    size_t avail_in = in.size();
+    uint8_t *next_out = outbuf;
+    size_t avail_out = sizeof(outbuf);
+
+    while (true) {
+        checkInterrupt();
+
+        auto ret = BrotliDecoderDecompressStream(s,
+                &avail_in, &next_in,
+                &avail_out, &next_out,
+                nullptr);
+
+        switch (ret) {
+        case BROTLI_DECODER_RESULT_ERROR:
+            throw CompressionError("error while decompressing brotli file");
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+            throw CompressionError("incomplete or corrupt brotli file");
+        case BROTLI_DECODER_RESULT_SUCCESS:
+            if (avail_in != 0)
+                throw CompressionError("unexpected input after brotli decompression");
+            break;
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+            // I'm not sure if this can happen, but abort if this happens with empty buffer
+            if (avail_out == sizeof(outbuf))
+                throw CompressionError("brotli decompression requires larger buffer");
+            break;
+        }
+
+        // Always ensure we have full buffer for next invocation
+        if (avail_out < sizeof(outbuf)) {
+            res->append((char*)outbuf, sizeof(outbuf) - avail_out);
+            next_out = outbuf;
+            avail_out = sizeof(outbuf);
+        }
+
+        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res;
+    }
+#endif // HAVE_BROTLI
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in)
@@ -270,33 +323,131 @@ struct BzipSink : CompressionSink
     }
 };
 
-struct BrotliSink : CompressionSink
+struct LambdaCompressionSink : CompressionSink
 {
     Sink & nextSink;
     std::string data;
+    using CompressFnTy = std::function<std::string(const std::string&)>;
+    CompressFnTy compressFn;
+    LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn)
+        : nextSink(nextSink)
+        , compressFn(std::move(compressFn))
+    {
+    };
+
+    void finish() override
+    {
+        flush();
+        nextSink(compressFn(data));
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        checkInterrupt();
+        this->data.append((const char *) data, len);
+    }
+};
+
+struct BrotliCmdSink : LambdaCompressionSink
+{
+    BrotliCmdSink(Sink& nextSink)
+        : LambdaCompressionSink(nextSink, [](const std::string& data) {
+            return runProgram(BROTLI, true, {}, data);
+        })
+    {
+    }
+};
+
+#if HAVE_BROTLI
+struct BrotliSink : CompressionSink
+{
+    Sink & nextSink;
+    uint8_t outbuf[BUFSIZ];
+    BrotliEncoderState *state;
+    bool finished = false;
 
     BrotliSink(Sink & nextSink) : nextSink(nextSink)
     {
+        state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
+        if (!state)
+            throw CompressionError("unable to initialise brotli encoder");
     }
 
     ~BrotliSink()
     {
+        BrotliEncoderDestroyInstance(state);
     }
 
-    // FIXME: use libbrotli
-
     void finish() override
     {
         flush();
-        nextSink(runProgram(BRO, true, {}, data));
+        assert(!finished);
+
+        const uint8_t *next_in = nullptr;
+        size_t avail_in = 0;
+        uint8_t *next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+        while (!finished) {
+            checkInterrupt();
+
+            if (!BrotliEncoderCompressStream(state,
+                        BROTLI_OPERATION_FINISH,
+                        &avail_in, &next_in,
+                        &avail_out, &next_out,
+                        nullptr))
+                throw CompressionError("error while finishing brotli file");
+
+            finished = BrotliEncoderIsFinished(state);
+            if (avail_out == 0 || finished) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+        }
     }
 
     void write(const unsigned char * data, size_t len) override
     {
-        checkInterrupt();
-        this->data.append((const char *) data, len);
+        assert(!finished);
+
+        // Don't feed brotli too much at once
+        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
+        while (len) {
+          size_t n = std::min(CHUNK_SIZE, len);
+          writeInternal(data, n);
+          data += n;
+          len -= n;
+        }
+    }
+  private:
+    void writeInternal(const unsigned char * data, size_t len)
+    {
+        assert(!finished);
+
+        const uint8_t *next_in = data;
+        size_t avail_in = len;
+        uint8_t *next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+
+        while (avail_in > 0) {
+            checkInterrupt();
+
+            if (!BrotliEncoderCompressStream(state,
+                      BROTLI_OPERATION_PROCESS,
+                      &avail_in, &next_in,
+                      &avail_out, &next_out,
+                      nullptr))
+                throw CompressionError("error while compressing brotli file");
+
+            if (avail_out < sizeof(outbuf) || avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+        }
     }
 };
+#endif // HAVE_BROTLI
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
 {
@@ -307,7 +458,11 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
+#if HAVE_BROTLI
         return make_ref<BrotliSink>(nextSink);
+#else
+        return make_ref<BrotliCmdSink>(nextSink);
+#endif
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 0721b21c208..5fc2aab569d 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,8 +6,8 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS)
 
 libutil_LIBS = libformat
 
-libutil_CXXFLAGS = -DBRO=\"$(bro)\"
+libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"
diff --git a/tests/brotli.sh b/tests/brotli.sh
new file mode 100644
index 00000000000..645dd4214ec
--- /dev/null
+++ b/tests/brotli.sh
@@ -0,0 +1,28 @@
+source common.sh
+
+
+# Only test if we found brotli libraries
+# (CLI tool is likely unavailable if libraries are missing)
+if [ -n "$HAVE_BROTLI" ]; then
+
+clearStore
+clearCache
+
+cacheURI="file://$cacheDir?compression=br"
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix copy --to $cacheURI $outPath
+
+HASH=$(nix hash-path $outPath)
+
+clearStore
+clearCacheCache
+
+nix copy --from $cacheURI $outPath --no-check-sigs
+
+HASH2=$(nix hash-path $outPath)
+
+[[ $HASH = $HASH2 ]]
+
+fi # HAVE_BROTLI
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 09f2949141a..83643d8b06b 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -32,6 +32,7 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
+export HAVE_BROTLI="@HAVE_BROTLI@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@
diff --git a/tests/local.mk b/tests/local.mk
index baf74224bb1..83154228e99 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -19,7 +19,8 @@ nix_tests = \
   fetchGit.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh
+  run.sh \
+  brotli.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From 5afee18726ce8241d56ea9690f228923441a8ea8 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 30 Dec 2017 21:29:23 -0600
Subject: [PATCH 0736/2196] run.sh: include lib64 in sandbox-paths to fix on
 ubuntu 16.XX

(cc #1769)
---
 tests/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/run.sh b/tests/run.sh
index 784d29183cf..194e767dd05 100644
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -18,10 +18,10 @@ if [[ $(uname) = Linux ]]; then
     # Note: we need the sandbox paths to ensure that the shell is
     # visible in the sandbox.
     nix run --sandbox-build-dir /build-tmp \
-        --sandbox-paths '/nix? /bin? /lib? /usr?' \
+        --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
         --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
 
-    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
+    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
 
     [[ $path/bin/hello = $path2 ]]
 

From 689b2783fcae0cd0559ebd1d37ccfc3a09c4b182 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 2 Jan 2018 12:22:52 -0500
Subject: [PATCH 0737/2196] Add hasContext primop

---
 src/libexpr/primops.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e3b5dfb420b..6029714273a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1653,6 +1653,14 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
 }
 
 
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
    builder without causing the derivation to be built (for instance,
    in the derivation that builds NARs in nix-push, when doing
@@ -2083,6 +2091,7 @@ void EvalState::createBaseEnv()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
+    addPrimOp("__hasContext", 1, prim_hasContext);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);

From 4cb5c513754ae709cd6dd2db746c43f7fafcbb0e Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Tue, 2 Jan 2018 23:39:42 -0500
Subject: [PATCH 0738/2196] Fix RPM builds by increasing VM memory size

The VM was running out of RAM while handling debug symbols, which caused the
eu-strip to fail while separating debug symbols.
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 68b58623289..ec1bebf61e9 100644
--- a/release.nix
+++ b/release.nix
@@ -313,7 +313,7 @@ let
         { extraPackages =
             [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
-      memSize = 1024;
+      memSize = 2048;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;

From 27788f406078489b2fc04749ebb86e267a19a6a5 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 3 Jan 2018 22:29:54 +0100
Subject: [PATCH 0739/2196] installer: don't touch /etc/profile

The default profile already loads /etc/bashrc.
---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 7bd00e42b97..91194a299a9 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -33,7 +33,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 readonly NIX_ROOT="/nix"
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
-readonly PROFILE_TARGETS=("/etc/profile" "/etc/bashrc" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 

From d15826164c34b71994c66ae8bf1203492b0fb44a Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 3 Jan 2018 22:34:34 +0100
Subject: [PATCH 0740/2196] installer: create 'enough' build users

---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 7bd00e42b97..515b4a87ed8 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -24,7 +24,7 @@ readonly YELLOW='\033[38;33m'
 readonly YELLOW_UL='\033[38;4;33m'
 
 readonly CORES=$(sysctl -n hw.ncpu)
-readonly NIX_USER_COUNT="$CORES"
+readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"

From 44272d87193e6ab17d15501ebdcd317cdd39f616 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Jan 2018 16:57:25 +0100
Subject: [PATCH 0741/2196] Rename "use-substitutes" to "substitute"

Commit c2154d4c8422ddc1c201d503bb52edff854af2ad renamed
"build-use-substitutes" to "use-substitutes", but that broke
"nix-copy-closure --use-substitutes".
---
 doc/manual/command-ref/conf-file.xml | 2 +-
 src/libstore/globals.hh              | 2 +-
 tests/shell.shebang.sh               | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index e52cbcd535e..87f05fb1c70 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -312,7 +312,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>use-substitutes</literal></term>
+  <varlistentry><term><literal>substitute</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 5c857cbb6a9..ae4b78a018e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -150,7 +150,7 @@ public:
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
         "Whether to call sync() before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "use-substitutes",
+    Setting<bool> useSubstitutes{this, true, "substitute",
         "Whether to use substitutes.",
         {"build-use-substitutes"}};
 
diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh
index c8e55ca9b90..f7132043de4 100755
--- a/tests/shell.shebang.sh
+++ b/tests/shell.shebang.sh
@@ -1,4 +1,4 @@
 #! @ENV_PROG@ nix-shell
-#! nix-shell -I nixpkgs=shell.nix --no-use-substitutes
+#! nix-shell -I nixpkgs=shell.nix --no-substitute
 #! nix-shell --pure -i bash -p foo bar
 echo "$(foo) $(bar) $@"

From 1882e802e760e61669c9cff6b4e933d87cf977ba Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Thu, 4 Jan 2018 19:44:32 -0500
Subject: [PATCH 0742/2196] Fix Fedora 25 i386 RPM build

---
 release.nix | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index ec1bebf61e9..2dd7264d912 100644
--- a/release.nix
+++ b/release.nix
@@ -313,7 +313,8 @@ let
         { extraPackages =
             [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
             ++ extraPackages; };
-      memSize = 2048;
+      # At most 2047MB can be simulated in qemu-system-i386
+      memSize = 2047;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;

From 7b9583680e9e1be1df2a5cd0d71ed8f9a7d45fad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 Jan 2018 19:13:48 +0100
Subject: [PATCH 0743/2196] Improve error message with --repair for untrusted
 users

---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 5629cc64b96..b5d49b6428a 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -411,7 +411,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
             /* Repairing is not atomic, so disallowed for "untrusted"
                clients.  */
             if (mode == bmRepair && !trusted)
-                throw Error("repairing is not supported when building through the Nix daemon");
+                throw Error("repairing is not allowed because you are not in 'trusted-users'");
         }
         logger->startWork();
         store->buildPaths(drvs, mode);

From 84d9e213d2dc8b09705b0668184f33ddd0a004b1 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 9 Jan 2018 08:58:19 -0600
Subject: [PATCH 0744/2196] fetchGit.sh: Test we don't "corrupt" cache if
 invoke w/o git avail

---
 tests/fetchGit.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 65d673c0885..b556fe594ce 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -119,3 +119,16 @@ path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 # Confirm same as 'dev' branch
 path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
+
+
+# Nuke the cache
+rm -rf $TEST_HOME/.cache/nix/git
+
+# Try again, but without 'git' on PATH
+NIX=$(command -v nix)
+# This should fail
+(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
+
+# Try again, with 'git' available.  This should work.
+path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+[[ $path3 = $path5 ]]

From 428680b3076a01f278ed629aa3b1744d11a2c231 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 9 Jan 2018 09:05:18 -0600
Subject: [PATCH 0745/2196] fetchGit: fix creation of uninitialized cache dir,
 let git create it

fetchGit test (as modified in previous commit) now passes.
---
 src/libexpr/primops/fetchGit.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 0d0b11958a4..fb664cffb5b 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -85,7 +85,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     Path cacheDir = getCacheDir() + "/nix/git";
 
     if (!pathExists(cacheDir)) {
-        createDirs(cacheDir);
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 

From b0328c244dd15da9e45f4e95d900c81fe0c47dda Mon Sep 17 00:00:00 2001
From: Renzo Carbonara <gnuk0001@gmail.com>
Date: Tue, 9 Jan 2018 22:40:07 +0100
Subject: [PATCH 0746/2196] nix.conf: builders-use-substitutes

Fixes #937
---
 doc/manual/command-ref/conf-file.xml | 14 ++++++++++++++
 src/build-remote/build-remote.cc     |  6 ++++--
 src/libstore/globals.hh              |  5 +++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 87f05fb1c70..2ddca991f19 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -321,6 +321,20 @@ false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>builders-use-substitutes</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, Nix will instruct
+    remote build machines to use their own binary substitutes if available. In
+    practical terms, this means that remote hosts will fetch as many build
+    dependencies as possible from their own substitutes (e.g, from
+    <literal>cache.nixos.org</literal>), instead of waiting for this host to
+    upload them all. This can drastically reduce build times if the network
+    connection between this computer and the remote build host is slow. Defaults
+    to <literal>false</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 445006b327f..df579729af2 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -218,9 +218,11 @@ int main (int argc, char * * argv)
             signal(SIGALRM, old);
         }
 
+        auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
+
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs);
+            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
@@ -240,7 +242,7 @@ int main (int argc, char * * argv)
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
         }
 
         return;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index ae4b78a018e..af72f7b1e35 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -138,6 +138,11 @@ public:
     Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
+    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
+        "Whether build machines should use their own substitutes for obtaining "
+        "build dependencies if possible, rather than waiting for this host to "
+        "upload them."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 

From 435ccc798077e6291fd34fd1720c6abcf3521557 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 10 Jan 2018 14:16:49 -0600
Subject: [PATCH 0747/2196] release: access fetchGit from builtins to fix eval
 w/1.11 (<1.12)

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 2dd7264d912..30bec2a6e3e 100644
--- a/release.nix
+++ b/release.nix
@@ -1,4 +1,4 @@
-{ nix ? fetchGit ./.
+{ nix ? builtins.fetchGit ./.
 , nixpkgs ? fetchTarball channel:nixos-17.09
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]

From 74f75c855837bce7f48491e9ce8ac03794e5b40d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jan 2018 17:31:08 +0100
Subject: [PATCH 0748/2196] import, builtins.readFile: Handle diverted stores

Fixes #1791
---
 src/libexpr/eval.cc       | 10 ++++++++++
 src/libexpr/eval.hh       |  9 +++++++++
 src/libexpr/primops.cc    | 12 ++++++------
 src/libstore/store-api.hh | 10 ++++++++--
 4 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 63de2d60a14..087a95ddef8 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -375,6 +375,16 @@ void EvalState::checkURI(const std::string & uri)
 }
 
 
+Path EvalState::toRealPath(const Path & path, const PathSet & context)
+{
+    // FIXME: check whether 'path' is in 'context'.
+    return
+        !context.empty() && store->isInStore(path)
+        ? store->toRealPath(path)
+        : path;
+};
+
+
 void EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f0ab1435bff..cc971ae80a4 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -112,6 +112,15 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /* When using a diverted store and 'path' is in the Nix store, map
+       'path' to the diverted location (e.g. /nix/store/foo is mapped
+       to /home/alice/my-nix/nix/store/foo). However, this is only
+       done if the context is not empty, since otherwise we're
+       probably trying to read from the actual /nix/store. This is
+       intended to distinguish between import-from-derivation and
+       sources stored in the actual /nix/store. */
+    Path toRealPath(const Path & path, const PathSet & context);
+
     /* Parse a Nix expression from the specified file. */
     Expr * parseExprFromFile(const Path & path);
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6029714273a..98fe2199e9f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -84,10 +84,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             % path % e.path % pos);
     }
 
-    path = state.checkSourcePath(path);
+    Path realPath = state.checkSourcePath(state.toRealPath(path, context));
 
     if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) {
-        Derivation drv = readDerivation(path);
+        Derivation drv = readDerivation(realPath);
         Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
         Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -114,7 +114,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     } else {
         state.forceAttrs(*args[0]);
         if (args[0]->attrs->empty())
-            state.evalFile(path, v);
+            state.evalFile(realPath, v);
         else {
             Env * env = &state.allocEnv(args[0]->attrs->size());
             env->up = &state.baseEnv;
@@ -127,8 +127,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
                 env->values[displ++] = attr.value;
             }
 
-            printTalkative("evaluating file '%1%'", path);
-            Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
+            printTalkative("evaluating file '%1%'", realPath);
+            Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
 
             e->eval(state, *env, v);
         }
@@ -863,7 +863,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
         throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
             % path % e.path % pos);
     }
-    string s = readFile(state.checkSourcePath(path));
+    string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
         throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
     mkString(v, s.c_str());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d1e1b5d6f45..c0e735cd314 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -597,6 +597,11 @@ public:
        "nix-cache-info" file. Lower value means higher priority. */
     virtual int getPriority() { return 0; }
 
+    virtual Path toRealPath(const Path & storePath)
+    {
+        return storePath;
+    }
+
 protected:
 
     Stats stats;
@@ -639,9 +644,10 @@ public:
 
     virtual Path getRealStoreDir() { return storeDir; }
 
-    Path toRealPath(const Path & storePath)
+    Path toRealPath(const Path & storePath) override
     {
-        return getRealStoreDir() + "/" + baseNameOf(storePath);
+        assert(isInStore(storePath));
+        return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
     }
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;

From a65376b01da1f5e612ecbc7232f23c0bebcabe22 Mon Sep 17 00:00:00 2001
From: Peter Stuart <peter.l.stuart@gmail.com>
Date: Fri, 12 Jan 2018 14:27:29 -0500
Subject: [PATCH 0749/2196] Remove extra space.

---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 91194a299a9..2e9d368c020 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -647,7 +647,7 @@ chat_about_sudo() {
         cat <<EOF
 This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
-run in  a headless fashion, like this:
+run in a headless fashion, like this:
 
   $ curl https://nixos.org/nix/install | sh
 

From 6454ca393a041345fd5b85f97bc452a696abfe4e Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 12 Jan 2018 14:45:05 -0600
Subject: [PATCH 0750/2196] nix log: use pager

---
 src/nix/log.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/log.cc b/src/nix/log.cc
index 966ad8b6508..f07ec4e93a1 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -50,6 +50,7 @@ struct CmdLog : InstallableCommand
 
         auto b = installable->toBuildable();
 
+        RunPager pager;
         for (auto & sub : subs) {
             auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr;
             for (auto & output : b.outputs) {

From 59086e459c2650b7dc42bd2fc4a9a98b23aaf6e7 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Sat, 13 Jan 2018 15:18:35 +0200
Subject: [PATCH 0751/2196] Fix tests using user namespaces on kernels that
 don't have it

Disable various tests if the kernel doesn't support unprivileged user
namespaces (e.g. Arch Linux disables them) or disable them via a sysctl
(Debian, Ubuntu).

Fixes #1521
Fixes #1625
---
 tests/build-remote.sh  |  2 +-
 tests/common.sh.in     | 18 ++++++++++++++++++
 tests/linux-sandbox.sh |  2 +-
 tests/run.sh           | 27 +++++++++++++--------------
 4 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index cf3bb463318..9bca0f4a385 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -2,7 +2,7 @@ source common.sh
 
 clearStore
 
-if [[ $(uname) != Linux ]]; then exit; fi
+if ! canUseSandbox; then exit; fi
 if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 83643d8b06b..186f9d6b955 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -87,6 +87,24 @@ killDaemon() {
     trap "" EXIT
 }
 
+canUseSandbox() {
+    if [[ $(uname) != Linux ]]; then return 1; fi
+
+    if [ ! -L /proc/self/ns/user ]; then
+        echo "Kernel doesn't support user namespaces, skipping this test..."
+        return 1
+    fi
+
+    if [ -e /proc/sys/kernel/unprivileged_userns_clone ]; then
+        if [ "$(cat /proc/sys/kernel/unprivileged_userns_clone)" != 1 ]; then
+            echo "Unprivileged user namespaces disabled by sysctl, skipping this test..."
+            return 1
+        fi
+    fi
+
+    return 0
+}
+
 fail() {
     echo "$1"
     exit 1
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 4a686bb59a3..acfd46c5417 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -2,7 +2,7 @@ source common.sh
 
 clearStore
 
-if [[ $(uname) != Linux ]]; then exit; fi
+if ! canUseSandbox; then exit; fi
 
 # Note: we need to bind-mount $SHELL into the chroot. Currently we
 # only support the case where $SHELL is in the Nix store, because
diff --git a/tests/run.sh b/tests/run.sh
index 194e767dd05..d1dbfd6bd4a 100644
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -6,24 +6,23 @@ clearCache
 nix run -f run.nix hello -c hello | grep 'Hello World'
 nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS'
 
-if [[ $(uname) = Linux ]]; then
+if ! canUseSandbox; then exit; fi
 
-    chmod -R u+w $TEST_ROOT/store0 || true
-    rm -rf $TEST_ROOT/store0
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
 
-    clearStore
+clearStore
 
-    path=$(nix eval --raw -f run.nix hello)
+path=$(nix eval --raw -f run.nix hello)
 
-    # Note: we need the sandbox paths to ensure that the shell is
-    # visible in the sandbox.
-    nix run --sandbox-build-dir /build-tmp \
-        --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
-        --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
+# Note: we need the sandbox paths to ensure that the shell is
+# visible in the sandbox.
+nix run --sandbox-build-dir /build-tmp \
+    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
+    --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
 
-    path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
+path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
 
-    [[ $path/bin/hello = $path2 ]]
+[[ $path/bin/hello = $path2 ]]
 
-    [[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]
-fi
+[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]

From ebc42f8b5944e6669362e7d13856f126b8e075d7 Mon Sep 17 00:00:00 2001
From: Iavael <iavael@users.noreply.github.com>
Date: Mon, 15 Jan 2018 00:43:39 +0300
Subject: [PATCH 0752/2196] Fix manpath detection

Checking for MANPATH without quotes always returns true, so that it breaks bash-completion for man pages on modern systems without MANPATH environment variable.
---
 scripts/nix-profile.sh.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 450d683c7f4..a5f52274fc7 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
-    if [ -n ${MANPATH} ]; then
+    if [ -n "${MANPATH}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 

From ba75c69e0014178cf524cc07427bb4e9eac333f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jan 2018 12:14:43 +0100
Subject: [PATCH 0753/2196] Barf when using a diverted store on macOS

Fixes #1792.
---
 src/libstore/build.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d4bd650baf2..523d737d9bf 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1810,8 +1810,13 @@ void DerivationGoal::startBuilder()
             useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
     }
 
-    if (worker.store.storeDir != worker.store.realStoreDir)
-        useChroot = true;
+    if (worker.store.storeDir != worker.store.realStoreDir) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
+    }
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */

From 23fa7e3606a2bee6e3622a61f07e66bdda9b5304 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2018 17:11:58 +0100
Subject: [PATCH 0754/2196] parseExprFromFile -> evalFile

parseExprFromFile() should be avoided since it doesn't cache anything.
---
 src/nix/installables.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index ae93c4ef649..c3b06c22eba 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -30,10 +30,8 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
     vSourceExpr = state.allocValue();
 
-    if (file != "") {
-        Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file)));
-        state.eval(e, *vSourceExpr);
-    }
+    if (file != "")
+        state.evalFile(lookupFileArg(state, file), *vSourceExpr);
 
     else {
 

From d4dcffd64349bb52ad5f1b184bee5cc7c2be73b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2018 18:50:38 +0100
Subject: [PATCH 0755/2196] Add pure evaluation mode

In this mode, the following restrictions apply:

* The builtins currentTime, currentSystem and storePath throw an
  error.

* $NIX_PATH and -I are ignored.

* fetchGit and fetchMercurial require a revision hash.

* fetchurl and fetchTarball require a sha256 attribute.

* No file system access is allowed outside of the paths returned by
  fetch{Git,Mercurial,url,Tarball}. Thus 'nix build -f ./foo.nix' is
  not allowed.

Thus, the evaluation result is completely reproducible from the
command line arguments. E.g.

  nix build --pure-eval '(
    let
      nix = fetchGit { url = https://github.com/NixOS/nixpkgs.git; rev = "9c927de4b179a6dd210dd88d34bda8af4b575680"; };
      nixpkgs = fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "release-17.09"; rev = "66b4de79e3841530e6d9c6baf98702aa1f7124e4"; };
    in (import (nix + "/release.nix") { inherit nix nixpkgs; }).build.x86_64-linux
  )'

The goal is to enable completely reproducible and traceable
evaluation. For example, a NixOS configuration could be fully
described by a single Git commit hash. 'nixos-rebuild' would do
something like

  nix build --pure-eval '(
    (import (fetchGit { url = file:///my-nixos-config; rev = "..."; })).system
  ')

where the Git repository /my-nixos-config would use further fetchGit
calls or Git externals to fetch Nixpkgs and whatever other
dependencies it has. Either way, the commit hash would uniquely
identify the NixOS configuration and allow it to reproduced.
---
 mk/tests.mk                            |  2 +-
 release.nix                            |  2 +-
 src/libexpr/eval.cc                    | 65 +++++++++++++++-----------
 src/libexpr/eval.hh                    |  8 ++--
 src/libexpr/primops.cc                 | 44 +++++++++++++----
 src/libexpr/primops/fetchGit.cc        | 15 ++++--
 src/libexpr/primops/fetchMercurial.cc  |  6 +++
 src/libstore/globals.hh                |  3 ++
 src/libutil/util.cc                    |  6 +++
 src/libutil/util.hh                    |  6 ++-
 src/nix-build/nix-build.cc             |  4 +-
 src/nix-instantiate/nix-instantiate.cc |  2 +-
 tests/fetchGit.sh                      |  7 +++
 tests/fetchMercurial.sh                |  7 +++
 tests/local.mk                         |  3 +-
 tests/pure-eval.nix                    |  3 ++
 tests/pure-eval.sh                     | 18 +++++++
 tests/restricted.nix                   |  1 +
 tests/restricted.sh                    | 10 +++-
 19 files changed, 159 insertions(+), 53 deletions(-)
 create mode 100644 tests/pure-eval.nix
 create mode 100644 tests/pure-eval.sh
 create mode 100644 tests/restricted.nix

diff --git a/mk/tests.mk b/mk/tests.mk
index e353d46a0d0..70c30661b95 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -39,7 +39,7 @@ installcheck:
 	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
 	  exit 1; \
 	else \
-		echo "$${green}All tests succeeded"; \
+		echo "$${green}All tests succeeded$$normal"; \
 	fi
 
 .PHONY: check installcheck
diff --git a/release.nix b/release.nix
index 30bec2a6e3e..04f1f836743 100644
--- a/release.nix
+++ b/release.nix
@@ -6,7 +6,7 @@
 
 let
 
-  pkgs = import nixpkgs {};
+  pkgs = import nixpkgs { system = "x86_64-linux"; };
 
   jobs = rec {
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 087a95ddef8..f8685e010e1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -300,16 +300,25 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 {
     countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
 
-    restricted = settings.restrictEval;
-
     assert(gcInitialised);
 
     /* Initialise the Nix expression search path. */
-    Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
-    for (auto & i : _searchPath) addToSearchPath(i);
-    for (auto & i : paths) addToSearchPath(i);
+    if (!settings.pureEval) {
+        Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
+        for (auto & i : _searchPath) addToSearchPath(i);
+        for (auto & i : paths) addToSearchPath(i);
+    }
     addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
 
+    if (settings.restrictEval || settings.pureEval) {
+        allowedPaths = PathSet();
+        for (auto & i : searchPath) {
+            auto r = resolveSearchPathElem(i);
+            if (!r.first) continue;
+            allowedPaths->insert(r.second);
+        }
+    }
+
     clearValue(vEmptySet);
     vEmptySet.type = tAttrs;
     vEmptySet.attrs = allocBindings(0);
@@ -326,38 +335,39 @@ EvalState::~EvalState()
 
 Path EvalState::checkSourcePath(const Path & path_)
 {
-    if (!restricted) return path_;
+    if (!allowedPaths) return path_;
+
+    auto doThrow = [&]() [[noreturn]] {
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
+    };
+
+    bool found = false;
+
+    for (auto & i : *allowedPaths) {
+        if (isDirOrInDir(path_, i)) {
+            found = true;
+            break;
+        }
+    }
+
+    if (!found) doThrow();
 
     /* Resolve symlinks. */
     debug(format("checking access to '%s'") % path_);
     Path path = canonPath(path_, true);
 
-    for (auto & i : searchPath) {
-        auto r = resolveSearchPathElem(i);
-        if (!r.first) continue;
-        if (path == r.second || isInDir(path, r.second))
+    for (auto & i : *allowedPaths) {
+        if (isDirOrInDir(path, i))
             return path;
     }
 
-    /* To support import-from-derivation, allow access to anything in
-       the store. FIXME: only allow access to paths that have been
-       constructed by this evaluation. */
-    if (store->isInStore(path)) return path;
-
-#if 0
-    /* Hack to support the chroot dependencies of corepkgs (see
-       corepkgs/config.nix.in). */
-    if (path == settings.nixPrefix && isStorePath(settings.nixPrefix))
-        return path;
-#endif
-
-    throw RestrictedPathError(format("access to path '%1%' is forbidden in restricted mode") % path_);
+    doThrow();
 }
 
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!restricted) return;
+    if (!settings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
@@ -396,7 +406,7 @@ void EvalState::addConstant(const string & name, Value & v)
 }
 
 
-void EvalState::addPrimOp(const string & name,
+Value * EvalState::addPrimOp(const string & name,
     unsigned int arity, PrimOpFun primOp)
 {
     Value * v = allocValue();
@@ -407,6 +417,7 @@ void EvalState::addPrimOp(const string & name,
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
+    return v;
 }
 
 
@@ -659,8 +670,10 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const Path & path, Value & v)
+void EvalState::evalFile(const Path & path_, Value & v)
 {
+    auto path = checkSourcePath(path_);
+
     FileEvalCache::iterator i;
     if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
         v = i->second;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index cc971ae80a4..9e3d30d95f4 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -76,9 +76,9 @@ public:
        already exist there. */
     RepairFlag repair;
 
-    /* If set, don't allow access to files outside of the Nix search
-       path or to environment variables. */
-    bool restricted;
+    /* The allowed filesystem paths in restricted or pure evaluation
+       mode. */
+    std::experimental::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
@@ -212,7 +212,7 @@ private:
 
     void addConstant(const string & name, Value & v);
 
-    void addPrimOp(const string & name,
+    Value * addPrimOp(const string & name,
         unsigned int arity, PrimOpFun primOp);
 
 public:
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 98fe2199e9f..0ec035b8624 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -439,7 +439,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, state.restricted ? "" : getEnv(name));
+    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -1929,7 +1929,14 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
+    if (settings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(res);
+
     mkString(v, res, PathSet({res}));
 }
 
@@ -1981,11 +1988,28 @@ void EvalState::createBaseEnv()
     mkNull(v);
     addConstant("null", v);
 
-    mkInt(v, time(0));
-    addConstant("__currentTime", v);
+    auto vThrow = addPrimOp("throw", 1, prim_throw);
 
-    mkString(v, settings.thisSystem);
-    addConstant("__currentSystem", v);
+    auto addPurityError = [&](const std::string & name) {
+        Value * v2 = allocValue();
+        mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
+        mkApp(v, *vThrow, *v2);
+        addConstant(name, v);
+    };
+
+    if (settings.pureEval)
+        addPurityError("__currentTime");
+    else {
+        mkInt(v, time(0));
+        addConstant("__currentTime", v);
+    }
+
+    if (settings.pureEval)
+        addPurityError("__currentSystem");
+    else {
+        mkString(v, settings.thisSystem);
+        addConstant("__currentSystem", v);
+    }
 
     mkString(v, nixVersion);
     addConstant("__nixVersion", v);
@@ -2001,10 +2025,10 @@ void EvalState::createBaseEnv()
     addConstant("__langVersion", v);
 
     // Miscellaneous
-    addPrimOp("scopedImport", 2, prim_scopedImport);
+    auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
     Value * v2 = allocValue();
     mkAttrs(*v2, 0);
-    mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2);
+    mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
     if (settings.enableNativeCode) {
@@ -2020,7 +2044,6 @@ void EvalState::createBaseEnv()
     addPrimOp("__isBool", 1, prim_isBool);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
-    addPrimOp("throw", 1, prim_throw);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
     addPrimOp("__tryEval", 1, prim_tryEval);
     addPrimOp("__getEnv", 1, prim_getEnv);
@@ -2035,7 +2058,10 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    addPrimOp("__storePath", 1, prim_storePath);
+    if (settings.pureEval)
+        addPurityError("__storePath");
+    else
+        addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__pathExists", 1, prim_pathExists);
     addPrimOp("baseNameOf", 1, prim_baseNameOf);
     addPrimOp("dirOf", 1, prim_dirOf);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index fb664cffb5b..2e3e2634db8 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -22,10 +22,15 @@ struct GitInfo
     uint64_t revCount = 0;
 };
 
+std::regex revRegex("^[0-9a-fA-F]{40}$");
+
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::experimental::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
+    if (settings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
 
         bool clean = true;
@@ -76,11 +81,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     if (!ref) ref = "master"s;
 
-    if (rev != "") {
-        std::regex revRegex("^[0-9a-fA-F]{40}$");
-        if (!std::regex_match(rev, revRegex))
-            throw Error("invalid Git revision '%s'", rev);
-    }
+    if (rev != "" && !std::regex_match(rev, revRegex))
+        throw Error("invalid Git revision '%s'", rev);
 
     Path cacheDir = getCacheDir() + "/nix/git";
 
@@ -231,6 +233,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
     v.attrs->sort();
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(gitInfo.storePath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a317476c582..5517d83df82 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -27,6 +27,9 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
+    if (settings.pureEval && rev == "")
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
     if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
 
         bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
@@ -196,6 +199,9 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
     v.attrs->sort();
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(hgInfo.storePath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index af72f7b1e35..81bb24a4eb3 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -232,6 +232,9 @@ public:
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};
 
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
         {"build-repeat"}};
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 197df0c44aa..27299739779 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -192,6 +192,12 @@ bool isInDir(const Path & path, const Path & dir)
 }
 
 
+bool isDirOrInDir(const Path & path, const Path & dir)
+{
+    return path == dir or isInDir(path, dir);
+}
+
+
 struct stat lstat(const Path & path)
 {
     struct stat st;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a3494e09b09..75eb9751524 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -53,10 +53,12 @@ Path dirOf(const Path & path);
    following the final `/'. */
 string baseNameOf(const Path & path);
 
-/* Check whether a given path is a descendant of the given
-   directory. */
+/* Check whether 'path' is a descendant of 'dir'. */
 bool isInDir(const Path & path, const Path & dir);
 
+/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */
+bool isDirOrInDir(const Path & path, const Path & dir);
+
 /* Get status of `path'. */
 struct stat lstat(const Path & path);
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 58366daa6e8..1b249427537 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -279,8 +279,8 @@ void mainWrapped(int argc, char * * argv)
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
-                exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state,
-                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))));
+                exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state,
+                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
         }
 
     /* Evaluate them into derivations. */
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 55ac007e868..e05040a42de 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -182,7 +182,7 @@ int main(int argc, char * * argv)
         for (auto & i : files) {
             Expr * e = fromArgs
                 ? state.parseExprFromString(i, absPath("."))
-                : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i)));
+                : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i))));
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index b556fe594ce..530ac7bb813 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -29,10 +29,17 @@ rev2=$(git -C $repo rev-parse HEAD)
 path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
 
+# In pure eval mode, fetchGit without a revision should fail.
+[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]]
+(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))")
+
 # Fetch using an explicit revision hash.
 path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
 
+# In pure eval mode, fetchGit with a revision should succeed.
+[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]]
+
 # Fetch again. This should be cached.
 mv $repo ${repo}-tmp
 path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 271350ecd17..4088dbd3979 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -29,10 +29,17 @@ rev2=$(hg log --cwd $repo -r tip --template '{node}')
 path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
 
+# In pure eval mode, fetchGit without a revision should fail.
+[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]]
+(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))")
+
 # Fetch using an explicit revision hash.
 path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
 
+# In pure eval mode, fetchGit with a revision should succeed.
+[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]]
+
 # Fetch again. This should be cached.
 mv $repo ${repo}-tmp
 path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
diff --git a/tests/local.mk b/tests/local.mk
index 83154228e99..82502a8e5f0 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -20,7 +20,8 @@ nix_tests = \
   fetchMercurial.sh \
   signing.sh \
   run.sh \
-  brotli.sh
+  brotli.sh \
+  pure-eval.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/pure-eval.nix b/tests/pure-eval.nix
new file mode 100644
index 00000000000..ed25b3d4563
--- /dev/null
+++ b/tests/pure-eval.nix
@@ -0,0 +1,3 @@
+{
+  x = 123;
+}
diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh
new file mode 100644
index 00000000000..49c8564487c
--- /dev/null
+++ b/tests/pure-eval.sh
@@ -0,0 +1,18 @@
+source common.sh
+
+clearStore
+
+nix eval --pure-eval '(assert 1 + 2 == 3; true)'
+
+[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]]
+
+(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)')
+
+(! nix eval --pure-eval '(builtins.currentTime)')
+(! nix eval --pure-eval '(builtins.currentSystem)')
+
+(! nix-instantiate --pure-eval ./simple.nix)
+
+[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]]
+(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)")
+nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)"
diff --git a/tests/restricted.nix b/tests/restricted.nix
new file mode 100644
index 00000000000..e0ef5840209
--- /dev/null
+++ b/tests/restricted.nix
@@ -0,0 +1 @@
+1 + 2
diff --git a/tests/restricted.sh b/tests/restricted.sh
index c063c8693d5..6c0392facf3 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -3,7 +3,8 @@ source common.sh
 clearStore
 
 nix-instantiate --restrict-eval --eval -E '1 + 2'
-(! nix-instantiate --restrict-eval ./simple.nix)
+(! nix-instantiate --restrict-eval ./restricted.nix)
+(! nix-instantiate --eval --restrict-eval <(echo '1 + 2'))
 nix-instantiate --restrict-eval ./simple.nix -I src=.
 nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh
 
@@ -28,3 +29,10 @@ nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval
 (! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
 (! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
 (! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval)
+
+ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix
+[[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]]
+(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix)
+(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT)
+(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .)
+nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I .

From 75b9670df61b3e82e6d60a0572316fdacc9cbd91 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2018 19:03:31 +0100
Subject: [PATCH 0756/2196] Make show-trace a config setting

---
 src/libmain/shared.cc   | 3 ---
 src/libstore/globals.hh | 3 ++-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 85d3c077ba5..90a4867163d 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -193,9 +193,6 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     mkFlag(0, "readonly-mode", "do not write to the Nix store",
         &settings.readOnlyMode);
 
-    mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors",
-        &settings.showTrace);
-
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 81bb24a4eb3..1e50e2d13e9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -211,7 +211,8 @@ public:
     bool lockCPU;
 
     /* Whether to show a stack trace if Nix evaluation fails. */
-    bool showTrace = false;
+    Setting<bool> showTrace{this, false, "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
 
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};

From 6ddfe9a999027f9867bbf61ae92c19c591c89a86 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2018 19:07:27 +0100
Subject: [PATCH 0757/2196] <nix/fetchurl.nix>: Don't access
 builtins.currentSystem

This doesn't work in pure evaluation mode.
---
 corepkgs/fetchurl.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix
index e135b947fdb..0ce1bab112f 100644
--- a/corepkgs/fetchurl.nix
+++ b/corepkgs/fetchurl.nix
@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem
+{ system ? "" # obsolete
 , url
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
@@ -17,7 +17,9 @@ derivation {
   inherit outputHashAlgo outputHash;
   outputHashMode = if unpack || executable then "recursive" else "flat";
 
-  inherit name system url executable unpack;
+  inherit name url executable unpack;
+
+  system = "builtin";
 
   # No need to double the amount of network traffic
   preferLocalBuild = true;

From d8b4cfad823524338f87a681b8790b8dbd3a7a1b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2018 11:53:08 +0100
Subject: [PATCH 0758/2196] Typo

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0ec035b8624..e90a1da25e0 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -39,7 +39,7 @@ std::pair<string, string> decodeContext(const string & s)
         size_t index = s.find("!", 1);
         return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
     } else
-        return std::pair<string, string>(s.at(0) == '/' ? s: string(s, 1), "");
+        return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
 }
 
 

From cfeff3b2731fd7e6dd030d4e4d56ff3a3aeac903 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2018 11:53:16 +0100
Subject: [PATCH 0759/2196] Move show-trace docs

---
 doc/manual/command-ref/conf-file.xml      | 8 ++++++++
 doc/manual/command-ref/opt-common-syn.xml | 1 -
 doc/manual/command-ref/opt-common.xml     | 7 -------
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 2ddca991f19..fff7994f28d 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -734,6 +734,14 @@ builtins.fetchurl {
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term>
+
+    <listitem><para>Causes Nix to print out a stack trace in case of Nix
+    expression evaluation errors.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
index 3aff4e1b635..168bef080f4 100644
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ b/doc/manual/command-ref/opt-common-syn.xml
@@ -47,7 +47,6 @@
 </arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
-<arg><option>--show-trace</option></arg>
 <arg>
   <option>-I</option>
   <replaceable>path</replaceable>
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 32d53c753a2..bcb60b30125 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -301,13 +301,6 @@
 
 </varlistentry>
 
-<varlistentry><term><option>--show-trace</option></term>
-
-  <listitem><para>Causes Nix to print out a stack trace in case of Nix
-  expression evaluation errors.</para></listitem>
-
-</varlistentry>
-
 
 <varlistentry xml:id="opt-I"><term><option>-I</option> <replaceable>path</replaceable></term>
 

From 16e0287556cbf04d2642c1091b4fe00fa6e352af Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2018 12:03:06 +0100
Subject: [PATCH 0760/2196] nix eval: Take only one argument

Thus --json no longer produces a list.
---
 src/nix/eval.cc | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 0fbeca1c121..2bc58b7dd4b 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -8,7 +8,7 @@
 
 using namespace nix;
 
-struct CmdEval : MixJSON, InstallablesCommand
+struct CmdEval : MixJSON, InstallableCommand
 {
     bool raw = false;
 
@@ -56,20 +56,16 @@ struct CmdEval : MixJSON, InstallablesCommand
 
         auto state = getEvalState();
 
-        auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr;
-
-        for (auto & i : installables) {
-            auto v = i->toValue(*state);
-            PathSet context;
-            if (raw) {
-                std::cout << state->coerceToString(noPos, *v, context);
-            } else if (json) {
-                auto jsonElem = jsonOut->placeholder();
-                printValueAsJSON(*state, true, *v, jsonElem, context);
-            } else {
-                state->forceValueDeep(*v);
-                std::cout << *v << "\n";
-            }
+        auto v = installable->toValue(*state);
+        PathSet context;
+        if (raw) {
+            std::cout << state->coerceToString(noPos, *v, context);
+        } else if (json) {
+            JSONPlaceholder jsonOut(std::cout);
+            printValueAsJSON(*state, true, *v, jsonOut, context);
+        } else {
+            state->forceValueDeep(*v);
+            std::cout << *v << "\n";
         }
     }
 };

From 27b510af5c3f8ad562dbc90dd39f82236fdf57db Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2018 12:04:44 +0100
Subject: [PATCH 0761/2196] nix eval: Stop progress bar before printing the
 result

---
 src/nix/eval.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 2bc58b7dd4b..b7058361cbe 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -5,6 +5,7 @@
 #include "eval.hh"
 #include "json.hh"
 #include "value-to-json.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
@@ -58,6 +59,9 @@ struct CmdEval : MixJSON, InstallableCommand
 
         auto v = installable->toValue(*state);
         PathSet context;
+
+        stopProgressBar();
+
         if (raw) {
             std::cout << state->coerceToString(noPos, *v, context);
         } else if (json) {

From 0c95776c3efdc63c4b957823d4e51d851a64ed84 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Jan 2018 16:37:39 +0100
Subject: [PATCH 0762/2196] Don't define builtins.{currentSystem,currentTime}
 in pure mode

This makes it easier to provide a default, e.g.

  system = builtins.currentSystem or "x86_64-linux";
---
 release.nix            | 2 +-
 src/libexpr/primops.cc | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/release.nix b/release.nix
index 04f1f836743..bdac283cfe3 100644
--- a/release.nix
+++ b/release.nix
@@ -6,7 +6,7 @@
 
 let
 
-  pkgs = import nixpkgs { system = "x86_64-linux"; };
+  pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
   jobs = rec {
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e90a1da25e0..975f0e8309e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1997,16 +1997,12 @@ void EvalState::createBaseEnv()
         addConstant(name, v);
     };
 
-    if (settings.pureEval)
-        addPurityError("__currentTime");
-    else {
+    if (!settings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
     }
 
-    if (settings.pureEval)
-        addPurityError("__currentSystem");
-    else {
+    if (!settings.pureEval) {
         mkString(v, settings.thisSystem);
         addConstant("__currentSystem", v);
     }

From 87e3d142cce442417b52610652b9d1a4fd7a6b4f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Jan 2018 13:58:28 +0100
Subject: [PATCH 0763/2196] Add a test for --check / --repeat

---
 tests/check.nix | 12 ++++++++++++
 tests/check.sh  | 18 ++++++++++++++++++
 tests/local.mk  |  3 ++-
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 tests/check.nix
 create mode 100644 tests/check.sh

diff --git a/tests/check.nix b/tests/check.nix
new file mode 100644
index 00000000000..b330ab9c984
--- /dev/null
+++ b/tests/check.nix
@@ -0,0 +1,12 @@
+with import ./config.nix;
+
+{
+  nondeterministic = mkDerivation {
+    name = "nondeterministic";
+    buildCommand =
+      ''
+        mkdir $out
+        date +%s.%N > $out/date
+      '';
+  };
+}
diff --git a/tests/check.sh b/tests/check.sh
new file mode 100644
index 00000000000..3efcef04492
--- /dev/null
+++ b/tests/check.sh
@@ -0,0 +1,18 @@
+source common.sh
+
+clearStore
+
+nix-build dependencies.nix --no-out-link
+nix-build dependencies.nix --no-out-link --check
+
+nix-build check.nix -A nondeterministic --no-out-link
+(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log)
+grep 'may not be deterministic' $TEST_ROOT/log
+
+clearStore
+
+nix-build dependencies.nix --no-out-link --repeat 3
+
+(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log)
+grep 'differs from previous round' $TEST_ROOT/log
+
diff --git a/tests/local.mk b/tests/local.mk
index 82502a8e5f0..e90b9f7da4a 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -21,7 +21,8 @@ nix_tests = \
   signing.sh \
   run.sh \
   brotli.sh \
-  pure-eval.sh
+  pure-eval.sh \
+  check.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From 2896bb682691e8ce2be055a386bdeef93ae05586 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Jan 2018 14:05:08 +0100
Subject: [PATCH 0764/2196] Don't retry CURLE_URL_MALFORMAT

---
 src/libstore/download.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4474dfd4b96..ef417685f1a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -303,6 +303,7 @@ struct CurlDownloader : public Downloader
                     // Don't bother retrying on certain cURL errors either
                     switch (code) {
                         case CURLE_FAILED_INIT:
+                        case CURLE_URL_MALFORMAT:
                         case CURLE_NOT_BUILT_IN:
                         case CURLE_REMOTE_ACCESS_DENIED:
                         case CURLE_FILE_COULDNT_READ_FILE:

From 3c4c30eadd879f512ac2075a7ba39c37ff77bf5c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Jan 2018 14:53:34 +0100
Subject: [PATCH 0765/2196] Rewrite builtin derivation environment

Also add a test.

Fixes #1803.
Closes #1805.
---
 src/libstore/build.cc |  7 ++++++-
 tests/check.nix       |  5 +++++
 tests/check.sh        | 14 ++++++++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 523d737d9bf..55066205db3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2933,8 +2933,13 @@ void DerivationGoal::runChild()
         if (drv->isBuiltin()) {
             try {
                 logger = makeJSONLogger(*logger);
+
+                BasicDerivation drv2(*drv);
+                for (auto & e : drv2.env)
+                    e.second = rewriteStrings(e.second, inputRewrites);
+
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv, netrcData);
+                    builtinFetchurl(drv2, netrcData);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
diff --git a/tests/check.nix b/tests/check.nix
index b330ab9c984..585d43032a9 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -9,4 +9,9 @@ with import ./config.nix;
         date +%s.%N > $out/date
       '';
   };
+
+  fetchurl = import <nix/fetchurl.nix> {
+    url = "file://" + toString ./lang/eval-okay-xml.out;
+    sha256 = "426fefcd2430e986551db13fcc2b1e45eeec17e68ffeb6ff155be2f8aaf5407e";
+  };
 }
diff --git a/tests/check.sh b/tests/check.sh
index 3efcef04492..a3067e06f14 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -16,3 +16,17 @@ nix-build dependencies.nix --no-out-link --repeat 3
 (! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log)
 grep 'differs from previous round' $TEST_ROOT/log
 
+path=$(nix-build check.nix -A fetchurl --no-out-link)
+
+chmod +w $path
+echo foo > $path
+chmod -w $path
+
+nix-build check.nix -A fetchurl --no-out-link --check
+
+# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
+[[ $(cat $path) = foo ]]
+
+nix-build check.nix -A fetchurl --no-out-link --repair
+
+[[ $(cat $path) != foo ]]

From 89a2a11d9f39f60097619212d3fed9ab8c216c8b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Jan 2018 14:58:26 +0100
Subject: [PATCH 0766/2196] Don't use [[noreturn]]

---
 src/libexpr/eval.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f8685e010e1..33a9bc61428 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -337,10 +337,6 @@ Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
 
-    auto doThrow = [&]() [[noreturn]] {
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
-    };
-
     bool found = false;
 
     for (auto & i : *allowedPaths) {
@@ -350,7 +346,8 @@ Path EvalState::checkSourcePath(const Path & path_)
         }
     }
 
-    if (!found) doThrow();
+    if (!found)
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
 
     /* Resolve symlinks. */
     debug(format("checking access to '%s'") % path_);
@@ -361,7 +358,7 @@ Path EvalState::checkSourcePath(const Path & path_)
             return path;
     }
 
-    doThrow();
+    throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
 }
 
 

From 9304fde8de3b9a54200025701199dcb51725c8b1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 22 Jan 2018 16:56:10 +0100
Subject: [PATCH 0767/2196] Don't access tarballs.nixos.org in a test

https://hydra.nixos.org/build/67806811
---
 tests/check.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/check.sh b/tests/check.sh
index a3067e06f14..b05e40ffbee 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -16,17 +16,17 @@ nix-build dependencies.nix --no-out-link --repeat 3
 (! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log)
 grep 'differs from previous round' $TEST_ROOT/log
 
-path=$(nix-build check.nix -A fetchurl --no-out-link)
+path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
 
 chmod +w $path
 echo foo > $path
 chmod -w $path
 
-nix-build check.nix -A fetchurl --no-out-link --check
+nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
 
 # Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
 [[ $(cat $path) = foo ]]
 
-nix-build check.nix -A fetchurl --no-out-link --repair
+nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
 
 [[ $(cat $path) != foo ]]

From c382866cd26e19900c638e3724af89cf599f1f46 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 22 Jan 2018 17:04:08 +0100
Subject: [PATCH 0768/2196] Fix test

https://hydra.nixos.org/build/67806811
---
 tests/check.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/check.nix b/tests/check.nix
index 585d43032a9..08aac2fb0a7 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -11,7 +11,7 @@ with import ./config.nix;
   };
 
   fetchurl = import <nix/fetchurl.nix> {
-    url = "file://" + toString ./lang/eval-okay-xml.out;
-    sha256 = "426fefcd2430e986551db13fcc2b1e45eeec17e68ffeb6ff155be2f8aaf5407e";
+    url = "file://" + toString ./lang/eval-okay-xml.exp.xml;
+    sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v";
   };
 }

From f7c26365ebc6b24dd41cf1c953b002737f9d6187 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 22 Jan 2018 12:19:50 -0600
Subject: [PATCH 0769/2196] nlohmann-json: 2.1.1 -> 3.0.1

---
 src/nlohmann/json.hpp | 1713 ++++++++++++++++++++++-------------------
 1 file changed, 932 insertions(+), 781 deletions(-)

diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
index 9754e464c76..5b0b0ea5b30 100644
--- a/src/nlohmann/json.hpp
+++ b/src/nlohmann/json.hpp
@@ -1,7 +1,7 @@
 /*
     __ _____ _____ _____
  __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 2.1.1
+|  |  |__   |  |  | | | |  version 3.0.1
 |_____|_____|_____|_|___|  https://github.com/nlohmann/json
 
 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
@@ -109,7 +109,7 @@ SOFTWARE.
     #define JSON_UNLIKELY(x)    x
 #endif
 
-// cpp language standard detection
+// C++ language standard detection
 #if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
     #define JSON_HAS_CPP_17
     #define JSON_HAS_CPP_14
@@ -128,20 +128,18 @@ template<typename = void, typename = void>
 struct adl_serializer;
 
 // forward declaration of basic_json (required to split the class)
-template<template<typename U, typename V, typename... Args> class ObjectType =
-         std::map,
-         template<typename U, typename... Args> class ArrayType = std::vector,
+template<template<typename, typename, typename...> class ObjectType = std::map,
+         template<typename, typename...> class ArrayType = std::vector,
          class StringType = std::string, class BooleanType = bool,
          class NumberIntegerType = std::int64_t,
          class NumberUnsignedType = std::uint64_t,
          class NumberFloatType = double,
-         template<typename U> class AllocatorType = std::allocator,
-         template<typename T, typename SFINAE = void> class JSONSerializer =
-         adl_serializer>
+         template<typename> class AllocatorType = std::allocator,
+         template<typename, typename = void> class JSONSerializer = adl_serializer>
 class basic_json;
 
-// Ugly macros to avoid uglier copy-paste when specializing basic_json
-// This is only temporary and will be removed in 3.0
+// Ugly macros to avoid uglier copy-paste when specializing basic_json. They
+// may be removed in the future once the class is split.
 
 #define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \
     template<template<typename, typename, typename...> class ObjectType,   \
@@ -227,7 +225,7 @@ class exception : public std::exception
 /*!
 @brief exception indicating a parse error
 
-This excpetion is thrown by the library when a parse error occurs. Parse errors
+This exception is thrown by the library when a parse error occurs. Parse errors
 can occur during the deserialization of JSON text, CBOR, MessagePack, as well
 as when using JSON Patch.
 
@@ -243,12 +241,12 @@ json.exception.parse_error.102 | parse error at 14: missing or wrong low surroga
 json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
 json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
 json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
-json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`.
+json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`.
 json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
 json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
 json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
 json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
-json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
+json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
 json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
 
 @note For an input with n bytes, 1 is the index of the first character and n+1
@@ -378,6 +376,7 @@ json.exception.type_error.312 | cannot use update() with string | The @ref updat
 json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
 json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
 json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
+json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
 
 @liveexample{The following code shows how a `type_error` exception can be
 caught.,type_error}
@@ -457,7 +456,6 @@ Exceptions have ids 5xx.
 name / id                      | example message | description
 ------------------------------ | --------------- | -------------------------
 json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
-json.exception.other_error.502 | invalid object size for conversion | Some conversions to user-defined types impose constraints on the object size (e.g. std::pair)
 
 @sa @ref exception for the base class of the library exceptions
 @sa @ref parse_error for exceptions indicating a parse error
@@ -540,20 +538,14 @@ Returns an ordering that is similar to Python:
 inline bool operator<(const value_t lhs, const value_t rhs) noexcept
 {
     static constexpr std::array<uint8_t, 8> order = {{
-            0, // null
-            3, // object
-            4, // array
-            5, // string
-            1, // boolean
-            2, // integer
-            2, // unsigned
-            2, // float
+            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
+            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
         }
     };
 
     const auto l_index = static_cast<std::size_t>(lhs);
     const auto r_index = static_cast<std::size_t>(rhs);
-    return (l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]);
+    return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
 }
 
 
@@ -591,17 +583,15 @@ struct merge_and_renumber;
 
 template<std::size_t... I1, std::size_t... I2>
 struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
-        : index_sequence < I1..., (sizeof...(I1) + I2)... >
-          {};
+        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
 
 template<std::size_t N>
 struct make_index_sequence
     : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
-      typename make_index_sequence < N - N / 2 >::type >
-{};
+      typename make_index_sequence < N - N / 2 >::type > {};
 
-template<> struct make_index_sequence<0> : index_sequence<> { };
-template<> struct make_index_sequence<1> : index_sequence<0> { };
+template<> struct make_index_sequence<0> : index_sequence<> {};
+template<> struct make_index_sequence<1> : index_sequence<0> {};
 
 template<typename... Ts>
 using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
@@ -624,7 +614,7 @@ template<class B1> struct conjunction<B1> : B1 {};
 template<class B1, class... Bn>
 struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
 
-template<class B> struct negation : std::integral_constant < bool, !B::value > {};
+template<class B> struct negation : std::integral_constant<bool, not B::value> {};
 
 // dispatch utility (taken from ranges-v3)
 template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
@@ -725,8 +715,7 @@ struct external_constructor<value_t::array>
     }
 
     template<typename BasicJsonType, typename CompatibleArrayType,
-             enable_if_t<not std::is_same<CompatibleArrayType,
-                                          typename BasicJsonType::array_t>::value,
+             enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
                          int> = 0>
     static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
     {
@@ -743,7 +732,7 @@ struct external_constructor<value_t::array>
         j.m_type = value_t::array;
         j.m_value = value_t::array;
         j.m_value.array->reserve(arr.size());
-        for (bool x : arr)
+        for (const bool x : arr)
         {
             j.m_value.array->push_back(x);
         }
@@ -782,8 +771,7 @@ struct external_constructor<value_t::object>
     }
 
     template<typename BasicJsonType, typename CompatibleObjectType,
-             enable_if_t<not std::is_same<CompatibleObjectType,
-                                          typename BasicJsonType::object_t>::value, int> = 0>
+             enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0>
     static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
     {
         using std::begin;
@@ -896,7 +884,7 @@ struct is_compatible_integer_type
         is_compatible_integer_type_impl <
         std::is_integral<CompatibleNumberIntegerType>::value and
         not std::is_same<bool, CompatibleNumberIntegerType>::value,
-        RealIntegerType, CompatibleNumberIntegerType > ::value;
+        RealIntegerType, CompatibleNumberIntegerType >::value;
 };
 
 
@@ -922,10 +910,8 @@ template<typename BasicJsonType, typename T>
 struct has_non_default_from_json
 {
   private:
-    template <
-        typename U,
-        typename = enable_if_t<std::is_same<
-                                   T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value >>
+    template<typename U, typename =
+             enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>>
     static int detect(U&&);
     static void detect(...);
 
@@ -954,22 +940,21 @@ struct has_to_json
 // to_json //
 /////////////
 
-template<typename BasicJsonType, typename T, enable_if_t<
-             std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
 void to_json(BasicJsonType& j, T b) noexcept
 {
     external_constructor<value_t::boolean>::construct(j, b);
 }
 
 template<typename BasicJsonType, typename CompatibleString,
-         enable_if_t<std::is_constructible<typename BasicJsonType::string_t,
-                     CompatibleString>::value, int> = 0>
+         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
 void to_json(BasicJsonType& j, const CompatibleString& s)
 {
     external_constructor<value_t::string>::construct(j, s);
 }
 
-template <typename BasicJsonType>
+template<typename BasicJsonType>
 void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
 {
     external_constructor<value_t::string>::construct(j, std::move(s));
@@ -982,19 +967,15 @@ void to_json(BasicJsonType& j, FloatType val) noexcept
     external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
 }
 
-template <
-    typename BasicJsonType, typename CompatibleNumberUnsignedType,
-    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t,
-                CompatibleNumberUnsignedType>::value, int> = 0 >
+template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
 void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
 {
     external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
 }
 
-template <
-    typename BasicJsonType, typename CompatibleNumberIntegerType,
-    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t,
-                CompatibleNumberIntegerType>::value, int> = 0 >
+template<typename BasicJsonType, typename CompatibleNumberIntegerType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
 void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
 {
     external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
@@ -1014,49 +995,43 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e)
     external_constructor<value_t::array>::construct(j, e);
 }
 
-template <
-    typename BasicJsonType, typename CompatibleArrayType,
-    enable_if_t <
-        is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
-        std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
-        int > = 0 >
+template<typename BasicJsonType, typename CompatibleArrayType,
+         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
+                     std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
+                     int> = 0>
 void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
 {
     external_constructor<value_t::array>::construct(j, arr);
 }
 
-template <typename BasicJsonType, typename T,
-          enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
 void to_json(BasicJsonType& j, std::valarray<T> arr)
 {
     external_constructor<value_t::array>::construct(j, std::move(arr));
 }
 
-template <typename BasicJsonType>
+template<typename BasicJsonType>
 void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
 {
     external_constructor<value_t::array>::construct(j, std::move(arr));
 }
 
-template <
-    typename BasicJsonType, typename CompatibleObjectType,
-    enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value,
-                int> = 0 >
+template<typename BasicJsonType, typename CompatibleObjectType,
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
 void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
 {
     external_constructor<value_t::object>::construct(j, obj);
 }
 
-template <typename BasicJsonType>
+template<typename BasicJsonType>
 void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
 {
     external_constructor<value_t::object>::construct(j, std::move(obj));
 }
 
 template<typename BasicJsonType, typename T, std::size_t N,
-         enable_if_t<not std::is_constructible<
-                         typename BasicJsonType::string_t, T (&)[N]>::value,
-                     int> = 0>
+         enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0>
 void to_json(BasicJsonType& j, T (&arr)[N])
 {
     external_constructor<value_t::array>::construct(j, arr);
@@ -1087,8 +1062,7 @@ void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
 // overloads for basic_json template parameters
 template<typename BasicJsonType, typename ArithmeticType,
          enable_if_t<std::is_arithmetic<ArithmeticType>::value and
-                     not std::is_same<ArithmeticType,
-                                      typename BasicJsonType::boolean_t>::value,
+                     not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
                      int> = 0>
 void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
 {
@@ -1351,6 +1325,13 @@ struct to_json_fn
     {
         static_assert(sizeof(BasicJsonType) == 0,
                       "could not find to_json() method in T's namespace");
+
+#ifdef _MSC_VER
+        // MSVC does not show a stacktrace for the above assert
+        using decayed = uncvref_t<T>;
+        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
+                      "forcing MSVC stacktrace to show which T we're talking about.");
+#endif
     }
 
   public:
@@ -1378,6 +1359,12 @@ struct from_json_fn
     {
         static_assert(sizeof(BasicJsonType) == 0,
                       "could not find from_json() method in T's namespace");
+#ifdef _MSC_VER
+        // MSVC does not show a stacktrace for the above assert
+        using decayed = uncvref_t<T>;
+        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
+                      "forcing MSVC stacktrace to show which T we're talking about.");
+#endif
     }
 
   public:
@@ -1448,7 +1435,7 @@ class input_stream_adapter : public input_adapter_protocol
     explicit input_stream_adapter(std::istream& i)
         : is(i), sb(*i.rdbuf())
     {
-        // ignore Byte Order Mark at start of input
+        // skip byte order mark
         std::char_traits<char>::int_type c;
         if ((c = get_character()) == 0xEF)
         {
@@ -1472,7 +1459,7 @@ class input_stream_adapter : public input_adapter_protocol
         }
         else if (c != std::char_traits<char>::eof())
         {
-            is.unget(); // Not BOM. Process as usual.
+            is.unget(); // no byte order mark; process as usual
         }
     }
 
@@ -1481,8 +1468,8 @@ class input_stream_adapter : public input_adapter_protocol
     input_stream_adapter& operator=(input_stream_adapter&) = delete;
 
     // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
-    // ensure that std::char_traits<char>::eof() and the character 0xff do not
-    // end up as the same value, eg. 0xffffffff.
+    // ensure that std::char_traits<char>::eof() and the character 0xFF do not
+    // end up as the same value, eg. 0xFFFFFFFF.
     std::char_traits<char>::int_type get_character() override
     {
         return sb.sbumpc();
@@ -1561,8 +1548,7 @@ class input_adapter
     template<typename CharT,
              typename std::enable_if<
                  std::is_pointer<CharT>::value and
-                 std::is_integral<
-                     typename std::remove_pointer<CharT>::type>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
                  sizeof(typename std::remove_pointer<CharT>::type) == 1,
                  int>::type = 0>
     input_adapter(CharT b, std::size_t l)
@@ -1574,8 +1560,7 @@ class input_adapter
     template<typename CharT,
              typename std::enable_if<
                  std::is_pointer<CharT>::value and
-                 std::is_integral<
-                     typename std::remove_pointer<CharT>::type>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
                  sizeof(typename std::remove_pointer<CharT>::type) == 1,
                  int>::type = 0>
     input_adapter(CharT b)
@@ -1585,8 +1570,7 @@ class input_adapter
     /// input adapter for iterator range with contiguous storage
     template<class IteratorType,
              typename std::enable_if<
-                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category,
-                              std::random_access_iterator_tag>::value,
+                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
                  int>::type = 0>
     input_adapter(IteratorType first, IteratorType last)
     {
@@ -1624,13 +1608,10 @@ class input_adapter
         : input_adapter(std::begin(array), std::end(array)) {}
 
     /// input adapter for contiguous container
-    template <
-        class ContiguousContainer,
-        typename std::enable_if <
-            not std::is_pointer<ContiguousContainer>::value and
-            std::is_base_of<std::random_access_iterator_tag,
-                            typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
-            int >::type = 0 >
+    template<class ContiguousContainer, typename
+             std::enable_if<not std::is_pointer<ContiguousContainer>::value and
+                            std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+                            int>::type = 0>
     input_adapter(const ContiguousContainer& c)
         : input_adapter(std::begin(c), std::end(c)) {}
 
@@ -1804,6 +1785,12 @@ class lexer
     checks if it is inside the range. If a violation was detected, set up an
     error message and return false. Otherwise, return true.
 
+    @param[in] ranges  list of integers; interpreted as list of pairs of
+                       inclusive lower and upper bound, respectively
+
+    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
+         1, 2, or 3 pairs. This precondition is enforced by an assertion.
+
     @return true if and only if no range violation was detected
     */
     bool next_byte_in_range(std::initializer_list<int> ranges)
@@ -1970,19 +1957,19 @@ class lexer
                             // result of the above calculation yields a proper codepoint
                             assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
 
-                            // translate code point to bytes
+                            // translate codepoint into bytes
                             if (codepoint < 0x80)
                             {
                                 // 1-byte characters: 0xxxxxxx (ASCII)
                                 add(codepoint);
                             }
-                            else if (codepoint <= 0x7ff)
+                            else if (codepoint <= 0x7FF)
                             {
                                 // 2-byte characters: 110xxxxx 10xxxxxx
                                 add(0xC0 | (codepoint >> 6));
                                 add(0x80 | (codepoint & 0x3F));
                             }
-                            else if (codepoint <= 0xffff)
+                            else if (codepoint <= 0xFFFF)
                             {
                                 // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
                                 add(0xE0 | (codepoint >> 12));
@@ -2021,12 +2008,12 @@ class lexer
                 case 0x07:
                 case 0x08:
                 case 0x09:
-                case 0x0a:
-                case 0x0b:
-                case 0x0c:
-                case 0x0d:
-                case 0x0e:
-                case 0x0f:
+                case 0x0A:
+                case 0x0B:
+                case 0x0C:
+                case 0x0D:
+                case 0x0E:
+                case 0x0F:
                 case 0x10:
                 case 0x11:
                 case 0x12:
@@ -2037,12 +2024,12 @@ class lexer
                 case 0x17:
                 case 0x18:
                 case 0x19:
-                case 0x1a:
-                case 0x1b:
-                case 0x1c:
-                case 0x1d:
-                case 0x1e:
-                case 0x1f:
+                case 0x1A:
+                case 0x1B:
+                case 0x1C:
+                case 0x1D:
+                case 0x1E:
+                case 0x1F:
                 {
                     error_message = "invalid string: control character must be escaped";
                     return token_type::parse_error;
@@ -2058,12 +2045,12 @@ class lexer
                 case 0x27:
                 case 0x28:
                 case 0x29:
-                case 0x2a:
-                case 0x2b:
-                case 0x2c:
-                case 0x2d:
-                case 0x2e:
-                case 0x2f:
+                case 0x2A:
+                case 0x2B:
+                case 0x2C:
+                case 0x2D:
+                case 0x2E:
+                case 0x2F:
                 case 0x30:
                 case 0x31:
                 case 0x32:
@@ -2074,12 +2061,12 @@ class lexer
                 case 0x37:
                 case 0x38:
                 case 0x39:
-                case 0x3a:
-                case 0x3b:
-                case 0x3c:
-                case 0x3d:
-                case 0x3e:
-                case 0x3f:
+                case 0x3A:
+                case 0x3B:
+                case 0x3C:
+                case 0x3D:
+                case 0x3E:
+                case 0x3F:
                 case 0x40:
                 case 0x41:
                 case 0x42:
@@ -2090,12 +2077,12 @@ class lexer
                 case 0x47:
                 case 0x48:
                 case 0x49:
-                case 0x4a:
-                case 0x4b:
-                case 0x4c:
-                case 0x4d:
-                case 0x4e:
-                case 0x4f:
+                case 0x4A:
+                case 0x4B:
+                case 0x4C:
+                case 0x4D:
+                case 0x4E:
+                case 0x4F:
                 case 0x50:
                 case 0x51:
                 case 0x52:
@@ -2106,11 +2093,11 @@ class lexer
                 case 0x57:
                 case 0x58:
                 case 0x59:
-                case 0x5a:
-                case 0x5b:
-                case 0x5d:
-                case 0x5e:
-                case 0x5f:
+                case 0x5A:
+                case 0x5B:
+                case 0x5D:
+                case 0x5E:
+                case 0x5F:
                 case 0x60:
                 case 0x61:
                 case 0x62:
@@ -2121,12 +2108,12 @@ class lexer
                 case 0x67:
                 case 0x68:
                 case 0x69:
-                case 0x6a:
-                case 0x6b:
-                case 0x6c:
-                case 0x6d:
-                case 0x6e:
-                case 0x6f:
+                case 0x6A:
+                case 0x6B:
+                case 0x6C:
+                case 0x6D:
+                case 0x6E:
+                case 0x6F:
                 case 0x70:
                 case 0x71:
                 case 0x72:
@@ -2137,48 +2124,48 @@ class lexer
                 case 0x77:
                 case 0x78:
                 case 0x79:
-                case 0x7a:
-                case 0x7b:
-                case 0x7c:
-                case 0x7d:
-                case 0x7e:
-                case 0x7f:
+                case 0x7A:
+                case 0x7B:
+                case 0x7C:
+                case 0x7D:
+                case 0x7E:
+                case 0x7F:
                 {
                     add(current);
                     break;
                 }
 
                 // U+0080..U+07FF: bytes C2..DF 80..BF
-                case 0xc2:
-                case 0xc3:
-                case 0xc4:
-                case 0xc5:
-                case 0xc6:
-                case 0xc7:
-                case 0xc8:
-                case 0xc9:
-                case 0xca:
-                case 0xcb:
-                case 0xcc:
-                case 0xcd:
-                case 0xce:
-                case 0xcf:
-                case 0xd0:
-                case 0xd1:
-                case 0xd2:
-                case 0xd3:
-                case 0xd4:
-                case 0xd5:
-                case 0xd6:
-                case 0xd7:
-                case 0xd8:
-                case 0xd9:
-                case 0xda:
-                case 0xdb:
-                case 0xdc:
-                case 0xdd:
-                case 0xde:
-                case 0xdf:
+                case 0xC2:
+                case 0xC3:
+                case 0xC4:
+                case 0xC5:
+                case 0xC6:
+                case 0xC7:
+                case 0xC8:
+                case 0xC9:
+                case 0xCA:
+                case 0xCB:
+                case 0xCC:
+                case 0xCD:
+                case 0xCE:
+                case 0xCF:
+                case 0xD0:
+                case 0xD1:
+                case 0xD2:
+                case 0xD3:
+                case 0xD4:
+                case 0xD5:
+                case 0xD6:
+                case 0xD7:
+                case 0xD8:
+                case 0xD9:
+                case 0xDA:
+                case 0xDB:
+                case 0xDC:
+                case 0xDD:
+                case 0xDE:
+                case 0xDF:
                 {
                     if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
                     {
@@ -2188,7 +2175,7 @@ class lexer
                 }
 
                 // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
-                case 0xe0:
+                case 0xE0:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
                     {
@@ -2199,20 +2186,20 @@ class lexer
 
                 // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
                 // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
-                case 0xe1:
-                case 0xe2:
-                case 0xe3:
-                case 0xe4:
-                case 0xe5:
-                case 0xe6:
-                case 0xe7:
-                case 0xe8:
-                case 0xe9:
-                case 0xea:
-                case 0xeb:
-                case 0xec:
-                case 0xee:
-                case 0xef:
+                case 0xE1:
+                case 0xE2:
+                case 0xE3:
+                case 0xE4:
+                case 0xE5:
+                case 0xE6:
+                case 0xE7:
+                case 0xE8:
+                case 0xE9:
+                case 0xEA:
+                case 0xEB:
+                case 0xEC:
+                case 0xEE:
+                case 0xEF:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
                     {
@@ -2222,7 +2209,7 @@ class lexer
                 }
 
                 // U+D000..U+D7FF: bytes ED 80..9F 80..BF
-                case 0xed:
+                case 0xED:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
                     {
@@ -2232,7 +2219,7 @@ class lexer
                 }
 
                 // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
-                case 0xf0:
+                case 0xF0:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
@@ -2242,9 +2229,9 @@ class lexer
                 }
 
                 // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
-                case 0xf1:
-                case 0xf2:
-                case 0xf3:
+                case 0xF1:
+                case 0xF2:
+                case 0xF3:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
@@ -2254,7 +2241,7 @@ class lexer
                 }
 
                 // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
-                case 0xf4:
+                case 0xF4:
                 {
                     if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
@@ -2772,9 +2759,9 @@ class lexer
     {
         // escape control characters
         std::string result;
-        for (auto c : token_string)
+        for (const auto c : token_string)
         {
-            if ('\x00' <= c and c <= '\x1f')
+            if ('\x00' <= c and c <= '\x1F')
             {
                 // escape control characters
                 std::stringstream ss;
@@ -2877,10 +2864,10 @@ class lexer
     std::size_t chars_read = 0;
 
     /// raw input token string (for error messages)
-    std::vector<char> token_string { };
+    std::vector<char> token_string {};
 
     /// buffer for variable-length tokens (numbers, strings)
-    std::string yytext { };
+    std::string yytext {};
 
     /// a description of occurred lexer errors
     const char* error_message = "";
@@ -3281,7 +3268,7 @@ class parser
     }
 
     /*!
-    @brief the acutal acceptor
+    @brief the actual acceptor
 
     @invariant 1. The last token is not yet processed. Therefore, the caller
                   of this function must make sure a token has been read.
@@ -3539,7 +3526,7 @@ class primitive_iterator_t
         return *this;
     }
 
-    primitive_iterator_t operator++(int)
+    primitive_iterator_t const operator++(int)
     {
         auto result = *this;
         m_it++;
@@ -3552,7 +3539,7 @@ class primitive_iterator_t
         return *this;
     }
 
-    primitive_iterator_t operator--(int)
+    primitive_iterator_t const operator--(int)
     {
         auto result = *this;
         m_it--;
@@ -3618,7 +3605,7 @@ This class implements a both iterators (iterator and const_iterator) for the
        iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
 */
 template<typename BasicJsonType>
-class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJsonType>
+class iter_impl
 {
     /// allow basic_json to access private members
     friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
@@ -3632,6 +3619,14 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso
                   "iter_impl only accepts (const) basic_json");
 
   public:
+
+    /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.
+    /// The C++ Standard has never required user-defined iterators to derive from std::iterator.
+    /// A user-defined iterator should provide publicly accessible typedefs named
+    /// iterator_category, value_type, difference_type, pointer, and reference.
+    /// Note that value_type is required to be non-const, even for constant iterators.
+    using iterator_category = std::bidirectional_iterator_tag;
+
     /// the type of the values when the iterator is dereferenced
     using value_type = typename BasicJsonType::value_type;
     /// a type to represent differences between iterators
@@ -3855,7 +3850,7 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso
     @brief post-increment (it++)
     @pre The iterator is initialized; i.e. `m_object != nullptr`.
     */
-    iter_impl operator++(int)
+    iter_impl const operator++(int)
     {
         auto result = *this;
         ++(*this);
@@ -3898,7 +3893,7 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso
     @brief post-decrement (it--)
     @pre The iterator is initialized; i.e. `m_object != nullptr`.
     */
-    iter_impl operator--(int)
+    iter_impl const operator--(int)
     {
         auto result = *this;
         --(*this);
@@ -4291,7 +4286,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
 {
   public:
     using difference_type = std::ptrdiff_t;
-    /// shortcut to the reverse iterator adaptor
+    /// shortcut to the reverse iterator adapter
     using base_iterator = std::reverse_iterator<Base>;
     /// the reference type for the pointed-to element
     using reference = typename Base::reference;
@@ -4304,7 +4299,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
     json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
 
     /// post-increment (it++)
-    json_reverse_iterator operator++(int)
+    json_reverse_iterator const operator++(int)
     {
         return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
     }
@@ -4316,7 +4311,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
     }
 
     /// post-decrement (it--)
-    json_reverse_iterator operator--(int)
+    json_reverse_iterator const operator--(int)
     {
         return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
     }
@@ -4576,12 +4571,12 @@ class binary_reader
             case 0x07:
             case 0x08:
             case 0x09:
-            case 0x0a:
-            case 0x0b:
-            case 0x0c:
-            case 0x0d:
-            case 0x0e:
-            case 0x0f:
+            case 0x0A:
+            case 0x0B:
+            case 0x0C:
+            case 0x0D:
+            case 0x0E:
+            case 0x0F:
             case 0x10:
             case 0x11:
             case 0x12:
@@ -4598,10 +4593,10 @@ class binary_reader
             case 0x19: // Unsigned integer (two-byte uint16_t follows)
                 return get_number<uint16_t>();
 
-            case 0x1a: // Unsigned integer (four-byte uint32_t follows)
+            case 0x1A: // Unsigned integer (four-byte uint32_t follows)
                 return get_number<uint32_t>();
 
-            case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
+            case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
                 return get_number<uint64_t>();
 
             // Negative integer -1-0x00..-1-0x17 (-1..-24)
@@ -4615,12 +4610,12 @@ class binary_reader
             case 0x27:
             case 0x28:
             case 0x29:
-            case 0x2a:
-            case 0x2b:
-            case 0x2c:
-            case 0x2d:
-            case 0x2e:
-            case 0x2f:
+            case 0x2A:
+            case 0x2B:
+            case 0x2C:
+            case 0x2D:
+            case 0x2E:
+            case 0x2F:
             case 0x30:
             case 0x31:
             case 0x32:
@@ -4642,12 +4637,12 @@ class binary_reader
                 return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
             }
 
-            case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
+            case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
             {
                 return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
             }
 
-            case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
+            case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
             {
                 return static_cast<number_integer_t>(-1) -
                        static_cast<number_integer_t>(get_number<uint64_t>());
@@ -4664,12 +4659,12 @@ class binary_reader
             case 0x67:
             case 0x68:
             case 0x69:
-            case 0x6a:
-            case 0x6b:
-            case 0x6c:
-            case 0x6d:
-            case 0x6e:
-            case 0x6f:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
             case 0x70:
             case 0x71:
             case 0x72:
@@ -4680,9 +4675,9 @@ class binary_reader
             case 0x77:
             case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
             case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
-            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
-            case 0x7f: // UTF-8 string (indefinite length)
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 0x7F: // UTF-8 string (indefinite length)
             {
                 return get_cbor_string();
             }
@@ -4698,12 +4693,12 @@ class binary_reader
             case 0x87:
             case 0x88:
             case 0x89:
-            case 0x8a:
-            case 0x8b:
-            case 0x8c:
-            case 0x8d:
-            case 0x8e:
-            case 0x8f:
+            case 0x8A:
+            case 0x8B:
+            case 0x8C:
+            case 0x8D:
+            case 0x8E:
+            case 0x8F:
             case 0x90:
             case 0x91:
             case 0x92:
@@ -4713,7 +4708,7 @@ class binary_reader
             case 0x96:
             case 0x97:
             {
-                return get_cbor_array(current & 0x1f);
+                return get_cbor_array(current & 0x1F);
             }
 
             case 0x98: // array (one-byte uint8_t for n follows)
@@ -4726,20 +4721,20 @@ class binary_reader
                 return get_cbor_array(get_number<uint16_t>());
             }
 
-            case 0x9a: // array (four-byte uint32_t for n follow)
+            case 0x9A: // array (four-byte uint32_t for n follow)
             {
                 return get_cbor_array(get_number<uint32_t>());
             }
 
-            case 0x9b: // array (eight-byte uint64_t for n follow)
+            case 0x9B: // array (eight-byte uint64_t for n follow)
             {
                 return get_cbor_array(get_number<uint64_t>());
             }
 
-            case 0x9f: // array (indefinite length)
+            case 0x9F: // array (indefinite length)
             {
                 BasicJsonType result = value_t::array;
-                while (get() != 0xff)
+                while (get() != 0xFF)
                 {
                     result.push_back(parse_cbor_internal(false));
                 }
@@ -4747,58 +4742,58 @@ class binary_reader
             }
 
             // map (0x00..0x17 pairs of data items follow)
-            case 0xa0:
-            case 0xa1:
-            case 0xa2:
-            case 0xa3:
-            case 0xa4:
-            case 0xa5:
-            case 0xa6:
-            case 0xa7:
-            case 0xa8:
-            case 0xa9:
-            case 0xaa:
-            case 0xab:
-            case 0xac:
-            case 0xad:
-            case 0xae:
-            case 0xaf:
-            case 0xb0:
-            case 0xb1:
-            case 0xb2:
-            case 0xb3:
-            case 0xb4:
-            case 0xb5:
-            case 0xb6:
-            case 0xb7:
-            {
-                return get_cbor_object(current & 0x1f);
-            }
-
-            case 0xb8: // map (one-byte uint8_t for n follows)
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            {
+                return get_cbor_object(current & 0x1F);
+            }
+
+            case 0xB8: // map (one-byte uint8_t for n follows)
             {
                 return get_cbor_object(get_number<uint8_t>());
             }
 
-            case 0xb9: // map (two-byte uint16_t for n follow)
+            case 0xB9: // map (two-byte uint16_t for n follow)
             {
                 return get_cbor_object(get_number<uint16_t>());
             }
 
-            case 0xba: // map (four-byte uint32_t for n follow)
+            case 0xBA: // map (four-byte uint32_t for n follow)
             {
                 return get_cbor_object(get_number<uint32_t>());
             }
 
-            case 0xbb: // map (eight-byte uint64_t for n follow)
+            case 0xBB: // map (eight-byte uint64_t for n follow)
             {
                 return get_cbor_object(get_number<uint64_t>());
             }
 
-            case 0xbf: // map (indefinite length)
+            case 0xBF: // map (indefinite length)
             {
                 BasicJsonType result = value_t::object;
-                while (get() != 0xff)
+                while (get() != 0xFF)
                 {
                     auto key = get_cbor_string();
                     result[key] = parse_cbor_internal();
@@ -4806,22 +4801,22 @@ class binary_reader
                 return result;
             }
 
-            case 0xf4: // false
+            case 0xF4: // false
             {
                 return false;
             }
 
-            case 0xf5: // true
+            case 0xF5: // true
             {
                 return true;
             }
 
-            case 0xf6: // null
+            case 0xF6: // null
             {
                 return value_t::null;
             }
 
-            case 0xf9: // Half-Precision Float (two-byte IEEE 754)
+            case 0xF9: // Half-Precision Float (two-byte IEEE 754)
             {
                 const int byte1 = get();
                 check_eof();
@@ -4837,8 +4832,8 @@ class binary_reader
                 // half-precision floating-point numbers in the C language
                 // is shown in Fig. 3.
                 const int half = (byte1 << 8) + byte2;
-                const int exp = (half >> 10) & 0x1f;
-                const int mant = half & 0x3ff;
+                const int exp = (half >> 10) & 0x1F;
+                const int mant = half & 0x3FF;
                 double val;
                 if (exp == 0)
                 {
@@ -4856,12 +4851,12 @@ class binary_reader
                 return (half & 0x8000) != 0 ? -val : val;
             }
 
-            case 0xfa: // Single-Precision Float (four-byte IEEE 754)
+            case 0xFA: // Single-Precision Float (four-byte IEEE 754)
             {
                 return get_number<float>();
             }
 
-            case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
+            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
             {
                 return get_number<double>();
             }
@@ -4869,7 +4864,7 @@ class binary_reader
             default: // anything else (0xFF is handled inside the other types)
             {
                 std::stringstream ss;
-                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
                 JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
             }
         }
@@ -4894,12 +4889,12 @@ class binary_reader
             case 0x07:
             case 0x08:
             case 0x09:
-            case 0x0a:
-            case 0x0b:
-            case 0x0c:
-            case 0x0d:
-            case 0x0e:
-            case 0x0f:
+            case 0x0A:
+            case 0x0B:
+            case 0x0C:
+            case 0x0D:
+            case 0x0E:
+            case 0x0F:
             case 0x10:
             case 0x11:
             case 0x12:
@@ -4910,12 +4905,12 @@ class binary_reader
             case 0x17:
             case 0x18:
             case 0x19:
-            case 0x1a:
-            case 0x1b:
-            case 0x1c:
-            case 0x1d:
-            case 0x1e:
-            case 0x1f:
+            case 0x1A:
+            case 0x1B:
+            case 0x1C:
+            case 0x1D:
+            case 0x1E:
+            case 0x1F:
             case 0x20:
             case 0x21:
             case 0x22:
@@ -4926,12 +4921,12 @@ class binary_reader
             case 0x27:
             case 0x28:
             case 0x29:
-            case 0x2a:
-            case 0x2b:
-            case 0x2c:
-            case 0x2d:
-            case 0x2e:
-            case 0x2f:
+            case 0x2A:
+            case 0x2B:
+            case 0x2C:
+            case 0x2D:
+            case 0x2E:
+            case 0x2F:
             case 0x30:
             case 0x31:
             case 0x32:
@@ -4942,12 +4937,12 @@ class binary_reader
             case 0x37:
             case 0x38:
             case 0x39:
-            case 0x3a:
-            case 0x3b:
-            case 0x3c:
-            case 0x3d:
-            case 0x3e:
-            case 0x3f:
+            case 0x3A:
+            case 0x3B:
+            case 0x3C:
+            case 0x3D:
+            case 0x3E:
+            case 0x3F:
             case 0x40:
             case 0x41:
             case 0x42:
@@ -4958,12 +4953,12 @@ class binary_reader
             case 0x47:
             case 0x48:
             case 0x49:
-            case 0x4a:
-            case 0x4b:
-            case 0x4c:
-            case 0x4d:
-            case 0x4e:
-            case 0x4f:
+            case 0x4A:
+            case 0x4B:
+            case 0x4C:
+            case 0x4D:
+            case 0x4E:
+            case 0x4F:
             case 0x50:
             case 0x51:
             case 0x52:
@@ -4974,12 +4969,12 @@ class binary_reader
             case 0x57:
             case 0x58:
             case 0x59:
-            case 0x5a:
-            case 0x5b:
-            case 0x5c:
-            case 0x5d:
-            case 0x5e:
-            case 0x5f:
+            case 0x5A:
+            case 0x5B:
+            case 0x5C:
+            case 0x5D:
+            case 0x5E:
+            case 0x5F:
             case 0x60:
             case 0x61:
             case 0x62:
@@ -4990,12 +4985,12 @@ class binary_reader
             case 0x67:
             case 0x68:
             case 0x69:
-            case 0x6a:
-            case 0x6b:
-            case 0x6c:
-            case 0x6d:
-            case 0x6e:
-            case 0x6f:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
             case 0x70:
             case 0x71:
             case 0x72:
@@ -5006,12 +5001,12 @@ class binary_reader
             case 0x77:
             case 0x78:
             case 0x79:
-            case 0x7a:
-            case 0x7b:
-            case 0x7c:
-            case 0x7d:
-            case 0x7e:
-            case 0x7f:
+            case 0x7A:
+            case 0x7B:
+            case 0x7C:
+            case 0x7D:
+            case 0x7E:
+            case 0x7F:
                 return static_cast<number_unsigned_t>(current);
 
             // fixmap
@@ -5025,14 +5020,14 @@ class binary_reader
             case 0x87:
             case 0x88:
             case 0x89:
-            case 0x8a:
-            case 0x8b:
-            case 0x8c:
-            case 0x8d:
-            case 0x8e:
-            case 0x8f:
+            case 0x8A:
+            case 0x8B:
+            case 0x8C:
+            case 0x8D:
+            case 0x8E:
+            case 0x8F:
             {
-                return get_msgpack_object(current & 0x0f);
+                return get_msgpack_object(current & 0x0F);
             }
 
             // fixarray
@@ -5046,154 +5041,154 @@ class binary_reader
             case 0x97:
             case 0x98:
             case 0x99:
-            case 0x9a:
-            case 0x9b:
-            case 0x9c:
-            case 0x9d:
-            case 0x9e:
-            case 0x9f:
+            case 0x9A:
+            case 0x9B:
+            case 0x9C:
+            case 0x9D:
+            case 0x9E:
+            case 0x9F:
             {
-                return get_msgpack_array(current & 0x0f);
+                return get_msgpack_array(current & 0x0F);
             }
 
             // fixstr
-            case 0xa0:
-            case 0xa1:
-            case 0xa2:
-            case 0xa3:
-            case 0xa4:
-            case 0xa5:
-            case 0xa6:
-            case 0xa7:
-            case 0xa8:
-            case 0xa9:
-            case 0xaa:
-            case 0xab:
-            case 0xac:
-            case 0xad:
-            case 0xae:
-            case 0xaf:
-            case 0xb0:
-            case 0xb1:
-            case 0xb2:
-            case 0xb3:
-            case 0xb4:
-            case 0xb5:
-            case 0xb6:
-            case 0xb7:
-            case 0xb8:
-            case 0xb9:
-            case 0xba:
-            case 0xbb:
-            case 0xbc:
-            case 0xbd:
-            case 0xbe:
-            case 0xbf:
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
                 return get_msgpack_string();
 
-            case 0xc0: // nil
+            case 0xC0: // nil
                 return value_t::null;
 
-            case 0xc2: // false
+            case 0xC2: // false
                 return false;
 
-            case 0xc3: // true
+            case 0xC3: // true
                 return true;
 
-            case 0xca: // float 32
+            case 0xCA: // float 32
                 return get_number<float>();
 
-            case 0xcb: // float 64
+            case 0xCB: // float 64
                 return get_number<double>();
 
-            case 0xcc: // uint 8
+            case 0xCC: // uint 8
                 return get_number<uint8_t>();
 
-            case 0xcd: // uint 16
+            case 0xCD: // uint 16
                 return get_number<uint16_t>();
 
-            case 0xce: // uint 32
+            case 0xCE: // uint 32
                 return get_number<uint32_t>();
 
-            case 0xcf: // uint 64
+            case 0xCF: // uint 64
                 return get_number<uint64_t>();
 
-            case 0xd0: // int 8
+            case 0xD0: // int 8
                 return get_number<int8_t>();
 
-            case 0xd1: // int 16
+            case 0xD1: // int 16
                 return get_number<int16_t>();
 
-            case 0xd2: // int 32
+            case 0xD2: // int 32
                 return get_number<int32_t>();
 
-            case 0xd3: // int 64
+            case 0xD3: // int 64
                 return get_number<int64_t>();
 
-            case 0xd9: // str 8
-            case 0xda: // str 16
-            case 0xdb: // str 32
+            case 0xD9: // str 8
+            case 0xDA: // str 16
+            case 0xDB: // str 32
                 return get_msgpack_string();
 
-            case 0xdc: // array 16
+            case 0xDC: // array 16
             {
                 return get_msgpack_array(get_number<uint16_t>());
             }
 
-            case 0xdd: // array 32
+            case 0xDD: // array 32
             {
                 return get_msgpack_array(get_number<uint32_t>());
             }
 
-            case 0xde: // map 16
+            case 0xDE: // map 16
             {
                 return get_msgpack_object(get_number<uint16_t>());
             }
 
-            case 0xdf: // map 32
+            case 0xDF: // map 32
             {
                 return get_msgpack_object(get_number<uint32_t>());
             }
 
             // positive fixint
-            case 0xe0:
-            case 0xe1:
-            case 0xe2:
-            case 0xe3:
-            case 0xe4:
-            case 0xe5:
-            case 0xe6:
-            case 0xe7:
-            case 0xe8:
-            case 0xe9:
-            case 0xea:
-            case 0xeb:
-            case 0xec:
-            case 0xed:
-            case 0xee:
-            case 0xef:
-            case 0xf0:
-            case 0xf1:
-            case 0xf2:
-            case 0xf3:
-            case 0xf4:
-            case 0xf5:
-            case 0xf6:
-            case 0xf7:
-            case 0xf8:
-            case 0xf9:
-            case 0xfa:
-            case 0xfb:
-            case 0xfc:
-            case 0xfd:
-            case 0xfe:
-            case 0xff:
+            case 0xE0:
+            case 0xE1:
+            case 0xE2:
+            case 0xE3:
+            case 0xE4:
+            case 0xE5:
+            case 0xE6:
+            case 0xE7:
+            case 0xE8:
+            case 0xE9:
+            case 0xEA:
+            case 0xEB:
+            case 0xEC:
+            case 0xED:
+            case 0xEE:
+            case 0xEF:
+            case 0xF0:
+            case 0xF1:
+            case 0xF2:
+            case 0xF3:
+            case 0xF4:
+            case 0xF5:
+            case 0xF6:
+            case 0xF7:
+            case 0xF8:
+            case 0xF9:
+            case 0xFA:
+            case 0xFB:
+            case 0xFC:
+            case 0xFD:
+            case 0xFE:
+            case 0xFF:
                 return static_cast<int8_t>(current);
 
             default: // anything else
             {
                 std::stringstream ss;
-                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
                 JSON_THROW(parse_error::create(112, chars_read,
                                                "error reading MessagePack; last byte: 0x" + ss.str()));
             }
@@ -5309,12 +5304,12 @@ class binary_reader
             case 0x67:
             case 0x68:
             case 0x69:
-            case 0x6a:
-            case 0x6b:
-            case 0x6c:
-            case 0x6d:
-            case 0x6e:
-            case 0x6f:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
             case 0x70:
             case 0x71:
             case 0x72:
@@ -5324,7 +5319,7 @@ class binary_reader
             case 0x76:
             case 0x77:
             {
-                return get_string(current & 0x1f);
+                return get_string(current & 0x1F);
             }
 
             case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
@@ -5337,20 +5332,20 @@ class binary_reader
                 return get_string(get_number<uint16_t>());
             }
 
-            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
             {
                 return get_string(get_number<uint32_t>());
             }
 
-            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
             {
                 return get_string(get_number<uint64_t>());
             }
 
-            case 0x7f: // UTF-8 string (indefinite length)
+            case 0x7F: // UTF-8 string (indefinite length)
             {
                 std::string result;
-                while (get() != 0xff)
+                while (get() != 0xFF)
                 {
                     check_eof();
                     result.push_back(static_cast<char>(current));
@@ -5361,7 +5356,7 @@ class binary_reader
             default:
             {
                 std::stringstream ss;
-                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
                 JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
             }
         }
@@ -5412,53 +5407,53 @@ class binary_reader
         switch (current)
         {
             // fixstr
-            case 0xa0:
-            case 0xa1:
-            case 0xa2:
-            case 0xa3:
-            case 0xa4:
-            case 0xa5:
-            case 0xa6:
-            case 0xa7:
-            case 0xa8:
-            case 0xa9:
-            case 0xaa:
-            case 0xab:
-            case 0xac:
-            case 0xad:
-            case 0xae:
-            case 0xaf:
-            case 0xb0:
-            case 0xb1:
-            case 0xb2:
-            case 0xb3:
-            case 0xb4:
-            case 0xb5:
-            case 0xb6:
-            case 0xb7:
-            case 0xb8:
-            case 0xb9:
-            case 0xba:
-            case 0xbb:
-            case 0xbc:
-            case 0xbd:
-            case 0xbe:
-            case 0xbf:
-            {
-                return get_string(current & 0x1f);
-            }
-
-            case 0xd9: // str 8
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
+            {
+                return get_string(current & 0x1F);
+            }
+
+            case 0xD9: // str 8
             {
                 return get_string(get_number<uint8_t>());
             }
 
-            case 0xda: // str 16
+            case 0xDA: // str 16
             {
                 return get_string(get_number<uint16_t>());
             }
 
-            case 0xdb: // str 32
+            case 0xDB: // str 32
             {
                 return get_string(get_number<uint32_t>());
             }
@@ -5466,7 +5461,7 @@ class binary_reader
             default:
             {
                 std::stringstream ss;
-                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
                 JSON_THROW(parse_error::create(113, chars_read,
                                                "expected a MessagePack string; last byte: 0x" + ss.str()));
             }
@@ -5562,15 +5557,15 @@ class binary_writer
         {
             case value_t::null:
             {
-                oa->write_character(static_cast<CharType>(0xf6));
+                oa->write_character(static_cast<CharType>(0xF6));
                 break;
             }
 
             case value_t::boolean:
             {
                 oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xf5)
-                                    : static_cast<CharType>(0xf4));
+                                    ? static_cast<CharType>(0xF5)
+                                    : static_cast<CharType>(0xF4));
                 break;
             }
 
@@ -5597,12 +5592,12 @@ class binary_writer
                     }
                     else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x1a));
+                        oa->write_character(static_cast<CharType>(0x1A));
                         write_number(static_cast<uint32_t>(j.m_value.number_integer));
                     }
                     else
                     {
-                        oa->write_character(static_cast<CharType>(0x1b));
+                        oa->write_character(static_cast<CharType>(0x1B));
                         write_number(static_cast<uint64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5627,12 +5622,12 @@ class binary_writer
                     }
                     else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
                     {
-                        oa->write_character(static_cast<CharType>(0x3a));
+                        oa->write_character(static_cast<CharType>(0x3A));
                         write_number(static_cast<uint32_t>(positive_number));
                     }
                     else
                     {
-                        oa->write_character(static_cast<CharType>(0x3b));
+                        oa->write_character(static_cast<CharType>(0x3B));
                         write_number(static_cast<uint64_t>(positive_number));
                     }
                 }
@@ -5657,12 +5652,12 @@ class binary_writer
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                 {
-                    oa->write_character(static_cast<CharType>(0x1a));
+                    oa->write_character(static_cast<CharType>(0x1A));
                     write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
                 }
                 else
                 {
-                    oa->write_character(static_cast<CharType>(0x1b));
+                    oa->write_character(static_cast<CharType>(0x1B));
                     write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
                 }
                 break;
@@ -5670,7 +5665,7 @@ class binary_writer
 
             case value_t::number_float: // Double-Precision Float
             {
-                oa->write_character(static_cast<CharType>(0xfb));
+                oa->write_character(static_cast<CharType>(0xFB));
                 write_number(j.m_value.number_float);
                 break;
             }
@@ -5683,25 +5678,25 @@ class binary_writer
                 {
                     write_number(static_cast<uint8_t>(0x60 + N));
                 }
-                else if (N <= 0xff)
+                else if (N <= 0xFF)
                 {
                     oa->write_character(static_cast<CharType>(0x78));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xffff)
+                else if (N <= 0xFFFF)
                 {
                     oa->write_character(static_cast<CharType>(0x79));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xffffffff)
+                else if (N <= 0xFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0x7a));
+                    oa->write_character(static_cast<CharType>(0x7A));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
+                else if (N <= 0xFFFFFFFFFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0x7b));
+                    oa->write_character(static_cast<CharType>(0x7B));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5721,25 +5716,25 @@ class binary_writer
                 {
                     write_number(static_cast<uint8_t>(0x80 + N));
                 }
-                else if (N <= 0xff)
+                else if (N <= 0xFF)
                 {
                     oa->write_character(static_cast<CharType>(0x98));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xffff)
+                else if (N <= 0xFFFF)
                 {
                     oa->write_character(static_cast<CharType>(0x99));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xffffffff)
+                else if (N <= 0xFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0x9a));
+                    oa->write_character(static_cast<CharType>(0x9A));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
+                else if (N <= 0xFFFFFFFFFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0x9b));
+                    oa->write_character(static_cast<CharType>(0x9B));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5758,27 +5753,27 @@ class binary_writer
                 const auto N = j.m_value.object->size();
                 if (N <= 0x17)
                 {
-                    write_number(static_cast<uint8_t>(0xa0 + N));
+                    write_number(static_cast<uint8_t>(0xA0 + N));
                 }
-                else if (N <= 0xff)
+                else if (N <= 0xFF)
                 {
-                    oa->write_character(static_cast<CharType>(0xb8));
+                    oa->write_character(static_cast<CharType>(0xB8));
                     write_number(static_cast<uint8_t>(N));
                 }
-                else if (N <= 0xffff)
+                else if (N <= 0xFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0xb9));
+                    oa->write_character(static_cast<CharType>(0xB9));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xffffffff)
+                else if (N <= 0xFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0xba));
+                    oa->write_character(static_cast<CharType>(0xBA));
                     write_number(static_cast<uint32_t>(N));
                 }
                 // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
+                else if (N <= 0xFFFFFFFFFFFFFFFF)
                 {
-                    oa->write_character(static_cast<CharType>(0xbb));
+                    oa->write_character(static_cast<CharType>(0xBB));
                     write_number(static_cast<uint64_t>(N));
                 }
                 // LCOV_EXCL_STOP
@@ -5806,15 +5801,15 @@ class binary_writer
         {
             case value_t::null: // nil
             {
-                oa->write_character(static_cast<CharType>(0xc0));
+                oa->write_character(static_cast<CharType>(0xC0));
                 break;
             }
 
             case value_t::boolean: // true and false
             {
                 oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xc3)
-                                    : static_cast<CharType>(0xc2));
+                                    ? static_cast<CharType>(0xC3)
+                                    : static_cast<CharType>(0xC2));
                 break;
             }
 
@@ -5833,25 +5828,25 @@ class binary_writer
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                     {
                         // uint 8
-                        oa->write_character(static_cast<CharType>(0xcc));
+                        oa->write_character(static_cast<CharType>(0xCC));
                         write_number(static_cast<uint8_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
                     {
                         // uint 16
-                        oa->write_character(static_cast<CharType>(0xcd));
+                        oa->write_character(static_cast<CharType>(0xCD));
                         write_number(static_cast<uint16_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                     {
                         // uint 32
-                        oa->write_character(static_cast<CharType>(0xce));
+                        oa->write_character(static_cast<CharType>(0xCE));
                         write_number(static_cast<uint32_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
                     {
                         // uint 64
-                        oa->write_character(static_cast<CharType>(0xcf));
+                        oa->write_character(static_cast<CharType>(0xCF));
                         write_number(static_cast<uint64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5866,28 +5861,28 @@ class binary_writer
                              j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
                     {
                         // int 8
-                        oa->write_character(static_cast<CharType>(0xd0));
+                        oa->write_character(static_cast<CharType>(0xD0));
                         write_number(static_cast<int8_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
                     {
                         // int 16
-                        oa->write_character(static_cast<CharType>(0xd1));
+                        oa->write_character(static_cast<CharType>(0xD1));
                         write_number(static_cast<int16_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
                     {
                         // int 32
-                        oa->write_character(static_cast<CharType>(0xd2));
+                        oa->write_character(static_cast<CharType>(0xD2));
                         write_number(static_cast<int32_t>(j.m_value.number_integer));
                     }
                     else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
                              j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
                     {
                         // int 64
-                        oa->write_character(static_cast<CharType>(0xd3));
+                        oa->write_character(static_cast<CharType>(0xD3));
                         write_number(static_cast<int64_t>(j.m_value.number_integer));
                     }
                 }
@@ -5904,25 +5899,25 @@ class binary_writer
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                 {
                     // uint 8
-                    oa->write_character(static_cast<CharType>(0xcc));
+                    oa->write_character(static_cast<CharType>(0xCC));
                     write_number(static_cast<uint8_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
                 {
                     // uint 16
-                    oa->write_character(static_cast<CharType>(0xcd));
+                    oa->write_character(static_cast<CharType>(0xCD));
                     write_number(static_cast<uint16_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
                 {
                     // uint 32
-                    oa->write_character(static_cast<CharType>(0xce));
+                    oa->write_character(static_cast<CharType>(0xCE));
                     write_number(static_cast<uint32_t>(j.m_value.number_integer));
                 }
                 else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
                 {
                     // uint 64
-                    oa->write_character(static_cast<CharType>(0xcf));
+                    oa->write_character(static_cast<CharType>(0xCF));
                     write_number(static_cast<uint64_t>(j.m_value.number_integer));
                 }
                 break;
@@ -5930,7 +5925,7 @@ class binary_writer
 
             case value_t::number_float: // float 64
             {
-                oa->write_character(static_cast<CharType>(0xcb));
+                oa->write_character(static_cast<CharType>(0xCB));
                 write_number(j.m_value.number_float);
                 break;
             }
@@ -5942,24 +5937,24 @@ class binary_writer
                 if (N <= 31)
                 {
                     // fixstr
-                    write_number(static_cast<uint8_t>(0xa0 | N));
+                    write_number(static_cast<uint8_t>(0xA0 | N));
                 }
                 else if (N <= 255)
                 {
                     // str 8
-                    oa->write_character(static_cast<CharType>(0xd9));
+                    oa->write_character(static_cast<CharType>(0xD9));
                     write_number(static_cast<uint8_t>(N));
                 }
                 else if (N <= 65535)
                 {
                     // str 16
-                    oa->write_character(static_cast<CharType>(0xda));
+                    oa->write_character(static_cast<CharType>(0xDA));
                     write_number(static_cast<uint16_t>(N));
                 }
                 else if (N <= 4294967295)
                 {
                     // str 32
-                    oa->write_character(static_cast<CharType>(0xdb));
+                    oa->write_character(static_cast<CharType>(0xDB));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -5979,16 +5974,16 @@ class binary_writer
                     // fixarray
                     write_number(static_cast<uint8_t>(0x90 | N));
                 }
-                else if (N <= 0xffff)
+                else if (N <= 0xFFFF)
                 {
                     // array 16
-                    oa->write_character(static_cast<CharType>(0xdc));
+                    oa->write_character(static_cast<CharType>(0xDC));
                     write_number(static_cast<uint16_t>(N));
                 }
-                else if (N <= 0xffffffff)
+                else if (N <= 0xFFFFFFFF)
                 {
                     // array 32
-                    oa->write_character(static_cast<CharType>(0xdd));
+                    oa->write_character(static_cast<CharType>(0xDD));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -6007,18 +6002,18 @@ class binary_writer
                 if (N <= 15)
                 {
                     // fixmap
-                    write_number(static_cast<uint8_t>(0x80 | (N & 0xf)));
+                    write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
                 }
                 else if (N <= 65535)
                 {
                     // map 16
-                    oa->write_character(static_cast<CharType>(0xde));
+                    oa->write_character(static_cast<CharType>(0xDE));
                     write_number(static_cast<uint16_t>(N));
                 }
                 else if (N <= 4294967295)
                 {
                     // map 32
-                    oa->write_character(static_cast<CharType>(0xdf));
+                    oa->write_character(static_cast<CharType>(0xDF));
                     write_number(static_cast<uint32_t>(N));
                 }
 
@@ -6363,9 +6358,9 @@ class serializer
                 case 0x05:
                 case 0x06:
                 case 0x07:
-                case 0x0b:
-                case 0x0e:
-                case 0x0f:
+                case 0x0B:
+                case 0x0E:
+                case 0x0F:
                 case 0x10:
                 case 0x11:
                 case 0x12:
@@ -6376,12 +6371,12 @@ class serializer
                 case 0x17:
                 case 0x18:
                 case 0x19:
-                case 0x1a:
-                case 0x1b:
-                case 0x1c:
-                case 0x1d:
-                case 0x1e:
-                case 0x1f:
+                case 0x1A:
+                case 0x1B:
+                case 0x1C:
+                case 0x1D:
+                case 0x1E:
+                case 0x1F:
                 {
                     // from c (1 byte) to \uxxxx (6 bytes)
                     res += 5;
@@ -6393,12 +6388,8 @@ class serializer
                     if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))
                     {
                         const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        if (bytes == std::string::npos)
-                        {
-                            // invalid characters are treated as is, so no
-                            // additional space will be used
-                            break;
-                        }
+                        // invalid characters will be detected by throw_if_invalid_utf8
+                        assert (bytes != std::string::npos);
 
                         if (bytes == 3)
                         {
@@ -6492,6 +6483,8 @@ class serializer
     */
     void dump_escaped(const string_t& s, const bool ensure_ascii) const
     {
+        throw_if_invalid_utf8(s);
+
         const auto space = extra_space(s, ensure_ascii);
         if (space == 0)
         {
@@ -6514,7 +6507,7 @@ class serializer
                     break;
                 }
 
-                case '\\': // reverse solidus (0x5c)
+                case '\\': // reverse solidus (0x5C)
                 {
                     // nothing to change
                     pos += 2;
@@ -6528,21 +6521,21 @@ class serializer
                     break;
                 }
 
-                case '\f': // formfeed (0x0c)
+                case '\f': // formfeed (0x0C)
                 {
                     result[pos + 1] = 'f';
                     pos += 2;
                     break;
                 }
 
-                case '\n': // newline (0x0a)
+                case '\n': // newline (0x0A)
                 {
                     result[pos + 1] = 'n';
                     pos += 2;
                     break;
                 }
 
-                case '\r': // carriage return (0x0d)
+                case '\r': // carriage return (0x0D)
                 {
                     result[pos + 1] = 'r';
                     pos += 2;
@@ -6564,21 +6557,18 @@ class serializer
                             (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)))
                     {
                         const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        if (bytes == std::string::npos)
-                        {
-                            // copy invalid character as is
-                            result[pos++] = s[i];
-                            break;
-                        }
+                        // invalid characters will be detected by throw_if_invalid_utf8
+                        assert (bytes != std::string::npos);
 
                         // check that the additional bytes are present
                         assert(i + bytes < s.size());
 
-                        // to use \uxxxx escaping, we first need to caluclate
+                        // to use \uxxxx escaping, we first need to calculate
                         // the codepoint from the UTF-8 bytes
                         int codepoint = 0;
 
-                        assert(0 <= bytes and bytes <= 3);
+                        // bytes is unsigned type:
+                        assert(bytes <= 3);
                         switch (bytes)
                         {
                             case 0:
@@ -6641,11 +6631,10 @@ class serializer
     @param[in] x  integer number (signed or unsigned) to dump
     @tparam NumberType either @a number_integer_t or @a number_unsigned_t
     */
-    template <
-        typename NumberType,
-        detail::enable_if_t<std::is_same<NumberType, number_unsigned_t>::value or
-                            std::is_same<NumberType, number_integer_t>::value,
-                            int> = 0 >
+    template<typename NumberType, detail::enable_if_t<
+                 std::is_same<NumberType, number_unsigned_t>::value or
+                 std::is_same<NumberType, number_integer_t>::value,
+                 int> = 0>
     void dump_integer(NumberType x)
     {
         // special case for "0"
@@ -6743,6 +6732,87 @@ class serializer
         }
     }
 
+    /*!
+    @brief check whether a string is UTF-8 encoded
+
+    The function checks each byte of a string whether it is UTF-8 encoded. The
+    result of the check is stored in the @a state parameter. The function must
+    be called initially with state 0 (accept). State 1 means the string must
+    be rejected, because the current byte is not allowed. If the string is
+    completely processed, but the state is non-zero, the string ended
+    prematurely; that is, the last byte indicated more bytes should have
+    followed.
+
+    @param[in,out] state  the state of the decoding
+    @param[in] byte       next byte to decode
+
+    @note The function has been edited: a std::array is used and the code
+          point is not calculated.
+
+    @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+    @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+    */
+    static void decode(uint8_t& state, const uint8_t byte)
+    {
+        static const std::array<uint8_t, 400> utf8d =
+        {
+            {
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
+                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
+                7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
+                8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
+                0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
+                0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
+                0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
+                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
+                1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
+                1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
+                1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
+            }
+        };
+
+        const uint8_t type = utf8d[byte];
+        state = utf8d[256u + state * 16u + type];
+    }
+
+    /*!
+    @brief throw an exception if a string is not UTF-8 encoded
+
+    @param[in] str  UTF-8 string to check
+    @throw type_error.316 if passed string is not UTF-8 encoded
+
+    @since version 3.0.0
+    */
+    static void throw_if_invalid_utf8(const std::string& str)
+    {
+        // start with state 0 (= accept)
+        uint8_t state = 0;
+
+        for (size_t i = 0; i < str.size(); ++i)
+        {
+            const auto byte = static_cast<uint8_t>(str[i]);
+            decode(state, byte);
+            if (state == 1)
+            {
+                // state 1 means reject
+                std::stringstream ss;
+                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte);
+                JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str()));
+            }
+        }
+
+        if (state != 0)
+        {
+            // we finish reading, but do not accept: string was incomplete
+            std::stringstream ss;
+            ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back()));
+            JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str()));
+        }
+    }
+
   private:
     /// the output of the serializer
     output_adapter_t<char> o = nullptr;
@@ -6771,27 +6841,20 @@ class json_ref
     using value_type = BasicJsonType;
 
     json_ref(value_type&& value)
-        : owned_value(std::move(value)),
-          value_ref(&owned_value),
-          is_rvalue(true)
+        : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true)
     {}
 
     json_ref(const value_type& value)
-        : value_ref(const_cast<value_type*>(&value)),
-          is_rvalue(false)
+        : value_ref(const_cast<value_type*>(&value)), is_rvalue(false)
     {}
 
     json_ref(std::initializer_list<json_ref> init)
-        : owned_value(init),
-          value_ref(&owned_value),
-          is_rvalue(true)
+        : owned_value(init), value_ref(&owned_value), is_rvalue(true)
     {}
 
-    template <class... Args>
+    template<class... Args>
     json_ref(Args&& ... args)
-        : owned_value(std::forward<Args>(args)...),
-          value_ref(&owned_value),
-          is_rvalue(true)
+        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true)
     {}
 
     // class should be movable only
@@ -6949,6 +7012,27 @@ class json_pointer
         return to_string();
     }
 
+    /*!
+    @param[in] s  reference token to be converted into an array index
+
+    @return integer representation of @a s
+
+    @throw out_of_range.404 if string @a s could not be converted to an integer
+    */
+    static int array_index(const std::string& s)
+    {
+        size_t processed_chars = 0;
+        const int res = std::stoi(s, &processed_chars);
+
+        // check if the string was completely read
+        if (JSON_UNLIKELY(processed_chars != s.size()))
+        {
+            JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
+        }
+
+        return res;
+    }
+
   private:
     /*!
     @brief remove and return last reference pointer
@@ -6984,7 +7068,6 @@ class json_pointer
         return result;
     }
 
-
     /*!
     @brief create and return a reference to the pointed to value
 
@@ -7320,11 +7403,11 @@ class basic_json
 
   public:
     using value_t = detail::value_t;
-    // forward declarations
+    /// @copydoc nlohmann::json_pointer
     using json_pointer = ::nlohmann::json_pointer;
     template<typename T, typename SFINAE>
     using json_serializer = JSONSerializer<T, SFINAE>;
-
+    /// helper type for initializer lists of basic_json values
     using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
 
     ////////////////
@@ -7436,7 +7519,7 @@ class basic_json
         result["url"] = "https://github.com/nlohmann/json";
         result["version"] =
         {
-            {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1}
+            {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1}
         };
 
 #ifdef _WIN32
@@ -7489,6 +7572,14 @@ class basic_json
     /// the template arguments passed to class @ref basic_json.
     /// @{
 
+#if defined(JSON_HAS_CPP_14)
+    // Use transparent comparator if possible, combined with perfect forwarding
+    // on find() and count() calls prevents unnecessary string construction.
+    using object_comparator_t = std::less<>;
+#else
+    using object_comparator_t = std::less<StringType>;
+#endif
+
     /*!
     @brief a type for an object
 
@@ -7572,14 +7663,6 @@ class basic_json
     7159](http://rfc7159.net/rfc7159), because any order implements the
     specified "unordered" nature of JSON objects.
     */
-
-#if defined(JSON_HAS_CPP_14)
-    // Use transparent comparator if possible, combined with perfect forwarding
-    // on find() and count() calls prevents unnecessary string construction.
-    using object_comparator_t = std::less<>;
-#else
-    using object_comparator_t = std::less<StringType>;
-#endif
     using object_t = ObjectType<StringType,
           basic_json,
           object_comparator_t,
@@ -7931,12 +8014,14 @@ class basic_json
     static T* create(Args&& ... args)
     {
         AllocatorType<T> alloc;
+        using AllocatorTraits = std::allocator_traits<AllocatorType<T>>;
+
         auto deleter = [&](T * object)
         {
-            alloc.deallocate(object, 1);
+            AllocatorTraits::deallocate(alloc, object, 1);
         };
-        std::unique_ptr<T, decltype(deleter)> object(alloc.allocate(1), deleter);
-        alloc.construct(object.get(), std::forward<Args>(args)...);
+        std::unique_ptr<T, decltype(deleter)> object(AllocatorTraits::allocate(alloc, 1), deleter);
+        AllocatorTraits::construct(alloc, object.get(), std::forward<Args>(args)...);
         assert(object != nullptr);
         return object.release();
     }
@@ -8054,7 +8139,7 @@ class basic_json
                     object = nullptr;  // silence warning, see #821
                     if (JSON_UNLIKELY(t == value_t::null))
                     {
-                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -8104,24 +8189,24 @@ class basic_json
                 case value_t::object:
                 {
                     AllocatorType<object_t> alloc;
-                    alloc.destroy(object);
-                    alloc.deallocate(object, 1);
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, object);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1);
                     break;
                 }
 
                 case value_t::array:
                 {
                     AllocatorType<array_t> alloc;
-                    alloc.destroy(array);
-                    alloc.deallocate(array, 1);
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, array);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1);
                     break;
                 }
 
                 case value_t::string:
                 {
                     AllocatorType<string_t> alloc;
-                    alloc.destroy(string);
-                    alloc.deallocate(string, 1);
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1);
                     break;
                 }
 
@@ -8154,6 +8239,21 @@ class basic_json
     // JSON parser callback //
     //////////////////////////
 
+    /*!
+    @brief parser event types
+
+    The parser callback distinguishes the following events:
+    - `object_start`: the parser read `{` and started to process a JSON object
+    - `key`: the parser read a key of a value in an object
+    - `object_end`: the parser read `}` and finished processing a JSON object
+    - `array_start`: the parser read `[` and started to process a JSON array
+    - `array_end`: the parser read `]` and finished processing a JSON array
+    - `value`: the parser finished reading a JSON value
+
+    @image html callback_events.png "Example when certain parse events are triggered"
+
+    @sa @ref parser_callback_t for more information and examples
+    */
     using parse_event_t = typename parser::parse_event_t;
 
     /*!
@@ -8280,7 +8380,7 @@ class basic_json
     @brief create a JSON value
 
     This is a "catch all" constructor for all compatible JSON types; that is,
-    types for which a `to_json()` method exsits. The constructor forwards the
+    types for which a `to_json()` method exists. The constructor forwards the
     parameter @a val to that method (to `json_serializer<U>::to_json` method
     with `U = uncvref_t<CompatibleType>`, to be exact).
 
@@ -8952,11 +9052,14 @@ class basic_json
     @param[in] indent_char The character to use for indentation if @a indent is
     greater than `0`. The default is ` ` (space).
     @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
-    in the output are escaped with \uXXXX sequences, and the result consists
+    in the output are escaped with `\uXXXX` sequences, and the result consists
     of ASCII characters only.
 
     @return string containing the serialization of the JSON value
 
+    @throw type_error.316 if a string stored inside the JSON value is not
+                          UTF-8 encoded
+
     @complexity Linear.
 
     @exceptionsafety Strong guarantee: if an exception is thrown, there are no
@@ -8968,8 +9071,8 @@ class basic_json
 
     @see https://docs.python.org/2/library/json.html#json.dump
 
-    @since version 1.0.0; indentation character @a indent_char and option
-           @a ensure_ascii added in version 3.0.0
+    @since version 1.0.0; indentation character @a indent_char, option
+           @a ensure_ascii and exceptions added in version 3.0.0
     */
     string_t dump(const int indent = -1, const char indent_char = ' ',
                   const bool ensure_ascii = false) const
@@ -9003,7 +9106,7 @@ class basic_json
             string                    | value_t::string
             number (integer)          | value_t::number_integer
             number (unsigned integer) | value_t::number_unsigned
-            number (foating-point)    | value_t::number_float
+            number (floating-point)   | value_t::number_float
             object                    | value_t::object
             array                     | value_t::array
             discarded                 | value_t::discarded
@@ -9507,11 +9610,9 @@ class basic_json
 
     @since version 2.1.0
     */
-    template <
-        typename BasicJsonType,
-        detail::enable_if_t<std::is_same<typename std::remove_const<BasicJsonType>::type,
-                                         basic_json_t>::value,
-                            int> = 0 >
+    template<typename BasicJsonType, detail::enable_if_t<
+                 std::is_same<typename std::remove_const<BasicJsonType>::type, basic_json_t>::value,
+                 int> = 0>
     basic_json get() const
     {
         return *this;
@@ -9556,14 +9657,12 @@ class basic_json
 
     @since version 2.1.0
     */
-    template <
-        typename ValueTypeCV,
-        typename ValueType = detail::uncvref_t<ValueTypeCV>,
-        detail::enable_if_t <
-            not std::is_same<basic_json_t, ValueType>::value and
-            detail::has_from_json<basic_json_t, ValueType>::value and
-            not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
-            int > = 0 >
+    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
+             detail::enable_if_t <
+                 not std::is_same<basic_json_t, ValueType>::value and
+                 detail::has_from_json<basic_json_t, ValueType>::value and
+                 not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+                 int> = 0>
     ValueType get() const noexcept(noexcept(
                                        JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
     {
@@ -9611,12 +9710,10 @@ class basic_json
 
     @since version 2.1.0
     */
-    template <
-        typename ValueTypeCV,
-        typename ValueType = detail::uncvref_t<ValueTypeCV>,
-        detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
-                            detail::has_non_default_from_json<basic_json_t,
-                                    ValueType>::value, int> = 0 >
+    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
+             detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
+                                 detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+                                 int> = 0>
     ValueType get() const noexcept(noexcept(
                                        JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>())))
     {
@@ -10110,7 +10207,7 @@ class basic_json
 
     @return const reference to the element at index @a idx
 
-    @throw type_error.305 if the JSON value is not an array; in that cases,
+    @throw type_error.305 if the JSON value is not an array; in that case,
     using the [] operator with an index makes no sense.
 
     @complexity Constant.
@@ -10193,7 +10290,7 @@ class basic_json
     @pre The element with key @a key must exist. **This precondition is
          enforced with an assertion.**
 
-    @throw type_error.305 if the JSON value is not an object; in that cases,
+    @throw type_error.305 if the JSON value is not an object; in that case,
     using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10282,7 +10379,7 @@ class basic_json
     @pre The element with key @a key must exist. **This precondition is
          enforced with an assertion.**
 
-    @throw type_error.305 if the JSON value is not an object; in that cases,
+    @throw type_error.305 if the JSON value is not an object; in that case,
     using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10342,7 +10439,7 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    @throw type_error.306 if the JSON value is not an object; in that case,
     using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10415,7 +10512,7 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    @throw type_error.306 if the JSON value is not an objec; in that case,
     using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10619,8 +10716,8 @@ class basic_json
                 if (is_string())
                 {
                     AllocatorType<string_t> alloc;
-                    alloc.destroy(m_value.string);
-                    alloc.deallocate(m_value.string, 1);
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
                     m_value.string = nullptr;
                 }
 
@@ -10725,8 +10822,8 @@ class basic_json
                 if (is_string())
                 {
                     AllocatorType<string_t> alloc;
-                    alloc.destroy(m_value.string);
-                    alloc.deallocate(m_value.string, 1);
+                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
+                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
                     m_value.string = nullptr;
                 }
 
@@ -11220,22 +11317,62 @@ class basic_json
     reference to the JSON values is returned, so there is no access to the
     underlying iterator.
 
+    For loop without iterator_wrapper:
+
+    @code{cpp}
+    for (auto it = j_object.begin(); it != j_object.end(); ++it)
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    Range-based for loop without iterator proxy:
+
+    @code{cpp}
+    for (auto it : j_object)
+    {
+        // "it" is of type json::reference and has no key() member
+        std::cout << "value: " << it << '\n';
+    }
+    @endcode
+
+    Range-based for loop with iterator proxy:
+
+    @code{cpp}
+    for (auto it : json::iterator_wrapper(j_object))
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    @note When iterating over an array, `key()` will return the index of the
+          element as string (see example).
+
+    @param[in] ref  reference to a JSON value
+    @return iteration proxy object wrapping @a ref with an interface to use in
+            range-based for loops
+
     @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @complexity Constant.
+
     @note The name of this function is not yet final and may change in the
     future.
     */
-    static iteration_proxy<iterator> iterator_wrapper(reference cont)
+    static iteration_proxy<iterator> iterator_wrapper(reference ref)
     {
-        return iteration_proxy<iterator>(cont);
+        return iteration_proxy<iterator>(ref);
     }
 
     /*!
     @copydoc iterator_wrapper(reference)
     */
-    static iteration_proxy<const_iterator> iterator_wrapper(const_reference cont)
+    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref)
     {
-        return iteration_proxy<const_iterator>(cont);
+        return iteration_proxy<const_iterator>(ref);
     }
 
     /// @}
@@ -12120,7 +12257,7 @@ class basic_json
             JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name())));
         }
 
-        for (auto it = j.begin(); it != j.end(); ++it)
+        for (auto it = j.cbegin(); it != j.cend(); ++it)
         {
             m_value.object->operator[](it.key()) = it.value();
         }
@@ -12341,7 +12478,7 @@ class basic_json
     [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39)
     could be used, for instance
     @code {.cpp}
-    template <typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
+    template<typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
     inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept
     {
         return std::abs(a - b) <= epsilon;
@@ -12769,7 +12906,7 @@ class basic_json
       `std::setw(4)` on @a o sets the indentation level to `4` and the
       serialization result is the same as calling `dump(4)`.
 
-    - The indentation characrer can be controlled with the member variable
+    - The indentation character can be controlled with the member variable
       `fill` of the output stream @a o. For instance, the manipulator
       `std::setfill('\\t')` sets indentation to use a tab character rather than
       the default space character.
@@ -12779,12 +12916,15 @@ class basic_json
 
     @return the stream @a o
 
+    @throw type_error.316 if a string stored inside the JSON value is not
+                          UTF-8 encoded
+
     @complexity Linear.
 
     @liveexample{The example below shows the serialization with different
     parameters to `width` to adjust the indentation level.,operator_serialize}
 
-    @since version 1.0.0; indentaction character added in version 3.0.0
+    @since version 1.0.0; indentation character added in version 3.0.0
     */
     friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
     {
@@ -13124,40 +13264,40 @@ class basic_json
 
     JSON value type | value/range                                | CBOR type                          | first byte
     --------------- | ------------------------------------------ | ---------------------------------- | ---------------
-    null            | `null`                                     | Null                               | 0xf6
-    boolean         | `true`                                     | True                               | 0xf5
-    boolean         | `false`                                    | False                              | 0xf4
-    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3b
-    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3a
+    null            | `null`                                     | Null                               | 0xF6
+    boolean         | `true`                                     | True                               | 0xF5
+    boolean         | `false`                                    | False                              | 0xF4
+    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3B
+    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3A
     number_integer  | -32768..-129                               | Negative integer (2 bytes follow)  | 0x39
     number_integer  | -128..-25                                  | Negative integer (1 byte follow)   | 0x38
     number_integer  | -24..-1                                    | Negative integer                   | 0x20..0x37
     number_integer  | 0..23                                      | Integer                            | 0x00..0x17
     number_integer  | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
     number_integer  | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
-    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
-    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
+    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
+    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
     number_unsigned | 0..23                                      | Integer                            | 0x00..0x17
     number_unsigned | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
     number_unsigned | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
-    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
-    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
-    number_float    | *any value*                                | Double-Precision Float             | 0xfb
+    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
+    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
+    number_float    | *any value*                                | Double-Precision Float             | 0xFB
     string          | *length*: 0..23                            | UTF-8 string                       | 0x60..0x77
     string          | *length*: 23..255                          | UTF-8 string (1 byte follow)       | 0x78
     string          | *length*: 256..65535                       | UTF-8 string (2 bytes follow)      | 0x79
-    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7a
-    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7b
+    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7A
+    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7B
     array           | *size*: 0..23                              | array                              | 0x80..0x97
     array           | *size*: 23..255                            | array (1 byte follow)              | 0x98
     array           | *size*: 256..65535                         | array (2 bytes follow)             | 0x99
-    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9a
-    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9b
-    object          | *size*: 0..23                              | map                                | 0xa0..0xb7
-    object          | *size*: 23..255                            | map (1 byte follow)                | 0xb8
-    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xb9
-    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xba
-    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xbb
+    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9A
+    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9B
+    object          | *size*: 0..23                              | map                                | 0xA0..0xB7
+    object          | *size*: 23..255                            | map (1 byte follow)                | 0xB8
+    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xB9
+    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xBA
+    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xBB
 
     @note The mapping is **complete** in the sense that any JSON value type
           can be converted to a CBOR value.
@@ -13167,20 +13307,20 @@ class basic_json
           function which serializes NaN or Infinity to `null`.
 
     @note The following CBOR types are not used in the conversion:
-          - byte strings (0x40..0x5f)
-          - UTF-8 strings terminated by "break" (0x7f)
-          - arrays terminated by "break" (0x9f)
-          - maps terminated by "break" (0xbf)
-          - date/time (0xc0..0xc1)
-          - bignum (0xc2..0xc3)
-          - decimal fraction (0xc4)
-          - bigfloat (0xc5)
-          - tagged items (0xc6..0xd4, 0xd8..0xdb)
-          - expected conversions (0xd5..0xd7)
-          - simple values (0xe0..0xf3, 0xf8)
-          - undefined (0xf7)
-          - half and single-precision floats (0xf9-0xfa)
-          - break (0xff)
+          - byte strings (0x40..0x5F)
+          - UTF-8 strings terminated by "break" (0x7F)
+          - arrays terminated by "break" (0x9F)
+          - maps terminated by "break" (0xBF)
+          - date/time (0xC0..0xC1)
+          - bignum (0xC2..0xC3)
+          - decimal fraction (0xC4)
+          - bigfloat (0xC5)
+          - tagged items (0xC6..0xD4, 0xD8..0xDB)
+          - expected conversions (0xD5..0xD7)
+          - simple values (0xE0..0xF3, 0xF8)
+          - undefined (0xF7)
+          - half and single-precision floats (0xF9-0xFA)
+          - break (0xFF)
 
     @param[in] j  JSON value to serialize
     @return MessagePack serialization as byte vector
@@ -13226,35 +13366,35 @@ class basic_json
 
     JSON value type | value/range                       | MessagePack type | first byte
     --------------- | --------------------------------- | ---------------- | ----------
-    null            | `null`                            | nil              | 0xc0
-    boolean         | `true`                            | true             | 0xc3
-    boolean         | `false`                           | false            | 0xc2
-    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xd3
-    number_integer  | -2147483648..-32769               | int32            | 0xd2
-    number_integer  | -32768..-129                      | int16            | 0xd1
-    number_integer  | -128..-33                         | int8             | 0xd0
-    number_integer  | -32..-1                           | negative fixint  | 0xe0..0xff
-    number_integer  | 0..127                            | positive fixint  | 0x00..0x7f
-    number_integer  | 128..255                          | uint 8           | 0xcc
-    number_integer  | 256..65535                        | uint 16          | 0xcd
-    number_integer  | 65536..4294967295                 | uint 32          | 0xce
-    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xcf
-    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7f
-    number_unsigned | 128..255                          | uint 8           | 0xcc
-    number_unsigned | 256..65535                        | uint 16          | 0xcd
-    number_unsigned | 65536..4294967295                 | uint 32          | 0xce
-    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xcf
-    number_float    | *any value*                       | float 64         | 0xcb
-    string          | *length*: 0..31                   | fixstr           | 0xa0..0xbf
-    string          | *length*: 32..255                 | str 8            | 0xd9
-    string          | *length*: 256..65535              | str 16           | 0xda
-    string          | *length*: 65536..4294967295       | str 32           | 0xdb
-    array           | *size*: 0..15                     | fixarray         | 0x90..0x9f
-    array           | *size*: 16..65535                 | array 16         | 0xdc
-    array           | *size*: 65536..4294967295         | array 32         | 0xdd
-    object          | *size*: 0..15                     | fix map          | 0x80..0x8f
-    object          | *size*: 16..65535                 | map 16           | 0xde
-    object          | *size*: 65536..4294967295         | map 32           | 0xdf
+    null            | `null`                            | nil              | 0xC0
+    boolean         | `true`                            | true             | 0xC3
+    boolean         | `false`                           | false            | 0xC2
+    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xD3
+    number_integer  | -2147483648..-32769               | int32            | 0xD2
+    number_integer  | -32768..-129                      | int16            | 0xD1
+    number_integer  | -128..-33                         | int8             | 0xD0
+    number_integer  | -32..-1                           | negative fixint  | 0xE0..0xFF
+    number_integer  | 0..127                            | positive fixint  | 0x00..0x7F
+    number_integer  | 128..255                          | uint 8           | 0xCC
+    number_integer  | 256..65535                        | uint 16          | 0xCD
+    number_integer  | 65536..4294967295                 | uint 32          | 0xCE
+    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xCF
+    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7F
+    number_unsigned | 128..255                          | uint 8           | 0xCC
+    number_unsigned | 256..65535                        | uint 16          | 0xCD
+    number_unsigned | 65536..4294967295                 | uint 32          | 0xCE
+    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xCF
+    number_float    | *any value*                       | float 64         | 0xCB
+    string          | *length*: 0..31                   | fixstr           | 0xA0..0xBF
+    string          | *length*: 32..255                 | str 8            | 0xD9
+    string          | *length*: 256..65535              | str 16           | 0xDA
+    string          | *length*: 65536..4294967295       | str 32           | 0xDB
+    array           | *size*: 0..15                     | fixarray         | 0x90..0x9F
+    array           | *size*: 16..65535                 | array 16         | 0xDC
+    array           | *size*: 65536..4294967295         | array 32         | 0xDD
+    object          | *size*: 0..15                     | fix map          | 0x80..0x8F
+    object          | *size*: 16..65535                 | map 16           | 0xDE
+    object          | *size*: 65536..4294967295         | map 32           | 0xDF
 
     @note The mapping is **complete** in the sense that any JSON value type
           can be converted to a MessagePack value.
@@ -13265,10 +13405,10 @@ class basic_json
           - objects with more than 4294967295 elements
 
     @note The following MessagePack types are not used in the conversion:
-          - bin 8 - bin 32 (0xc4..0xc6)
-          - ext 8 - ext 32 (0xc7..0xc9)
-          - float 32 (0xca)
-          - fixext 1 - fixext 16 (0xd4..0xd8)
+          - bin 8 - bin 32 (0xC4..0xC6)
+          - ext 8 - ext 32 (0xC7..0xC9)
+          - float 32 (0xCA)
+          - fixext 1 - fixext 16 (0xD4..0xD8)
 
     @note Any MessagePack output created @ref to_msgpack can be successfully
           parsed by @ref from_msgpack.
@@ -13322,51 +13462,51 @@ class basic_json
     Integer                | number_unsigned | 0x00..0x17
     Unsigned integer       | number_unsigned | 0x18
     Unsigned integer       | number_unsigned | 0x19
-    Unsigned integer       | number_unsigned | 0x1a
-    Unsigned integer       | number_unsigned | 0x1b
+    Unsigned integer       | number_unsigned | 0x1A
+    Unsigned integer       | number_unsigned | 0x1B
     Negative integer       | number_integer  | 0x20..0x37
     Negative integer       | number_integer  | 0x38
     Negative integer       | number_integer  | 0x39
-    Negative integer       | number_integer  | 0x3a
-    Negative integer       | number_integer  | 0x3b
+    Negative integer       | number_integer  | 0x3A
+    Negative integer       | number_integer  | 0x3B
     Negative integer       | number_integer  | 0x40..0x57
     UTF-8 string           | string          | 0x60..0x77
     UTF-8 string           | string          | 0x78
     UTF-8 string           | string          | 0x79
-    UTF-8 string           | string          | 0x7a
-    UTF-8 string           | string          | 0x7b
-    UTF-8 string           | string          | 0x7f
+    UTF-8 string           | string          | 0x7A
+    UTF-8 string           | string          | 0x7B
+    UTF-8 string           | string          | 0x7F
     array                  | array           | 0x80..0x97
     array                  | array           | 0x98
     array                  | array           | 0x99
-    array                  | array           | 0x9a
-    array                  | array           | 0x9b
-    array                  | array           | 0x9f
-    map                    | object          | 0xa0..0xb7
-    map                    | object          | 0xb8
-    map                    | object          | 0xb9
-    map                    | object          | 0xba
-    map                    | object          | 0xbb
-    map                    | object          | 0xbf
-    False                  | `false`         | 0xf4
-    True                   | `true`          | 0xf5
-    Nill                   | `null`          | 0xf6
-    Half-Precision Float   | number_float    | 0xf9
-    Single-Precision Float | number_float    | 0xfa
-    Double-Precision Float | number_float    | 0xfb
+    array                  | array           | 0x9A
+    array                  | array           | 0x9B
+    array                  | array           | 0x9F
+    map                    | object          | 0xA0..0xB7
+    map                    | object          | 0xB8
+    map                    | object          | 0xB9
+    map                    | object          | 0xBA
+    map                    | object          | 0xBB
+    map                    | object          | 0xBF
+    False                  | `false`         | 0xF4
+    True                   | `true`          | 0xF5
+    Nill                   | `null`          | 0xF6
+    Half-Precision Float   | number_float    | 0xF9
+    Single-Precision Float | number_float    | 0xFA
+    Double-Precision Float | number_float    | 0xFB
 
     @warning The mapping is **incomplete** in the sense that not all CBOR
              types can be converted to a JSON value. The following CBOR types
              are not supported and will yield parse errors (parse_error.112):
-             - byte strings (0x40..0x5f)
-             - date/time (0xc0..0xc1)
-             - bignum (0xc2..0xc3)
-             - decimal fraction (0xc4)
-             - bigfloat (0xc5)
-             - tagged items (0xc6..0xd4, 0xd8..0xdb)
-             - expected conversions (0xd5..0xd7)
-             - simple values (0xe0..0xf3, 0xf8)
-             - undefined (0xf7)
+             - byte strings (0x40..0x5F)
+             - date/time (0xC0..0xC1)
+             - bignum (0xC2..0xC3)
+             - decimal fraction (0xC4)
+             - bigfloat (0xC5)
+             - tagged items (0xC6..0xD4, 0xD8..0xDB)
+             - expected conversions (0xD5..0xD7)
+             - simple values (0xE0..0xF3, 0xF8)
+             - undefined (0xF7)
 
     @warning CBOR allows map keys of any type, whereas JSON only allows
              strings as keys in object values. Therefore, CBOR maps with keys
@@ -13426,38 +13566,38 @@ class basic_json
 
     MessagePack type | JSON value type | first byte
     ---------------- | --------------- | ----------
-    positive fixint  | number_unsigned | 0x00..0x7f
-    fixmap           | object          | 0x80..0x8f
-    fixarray         | array           | 0x90..0x9f
-    fixstr           | string          | 0xa0..0xbf
-    nil              | `null`          | 0xc0
-    false            | `false`         | 0xc2
-    true             | `true`          | 0xc3
-    float 32         | number_float    | 0xca
-    float 64         | number_float    | 0xcb
-    uint 8           | number_unsigned | 0xcc
-    uint 16          | number_unsigned | 0xcd
-    uint 32          | number_unsigned | 0xce
-    uint 64          | number_unsigned | 0xcf
-    int 8            | number_integer  | 0xd0
-    int 16           | number_integer  | 0xd1
-    int 32           | number_integer  | 0xd2
-    int 64           | number_integer  | 0xd3
-    str 8            | string          | 0xd9
-    str 16           | string          | 0xda
-    str 32           | string          | 0xdb
-    array 16         | array           | 0xdc
-    array 32         | array           | 0xdd
-    map 16           | object          | 0xde
-    map 32           | object          | 0xdf
-    negative fixint  | number_integer  | 0xe0-0xff
+    positive fixint  | number_unsigned | 0x00..0x7F
+    fixmap           | object          | 0x80..0x8F
+    fixarray         | array           | 0x90..0x9F
+    fixstr           | string          | 0xA0..0xBF
+    nil              | `null`          | 0xC0
+    false            | `false`         | 0xC2
+    true             | `true`          | 0xC3
+    float 32         | number_float    | 0xCA
+    float 64         | number_float    | 0xCB
+    uint 8           | number_unsigned | 0xCC
+    uint 16          | number_unsigned | 0xCD
+    uint 32          | number_unsigned | 0xCE
+    uint 64          | number_unsigned | 0xCF
+    int 8            | number_integer  | 0xD0
+    int 16           | number_integer  | 0xD1
+    int 32           | number_integer  | 0xD2
+    int 64           | number_integer  | 0xD3
+    str 8            | string          | 0xD9
+    str 16           | string          | 0xDA
+    str 32           | string          | 0xDB
+    array 16         | array           | 0xDC
+    array 32         | array           | 0xDD
+    map 16           | object          | 0xDE
+    map 32           | object          | 0xDF
+    negative fixint  | number_integer  | 0xE0-0xFF
 
     @warning The mapping is **incomplete** in the sense that not all
              MessagePack types can be converted to a JSON value. The following
              MessagePack types are not supported and will yield parse errors:
-              - bin 8 - bin 32 (0xc4..0xc6)
-              - ext 8 - ext 32 (0xc7..0xc9)
-              - fixext 1 - fixext 16 (0xd4..0xd8)
+              - bin 8 - bin 32 (0xC4..0xC6)
+              - ext 8 - ext 32 (0xC7..0xC9)
+              - fixext 1 - fixext 16 (0xD4..0xD8)
 
     @note Any MessagePack output created @ref to_msgpack can be successfully
           parsed by @ref from_msgpack.
@@ -13601,6 +13741,9 @@ class basic_json
     pointer @a ptr. As `at` provides checked access (and no elements are
     implicitly inserted), the index '-' is always invalid. See example below.
 
+    @throw out_of_range.403 if the JSON pointer describes a key of an object
+    which cannot be found. See example below.
+
     @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
     See example below.
 
@@ -13641,6 +13784,9 @@ class basic_json
     pointer @a ptr. As `at` provides checked access (and no elements are
     implicitly inserted), the index '-' is always invalid. See example below.
 
+    @throw out_of_range.403 if the JSON pointer describes a key of an object
+    which cannot be found. See example below.
+
     @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
     See example below.
 
@@ -13856,7 +14002,7 @@ class basic_json
                         }
                         else
                         {
-                            const auto idx = std::stoi(last_path);
+                            const auto idx = json_pointer::array_index(last_path);
                             if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
                             {
                                 // avoid undefined behavior
@@ -13904,7 +14050,7 @@ class basic_json
             else if (parent.is_array())
             {
                 // note erase performs range check
-                parent.erase(static_cast<size_type>(std::stoi(last_path)));
+                parent.erase(static_cast<size_type>(json_pointer::array_index(last_path)));
             }
         };
 
@@ -13999,7 +14145,12 @@ class basic_json
                     const json_pointer from_ptr(from_path);
 
                     // the "from" location must exist - use at()
-                    result[ptr] = result.at(from_ptr);
+                    basic_json v = result.at(from_ptr);
+
+                    // The copy is functionally identical to an "add"
+                    // operation at the target location using the value
+                    // specified in the "from" member.
+                    operation_add(ptr, v);
                     break;
                 }
 
@@ -14141,7 +14292,7 @@ class basic_json
                 case value_t::object:
                 {
                     // first pass: traverse this object's elements
-                    for (auto it = source.begin(); it != source.end(); ++it)
+                    for (auto it = source.cbegin(); it != source.cend(); ++it)
                     {
                         // escape the key name to be used in a JSON patch
                         const auto key = json_pointer::escape(it.key());
@@ -14163,7 +14314,7 @@ class basic_json
                     }
 
                     // second pass: traverse other object's elements
-                    for (auto it = target.begin(); it != target.end(); ++it)
+                    for (auto it = target.cbegin(); it != target.cend(); ++it)
                     {
                         if (source.find(it.key()) == source.end())
                         {
@@ -14256,7 +14407,7 @@ json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const
                 // create an entry in the array
                 JSON_TRY
                 {
-                    result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                    result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
                 }
                 JSON_CATCH(std::invalid_argument&)
                 {
@@ -14333,7 +14484,7 @@ json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const
                     JSON_TRY
                     {
                         ptr = &ptr->operator[](
-                            static_cast<size_type>(std::stoi(reference_token)));
+                            static_cast<size_type>(array_index(reference_token)));
                     }
                     JSON_CATCH(std::invalid_argument&)
                     {
@@ -14388,7 +14539,7 @@ json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const
                 // note: at performs range check
                 JSON_TRY
                 {
-                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
                 }
                 JSON_CATCH(std::invalid_argument&)
                 {
@@ -14443,7 +14594,7 @@ json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
                 JSON_TRY
                 {
                     ptr = &ptr->operator[](
-                        static_cast<size_type>(std::stoi(reference_token)));
+                        static_cast<size_type>(array_index(reference_token)));
                 }
                 JSON_CATCH(std::invalid_argument&)
                 {
@@ -14497,7 +14648,7 @@ json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
                 // note: at performs range check
                 JSON_TRY
                 {
-                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
                 }
                 JSON_CATCH(std::invalid_argument&)
                 {

From d43a8b25f044dffdbdd94777924e270cae3e0c8f Mon Sep 17 00:00:00 2001
From: Dan Peebles <pumpkin@me.com>
Date: Tue, 23 Jan 2018 14:13:30 -0500
Subject: [PATCH 0770/2196] Fix obscure corner case in name resolution for
 builtin:fetchurl in sandboxed environments

---
 src/libstore/build.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 55066205db3..cca51f17ee2 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
@@ -1777,6 +1778,19 @@ PathSet exportReferences(Store & store, PathSet storePaths)
     return paths;
 }
 
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid");
+        request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning
+        try { getDownloader()->download(request); } catch (...) {}
+    });
+}
 
 void DerivationGoal::startBuilder()
 {
@@ -1787,6 +1801,9 @@ void DerivationGoal::startBuilder()
             % drv->platform % settings.thisSystem % drvPath);
     }
 
+    if (drv->isBuiltin())
+        preloadNSS();
+
 #if __APPLE__
     additionalSandboxProfile = get(drv->env, "__sandboxProfile");
 #endif

From e09161d05cfbd7c6d4cf41a35765e3fe346ea181 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Jan 2018 17:10:52 +0100
Subject: [PATCH 0771/2196] Remove signed-binary-caches as the default for
 require-sigs

This was for backward compatibility. However, with security-related
configuration settings, it's best not to have any confusion.

Issue #495.
---
 src/libstore/globals.hh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 1e50e2d13e9..20ac8fe4e9a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -287,10 +287,7 @@ public:
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
         "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
-    Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches",
-        "Obsolete."};
-
-    Setting<bool> requireSigs{this, signedBinaryCaches == "*", "require-sigs",
+    Setting<bool> requireSigs{this, true, "require-sigs",
         "Whether to check that any non-content-addressed path added to the "
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};

From 1d5d277ac7bf8a4bc601358b38746005416e935e Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Fri, 26 Jan 2018 11:12:30 -0800
Subject: [PATCH 0772/2196] HttpBinaryCacheStore: Support upsertFile with PUT.

Some servers, such as Artifactory, allow uploading with PUT and BASIC
auth. This allows nix copy to work to upload binaries to those
servers.

Worked on together with @adelbertc
---
 src/libstore/download.cc                | 28 ++++++++++++++++++++++++-
 src/libstore/download.hh                |  6 ++++--
 src/libstore/http-binary-cache-store.cc |  8 ++++++-
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index ef417685f1a..4b37826c46c 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -22,6 +22,7 @@
 #include <thread>
 #include <cmath>
 #include <random>
+#include <algorithm>
 
 using namespace std::string_literals;
 
@@ -91,6 +92,8 @@ struct CurlDownloader : public Downloader
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+            if (!request.mimeType.empty())
+                requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
         ~DownloadItem()
@@ -185,6 +188,22 @@ struct CurlDownloader : public Downloader
             return 0;
         }
 
+        size_t readOffset = 0;
+        int readCallback(char *buffer, size_t size, size_t nitems)
+        {
+            if (readOffset == request.data->length())
+                return 0;
+            auto count = std::min(size * nitems, request.data->length() - readOffset);
+            memcpy(buffer, request.data->data() + readOffset, count);
+            readOffset += count;
+            return count;
+        }
+
+        static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
+        {
+            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+        }
+
         long lowSpeedTimeout = 300;
 
         void init()
@@ -225,6 +244,13 @@ struct CurlDownloader : public Downloader
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
+            if (request.data) {
+                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
+                curl_easy_setopt(req, CURLOPT_READDATA, this);
+                curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+            }
+
             if (request.verifyTLS) {
                 if (settings.caFile != "")
                     curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
@@ -265,7 +291,7 @@ struct CurlDownloader : public Downloader
             }
 
             if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index f2d65ad8d61..0a278a05e0e 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -18,9 +18,11 @@ struct DownloadRequest
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
+    std::shared_ptr<std::string> data;
+    std::string mimeType;
 
-    DownloadRequest(const std::string & uri)
-        : uri(uri), parentAct(curActivity) { }
+    DownloadRequest(const std::string & uri, std::shared_ptr<std::string> data = nullptr, std::string mimeType = "")
+        : uri(uri), parentAct(curActivity), data(std::move(data)), mimeType(std::move(mimeType)) { }
 };
 
 struct DownloadResult
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 05733768579..93bd3e5d598 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -67,7 +67,13 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         const std::string & data,
         const std::string & mimeType) override
     {
-        throw UploadToHTTP("uploading to an HTTP binary cache is not supported");
+        auto data_ = std::make_shared<string>(data);
+        auto req = DownloadRequest(cacheUri + "/" + path, data_, mimeType);
+        try {
+            getDownloader()->download(req);
+        } catch (DownloadError & e) {
+            throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg());
+        }
     }
 
     void getFile(const std::string & path,

From 746f8aed86a990a6a8277cd2596f83166a73d718 Mon Sep 17 00:00:00 2001
From: Spencer Baugh <sbaugh@catern.com>
Date: Sun, 14 Jan 2018 21:20:22 -0500
Subject: [PATCH 0773/2196] remote_store: register for NIX_REMOTE=unix://path

This allows overriding the socket path so the daemon may be listening at
an arbitrary Unix domain socket location.

Fixes #1800
---
 src/libstore/remote-store.cc | 26 ++++++++++++++++++++++++--
 src/libstore/remote-store.hh |  2 ++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 77b41b6bf8a..8f0b65557ac 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -78,9 +78,22 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 }
 
 
+UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
+    : Store(params)
+    , LocalFSStore(params)
+    , RemoteStore(params)
+    , path(socket_path)
+{
+}
+
+
 std::string UDSRemoteStore::getUri()
 {
-    return "daemon";
+    if (path) {
+        return std::string("unix://") + *path;
+    } else {
+        return "daemon";
+    }
 }
 
 
@@ -98,7 +111,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(conn->fd.get());
 
-    string socketPath = settings.nixDaemonSocketFile;
+    string socketPath = path ? *path : settings.nixDaemonSocketFile;
 
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
@@ -721,5 +734,14 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
     }
 }
 
+static std::string uriScheme = "unix://";
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
+});
 
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 30c6beae6ff..7f36e206416 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -134,6 +134,7 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore
 public:
 
     UDSRemoteStore(const Params & params);
+    UDSRemoteStore(std::string path, const Params & params);
 
     std::string getUri() override;
 
@@ -145,6 +146,7 @@ private:
     };
 
     ref<RemoteStore::Connection> openConnection() override;
+    std::experimental::optional<std::string> path;
 };
 
 

From 0167eac571d6d92f18640d05cfb7fa10a4cd0fd9 Mon Sep 17 00:00:00 2001
From: Eric Wolf <ericwolf42@gmail.com>
Date: Sat, 27 Jan 2018 15:10:51 +0100
Subject: [PATCH 0774/2196] Improve manual on inheriting attributes

Expands first paragraph a bit
Adds a more comprehensive example
---
 .../expressions/language-constructs.xml       | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 2f0027d479c..47d95f8a13e 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -61,7 +61,7 @@ evaluates to <literal>"foobar"</literal>.
 
 <simplesect><title>Inheriting attributes</title>
 
-<para>When defining a set it is often convenient to copy variables
+<para>When defining a set or in a let-expression it is often convenient to copy variables
 from the surrounding lexical scope (e.g., when you want to propagate
 attributes).  This can be shortened using the
 <literal>inherit</literal> keyword.  For instance,
@@ -72,7 +72,15 @@ let x = 123; in
   y = 456;
 }</programlisting>
 
-evaluates to <literal>{ x = 123; y = 456; }</literal>.  (Note that
+is equivalent to
+
+<programlisting>
+let x = 123; in
+{ x = x;
+  y = 456;
+}</programlisting>
+
+and both evaluate to <literal>{ x = 123; y = 456; }</literal>. (Note that
 this works because <varname>x</varname> is added to the lexical scope
 by the <literal>let</literal> construct.)  It is also possible to
 inherit attributes from another set.  For instance, in this fragment
@@ -101,6 +109,26 @@ variables from the surrounding scope (<varname>fetchurl</varname>
 <varname>libXaw</varname> (the X Athena Widgets) from the
 <varname>xlibs</varname> (X11 client-side libraries) set.</para>
 
+<para>
+Summarizing the fragment
+
+<programlisting>
+...
+inherit x y z;
+inherit (src-set) a b c;
+...</programlisting>
+
+is equivalent to
+
+<programlisting>
+...
+x = x; y = y; z = z;
+a = src-set.a; b = src-set.b; c = src-set.c;
+...</programlisting>
+
+when used while defining local variables in a let-expression or
+while defining a set.</para>
+
 </simplesect>
 
 

From f93e890b4d19f48fefde449c46c29e8330b23998 Mon Sep 17 00:00:00 2001
From: Ben Gamari <ben@smart-cactus.org>
Date: Sun, 28 Jan 2018 14:19:56 -0500
Subject: [PATCH 0775/2196] configure: Use $CPP instead of cpp directly

The latter breaks in the case of cross-compilation, when `cpp` bears a
target prefix.
---
 configure.ac | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 9db92ce9140..83b2346d065 100644
--- a/configure.ac
+++ b/configure.ac
@@ -61,6 +61,7 @@ CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
+AC_PROG_CPP
 AX_CXX_COMPILE_STDCXX_11
 
 
@@ -199,7 +200,7 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | cpp -E | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
   AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
   AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
 fi

From 9f9393df5504c43be95df55ed1c5d5a90ea214a2 Mon Sep 17 00:00:00 2001
From: Giorgio Gallo <giorgio.gallo@bitnic.it>
Date: Mon, 29 Jan 2018 21:33:17 +0100
Subject: [PATCH 0776/2196] solves #1582

---
 scripts/install-darwin-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 2e9d368c020..8d59c1c2b9a 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -695,7 +695,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt "$(pwd)/store/" "$NIX_ROOT/store/"
+              rsync -rlpt ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

From 478e3e46497322d00430c77d27f36eb64be7f9ba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 29 Jan 2018 16:22:59 +0100
Subject: [PATCH 0777/2196] Indent properly

---
 src/libstore/download.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index ef417685f1a..3121e6441f4 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -312,10 +312,10 @@ struct CurlDownloader : public Downloader
                         case CURLE_BAD_FUNCTION_ARGUMENT:
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
-                        err = Misc;
-                        break;
+                            err = Misc;
+                            break;
                         default: // Shut up warnings
-                        break;
+                            break;
                     }
                 }
 

From f8e8dd827eb9005fcc3ba42fd0855729ef77d9fb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 Jan 2018 15:08:46 +0100
Subject: [PATCH 0778/2196] Manual: Remove old cruft

---
 doc/manual/expressions/debug-build.xml        | 34 ---------------
 .../expressions/simple-building-testing.xml   |  2 -
 doc/manual/installation/installing-binary.xml | 10 -----
 .../installation/supported-platforms.xml      |  3 --
 doc/manual/introduction/quick-start.xml       |  2 +-
 doc/manual/manual.xml                         |  6 +--
 .../troubleshooting/collisions-nixenv.xml     | 38 ----------------
 .../troubleshooting/links-nix-store.xml       | 43 -------------------
 .../troubleshooting/troubleshooting.xml       | 16 -------
 9 files changed, 2 insertions(+), 152 deletions(-)
 delete mode 100644 doc/manual/expressions/debug-build.xml
 delete mode 100644 doc/manual/troubleshooting/collisions-nixenv.xml
 delete mode 100644 doc/manual/troubleshooting/links-nix-store.xml
 delete mode 100644 doc/manual/troubleshooting/troubleshooting.xml

diff --git a/doc/manual/expressions/debug-build.xml b/doc/manual/expressions/debug-build.xml
deleted file mode 100644
index 0c1f4e6719b..00000000000
--- a/doc/manual/expressions/debug-build.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-debug-build">
-
-<title>Debugging Build Failures</title>
-
-<para>At the beginning of each phase of the build (such as unpacking,
-building or installing), the set of all shell variables is written to
-the file <filename>env-vars</filename> at the top-level build
-directory.  This is useful for debugging: it allows you to recreate
-the environment in which a build was performed.  For instance, if a
-build fails, then assuming you used the <option>-K</option> flag, you
-can go to the output directory and <quote>switch</quote> to the
-environment of the builder:
-
-<screen>
-$ nix-build -K ./foo.nix
-... fails, keeping build directory `/tmp/nix-1234-0'
-
-$ cd /tmp/nix-1234-0
-
-$ source env-vars
-
-<lineannotation>(edit some files...)</lineannotation>
-
-$ make
-
-<lineannotation>(execution continues with the same GCC, make, etc.)</lineannotation></screen>
-
-</para>
-
-</section>
diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
index bd3901a1335..0348c082b20 100644
--- a/doc/manual/expressions/simple-building-testing.xml
+++ b/doc/manual/expressions/simple-building-testing.xml
@@ -81,6 +81,4 @@ Just pass the option <link linkend='opt-max-jobs'><option>-j
 in parallel, or set.  Typically this should be the number of
 CPUs.</para>
 
-<xi:include href="debug-build.xml" />
-
 </section>
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 24e76eafeb1..7e8dfb0db3d 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -79,16 +79,6 @@ alice$ ./install
 
 </para>
 
-<para>Nix can be uninstalled using <command>rpm -e nix</command> or
-<command>dpkg -r nix</command> on RPM- and Dpkg-based systems,
-respectively.  After this you should manually remove the Nix store and
-other auxiliary data, if desired:
-
-<screen>
-$ rm -rf /nix</screen>
-
-</para>
-
 <para>You can uninstall Nix simply by running:
 
 <screen>
diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml
index a468a564063..6858573ff40 100644
--- a/doc/manual/installation/supported-platforms.xml
+++ b/doc/manual/installation/supported-platforms.xml
@@ -33,7 +33,4 @@
 
 </para>
 
-<para>Nix is fairly portable, so it should work on most platforms that
-support POSIX threads and have a C++11 compiler.</para>
-
 </chapter>
diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml
index aa239b7538b..1ce6c8d50a1 100644
--- a/doc/manual/introduction/quick-start.xml
+++ b/doc/manual/introduction/quick-start.xml
@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ curl https://nixos.org/nix/install | sh
+$ bash &lt;(curl https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script
diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml
index 61205d91699..b5a6af7d0c3 100644
--- a/doc/manual/manual.xml
+++ b/doc/manual/manual.xml
@@ -12,14 +12,11 @@
         <firstname>Eelco</firstname>
         <surname>Dolstra</surname>
       </personname>
-      <affiliation>
-        <orgname>LogicBlox</orgname>
-      </affiliation>
       <contrib>Author</contrib>
     </author>
 
     <copyright>
-      <year>2004-2014</year>
+      <year>2004-2017</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
@@ -41,7 +38,6 @@
   <xi:include href="expressions/writing-nix-expressions.xml" />
   <xi:include href="advanced-topics/advanced-topics.xml" />
   <xi:include href="command-ref/command-ref.xml" />
-  <xi:include href="troubleshooting/troubleshooting.xml" />
   <xi:include href="glossary/glossary.xml" />
   <xi:include href="hacking.xml" />
   <xi:include href="release-notes/release-notes.xml" />
diff --git a/doc/manual/troubleshooting/collisions-nixenv.xml b/doc/manual/troubleshooting/collisions-nixenv.xml
deleted file mode 100644
index 23cc43faf08..00000000000
--- a/doc/manual/troubleshooting/collisions-nixenv.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-collisions-nixenv">
-
-<title>Collisions in <command>nix-env</command></title>
-
-<para>Symptom: when installing or upgrading, you get an error message such as
-
-<screen>
-$ nix-env -i docbook-xml
-...
-adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
-collision between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
-  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
-  at /nix/store/...-builder.pl line 62.</screen>
-
-</para>
-
-<para>The cause is that two installed packages in the user environment
-have overlapping filenames (e.g.,
-<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
-happens when you accidentally try to install two versions of the same
-package.  For instance, in the example above, the Nix Packages
-collection contains two versions of <literal>docbook-xml</literal>, so
-<command>nix-env -i</command> will try to install both.  The default
-user environment builder has no way to way to resolve such conflicts,
-so it just gives up.</para>
-
-<para>Solution: remove one of the offending packages from the user
-environment (if already installed) using <command>nix-env
--e</command>, or specify exactly which version should be installed
-(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
-
-<!-- FIXME: describe priorities -->
-
-</section>
diff --git a/doc/manual/troubleshooting/links-nix-store.xml b/doc/manual/troubleshooting/links-nix-store.xml
deleted file mode 100644
index c768889567d..00000000000
--- a/doc/manual/troubleshooting/links-nix-store.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="sec-links-nix-store">
-
-<title><quote>Too many links</quote> Error in the Nix store</title>
-
-
-<para>Symptom: when building something, you get an error message such as
-
-<screen>
-...
-<literal>mkdir: cannot create directory `/nix/store/<replaceable>name</replaceable>': Too many links</literal></screen>
-
-</para>
-
-<para>This is usually because you have more than 32,000 subdirectories
-in <filename>/nix/store</filename>, as can be seen using <command>ls
--l</command>:
-
-<screen>
-$ ls -ld /nix/store
-drwxrwxrwt 32000 nix nix 4620288 Sep 8 15:08 store</screen>
-
-The <literal>ext2</literal> file system is limited to an inode link
-count of 32,000 (each subdirectory increasing the count by one).
-Furthermore, the <literal>st_nlink</literal> field of the
-<function>stat</function> system call is a 16-bit value.</para>
-
-<para>This only happens on very large Nix installations (such as build
-machines).</para>
-
-<para>Quick solution: run the garbage collector.  You may want to use
-the <option>--max-links</option> option.</para>
-
-<para>Real solution: put the Nix store on a file system that supports
-more than 32,000 subdirectories per directory, such as ext4.  (This
-doesn’t solve the <literal>st_nlink</literal> limit, but ext4 lies to
-the kernel by reporting a link count of 1 if it exceeds the
-limit.)</para>
-
-</section>
diff --git a/doc/manual/troubleshooting/troubleshooting.xml b/doc/manual/troubleshooting/troubleshooting.xml
deleted file mode 100644
index 1e973a192b1..00000000000
--- a/doc/manual/troubleshooting/troubleshooting.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude"
-          version="5.0"
-          xml:id="ch-troubleshooting">
-
-<title>Troubleshooting</title>
-
-<para>This section provides solutions for some common problems.  See
-the <link xlink:href="https://github.com/NixOS/nix/issues">Nix bug
-tracker</link> for a list of currently known issues.</para>
-
-<xi:include href="collisions-nixenv.xml" />
-<xi:include href="links-nix-store.xml" />
-
-</appendix>

From 30370f168fef6fd1dd2e888a680e3cabbb7215c1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 Jan 2018 15:12:27 +0100
Subject: [PATCH 0779/2196] Cleanup

---
 src/libstore/download.hh                | 4 ++--
 src/libstore/http-binary-cache-store.cc | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 0a278a05e0e..d9d525d4e65 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -21,8 +21,8 @@ struct DownloadRequest
     std::shared_ptr<std::string> data;
     std::string mimeType;
 
-    DownloadRequest(const std::string & uri, std::shared_ptr<std::string> data = nullptr, std::string mimeType = "")
-        : uri(uri), parentAct(curActivity), data(std::move(data)), mimeType(std::move(mimeType)) { }
+    DownloadRequest(const std::string & uri)
+        : uri(uri), parentAct(curActivity) { }
 };
 
 struct DownloadResult
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 93bd3e5d598..b9e9cd5daba 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -38,7 +38,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
-                throw Error(format("'%s' does not appear to be a binary cache") % cacheUri);
+                throw Error("'%s' does not appear to be a binary cache", cacheUri);
             }
             diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
         }
@@ -67,8 +67,9 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         const std::string & data,
         const std::string & mimeType) override
     {
-        auto data_ = std::make_shared<string>(data);
-        auto req = DownloadRequest(cacheUri + "/" + path, data_, mimeType);
+        auto req = DownloadRequest(cacheUri + "/" + path);
+        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        req.mimeType = mimeType;
         try {
             getDownloader()->download(req);
         } catch (DownloadError & e) {

From 6fa690291a0ca81ded59ab3db1745ce2dfef84fd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 Jan 2018 16:14:47 +0100
Subject: [PATCH 0780/2196] Add 'nix upgrade-nix' command

This command upgrades Nix to the latest stable version by installing a
store path obtained from

  https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix

which is the same store path that the installer at
https://nixos.org/nix/install.sh uses.

The upgrade fails if Nix is not installed in a profile (e.g. on NixOS,
or when installed outside of the Nix store).
---
 doc/manual/release-notes/rl-1.12.xml |   5 +
 src/nix/command.hh                   |   2 +
 src/nix/main.cc                      |   5 +-
 src/nix/upgrade-nix.cc               | 131 +++++++++++++++++++++++++++
 4 files changed, 142 insertions(+), 1 deletion(-)
 create mode 100644 src/nix/upgrade-nix.cc

diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml
index 29943e3e6e9..89ec920949b 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-1.12.xml
@@ -79,6 +79,11 @@
 
       <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem>
 
+      <listitem><para><command>nix upgrade-nix</command> upgrades Nix
+      to the latest stable version. This requires that Nix is
+      installed in a profile. (Thus it won’t work on NixOS, or if it’s
+      installed outside of the Nix store.)</para></listitem>
+
       <listitem><para>Progress indicator.</para></listitem>
 
       <listitem><para>All options are available as flags now
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 6b34e3881e7..a7863c49f37 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -5,6 +5,8 @@
 
 namespace nix {
 
+extern std::string programPath;
+
 struct Value;
 class Bindings;
 class EvalState;
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 06bb8a1c304..8f6bbe8f51a 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -16,6 +16,8 @@ void chrootHelper(int argc, char * * argv);
 
 namespace nix {
 
+std::string programPath;
+
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
@@ -78,7 +80,8 @@ void mainWrapped(int argc, char * * argv)
     initNix();
     initGC();
 
-    string programName = baseNameOf(argv[0]);
+    programPath = argv[0];
+    string programName = baseNameOf(programPath);
 
     {
         auto legacy = (*RegisterLegacyCommand::commands)[programName];
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
new file mode 100644
index 00000000000..758bbbc688b
--- /dev/null
+++ b/src/nix/upgrade-nix.cc
@@ -0,0 +1,131 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "download.hh"
+#include "eval.hh"
+#include "attr-path.hh"
+
+using namespace nix;
+
+struct CmdUpgradeNix : StoreCommand
+{
+    Path profileDir;
+
+    CmdUpgradeNix()
+    {
+        mkFlag()
+            .longName("profile")
+            .shortName('p')
+            .labels({"profile-dir"})
+            .description("the Nix profile to upgrade")
+            .dest(&profileDir);
+    }
+
+    std::string name() override
+    {
+        return "upgrade-nix";
+    }
+
+    std::string description() override
+    {
+        return "upgrade Nix to the latest stable version";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To upgrade Nix to the latest stable version:",
+                "nix upgrade-nix"
+            },
+            Example{
+                "To upgrade Nix in a specific profile:",
+                "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        settings.pureEval = true;
+
+        if (profileDir == "")
+            profileDir = getProfileDir(store);
+
+        printInfo("upgrading Nix in profile '%s'", profileDir);
+
+        Path storePath;
+        {
+            Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
+            storePath = getLatestNix(store);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
+            store->ensurePath(storePath);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath));
+            auto program = storePath + "/bin/nix-env";
+            auto s = runProgram(program, false, {"--version"});
+            if (s.find("Nix") == std::string::npos)
+                throw Error("could not verify that '%s' works", program);
+        }
+
+        {
+            Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
+            runProgram(settings.nixBinDir + "/nix-env", false,
+                {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
+        }
+    }
+
+    /* Return the profile in which Nix is installed. */
+    Path getProfileDir(ref<Store> store)
+    {
+        Path where;
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+            if (pathExists(dir + "/nix-env")) {
+                where = dir;
+                break;
+            }
+
+        if (where == "")
+            throw Error("couldn't figure out how Nix is installed, so I can't upgrade it");
+
+        printInfo("found Nix in '%s'", where);
+
+        if (hasPrefix(where, "/run/current-system"))
+            throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'");
+
+        Path profileDir;
+        Path userEnv;
+
+        if (baseNameOf(where) != "bin" ||
+            !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment"))
+            throw Error("directory '%s' does not appear to be part of a Nix profile", where);
+
+        if (!store->isValidPath(userEnv))
+            throw Error("directory '%s' is not in the Nix store", userEnv);
+
+        return profileDir;
+    }
+
+    /* Return the store path of the latest stable Nix. */
+    Path getLatestNix(ref<Store> store)
+    {
+        // FIXME: use nixos.org?
+        auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix");
+        auto res = getDownloader()->download(req);
+
+        EvalState state(Strings(), store);
+        auto v = state.allocValue();
+        state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v);
+        Bindings & bindings(*state.allocBindings(0));
+        auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v);
+
+        return state.forceString(*v2);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdUpgradeNix>());

From c287d7312103bae5e154c0c4dd493371a22ea207 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 Jan 2018 18:58:45 +0100
Subject: [PATCH 0781/2196] Rename 1.12 -> 2.0

Following discussion with Shea and Graham. It's a big enough change
from the last release. Also, from a semver perspective, 2.0 makes more
sense because we did remove some interfaces (like nix-pull/nix-push).
---
 .github/ISSUE_TEMPLATE.md                            | 4 ++--
 doc/manual/release-notes/release-notes.xml           | 2 +-
 doc/manual/release-notes/{rl-1.12.xml => rl-2.0.xml} | 4 ++--
 src/libstore/local-store.hh                          | 2 +-
 version                                              | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)
 rename doc/manual/release-notes/{rl-1.12.xml => rl-2.0.xml} (99%)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index cdf642e6abd..3372b1f03f7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -14,7 +14,7 @@ Examples of _Nix_ issues:
 
 - Nix segfaults when I run `nix-build -A blahblah`
 - The Nix language needs a new builtin: `builtins.foobar`
-- Regression in the behavior of `nix-env` in Nix 1.12
+- Regression in the behavior of `nix-env` in Nix 2.0
 
 Examples of _nixpkgs_ issues:
 
@@ -24,4 +24,4 @@ Examples of _nixpkgs_ issues:
 
 Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
 
--->
\ No newline at end of file
+-->
diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index c4b14bc5499..b8392a647af 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,7 +12,7 @@
 </partintro>
 -->
 
-<xi:include href="rl-1.12.xml" />
+<xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />
 <xi:include href="rl-1.11.xml" />
 <xi:include href="rl-1.10.xml" />
diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-2.0.xml
similarity index 99%
rename from doc/manual/release-notes/rl-1.12.xml
rename to doc/manual/release-notes/rl-2.0.xml
index 89ec920949b..0d485c36928 100644
--- a/doc/manual/release-notes/rl-1.12.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -2,9 +2,9 @@
       xmlns:xlink="http://www.w3.org/1999/xlink"
       xmlns:xi="http://www.w3.org/2001/XInclude"
       version="5.0"
-      xml:id="ssec-relnotes-1.12">
+      xml:id="ssec-relnotes-2.0">
 
-<title>Release 1.12 (TBA)</title>
+<title>Release 2.0 (2018-02-??)</title>
 
 <para>This release has the following new features:</para>
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d35cd1a949e..30bef3a799d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -19,7 +19,7 @@ namespace nix {
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
    Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
-   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 1.12. */
+   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 2.0. */
 const int nixSchemaVersion = 10;
 
 
diff --git a/version b/version
index 35d51f33b34..415b19fc362 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-1.12
\ No newline at end of file
+2.0
\ No newline at end of file

From 88f4f0231b805d4d57ab4c03bd56650fe8b8fd4c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 31 Jan 2018 22:34:19 +0100
Subject: [PATCH 0782/2196] Fix building without aws-sdk-cpp

---
 src/libstore/download.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index a8bcab2f159..258d7937cc3 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -17,12 +17,13 @@
 
 #include <curl/curl.h>
 
-#include <queue>
-#include <iostream>
-#include <thread>
+#include <algorithm>
 #include <cmath>
+#include <cstring>
+#include <iostream>
+#include <queue>
 #include <random>
-#include <algorithm>
+#include <thread>
 
 using namespace std::string_literals;
 

From e5432574e2d54388eb52292da540f2d4de07e9dc Mon Sep 17 00:00:00 2001
From: Spencer Baugh <sbaugh@catern.com>
Date: Wed, 31 Jan 2018 22:47:16 +0000
Subject: [PATCH 0783/2196] document ability to set
 NIX_REMOTE=unix://path/to/socket

---
 doc/manual/command-ref/env-common.xml | 2 ++
 src/libstore/store-api.hh             | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index a83aeaf2e57..361d3e2b033 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -154,6 +154,8 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <literal>daemon</literal> if you want to use the Nix daemon to
   execute Nix operations. This is necessary in <link
   linkend="ssec-multi-user">multi-user Nix installations</link>.
+  If the Nix daemon's Unix socket is at some non-standard path,
+  this variable should be set to <literal>unix://path/to/socket</literal>.
   Otherwise, it should be left unset.</para></listitem>
 
 </varlistentry>
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c0e735cd314..b21d3dd8a93 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -705,6 +705,9 @@ void removeTempRoots();
    * ‘daemon’: The Nix store accessed via a Unix domain socket
      connection to nix-daemon.
 
+   * ‘unix://<path>’: The Nix store accessed via a Unix domain socket
+     connection to nix-daemon, with the socket located at <path>.
+
    * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
      whether the user has write access to the local Nix
      store/database.

From 855699855fd271e0b8e43001bf1793ca3ef9b71c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Feb 2018 10:39:16 +0100
Subject: [PATCH 0784/2196] Remove obsolete references to manifests

Closes #323.
---
 src/libstore/gc.cc | 4 +---
 tests/common.sh.in | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ab2c5ca0274..943b16c28fa 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -324,10 +324,8 @@ Roots LocalStore::findRootsNoTemp()
 {
     Roots roots;
 
-    /* Process direct roots in {gcroots,manifests,profiles}. */
+    /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
-    if (pathExists(stateDir + "/manifests"))
-        findRoots(stateDir + "/manifests", DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
     /* Add additional roots returned by the program specified by the
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 186f9d6b955..195205988af 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -11,7 +11,6 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
-export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1

From e7b23eb5ab418ca7194f6575014f40bf4d9e6afd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Feb 2018 16:40:58 +0100
Subject: [PATCH 0785/2196] Remove docs on removed --drv-link and
 --add-drv-link options

---
 doc/manual/command-ref/nix-build.xml | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml
index d6b2e5e5adb..40fe7a43f10 100644
--- a/doc/manual/command-ref/nix-build.xml
+++ b/doc/manual/command-ref/nix-build.xml
@@ -29,8 +29,6 @@
       </group>
       <replaceable>attrPath</replaceable>
     </arg>
-    <arg><option>--drv-link</option> <replaceable>drvlink</replaceable></arg>
-    <arg><option>--add-drv-link</option></arg>
     <arg><option>--no-out-link</option></arg>
     <arg>
       <group choice='req'>
@@ -91,25 +89,6 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 <variablelist>
 
-  <varlistentry><term><option>--drv-link</option> <replaceable>drvlink</replaceable></term>
-
-    <listitem><para>Add a symlink named
-    <replaceable>drvlink</replaceable> to the store derivation
-    produced by <command>nix-instantiate</command>.  The derivation is
-    a root of the garbage collector until the symlink is deleted or
-    renamed.  If there are multiple derivations, numbers are suffixed
-    to <replaceable>drvlink</replaceable> to distinguish between
-    them.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--add-drv-link</option></term>
-
-    <listitem><para>Shorthand for <option>--drv-link</option>
-    <filename>./derivation</filename>.</para></listitem>
-
-  </varlistentry>
-
   <varlistentry><term><option>--no-out-link</option></term>
 
     <listitem><para>Do not create a symlink to the output path.  Note

From 19477e8815a7c9b91c82278ec0e22957c97bec16 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Feb 2018 17:09:56 +0100
Subject: [PATCH 0786/2196] nix-build: Ignore --indirect

Note that nix-build always creates indirect roots.

Fixes #1830.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 1b249427537..3d02276bf42 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -141,7 +141,7 @@ void mainWrapped(int argc, char * * argv)
         else if (*arg == "--version")
             printVersion(myName);
 
-        else if (*arg == "--add-drv-link")
+        else if (*arg == "--add-drv-link" || *arg == "--indirect")
             ; // obsolete
 
         else if (*arg == "--no-out-link" || *arg == "--no-link")

From 84722d67d2b09b5c28e1c2d9dd438ba592df2296 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 3 Feb 2018 10:04:29 +0100
Subject: [PATCH 0787/2196] Remove nix-build --hash

Instead, if a fixed-output derivation produces has an incorrect output
hash, we now unconditionally move the outputs to the path
corresponding with the actual hash and register it as valid. Thus,
after correcting the hash in the Nix expression (e.g. in a fetchurl
call), the fixed-output derivation doesn't have to be built again.

It would still be good to have a command for reporting the actual hash
of a fixed-output derivation (instead of throwing an error), but
"nix-build --hash" didn't do that.
---
 doc/manual/release-notes/rl-2.0.xml | 10 +++---
 src/libstore/build.cc               | 47 +++++++++++++++++------------
 src/libstore/store-api.hh           |  2 +-
 src/nix-build/nix-build.cc          |  3 --
 src/nix-store/nix-store.cc          |  1 -
 tests/fixed.sh                      | 13 ++++++--
 6 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 0d485c36928..32cdb1d0cef 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -99,11 +99,11 @@
   </listitem>
 
   <listitem>
-    <para>New build mode <command>nix-build --hash</command> that
-    builds a derivation, computes the hash of the output, and moves
-    the output to the store path corresponding to what a fixed-output
-    derivation with that hash would produce.
-    (Add docs and examples; see d367b8e7875161e655deaa96bf8a5dd0bcf8229e)</para>
+    <para>If a fixed-output derivation produces a result with an
+    incorrect hash, the output path will be moved to the location
+    corresponding to the actual hash and registered as valid. Thus, a
+    subsequent build of the fixed-output derivation with the correct
+    hash is unnecessary.</para>
   </listitem>
 
   <listitem>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cca51f17ee2..d4b93b5104c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1124,11 +1124,6 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    /* Reject doing a hash build of anything other than a fixed-output
-       derivation. */
-    if (buildMode == bmHash && !drv->isFixedOutput())
-        throw Error("cannot do a hash build of non-fixed-output derivation '%1%'", drvPath);
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1320,9 +1315,7 @@ void DerivationGoal::inputsRealised()
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
     /* Is this a fixed-output derivation? */
-    fixedOutput = true;
-    for (auto & i : drv->outputs)
-        if (i.second.hash == "") fixedOutput = false;
+    fixedOutput = drv->isFixedOutput();
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
@@ -3019,6 +3012,8 @@ void DerivationGoal::registerOutputs()
     bool runDiffHook = settings.runDiffHook;
     bool keepPreviousRound = settings.keepFailed || runDiffHook;
 
+    std::exception_ptr delayedException;
+
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
@@ -3093,7 +3088,7 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
-        if (i.second.hash != "") {
+        if (fixedOutput) {
 
             bool recursive; Hash h;
             i.second.parseHashInfo(recursive, h);
@@ -3109,27 +3104,34 @@ void DerivationGoal::registerOutputs()
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
-            if (buildMode == bmHash) {
-                Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
-                printError(format("build produced path '%1%' with %2% hash '%3%'")
-                    % dest % printHashType(h.type) % printHash16or32(h2));
-                if (worker.store.isValidPath(dest))
-                    return;
+
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
+
+            if (h != h2) {
+
+                /* Throw an error after registering the path as
+                   valid. */
+                delayedException = std::make_exception_ptr(
+                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
+                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
+
                 Path actualDest = worker.store.toRealPath(dest);
+
+                if (worker.store.isValidPath(dest))
+                    std::rethrow_exception(delayedException);
+
                 if (actualPath != actualDest) {
                     PathLocks outputLocks({actualDest});
                     deletePath(actualDest);
                     if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
                         throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest);
                 }
+
                 path = dest;
                 actualPath = actualDest;
-            } else {
-                if (h != h2)
-                    throw BuildError(
-                        format("output path '%1%' has %2% hash '%3%' when '%4%' was expected")
-                        % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h));
             }
+            else
+                assert(path == dest);
 
             info.ca = makeFixedOutputCA(recursive, h2);
         }
@@ -3306,6 +3308,11 @@ void DerivationGoal::registerOutputs()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     worker.store.registerValidPaths(infos);
+
+    /* In case of a fixed-output derivation hash mismatch, throw an
+       exception now that we have registered the output as valid. */
+    if (delayedException)
+        std::rethrow_exception(delayedException);
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b21d3dd8a93..70f23e1fcaf 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -192,7 +192,7 @@ struct ValidPathInfo
 typedef list<ValidPathInfo> ValidPathInfos;
 
 
-enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash };
+enum BuildMode { bmNormal, bmRepair, bmCheck };
 
 
 struct BuildResult
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 3d02276bf42..1581c282c75 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -167,9 +167,6 @@ void mainWrapped(int argc, char * * argv)
             buildMode = bmRepair;
         }
 
-        else if (*arg == "--hash")
-            buildMode = bmHash;
-
         else if (*arg == "--run-env") // obsolete
             runEnv = true;
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index f6f276dd179..4fc3421c0dd 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -122,7 +122,6 @@ static void opRealise(Strings opFlags, Strings opArgs)
         if (i == "--dry-run") dryRun = true;
         else if (i == "--repair") buildMode = bmRepair;
         else if (i == "--check") buildMode = bmCheck;
-        else if (i == "--hash") buildMode = bmHash;
         else if (i == "--ignore-unknown") ignoreUnknown = true;
         else throw UsageError(format("unknown flag '%1%'") % i);
 
diff --git a/tests/fixed.sh b/tests/fixed.sh
index cac3f0be91b..8f51403a707 100644
--- a/tests/fixed.sh
+++ b/tests/fixed.sh
@@ -5,15 +5,22 @@ clearStore
 export IMPURE_VAR1=foo
 export IMPURE_VAR2=bar
 
+path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0))
+
+echo 'testing bad...'
+nix-build fixed.nix -A bad --no-out-link && fail "should fail"
+
+# Building with the bad hash should produce the "good" output path as
+# a side-effect.
+[[ -e $path ]]
+nix path-info --json $path | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd
+
 echo 'testing good...'
 nix-build fixed.nix -A good --no-out-link
 
 echo 'testing good2...'
 nix-build fixed.nix -A good2 --no-out-link
 
-echo 'testing bad...'
-nix-build fixed.nix -A bad --no-out-link && fail "should fail"
-
 echo 'testing reallyBad...'
 nix-instantiate fixed.nix -A reallyBad && fail "should fail"
 

From 0ffa615420fc1ee67a03dcb1a491fed9e8cdb782 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 5 Feb 2018 10:25:26 -0600
Subject: [PATCH 0788/2196] busybox shell: enable various useful/expected
 features

Matches changes made in nixpkgs:
https://github.com/NixOS/nixpkgs/pull/34628
---
 release-common.nix | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/release-common.nix b/release-common.nix
index 4553118e1f5..de5af452c0d 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -6,10 +6,22 @@ rec {
     enableStatic = true;
     enableMinimal = true;
     extraConfig = ''
+      CONFIG_FEATURE_FANCY_ECHO y
+      CONFIG_FEATURE_SH_MATH y
+      CONFIG_FEATURE_SH_MATH_64 y
+
       CONFIG_ASH y
+      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+      CONFIG_ASH_ALIAS y
+      CONFIG_ASH_BASH_COMPAT y
+      CONFIG_ASH_CMDCMD y
       CONFIG_ASH_ECHO y
+      CONFIG_ASH_GETOPTS y
+      CONFIG_ASH_INTERNAL_GLOB y
+      CONFIG_ASH_JOB_CONTROL y
+      CONFIG_ASH_PRINTF y
       CONFIG_ASH_TEST y
-      CONFIG_ASH_OPTIMIZE_FOR_SIZE y
     '';
   };
 

From 2175eee9fec07dea32e07471946d26a242a07760 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Feb 2018 17:46:43 +0100
Subject: [PATCH 0789/2196] Fix segfault using non-binary cache stores as
 substituters

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d4b93b5104c..5540d57a867 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3670,7 +3670,7 @@ void SubstitutionGoal::tryNext()
     /* Update the total expected download size. */
     auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize);
+    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
 
     maintainExpectedDownload =
         narInfo && narInfo->fileSize

From bb1d046f5c37a01ee85fc30d4602e8be8213eb84 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Feb 2018 18:08:30 +0100
Subject: [PATCH 0790/2196] Allow substituters to be marked as trusted

This is needed by nixos-install, which uses the Nix store on the
installation CD as a substituter. We don't want to disable signature
checking entirely because substitutes from cache.nixos.org should
still be checked. So now we can pas "local?trusted=1" to mark only the
Nix store in /nix as not requiring signatures.

Fixes #1819.
---
 src/libstore/build.cc     | 7 +++++--
 src/libstore/store-api.hh | 2 ++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5540d57a867..5be7ce60dab 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3684,7 +3684,10 @@ void SubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) {
+    if (worker.store.requireSigs
+        && !sub->isTrusted
+        && !info->checkSignatures(worker.store, worker.store.publicKeys))
+    {
         printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
             % sub->getUri() % storePath);
         tryNext();
@@ -3752,7 +3755,7 @@ void SubstitutionGoal::tryToRun()
             PushActivity pact(act.id);
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                storePath, repair);
+                storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 70f23e1fcaf..aa83c2ded8c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -248,6 +248,8 @@ public:
 
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
 
+    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
+
 protected:
 
     struct State

From 55012ec0b93351a0747b8d5d58b83ebc29dfaf61 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Feb 2018 18:32:23 +0100
Subject: [PATCH 0791/2196] Tweak progress bar message

Say "copying" instead of "fetching" when copying from another local
store. Nice for nixos-install.
---
 src/nix/progress-bar.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index fb9955190b4..8bffda54a55 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -167,7 +167,12 @@ class ProgressBar : public Logger
 
         if (type == actSubstitute) {
             auto name = storePathToName(getS(fields, 0));
-            i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1));
+            auto sub = getS(fields, 1);
+            i->s = fmt(
+                hasPrefix(sub, "local")
+                ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s"
+                : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s",
+                name, sub);
         }
 
         if (type == actQueryPathInfo) {

From 47dc6076afa6c7b919320353ac299f9551acd39e Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 5 Feb 2018 11:28:27 -0600
Subject: [PATCH 0792/2196] release-common: use shell from nixpkgs, provide
 fallback for compat

---
 release-common.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index 4553118e1f5..a4ae24ba482 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -1,7 +1,9 @@
 { pkgs }:
 
 rec {
-  sh = pkgs.busybox.override {
+  # Use "busybox-sandbox-shell" if present,
+  # if not (legacy) fallback and hope it's sufficient.
+  sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
     useMusl = true;
     enableStatic = true;
     enableMinimal = true;
@@ -11,7 +13,7 @@ rec {
       CONFIG_ASH_TEST y
       CONFIG_ASH_OPTIMIZE_FOR_SIZE y
     '';
-  };
+  });
 
   configureFlags =
     [ "--disable-init-state"

From f539085e651b133f023e8d02a8036124ac47d36c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Feb 2018 21:48:09 +0100
Subject: [PATCH 0793/2196] Fix evaluation

---
 tests/nix-copy-closure.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index 0bf5b42d84a..be0a4a683cd 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -2,7 +2,7 @@
 
 { nixpkgs, system, nix }:
 
-with import (nixpkgs + /nixos/lib/testing.nix) { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
 
 makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 

From f24e726ba53e23235d33d8bdc7877ad3a8632fde Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Feb 2018 14:35:14 +0100
Subject: [PATCH 0794/2196] checkURI(): Check file URIs against allowedPaths

This makes e.g. 'fetchGit ./.' work (assuming that ./. is an allowed
path).
---
 src/libexpr/eval.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 33a9bc61428..7775cbe53cc 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -378,6 +378,18 @@ void EvalState::checkURI(const std::string & uri)
             && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
             return;
 
+    /* If the URI is a path, then check it against allowedPaths as
+       well. */
+    if (hasPrefix(uri, "/")) {
+        checkSourcePath(uri);
+        return;
+    }
+
+    if (hasPrefix(uri, "file://")) {
+        checkSourcePath(std::string(uri, 7));
+        return;
+    }
+
     throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri);
 }
 

From 43f8ef73c6aeb23aee40d485556004d6262d4e3b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Feb 2018 15:38:45 +0100
Subject: [PATCH 0795/2196] realiseContext(): Add derivation outputs to the
 allowed paths

This makes import-from-derivation work in restricted mode again.
---
 src/libexpr/primops.cc | 34 ++++++++++++++++++++++++----------
 tests/restricted.sh    |  2 ++
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 975f0e8309e..5fe7da21643 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -49,24 +49,38 @@ InvalidPathError::InvalidPathError(const Path & path) :
 void EvalState::realiseContext(const PathSet & context)
 {
     PathSet drvs;
+
     for (auto & i : context) {
         std::pair<string, string> decoded = decodeContext(i);
         Path ctx = decoded.first;
         assert(store->isStorePath(ctx));
         if (!store->isValidPath(ctx))
             throw InvalidPathError(ctx);
-        if (!decoded.second.empty() && nix::isDerivation(ctx))
+        if (!decoded.second.empty() && nix::isDerivation(ctx)) {
             drvs.insert(decoded.first + "!" + decoded.second);
+
+            /* Add the output of this derivation to the allowed
+               paths. */
+            if (allowedPaths) {
+                auto drv = store->derivationFromPath(decoded.first);
+                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
+                if (i == drv.outputs.end())
+                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
+                allowedPaths->insert(i->second.path);
+            }
+        }
     }
-    if (!drvs.empty()) {
-        if (!settings.enableImportFromDerivation)
-            throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
-        /* For performance, prefetch all substitute info. */
-        PathSet willBuild, willSubstitute, unknown;
-        unsigned long long downloadSize, narSize;
-        store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-        store->buildPaths(drvs);
-    }
+
+    if (drvs.empty()) return;
+
+    if (!settings.enableImportFromDerivation)
+        throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+
+    /* For performance, prefetch all substitute info. */
+    PathSet willBuild, willSubstitute, unknown;
+    unsigned long long downloadSize, narSize;
+    store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+    store->buildPaths(drvs);
 }
 
 
diff --git a/tests/restricted.sh b/tests/restricted.sh
index 6c0392facf3..0605383cc86 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -36,3 +36,5 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix
 (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT)
 (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .)
 nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I .
+
+[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]]

From 6f6bfc820544c3fe9cc35ec67ed3f9d4c6a293a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 6 Feb 2018 20:51:37 +0100
Subject: [PATCH 0796/2196] Update the progress bar at most 20 times per second

Fixes #1834.
---
 src/nix/local.mk        |  2 ++
 src/nix/progress-bar.cc | 43 +++++++++++++++++++++++++++--------------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index bddd53b168d..f76da194467 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
+nix_LDFLAGS = -pthread
+
 $(eval $(call install-symlink, nix, $(bindir)/nix-hash))
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 8bffda54a55..252d12c5d37 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -3,8 +3,9 @@
 #include "sync.hh"
 #include "store-api.hh"
 
-#include <map>
 #include <atomic>
+#include <map>
+#include <thread>
 
 namespace nix {
 
@@ -101,15 +102,28 @@ class ProgressBar : public Logger
 
     Sync<State> state_;
 
+    std::thread updateThread;
+
+    std::condition_variable quitCV, updateCV;
+
 public:
 
     ProgressBar()
     {
+        updateThread = std::thread([&]() {
+            auto state(state_.lock());
+            while (state->active) {
+                state.wait(updateCV);
+                draw(*state);
+                state.wait_for(quitCV, std::chrono::milliseconds(50));
+            }
+        });
     }
 
     ~ProgressBar()
     {
         stop();
+        updateThread.join();
     }
 
     void stop()
@@ -121,6 +135,8 @@ class ProgressBar : public Logger
         writeToStderr("\r\e[K");
         if (status != "")
             writeToStderr("[" + status + "]\n");
+        updateCV.notify_one();
+        quitCV.notify_one();
     }
 
     void log(Verbosity lvl, const FormatOrString & fs) override
@@ -132,7 +148,7 @@ class ProgressBar : public Logger
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
-        update(state);
+        draw(state);
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
@@ -185,7 +201,7 @@ class ProgressBar : public Logger
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
-        update(*state);
+        update();
     }
 
     /* Check whether an activity has an ancestore with the specified
@@ -220,7 +236,7 @@ class ProgressBar : public Logger
             state->its.erase(i);
         }
 
-        update(*state);
+        update();
     }
 
     void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
@@ -230,7 +246,7 @@ class ProgressBar : public Logger
         if (type == resFileLinked) {
             state->filesLinked++;
             state->bytesLinked += getI(fields, 0);
-            update(*state);
+            update();
         }
 
         else if (type == resBuildLogLine) {
@@ -243,25 +259,25 @@ class ProgressBar : public Logger
                 info.lastLine = lastLine;
                 state->activities.emplace_back(info);
                 i->second = std::prev(state->activities.end());
-                update(*state);
+                update();
             }
         }
 
         else if (type == resUntrustedPath) {
             state->untrustedPaths++;
-            update(*state);
+            update();
         }
 
         else if (type == resCorruptedPath) {
             state->corruptedPaths++;
-            update(*state);
+            update();
         }
 
         else if (type == resSetPhase) {
             auto i = state->its.find(act);
             assert(i != state->its.end());
             i->second->phase = getS(fields, 0);
-            update(*state);
+            update();
         }
 
         else if (type == resProgress) {
@@ -272,7 +288,7 @@ class ProgressBar : public Logger
             actInfo.expected = getI(fields, 1);
             actInfo.running = getI(fields, 2);
             actInfo.failed = getI(fields, 3);
-            update(*state);
+            update();
         }
 
         else if (type == resSetExpected) {
@@ -284,17 +300,16 @@ class ProgressBar : public Logger
             state->activitiesByType[type].expected -= j;
             j = getI(fields, 1);
             state->activitiesByType[type].expected += j;
-            update(*state);
+            update();
         }
     }
 
     void update()
     {
-        auto state(state_.lock());
-        update(*state);
+        updateCV.notify_one();
     }
 
-    void update(State & state)
+    void draw(State & state)
     {
         if (!state.active) return;
 

From bc7e3a4dd62baa99dbd1985d329a2a806d59a422 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Tue, 6 Feb 2018 22:42:02 +0100
Subject: [PATCH 0797/2196] support multi threaded xz encoder, this might be
 particularly useful in the case of hydra where the overhead of single
 threaded encoding is more noticeable e.g most of the time spent in "Sending
 inputs"/"Receiving outputs" is due to compression while the actual upload to
 the binary cache seems to be negligible.

---
 src/libutil/compression.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5e2631ba340..aad6e9b5bc1 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -191,8 +191,13 @@ struct XzSink : CompressionSink
 
     XzSink(Sink & nextSink) : nextSink(nextSink)
     {
-        lzma_ret ret = lzma_easy_encoder(
-            &strm, 6, LZMA_CHECK_CRC64);
+        lzma_mt mt_options = {};
+        mt_options.flags = 0;
+        mt_options.timeout = 300;
+        mt_options.check = LZMA_CHECK_CRC64;
+        mt_options.threads = lzma_cputhreads();
+        lzma_ret ret = lzma_stream_encoder_mt(
+            &strm, &mt_options);
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?

From 69d82e5c58bf6d7e16fc296f598c352da2a618d0 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 25 Jan 2018 07:05:57 -0800
Subject: [PATCH 0798/2196] Add path primop.

builtins.path allows specifying the name of a path (which makes paths
with store-illegal names now addable), allows adding paths with flat
instead of recursive hashes, allows specifying a filter (so is a
generalization of filterSource), and allows specifying an expected
hash (enabling safe path adding in pure mode).
---
 doc/manual/expressions/builtins.xml | 74 ++++++++++++++++++++++-
 src/libexpr/eval.cc                 |  2 +-
 src/libexpr/primops.cc              | 93 +++++++++++++++++++++++------
 src/libstore/store-api.cc           |  5 +-
 src/libstore/store-api.hh           |  6 +-
 tests/lang/data                     |  1 +
 tests/lang/eval-okay-path.exp       |  1 +
 tests/lang/eval-okay-path.nix       |  7 +++
 8 files changed, 162 insertions(+), 27 deletions(-)
 create mode 100644 tests/lang/data
 create mode 100644 tests/lang/eval-okay-path.exp
 create mode 100644 tests/lang/eval-okay-path.nix

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 5a3a8645c1d..81770bcf629 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -308,8 +308,9 @@ stdenv.mkDerivation { … }
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.filterSource</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-filterSource'>
+    <term><function>builtins.filterSource</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem>
 
@@ -768,6 +769,75 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry>
+    <term>
+      <function>builtins.path</function>
+      <replaceable>args</replaceable>
+    </term>
+
+    <listitem>
+      <para>
+        An enrichment of the built-in path type, based on the attributes
+        present in <replaceable>args</replaceable>. All are optional
+        except <varname>path</varname>:
+      </para>
+
+      <variablelist>
+        <varlistentry>
+          <term>path</term>
+          <listitem>
+            <para>The underlying path.</para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>name</term>
+          <listitem>
+            <para>
+              The name of the path when added to the store. This can
+              used to reference paths that have nix-illegal characters
+              in their names, like <literal>@</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>filter</term>
+          <listitem>
+            <para>
+              A function of the type expected by
+              <link linkend="builtin-filterSource">builtins.filterSource</link>,
+              with the same semantics.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>recursive</term>
+          <listitem>
+            <para>
+              When <literal>false</literal>, when
+              <varname>path</varname> is added to the store it is with a
+              flat hash, rather than a hash of the NAR serialization of
+              the file. Thus, <varname>path</varname> must refer to a
+              regular file, not a directory. This allows similar
+              behavior to <literal>fetchurl</literal>. Defaults to
+              <literal>true</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>sha256</term>
+          <listitem>
+            <para>
+              When provided, this is the expected hash of the file at
+              the path. Evaluation will fail if the hash is incorrect,
+              and providing a hash allows
+              <literal>builtins.path</literal> to be used even when the
+              <literal>pure-eval</literal> nix config option is on.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+    </listitem>
+  </varlistentry>
 
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 33a9bc61428..9499ebe7098 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1566,7 +1566,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         dstPath = srcToStore[path];
     else {
         dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(checkSourcePath(path)).first
+            ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
             : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         srcToStore[path] = dstPath;
         printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 975f0e8309e..5c8dfd9dfe4 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1009,20 +1009,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 }
 
 
-static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
+    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    PathSet context;
-    Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
-
-    state.forceValue(*args[0]);
-    if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
-
-    path = state.checkSourcePath(path);
-
-    PathFilter filter = [&](const Path & path) {
+    const auto path = settings.pureEval && expectedHash ?
+        path_ :
+        state.checkSourcePath(path_);
+    PathFilter filter = filterFun ? ([&](const Path & path) {
         auto st = lstat(path);
 
         /* Call the filter function.  The first argument is the path,
@@ -1031,7 +1024,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         mkString(arg1, path);
 
         Value fun2;
-        state.callFunction(*args[0], arg1, fun2, noPos);
+        state.callFunction(*filterFun, arg1, fun2, noPos);
 
         Value arg2;
         mkString(arg2,
@@ -1044,16 +1037,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
-    };
+    }) : defaultPathFilter;
 
-    Path dstPath = settings.readOnlyMode
-        ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first
-        : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair);
+    Path expectedStorePath;
+    if (expectedHash) {
+        expectedStorePath =
+            state.store->makeFixedOutputPath(recursive, expectedHash, name);
+    }
+    Path dstPath;
+    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
+        dstPath = settings.readOnlyMode
+            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
+        if (expectedHash && expectedStorePath != dstPath) {
+            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
+        }
+    } else
+        dstPath = expectedStorePath;
 
     mkString(v, dstPath, {dstPath});
 }
 
 
+static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    if (!context.empty())
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+
+    state.forceValue(*args[0]);
+    if (args[0]->type != tLambda)
+        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+
+    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+}
+
+static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+    Path path;
+    string name;
+    Value * filterFun = nullptr;
+    auto recursive = true;
+    Hash expectedHash;
+
+    for (auto & attr : *args[0]->attrs) {
+        const string & n(attr.name);
+        if (n == "path") {
+            PathSet context;
+            path = state.coerceToPath(*attr.pos, *attr.value, context);
+            if (!context.empty())
+                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+        } else if (attr.name == state.sName)
+            name = state.forceStringNoCtx(*attr.value, *attr.pos);
+        else if (n == "filter") {
+            state.forceValue(*attr.value);
+            filterFun = attr.value;
+        } else if (n == "recursive")
+            recursive = state.forceBool(*attr.value, *attr.pos);
+        else if (n == "sha256")
+            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+        else
+            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+    }
+    if (path.empty())
+        throw EvalError(format("'path' required, at %1%") % pos);
+    if (name.empty())
+        name = baseNameOf(path);
+
+    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+}
+
+
 /*************************************************************
  * Sets
  *************************************************************/
@@ -2071,6 +2127,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__fromJSON", 1, prim_fromJSON);
     addPrimOp("__toFile", 2, prim_toFile);
     addPrimOp("__filterSource", 2, prim_filterSource);
+    addPrimOp("__path", 1, prim_path);
 
     // Sets
     addPrimOp("__attrNames", 1, prim_attrNames);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 77ab87ef728..7abb300a9bb 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
 }
 
 
-std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter) const
+std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
+    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
 {
     Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    string name = baseNameOf(srcPath);
     Path dstPath = makeFixedOutputPath(recursive, h, name);
     return std::pair<Path, Hash>(dstPath, h);
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c0e735cd314..bf0862ef1bb 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -305,9 +305,9 @@ public:
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
-    std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter) const;
+    std::pair<Path, Hash> computeStorePathForPath(const string & name,
+        const Path & srcPath, bool recursive = true,
+        HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
 
diff --git a/tests/lang/data b/tests/lang/data
new file mode 100644
index 00000000000..257cc5642cb
--- /dev/null
+++ b/tests/lang/data
@@ -0,0 +1 @@
+foo
diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp
new file mode 100644
index 00000000000..6827d49ffa1
--- /dev/null
+++ b/tests/lang/eval-okay-path.exp
@@ -0,0 +1 @@
+"/run/user/1000/nix-test/store/wjagrv37lfvfx92g2gf3yqflwypj0q1y-output"
diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix
new file mode 100644
index 00000000000..e67168cf3ed
--- /dev/null
+++ b/tests/lang/eval-okay-path.nix
@@ -0,0 +1,7 @@
+builtins.path
+  { path = ./.;
+    filter = path: _: baseNameOf path == "data";
+    recursive = true;
+    sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
+    name = "output";
+  }

From 9d1e22f743ea9ca232d39d498b675d7e5ac1ca87 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Wed, 7 Feb 2018 11:18:55 +0100
Subject: [PATCH 0799/2196] set block size to 0 to let the lzma lib choose the
 right one, add some comments about possible improvements wrt memory
 usage/threading.

---
 src/libutil/compression.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index aad6e9b5bc1..a36c4405f8b 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -193,9 +193,14 @@ struct XzSink : CompressionSink
     {
         lzma_mt mt_options = {};
         mt_options.flags = 0;
-        mt_options.timeout = 300;
+        mt_options.timeout = 300; // Using the same setting as the xz cmd line
         mt_options.check = LZMA_CHECK_CRC64;
         mt_options.threads = lzma_cputhreads();
+        mt_options.block_size = 0;
+        if (mt_options.threads == 0)
+            mt_options.threads = 1;
+        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+        // number of threads.
         lzma_ret ret = lzma_stream_encoder_mt(
             &strm, &mt_options);
         if (ret != LZMA_OK)

From cfdfad5c3451731879a5a693059c094f107c2bc8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Feb 2018 14:15:20 +0100
Subject: [PATCH 0800/2196] Simplify

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index d9c4f1efb4b..9e04f0b67f9 100644
--- a/release.nix
+++ b/release.nix
@@ -225,7 +225,7 @@ let
     });
 
     tests.setuid = pkgs.lib.genAttrs
-      (pkgs.lib.filter (system: system == "x86_64-linux" || system == "i686-linux") systems)
+      ["i686-linux" "x86_64-linux"]
       (system:
         import ./tests/setuid.nix rec {
           inherit nixpkgs;

From 84989d3af23c717744b8ddeacd6828bc87e7eda1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Feb 2018 15:19:10 +0100
Subject: [PATCH 0801/2196] Improve filtering of ANSI escape sequences in build
 logs

All ANSI sequences except color setting are now filtered out. In
particular, terminal resets (such as from NixOS VM tests) are filtered
out.

Also, fix the completely broken tab character handling.
---
 src/libstore/build.cc   |  2 +-
 src/libutil/logging.cc  |  2 +-
 src/libutil/util.cc     | 67 +++++++++++++++++++++++++----------------
 src/libutil/util.hh     | 10 +++---
 src/nix/progress-bar.cc | 43 +++-----------------------
 tests/misc.sh           |  2 +-
 6 files changed, 54 insertions(+), 72 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5be7ce60dab..9f669f7e464 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine()
     else {
         if (settings.verboseBuild &&
             (settings.printRepeatedBuilds || curRound == 1))
-            printError(filterANSIEscapes(currentLogLine, true));
+            printError(currentLogLine);
         else {
             logTail.push_back(currentLogLine);
             if (logTail.size() > settings.logLines) logTail.pop_front();
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 6924e008047..27a631a37d1 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -44,7 +44,7 @@ class SimpleLogger : public Logger
             prefix = std::string("<") + c + ">";
         }
 
-        writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
+        writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 27299739779..f7a12d21b24 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1178,36 +1178,51 @@ void ignoreException()
 }
 
 
-string filterANSIEscapes(const string & s, bool nixOnly)
-{
-    string t, r;
-    enum { stTop, stEscape, stCSI } state = stTop;
-    for (auto c : s) {
-        if (state == stTop) {
-            if (c == '\e') {
-                state = stEscape;
-                r = c;
-            } else
-                t += c;
-        } else if (state == stEscape) {
-            r += c;
-            if (c == '[')
-                state = stCSI;
-            else {
-                t += r;
-                state = stTop;
+std::string filterANSIEscapes(const std::string & s, unsigned int width)
+{
+    std::string t, e;
+    size_t w = 0;
+    auto i = s.begin();
+
+    while (w < (size_t) width && i != s.end()) {
+
+        if (*i == '\e') {
+            std::string e;
+            e += *i++;
+            char last = 0;
+
+            if (i != s.end() && *i == '[') {
+                e += *i++;
+                // eat parameter bytes
+                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
+                // eat intermediate bytes
+                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
+                // eat final byte
+                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
+            } else {
+                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
             }
-        } else {
-            r += c;
-            if (c >= 0x40 && c <= 0x7e) {
-                if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b'))
-                    t += r;
-                state = stTop;
-                r.clear();
+
+            if (last == 'm')
+                t += e;
+        }
+
+        else if (*i == '\t') {
+            i++; t += ' '; w++;
+            while (w < (size_t) width && w % 8) {
+                t += ' '; w++;
             }
         }
+
+        else if (*i == '\r')
+            // do nothing for now
+            ;
+
+        else {
+            t += *i++; w++;
+        }
     }
-    t += r;
+
     return t;
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 75eb9751524..47e02bc898a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -388,10 +388,12 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
-/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is
-   set, only filter escape codes generated by Nixpkgs' stdenv (used to
-   denote nesting etc.). */
-string filterANSIEscapes(const string & s, bool nixOnly = false);
+/* Truncate a string to 'width' printable characters. Certain ANSI
+   escape sequences (such as colour setting) are copied but not
+   included in the character count. Other ANSI escape sequences are
+   filtered. Also, tabs are expanded to spaces. */
+std::string filterANSIEscapes(const std::string & s,
+    unsigned int width = std::numeric_limits<unsigned int>::max());
 
 
 /* Base64 encoding/decoding. */
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 252d12c5d37..e6553c06f4a 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -23,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
     return fields[n].i;
 }
 
-/* Truncate a string to 'width' printable characters. ANSI escape
-   sequences are copied but not included in the character count. Also,
-   tabs are expanded to spaces. */
-static std::string ansiTruncate(const std::string & s, int width)
-{
-    if (width <= 0) return s;
-
-    std::string t;
-    size_t w = 0;
-    auto i = s.begin();
-
-    while (w < (size_t) width && i != s.end()) {
-        if (*i == '\e') {
-            t += *i++;
-            if (i != s.end() && *i == '[') {
-                t += *i++;
-                while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
-                    t += *i++;
-                }
-                if (i != s.end()) t += *i++;
-            }
-        }
-
-        else if (*i == '\t') {
-            t += ' '; w++;
-            while (w < (size_t) width && w & 8) {
-                t += ' '; w++;
-            }
-        }
-
-        else {
-            t += *i++; w++;
-        }
-    }
-
-    return t;
-}
-
 class ProgressBar : public Logger
 {
 private:
@@ -343,7 +305,10 @@ class ProgressBar : public Logger
             }
         }
 
-        writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K");
+        auto width = getWindowSize().second;
+        if (width <= 0) std::numeric_limits<decltype(width)>::max();
+
+        writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
     }
 
     std::string getStatus(State & state)
diff --git a/tests/misc.sh b/tests/misc.sh
index 6d0ab3adcec..eda0164167f 100644
--- a/tests/misc.sh
+++ b/tests/misc.sh
@@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$"
+nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"

From 48c192ca2d5bc65b69d2336c8577258f8eb80cf8 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 7 Feb 2018 10:26:53 -0500
Subject: [PATCH 0802/2196] builtins.path test: Don't rely on shlevy's
 XDG_RUNTIME_DIR

---
 tests/lang/eval-okay-path.exp | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 tests/lang/eval-okay-path.exp

diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp
deleted file mode 100644
index 6827d49ffa1..00000000000
--- a/tests/lang/eval-okay-path.exp
+++ /dev/null
@@ -1 +0,0 @@
-"/run/user/1000/nix-test/store/wjagrv37lfvfx92g2gf3yqflwypj0q1y-output"

From 55ecdfe2a83a161c27d6497733cdc60fa112a43d Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Wed, 7 Feb 2018 17:54:08 +0100
Subject: [PATCH 0803/2196] make multi threaded compression configurable and
 use single threaded by default.

---
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/globals.hh            |  3 +++
 src/libutil/compression.cc         | 42 ++++++++++++++++++------------
 src/libutil/compression.hh         |  4 +--
 4 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ab971dd8b6d..d34adbd60c7 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar);
+    auto narCompressed = compress(compression, *nar, settings.parallelCompression);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 20ac8fe4e9a..aafec2ea269 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -174,6 +174,9 @@ public:
         "Whether to compress logs.",
         {"build-compress-log"}};
 
+    Setting<bool> parallelCompression{this, false, "parallel-compression",
+        "Whether to enable parallel compression, only supported with xz currently"};
+
     Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
         "Maximum number of bytes a builder can write to stdout/stderr "
         "before being killed (0 means no limit).",
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a36c4405f8b..ed15761b32a 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -151,10 +151,10 @@ static ref<std::string> decompressBrotli(const std::string & in)
 #endif // HAVE_BROTLI
 }
 
-ref<std::string> compress(const std::string & method, const std::string & in)
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
 {
     StringSink ssink;
-    auto sink = makeCompressionSink(method, ssink);
+    auto sink = makeCompressionSink(method, ssink, parallel);
     (*sink)(in);
     sink->finish();
     return ssink.s;
@@ -189,20 +189,28 @@ struct XzSink : CompressionSink
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    XzSink(Sink & nextSink) : nextSink(nextSink)
+    XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
     {
-        lzma_mt mt_options = {};
-        mt_options.flags = 0;
-        mt_options.timeout = 300; // Using the same setting as the xz cmd line
-        mt_options.check = LZMA_CHECK_CRC64;
-        mt_options.threads = lzma_cputhreads();
-        mt_options.block_size = 0;
-        if (mt_options.threads == 0)
-            mt_options.threads = 1;
-        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-        // number of threads.
-        lzma_ret ret = lzma_stream_encoder_mt(
-            &strm, &mt_options);
+        lzma_ret ret;
+        if (parallel) {
+            lzma_mt mt_options = {};
+            mt_options.flags = 0;
+            mt_options.timeout = 300; // Using the same setting as the xz cmd line
+            mt_options.preset = LZMA_PRESET_DEFAULT;
+            mt_options.filters = NULL;
+            mt_options.check = LZMA_CHECK_CRC64;
+            mt_options.threads = lzma_cputhreads();
+            mt_options.block_size = 0;
+            if (mt_options.threads == 0)
+                mt_options.threads = 1;
+            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+            // number of threads.
+            ret = lzma_stream_encoder_mt(
+                &strm, &mt_options);
+        } else
+            ret = lzma_easy_encoder(
+                &strm, 6, LZMA_CHECK_CRC64);
+
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
@@ -459,12 +467,12 @@ struct BrotliSink : CompressionSink
 };
 #endif // HAVE_BROTLI
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink);
+        return make_ref<XzSink>(nextSink, parallel);
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index e3e6f5a9930..a0d7530d74f 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,7 +8,7 @@
 
 namespace nix {
 
-ref<std::string> compress(const std::string & method, const std::string & in);
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
@@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink
     virtual void finish() = 0;
 };
 
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink);
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
 
 MakeError(UnknownCompressionMethod, Error);
 

From 88b5d0c8e89afefbc547b6243c5aa5a3ec8176e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Feb 2018 19:07:38 +0100
Subject: [PATCH 0804/2196] Prevent accidental recursive Nix

---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index b5d49b6428a..0e5ff370cf7 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -994,7 +994,7 @@ static void daemonLoop(char * * argv)
             if (matchUser(user, group, trustedUsers))
                 trusted = true;
 
-            if (!trusted && !matchUser(user, group, allowedUsers))
+            if ((!trusted && !matchUser(user, group, allowedUsers)) || group == "nixbld")
                 throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
 
             printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""))

From 47ad88099b1cd2b19bdf3eef35c21baf35cc7e82 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Wed, 7 Feb 2018 21:06:11 +0100
Subject: [PATCH 0805/2196] move the parallel-compression setting to
 binary-cache-store, the setting can be done now from the url e.g
 s3://nix-cache?parallel-compression=1 instead of nix.conf.

---
 src/libstore/binary-cache-store.cc | 2 +-
 src/libstore/binary-cache-store.hh | 2 ++
 src/libstore/globals.hh            | 3 ---
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index d34adbd60c7..d1b278b8efb 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, settings.parallelCompression);
+    auto narCompressed = compress(compression, *nar, parallelCompression);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 8492ff600eb..e20b968442b 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -19,6 +19,8 @@ public:
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+        "enable multi-threading compression, available for xz only currently"};
 
 private:
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index aafec2ea269..20ac8fe4e9a 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -174,9 +174,6 @@ public:
         "Whether to compress logs.",
         {"build-compress-log"}};
 
-    Setting<bool> parallelCompression{this, false, "parallel-compression",
-        "Whether to enable parallel compression, only supported with xz currently"};
-
     Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
         "Maximum number of bytes a builder can write to stdout/stderr "
         "before being killed (0 means no limit).",

From f201b7733e22cc236a41093a88cc789239d994bd Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 7 Feb 2018 15:17:44 -0600
Subject: [PATCH 0806/2196] More completely fix recursive nix, unbreak tests

See:
https://github.com/NixOS/nix/commit/88b5d0c8e89afefbc547b6243c5aa5a3ec8176e9#commitcomment-27406365
---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 0e5ff370cf7..d3a8ebbdda5 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -994,7 +994,7 @@ static void daemonLoop(char * * argv)
             if (matchUser(user, group, trustedUsers))
                 trusted = true;
 
-            if ((!trusted && !matchUser(user, group, allowedUsers)) || group == "nixbld")
+            if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup)
                 throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
 
             printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""))

From 3780435a0e3166ff898b33b8eb9a057ffa16ce67 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 7 Feb 2018 14:57:44 -0600
Subject: [PATCH 0807/2196] tests: Add (failing) tests for reported --dry-run
 issues.

---
 tests/build-dry.sh | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/local.mk     |  1 +
 2 files changed, 50 insertions(+)
 create mode 100644 tests/build-dry.sh

diff --git a/tests/build-dry.sh b/tests/build-dry.sh
new file mode 100644
index 00000000000..9a2a70d5f11
--- /dev/null
+++ b/tests/build-dry.sh
@@ -0,0 +1,49 @@
+source common.sh
+
+###################################################
+# Check that --dry-run isn't confused with read-only mode
+# https://github.com/NixOS/nix/issues/1795
+
+clearStore
+clearCache
+
+# Ensure this builds successfully first
+nix build -f dependencies.nix
+
+clearStore
+clearCache
+
+# Try --dry-run using old command first
+nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+# Now new command:
+nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
+
+clearStore
+clearCache
+
+# Try --dry-run using new command first
+nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
+# Now old command:
+nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+
+
+###################################################
+# Check --dry-run doesn't create links with --dry-run
+# https://github.com/NixOS/nix/issues/1849
+clearStore
+clearCache
+
+RESULT=$TEST_ROOT/result-link
+rm -f $RESULT
+
+nix-build dependencies.nix -o $RESULT --dry-run
+
+[[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link"
+
+nix build -f dependencies.nix -o $RESULT --dry-run
+
+[[ ! -h $RESULT ]] || fail "nix build --dry-run created output link"
+
+nix build -f dependencies.nix -o $RESULT
+
+[[ -h $RESULT ]]
diff --git a/tests/local.mk b/tests/local.mk
index e90b9f7da4a..0035dca2d61 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -13,6 +13,7 @@ nix_tests = \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
   placeholders.sh nix-shell.sh \
   linux-sandbox.sh \
+  build-dry.sh \
   build-remote.sh \
   nar-access.sh \
   structured-attrs.sh \

From 98031b6050000bccef915b99cf3a2ed810a1b35e Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 7 Feb 2018 14:58:38 -0600
Subject: [PATCH 0808/2196] nix build: Don't create output links with
 --dry-run.

Fixes #1849.
---
 src/nix/build.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index f7c99f12dbb..093415a0de1 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -52,6 +52,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand
     {
         auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
 
+        if (dryRun) return;
+
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);
 

From c7e0be1bfc79909cafd52645c88ceb8bcf2a588a Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 7 Feb 2018 15:08:13 -0600
Subject: [PATCH 0809/2196] build-dry: disable failing portion of test until
 issue is fixed

---
 tests/build-dry.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/build-dry.sh b/tests/build-dry.sh
index 9a2a70d5f11..610e6070c5d 100644
--- a/tests/build-dry.sh
+++ b/tests/build-dry.sh
@@ -18,6 +18,9 @@ nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now new command:
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
+# TODO: XXX: FIXME: #1793
+# Disable this part of the test until the problem is resolved:
+if [ -n "$ISSUE_1795_IS_FIXED" ]; then
 clearStore
 clearCache
 
@@ -25,7 +28,7 @@ clearCache
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now old command:
 nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
-
+fi
 
 ###################################################
 # Check --dry-run doesn't create links with --dry-run

From 444bae44ef3c27959d837d2e1f28131290cb52a7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Feb 2018 13:46:23 +0100
Subject: [PATCH 0810/2196] dsa -> ed25519

DSS is disabled by default in NixOS 18.03.

https://hydra.nixos.org/build/68788560
---
 tests/nix-copy-closure.nix |  6 +++---
 tests/remote-builds.nix    | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index be0a4a683cd..0dc147fb34e 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -29,10 +29,10 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
       startAll;
 
       # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
       $client->succeed("mkdir -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
-      $client->succeed("chmod 600 /root/.ssh/id_dsa");
+      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
 
       # Install the SSH key on the server.
       $server->succeed("mkdir -m 700 /root/.ssh");
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 75704ace2db..d7a4b21989e 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -46,13 +46,13 @@ in
           nix.buildMachines =
             [ { hostName = "slave1";
                 sshUser = "root";
-                sshKey = "/root/.ssh/id_dsa";
+                sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
                 maxJobs = 1;
               }
               { hostName = "slave2";
                 sshUser = "root";
-                sshKey = "/root/.ssh/id_dsa";
+                sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
                 maxJobs = 1;
               }
@@ -70,10 +70,10 @@ in
       startAll;
 
       # Create an SSH key on the client.
-      my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+      my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
       $client->succeed("mkdir -p -m 700 /root/.ssh");
-      $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
-      $client->succeed("chmod 600 /root/.ssh/id_dsa");
+      $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+      $client->succeed("chmod 600 /root/.ssh/id_ed25519");
 
       # Install the SSH key on the slaves.
       $client->waitForUnit("network.target");

From ad97a218344937d8586d0f4f710f3785da77f5bd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Feb 2018 15:25:03 +0100
Subject: [PATCH 0811/2196] nix-env: Fix parsing of --system

https://hydra.nixos.org/build/68827814
---
 src/libmain/common-args.cc | 4 ++++
 tests/user-envs.sh         | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index d3aac6aba1f..bcc05c2cdad 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     std::string cat = "config";
     settings.convertToArgs(*this, cat);
+
+    // Backward compatibility hack: nix-env already had a --system flag.
+    if (programName == "nix-env") longFlags.erase("system");
+
     hiddenCategories.insert(cat);
 }
 
diff --git a/tests/user-envs.sh b/tests/user-envs.sh
index c4192fdc59b..ba63923113d 100644
--- a/tests/user-envs.sh
+++ b/tests/user-envs.sh
@@ -24,6 +24,9 @@ rm -f $HOME/.nix-defexpr
 ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr
 nix-env -qa '*' --description | grep -q silly
 
+# Query the system.
+nix-env -qa '*' --system | grep -q $system
+
 # Install "foo-1.0".
 nix-env -i foo-1.0
 

From 88cd2d41acb994684a3e4ead1b1676019f43b4b6 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 8 Feb 2018 11:26:18 -0500
Subject: [PATCH 0812/2196] Add plugins to make Nix more extensible.

All plugins in plugin-files will be dlopened, allowing them to
statically construct instances of the various Register* types Nix
supports.
---
 .gitignore                                    |  1 +
 Makefile                                      |  3 ++-
 doc/manual/command-ref/conf-file.xml          | 27 +++++++++++++++++++
 doc/manual/release-notes/rl-2.0.xml           |  7 +++++
 mk/libraries.mk                               |  7 +++++
 src/build-remote/build-remote.cc              |  2 ++
 src/libmain/shared.hh                         |  1 +
 src/libstore/globals.cc                       | 15 +++++++++++
 src/libstore/globals.hh                       |  7 +++++
 src/libstore/local.mk                         |  3 +++
 src/nix-build/nix-build.cc                    |  2 ++
 src/nix-channel/nix-channel.cc                |  3 +++
 .../nix-collect-garbage.cc                    |  2 ++
 src/nix-copy-closure/nix-copy-closure.cc      |  2 ++
 src/nix-daemon/nix-daemon.cc                  |  2 ++
 src/nix-env/nix-env.cc                        |  2 ++
 src/nix-instantiate/nix-instantiate.cc        |  2 ++
 src/nix-prefetch-url/nix-prefetch-url.cc      |  2 ++
 src/nix-store/nix-store.cc                    |  2 ++
 src/nix/main.cc                               |  2 ++
 tests/local.mk                                |  5 ++--
 tests/plugins.sh                              |  7 +++++
 tests/plugins/local.mk                        |  9 +++++++
 tests/plugins/plugintest.cc                   | 10 +++++++
 24 files changed, 122 insertions(+), 3 deletions(-)
 create mode 100644 tests/plugins.sh
 create mode 100644 tests/plugins/local.mk
 create mode 100644 tests/plugins/plugintest.cc

diff --git a/.gitignore b/.gitignore
index ce22fa007dc..0a959937856 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ perl/Makefile.config
 /scripts/nix-copy-closure
 /scripts/nix-reduce-build
 /scripts/nix-http-export.cgi
+/scripts/nix-profile-daemon.sh
 
 # /src/libexpr/
 /src/libexpr/lexer-tab.cc
diff --git a/Makefile b/Makefile
index 5d8e990cc5c..c867823fc48 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,8 @@ makefiles = \
   misc/launchd/local.mk \
   misc/upstart/local.mk \
   doc/manual/local.mk \
-  tests/local.mk
+  tests/local.mk \
+  tests/plugins/local.mk
 
 GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
 
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index fff7994f28d..cede6db3cea 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -742,6 +742,33 @@ builtins.fetchurl {
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-plugin-files">
+    <term><literal>plugin-files</literal></term>
+    <listitem>
+      <para>
+        A list of plugin files to be loaded by Nix. Each of these
+        files will be dlopened by Nix, allowing them to affect
+        execution through static initialization. In particular, these
+        plugins may construct static instances of RegisterPrimOp to
+        add new primops to the expression language,
+        RegisterStoreImplementation to add new store implementations,
+        and RegisterCommand to add new subcommands to the
+        <literal>nix</literal> command. See the constructors for those
+        types for more details.
+      </para>
+      <para>
+	Since these files are loaded into the same address space as
+        Nix itself, they must be DSOs compatible with the instance of
+        Nix running at the time (i.e. compiled against the same
+        headers, not linked to any incompatible libraries). They
+        should not be linked to any Nix libs directly, as those will
+        be available already at load time.
+      </para>
+    </listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 32cdb1d0cef..effd2e39d30 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -389,6 +389,13 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     </para>
   </listitem>
 
+  <listitem>
+    <para>
+      Nix can now be extended with plugins. See the documentation of
+      the 'plugin-files' option for more details.
+    </para>
+  </listitem>
+
 </itemizedlist>
 
 <para>Some features were removed:</para>
diff --git a/mk/libraries.mk b/mk/libraries.mk
index 3cd7a53107b..14c95fa91cf 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -45,6 +45,11 @@ endif
 # - $(1)_INSTALL_DIR: the directory where the library will be
 #   installed.  Defaults to $(libdir).
 #
+# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not
+#   be automatically marked as a dependency of the top-level all
+#   target andwill not be listed in the make help output. This is
+#   useful for libraries built solely for testing, for example.
+#
 # - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be
 #   built, otherwise a static library.
 define build-library
@@ -149,7 +154,9 @@ define build-library
   $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn)))
   -include $$($(1)_DEPS)
 
+  ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST
   libs-list += $$($(1)_PATH)
+  endif
   clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
 endef
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index df579729af2..50eb6b29e51 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -64,6 +64,8 @@ int main (int argc, char * * argv)
 
         settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
+        initPlugins();
+
         auto store = openStore().cast<LocalStore>();
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 1dcc4f0ac94..8e4861232db 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -22,6 +22,7 @@ public:
 
 int handleExceptions(const string & programName, std::function<void()> fun);
 
+/* Don't forget to call initPlugins() after settings are initialized! */
 void initNix();
 
 void parseCmdLine(int argc, char * * argv,
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index d3c96ddd6e6..21ab0e6296e 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <map>
 #include <thread>
+#include <dlfcn.h>
 
 
 namespace nix {
@@ -137,4 +138,18 @@ void MaxBuildJobsSetting::set(const std::string & str)
         throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
 }
 
+
+void initPlugins()
+{
+    for (const auto & pluginFile : settings.pluginFiles.get()) {
+        /* handle is purposefully leaked as there may be state in the
+           DSO needed by the action of the plugin. */
+        void *handle =
+            dlopen(pluginFile.c_str(), RTLD_LAZY | RTLD_LOCAL);
+        if (!handle)
+            throw Error(format("could not dynamically open plugin file '%1%': %2%") % pluginFile % dlerror());
+    }
+}
+
+
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 20ac8fe4e9a..508084d08ac 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -367,12 +367,19 @@ public:
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<Paths> pluginFiles{this, {}, "plugin-files",
+        "Plugins to dynamically load at nix initialization time."};
 };
 
 
 // FIXME: don't use a global variable.
 extern Settings settings;
 
+/* This should be called after settings are initialized, but before
+   anything else */
+void initPlugins();
+
 
 extern const string nixVersion;
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 50c46ce6fe9..239356aee8d 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc)
 libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+ifneq ($(OS), FreeBSD)
+ libstore_LDFLAGS += -ldl
+endif
 
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 1581c282c75..99f773451ff 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -232,6 +232,8 @@ void mainWrapped(int argc, char * * argv)
 
     myArgs.parseCmdline(args);
 
+    initPlugins();
+
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 370f216abcc..ec9a7174ecb 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -213,6 +213,9 @@ int main(int argc, char ** argv)
             }
             return true;
         });
+
+        initPlugins();
+
         switch (cmd) {
             case cNone:
                 throw UsageError("no command specified");
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index cc663a96924..37fe22f4813 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -77,6 +77,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         auto profilesDir = settings.nixStateDir + "/profiles";
         if (removeOld) removeOldGenerations(profilesDir);
 
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 861fc2e5cd6..dfb1b8fc5dc 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -44,6 +44,8 @@ int main(int argc, char ** argv)
             return true;
         });
 
+        initPlugins();
+
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index d3a8ebbdda5..890bffa19aa 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -1060,6 +1060,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (stdio) {
             if (getStoreType() == tDaemon) {
                 /* Forward on this connection to the real daemon */
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 016caf6d234..97e66cbd937 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1393,6 +1393,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index e05040a42de..dd262bea091 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -151,6 +151,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index fef3eaa4553..fa7ee254500 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -89,6 +89,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4fc3421c0dd..efef7f15c09 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -1052,6 +1052,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         if (op != opDump && op != opRestore) /* !!! hack */
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 8f6bbe8f51a..bb107ec7d3f 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -92,6 +92,8 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
+    initPlugins();
+
     if (!args.command) args.showHelpAndExit();
 
     Finally f([]() { stopProgressBar(); });
diff --git a/tests/local.mk b/tests/local.mk
index e90b9f7da4a..51bc09dd432 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -22,7 +22,8 @@ nix_tests = \
   run.sh \
   brotli.sh \
   pure-eval.sh \
-  check.sh
+  check.sh \
+  plugins.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
@@ -31,4 +32,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh
+installcheck: $(d)/common.sh $(d)/plugins/plugintest.so
diff --git a/tests/plugins.sh b/tests/plugins.sh
new file mode 100644
index 00000000000..6d18d1da0d1
--- /dev/null
+++ b/tests/plugins.sh
@@ -0,0 +1,7 @@
+source common.sh
+
+set -o pipefail
+
+res=$(nix eval '(builtins.constNull true)' --option plugin-files $PWD/plugins/plugintest.so)
+
+[ "$res"x = "nullx" ]
diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk
new file mode 100644
index 00000000000..a5f19b087c8
--- /dev/null
+++ b/tests/plugins/local.mk
@@ -0,0 +1,9 @@
+libraries += plugintest
+
+plugintest_DIR := $(d)
+
+plugintest_SOURCES := $(d)/plugintest.cc
+
+plugintest_ALLOW_UNDEFINED := 1
+
+plugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc
new file mode 100644
index 00000000000..f788c481432
--- /dev/null
+++ b/tests/plugins/plugintest.cc
@@ -0,0 +1,10 @@
+#include "primops.hh"
+
+using namespace nix;
+
+static void prim_constNull (EvalState & state, const Pos & pos, Value ** args, Value & v)
+{
+    mkNull(v);
+}
+
+static RegisterPrimOp r("constNull", 1, prim_constNull);

From 081f14a169d36243f97263acb41fb108af243619 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 8 Feb 2018 13:00:53 -0500
Subject: [PATCH 0813/2196] Allow using RegisterPrimop to define constants.

This enables plugins to add new constants, as well as new primops.
---
 doc/manual/command-ref/conf-file.xml | 2 +-
 src/libexpr/eval.cc                  | 8 +++++++-
 src/libexpr/eval.hh                  | 2 +-
 src/libexpr/primops.hh               | 3 +++
 tests/plugins.sh                     | 2 +-
 tests/plugins/plugintest.cc          | 4 ++--
 6 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index cede6db3cea..5c4561f66d8 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -750,7 +750,7 @@ builtins.fetchurl {
         files will be dlopened by Nix, allowing them to affect
         execution through static initialization. In particular, these
         plugins may construct static instances of RegisterPrimOp to
-        add new primops to the expression language,
+        add new primops or constants to the expression language,
         RegisterStoreImplementation to add new store implementations,
         and RegisterCommand to add new subcommands to the
         <literal>nix</literal> command. See the constructors for those
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0b0a0f7b179..1f3bbc0a53b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -404,7 +404,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context)
 };
 
 
-void EvalState::addConstant(const string & name, Value & v)
+Value * EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
     *v2 = v;
@@ -412,12 +412,18 @@ void EvalState::addConstant(const string & name, Value & v)
     baseEnv.values[baseEnvDispl++] = v2;
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
+    return v2;
 }
 
 
 Value * EvalState::addPrimOp(const string & name,
     unsigned int arity, PrimOpFun primOp)
 {
+    if (arity == 0) {
+        Value v;
+        primOp(*this, noPos, nullptr, v);
+        return addConstant(name, v);
+    }
     Value * v = allocValue();
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9e3d30d95f4..51905d7e1c6 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -210,7 +210,7 @@ private:
 
     void createBaseEnv();
 
-    void addConstant(const string & name, Value & v);
+    Value * addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
         unsigned int arity, PrimOpFun primOp);
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 39d23b04a5c..31bf3f84f6c 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -9,6 +9,9 @@ struct RegisterPrimOp
 {
     typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
     static PrimOps * primOps;
+    /* You can register a constant by passing an arity of 0. fun
+       will get called during EvalState initialization, so there
+       may be primops not yet added and builtins is not yet sorted. */
     RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
 };
 
diff --git a/tests/plugins.sh b/tests/plugins.sh
index 6d18d1da0d1..23caf04f338 100644
--- a/tests/plugins.sh
+++ b/tests/plugins.sh
@@ -2,6 +2,6 @@ source common.sh
 
 set -o pipefail
 
-res=$(nix eval '(builtins.constNull true)' --option plugin-files $PWD/plugins/plugintest.so)
+res=$(nix eval '(builtins.anotherNull)' --option plugin-files $PWD/plugins/plugintest.so)
 
 [ "$res"x = "nullx" ]
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc
index f788c481432..6b5e6d7cde2 100644
--- a/tests/plugins/plugintest.cc
+++ b/tests/plugins/plugintest.cc
@@ -2,9 +2,9 @@
 
 using namespace nix;
 
-static void prim_constNull (EvalState & state, const Pos & pos, Value ** args, Value & v)
+static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v)
 {
     mkNull(v);
 }
 
-static RegisterPrimOp r("constNull", 1, prim_constNull);
+static RegisterPrimOp r("anotherNull", 0, prim_anotherNull);

From aa02cdc33ceccce35e56d61535fdb2c55f13cc44 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Feb 2018 14:36:38 +0100
Subject: [PATCH 0814/2196] getDefaultSubstituters(): Skip broken substituters

Fixes #1340.
---
 src/libstore/store-api.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7abb300a9bb..4d43ef082d5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -896,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters()
         auto addStore = [&](const std::string & uri) {
             if (done.count(uri)) return;
             done.insert(uri);
-            stores.push_back(openStore(uri));
+            try {
+                stores.push_back(openStore(uri));
+            } catch (Error & e) {
+                printError("warning: %s", e.what());
+            }
         };
 
         for (auto uri : settings.substituters.get())

From 960e9c560e335c65d378805fdb2594e7079a0c45 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Feb 2018 15:05:41 +0100
Subject: [PATCH 0815/2196] nix: Ensure that the user sees errors from
 substituters

---
 src/libstore/build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9f669f7e464..392b494e65e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3688,8 +3688,8 @@ void SubstitutionGoal::tryNext()
         && !sub->isTrusted
         && !info->checkSignatures(worker.store, worker.store.publicKeys))
     {
-        printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
-            % sub->getUri() % storePath);
+        printError("warning: substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), storePath);
         tryNext();
         return;
     }
@@ -3779,7 +3779,7 @@ void SubstitutionGoal::finished()
     try {
         promise.get_future().get();
     } catch (Error & e) {
-        printInfo(e.msg());
+        printError(e.msg());
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;

From 5a082ad15a0f46dc1cbfd8aa5cb3ad9d94b5f178 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 9 Feb 2018 20:57:31 -0600
Subject: [PATCH 0816/2196] configure.ac: check if lzma has MT support, fix deb
 build/etc.

---
 configure.ac               | 2 ++
 src/libutil/compression.cc | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/configure.ac b/configure.ac
index 83b2346d065..c7498fb5240 100644
--- a/configure.ac
+++ b/configure.ac
@@ -175,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
+AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
+  [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
 
 # Look for libbrotli{enc,dec}, optional dependencies
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index ed15761b32a..c509472b384 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -192,6 +192,7 @@ struct XzSink : CompressionSink
     XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
     {
         lzma_ret ret;
+#ifdef HAVE_LZMA_MT
         if (parallel) {
             lzma_mt mt_options = {};
             mt_options.flags = 0;
@@ -208,6 +209,7 @@ struct XzSink : CompressionSink
             ret = lzma_stream_encoder_mt(
                 &strm, &mt_options);
         } else
+#endif
             ret = lzma_easy_encoder(
                 &strm, 6, LZMA_CHECK_CRC64);
 

From 60eca585339b5cca7d04d82c48b9d5feb2d628ae Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Sun, 11 Feb 2018 14:37:50 +0100
Subject: [PATCH 0817/2196] Nix stats: flatten statistics

Flattens the list of statistics as suggested in
https://github.com/NixOS/ofborg/issues/67. This makes it easier to work
with.
---
 src/libexpr/eval.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0b0a0f7b179..11195af7793 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1700,10 +1700,13 @@ void EvalState::printStats()
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
     printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs);
-    printMsg(v, format("  list elements: %1% (%2% bytes)") % nrListElems % bLists);
+    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
+    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
+    printMsg(v, format("  list elements count: %1%") % nrListElems);
+    printMsg(v, format("  list elements bytes: %1%") % bLists);
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated: %1% (%2% bytes)") % nrValues % bValues);
+    printMsg(v, format("  values allocated count: %1%") % nrValues);
+    printMsg(v, format("  values allocated bytes: %1%") % bValues);
     printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);

From a0bdc96726b15b7f529156bccd60d0f8dd5544f3 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sun, 11 Feb 2018 12:47:42 -0600
Subject: [PATCH 0818/2196] compression: print warning if parallel requested
 but not supported

---
 src/libutil/compression.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index c509472b384..0b0ff110253 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -1,6 +1,7 @@
 #include "compression.hh"
 #include "util.hh"
 #include "finally.hh"
+#include "logging.hh"
 
 #include <lzma.h>
 #include <bzlib.h>
@@ -192,8 +193,8 @@ struct XzSink : CompressionSink
     XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
     {
         lzma_ret ret;
-#ifdef HAVE_LZMA_MT
         if (parallel) {
+#ifdef HAVE_LZMA_MT
             lzma_mt mt_options = {};
             mt_options.flags = 0;
             mt_options.timeout = 300; // Using the same setting as the xz cmd line
@@ -209,6 +210,9 @@ struct XzSink : CompressionSink
             ret = lzma_stream_encoder_mt(
                 &strm, &mt_options);
         } else
+#else
+            printMsg(lvlError, "Warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
+        }
 #endif
             ret = lzma_easy_encoder(
                 &strm, 6, LZMA_CHECK_CRC64);
@@ -471,6 +475,9 @@ struct BrotliSink : CompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
+    if (parallel && method != "xz")
+        printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
+
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")

From c6209030c424424ebd51283326d5e5df68a48533 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sun, 11 Feb 2018 13:23:31 -0600
Subject: [PATCH 0819/2196] compression: make parallel sink separate class

---
 src/libutil/compression.cc | 63 ++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 29 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 0b0ff110253..470c925ed7a 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -190,33 +190,9 @@ struct XzSink : CompressionSink
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
-    {
-        lzma_ret ret;
-        if (parallel) {
-#ifdef HAVE_LZMA_MT
-            lzma_mt mt_options = {};
-            mt_options.flags = 0;
-            mt_options.timeout = 300; // Using the same setting as the xz cmd line
-            mt_options.preset = LZMA_PRESET_DEFAULT;
-            mt_options.filters = NULL;
-            mt_options.check = LZMA_CHECK_CRC64;
-            mt_options.threads = lzma_cputhreads();
-            mt_options.block_size = 0;
-            if (mt_options.threads == 0)
-                mt_options.threads = 1;
-            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-            // number of threads.
-            ret = lzma_stream_encoder_mt(
-                &strm, &mt_options);
-        } else
-#else
-            printMsg(lvlError, "Warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
-        }
-#endif
-            ret = lzma_easy_encoder(
-                &strm, 6, LZMA_CHECK_CRC64);
-
+    template <typename F>
+    XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
+        lzma_ret ret = initEncoder();
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
         // FIXME: apply the x86 BCJ filter?
@@ -224,6 +200,9 @@ struct XzSink : CompressionSink
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
     }
+    XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
+        return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
+    }) {}
 
     ~XzSink()
     {
@@ -277,6 +256,27 @@ struct XzSink : CompressionSink
     }
 };
 
+#ifdef HAVE_LZMA_MT
+struct ParallelXzSink : public XzSink
+{
+  ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
+        lzma_mt mt_options = {};
+        mt_options.flags = 0;
+        mt_options.timeout = 300; // Using the same setting as the xz cmd line
+        mt_options.preset = LZMA_PRESET_DEFAULT;
+        mt_options.filters = NULL;
+        mt_options.check = LZMA_CHECK_CRC64;
+        mt_options.threads = lzma_cputhreads();
+        mt_options.block_size = 0;
+        if (mt_options.threads == 0)
+            mt_options.threads = 1;
+        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+        // number of threads.
+        return lzma_stream_encoder_mt(&strm, &mt_options);
+  }) {}
+};
+#endif
+
 struct BzipSink : CompressionSink
 {
     Sink & nextSink;
@@ -475,13 +475,18 @@ struct BrotliSink : CompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
-    if (parallel && method != "xz")
+    if (parallel) {
+#ifdef HAVE_LZMA_MT
+        if (method == "xz")
+            return make_ref<ParallelXzSink>(nextSink);
+#endif
         printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
+    }
 
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink, parallel);
+        return make_ref<XzSink>(nextSink);
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")

From 35fd31770c589f28c569e7868f868a5b558cf83d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Feb 2018 16:42:32 +0100
Subject: [PATCH 0820/2196] toBuildables -> build

---
 src/nix/build.cc        | 2 +-
 src/nix/command.hh      | 2 +-
 src/nix/installables.cc | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix/build.cc b/src/nix/build.cc
index f7c99f12dbb..b4f21b32d78 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -50,7 +50,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
     void run(ref<Store> store) override
     {
-        auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
+        auto buildables = build(store, dryRun ? DryRun : Build, installables);
 
         for (size_t i = 0; i < buildables.size(); ++i) {
             auto & b(buildables[i]);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a7863c49f37..97a6fee7fd2 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -198,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable(
     SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
     bool useDefaultInstallables);
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables);
 
 PathSet toStorePaths(ref<Store> store, RealiseMode mode,
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index c3b06c22eba..a3fdd8a2808 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -253,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable(
     return installables.front();
 }
 
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables)
 {
     if (mode != Build)
@@ -291,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode,
 {
     PathSet outPaths;
 
-    for (auto & b : toBuildables(store, mode, installables))
+    for (auto & b : build(store, mode, installables))
         for (auto & output : b.outputs)
             outPaths.insert(output.second);
 

From 4f09ce7940689887a18d4aa44367d2e6abeaa3cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Feb 2018 16:56:12 +0100
Subject: [PATCH 0821/2196] Fix 'deadlock: trying to re-acquire self-held lock'

This was caused by derivations with 'allowSubstitutes = false'. Such
derivations will be built locally. However, if there is another
SubstitionGoal that has the output of the first derivation in its
closure, then the path will be simultaneously built and substituted.

There was a check to catch this situation (via pathIsLockedByMe()),
but it no longer worked reliably because substitutions are now done in
another thread. (Thus the comment 'It can't happen between here and
the lockPaths() call below because we're not allowing multi-threading'
was no longer valid.)

The fix is to handle the path already being locked in both
SubstitutionGoal and DerivationGoal.
---
 src/libstore/build.cc       | 30 +++++++++++++++++-------------
 src/libstore/local-store.cc |  4 ++--
 src/libstore/pathlocks.cc   |  6 ++++--
 src/libstore/pathlocks.hh   |  6 +-----
 4 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 392b494e65e..cc69ff1c74b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1335,19 +1335,6 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* Check for the possibility that some other goal in this process
-       has locked the output since we checked in haveDerivation().
-       (It can't happen between here and the lockPaths() call below
-       because we're not allowing multi-threading.)  If so, put this
-       goal to sleep until another goal finishes, then try again. */
-    for (auto & i : drv->outputs)
-        if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
-            debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
-                % drvPath % i.second.path);
-            worker.waitForAnyGoal(shared_from_this());
-            return;
-        }
-
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -3739,6 +3726,17 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
+    /* If the store path is already locked (probably by a
+       DerivationGoal), then put this goal to sleep. Note: we don't
+       acquire a lock here since that breaks addToStore(), so below we
+       handle an AlreadyLocked exception from addToStore(). The check
+       here is just an optimisation to prevent having to redo a
+       download due to a locked path. */
+    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
+        worker.waitForAWhile(shared_from_this());
+        return;
+    }
+
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -3778,6 +3776,12 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
+    } catch (AlreadyLocked & e) {
+        /* Probably a DerivationGoal is already building this store
+           path. Sleep for a while and try again. */
+        state = &SubstitutionGoal::init;
+        worker.waitForAWhile(shared_from_this());
+        return;
     } catch (Error & e) {
         printError(e.msg());
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7afecc1cfc6..8a79fc7235f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -992,8 +992,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
-        if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
+        static auto locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+        if (!locksHeld.count(info.path))
             outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 587f2959885..08d1efdbeb0 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths,
 
         {
             auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath))
-                throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            if (lockedPaths->count(lockPath)) {
+                if (!wait) return false;
+                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            }
             lockedPaths->insert(lockPath);
         }
 
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 2a7de611446..db51f950a32 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -2,10 +2,8 @@
 
 #include "util.hh"
 
-
 namespace nix {
 
-
 /* Open (possibly create) a lock file and return the file descriptor.
    -1 is returned if create is false and the lock could not be opened
    because it doesn't exist.  Any other error throws an exception. */
@@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
+MakeError(AlreadyLocked, Error);
 
 class PathLocks
 {
@@ -38,9 +37,6 @@ public:
     void setDeletion(bool deletePaths);
 };
 
-
-// FIXME: not thread-safe!
 bool pathIsLockedByMe(const Path & path);
 
-
 }

From 9bcb4d2dd99ba7509c27479eecf1e7ac88244fa8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 12 Feb 2018 22:48:55 +0100
Subject: [PATCH 0822/2196] Fix hang in build-remote

---
 src/build-remote/build-remote.cc | 2 +-
 src/libstore/local-store.cc      | 1 -
 src/libstore/local-store.hh      | 3 +++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index df579729af2..c6e75e8cc70 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -241,7 +241,7 @@ int main (int argc, char * * argv)
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
+            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
         }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 8a79fc7235f..4afe51ea91e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -992,7 +992,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        static auto locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
         if (!locksHeld.count(info.path))
             outputLock.lockPaths({realPath});
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 30bef3a799d..bbd50e1c145 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -104,6 +104,9 @@ private:
 
 public:
 
+    // Hack for build-remote.cc.
+    PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+
     /* Initialise the local store, upgrading the schema if
        necessary. */
     LocalStore(const Params & params);

From 6eb1040e909ab83fbc03983724d9c6ec223c4495 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 13 Feb 2018 08:16:32 -0500
Subject: [PATCH 0823/2196] Allow includes from nix.conf

---
 doc/manual/command-ref/conf-file.xml |  7 ++++++-
 src/libutil/config.cc                | 26 +++++++++++++++++++++++++-
 tests/init.sh                        |  5 +++++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 5c4561f66d8..8ed614963a4 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -40,7 +40,12 @@
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =
-<replaceable>value</replaceable></literal> pairs, one per line.
+<replaceable>value</replaceable></literal> pairs, one per line. Other
+files can be included with a line like <literal>include
+<replaceable>path</replaceable></literal>, where
+<replaceable>path</replaceable> is interpreted relative to the current
+conf file and a missing file is an error unless
+<literal>!include</literal> is used instead.
 Comments start with a <literal>#</literal> character.  Here is an
 example configuration file:</para>
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index d46ca65a386..0e502769edf 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -80,7 +80,31 @@ void Config::applyConfigFile(const Path & path, bool fatal)
             vector<string> tokens = tokenizeString<vector<string> >(line);
             if (tokens.empty()) continue;
 
-            if (tokens.size() < 2 || tokens[1] != "=")
+            if (tokens.size() < 2)
+                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+
+            auto include = false;
+            auto ignoreMissing = false;
+            if (tokens[0] == "include")
+                include = true;
+            else if (tokens[0] == "!include") {
+                include = true;
+                ignoreMissing = true;
+            }
+
+            if (include) {
+                if (tokens.size() != 2)
+                    throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+                auto p = absPath(tokens[1], dirOf(path));
+                if (pathExists(p)) {
+                    applyConfigFile(p, fatal);
+                } else if (!ignoreMissing) {
+                    throw Error("file '%1%' included from '%2%' not found", p, path);
+                }
+                continue;
+            }
+
+            if (tokens[1] != "=")
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
             string name = tokens[0];
diff --git a/tests/init.sh b/tests/init.sh
index 41cca047d8f..e5353598bcc 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -16,7 +16,12 @@ mkdir "$NIX_CONF_DIR"
 cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
+include nix.conf.extra
+EOF
+
+cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF
 fsync-metadata = false
+!include nix.conf.extra.not-there
 EOF
 
 # Initialise the database.

From 78ac3eb4ebc04e180abeb01bcc6f9d4103522eab Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 13 Feb 2018 11:05:25 +0000
Subject: [PATCH 0824/2196] Fix #1762

nix-store --export, nix-store --dump, and nix dump-path would previously
fail silently if writing the data out failed, because
 a) FdSink::write ignored exceptions, and
 b) the commands relied on FdSink's destructor, which ignores
    exceptions, to flush the data out.

This could cause rather opaque issues with installing nixos, because
nix-store --export would happily proceed even if it couldn't write its
data out (e.g. if nix-store --import on the other side of the pipe
failed).

This commit adds tests that expose these issues in the nix-store
commands, and fixes them for all three.
---
 src/libutil/serialise.cc   | 3 ++-
 src/nix-store/nix-store.cc | 2 ++
 src/nix/dump-path.cc       | 1 +
 tests/export.sh            | 5 +++++
 tests/nar-access.sh        | 6 ++++++
 5 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 950e6362a24..9e2a502afaf 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -67,7 +67,8 @@ void FdSink::write(const unsigned char * data, size_t len)
     try {
         writeFull(fd, data, len);
     } catch (SysError & e) {
-        _good = true;
+        _good = false;
+        throw;
     }
 }
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4fc3421c0dd..4bea1316134 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -631,6 +631,7 @@ static void opDump(Strings opFlags, Strings opArgs)
     FdSink sink(STDOUT_FILENO);
     string path = *opArgs.begin();
     dumpPath(path, sink);
+    sink.flush();
 }
 
 
@@ -656,6 +657,7 @@ static void opExport(Strings opFlags, Strings opArgs)
 
     FdSink sink(STDOUT_FILENO);
     store->exportPaths(opArgs, sink);
+    sink.flush();
 }
 
 
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 1a1866437b0..f411c0cb7c8 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -29,6 +29,7 @@ struct CmdDumpPath : StorePathCommand
     {
         FdSink sink(STDOUT_FILENO);
         store->narFromPath(storePath, sink);
+        sink.flush();
     }
 };
 
diff --git a/tests/export.sh b/tests/export.sh
index ec7560f1972..2238539bcca 100644
--- a/tests/export.sh
+++ b/tests/export.sh
@@ -8,6 +8,11 @@ nix-store --export $outPath > $TEST_ROOT/exp
 
 nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all
 
+if nix-store --export $outPath >/dev/full ; then
+    echo "exporting to a bad file descriptor should fail"
+    exit 1
+fi
+
 
 clearStore
 
diff --git a/tests/nar-access.sh b/tests/nar-access.sh
index bd849cbfab1..553d6ca89d7 100644
--- a/tests/nar-access.sh
+++ b/tests/nar-access.sh
@@ -36,3 +36,9 @@ diff -u baz.cat-nar $storePath/foo/baz
 # Test missing files.
 nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
 nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist'
+
+# Test failure to dump.
+if nix-store --dump $storePath >/dev/full ; then
+    echo "dumping to /dev/full should fail"
+    exit -1
+fi

From b8739f2fb33a28a250cd2053c013b977a3f096e8 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 13 Feb 2018 07:51:52 -0500
Subject: [PATCH 0825/2196] Enable specifying directories in plugin-files.

---
 doc/manual/command-ref/conf-file.xml |  4 ++++
 src/libstore/globals.cc              | 24 ++++++++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 5c4561f66d8..2b7a69a0c59 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -764,6 +764,10 @@ builtins.fetchurl {
         should not be linked to any Nix libs directly, as those will
         be available already at load time.
       </para>
+      <para>
+	If an entry in the list is a directory, all files in the
+	directory are loaded as plugins (non-recursively).
+      </para>
     </listitem>
 
   </varlistentry>
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 21ab0e6296e..c6b508cbe82 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -142,12 +142,24 @@ void MaxBuildJobsSetting::set(const std::string & str)
 void initPlugins()
 {
     for (const auto & pluginFile : settings.pluginFiles.get()) {
-        /* handle is purposefully leaked as there may be state in the
-           DSO needed by the action of the plugin. */
-        void *handle =
-            dlopen(pluginFile.c_str(), RTLD_LAZY | RTLD_LOCAL);
-        if (!handle)
-            throw Error(format("could not dynamically open plugin file '%1%': %2%") % pluginFile % dlerror());
+        Paths pluginFiles;
+        try {
+            auto ents = readDirectory(pluginFile);
+            for (const auto & ent : ents)
+                pluginFiles.emplace_back(pluginFile + "/" + ent.name);
+        } catch (SysError & e) {
+            if (e.errNo != ENOTDIR)
+                throw;
+            pluginFiles.emplace_back(pluginFile);
+        }
+        for (const auto & file : pluginFiles) {
+            /* handle is purposefully leaked as there may be state in the
+               DSO needed by the action of the plugin. */
+            void *handle =
+                dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
+            if (!handle)
+                throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror());
+        }
     }
 }
 

From 3fe9767dd33499c2560d209dc13a01f5fcead1f0 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 13 Feb 2018 12:49:14 -0500
Subject: [PATCH 0826/2196] Fix plugin tests on darwin

---
 tests/local.mk         |  2 +-
 tests/plugins.sh       |  2 +-
 tests/plugins/local.mk | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/local.mk b/tests/local.mk
index 51bc09dd432..2a4832e1304 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -32,4 +32,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh $(d)/plugins/plugintest.so
+installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT)
diff --git a/tests/plugins.sh b/tests/plugins.sh
index 23caf04f338..0fad4f773a2 100644
--- a/tests/plugins.sh
+++ b/tests/plugins.sh
@@ -2,6 +2,6 @@ source common.sh
 
 set -o pipefail
 
-res=$(nix eval '(builtins.anotherNull)' --option plugin-files $PWD/plugins/plugintest.so)
+res=$(nix eval '(builtins.anotherNull)' --option plugin-files $PWD/plugins/libplugintest*)
 
 [ "$res"x = "nullx" ]
diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk
index a5f19b087c8..1d2bac052fd 100644
--- a/tests/plugins/local.mk
+++ b/tests/plugins/local.mk
@@ -1,9 +1,9 @@
-libraries += plugintest
+libraries += libplugintest
 
-plugintest_DIR := $(d)
+libplugintest_DIR := $(d)
 
-plugintest_SOURCES := $(d)/plugintest.cc
+libplugintest_SOURCES := $(d)/plugintest.cc
 
-plugintest_ALLOW_UNDEFINED := 1
+libplugintest_ALLOW_UNDEFINED := 1
 
-plugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
+libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1

From de4934ab3b26aa851b7044e9884102cc054dc092 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 13 Feb 2018 14:43:32 -0500
Subject: [PATCH 0827/2196] Allow plugins to define new settings.

---
 doc/manual/command-ref/conf-file.xml |  7 +++--
 src/libstore/globals.cc              | 16 +++++++++++
 src/libstore/globals.hh              |  7 +++++
 src/libstore/store-api.cc            |  2 +-
 src/libutil/config.cc                | 42 ++++++++++++++--------------
 src/libutil/config.hh                |  8 +++---
 tests/plugins.sh                     |  2 +-
 tests/plugins/plugintest.cc          | 13 +++++++--
 8 files changed, 65 insertions(+), 32 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 42906ddff5e..c14a4d206b8 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -757,9 +757,10 @@ builtins.fetchurl {
         plugins may construct static instances of RegisterPrimOp to
         add new primops or constants to the expression language,
         RegisterStoreImplementation to add new store implementations,
-        and RegisterCommand to add new subcommands to the
-        <literal>nix</literal> command. See the constructors for those
-        types for more details.
+        RegisterCommand to add new subcommands to the
+        <literal>nix</literal> command, and RegisterSetting to add new
+        nix config settings. See the constructors for those types for
+        more details.
       </para>
       <para>
 	Since these files are loaded into the same address space as
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index c6b508cbe82..c5a4536ef2f 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -161,6 +161,22 @@ void initPlugins()
                 throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror());
         }
     }
+    /* We handle settings registrations here, since plugins can add settings */
+    if (RegisterSetting::settingRegistrations) {
+        for (auto & registration : *RegisterSetting::settingRegistrations)
+            settings.addSetting(registration);
+        delete RegisterSetting::settingRegistrations;
+    }
+    settings.handleUnknownSettings();
+}
+
+RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations;
+
+RegisterSetting::RegisterSetting(AbstractSetting * s)
+{
+    if (!settingRegistrations)
+        settingRegistrations = new SettingRegistrations;
+    settingRegistrations->emplace_back(s);
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 508084d08ac..1d019aab9d2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -383,5 +383,12 @@ void initPlugins();
 
 extern const string nixVersion;
 
+struct RegisterSetting
+{
+    typedef std::vector<AbstractSetting *> SettingRegistrations;
+    static SettingRegistrations * settingRegistrations;
+    RegisterSetting(AbstractSetting * s);
+};
+
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 4d43ef082d5..8830edcc344 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -839,7 +839,7 @@ ref<Store> openStore(const std::string & uri_,
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->warnUnknownSettings();
+            store->handleUnknownSettings();
             return ref<Store>(store);
         }
     }
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 0e502769edf..cdd1e0a15aa 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -7,10 +7,12 @@ namespace nix {
 void Config::set(const std::string & name, const std::string & value)
 {
     auto i = _settings.find(name);
-    if (i == _settings.end())
-        throw UsageError("unknown setting '%s'", name);
-    i->second.setting->set(value);
-    i->second.setting->overriden = true;
+    if (i == _settings.end()) {
+        extras.emplace(name, value);
+    } else {
+        i->second.setting->set(value);
+        i->second.setting->overriden = true;
+    }
 }
 
 void Config::addSetting(AbstractSetting * setting)
@@ -21,34 +23,37 @@ void Config::addSetting(AbstractSetting * setting)
 
     bool set = false;
 
-    auto i = initials.find(setting->name);
-    if (i != initials.end()) {
+    auto i = extras.find(setting->name);
+    if (i != extras.end()) {
         setting->set(i->second);
         setting->overriden = true;
-        initials.erase(i);
+        extras.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = initials.find(alias);
-        if (i != initials.end()) {
+        auto i = extras.find(alias);
+        if (i != extras.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
                 setting->set(i->second);
                 setting->overriden = true;
-                initials.erase(i);
+                extras.erase(i);
                 set = true;
             }
         }
     }
 }
 
-void Config::warnUnknownSettings()
+void Config::handleUnknownSettings(bool fatal)
 {
-    for (auto & i : initials)
-        warn("unknown setting '%s'", i.first);
+    for (auto & s : extras)
+        if (fatal)
+            throw UsageError("unknown setting '%s%'", s.first);
+        else
+            warn("unknown setting '%s'", s.first);
 }
 
 StringMap Config::getSettings(bool overridenOnly)
@@ -60,7 +65,7 @@ StringMap Config::getSettings(bool overridenOnly)
     return res;
 }
 
-void Config::applyConfigFile(const Path & path, bool fatal)
+void Config::applyConfigFile(const Path & path)
 {
     try {
         string contents = readFile(path);
@@ -97,7 +102,7 @@ void Config::applyConfigFile(const Path & path, bool fatal)
                     throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
                 auto p = absPath(tokens[1], dirOf(path));
                 if (pathExists(p)) {
-                    applyConfigFile(p, fatal);
+                    applyConfigFile(p);
                 } else if (!ignoreMissing) {
                     throw Error("file '%1%' included from '%2%' not found", p, path);
                 }
@@ -112,12 +117,7 @@ void Config::applyConfigFile(const Path & path, bool fatal)
             vector<string>::iterator i = tokens.begin();
             advance(i, 2);
 
-            try {
-                set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
-            } catch (UsageError & e) {
-                if (fatal) throw;
-                warn("in configuration file '%s': %s", path, e.what());
-            }
+            set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
         };
     } catch (SysError &) { }
 }
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 9a32af528ec..c6783e13c2b 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -48,25 +48,25 @@ private:
 
     Settings _settings;
 
-    StringMap initials;
+    StringMap extras;
 
 public:
 
     Config(const StringMap & initials)
-        : initials(initials)
+        : extras(initials)
     { }
 
     void set(const std::string & name, const std::string & value);
 
     void addSetting(AbstractSetting * setting);
 
-    void warnUnknownSettings();
+    void handleUnknownSettings(bool fatal = false);
 
     StringMap getSettings(bool overridenOnly = false);
 
     const Settings & _getSettings() { return _settings; }
 
-    void applyConfigFile(const Path & path, bool fatal = false);
+    void applyConfigFile(const Path & path);
 
     void resetOverriden();
 
diff --git a/tests/plugins.sh b/tests/plugins.sh
index 0fad4f773a2..4b1baeddce3 100644
--- a/tests/plugins.sh
+++ b/tests/plugins.sh
@@ -2,6 +2,6 @@ source common.sh
 
 set -o pipefail
 
-res=$(nix eval '(builtins.anotherNull)' --option plugin-files $PWD/plugins/libplugintest*)
+res=$(nix eval '(builtins.anotherNull)' --option setting-set true --option plugin-files $PWD/plugins/libplugintest*)
 
 [ "$res"x = "nullx" ]
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc
index 6b5e6d7cde2..8da15ebabd7 100644
--- a/tests/plugins/plugintest.cc
+++ b/tests/plugins/plugintest.cc
@@ -1,10 +1,19 @@
+#include "globals.hh"
 #include "primops.hh"
 
 using namespace nix;
 
+static BaseSetting<bool> settingSet{false, "setting-set",
+        "Whether the plugin-defined setting was set"};
+
+static RegisterSetting rs(&settingSet);
+
 static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v)
 {
-    mkNull(v);
+    if (settingSet)
+        mkNull(v);
+    else
+        mkBool(v, false);
 }
 
-static RegisterPrimOp r("anotherNull", 0, prim_anotherNull);
+static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull);

From b095c06139fa267e6050e4c95208c627cc6251b8 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 13 Feb 2018 18:28:27 -0500
Subject: [PATCH 0828/2196] Add splitVersion primop.

Fixes #1868.
---
 doc/manual/expressions/builtins.xml   | 11 +++++++++++
 src/libexpr/names.cc                  |  2 +-
 src/libexpr/names.hh                  |  2 ++
 src/libexpr/primops.cc                | 21 +++++++++++++++++++++
 tests/lang/eval-okay-splitversion.exp |  1 +
 tests/lang/eval-okay-splitversion.nix |  1 +
 6 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 tests/lang/eval-okay-splitversion.exp
 create mode 100644 tests/lang/eval-okay-splitversion.nix

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 81770bcf629..8a32ed8b5c9 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -126,6 +126,17 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.splitVersion</function>
+  <replaceable>s</replaceable></term>
+
+    <listitem><para>Split a string representing a version into its
+    components, by the same version splitting logic underlying the
+    version comparison in <link linkend="ssec-version-comparisons">
+    <command>nix-env -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><function>builtins.concatLists</function>
   <replaceable>lists</replaceable></term>
 
diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc
index 6d78d211612..382088c7887 100644
--- a/src/libexpr/names.cc
+++ b/src/libexpr/names.cc
@@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n)
 }
 
 
-static string nextComponent(string::const_iterator & p,
+string nextComponent(string::const_iterator & p,
     const string::const_iterator end)
 {
     /* Skip any dots and dashes (component separators). */
diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh
index 9667fc96fd0..13c3093e77b 100644
--- a/src/libexpr/names.hh
+++ b/src/libexpr/names.hh
@@ -24,6 +24,8 @@ private:
 
 typedef list<DrvName> DrvNames;
 
+string nextComponent(string::const_iterator & p,
+    const string::const_iterator end);
 int compareVersions(const string & v1, const string & v2);
 DrvNames drvNamesFromArgs(const Strings & opArgs);
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 466fd13e869..ca97b2b28bb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1961,6 +1961,26 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
 }
 
 
+static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string version = state.forceStringNoCtx(*args[0], pos);
+    auto iter = version.cbegin();
+    Strings components;
+    while (iter != version.cend()) {
+        auto component = nextComponent(iter, version.cend());
+        if (component.empty())
+            break;
+        components.emplace_back(std::move(component));
+    }
+    state.mkList(v, components.size());
+    unsigned int n = 0;
+    for (auto & component : components) {
+        auto listElem = v.listElems()[n++] = state.allocValue();
+        mkString(*listElem, std::move(component));
+    }
+}
+
+
 /*************************************************************
  * Networking
  *************************************************************/
@@ -2196,6 +2216,7 @@ void EvalState::createBaseEnv()
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);
     addPrimOp("__compareVersions", 2, prim_compareVersions);
+    addPrimOp("__splitVersion", 1, prim_splitVersion);
 
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/lang/eval-okay-splitversion.exp
new file mode 100644
index 00000000000..153ceb8186a
--- /dev/null
+++ b/tests/lang/eval-okay-splitversion.exp
@@ -0,0 +1 @@
+[ "1" "2" "3" ]
diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/lang/eval-okay-splitversion.nix
new file mode 100644
index 00000000000..9e5c99d2e7f
--- /dev/null
+++ b/tests/lang/eval-okay-splitversion.nix
@@ -0,0 +1 @@
+builtins.splitVersion "1.2.3"

From ac973a6d3c0ff2f505dece8e9f1508c6f77553a5 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 14 Feb 2018 12:35:07 -0500
Subject: [PATCH 0829/2196] monitor-fds: Fix on macOS.

Fixes #1871.
---
 src/libutil/monitor-fd.hh | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh
index e0ec66c0180..5ee0b88ef50 100644
--- a/src/libutil/monitor-fd.hh
+++ b/src/libutil/monitor-fd.hh
@@ -21,13 +21,29 @@ public:
     MonitorFdHup(int fd)
     {
         thread = std::thread([fd]() {
-            /* Wait indefinitely until a POLLHUP occurs. */
-            struct pollfd fds[1];
-            fds[0].fd = fd;
-            fds[0].events = 0;
-            if (poll(fds, 1, -1) == -1) abort(); // can't happen
-            assert(fds[0].revents & POLLHUP);
-            triggerInterrupt();
+            while (true) {
+              /* Wait indefinitely until a POLLHUP occurs. */
+              struct pollfd fds[1];
+              fds[0].fd = fd;
+              /* This shouldn't be necessary, but macOS doesn't seem to
+                 like a zeroed out events field.
+                 See rdar://37537852.
+              */
+              fds[0].events = POLLHUP;
+              auto count = poll(fds, 1, -1);
+              if (count == -1) abort(); // can't happen
+              /* This shouldn't happen, but can on macOS due to a bug.
+                 See rdar://37550628.
+
+                 This may eventually need a delay or further
+                 coordination with the main thread if spinning proves
+                 too harmful.
+               */
+              if (count == 0) continue;
+              assert(fds[0].revents & POLLHUP);
+              triggerInterrupt();
+              break;
+            }
         });
     };
 

From 8f186722a97882ba41330684f37bfe8b8637eba8 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 14 Feb 2018 16:05:55 -0600
Subject: [PATCH 0830/2196] =?UTF-8?q?Set=20backup=20MANPATH=20in=20case=20?=
 =?UTF-8?q?man=20path=20isn=E2=80=99t=20set=20correctly.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously, this would fail at startup for non-NixOS installs:

nix-env --help

The fix for this is to just use "nixManDir" as the value for MANPATH
when spawning "man".

To test this, I’m using the following:

$ nix-build release.nix -A build
$ MANPATH= ./result/bin/nix-env --help

Fixes #1627
---
 src/libmain/shared.cc   | 1 +
 src/libstore/globals.cc | 1 +
 src/libstore/globals.hh | 3 +++
 src/libstore/local.mk   | 1 +
 4 files changed, 6 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 90a4867163d..7d888202bbf 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -262,6 +262,7 @@ void printVersion(const string & programName)
 void showManPage(const string & name)
 {
     restoreSignals();
+    setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), NULL);
     throw SysError(format("command 'man %1%' failed") % name.c_str());
 }
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index c6b508cbe82..247040606fb 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -38,6 +38,7 @@ Settings::Settings()
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
+    , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 508084d08ac..fd3fbec9f01 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -82,6 +82,9 @@ public:
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
+    /* The directory where the man pages are stored. */
+    Path nixManDir;
+
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 239356aee8d..c7ac534e238 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -38,6 +38,7 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
+ -DNIX_MAN_DIR=\"$(mandir)\" \
  -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 

From f67a7007a230c84015793794277c0449e682ab54 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Tue, 13 Feb 2018 01:32:29 +0200
Subject: [PATCH 0831/2196] libexpr: Pre-reserve space in string in
 unescapeStr()

Avoids some malloc() traffic.
---
 src/libexpr/lexer.l | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 28a0a6a8789..e5e01fb5831 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -49,9 +49,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-static Expr * unescapeStr(SymbolTable & symbols, const char * s)
+static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 {
     string t;
+    t.reserve(length);
     char c;
     while ((c = *s++)) {
         if (c == '\\') {
@@ -150,7 +151,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->e = unescapeStr(data->symbols, yytext);
+                yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
@@ -178,7 +179,7 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\. {
-                   yylval->e = unescapeStr(data->symbols, yytext + 2);
+                   yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }

From b8bed7da14b26dcc328075522842dd16aa71b434 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Fri, 16 Feb 2018 05:14:35 +0200
Subject: [PATCH 0832/2196] libexpr: Optimize prim_attrNames a bit

Instead of having lexicographicOrder() create a temporary sorted array
of Attr*:s and copying attr names from that, copy the attr names
first and then sort that.
---
 src/libexpr/primops.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ca97b2b28bb..89e984d2e58 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1138,8 +1138,11 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
     state.mkList(v, args[0]->attrs->size());
 
     size_t n = 0;
-    for (auto & i : args[0]->attrs->lexicographicOrder())
-        mkString(*(v.listElems()[n++] = state.allocValue()), i->name);
+    for (auto & i : *args[0]->attrs)
+        mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
+
+    std::sort(v.listElems(), v.listElems() + n,
+              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
 }
 
 

From 0845cdf9443a6b304c1bcec304a462ae4995c744 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Tue, 13 Feb 2018 05:00:17 +0200
Subject: [PATCH 0833/2196] libexpr: Rely on Boehm returning zeroed memory in
 EvalState::allocEnv()

Boehm guarantees that memory returned by GC_malloc() is zeroed, so take
advantage of that.
---
 src/libexpr/attr-set.cc | 3 ++-
 src/libexpr/eval.cc     | 7 +++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index 910428c0268..b284daa3c2f 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -7,13 +7,14 @@
 namespace nix {
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
 static void * allocBytes(size_t n)
 {
     void * p;
 #if HAVE_BOEHMGC
     p = GC_malloc(n);
 #else
-    p = malloc(n);
+    p = calloc(n, 1);
 #endif
     if (!p) throw std::bad_alloc();
     return p;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b94bc597b01..48542d8e500 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -43,13 +43,14 @@ static char * dupString(const char * s)
 }
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
 static void * allocBytes(size_t n)
 {
     void * p;
 #if HAVE_BOEHMGC
     p = GC_malloc(n);
 #else
-    p = malloc(n);
+    p = calloc(n, 1);
 #endif
     if (!p) throw std::bad_alloc();
     return p;
@@ -582,9 +583,7 @@ Env & EvalState::allocEnv(unsigned int size)
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
     env->size = size;
 
-    /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */
-    for (unsigned i = 0; i < size; ++i)
-        env->values[i] = 0;
+    /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
 }

From 7e0360504d1a964ad5bd0da996045bc3868d0d7d Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Sat, 4 Mar 2017 15:24:06 +0200
Subject: [PATCH 0834/2196] libexpr: Optimize prim_derivationStrict by using
 more symbol comparisons

---
 src/libexpr/eval.cc    |  4 ++++
 src/libexpr/eval.hh    |  3 ++-
 src/libexpr/primops.cc | 18 +++++++++---------
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 48542d8e500..2144d345257 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -294,6 +294,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sWrong(symbols.create("wrong"))
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
     , sBuilder(symbols.create("builder"))
+    , sArgs(symbols.create("args"))
+    , sOutputHash(symbols.create("outputHash"))
+    , sOutputHashAlgo(symbols.create("outputHashAlgo"))
+    , sOutputHashMode(symbols.create("outputHashMode"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 51905d7e1c6..9d8799b7906 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -69,7 +69,8 @@ public:
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs, sBuilder;
+        sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sOutputHash, sOutputHashAlgo, sOutputHashMode;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 89e984d2e58..317623b22f7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -589,7 +589,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            if (key == "args") {
+            if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
                 for (unsigned int n = 0; n < i->value->listSize(); ++n) {
                     string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@@ -614,13 +614,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
                     else if (i->name == state.sName)
                         drvName = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHash")
+                    else if (i->name == state.sOutputHash)
                         outputHash = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHashAlgo")
+                    else if (i->name == state.sOutputHashAlgo)
                         outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (key == "outputHashMode")
+                    else if (i->name == state.sOutputHashMode)
                         handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
-                    else if (key == "outputs") {
+                    else if (i->name == state.sOutputs) {
                         /* Require ‘outputs’ to be a list of strings. */
                         state.forceList(*i->value, posDrvName);
                         Strings ss;
@@ -638,10 +638,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         drvName = s;
                         printMsg(lvlVomit, format("derivation name is '%1%'") % drvName);
                     }
-                    else if (key == "outputHash") outputHash = s;
-                    else if (key == "outputHashAlgo") outputHashAlgo = s;
-                    else if (key == "outputHashMode") handleHashMode(s);
-                    else if (key == "outputs")
+                    else if (i->name == state.sOutputHash) outputHash = s;
+                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
+                    else if (i->name == state.sOutputHashMode) handleHashMode(s);
+                    else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
                 }
 

From 66eeff33456d544e5852e580d8cac21f0c38c11f Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Sat, 17 Feb 2018 16:51:10 +0200
Subject: [PATCH 0835/2196] libexpr: Remove unnecessary drvName assignment in
 prim_derivationStrict

drvName is already assigned to the same value right at the start of the
function.
---
 src/libexpr/primops.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 317623b22f7..f6c34d52576 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -612,8 +612,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         drv.builder = state.forceString(*i->value, context, posDrvName);
                     else if (i->name == state.sSystem)
                         drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
-                    else if (i->name == state.sName)
-                        drvName = state.forceStringNoCtx(*i->value, posDrvName);
                     else if (i->name == state.sOutputHash)
                         outputHash = state.forceStringNoCtx(*i->value, posDrvName);
                     else if (i->name == state.sOutputHashAlgo)
@@ -634,10 +632,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     drv.env.emplace(key, s);
                     if (i->name == state.sBuilder) drv.builder = s;
                     else if (i->name == state.sSystem) drv.platform = s;
-                    else if (i->name == state.sName) {
-                        drvName = s;
-                        printMsg(lvlVomit, format("derivation name is '%1%'") % drvName);
-                    }
                     else if (i->name == state.sOutputHash) outputHash = s;
                     else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
                     else if (i->name == state.sOutputHashMode) handleHashMode(s);

From 37264ed0ad898cdd6880de8ce6e5dda7977eed5f Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Fri, 16 Feb 2018 05:13:39 +0200
Subject: [PATCH 0836/2196] libexpr: Avoid an unnecessary string copy in
 prim_derivationStrict

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index f6c34d52576..a800d24290a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -553,7 +553,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     for (auto & i : args[0]->attrs->lexicographicOrder()) {
         if (i->name == state.sIgnoreNulls) continue;
-        string key = i->name;
+        const string & key = i->name;
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {

From 690ac7c90b5bf3c599e210c53365c7d229c8b0ff Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 18 Feb 2018 02:35:01 -0500
Subject: [PATCH 0837/2196] configure: Add a flag to disable seccomp.

This is needed for new arches where libseccomp support doesn't exist
yet.

Fixes #1878.
---
 Makefile.config.in    |  1 +
 configure.ac          | 16 ++++++++++++++--
 src/libstore/build.cc |  4 +++-
 src/libstore/local.mk |  2 +-
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index fab82194656..a9785dc7395 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -7,6 +7,7 @@ ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
 HAVE_BROTLI = @HAVE_BROTLI@
+HAVE_SECCOMP = @HAVE_SECCOMP@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
diff --git a/configure.ac b/configure.ac
index 83b2346d065..14f742cf3ff 100644
--- a/configure.ac
+++ b/configure.ac
@@ -186,9 +186,21 @@ AC_SUBST(HAVE_BROTLI, [$have_brotli])
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
-  PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
-                    [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+  AC_ARG_ENABLE([seccomp-sandboxing],
+                AC_HELP_STRING([--disable-seccomp-sandboxing],
+                               [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)]
+                              ))
+  if test "x$enable_seccomp_sandboxing" != "xno"; then
+    PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
+                      [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+    have_seccomp=1
+  else
+    have_seccomp=
+  fi
+else
+  have_seccomp=
 fi
+AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 
 # Look for aws-cpp-sdk-s3.
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cc69ff1c74b..9b7abaa3d1e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -49,7 +49,9 @@
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/syscall.h>
+#if HAVE_SECCOMP
 #include <seccomp.h>
+#endif
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
@@ -2469,7 +2471,7 @@ void DerivationGoal::chownToBuilder(const Path & path)
 
 void setupSeccomp()
 {
-#if __linux__
+#if __linux__ && HAVE_SECCOMP
     if (!settings.filterSyscalls) return;
 
     scmp_filter_ctx ctx;
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index c7ac534e238..e11efa5c2b5 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -25,7 +25,7 @@ ifeq ($(OS), SunOS)
 	libstore_LDFLAGS += -lsocket
 endif
 
-ifeq ($(OS), Linux)
+ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 

From ed73d40c3b19dc0581bbf28ef29aad50cab3aaf2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Feb 2018 14:00:34 +0100
Subject: [PATCH 0838/2196] Config::handleUnknownSettings(): Remove unused
 'fatal' argument

---
 src/libutil/config.cc | 7 ++-----
 src/libutil/config.hh | 2 +-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index cdd1e0a15aa..ce6858f0d65 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -47,13 +47,10 @@ void Config::addSetting(AbstractSetting * setting)
     }
 }
 
-void Config::handleUnknownSettings(bool fatal)
+void Config::handleUnknownSettings()
 {
     for (auto & s : extras)
-        if (fatal)
-            throw UsageError("unknown setting '%s%'", s.first);
-        else
-            warn("unknown setting '%s'", s.first);
+        warn("unknown setting '%s'", s.first);
 }
 
 StringMap Config::getSettings(bool overridenOnly)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index c6783e13c2b..d2e7faf1743 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -60,7 +60,7 @@ public:
 
     void addSetting(AbstractSetting * setting);
 
-    void handleUnknownSettings(bool fatal = false);
+    void handleUnknownSettings();
 
     StringMap getSettings(bool overridenOnly = false);
 

From e59a8a63e10992834a48e2b43c854c2e4f990dbe Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 19 Feb 2018 09:56:24 -0500
Subject: [PATCH 0839/2196] Don't silently succeed seccomp setup when
 !HAVE_SECCOMP.

Running Nix with build users without seccomp on Linux is dangerous,
and administrators should very explicitly opt-in to it.
---
 src/libstore/build.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9b7abaa3d1e..30f22833fa7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2471,9 +2471,9 @@ void DerivationGoal::chownToBuilder(const Path & path)
 
 void setupSeccomp()
 {
-#if __linux__ && HAVE_SECCOMP
+#if __linux__
     if (!settings.filterSyscalls) return;
-
+#if HAVE_SECCOMP
     scmp_filter_ctx ctx;
 
     if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
@@ -2519,6 +2519,11 @@ void setupSeccomp()
 
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
+#else
+    throw Error("%s\n%s",
+        "seccomp is not supported on this platform"
+        "you can avoid this by setting the filter-syscalls option to false, but note that untrusted builds can then create setuid binaries!");
+#endif
 #endif
 }
 

From a6c0b773b72d4e30690e01f1f1dcffc28f2d9ea1 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 19 Feb 2018 12:13:51 -0600
Subject: [PATCH 0840/2196] configure.ac: define HAVE_SECCOMP macro when using
 seccomp, fix build/tests

Happily the failing tests should prevent anyone from using such a Nix
in situations where they expect sandboxing to be on,
which would otherwise be a risk.
---
 configure.ac | 1 +
 1 file changed, 1 insertion(+)

diff --git a/configure.ac b/configure.ac
index 4102f32166f..54322d463ae 100644
--- a/configure.ac
+++ b/configure.ac
@@ -196,6 +196,7 @@ if test "$sys_name" = linux; then
     PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
                       [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
     have_seccomp=1
+    AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
   else
     have_seccomp=
   fi

From 70eb64147e073a0a7cf1d44af6815cfd4f8d507f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Feb 2018 20:38:06 +0100
Subject: [PATCH 0841/2196] Update release notes

Also add some examples to nix --help.
---
 doc/manual/release-notes/rl-2.0.xml | 983 +++++++++++++++++++++-------
 doc/manual/style.css                |  12 +-
 src/nix/ls.cc                       |  20 +
 src/nix/run.cc                      |   4 +
 4 files changed, 755 insertions(+), 264 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index effd2e39d30..0ad17373ef4 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -6,165 +6,572 @@
 
 <title>Release 2.0 (2018-02-??)</title>
 
-<para>This release has the following new features:</para>
+<para>The following incompatible changes have been made:</para>
 
 <itemizedlist>
 
   <listitem>
-    <para>Start of new <command>nix</command> command line
-    interface. This is a work in progress and the interface is subject
-    to change.</para>
+    <para>The manifest-based substituter mechanism
+    (<command>download-using-manifests</command>) has been <link
+    xlink:href="https://github.com/NixOS/nix/commit/867967265b80946dfe1db72d40324b4f9af988ed">removed</link>. It
+    has been superseded by the binary cache substituter mechanism
+    since several years. As a result, the following programs have been
+    removed:
 
     <itemizedlist>
+      <listitem><para><command>nix-pull</command></para></listitem>
+      <listitem><para><command>nix-generate-patches</command></para></listitem>
+      <listitem><para><command>bsdiff</command></para></listitem>
+      <listitem><para><command>bspatch</command></para></listitem>
+    </itemizedlist>
+    </para>
+  </listitem>
 
-      <listitem><para>Self-documenting: <option>--help</option> shows
-      all available command-line arguments.</para></listitem>
-
-      <listitem><para><option>--help-config</option> shows all
-      configuration options.</para></listitem>
+  <listitem>
+    <para>The “copy from other stores” substituter mechanism
+    (<command>copy-from-other-stores</command> and the
+    <envar>NIX_OTHER_STORES</envar> environment variable) has been
+    removed. It was primarily used by the NixOS installer to copy
+    available paths from the installation medium. The replacement is
+    to use a chroot store as a substituter
+    (e.g. <literal>--substituters /mnt</literal>), or to build into a
+    chroot store (e.g. <literal>--store /mnt --substituter /</literal>).</para>
+  </listitem>
 
-      <listitem><para><command>nix build</command>: Replacement for
-      <command>nix-build</command>.</para></listitem>
+  <listitem>
+    <para>The command <command>nix-push</command> has been removed as
+    part of the effort to eliminate Nix's dependency on Perl. You can
+    use <command>nix copy</command> instead, e.g. <literal>nix copy
+    --to /tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
+  </listitem>
 
-      <listitem><para><command>nix ls-store</command> and <command>nix
-      ls-nar</command> allow listing the contents of a store path or
-      NAR file.</para></listitem>
+  <listitem>
+    <para>The “nested” log output feature (<option>--log-type
+    pretty</option>) has been removed. As a result,
+    <command>nix-log2xml</command> was also removed.</para>
+  </listitem>
 
-      <listitem><para><command>nix cat-store</command> and
-      <command>nix cat-nar</command> allow extracting a file from a
-      store path or NAR file.</para></listitem>
+  <listitem>
+    <para>OpenSSL-based signing has been <link
+    xlink:href="https://github.com/NixOS/nix/commit/f435f8247553656774dd1b2c88e9de5d59cab203">removed</link>. This
+    feature was never well-supported. A better alternative is provided
+    by the <option>secret-key-files</option> and
+    <option>trusted-public-keys</option> options.</para>
+  </listitem>
 
-      <listitem><para><command>nix verify</command> checks whether a
-      store path is unmodified and/or is trusted.</para></listitem>
+  <listitem>
+    <para>Failed build caching has been <link
+    xlink:href="https://github.com/NixOS/nix/commit/8cffec84859cec8b610a2a22ab0c4d462a9351ff">removed</link>. This
+    feature was introduced to support the Hydra continuous build
+    system, but Hydra no longer uses it.</para>
+  </listitem>
 
-      <listitem><para><command>nix copy-sigs</command> copies
-      signatures from one store to another.</para></listitem>
+  <listitem>
+    <para><filename>nix-mode.el</filename> has been removed from
+    Nix. It is now <link
+    xlink:href="https://github.com/NixOS/nix-mode">a separate
+    repository</link> and can be installed through the MELPA package
+    repository.</para>
+  </listitem>
 
-      <listitem><para><command>nix sign-paths</command> signs store
-      paths.</para></listitem>
+</itemizedlist>
 
-      <listitem><para><command>nix copy</command> copies paths between
-      arbitrary Nix stores, generalising
-      <command>nix-copy-closure</command> and
-      <command>nix-push</command>.</para></listitem>
+<para>This release has the following new features:</para>
 
-      <listitem><para><command>nix path-info</command> shows
-      information about store paths.</para></listitem>
+<itemizedlist>
 
-      <listitem><para><command>nix run</command> starts a shell in
-      which the specified packages are available.</para></listitem>
+  <listitem>
+    <para>It introduces a new command named <command>nix</command>,
+    which is intended to eventually replace all
+    <command>nix-*</command> commands with a more consistent and
+    better designed user interface. It currently provides replacements
+    for some (but not all) of the functionality provided by
+    <command>nix-store</command>, <command>nix-build</command>,
+    <command>nix-shell -p</command>, <command>nix-env -qa</command>,
+    <command>nix-instantiate --eval</command>,
+    <command>nix-push</command> and
+    <command>nix-copy-closure</command>. It has the following major
+    features:</para>
 
-      <listitem><para><command>nix log</command> shows the build log
-      of a package or path. If the build log is not available locally,
-      it will try to obtain it from a binary cache.</para></listitem>
+    <itemizedlist>
 
-      <listitem><para><command>nix eval</command> replaces
-      <command>nix-instantiate --eval</command>.</para></listitem>
+      <listitem>
+        <para>Unlike the legacy commands, it has a consistent way to
+        refer to packages and package-like argumements (like store
+        paths). For example, the following commands all copy the GNU
+        Hello package to a remote machine:
+
+        <screen>nix copy --to ssh://machine nixpkgs.hello</screen>
+        <screen>nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen>
+        <screen>nix copy --to ssh://machine '(with import &lt;nixpkgs> {}; hello)'</screen>
+
+        By contrast, <command>nix-copy-closure</command> only accepted
+        store paths as arguments.</para>
+      </listitem>
+
+      <listitem>
+        <para>It is self-documenting: <option>--help</option> shows
+        all available command-line arguments. If
+        <option>--help</option> is given after a subcommand, it shows
+        examples for that subcommand. <command>nix
+        --help-config</command> shows all configuration
+        options.</para>
+      </listitem>
+
+      <listitem>
+        <para>It is much less verbose. By default, it displays a
+        single-line progress indicator that shows how many packages
+        are left to be built or downloaded, and (if there are running
+        builds) the most recent line of builder output. If a build
+        fails, it shows the last few lines of builder output. The full
+        build log can be retrieved using <command>nix
+        log</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para>It <link
+        xlink:href="https://github.com/NixOS/nix/commit/b8283773bd64d7da6859ed520ee19867742a03ba">provides</link>
+        all <filename>nix.conf</filename> configuration options as
+        command line flags. For example, instead of <literal>--option
+        http-connections 100</literal> you can write
+        <literal>--http-connections 100</literal>. Boolean options can
+        be written as
+        <literal>--<replaceable>foo</replaceable></literal> or
+        <literal>--no-<replaceable>foo</replaceable></literal>
+        (e.g. <option>--no-auto-optimise-store</option>).</para>
+      </listitem>
+
+      <listitem>
+        <para>Many subcommands have a <option>--json</option> flag to
+        write results to stdout in JSON format.</para>
+      </listitem>
 
-      <listitem><para><command>nix dump-path</command> to get a NAR
-      from a store path.</para></listitem>
+    </itemizedlist>
 
-      <listitem><para><command>nix edit</command> opens the source
-      code of a package in an editor.</para></listitem>
+    <warning><para>Please note that the <command>nix</command> command
+    is a work in progress and the interface is subject to
+    change.</para></warning>
 
-      <listitem><para><command>nix search</command> replaces
-      <command>nix-env -qa</command>. It searches the available
-      packages for occurences of a search string in the attribute
-      name, package name or description. It caches available packages
-      to speed up searches.</para></listitem>
+    <para>It provides the following high-level (“porcelain”)
+    subcommands:</para>
 
-      <listitem><para><command>nix why-depends</command> (d41c5eb13f4f3a37d80dbc6d3888644170c3b44a).</para></listitem>
+    <itemizedlist>
 
-      <listitem><para><command>nix show-derivation</command> (e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2).</para></listitem>
+      <listitem>
+        <para><command>nix build</command> is a replacement for
+        <command>nix-build</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix run</command> executes a command in an
+        environment in which the specified packages are available. It
+        is (roughly) a replacement for <command>nix-shell
+        -p</command>. Unlike that command, it does not execute the
+        command in a shell, and has a flag (<command>-c</command>)
+        that specifies the unquoted command line to be
+        executed.</para>
+
+        <para>It is particularly useful in conjunction with chroot
+        stores, allowing Linux users who do not have permission to
+        install Nix in <command>/nix/store</command> to still use
+        binary substitutes that assume
+        <command>/nix/store</command>. For example,
+
+        <screen>nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!'</screen>
+
+        downloads (or if not substitutes are available, builds) the
+        GNU Hello package into
+        <filename>~/my-nix/nix/store</filename>, then runs
+        <command>hello</command> in a mount namespace where
+        <filename>~/my-nix/nix/store</filename> is mounted onto
+        <command>/nix/store</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix search</command> replaces <command>nix-env
+        -qa</command>. It searches the available packages for
+        occurences of a search string in the attribute name, package
+        name or description. Unlike <command>nix-env -qa</command>, it
+        has a cache to speed up subsequent searches.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix copy</command> copies paths between
+        arbitrary Nix stores, generalising
+        <command>nix-copy-closure</command> and
+        <command>nix-push</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix repl</command> replaces the external
+        program <command>nix-repl</command>. It provides an
+        interactive environment for evaluating and building Nix
+        expressions. Note that it uses <literal>linenoise-ng</literal>
+        instead of GNU Readline.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix upgrade-nix</command> upgrades Nix to the
+        latest stable version. This requires that Nix is installed in
+        a profile. (Thus it won’t work on NixOS, or if it’s installed
+        outside of the Nix store.)</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix verify</command> checks whether store paths
+        are unmodified and/or “trusted” (see below). It replaces
+        <command>nix-store --verify</command> and <command>nix-store
+        --verify-path</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix log</command> shows the build log of a
+        package or path. If the build log is not available locally, it
+        will try to obtain it from the configured substituters (such
+        as <uri>cache.nixos.org</uri>, which now provides build
+        logs).</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix edit</command> opens the source code of a
+        package in your editor.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix eval</command> replaces
+        <command>nix-instantiate --eval</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command
+        xlink:href="https://github.com/NixOS/nix/commit/d41c5eb13f4f3a37d80dbc6d3888644170c3b44a">nix
+        why-depends</command> shows why one store path has another in
+        its closure. This is primarily useful to finding the causes of
+        closure bloat. For example,
+
+        <screen>nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev</screen>
+
+        shows a chain of files and fragments of file contents that
+        cause the VLC package to have the “dev” output of
+        <literal>libdrm</literal> in its closure — an undesirable
+        situation.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix path-info</command> shows information about
+        store paths, replacing <command>nix-store -q</command>. A
+        useful feature is the option <option>--closure-size</option>
+        (<option>-S</option>). For example, the following command show
+        the closure sizes of every path in the current NixOS system
+        closure, sorted by size:
+
+        <screen>nix path-info -rS /run/current-system | sort -nk2</screen>
+
+        </para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix optimise-store</command> replaces
+        <command>nix-store --optimise</command>. The main difference
+        is that it has a progress indicator.</para>
+      </listitem>
 
-      <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem>
+    </itemizedlist>
 
-      <listitem><para><command>nix upgrade-nix</command> upgrades Nix
-      to the latest stable version. This requires that Nix is
-      installed in a profile. (Thus it won’t work on NixOS, or if it’s
-      installed outside of the Nix store.)</para></listitem>
+    <para>A number of low-level (“plumbing”) commands are also
+    available:</para>
 
-      <listitem><para>Progress indicator.</para></listitem>
+    <itemizedlist>
 
-      <listitem><para>All options are available as flags now
-      (b8283773bd64d7da6859ed520ee19867742a03ba).</para></listitem>
+      <listitem>
+        <para><command>nix ls-store</command> and <command>nix
+        ls-nar</command> list the contents of a store path or NAR
+        file. The former is primarily useful in conjunction with
+        remote stores, e.g.
+
+        <screen>nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen>
+
+        lists the contents of path in a binary cache.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix cat-store</command> and <command>nix
+        cat-nar</command> allow extracting a file from a store path or
+        NAR file.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix dump-path</command> writes the contents of
+        a store path to stdout in NAR format. This replaces
+        <command>nix-store --dump</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command
+        xlink:href="https://github.com/NixOS/nix/commit/e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2">nix
+        show-derivation</command> displays a store derivation in JSON
+        format. This is an alternative to
+        <command>pp-aterm</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command
+        xlink:href="https://github.com/NixOS/nix/commit/970366266b8df712f5f9cedb45af183ef5a8357f">nix
+        add-to-store</command> replaces <command>nix-store
+        --add</command>.</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix sign-paths</command> signs store
+        paths. (TODO: add examples)</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix copy-sigs</command> copies signatures from
+        one store to another. (TODO: add examples and
+        tests)</para>
+      </listitem>
+
+      <listitem>
+        <para><command>nix show-config</command> shows all
+        configuration options and their current values.</para>
+      </listitem>
 
     </itemizedlist>
 
   </listitem>
 
   <listitem>
-    <para>The external program <command>nix-repl</command> has been
-    integrated into Nix as <command>nix repl</command>.</para>
-  </listitem>
+    <para>The store abstraction that Nix has had for a long time to
+    support store access via the Nix daemon has been extended
+    significantly. In particular, substituters (which used to be
+    external programs such as
+    <command>download-from-binary-cache</command>) are now subclasses
+    of the abstract <classname>Store</classname> class. This allows
+    many Nix commands to operate on such store types. For example,
+    <command>nix path-info</command> shows information about paths in
+    your local Nix store, while <command>nix path-info --store
+    https://cache.nixos.org/</command> shows information about paths
+    in the specified binary cache. Similarly,
+    <command>nix-copy-closure</command>, <command>nix-push</command>
+    and substitution are all instances of the general notion of
+    copying paths between different kinds of Nix stores.</para>
 
-  <listitem>
-    <para>If a fixed-output derivation produces a result with an
-    incorrect hash, the output path will be moved to the location
-    corresponding to the actual hash and registered as valid. Thus, a
-    subsequent build of the fixed-output derivation with the correct
-    hash is unnecessary.</para>
-  </listitem>
+    <para>Stores are specified using an URI-like syntax,
+    e.g. <uri>https://cache.nixos.org/</uri> or
+    <uri>ssh://machine</uri>. The following store types are supported:
+
+    <itemizedlist>
+
+      <listitem>
+
+        <para><classname>LocalStore</classname> (stori URI
+        <literal>local</literal> or an absolute path) and the misnamed
+        <classname>RemoteStore</classname> (<literal>daemon</literal>)
+        provide access to a local Nix store, the latter via the Nix
+        daemon. You can use <literal>auto</literal> or the empty
+        string to auto-select a local or daemon store depending on
+        whether you have write permission to the Nix store. It is no
+        longer necessary to set the <envar>NIX_REMOTE</envar>
+        environment variable to use the Nix daemon.</para>
+
+        <para>As noted above, <classname>LocalStore</classname> now
+        supports chroot builds, allowing the “physical” location of
+        the Nix store
+        (e.g. <filename>/home/alice/nix/store</filename>) to differ
+        from its “logical” location (typically
+        <filename>/nix/store</filename>). This allows non-root users
+        to use Nix while still getting the benefits from prebuilt
+        binaries from <uri>cache.nixos.org</uri>.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>BinaryCacheStore</classname> is the abstract
+        superclass of all binary cache stores. It supports writing
+        build logs and NAR content listings in JSON format.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>HttpBinaryCacheStore</classname>
+        (<literal>http://</literal>, <literal>https://</literal>)
+        supports binary caches via HTTP or HTTPS. If the server
+        supports <literal>PUT</literal> requests, it supports
+        uploading store paths via commands such as <command>nix
+        copy</command>.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>LocalBinaryCacheStore</classname>
+        (<literal>file://</literal>) supports binary caches in the
+        local filesystem.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>S3BinaryCacheStore</classname>
+        (<literal>s3://</literal>) supports binary caches stored in
+        Amazon S3, if enabled at compile time.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>LegacySSHStore</classname> (<literal>ssh://</literal>)
+        is used to implement remote builds and
+        <command>nix-copy-closure</command>.</para>
+
+      </listitem>
+
+      <listitem>
+
+        <para><classname>SSHStore</classname>
+        (<literal>ssh-ng://</literal>) supports arbitrary Nix
+        operations on a remote machine via the same protocol used by
+        <command>nix-daemon</command>.</para>
+
+      </listitem>
+
+    </itemizedlist>
+
+    </para>
 
-  <listitem>
-    <para>It is no longer necessary to set the
-    <envar>NIX_REMOTE</envar> environment variable if you need to use
-    the Nix daemon. Nix will use the daemon automatically if you don’t
-    have write access to the Nix database.</para>
   </listitem>
 
   <listitem>
-    <para>The Nix language now supports floating point numbers. They are
-    based on regular C++ <literal>float</literal> and compatible with
-    existing integers and number-related operations. Export and import to and
-    from JSON and XML works, too.</para>
+
+    <para>Security has been improved in various ways:
+
+    <itemizedlist>
+
+      <listitem>
+        <para>Nix now stores signatures for local store
+        paths. When paths are copied between stores (e.g., copied from
+        a binary cache to a local store), signatures are
+        propagated.</para>
+
+        <para>Locally-built paths are signed automatically using the
+        secret keys specified by the <option>secret-key-files</option>
+        store option. Secret/public key pairs can be generated using
+        <command>nix-store
+        --generate-binary-cache-key</command>. (TODO: rename)</para>
+
+        <para>In addition, locally-built store paths are marked as
+        “ultimately trusted”, but this bit is not propagated when
+        paths are copied between stores.</para>
+      </listitem>
+
+      <listitem>
+        <para>Content-addressable store paths no longer require
+        signatures — they can be imported into a store by unprivileged
+        users even if they lack signatures.</para>
+      </listitem>
+
+      <listitem>
+        <para>The command <command>nix verify</command> checks whether
+        the specified paths are trusted, i.e., have a certain number
+        of trusted signatures, are ultimately trusted, or are
+        content-addressed.</para>
+      </listitem>
+
+      <listitem>
+        <para>Substitutions from binary caches <link
+        xlink:href="https://github.com/NixOS/nix/commit/ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b">now</link>
+        require signatures by default. This was already the case on
+        NixOS.</para>
+      </listitem>
+
+      <listitem>
+        <para>In Linux sandbox builds, we <link
+        xlink:href="https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b">now</link>
+        use <filename>/build</filename> instead of
+        <filename>/tmp</filename> as the temporary build
+        directory. This fixes potential security problems when a build
+        accidentally stores its <envar>TMPDIR</envar> in some
+        security-sensitive place, such as an RPATH.</para>
+      </listitem>
+
+    </itemizedlist>
+
+    </para>
+
   </listitem>
 
   <listitem>
-    <para><command>nix-shell</command> now sets the
-    <varname>IN_NIX_SHELL</varname> environment variable during
-    evaluation and in the shell itself. This can be used to perform
-    different actions depending on whether you’re in a Nix shell or in
-    a regular build. Nixpkgs provides
-    <varname>lib.inNixShell</varname> to check this variable during
-    evaluation. (bb36a1a3cf3fbe6bc9d0afcc5fa0f928bed03170)</para>
+    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
+    of the existing restricted evaluation mode. In pure mode, the Nix
+    evaluator forbids access to anything that could cause different
+    evaluations of the same command line arguments to produce a
+    different result. This includes builtin functions such as
+    <function>builtins.getEnv</function>, but more importantly,
+    <emphasis>all</emphasis> filesystem or network access unless a
+    content hash or commit hash is specified. For example, calls to
+    <function>builtins.fetchGit</function> are only allowed if a
+    <varname>rev</varname> attribute is specified.</para>
+
+    <para>The goal of this feature is to enable true reproducibility
+    and traceability of builds (including NixOS system configurations)
+    at the evaluation level. For example, in the future,
+    <command>nixos-rebuild</command> might build configurations from a
+    Nix expression in a Git repository in pure mode. That expression
+    might fetch other repositories such as Nixpkgs via
+    <function>builtins.fetchGit</function>. The commit hash of the
+    top-level repository then uniquely identifies a running system,
+    and, in conjunction with that repository, allows it to be
+    reproduced or modified.</para>
+
   </listitem>
 
   <listitem>
-    <para>Internal: all <classname>Store</classname> classes are now
-    thread-safe. <classname>RemoteStore</classname> supports multiple
-    concurrent connections to the daemon. This is primarily useful in
-    multi-threaded programs such as
-    <command>hydra-queue-runner</command>.</para>
+    <para>There are several new features to support binary
+    reproducibility (i.e. to help ensure that multiple builds of the
+    same derivation produce exactly the same output). When
+    <option>enforce-determinism</option> is set to
+    <literal>false</literal>, it’s <link
+    xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no
+    longer</link> a fatal error if build rounds produce different
+    output. Also, a hook named <option>diff-hook</option> is <link
+    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169w">provided</link>
+    to allow you to run tools such as <command>diffoscope</command>
+    when build rounds produce different output.</para>
   </listitem>
 
   <listitem>
-    <para>The dependency on Perl has been removed. As a result, some
-    (obsolete) programs have been removed: <command>nix-push</command>
-    (replaced by <command>nix copy</command>),
-    <command>nix-pull</command> (obsoleted by binary caches),
-    <command>nix-generate-patches</command>,
-    <command>bsdiff</command>, <command>bspatch</command>.</para>
+    <para>Configuring remote builds is a lot easier now. Provided you
+    are not using the Nix daemon, you can now just specify a remote
+    build machine on the command line, e.g. <literal>--option builders
+    'ssh://my-mac x86_64-darwin'</literal>. The environment variable
+    <envar>NIX_BUILD_HOOK</envar> has been removed and is no longer
+    needed. The environment variable <envar>NIX_REMOTE_SYSTEMS</envar>
+    is still supported for compatibility, but it is also possible to
+    specify builders in <command>nix.conf</command> by setting the
+    option <literal>builders =
+    @<replaceable>path</replaceable></literal>.</para>
   </listitem>
 
   <listitem>
-    <para>Improved store abstraction. Substituters
-    eliminated. BinaryCacheStore, LocalBinaryCacheStore,
-    HttpBinaryCacheStore, S3BinaryCacheStore (compile-time
-    optional), SSHStore. Add docs + examples?
-    </para>
+    <para>If a fixed-output derivation produces a result with an
+    incorrect hash, the output path is moved to the location
+    corresponding to the actual hash and registered as valid. Thus, a
+    subsequent build of the fixed-output derivation with the correct
+    hash is unnecessary.</para>
   </listitem>
 
   <listitem>
-    <para>Nix now stores signatures for local store
-    paths. Locally-built paths are now signed automatically using the
-    secret keys specified by the <option>secret-key-files</option>
-    store option.</para>
-
-    <para>In addition, store paths that have been built locally are
-    marked as “ultimately trusted”, and content-addressable store
-    paths carry a “content-addressability assertion” that allow them
-    to be trusted without any signatures.</para>
+    <para><command>nix-shell</command> <link
+    xlink:href="https://github.com/NixOS/nix/commit/ea59f39326c8e9dc42dfed4bcbf597fbce58797c">now</link>
+    sets the <varname>IN_NIX_SHELL</varname> environment variable
+    during evaluation and in the shell itself. This can be used to
+    perform different actions depending on whether you’re in a Nix
+    shell or in a regular build. Nixpkgs provides
+    <varname>lib.inNixShell</varname> to check this variable during
+    evaluation.</para>
   </listitem>
 
   <listitem>
@@ -179,7 +586,8 @@
     <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For
     example, <literal>nix-build channel:nixos-15.09 -A hello</literal>
     will build the GNU Hello package from the
-    <literal>nixos-15.09</literal> channel.</para>
+    <literal>nixos-15.09</literal> channel. In the future, this may
+    use Git to fetch updates more efficiently.</para>
   </listitem>
 
   <listitem>
@@ -189,45 +597,119 @@
   </listitem>
 
   <listitem>
-    <para><function>builtins.fetchGit</function>.
-    (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para>
+    <para>Networking has been improved:
+
+    <itemizedlist>
+
+      <listitem>
+        <para>HTTP/2 is now supported. This makes binary cache lookups
+        <link
+        xlink:href="https://github.com/NixOS/nix/commit/90ad02bf626b885a5dd8967894e2eafc953bdf92">much
+        more efficient</link>.</para>
+      </listitem>
+
+      <listitem>
+        <para>We now retry downloads on many HTTP errors, making
+        binary caches substituters more resilient to temporary
+        failures.</para>
+      </listitem>
+
+      <listitem>
+        <para>HTTP credentials can now be configured via the standard
+        <filename>netrc</filename> mechanism.</para>
+      </listitem>
+
+      <listitem>
+        <para>If S3 support is enabled at compile time,
+        <uri>s3://</uri> URIs are <link
+        xlink:href="https://github.com/NixOS/nix/commit/9ff9c3f2f80ba4108e9c945bbfda2c64735f987b">supported</link>
+        in all places where Nix allows URIs.</para>
+      </listitem>
+
+      <listitem>
+        <para>Brotli compression is now supported. In particular,
+        <uri>cache.nixos.org</uri> build logs are now compressed using
+        Brotli.</para>
+      </listitem>
+
+    </itemizedlist>
+
+    </para>
+
   </listitem>
 
   <listitem>
-    <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
-    content-addressable tarball cache at
-    <uri>http://tarballs.nixos.org/</uri>, just like
-    <function>fetchurl</function> in
-    Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para>
+    <para><command>nix-env</command> <link
+    xlink:href="https://github.com/NixOS/nix/commit/b0cb11722626e906a73f10dd9a0c9eea29faf43a">now</link>
+    ignores packages with bad derivation names (in particular those
+    starting with a digit or containing a dot).</para>
   </listitem>
 
   <listitem>
-    <para>Chroot Nix stores: allow the “physical” location of the Nix
-    store (e.g. <filename>/home/alice/nix/store</filename>) to differ
-    from its “logical” location (typically
-    <filename>/nix/store</filename>). This allows non-root users to
-    use Nix while still getting the benefits from prebuilt binaries
-    from
-    <uri>cache.nixos.org</uri>. (4494000e04122f24558e1436e66d20d89028b4bd,
-    3eb621750848e0e6b30e5a79f76afbb096bb6c8a)</para>
+    <para>Many configuration options have been renamed, either because
+    they were unnecessarily verbose
+    (e.g. <option>build-use-sandbox</option> is now just
+    <option>sandbox</option>) or to reflect generalised behaviour
+    (e.g. <option>binary-caches</option> is now
+    <option>substituters</option> because it allows arbitrary store
+    URIs). The old names are still supported for compatibility.</para>
   </listitem>
 
   <listitem>
-    <para>On Linux, builds are now executed in a user
-    namespace with uid 1000 and gid 100.</para>
+    <para>The <option>max-jobs</option> option can <link
+    xlink:href="https://github.com/NixOS/nix/commit/7251d048fa812d2551b7003bc9f13a8f5d4c95a5">now</link>
+    be set to <literal>auto</literal> to use the number of CPUs in the
+    system.</para>
   </listitem>
 
   <listitem>
-    <para><function>builtins.fetchurl</function> and
-    <function>builtins.fetchTarball</function> now support
-    <varname>sha256</varname> and <varname>name</varname>
-    attributes.</para>
+    <para>Hashes can <link
+    xlink:href="https://github.com/NixOS/nix/commit/c0015e87af70f539f24d2aa2bc224a9d8b84276b">now</link>
+    be specified in base-64 format, in addition to base-16 and the
+    non-standard base-32.</para>
   </listitem>
 
   <listitem>
-    <para><literal>HttpBinaryCacheStore</literal> (the replacement of
-    <command>download-from-binary-cache</command>) now retries
-    automatically on certain HTTP error codes.</para>
+    <para><command>nix-shell</command> now uses
+    <varname>bashInteractive</varname> from Nixpkgs, rather than the
+    <command>bash</command> command that happens to be in the caller’s
+    <envar>PATH</envar>. This is especially important on macOS where
+    the <command>bash</command> provided by the system is seriously
+    outdated and cannot execute <literal>stdenv</literal>’s setup
+    script.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix can now automatically trigger a garbage collection if
+    free disk space drops below a certain level during a build. This
+    is configured using the <option>min-free</option> and
+    <option>max-free</option> options.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-store -q --roots</command> and
+    <command>nix-store --gc --print-roots</command> now show temporary
+    and in-memory roots.</para>
+  </listitem>
+
+  <listitem>
+    <para>
+      Nix can now be extended with plugins. See the documentation of
+      the <option>plugin-files</option> option for more details.
+    </para>
+  </listitem>
+
+</itemizedlist>
+
+<para>The Nix language has the following new features:
+
+<itemizedlist>
+
+  <listitem>
+    <para>It supports floating point numbers. They are based on the
+    C++ <literal>float</literal> type and are supported by the
+    existing numerical operators. Export and import to and from JSON
+    and XML works, too.</para>
   </listitem>
 
   <listitem>
@@ -245,194 +727,187 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     add docs.</para>
   </listitem>
 
-  <listitem>
-    <para>Support for HTTP/2. This makes binary cache lookups much
-    more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para>
-  </listitem>
+</itemizedlist>
 
-  <listitem>
-    <para>The <option>build-sandbox-paths</option> configuration
-    option can now specify optional paths by appending a
-    <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will
-    bind-mount <varname>/dev/nvidiactl</varname> only if it
-    exists.</para>
-  </listitem>
+</para>
 
-  <listitem>
-    <para>More support for testing build reproducibility: when
-    <option>enforce-determinism</option> is set to
-    <literal>false</literal>, it’s no longer a fatal error build
-    rounds produce different output
-    (8bdf83f936adae6f2c907a6d2541e80d4120f051); add a hook to run
-    diffoscope when build rounds produce different output
-    (9a313469a4bdea2d1e8df24d16289dc2a172a169w).</para>
-  </listitem>
+<para>The following builtin functions are new or extended:
 
-  <listitem>
-    <para>Kill builds as soon as stdout/stderr is closed. This fixes a
-    bug that allowed builds to hang Nix indefinitely (regardless of
-    timeouts). (21948deed99a3295e4d5666e027a6ca42dc00b40)</para>
-  </listitem>
+<itemizedlist>
 
   <listitem>
-    <para>Add support for passing structured data to builders. TODO:
-    document. (6de33a9c675b187437a2e1abbcb290981a89ecb1)</para>
-  </listitem>
+    <para><function
+    xlink:href="https://github.com/NixOS/nix/commit/38539b943a060d9cdfc24d6e5d997c0885b8aa2f">builtins.fetchGit</function>
+    allows Git repositories to be fetched at evaluation time. Thus it
+    differs from the <function>fetchgit</function> function in
+    Nixpkgs, which fetches at build time and cannot be used to fetch
+    Nix expressions during evaluation. A typical use case is to import
+    external NixOS modules from your configuration, e.g.
 
-  <listitem>
-    <para><varname>exportReferencesGraph</varname>: Export more
-    complete info in JSON
-    format. (c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a)</para>
-  </listitem>
+    <programlisting>imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ];</programlisting>
 
-  <listitem>
-    <para>Support for
-    netrc. (e6e74f987f0fa284d220432d426eb965269a97d6,
-    302386f775eea309679654e5ea7c972fb6e7b9af)</para>
+    </para>
   </listitem>
 
   <listitem>
-    <para>Support <uri>s3://</uri> URIs in all places where Nix allows
-    URIs. (9ff9c3f2f80ba4108e9c945bbfda2c64735f987b)</para>
+    <para>Similarly, <function>builtins.fetchMercurial</function>
+    allows you to fetch Mercurial repositories.</para>
   </listitem>
 
   <listitem>
-    <para>The <option>build-max-jobs</option> option can be set to
-    <literal>auto</literal> to use the number of CPUs in the
-    system. (7251d048fa812d2551b7003bc9f13a8f5d4c95a5)</para>
+    <para><function>builtins.path</function> generalises
+    <function>builtins.filterSource</function> and path literals
+    (e.g. <literal>./foo</literal>). It allows specifying a store path
+    name that differs from the source path name
+    (e.g. <literal>builtins.path { path = ./foo; name = "bar";
+    }</literal>) and also supports filtering out unwanted
+    files.</para>
   </listitem>
 
   <listitem>
-    <para>Add support for Brotli compression.
-    <uri>cache.nixos.org</uri> compresses build logs using
-    Brotli.</para>
+    <para><function>builtins.fetchurl</function> and
+    <function>builtins.fetchTarball</function> now support
+    <varname>sha256</varname> and <varname>name</varname>
+    attributes.</para>
   </listitem>
 
   <listitem>
-    <para>Substitutions from binary caches now require signatures by
-    default. This was already the case on
-    NixOS. (ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b)</para>
+    <para><function
+    xlink:href="https://github.com/NixOS/nix/commit/b8867a0239b1930a16f9ef3f7f3e864b01416dff">builtins.split</function>
+    splits a string using a POSIX extended regular expression as the
+    separator.</para>
   </listitem>
 
   <listitem>
-    <para><command>nix-env</command> now ignores packages with bad
-    derivation names (in particular those starting with a digit or
-    containing a
-    dot). (b0cb11722626e906a73f10dd9a0c9eea29faf43a)</para>
+    <para><function
+    xlink:href="https://github.com/NixOS/nix/commit/26d92017d3b36cff940dcb7d1611c42232edb81a">builtins.partition</function>
+    partitions the elements of a list into two lists, depending on a
+    Boolean predicate.</para>
   </listitem>
 
   <listitem>
-    <para>Renamed various configuration options. (TODO: in progress)</para>
+    <para><literal>&lt;nix/fetchurl.nix&gt;</literal> now uses the
+    content-addressable tarball cache at
+    <uri>http://tarballs.nixos.org/</uri>, just like
+    <function>fetchurl</function> in
+    Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para>
   </listitem>
 
   <listitem>
-    <para>Remote machines can now be specified on the command
-    line. TODO:
-    document. (1a68710d4dff609bbaf61db3e17a2573f0aadf17)</para>
+    <para>In restricted and pure evaluation mode, builtin functions
+    that download from the network (such as
+    <function>fetchGit</function>) are permitted to fetch underneath a
+    list of URI prefixes specified in the option
+    <option>allowed-uris</option>.</para>
   </listitem>
 
-  <listitem>
-    <para>In Linux sandbox builds, we now use
-    <filename>/build</filename> instead of <filename>/tmp</filename>
-    as the temporary build directory. This fixes potential security
-    problems when a build accidentally stores its
-    <envar>TMPDIR</envar> in some critical place, such as an
-    RPATH. (eba840c8a13b465ace90172ff76a0db2899ab11b)</para>
-  </listitem>
+</itemizedlist>
 
-  <listitem>
-    <para>In Linux sandbox builds, we now provide a default
-    <filename>/bin/sh</filename> (namely <filename>ash</filename> from
-    BusyBox). (a2d92bb20e82a0957067ede60e91fab256948b41)</para>
-  </listitem>
+</para>
 
-  <listitem>
-    <para>Make all configuration options available as command line
-    flags (b8283773bd64d7da6859ed520ee19867742a03ba).</para>
-  </listitem>
+<para>The Nix build environment has the following changes:
 
-  <listitem>
-    <para>Support base-64
-    hashes. (c0015e87af70f539f24d2aa2bc224a9d8b84276b)</para>
-  </listitem>
+<itemizedlist>
 
   <listitem>
-    <para><command>nix-shell</command> now uses
-    <varname>bashInteractive</varname> from Nixpkgs, rather than the
-    <command>bash</command> command that happens to be in the caller’s
-    <envar>PATH</envar>. This is especially important on macOS where
-    the <command>bash</command> provided by the system is seriously
-    outdated and cannot execute <literal>stdenv</literal>’s setup
-    script.</para>
+    <para>Values such as Booleans, integers, (nested) lists and
+    attribute sets can <link
+    xlink:href="https://github.com/NixOS/nix/commit/6de33a9c675b187437a2e1abbcb290981a89ecb1">now</link>
+    be passed to builders in a non-lossy way. If the special attribute
+    <varname>__structuredAttrs</varname> is set to
+    <literal>true</literal>, the other derivation attributes are
+    serialised in JSON format and made available to the builder via
+    the file <envar>.attrs.json</envar> in the builder’s temporary
+    directory. This obviates the need for
+    <varname>passAsFile</varname> since JSON files have no size
+    restrictions, unlike process environments.</para>
+
+    <para><link
+    xlink:href="https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699">As
+    a convenience to Bash builders</link>, Nix writes a script named
+    <envar>.attrs.sh</envar> to the builder’s directory that
+    initialises shell variables corresponding to all attributes that
+    are representable in Bash. This includes non-nested (associative)
+    arrays. For example, the attribute <literal>hardening.format =
+    true</literal> ends up as the Bash associative array element
+    <literal>${hardening[format]}</literal>.</para>
+  </listitem>
+
+  <listitem>
+    <para>Builders can <link
+    xlink:href="https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3">now</link>
+    communicate what build phase they are in by writing messages to
+    the file descriptor specified in <envar>NIX_LOG_FD</envar>. The
+    current phase is shown by the <command>nix</command> progress
+    indicator.
+    </para>
   </listitem>
 
   <listitem>
-    <para>New builtin functions: <function>builtins.split</function>
-    (b8867a0239b1930a16f9ef3f7f3e864b01416dff),
-    <function>builtins.partition</function>.</para>
+    <para>In Linux sandbox builds, we <link
+    xlink:href="https://github.com/NixOS/nix/commit/a2d92bb20e82a0957067ede60e91fab256948b41">now</link>
+    provide a default <filename>/bin/sh</filename> (namely
+    <filename>ash</filename> from BusyBox).</para>
   </listitem>
 
   <listitem>
-    <para>Automatic garbage collection.</para>
+    <para>In structured attribute mode,
+    <varname>exportReferencesGraph</varname> <link
+    xlink:href="https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a">exports</link>
+    extended information about closures in JSON format. In particular,
+    it includes the sizes and hashes of paths. This is primarily
+    useful for NixOS image builders.</para>
   </listitem>
 
   <listitem>
-    <para><command>nix-store -q --roots</command> and
-    <command>nix-store --gc --print-roots</command> now show temporary
-    and in-memory roots.</para>
+    <para>Builds are <link
+    xlink:href="https://github.com/NixOS/nix/commit/21948deed99a3295e4d5666e027a6ca42dc00b40">now</link>
+    killed as soon as Nix receives EOF on the builder’s stdout or
+    stderr. This fixes a bug that allowed builds to hang Nix
+    indefinitely, regardless of
+    timeouts.</para>
   </listitem>
 
   <listitem>
-    <para>Builders can now communicate what build phase they are in by
-    writing messages to the file descriptor specified in
-    <envar>NIX_LOG_FD</envar>. (88e6bb76de5564b3217be9688677d1c89101b2a3)
-    </para>
+    <para>The <option>sandbox-paths</option> configuration
+    option can now specify optional paths by appending a
+    <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will
+    bind-mount <varname>/dev/nvidiactl</varname> only if it
+    exists.</para>
   </listitem>
 
   <listitem>
-    <para>
-      Nix can now be extended with plugins. See the documentation of
-      the 'plugin-files' option for more details.
-    </para>
+    <para>On Linux, builds are now executed in a user
+    namespace with uid 1000 and gid 100.</para>
   </listitem>
 
 </itemizedlist>
 
-<para>Some features were removed:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>“Nested” log output. As a result,
-    <command>nix-log2xml</command> was also removed.</para>
-  </listitem>
+</para>
 
-  <listitem>
-    <para>OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para>
-  </listitem>
+<para>A number of significant internal changes were made:
 
-  <listitem>
-    <para>Caching of failed
-    builds. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para>
-  </listitem>
+<itemizedlist>
 
   <listitem>
-    <para><filename>nix-mode.el</filename> has been removed from
-    Nix. It is now a separate repository in
-    <uri>https://github.com/NixOS/nix-mode</uri> and can be installed
-    through the MELPA package repository.</para>
+    <para>Nix no longer depends on Perl and all Perl components have
+    been rewritten in C++ or removed. The Perl bindings that used to
+    be part of Nix have been moved to a separate package,
+    <literal>nix-perl</literal>.</para>
   </listitem>
 
   <listitem>
-    <para>In restricted evaluation mode
-    (<option>--restrict-eval</option>), builtin functions that
-    download from the network (such as <function>fetchGit</function>)
-    are permitted to fetch underneath the list of URI prefixes
-    specified in the option <option>allowed-uris</option>.</para>
+    <para>All <classname>Store</classname> classes are now
+    thread-safe. <classname>RemoteStore</classname> supports multiple
+    concurrent connections to the daemon. This is primarily useful in
+    multi-threaded programs such as
+    <command>hydra-queue-runner</command>.</para>
   </listitem>
 
 </itemizedlist>
 
+</para>
+
 <para>This release has contributions from TBD.</para>
 
 </section>
diff --git a/doc/manual/style.css b/doc/manual/style.css
index 53fd9d5709c..592583ab086 100644
--- a/doc/manual/style.css
+++ b/doc/manual/style.css
@@ -96,7 +96,6 @@ div.example
     margin-right: 1.5em;
     background: #f4f4f8;
     border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.example p.title
@@ -106,7 +105,6 @@ div.example p.title
 
 div.example pre
 {
-    box-shadow: none;
 }
 
 
@@ -116,15 +114,12 @@ div.example pre
 
 pre.screen, pre.programlisting
 {
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
+    padding: 6px 6px;
     margin-left: 1.5em;
     margin-right: 1.5em;
     color: #600000;
     background: #f4f4f8;
     font-family: monospace;
-    border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.example pre.programlisting
@@ -149,7 +144,6 @@ div.example pre.programlisting
     padding: 0.3em 0.3em 0.3em 0.3em;
     background: #fffff5;
     border-radius: 0.4em;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.note, div.warning
@@ -256,16 +250,14 @@ span.command strong
 
 div.calloutlist table
 {
-    box-shadow: none;
 }
 
 table
 {
     border-collapse: collapse;
-    box-shadow: 0.4em 0.4em 0.5em #e0e0e0;
 }
 
 div.affiliation
 {
     font-style: italic;
-}
\ No newline at end of file
+}
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 69620595d8c..e99622faf47 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -90,6 +90,16 @@ struct CmdLsStore : StoreCommand, MixLs
         expectArg("path", &path);
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To list the contents of a store path in a binary cache:",
+                "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
+            },
+        };
+    }
+
     std::string name() override
     {
         return "ls-store";
@@ -116,6 +126,16 @@ struct CmdLsNar : Command, MixLs
         expectArg("path", &path);
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To list a specific file in a NAR:",
+                "nix ls-nar -l hello.nar /bin/hello"
+            },
+        };
+    }
+
     std::string name() override
     {
         return "ls-nar";
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ade87e63a49..822654daf48 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -85,6 +85,10 @@ struct CmdRun : InstallablesCommand
                 "To run GNU Hello:",
                 "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"
             },
+            Example{
+                "To run GNU Hello in a chroot store:",
+                "nix run --store ~/my-nix nixpkgs.hello -c hello"
+            },
         };
     }
 

From d7fdfe322b81f4b8578e55d95978484ec84cc46c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Feb 2018 20:40:25 +0100
Subject: [PATCH 0842/2196] Remove macOS multi-user instructions

This is already handled by the installer.
---
 doc/manual/installation/multi-user.xml | 28 --------------------------
 1 file changed, 28 deletions(-)

diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml
index a13e3c89be7..69ae1ef2704 100644
--- a/doc/manual/installation/multi-user.xml
+++ b/doc/manual/installation/multi-user.xml
@@ -52,34 +52,6 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if
 you expect to do many builds at the same time.</para>
 
-<para>On macOS, you can create the required group and users by
-running the following script:
-
-<programlisting>
-#! /bin/bash -e
-
-dseditgroup -o create nixbld -q
-
-gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }')
-
-echo "created nixbld group with gid $gid"
-
-for i in $(seq 1 10); do
-    user=/Users/nixbld$i
-    uid="$((30000 + $i))"
-    dscl . create $user
-    dscl . create $user RealName "Nix build user $i"
-    dscl . create $user PrimaryGroupID "$gid"
-    dscl . create $user UserShell /usr/bin/false
-    dscl . create $user NFSHomeDirectory /var/empty
-    dscl . create $user UniqueID "$uid"
-    dseditgroup -o edit -a nixbld$i -t user nixbld
-    echo "created nixbld$i user with uid $uid"
-done
-</programlisting>
-
-</para>
-
 </simplesect>
 
 

From d4e93532e2695fa7204937a96573b686a3b1a1ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Feb 2018 20:46:39 +0100
Subject: [PATCH 0843/2196] Fix incorrect (and unnecessary) format string

https://hydra.nixos.org/eval/1434547#tabs-now-fail
---
 src/libstore/build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 30f22833fa7..1d611ffbaba 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2520,9 +2520,9 @@ void setupSeccomp()
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
 #else
-    throw Error("%s\n%s",
-        "seccomp is not supported on this platform"
-        "you can avoid this by setting the filter-syscalls option to false, but note that untrusted builds can then create setuid binaries!");
+    throw Error(
+        "seccomp is not supported on this platform; "
+        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
 #endif
 #endif
 }

From 056d28a60110975332d8cb0c08990a1d3d8060a7 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 19 Feb 2018 22:47:25 +0200
Subject: [PATCH 0844/2196] libexpr: Don't create lots of temporary strings in
 Bindings::lexicographicOrder

Avoids ~180,000 string temporaries created when evaluating a headless
NixOS system.
---
 src/libexpr/attr-set.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index e1fc2bf6d79..3119a1848af 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -83,7 +83,7 @@ public:
         for (size_t n = 0; n < size_; n++)
             res.emplace_back(&attrs[n]);
         std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) {
-            return (string) a->name < (string) b->name;
+            return (const string &) a->name < (const string &) b->name;
         });
         return res;
     }

From 1d0e42879fa687a7b6856b1a63070e44bd8ed5c4 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 19 Feb 2018 17:32:11 +0200
Subject: [PATCH 0845/2196] libutil: Fix infinite loop in filterANSIEscapes on
 '\r'

E.g. nix-instantiate --eval -E 'abort "\r"' hangs.

Found by afl-fuzz.
---
 src/libutil/util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index f7a12d21b24..341dedfdf03 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1216,7 +1216,7 @@ std::string filterANSIEscapes(const std::string & s, unsigned int width)
 
         else if (*i == '\r')
             // do nothing for now
-            ;
+            i++;
 
         else {
             t += *i++; w++;

From 4ea9707591beceacf9988b3c185faf50da238403 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 19 Feb 2018 17:52:33 +0200
Subject: [PATCH 0846/2196] libexpr: Fix prim_replaceStrings() to work on an
 empty source string

Otherwise, running e.g.

nix-instantiate --eval -E --strict 'builtins.replaceStrings [""] ["X"] "abc"'

would just hang in an infinite loop.

Found by afl-fuzz.
---
 src/libexpr/primops.cc                  | 11 ++++++++---
 tests/lang/eval-okay-replacestrings.exp |  2 +-
 tests/lang/eval-okay-replacestrings.nix |  2 ++
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a800d24290a..ab9351f1185 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1913,21 +1913,26 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     auto s = state.forceString(*args[2], context, pos);
 
     string res;
-    for (size_t p = 0; p < s.size(); ) {
+    // Loops one past last character to handle the case where 'from' contains an empty string.
+    for (size_t p = 0; p <= s.size(); ) {
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
         for (; i != from.end(); ++i, ++j)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
-                p += i->size();
                 res += j->first;
+                if (i->empty()) {
+                    res += s[p++];
+                } else {
+                    p += i->size();
+                }
                 for (auto& path : j->second)
                     context.insert(path);
                 j->second.clear();
                 break;
             }
-        if (!found) res += s[p++];
+        if (!found && p < s.size()) res += s[p++];
     }
 
     mkString(v, res, context);
diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp
index a2add1b7b14..590c281ac86 100644
--- a/tests/lang/eval-okay-replacestrings.exp
+++ b/tests/lang/eval-okay-replacestrings.exp
@@ -1 +1 @@
-[ "faabar" "fbar" "fubar" "faboor" "fubar" ]
+[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" ]
diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix
index 6284a0e660a..c84e9269ff3 100644
--- a/tests/lang/eval-okay-replacestrings.nix
+++ b/tests/lang/eval-okay-replacestrings.nix
@@ -5,4 +5,6 @@ with builtins;
   (replaceStrings ["oo"] ["u"] "foobar")
   (replaceStrings ["oo" "a"] ["a" "oo"] "foobar")
   (replaceStrings ["oo" "oo"] ["u" "i"] "foobar")
+  (replaceStrings [""] ["X"] "abc")
+  (replaceStrings [""] ["X"] "")
 ]

From 546f98dace5c3569211caf392c9dde06a20aa7b0 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 19 Feb 2018 18:44:30 +0200
Subject: [PATCH 0847/2196] libutil: Fix invalid assert on decoding base64
 hashes

The assertion is broken because there is no one-to-one mapping from
length of a base64 string to the length of the output.

E.g.

"1q69lz7Empb06nzfkj651413n9icx0njmyr3xzq1j9q=" results in a 32-byte output.
"1q69lz7Empb06nzfkj651413n9icx0njmyr3xzq1j9qy" results in a 33-byte output.

To reproduce, evaluate:

builtins.derivationStrict {
    name = "0";
    builder = "0";
    system = "0";
    outputHashAlgo = "sha256";
    outputHash = "1q69lz7Empb06nzfkj651413n9icx0njmyr3xzq1j9qy";
}

Found by afl-fuzz.
---
 src/libutil/hash.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 11e3c9dca58..75e4767550f 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -189,7 +189,8 @@ Hash::Hash(const std::string & s, HashType type)
 
     else if (size == base64Len()) {
         auto d = base64Decode(std::string(s, pos));
-        assert(d.size() == hashSize);
+        if (d.size() != hashSize)
+            throw BadHash("invalid base-64 hash '%s'", s);
         memcpy(hash, d.data(), hashSize);
     }
 

From cea4fb3a31de8ae04faf8437c7b0ac6b964a1aad Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Feb 2018 12:33:32 +0100
Subject: [PATCH 0848/2196] Fix evaluation of binaryTarball.aarch64-linux

---
 release.nix | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/release.nix b/release.nix
index 9e04f0b67f9..3f8d5da4721 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? fetchTarball channel:nixos-17.09
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -127,7 +127,6 @@ let
 
     binaryTarball = pkgs.lib.genAttrs systems (system:
 
-      # FIXME: temporarily use a different branch for the Darwin build.
       with import nixpkgs { inherit system; };
 
       let
@@ -137,7 +136,7 @@ let
 
       runCommand "nix-binary-tarball-${version}"
         { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ];
-          buildInputs = [ perl shellcheck ];
+          buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
@@ -150,8 +149,10 @@ let
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
 
-          shellcheck -e SC1090 $TMPDIR/install
-          shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+          if type -p shellcheck; then
+            shellcheck -e SC1090 $TMPDIR/install
+            shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+          fi
 
           chmod +x $TMPDIR/install
           chmod +x $TMPDIR/install-darwin-multi-user

From 4e44025ac5e280c50641d18f1e394c9f120f8bf7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Feb 2018 15:19:46 +0100
Subject: [PATCH 0849/2196] Release notes: Add contributors

---
 doc/manual/release-notes/rl-2.0.xml | 109 +++++++++++++++++++++++++++-
 1 file changed, 105 insertions(+), 4 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 0ad17373ef4..0d5296cc900 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -97,7 +97,7 @@
 
       <listitem>
         <para>Unlike the legacy commands, it has a consistent way to
-        refer to packages and package-like argumements (like store
+        refer to packages and package-like arguments (like store
         paths). For example, the following commands all copy the GNU
         Hello package to a remote machine:
 
@@ -190,7 +190,7 @@
       <listitem>
         <para><command>nix search</command> replaces <command>nix-env
         -qa</command>. It searches the available packages for
-        occurences of a search string in the attribute name, package
+        occurrences of a search string in the attribute name, package
         name or description. Unlike <command>nix-env -qa</command>, it
         has a cache to speed up subsequent searches.</para>
       </listitem>
@@ -878,7 +878,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
   <listitem>
     <para>On Linux, builds are now executed in a user
-    namespace with uid 1000 and gid 100.</para>
+    namespace with UID 1000 and GID 100.</para>
   </listitem>
 
 </itemizedlist>
@@ -908,6 +908,107 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
 </para>
 
-<para>This release has contributions from TBD.</para>
+<para>This release has contributions from
+
+Adrien Devresse,
+Alexander Ried,
+Alex Cruice,
+Alexey Shmalko,
+AmineChikhaoui,
+Andy Wingo,
+Aneesh Agrawal,
+Anthony Cowley,
+Armijn Hemel,
+aszlig,
+Ben Gamari,
+Benjamin Hipple,
+Benjamin Staffin,
+Benno Fünfstück,
+Bjørn Forsman,
+Brian McKenna,
+Charles Strahan,
+Chase Adams,
+Chris Martin,
+Christian Theune,
+Chris Warburton,
+Daiderd Jordan,
+Dan Connolly,
+Daniel Peebles,
+Dan Peebles,
+davidak,
+David McFarland,
+Dmitry Kalinkin,
+Domen Kožar,
+Eelco Dolstra,
+Emery Hemingway,
+Eric Litak,
+Eric Wolf,
+Fabian Schmitthenner,
+Frederik Rietdijk,
+Gabriel Gonzalez,
+Giorgio Gallo,
+Graham Christensen,
+Guillaume Maudoux,
+Harmen,
+Iavael,
+James Broadhead,
+James Earl Douglas,
+Janus Troelsen,
+Jeremy Shaw,
+Joachim Schiele,
+Joe Hermaszewski,
+Joel Moberg,
+Johannes 'fish' Ziemke,
+Jörg Thalheim,
+Jude Taylor,
+kballou,
+Keshav Kini,
+Kjetil Orbekk,
+Langston Barrett,
+Linus Heckemann,
+Ludovic Courtès,
+Manav Rathi,
+Marc Scholten,
+Markus Hauck,
+Matt Audesse,
+Matthew Bauer,
+Matthias Beyer,
+Matthieu Coudron,
+N1X,
+Nathan Zadoks,
+Neil Mayhew,
+Nicolas B. Pierron,
+Niklas Hambüchen,
+Nikolay Amiantov,
+Ole Jørgen Brønner,
+Orivej Desh,
+Peter Simons,
+Peter Stuart,
+Pyry Jahkola,
+regnat,
+Renzo Carbonara,
+Rhys,
+Robert Vollmert,
+Scott Olson,
+Scott R. Parish,
+Sergei Trofimovich,
+Shea Levy,
+Sheena Artrip,
+Spencer Baugh,
+Stefan Junker,
+Susan Potter,
+Thomas Tuegel,
+Timothy Allen,
+Tristan Hume,
+Tuomas Tynkkynen,
+tv,
+Tyson Whitehead,
+Vladimír Čunát,
+Will Dietz,
+wmertens,
+Wout Mertens,
+zimbatm and
+Zoran Plesivčak.
+</para>
 
 </section>

From fa7fd76c5e8edf3b6888ec0f689e8130448987e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 21 Feb 2018 01:12:47 +0000
Subject: [PATCH 0850/2196] nix-copy: fix examples

maybe a left-over from nix-store -r ?
---
 src/nix/copy.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 2ddea9e70a6..f29429c1ac4 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -57,15 +57,15 @@ struct CmdCopy : StorePathsCommand
         return {
             Example{
                 "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
-                "nix copy --to file:///tmp/cache -r $(type -p firefox)"
+                "nix copy --to file:///tmp/cache $(type -p firefox)"
             },
             Example{
                 "To copy the entire current NixOS system closure to another machine via SSH:",
-                "nix copy --to ssh://server -r /run/current-system"
+                "nix copy --to ssh://server /run/current-system"
             },
             Example{
                 "To copy a closure from another machine via SSH:",
-                "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
+                "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
             },
         };
     }

From e2d71bd1862cdda65f957c6a6c41016258d9e003 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Feb 2018 15:34:40 +0100
Subject: [PATCH 0851/2196] Revert "libexpr: Fix prim_replaceStrings() to work
 on an empty source string"

This reverts commit 4ea9707591beceacf9988b3c185faf50da238403.

It causes an infinite loop in Nixpkgs evaluation,
e.g. "nix-instantiate -A hello" hung.

PR #1886.
---
 src/libexpr/primops.cc                  | 11 +++--------
 tests/lang/eval-okay-replacestrings.exp |  2 +-
 tests/lang/eval-okay-replacestrings.nix |  2 --
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ab9351f1185..a800d24290a 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1913,26 +1913,21 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     auto s = state.forceString(*args[2], context, pos);
 
     string res;
-    // Loops one past last character to handle the case where 'from' contains an empty string.
-    for (size_t p = 0; p <= s.size(); ) {
+    for (size_t p = 0; p < s.size(); ) {
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
         for (; i != from.end(); ++i, ++j)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
+                p += i->size();
                 res += j->first;
-                if (i->empty()) {
-                    res += s[p++];
-                } else {
-                    p += i->size();
-                }
                 for (auto& path : j->second)
                     context.insert(path);
                 j->second.clear();
                 break;
             }
-        if (!found && p < s.size()) res += s[p++];
+        if (!found) res += s[p++];
     }
 
     mkString(v, res, context);
diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp
index 590c281ac86..a2add1b7b14 100644
--- a/tests/lang/eval-okay-replacestrings.exp
+++ b/tests/lang/eval-okay-replacestrings.exp
@@ -1 +1 @@
-[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" ]
+[ "faabar" "fbar" "fubar" "faboor" "fubar" ]
diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix
index c84e9269ff3..6284a0e660a 100644
--- a/tests/lang/eval-okay-replacestrings.nix
+++ b/tests/lang/eval-okay-replacestrings.nix
@@ -5,6 +5,4 @@ with builtins;
   (replaceStrings ["oo"] ["u"] "foobar")
   (replaceStrings ["oo" "a"] ["a" "oo"] "foobar")
   (replaceStrings ["oo" "oo"] ["u" "i"] "foobar")
-  (replaceStrings [""] ["X"] "abc")
-  (replaceStrings [""] ["X"] "")
 ]

From 0d54671b7b3aa96ab45347e65352979d874346ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Feb 2018 16:22:49 +0100
Subject: [PATCH 0852/2196] Manual: Update chapter on remote builds

Alos add a command "nix ping-store" to make it easier to see if Nix
can connect to a remote builder (e.g. 'nix ping-store --store
ssh://mac').
---
 .../advanced-topics/distributed-builds.xml    | 211 ++++++++++++------
 doc/manual/command-ref/conf-file.xml          |  15 +-
 src/nix/ping-store.cc                         |  35 +++
 3 files changed, 191 insertions(+), 70 deletions(-)
 create mode 100644 src/nix/ping-store.cc

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index 1957e1105e6..20fd6a0cfb0 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -4,71 +4,109 @@
       version="5.0"
       xml:id='chap-distributed-builds'>
 
-<title>Distributed Builds</title>
-
-<para>Nix supports distributed builds, where a local Nix installation can
-forward Nix builds to other machines over the network.  This allows
-multiple builds to be performed in parallel (thus improving
-performance) and allows Nix to perform multi-platform builds in a
-semi-transparent way.  For instance, if you perform a build for a
-<literal>x86_64-darwin</literal> on an <literal>i686-linux</literal>
-machine, Nix can automatically forward the build to a
-<literal>x86_64-darwin</literal> machine, if available.</para>
-
-<para>You can enable distributed builds by setting the environment
-variable <envar>NIX_BUILD_HOOK</envar> to point to a program that Nix
-will call whenever it wants to build a derivation.  The build hook
-(typically a shell or Perl script) can decline the build, in which Nix
-will perform it in the usual way if possible, or it can accept it, in
-which case it is responsible for somehow getting the inputs of the
-build to another machine, doing the build there, and getting the
-results back.</para>
-
-<example xml:id='ex-remote-systems'><title>Remote machine configuration:
-<filename>remote-systems.conf</filename></title>
-<programlisting>
-nix@mcflurry.labs.cs.uu.nl  x86_64-darwin   /home/nix/.ssh/id_quarterpounder_auto  2
-nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
-nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
-nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2 kvm perf
-</programlisting>
-</example>
-
-<para>Nix ships with a build hook that should be suitable for most
-purposes.  It uses <command>ssh</command> and
-<command>nix-copy-closure</command> to copy the build inputs and
-outputs and perform the remote build.  To use it, you should set
-<envar>NIX_BUILD_HOOK</envar> to
-<filename><replaceable>prefix</replaceable>/libexec/nix/build-remote</filename>.
-You should also define a list of available build machines and point
-the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to
-it. <envar>NIX_REMOTE_SYSTEMS</envar> must be an absolute path. An
-example configuration is shown in <xref linkend='ex-remote-systems'
-/>.  Each line in the file specifies a machine, with the following
-bits of information:
+<title>Remote Builds</title>
+
+<para>Nix supports remote builds, where a local Nix installation can
+forward Nix builds to other machines.  This allows multiple builds to
+be performed in parallel and allows Nix to perform multi-platform
+builds in a semi-transparent way.  For instance, if you perform a
+build for a <literal>x86_64-darwin</literal> on an
+<literal>i686-linux</literal> machine, Nix can automatically forward
+the build to a <literal>x86_64-darwin</literal> machine, if
+available.</para>
+
+<para>To forward a build to a remote machine, it’s required that the
+remote machine is accessible via SSH and that it has Nix
+installed. You can test whether connecting to the remote Nix instance
+works, e.g.
+
+<screen>
+$ nix ping-store --store ssh://mac
+</screen>
+
+will try to connect to the machine named <literal>mac</literal>. It is
+possible to specify an SSH identity file as part of the remote store
+URI, e.g.
+
+<screen>
+$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key
+</screen>
+
+Since builds should be non-interactive, the key should not have a
+passphrase. Alternatively, you can load identities ahead of time into
+<command>ssh-agent</command> or <command>gpg-agent</command>.</para>
+
+<para>If you get the error
+
+<screen>
+bash: nix-store: command not found
+error: cannot connect to 'mac'
+</screen>
+
+then you need to ensure that the <envar>PATH</envar> of
+non-interactive login shells contains Nix.</para>
+
+<warning><para>If you are building via the Nix daemon, it is the Nix
+daemon user account (that is, <literal>root</literal>) that should
+have SSH access to the remote machine. If you can’t or don’t want to
+configure <literal>root</literal> to be able to access to remote
+machine, you can use a private Nix store instead by passing
+e.g. <literal>--store ~/my-nix</literal>.</para></warning>
+
+<para>The list of remote machines can be specified on the command line
+or in the Nix configuration file. The former is convenient for
+testing. For example, the following command allows you to build a
+derivation for <literal>x86_64-darwin</literal> on a Linux machine:
+
+<screen>
+$ uname
+Linux
+
+$ nix build \
+  '(with import &lt;nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+  --builders 'ssh://mac x86_64-darwin'
+[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
+
+$ cat ./result
+Darwin
+</screen>
+
+It is possible to specify multiple builders separated by a semicolon
+or a newline, e.g.
+
+<screen>
+  --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd'
+</screen>
+</para>
+
+<para>Each machine specification consists of the following elements,
+separated by spaces. Only the first element is required.
 
 <orderedlist>
 
-  <listitem><para>The name of the remote machine, with optionally the
-  user under which the remote build should be performed.  This is
-  actually passed as an argument to <command>ssh</command>, so it can
-  be an alias defined in your
+  <listitem><para>The URI of the remote store in the format
+  <literal>ssh://[<replaceable>username</replaceable>@]<replaceable>hostname</replaceable></literal>,
+  e.g. <literal>ssh://nix@mac</literal> or
+  <literal>ssh://mac</literal>. For backward compatibility,
+  <literal>ssh://</literal> may be omitted. The hostname may be an
+  alias defined in your
   <filename>~/.ssh/config</filename>.</para></listitem>
 
   <listitem><para>A comma-separated list of Nix platform type
   identifiers, such as <literal>x86_64-darwin</literal>.  It is
   possible for a machine to support multiple platform types, e.g.,
-  <literal>i686-linux,x86_64-linux</literal>.</para></listitem>
+  <literal>i686-linux,x86_64-linux</literal>. If omitted, this
+  defaults to the local platform type.</para></listitem>
 
-  <listitem><para>The SSH private key to be used to log in to the
-  remote machine.  Since builds should be non-interactive, this key
-  should not have a passphrase!</para></listitem>
+  <listitem><para>The SSH identity file to be used to log in to the
+  remote machine. If omitted, SSH will use its regular
+  identities.</para></listitem>
 
-  <listitem><para>The maximum number of builds that
-  <filename>build-remote</filename> will execute in parallel on the
-  machine.  Typically this should be equal to the number of CPU cores.
-  For instance, the machine <literal>itchy</literal> in the example
-  will execute up to 8 builds in parallel.</para></listitem>
+  <listitem><para>The maximum number of builds that Nix will execute
+  in parallel on the machine.  Typically this should be equal to the
+  number of CPU cores.  For instance, the machine
+  <literal>itchy</literal> in the example will execute up to 8 builds
+  in parallel.</para></listitem>
 
   <listitem><para>The “speed factor”, indicating the relative speed of
   the machine.  If there are multiple machines of the right type, Nix
@@ -76,30 +114,69 @@ bits of information:
 
   <listitem><para>A comma-separated list of <emphasis>supported
   features</emphasis>.  If a derivation has the
-  <varname>requiredSystemFeatures</varname> attribute, then
-  <filename>build-remote</filename> will only perform the
-  derivation on a machine that has the specified features.  For
-  instance, the attribute
+  <varname>requiredSystemFeatures</varname> attribute, then Nix will
+  only perform the derivation on a machine that has the specified
+  features.  For instance, the attribute
 
 <programlisting>
 requiredSystemFeatures = [ "kvm" ];
 </programlisting>
 
   will cause the build to be performed on a machine that has the
-  <literal>kvm</literal> feature (i.e., <literal>scratchy</literal> in
-  the example above).</para></listitem>
+  <literal>kvm</literal> feature.</para></listitem>
 
   <listitem><para>A comma-separated list of <emphasis>mandatory
   features</emphasis>.  A machine will only be used to build a
   derivation if all of the machine’s mandatory features appear in the
-  derivation’s <varname>requiredSystemFeatures</varname> attribute.
-  Thus, in the example, the machine <literal>poochie</literal> will
-  only do derivations that have
-  <varname>requiredSystemFeatures</varname> set to <literal>["kvm"
-  "perf"]</literal> or <literal>["perf"]</literal>.</para></listitem>
+  derivation’s <varname>requiredSystemFeatures</varname>
+  attribute..</para></listitem>
 
 </orderedlist>
 
-</para>
+For example, the machine specification
+
+<programlisting>
+nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        8 1 kvm
+nix@itchy.labs.cs.uu.nl     i686-linux      /home/nix/.ssh/id_scratchy_auto        8 2
+nix@poochie.labs.cs.uu.nl   i686-linux      /home/nix/.ssh/id_scratchy_auto        1 2 kvm benchmark
+</programlisting>
+
+specifies several machines that can perform
+<literal>i686-linux</literal> builds. However,
+<literal>poochie</literal> will only do builds that have the attribute
+
+<programlisting>
+requiredSystemFeatures = [ "benchmark" ];
+</programlisting>
+
+or
+
+<programlisting>
+requiredSystemFeatures = [ "benchmark" "kvm" ];
+</programlisting>
+
+<literal>itchy</literal> cannot do builds that require
+<literal>kvm</literal>, but <literal>scratchy</literal> does support
+such builds. For regular builds, <literal>itchy</literal> will be
+preferred over <literal>scratchy</literal> because it has a higher
+speed factor.</para>
+
+<para>Remote builders can also be configured in
+<filename>nix.conf</filename>, e.g.
+
+<programlisting>
+builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd
+</programlisting>
+
+Finally, remote builders can be configured in a separate configuration
+file included in <option>builders</option> via the syntax
+<literal>@<replaceable>file</replaceable></literal>. For example,
+
+<programlisting>
+builders = @/etc/nix/machines
+</programlisting>
+
+causes the list of machines in <filename>/etc/nix/machines</filename>
+to be included. (This is the default.)</para>
 
 </chapter>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c14a4d206b8..f658058994c 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -763,7 +763,7 @@ builtins.fetchurl {
         more details.
       </para>
       <para>
-	Since these files are loaded into the same address space as
+        Since these files are loaded into the same address space as
         Nix itself, they must be DSOs compatible with the instance of
         Nix running at the time (i.e. compiled against the same
         headers, not linked to any incompatible libraries). They
@@ -771,14 +771,23 @@ builtins.fetchurl {
         be available already at load time.
       </para>
       <para>
-	If an entry in the list is a directory, all files in the
-	directory are loaded as plugins (non-recursively).
+        If an entry in the list is a directory, all files in the
+        directory are loaded as plugins (non-recursively).
       </para>
     </listitem>
 
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-builders">
+    <term><literal>builders</literal></term>
+    <listitem>
+      <para>A list of machines on which to perform builds. <phrase
+      condition="manual">See <xref linkend="chap-distributed-builds"
+      /> for details.</phrase></para>
+    </listitem>
+  </varlistentry>
+
 </variablelist>
 
 </para>
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
new file mode 100644
index 00000000000..310942574a2
--- /dev/null
+++ b/src/nix/ping-store.cc
@@ -0,0 +1,35 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdPingStore : StoreCommand
+{
+    std::string name() override
+    {
+        return "ping-store";
+    }
+
+    std::string description() override
+    {
+        return "test whether a store can be opened";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To test whether connecting to a remote Nix store via SSH works:",
+                "nix ping-store --store ssh://mac1"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        store->connect();
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdPingStore>());

From 88c90d5e6d3865dffedef8a83f24473aefc08646 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Feb 2018 18:08:47 +0100
Subject: [PATCH 0853/2196] Manual: Put configuration options in sorted order

---
 doc/manual/command-ref/conf-file.xml | 853 +++++++++++++--------------
 1 file changed, 419 insertions(+), 434 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index f658058994c..c76640c97e7 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -63,147 +63,99 @@ false</literal>.</para>
 <variablelist>
 
 
-  <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
-
-    <listitem><para>If <literal>true</literal>, the garbage collector
-    will keep the outputs of non-garbage derivations.  If
-    <literal>false</literal> (default), outputs will be deleted unless
-    they are GC roots themselves (or reachable from other roots).</para>
-
-    <para>In general, outputs must be registered as roots separately.
-    However, even if the output of a derivation is registered as a
-    root, the collector will still delete store paths that are used
-    only at build time (e.g., the C compiler, or source tarballs
-    downloaded from the network).  To prevent it from doing so, set
-    this option to <literal>true</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term>
-
-    <listitem><para>If <literal>true</literal> (default), the garbage
-    collector will keep the derivations from which non-garbage store
-    paths were built.  If <literal>false</literal>, they will be
-    deleted unless explicitly registered as a root (or reachable from
-    other roots).</para>
-
-    <para>Keeping derivation around is useful for querying and
-    traceability (e.g., it allows you to ask with what dependencies or
-    options a store path was built), so by default this option is on.
-    Turn it off to save a bit of disk space (or a lot if
-    <literal>keep-outputs</literal> is also turned on).</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>keep-env-derivations</literal></term>
+  <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term>
 
-    <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivation
-    any build-time-only dependencies may be garbage-collected.</para>
+    <listitem>
 
-    <para>If <literal>true</literal>, when you add a Nix derivation to
-    a user environment, the path of the derivation is stored in the
-    user environment.  Thus, the derivation will not be
-    garbage-collected until the user environment generation is deleted
-    (<command>nix-env --delete-generations</command>).  To prevent
-    build-time-only dependencies from being collected, you should also
-    turn on <literal>keep-outputs</literal>.</para>
+      <para>A list of URI prefixes to which access is allowed in
+      restricted evaluation mode. For example, when set to
+      <literal>https://github.com/NixOS</literal>, builtin functions
+      such as <function>fetchGit</function> are allowed to access
+      <literal>https://github.com/NixOS/patchelf.git</literal>.</para>
 
-    <para>The difference between this option and
-    <literal>keep-derivations</literal> is that this one is
-    “sticky”: it applies to any user environment created while this
-    option was enabled, while <literal>keep-derivations</literal>
-    only applies at the moment the garbage collector is
-    run.</para></listitem>
+    </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
+  <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
 
-    <listitem><para>This option defines the maximum number of jobs
-    that Nix will try to build in parallel.  The default is
-    <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  It can be
-    overridden using the <option
-    linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
-    command line switch.</para></listitem>
+    <listitem><para>By default, Nix allows you to <function>import</function> from a derivation,
+    allowing building at evaluation time. With this option set to false, Nix will throw an error
+    when evaluating an expression that uses this feature, allowing users to ensure their evaluation
+    will not require any builds to take place.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
+  <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term>
 
-    <listitem><para>Sets the value of the
-    <envar>NIX_BUILD_CORES</envar> environment variable in the
-    invocation of builders.  Builders can use this variable at their
-    discretion to control the maximum amount of parallelism.  For
-    instance, in Nixpkgs, if the derivation attribute
-    <varname>enableParallelBuilding</varname> is set to
-    <literal>true</literal>, the builder passes the
-    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
-    It can be overridden using the <option
-    linkend='opt-cores'>--cores</option> command line switch and
-    defaults to <literal>1</literal>.  The value <literal>0</literal>
-    means that the builder should use all available CPU cores in the
-    system.</para></listitem>
+    <listitem><para>(Linux-specific.) By default, builders on Linux
+    cannot acquire new privileges by calling setuid/setgid programs or
+    programs that have file capabilities. For example, programs such
+    as <command>sudo</command> or <command>ping</command> will
+    fail. (Note that in sandbox builds, no such programs are available
+    unless you bind-mount them into the sandbox via the
+    <option>sandbox-paths</option> option.) You can allow the
+    use of such programs by enabling this option. This is impure and
+    usually undesirable, but may be useful in certain scenarios
+    (e.g. to spin up containers or set up userspace network interfaces
+    in tests).</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
+  <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term>
 
     <listitem>
 
-      <para>This option defines the maximum number of seconds that a
-      builder can go without producing any data on standard output or
-      standard error.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite
-      loop, or to catch remote builds that are hanging due to network
-      problems.  It can be overridden using the <option
-      linkend="opt-max-silent-time">--max-silent-time</option> command
-      line switch.</para>
+      <para>A list of names of users (separated by whitespace) that
+      are allowed to connect to the Nix daemon. As with the
+      <option>trusted-users</option> option, you can specify groups by
+      prefixing them with <literal>@</literal>. Also, you can allow
+      all users by specifying <literal>*</literal>. The default is
+      <literal>*</literal>.</para>
 
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
+      <para>Note that trusted users are always allowed to connect.</para>
 
     </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
+  <varlistentry><term><literal>auto-optimise-store</literal></term>
 
-    <listitem>
+    <listitem><para>If set to <literal>true</literal>, Nix
+    automatically detects files in the store that have identical
+    contents, and replaces them with hard links to a single copy.
+    This saves disk space.  If set to <literal>false</literal> (the
+    default), you can still run <command>nix-store
+    --optimise</command> to get rid of duplicate
+    files.</para></listitem>
 
-      <para>This option defines the maximum number of seconds that a
-      builder can run.  This is useful (for instance in an automated
-      build system) to catch builds that are stuck in an infinite loop
-      but keep writing to their standard output or standard error.  It
-      can be overridden using the <option
-      linkend="opt-timeout">--timeout</option> command line
-      switch.</para>
+  </varlistentry>
 
-      <para>The value <literal>0</literal> means that there is no
-      timeout.  This is also the default.</para>
 
+  <varlistentry xml:id="conf-builders">
+    <term><literal>builders</literal></term>
+    <listitem>
+      <para>A list of machines on which to perform builds. <phrase
+      condition="manual">See <xref linkend="chap-distributed-builds"
+      /> for details.</phrase></para>
     </listitem>
-
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
-
-    <listitem>
-
-      <para>This option defines the maximum number of bytes that a
-      builder can write to its stdout/stderr.  If the builder exceeds
-      this limit, it’s killed.  A value of <literal>0</literal> (the
-      default) means that there is no limit.</para>
+  <varlistentry><term><literal>builders-use-substitutes</literal></term>
 
-    </listitem>
+    <listitem><para>If set to <literal>true</literal>, Nix will instruct
+    remote build machines to use their own binary substitutes if available. In
+    practical terms, this means that remote hosts will fetch as many build
+    dependencies as possible from their own substitutes (e.g, from
+    <literal>cache.nixos.org</literal>), instead of waiting for this host to
+    upload them all. This can drastically reduce build times if the network
+    connection between this computer and the remote build host is slow. Defaults
+    to <literal>false</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -249,66 +201,51 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>sandbox</literal></term>
+  <varlistentry><term><literal>compress-build-log</literal></term>
 
-    <listitem><para>If set to <literal>true</literal>, builds will be
-    performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
-    they’re isolated from the normal file system hierarchy and will
-    only see their dependencies in the Nix store, the temporary build
-    directory, private versions of <filename>/proc</filename>,
-    <filename>/dev</filename>, <filename>/dev/shm</filename> and
-    <filename>/dev/pts</filename> (on Linux), and the paths configured with the
-    <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal>
-    option</link>. This is useful to prevent undeclared dependencies
-    on files in directories such as <filename>/usr/bin</filename>. In
-    addition, on Linux, builds run in private PID, mount, network, IPC
-    and UTS namespaces to isolate them from other processes in the
-    system (except that fixed-output derivations do not run in private
-    network namespace to ensure they can access the network).</para>
+    <listitem><para>If set to <literal>true</literal> (the default),
+    build logs written to <filename>/nix/var/log/nix/drvs</filename>
+    will be compressed on the fly using bzip2.  Otherwise, they will
+    not be compressed.</para></listitem>
 
-    <para>Currently, sandboxing only work on Linux and macOS. The use
-    of a sandbox requires that Nix is run as root (so you should use
-    the <link linkend='conf-build-users-group'>“build users”
-    feature</link> to perform the actual builds under different users
-    than root).</para>
+  </varlistentry>
 
-    <para>If this option is set to <literal>relaxed</literal>, then
-    fixed-output derivations and derivations that have the
-    <varname>__noChroot</varname> attribute set to
-    <literal>true</literal> do not run in sandboxes.</para>
 
-    <para>The default is <literal>false</literal>.</para>
+  <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
+
+    <listitem>
+
+      <para>The timeout (in seconds) for establishing connections in
+      the binary cache substituter.  It corresponds to
+      <command>curl</command>’s <option>--connect-timeout</option>
+      option.</para>
 
     </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-sandbox-paths">
-    <term><literal>sandbox-paths</literal></term>
-
-    <listitem><para>A list of paths bind-mounted into Nix sandbox
-    environments. You can use the syntax
-    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
-    to mount a path in a different location in the sandbox; for
-    instance, <literal>/bin=/nix-bin</literal> will mount the path
-    <literal>/nix-bin</literal> as <literal>/bin</literal> inside the
-    sandbox. If <replaceable>source</replaceable> is followed by
-    <literal>?</literal>, then it is not an error if
-    <replaceable>source</replaceable> does not exist; for example,
-    <literal>/dev/nvidiactl?</literal> specifies that
-    <filename>/dev/nvidiactl</filename> will only be mounted in the
-    sandbox if it exists in the host filesystem.</para>
+  <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term>
 
-    <para>Depending on how Nix was built, the default value for this option
-    may be empty or provide <filename>/bin/sh</filename> as a
-    bind-mount of <command>bash</command>.</para></listitem>
+    <listitem><para>Sets the value of the
+    <envar>NIX_BUILD_CORES</envar> environment variable in the
+    invocation of builders.  Builders can use this variable at their
+    discretion to control the maximum amount of parallelism.  For
+    instance, in Nixpkgs, if the derivation attribute
+    <varname>enableParallelBuilding</varname> is set to
+    <literal>true</literal>, the builder passes the
+    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+    It can be overridden using the <option
+    linkend='opt-cores'>--cores</option> command line switch and
+    defaults to <literal>1</literal>.  The value <literal>0</literal>
+    means that the builder should use all available CPU cores in the
+    system.</para></listitem>
 
   </varlistentry>
 
 
   <varlistentry xml:id="conf-extra-sandbox-paths">
-    <term><literal>build-extra-sandbox-paths</literal></term>
+    <term><literal>extra-sandbox-paths</literal></term>
 
     <listitem><para>A list of additional paths appended to
     <option>sandbox-paths</option>. Useful if you want to extend
@@ -317,30 +254,18 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>substitute</literal></term>
+  <varlistentry><term><literal>extra-substituters</literal></term>
 
-    <listitem><para>If set to <literal>true</literal> (default), Nix
-    will use binary substitutes if available.  This option can be
-    disabled to force building from source.</para></listitem>
+    <listitem><para>Additional binary caches appended to those
+    specified in <option>substituters</option>.  When used by
+    unprivileged users, untrusted substituters (i.e. those not listed
+    in <option>trusted-substituters</option>) are silently
+    ignored.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>builders-use-substitutes</literal></term>
-
-    <listitem><para>If set to <literal>true</literal>, Nix will instruct
-    remote build machines to use their own binary substitutes if available. In
-    practical terms, this means that remote hosts will fetch as many build
-    dependencies as possible from their own substitutes (e.g, from
-    <literal>cache.nixos.org</literal>), instead of waiting for this host to
-    upload them all. This can drastically reduce build times if the network
-    connection between this computer and the remote build host is slow. Defaults
-    to <literal>false</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>fallback</literal></term>
+  <varlistentry><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
     back to building from source if a binary substitute fails.  This
@@ -350,110 +275,41 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>keep-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    Nix will write the build log of a derivation (i.e. the standard
-    output and error of its builder) to the directory
-    <filename>/nix/var/log/nix/drvs</filename>.  The build log can be
-    retrieved using the command <command>nix-store -l
-    <replaceable>path</replaceable></command>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>compress-build-log</literal></term>
-
-    <listitem><para>If set to <literal>true</literal> (the default),
-    build logs written to <filename>/nix/var/log/nix/drvs</filename>
-    will be compressed on the fly using bzip2.  Otherwise, they will
-    not be compressed.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  The default is
-    <literal>https://cache.nixos.org</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <!--
-  <varlistentry><term><literal>binary-caches-files</literal></term>
-
-    <listitem><para>A list of names of files that will be read to
-    obtain additional binary cache URLs.  The default is
-    <literal>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels/binary-caches/*</literal>.
-    Note that when you’re using the Nix daemon,
-    <replaceable>username</replaceable> is always equal to
-    <literal>root</literal>, so Nix will only use the binary caches
-    provided by the channels installed by root.  Do not set this
-    option to read files created by untrusted users!</para></listitem>
-
-  </varlistentry>
-  -->
-
-
-  <varlistentry><term><literal>trusted-substituters</literal></term>
-
-    <listitem><para>A list of URLs of substituters, separated by
-    whitespace.  These are not used by default, but can be enabled by
-    users of the Nix daemon by specifying <literal>--option
-    substituters <replaceable>urls</replaceable></literal> on the
-    command line.  Unprivileged users are only allowed to pass a
-    subset of the URLs listed in <literal>substituters</literal> and
-    <literal>trusted-substituters</literal>.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>extra-substituters</literal></term>
-
-    <listitem><para>Additional binary caches appended to those
-    specified in <option>substituters</option>.  When used by
-    unprivileged users, untrusted substituters (i.e. those not listed
-    in <option>trusted-substituters</option>) are silently
-    ignored.</para></listitem>
-
-  </varlistentry>
-
-
-  <varlistentry><term><literal>require-sigs</literal></term>
+  <varlistentry><term><literal>fsync-metadata</literal></term>
 
-    <listitem><para>If set to <literal>true</literal> (the default),
-    any non-content-addressed path added or copied to the Nix store
-    (e.g. when substituting from a binary cache) must have a valid
-    signature, that is, be signed using one of the keys listed in
-    <option>trusted-public-keys</option> or
-    <option>secret-key-files</option>. Set to <literal>false</literal>
-    to disable signature checking.</para></listitem>
+    <listitem><para>If set to <literal>true</literal>, changes to the
+    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
+    synchronously flushed to disk.  This improves robustness in case
+    of system crashes, but reduces performance.  The default is
+    <literal>true</literal>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>trusted-public-keys</literal></term>
-
-    <listitem><para>A whitespace-separated list of public keys. When
-    paths are copied from another Nix store (such as a binary cache),
-    they must be signed with one of these keys. For example:
-    <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
-
-  </varlistentry>
+  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
 
+    <listitem><para>A list of web servers used by
+    <function>builtins.fetchurl</function> to obtain files by
+    hash. The default is
+    <literal>http://tarballs.nixos.org/</literal>. Given a hash type
+    <replaceable>ht</replaceable> and a base-16 hash
+    <replaceable>h</replaceable>, Nix will try to download the file
+    from
+    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
+    This allows files to be downloaded even if they have disappeared
+    from their original URI. For example, given the default mirror
+    <literal>http://tarballs.nixos.org/</literal>, when building the derivation
 
-  <varlistentry><term><literal>secret-key-files</literal></term>
+<programlisting>
+builtins.fetchurl {
+  url = https://example.org/foo-1.2.3.tar.xz;
+  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+}
+</programlisting>
 
-    <listitem><para>A whitespace-separated list of files containing
-    secret (private) keys. These are used to sign locally-built
-    paths. They can be generated using <command>nix-store
-    --generate-binary-cache-key</command>. The corresponding public
-    key can be distributed to other users, who can add it to
-    <option>trusted-public-keys</option> in their
-    <filename>nix.conf</filename>.</para></listitem>
+    Nix will attempt to download this file from
+    <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
+    first. If it is not available there, if will try the original URI.</para></listitem>
 
   </varlistentry>
 
@@ -467,157 +323,174 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>netrc-file</literal></term>
-
-    <listitem><para>If set to an absolute path to a <filename>netrc</filename>
-    file, Nix will use the HTTP authentication credentials in this file when
-    trying to download from a remote host through HTTP or HTTPS. Defaults to
-    <filename>$NIX_CONF_DIR/netrc</filename>.</para>
-
-    <para>The <filename>netrc</filename> file consists of a list of
-    accounts in the following format:
-
-<screen>
-machine <replaceable>my-machine</replaceable>
-login <replaceable>my-username</replaceable>
-password <replaceable>my-password</replaceable>
-</screen>
+  <varlistentry><term><literal>keep-build-log</literal></term>
 
-    For the exact syntax, see <link
-    xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
-    <literal>curl</literal> documentation.</link></para></listitem>
+    <listitem><para>If set to <literal>true</literal> (the default),
+    Nix will write the build log of a derivation (i.e. the standard
+    output and error of its builder) to the directory
+    <filename>/nix/var/log/nix/drvs</filename>.  The build log can be
+    retrieved using the command <command>nix-store -l
+    <replaceable>path</replaceable></command>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>system</literal></term>
+  <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term>
 
-    <listitem><para>This option specifies the canonical Nix system
-    name of the current installation, such as
-    <literal>i686-linux</literal> or
-    <literal>x86_64-darwin</literal>.  Nix can only build derivations
-    whose <literal>system</literal> attribute equals the value
-    specified here.  In general, it never makes sense to modify this
-    value from its default, since you can use it to ‘lie’ about the
-    platform you are building on (e.g., perform a Mac OS build on a
-    Linux machine; the result would obviously be wrong).  It only
-    makes sense if the Nix binaries can run on multiple platforms,
-    e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and
-    <literal>i686-linux</literal>.</para>
+    <listitem><para>If <literal>true</literal> (default), the garbage
+    collector will keep the derivations from which non-garbage store
+    paths were built.  If <literal>false</literal>, they will be
+    deleted unless explicitly registered as a root (or reachable from
+    other roots).</para>
 
-    <para>It defaults to the canonical Nix system name detected by
-    <filename>configure</filename> at build time.</para></listitem>
+    <para>Keeping derivation around is useful for querying and
+    traceability (e.g., it allows you to ask with what dependencies or
+    options a store path was built), so by default this option is on.
+    Turn it off to save a bit of disk space (or a lot if
+    <literal>keep-outputs</literal> is also turned on).</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>fsync-metadata</literal></term>
+  <varlistentry><term><literal>keep-env-derivations</literal></term>
 
-    <listitem><para>If set to <literal>true</literal>, changes to the
-    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
-    synchronously flushed to disk.  This improves robustness in case
-    of system crashes, but reduces performance.  The default is
-    <literal>true</literal>.</para></listitem>
+    <listitem><para>If <literal>false</literal> (default), derivations
+    are not stored in Nix user environments.  That is, the derivation
+    any build-time-only dependencies may be garbage-collected.</para>
+
+    <para>If <literal>true</literal>, when you add a Nix derivation to
+    a user environment, the path of the derivation is stored in the
+    user environment.  Thus, the derivation will not be
+    garbage-collected until the user environment generation is deleted
+    (<command>nix-env --delete-generations</command>).  To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on <literal>keep-outputs</literal>.</para>
+
+    <para>The difference between this option and
+    <literal>keep-derivations</literal> is that this one is
+    “sticky”: it applies to any user environment created while this
+    option was enabled, while <literal>keep-derivations</literal>
+    only applies at the moment the garbage collector is
+    run.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry><term><literal>auto-optimise-store</literal></term>
+  <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
 
-    <listitem><para>If set to <literal>true</literal>, Nix
-    automatically detects files in the store that have identical
-    contents, and replaces them with hard links to a single copy.
-    This saves disk space.  If set to <literal>false</literal> (the
-    default), you can still run <command>nix-store
-    --optimise</command> to get rid of duplicate
-    files.</para></listitem>
+    <listitem><para>If <literal>true</literal>, the garbage collector
+    will keep the outputs of non-garbage derivations.  If
+    <literal>false</literal> (default), outputs will be deleted unless
+    they are GC roots themselves (or reachable from other roots).</para>
+
+    <para>In general, outputs must be registered as roots separately.
+    However, even if the output of a derivation is registered as a
+    root, the collector will still delete store paths that are used
+    only at build time (e.g., the C compiler, or source tarballs
+    downloaded from the network).  To prevent it from doing so, set
+    this option to <literal>true</literal>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
+  <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
 
     <listitem>
 
-      <para>The timeout (in seconds) for establishing connections in
-      the binary cache substituter.  It corresponds to
-      <command>curl</command>’s <option>--connect-timeout</option>
-      option.</para>
+      <para>This option defines the maximum number of bytes that a
+      builder can write to its stdout/stderr.  If the builder exceeds
+      this limit, it’s killed.  A value of <literal>0</literal> (the
+      default) means that there is no limit.</para>
 
     </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
-
-    <listitem>
-
-      <para>A list of names of users (separated by whitespace) that
-      have additional rights when connecting to the Nix daemon, such
-      as the ability to specify additional binary caches, or to import
-      unsigned NARs. You can also specify groups by prefixing them
-      with <literal>@</literal>; for instance,
-      <literal>@wheel</literal> means all users in the
-      <literal>wheel</literal> group. The default is
-      <literal>root</literal>.</para>
-
-      <warning><para>Adding a user to <option>trusted-users</option>
-      is essentially equivalent to giving that user root access to the
-      system. For example, the user can set
-      <option>sandbox-paths</option> and thereby obtain read access to
-      directories that are otherwise inacessible to
-      them.</para></warning>
+  <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
 
-    </listitem>
+    <listitem><para>This option defines the maximum number of jobs
+    that Nix will try to build in parallel.  The default is
+    <literal>1</literal>. The special value <literal>auto</literal>
+    causes Nix to use the number of CPUs in your system.  It can be
+    overridden using the <option
+    linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
+    command line switch.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term>
+  <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
 
     <listitem>
 
-      <para>A list of names of users (separated by whitespace) that
-      are allowed to connect to the Nix daemon. As with the
-      <option>trusted-users</option> option, you can specify groups by
-      prefixing them with <literal>@</literal>. Also, you can allow
-      all users by specifying <literal>*</literal>. The default is
-      <literal>*</literal>.</para>
+      <para>This option defines the maximum number of seconds that a
+      builder can go without producing any data on standard output or
+      standard error.  This is useful (for instance in an automated
+      build system) to catch builds that are stuck in an infinite
+      loop, or to catch remote builds that are hanging due to network
+      problems.  It can be overridden using the <option
+      linkend="opt-max-silent-time">--max-silent-time</option> command
+      line switch.</para>
 
-      <para>Note that trusted users are always allowed to connect.</para>
+      <para>The value <literal>0</literal> means that there is no
+      timeout.  This is also the default.</para>
 
     </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term>
+  <varlistentry><term><literal>netrc-file</literal></term>
 
-    <listitem>
+    <listitem><para>If set to an absolute path to a <filename>netrc</filename>
+    file, Nix will use the HTTP authentication credentials in this file when
+    trying to download from a remote host through HTTP or HTTPS. Defaults to
+    <filename>$NIX_CONF_DIR/netrc</filename>.</para>
 
-      <para>If set to <literal>true</literal>, the Nix evaluator will
-      not allow access to any files outside of the Nix search path (as
-      set via the <envar>NIX_PATH</envar> environment variable or the
-      <option>-I</option> option), or to URIs outside of
-      <option>allowed-uri</option>. The default is
-      <literal>false</literal>.</para>
+    <para>The <filename>netrc</filename> file consists of a list of
+    accounts in the following format:
 
-    </listitem>
+<screen>
+machine <replaceable>my-machine</replaceable>
+login <replaceable>my-username</replaceable>
+password <replaceable>my-password</replaceable>
+</screen>
 
-  </varlistentry>
+    For the exact syntax, see <link
+    xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
+    <literal>curl</literal> documentation.</link></para></listitem>
 
+  </varlistentry>
 
-  <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term>
 
+  <varlistentry xml:id="conf-plugin-files">
+    <term><literal>plugin-files</literal></term>
     <listitem>
-
-      <para>A list of URI prefixes to which access is allowed in
-      restricted evaluation mode. For example, when set to
-      <literal>https://github.com/NixOS</literal>, builtin functions
-      such as <function>fetchGit</function> are allowed to access
-      <literal>https://github.com/NixOS/patchelf.git</literal>.</para>
-
+      <para>
+        A list of plugin files to be loaded by Nix. Each of these
+        files will be dlopened by Nix, allowing them to affect
+        execution through static initialization. In particular, these
+        plugins may construct static instances of RegisterPrimOp to
+        add new primops or constants to the expression language,
+        RegisterStoreImplementation to add new store implementations,
+        RegisterCommand to add new subcommands to the
+        <literal>nix</literal> command, and RegisterSetting to add new
+        nix config settings. See the constructors for those types for
+        more details.
+      </para>
+      <para>
+        Since these files are loaded into the same address space as
+        Nix itself, they must be DSOs compatible with the instance of
+        Nix running at the time (i.e. compiled against the same
+        headers, not linked to any incompatible libraries). They
+        should not be linked to any Nix libs directly, as those will
+        be available already at load time.
+      </para>
+      <para>
+        If an entry in the list is a directory, all files in the
+        directory are loaded as plugins (non-recursively).
+      </para>
     </listitem>
 
   </varlistentry>
@@ -671,6 +544,70 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry><term><literal>require-sigs</literal></term>
+
+    <listitem><para>If set to <literal>true</literal> (the default),
+    any non-content-addressed path added or copied to the Nix store
+    (e.g. when substituting from a binary cache) must have a valid
+    signature, that is, be signed using one of the keys listed in
+    <option>trusted-public-keys</option> or
+    <option>secret-key-files</option>. Set to <literal>false</literal>
+    to disable signature checking.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term>
+
+    <listitem>
+
+      <para>If set to <literal>true</literal>, the Nix evaluator will
+      not allow access to any files outside of the Nix search path (as
+      set via the <envar>NIX_PATH</envar> environment variable or the
+      <option>-I</option> option), or to URIs outside of
+      <option>allowed-uri</option>. The default is
+      <literal>false</literal>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>sandbox</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, builds will be
+    performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
+    they’re isolated from the normal file system hierarchy and will
+    only see their dependencies in the Nix store, the temporary build
+    directory, private versions of <filename>/proc</filename>,
+    <filename>/dev</filename>, <filename>/dev/shm</filename> and
+    <filename>/dev/pts</filename> (on Linux), and the paths configured with the
+    <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal>
+    option</link>. This is useful to prevent undeclared dependencies
+    on files in directories such as <filename>/usr/bin</filename>. In
+    addition, on Linux, builds run in private PID, mount, network, IPC
+    and UTS namespaces to isolate them from other processes in the
+    system (except that fixed-output derivations do not run in private
+    network namespace to ensure they can access the network).</para>
+
+    <para>Currently, sandboxing only work on Linux and macOS. The use
+    of a sandbox requires that Nix is run as root (so you should use
+    the <link linkend='conf-build-users-group'>“build users”
+    feature</link> to perform the actual builds under different users
+    than root).</para>
+
+    <para>If this option is set to <literal>relaxed</literal>, then
+    fixed-output derivations and derivations that have the
+    <varname>__noChroot</varname> attribute set to
+    <literal>true</literal> do not run in sandboxes.</para>
+
+    <para>The default is <literal>false</literal>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
 
     <listitem><para>This option determines the maximum size of the
@@ -684,110 +621,158 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term>
+  <varlistentry xml:id="conf-sandbox-paths">
+    <term><literal>sandbox-paths</literal></term>
 
-    <listitem><para>By default, Nix allows you to <function>import</function> from a derivation,
-    allowing building at evaluation time. With this option set to false, Nix will throw an error
-    when evaluating an expression that uses this feature, allowing users to ensure their evaluation
-    will not require any builds to take place.</para></listitem>
+    <listitem><para>A list of paths bind-mounted into Nix sandbox
+    environments. You can use the syntax
+    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
+    to mount a path in a different location in the sandbox; for
+    instance, <literal>/bin=/nix-bin</literal> will mount the path
+    <literal>/nix-bin</literal> as <literal>/bin</literal> inside the
+    sandbox. If <replaceable>source</replaceable> is followed by
+    <literal>?</literal>, then it is not an error if
+    <replaceable>source</replaceable> does not exist; for example,
+    <literal>/dev/nvidiactl?</literal> specifies that
+    <filename>/dev/nvidiactl</filename> will only be mounted in the
+    sandbox if it exists in the host filesystem.</para>
+
+    <para>Depending on how Nix was built, the default value for this option
+    may be empty or provide <filename>/bin/sh</filename> as a
+    bind-mount of <command>bash</command>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term>
+  <varlistentry><term><literal>secret-key-files</literal></term>
 
-    <listitem><para>(Linux-specific.) By default, builders on Linux
-    cannot acquire new privileges by calling setuid/setgid programs or
-    programs that have file capabilities. For example, programs such
-    as <command>sudo</command> or <command>ping</command> will
-    fail. (Note that in sandbox builds, no such programs are available
-    unless you bind-mount them into the sandbox via the
-    <option>sandbox-paths</option> option.) You can allow the
-    use of such programs by enabling this option. This is impure and
-    usually undesirable, but may be useful in certain scenarios
-    (e.g. to spin up containers or set up userspace network interfaces
-    in tests).</para></listitem>
+    <listitem><para>A whitespace-separated list of files containing
+    secret (private) keys. These are used to sign locally-built
+    paths. They can be generated using <command>nix-store
+    --generate-binary-cache-key</command>. The corresponding public
+    key can be distributed to other users, who can add it to
+    <option>trusted-public-keys</option> in their
+    <filename>nix.conf</filename>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
+  <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term>
 
-    <listitem><para>A list of web servers used by
-    <function>builtins.fetchurl</function> to obtain files by
-    hash. The default is
-    <literal>http://tarballs.nixos.org/</literal>. Given a hash type
-    <replaceable>ht</replaceable> and a base-16 hash
-    <replaceable>h</replaceable>, Nix will try to download the file
-    from
-    <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
-    This allows files to be downloaded even if they have disappeared
-    from their original URI. For example, given the default mirror
-    <literal>http://tarballs.nixos.org/</literal>, when building the derivation
+    <listitem><para>Causes Nix to print out a stack trace in case of Nix
+    expression evaluation errors.</para></listitem>
 
-<programlisting>
-builtins.fetchurl {
-  url = https://example.org/foo-1.2.3.tar.xz;
-  sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
+  </varlistentry>
 
-    Nix will attempt to download this file from
-    <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
-    first. If it is not available there, if will try the original URI.</para></listitem>
+
+  <varlistentry><term><literal>substitute</literal></term>
+
+    <listitem><para>If set to <literal>true</literal> (default), Nix
+    will use binary substitutes if available.  This option can be
+    disabled to force building from source.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term>
+  <varlistentry><term><literal>substituters</literal></term>
 
-    <listitem><para>Causes Nix to print out a stack trace in case of Nix
-    expression evaluation errors.</para></listitem>
+    <listitem><para>A list of URLs of substituters, separated by
+    whitespace.  The default is
+    <literal>https://cache.nixos.org</literal>.</para></listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-plugin-files">
-    <term><literal>plugin-files</literal></term>
+  <varlistentry><term><literal>system</literal></term>
+
+    <listitem><para>This option specifies the canonical Nix system
+    name of the current installation, such as
+    <literal>i686-linux</literal> or
+    <literal>x86_64-darwin</literal>.  Nix can only build derivations
+    whose <literal>system</literal> attribute equals the value
+    specified here.  In general, it never makes sense to modify this
+    value from its default, since you can use it to ‘lie’ about the
+    platform you are building on (e.g., perform a Mac OS build on a
+    Linux machine; the result would obviously be wrong).  It only
+    makes sense if the Nix binaries can run on multiple platforms,
+    e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and
+    <literal>i686-linux</literal>.</para>
+
+    <para>It defaults to the canonical Nix system name detected by
+    <filename>configure</filename> at build time.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
+
     <listitem>
-      <para>
-        A list of plugin files to be loaded by Nix. Each of these
-        files will be dlopened by Nix, allowing them to affect
-        execution through static initialization. In particular, these
-        plugins may construct static instances of RegisterPrimOp to
-        add new primops or constants to the expression language,
-        RegisterStoreImplementation to add new store implementations,
-        RegisterCommand to add new subcommands to the
-        <literal>nix</literal> command, and RegisterSetting to add new
-        nix config settings. See the constructors for those types for
-        more details.
-      </para>
-      <para>
-        Since these files are loaded into the same address space as
-        Nix itself, they must be DSOs compatible with the instance of
-        Nix running at the time (i.e. compiled against the same
-        headers, not linked to any incompatible libraries). They
-        should not be linked to any Nix libs directly, as those will
-        be available already at load time.
-      </para>
-      <para>
-        If an entry in the list is a directory, all files in the
-        directory are loaded as plugins (non-recursively).
-      </para>
+
+      <para>This option defines the maximum number of seconds that a
+      builder can run.  This is useful (for instance in an automated
+      build system) to catch builds that are stuck in an infinite loop
+      but keep writing to their standard output or standard error.  It
+      can be overridden using the <option
+      linkend="opt-timeout">--timeout</option> command line
+      switch.</para>
+
+      <para>The value <literal>0</literal> means that there is no
+      timeout.  This is also the default.</para>
+
     </listitem>
 
   </varlistentry>
 
 
-  <varlistentry xml:id="conf-builders">
-    <term><literal>builders</literal></term>
+  <varlistentry><term><literal>trusted-public-keys</literal></term>
+
+    <listitem><para>A whitespace-separated list of public keys. When
+    paths are copied from another Nix store (such as a binary cache),
+    they must be signed with one of these keys. For example:
+    <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>trusted-substituters</literal></term>
+
+    <listitem><para>A list of URLs of substituters, separated by
+    whitespace.  These are not used by default, but can be enabled by
+    users of the Nix daemon by specifying <literal>--option
+    substituters <replaceable>urls</replaceable></literal> on the
+    command line.  Unprivileged users are only allowed to pass a
+    subset of the URLs listed in <literal>substituters</literal> and
+    <literal>trusted-substituters</literal>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
+
     <listitem>
-      <para>A list of machines on which to perform builds. <phrase
-      condition="manual">See <xref linkend="chap-distributed-builds"
-      /> for details.</phrase></para>
+
+      <para>A list of names of users (separated by whitespace) that
+      have additional rights when connecting to the Nix daemon, such
+      as the ability to specify additional binary caches, or to import
+      unsigned NARs. You can also specify groups by prefixing them
+      with <literal>@</literal>; for instance,
+      <literal>@wheel</literal> means all users in the
+      <literal>wheel</literal> group. The default is
+      <literal>root</literal>.</para>
+
+      <warning><para>Adding a user to <option>trusted-users</option>
+      is essentially equivalent to giving that user root access to the
+      system. For example, the user can set
+      <option>sandbox-paths</option> and thereby obtain read access to
+      directories that are otherwise inacessible to
+      them.</para></warning>
+
     </listitem>
+
   </varlistentry>
 
+
 </variablelist>
 
 </para>

From ddc58e789636e1b94149c342575d92583251fbf6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 12:27:25 +0100
Subject: [PATCH 0854/2196] Enable sandbox builds on Linux by default

The overhead of sandbox builds is a problem on NixOS (since building a
NixOS configuration involves a lot of small derivations) but not for
typical non-NixOS use cases. So outside of NixOS we can enable it.

Issue #179.
---
 doc/manual/manual.xml               | 4 +---
 doc/manual/release-notes/rl-2.0.xml | 4 ++++
 src/libstore/globals.hh             | 8 +++++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml
index b5a6af7d0c3..b408b681772 100644
--- a/doc/manual/manual.xml
+++ b/doc/manual/manual.xml
@@ -16,12 +16,10 @@
     </author>
 
     <copyright>
-      <year>2004-2017</year>
+      <year>2004-2018</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>November 2014</date>
-
   </info>
 
   <!--
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 0d5296cc900..f0f44ae260b 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -809,6 +809,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
 <itemizedlist>
 
+  <listitem>
+    <para>Sandbox builds are now enabled by default on Linux.</para>
+  </listitem>
+
   <listitem>
     <para>Values such as Booleans, integers, (nested) lists and
     attribute sets can <link
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index dd01f832df0..12917a438fa 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -220,7 +220,13 @@ public:
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
+    Setting<SandboxMode> sandboxMode{this,
+        #if __linux__
+          smEnabled
+        #else
+          smDisabled
+        #endif
+        , "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
 

From eaa52c34b40aa2f899280e6aa99d5fa0a420ad3a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 12:44:46 +0100
Subject: [PATCH 0855/2196] Set release date

---
 doc/manual/release-notes/rl-2.0.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index f0f44ae260b..bf06d3786c2 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -4,7 +4,7 @@
       version="5.0"
       xml:id="ssec-relnotes-2.0">
 
-<title>Release 2.0 (2018-02-??)</title>
+<title>Release 2.0 (2018-02-22)</title>
 
 <para>The following incompatible changes have been made:</para>
 

From ddbcd01c835419cc40f648670f28646a8f51bd15 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 25 Jan 2018 06:52:51 -0800
Subject: [PATCH 0856/2196] Fix restricted mode when installing in
 non-canonical data dir

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 33a9bc61428..60f22a2033f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -308,7 +308,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
+    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs"));
 
     if (settings.restrictEval || settings.pureEval) {
         allowedPaths = PathSet();

From e8d53bfdc94eea142542de92ff0822eb283d293f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 14:19:20 +0100
Subject: [PATCH 0857/2196] Revert "Enable sandbox builds on Linux by default"

This reverts commit ddc58e789636e1b94149c342575d92583251fbf6.

https://hydra.nixos.org/eval/1435322
---
 doc/manual/release-notes/rl-2.0.xml | 4 ----
 src/libstore/globals.hh             | 8 +-------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index bf06d3786c2..25b9beb009b 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -809,10 +809,6 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
 <itemizedlist>
 
-  <listitem>
-    <para>Sandbox builds are now enabled by default on Linux.</para>
-  </listitem>
-
   <listitem>
     <para>Values such as Booleans, integers, (nested) lists and
     attribute sets can <link
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 12917a438fa..dd01f832df0 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -220,13 +220,7 @@ public:
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<SandboxMode> sandboxMode{this,
-        #if __linux__
-          smEnabled
-        #else
-          smDisabled
-        #endif
-        , "sandbox",
+    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
 

From b6dec2f3ca312431e2e06f0f3564880ef78e6bc6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 17:11:17 +0100
Subject: [PATCH 0858/2196] Release script tweaks

---
 maintainers/upload-release.pl | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 313ff93d4c8..aa7633a709c 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -6,6 +6,7 @@
 use File::Basename;
 use File::Path;
 use File::Slurp;
+use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
 
@@ -54,7 +55,7 @@ sub downloadFile {
 
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
 
-    my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die;
+    my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
     my $dstFile = "$releaseDir/" . $dstName;
 
@@ -78,9 +79,8 @@ sub downloadFile {
     return ($dstFile, $sha256_expected);
 }
 
-downloadFile("tarball", "2"); # PDF
-downloadFile("tarball", "3"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "4"); # .tar.xz
+downloadFile("tarball", "2"); # .tar.bz2
+my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
 my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1");
 my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1");
 my ($tarball_aarch64_linux, $tarball_aarch64_linux_hash) = downloadFile("binaryTarball.aarch64-linux", "1");
@@ -152,4 +152,9 @@ sub getStorePath {
 
 system("cd $siteDir && nix-shell --run 'make nix/install nix/install.sig'") == 0 or die;
 
+copy("$siteDir/nix/install", "$siteDir/nix/install-$version") or die;
+copy("$siteDir/nix/install.sig", "$siteDir/nix/install-$version.sig") or die;
+
+system("cd $siteDir && git add nix/install-$version nix/install-$version.sig") == 0 or die;
+
 system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

From 2691d51a33b73d2aa22a520fe7feab85e170d104 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 17:14:35 +0100
Subject: [PATCH 0859/2196] Doh

---
 doc/manual/release-notes/rl-2.0.xml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 25b9beb009b..94098bd864c 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -323,13 +323,12 @@
 
       <listitem>
         <para><command>nix sign-paths</command> signs store
-        paths. (TODO: add examples)</para>
+        paths.</para>
       </listitem>
 
       <listitem>
         <para><command>nix copy-sigs</command> copies signatures from
-        one store to another. (TODO: add examples and
-        tests)</para>
+        one store to another.</para>
       </listitem>
 
       <listitem>
@@ -460,7 +459,7 @@
         secret keys specified by the <option>secret-key-files</option>
         store option. Secret/public key pairs can be generated using
         <command>nix-store
-        --generate-binary-cache-key</command>. (TODO: rename)</para>
+        --generate-binary-cache-key</command>.</para>
 
         <para>In addition, locally-built store paths are marked as
         “ultimately trusted”, but this bit is not propagated when
@@ -723,8 +722,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
 
     will cause the <envar>configureFlags</envar> environment variable
     to contain the actual store paths corresponding to the
-    <literal>out</literal> and <literal>dev</literal> outputs. TODO:
-    add docs.</para>
+    <literal>out</literal> and <literal>dev</literal> outputs.</para>
   </listitem>
 
 </itemizedlist>

From 39b4177500e8375554bd2012f6f51f6021c881c1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Feb 2018 17:42:06 +0100
Subject: [PATCH 0860/2196] Fix example in release notes

---
 doc/manual/release-notes/rl-2.0.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index 94098bd864c..bf2e93e2e16 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -42,7 +42,7 @@
     <para>The command <command>nix-push</command> has been removed as
     part of the effort to eliminate Nix's dependency on Perl. You can
     use <command>nix copy</command> instead, e.g. <literal>nix copy
-    --to /tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
+    --to file:///tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para>
   </listitem>
 
   <listitem>

From 64e486ab63a87b18922bbdb8d2414e74afabb8db Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 23 Feb 2018 13:00:42 +0100
Subject: [PATCH 0861/2196] Fix downloadCached() with a chroot store

E.g.

  nix run --store ~/my-nix -f channel:nixos-17.03 hello -c hello

This problem was mentioned in #1897.
---
 src/libstore/download.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 258d7937cc3..5ab625f4228 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -717,7 +717,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"});
+            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
             unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
@@ -727,7 +727,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     if (expectedStorePath != "" && storePath != expectedStorePath)
         throw nix::Error("store path mismatch in file downloaded from '%s'", url);
 
-    return storePath;
+    return store->toRealPath(storePath);
 }
 
 

From d9d8a84a96bc09443c4a12266c6f511807c07d15 Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Fri, 23 Feb 2018 23:45:42 -0500
Subject: [PATCH 0862/2196] Fix a small typo in the release notes

---
 doc/manual/release-notes/rl-2.0.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index bf2e93e2e16..8aa449ea8a5 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -35,7 +35,7 @@
     available paths from the installation medium. The replacement is
     to use a chroot store as a substituter
     (e.g. <literal>--substituters /mnt</literal>), or to build into a
-    chroot store (e.g. <literal>--store /mnt --substituter /</literal>).</para>
+    chroot store (e.g. <literal>--store /mnt --substituters /</literal>).</para>
   </listitem>
 
   <listitem>

From bd94e63853e995608da30548890aa2baa26f3fb9 Mon Sep 17 00:00:00 2001
From: Michael Weiss <dev.primeos@gmail.com>
Date: Sat, 24 Feb 2018 16:06:17 +0100
Subject: [PATCH 0863/2196] doc: Fix a URL in the release notes for Nix 2.0

---
 doc/manual/release-notes/rl-2.0.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index bf2e93e2e16..1cd1585e99a 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -536,7 +536,7 @@
     xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no
     longer</link> a fatal error if build rounds produce different
     output. Also, a hook named <option>diff-hook</option> is <link
-    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169w">provided</link>
+    xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169">provided</link>
     to allow you to run tools such as <command>diffoscope</command>
     when build rounds produce different output.</para>
   </listitem>

From 8282c60d742dc6725723370249a341a944575d6c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sun, 25 Feb 2018 16:33:17 -0600
Subject: [PATCH 0864/2196] tests: test nix search behavior

---
 tests/local.mk   |  3 ++-
 tests/search.nix | 25 +++++++++++++++++++++++++
 tests/search.sh  | 42 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 tests/search.nix
 create mode 100644 tests/search.sh

diff --git a/tests/local.mk b/tests/local.mk
index ec7ebfb0ded..9df0adf1bfd 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -24,7 +24,8 @@ nix_tests = \
   brotli.sh \
   pure-eval.sh \
   check.sh \
-  plugins.sh
+  plugins.sh \
+  search.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/search.nix b/tests/search.nix
new file mode 100644
index 00000000000..fea6e7a7a64
--- /dev/null
+++ b/tests/search.nix
@@ -0,0 +1,25 @@
+with import ./config.nix;
+
+{
+  hello = mkDerivation rec {
+    name = "hello-${version}";
+    version = "0.1";
+    buildCommand = "touch $out";
+    meta.description = "Empty file";
+  };
+  foo = mkDerivation rec {
+    name = "foo-5";
+    buildCommand = ''
+      mkdir -p $out
+      echo ${name} > $out/${name}
+    '';
+  };
+  bar = mkDerivation rec {
+    name = "bar-3";
+    buildCommand = ''
+      echo "Does not build successfully"
+      exit 1
+    '';
+    meta.description = "broken bar";
+  };
+}
diff --git a/tests/search.sh b/tests/search.sh
new file mode 100644
index 00000000000..53a69a0bb01
--- /dev/null
+++ b/tests/search.sh
@@ -0,0 +1,42 @@
+source common.sh
+
+clearStore
+clearCache
+
+# No packages
+(( $(NIX_PATH= nix search -u|wc -l) == 0 ))
+
+# Haven't updated cache, still nothing
+(( $(nix search -f search.nix hello|wc -l) == 0 ))
+(( $(nix search -f search.nix |wc -l) == 0 ))
+
+# Update cache, search should work
+(( $(nix search -f search.nix -u hello|wc -l) > 0 ))
+
+# Use cache
+(( $(nix search -f search.nix foo|wc -l) > 0 ))
+(( $(nix search foo|wc -l) > 0 ))
+
+# Test --no-cache works
+# No results from cache
+(( $(nix search --no-cache foo |wc -l) == 0 ))
+# Does find results from file pointed at
+(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 ))
+
+# Check descriptions are searched
+(( $(nix search broken | wc -l) > 0 ))
+
+# Check search that matches nothing
+(( $(nix search nosuchpackageexists | wc -l) == 0 ))
+
+
+## Search expressions
+
+# Check that empty search string matches all
+EMPTY=$(nix search)
+BOL=$(nix search "^")
+ALL=$(nix search ".*")
+
+diff <(printf "%s" $EMPTY) <(printf "%s" $BOL)
+# "ALL" highlights differently, not sure preferred behavior
+# diff <(printf "%s" $EMPTY) <(printf "%s" $ALL)

From 3cac8734acb098488bd232bcee92099410d4d167 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sun, 25 Feb 2018 16:32:36 -0600
Subject: [PATCH 0865/2196] nix search: fix bug where we wrote to cache when
 shouldn't, breaking

This is exposed by the tests added previously,
and resolves the error reported in #1892:
"expected JSON value".
---
 src/nix/search.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 87cdb2d7ed8..1adde28cb3c 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -237,7 +237,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                     throw Error("error writing to %s", tmpFile);
             }
 
-            if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
+            if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
     }

From 009cf9cd23e5b9198cf8977cca7c823b6b95ad5b Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sun, 25 Feb 2018 16:38:13 -0600
Subject: [PATCH 0866/2196] nix search: explicitly handle empty search string,
 fixes #1893

This is important since this is given as an example.
Other patterns containing "empty search string" will still
be handled differently on different platforms ("asdf|")
but that's less of an issue.
---
 src/nix/search.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 1adde28cb3c..009dfae9cf4 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -78,6 +78,11 @@ struct CmdSearch : SourceExprCommand, MixJSON
     {
         settings.readOnlyMode = true;
 
+        // Empty search string should match all packages
+        // Use "^" here instead of ".*" due to differences in resulting highlighting
+        // (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
+        if (re.empty()) re = "^";
+
         std::regex regex(re, std::regex::extended | std::regex::icase);
 
         auto state = getEvalState();

From 24ec7500032c2434e450b7bc3f77ff4c1f12c41c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Feb 2018 18:29:40 +0100
Subject: [PATCH 0867/2196] nix run: Fix segfault on macOS

Note that clearenv() is not available on macOS.

Fixes #1907.
---
 src/libutil/util.cc | 7 +++++++
 src/libutil/util.hh | 3 +++
 src/nix/run.cc      | 4 +---
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 341dedfdf03..2391e14a94b 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -73,6 +73,13 @@ std::map<std::string, std::string> getEnv()
 }
 
 
+void clearEnv()
+{
+    for (auto & name : getEnv())
+        unsetenv(name.first.c_str());
+}
+
+
 Path absPath(Path path, Path dir)
 {
     if (path[0] != '/') {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 47e02bc898a..c5c537ee63d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -32,6 +32,9 @@ string getEnv(const string & key, const string & def = "");
 /* Get the entire environment. */
 std::map<std::string, std::string> getEnv();
 
+/* Clear the environment. */
+void clearEnv();
+
 /* Return an absolutized path, resolving paths relative to the
    specified directory, or the current directory otherwise.  The path
    is also canonicalised. */
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 822654daf48..d04e106e037 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -16,8 +16,6 @@ using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
 
-extern char * * environ;
-
 struct CmdRun : InstallablesCommand
 {
     std::vector<std::string> command = { "bash" };
@@ -109,7 +107,7 @@ struct CmdRun : InstallablesCommand
                 if (s) kept[var] = s;
             }
 
-            environ = nullptr;
+            clearEnv();
 
             for (auto & var : kept)
                 setenv(var.first.c_str(), var.second.c_str(), 1);

From 77e9e1ed91182cb8409d325e225a99decb49b3d5 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 19 Feb 2018 17:52:33 +0200
Subject: [PATCH 0868/2196] libexpr: Fix prim_replaceStrings() to work on an
 empty source string

Otherwise, running e.g.

nix-instantiate --eval -E --strict 'builtins.replaceStrings [""] ["X"] "abc"'

would just hang in an infinite loop.

Found by afl-fuzz.

First attempt of this was reverted in e2d71bd1862cdda because it caused
another infinite loop, which is fixed now and a test added.
---
 src/libexpr/primops.cc                  | 17 ++++++++++++++---
 tests/lang/eval-okay-replacestrings.exp |  2 +-
 tests/lang/eval-okay-replacestrings.nix |  3 +++
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a800d24290a..db258ea186e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1913,21 +1913,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     auto s = state.forceString(*args[2], context, pos);
 
     string res;
-    for (size_t p = 0; p < s.size(); ) {
+    // Loops one past last character to handle the case where 'from' contains an empty string.
+    for (size_t p = 0; p <= s.size(); ) {
         bool found = false;
         auto i = from.begin();
         auto j = to.begin();
         for (; i != from.end(); ++i, ++j)
             if (s.compare(p, i->size(), *i) == 0) {
                 found = true;
-                p += i->size();
                 res += j->first;
+                if (i->empty()) {
+                    if (p < s.size())
+                        res += s[p];
+                    p++;
+                } else {
+                    p += i->size();
+                }
                 for (auto& path : j->second)
                     context.insert(path);
                 j->second.clear();
                 break;
             }
-        if (!found) res += s[p++];
+        if (!found) {
+            if (p < s.size())
+                res += s[p];
+            p++;
+        }
     }
 
     mkString(v, res, context);
diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp
index a2add1b7b14..72e8274d8c5 100644
--- a/tests/lang/eval-okay-replacestrings.exp
+++ b/tests/lang/eval-okay-replacestrings.exp
@@ -1 +1 @@
-[ "faabar" "fbar" "fubar" "faboor" "fubar" ]
+[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ]
diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix
index 6284a0e660a..bd8031fc004 100644
--- a/tests/lang/eval-okay-replacestrings.nix
+++ b/tests/lang/eval-okay-replacestrings.nix
@@ -5,4 +5,7 @@ with builtins;
   (replaceStrings ["oo"] ["u"] "foobar")
   (replaceStrings ["oo" "a"] ["a" "oo"] "foobar")
   (replaceStrings ["oo" "oo"] ["u" "i"] "foobar")
+  (replaceStrings [""] ["X"] "abc")
+  (replaceStrings [""] ["X"] "")
+  (replaceStrings ["-"] ["_"] "a-b")
 ]

From eb03a296c1a538111056ce0d554911410c4ccb48 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 20 Feb 2018 12:34:50 +0000
Subject: [PATCH 0869/2196] Add build-extra-platforms setting

This allows specifying additional systems that a machine is able to
build for. This may apply on some armv7-capable aarch64 processors, or
on systems using qemu-user with binfmt-misc to support transparent
execution of foreign-arch programs.

This removes the previous hard-coded assumptions about which systems are
ABI-compatible with which other systems, and instead relies on the user
to specify any additional platforms that they have ensured compatibility
for and wish to build for locally.

NixOS should probably add i686-linux on x86_64-linux systems for this
setting by default.
---
 src/libstore/derivations.cc | 12 ++----------
 src/libstore/globals.hh     |  3 +++
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index a0a0d78b7d3..74b861281ee 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -57,16 +57,8 @@ bool BasicDerivation::isBuiltin() const
 bool BasicDerivation::canBuildLocally() const
 {
     return platform == settings.thisSystem
-        || isBuiltin()
-#if __linux__
-        || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
-        || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux")
-        || (platform == "armv5tel-linux" && (settings.thisSystem == "armv7l-linux" || settings.thisSystem == "armv6l-linux"))
-#elif __FreeBSD__
-        || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd")
-        || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd")
-#endif
-        ;
+        || settings.extraPlatforms.get().count(platform) > 0
+        || isBuiltin();
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index dd01f832df0..b382da01e97 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -295,6 +295,9 @@ public:
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};
 
+    Setting<StringSet> extraPlatforms{this, StringSet{}, "build-extra-platforms",
+        "Additional platforms that can be built on the local system, e.g. using qemu-user."};
+
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",

From 919c3c20b3ebbf8b83b8de28b612d09270e7a2a6 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 20 Feb 2018 12:30:03 +0000
Subject: [PATCH 0870/2196] seccomp: add 32-bit ARM on aarch64-linux

This allows building armv[67]l-linux derivations on compatible aarch64
machines. Failure to add the architecture may result from missing
hardware support, in which case we can't run 32-bit binaries and don't
need to restrict them with seccomp anyway,
---
 src/libstore/build.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1d611ffbaba..2466f9bd6b0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2491,6 +2491,10 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
         throw SysError("unable to add X32 seccomp architecture");
 
+    if (settings.thisSystem == "aarch64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
+        printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes.");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,

From 14ca85688cdd17dfcf4b2bd8c324f21d6f3d1dde Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 28 Feb 2018 06:19:40 -0500
Subject: [PATCH 0871/2196] Actually fix nixDataDir in non-canonical path

---
 src/libexpr/eval.cc    | 2 +-
 src/libexpr/primops.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 63afccbec18..f94c23ea72b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -313,7 +313,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs"));
+    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
     if (settings.restrictEval || settings.pureEval) {
         allowedPaths = PathSet();
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index db258ea186e..6778023f506 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2236,7 +2236,7 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix";
+    string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
     sDerivationNix = symbols.create(path);
     evalFile(path, v);
     addConstant("derivation", v);

From 05cb8e5c5a954fa999ac5156cdf19148a05fdb2b Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 28 Feb 2018 19:58:48 +0100
Subject: [PATCH 0872/2196] launchd: enable keepalive for the nix-daemon
 service

Without this the daemon won't be restarted if the process ever dies, for
example when sending a SIGHUP to reload nix.conf.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 66fcd155ee9..549619a57d5 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -4,6 +4,8 @@
   <dict>
     <key>Label</key>
     <string>org.nixos.nix-daemon</string>
+    <key>KeepAlive</key>
+    <true/>
     <key>RunAtLoad</key>
     <true/>
     <key>Program</key>

From e89d02bf031277bd81a8ce255f49d352763e5c71 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 28 Feb 2018 15:03:54 -0600
Subject: [PATCH 0873/2196] fetchGit: use "HEAD" as default ref

---
 src/libexpr/primops/fetchGit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 2e3e2634db8..9fc0d46626d 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -79,7 +79,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         ref = "HEAD"s;
     }
 
-    if (!ref) ref = "master"s;
+    if (!ref) ref = "HEAD"s;
 
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);

From ad97d1a78653373fad8dc3ed98c3607862581b76 Mon Sep 17 00:00:00 2001
From: Michael Fiano <mfiano@users.noreply.github.com>
Date: Thu, 1 Mar 2018 00:27:25 -0500
Subject: [PATCH 0874/2196] nix-channel grammar and punctuation

Minor changes to the nix-channel manpage for my first contribution
---
 doc/manual/command-ref/nix-channel.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
index 9acf44e5298..ff4021a765e 100644
--- a/doc/manual/command-ref/nix-channel.xml
+++ b/doc/manual/command-ref/nix-channel.xml
@@ -31,7 +31,7 @@
 
 <refsection><title>Description</title>
 
-<para>A Nix channel is mechanism that allows you to automatically stay
+<para>A Nix channel is a mechanism that allows you to automatically stay
 up-to-date with a set of pre-built Nix expressions.  A Nix channel is
 just a URL that points to a place containing both a set of Nix
 expressions and a pointer to a binary cache.  <phrase
@@ -165,8 +165,8 @@ following files:</para>
   <varlistentry><term><filename>nixexprs.tar.xz</filename></term>
 
     <listitem><para>A tarball containing Nix expressions and files
-    referenced by them (such as build scripts and patches). At
-    top-level, the tarball should contain a single directory. That
+    referenced by them (such as build scripts and patches). At the
+    top level, the tarball should contain a single directory. That
     directory must contain a file <filename>default.nix</filename>
     that serves as the channel’s “entry point”.</para></listitem>
 
@@ -175,7 +175,7 @@ following files:</para>
   <varlistentry><term><filename>binary-cache-url</filename></term>
 
     <listitem><para>A file containing the URL to a binary cache (such
-    as <uri>https://cache.nixos.org</uri>. Nix will automatically
+    as <uri>https://cache.nixos.org</uri>). Nix will automatically
     check this cache for pre-built binaries, if the user has
     sufficient rights to add binary caches. For instance, in a
     multi-user Nix setup, the binary caches provided by the channels

From c577186f5916c90193368492f1c4180a1386febc Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 1 Mar 2018 13:24:03 -0600
Subject: [PATCH 0875/2196] tests/search.sh: simplify, don't rely on consistent
 ordering

---
 tests/search.sh | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/search.sh b/tests/search.sh
index 53a69a0bb01..d83427247d3 100644
--- a/tests/search.sh
+++ b/tests/search.sh
@@ -33,10 +33,6 @@ clearCache
 ## Search expressions
 
 # Check that empty search string matches all
-EMPTY=$(nix search)
-BOL=$(nix search "^")
-ALL=$(nix search ".*")
-
-diff <(printf "%s" $EMPTY) <(printf "%s" $BOL)
-# "ALL" highlights differently, not sure preferred behavior
-# diff <(printf "%s" $EMPTY) <(printf "%s" $ALL)
+nix search|grep -q foo
+nix search|grep -q bar
+nix search|grep -q hello

From 429154b74c7f4b4d679ae79bbc939049f537bfaf Mon Sep 17 00:00:00 2001
From: Matthew O'Gorman <mog@rldn.net>
Date: Wed, 6 Jan 2016 20:15:19 -0500
Subject: [PATCH 0876/2196] Implement --delete-generations + flag for keeping
 last N number of generations

---
 doc/manual/command-ref/nix-env.xml |  7 +++++--
 src/libstore/profiles.cc           | 23 +++++++++++++++++++++++
 src/libstore/profiles.hh           |  2 ++
 src/nix-env/nix-env.cc             |  2 ++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index d4563ac4755..f0e70a41c02 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1346,9 +1346,10 @@ $ nix-env --list-generations
 <para>This operation deletes the specified generations of the current
 profile.  The generations can be a list of generation numbers, the
 special value <literal>old</literal> to delete all non-current
-generations, or a value such as <literal>30d</literal> to delete all
+generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
-generation that was active at that point in time).
+generation that was active at that point in time), or a value such as.
+<literal>+5</literal> to delete all but the number of items specified.
 Periodically deleting old generations is important to make garbage
 collection effective.</para>
 
@@ -1359,6 +1360,8 @@ collection effective.</para>
 <screen>
 $ nix-env --delete-generations 3 4 8
 
+$ nix-env --delete-generations +5
+
 $ nix-env --delete-generations 30d
 
 $ nix-env -p other_profile --delete-generations old</screen>
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 4a607b58450..e6300cf0555 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -157,6 +157,29 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     }
 }
 
+void deleteGenerationsGreaterThan(const Path & profile, const string & max, bool dryRun)
+{
+    int max_keep = 0;
+    PathLocks lock;
+    if(max.size() < 2)
+     throw Error(format("invalid number of generations ‘%1%’") % max);
+    string str_max = string(max, 1, max.size());
+    if (!string2Int(str_max, max_keep) || max_keep == 0)
+      throw Error(format("invalid number of generations to keep ‘%1%’") % max);
+
+    lockProfile(lock, profile);
+
+    int curGen;
+    Generations gens = findGenerations(profile, curGen);
+
+    for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
+      if (max_keep) {
+	max_keep--;
+	continue;
+      }
+      deleteGeneration2(profile, i->number, dryRun);
+    }
+}
 
 void deleteOldGenerations(const Path & profile, bool dryRun)
 {
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 1d4e6d3037d..3cad08dc526 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -39,6 +39,8 @@ void deleteGeneration(const Path & profile, unsigned int gen);
 
 void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
 
+void deleteGenerationsGreaterThan(const Path & profile, const string & max, bool dryRun);
+
 void deleteOldGenerations(const Path & profile, bool dryRun);
 
 void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 97e66cbd937..1440b7744fd 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1284,6 +1284,8 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         deleteOldGenerations(globals.profile, globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) {
         deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
+    } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) {
+        deleteGenerationsGreaterThan(globals.profile, opArgs.front(), globals.dryRun);
     } else {
         std::set<unsigned int> gens;
         for (auto & i : opArgs) {

From 0312d30315a3a77ab659b742b76ec32685145715 Mon Sep 17 00:00:00 2001
From: Matthew O'Gorman <mog@rldn.net>
Date: Thu, 19 May 2016 15:42:54 -0400
Subject: [PATCH 0877/2196] this updates issues that were addressed by people
 in pr

---
 src/libstore/profiles.cc | 17 ++++++-----------
 src/libstore/profiles.hh |  2 +-
 src/nix-env/nix-env.cc   |  8 +++++++-
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index e6300cf0555..44c3c6e0d47 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -157,15 +157,10 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     }
 }
 
-void deleteGenerationsGreaterThan(const Path & profile, const string & max, bool dryRun)
+void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
 {
     int max_keep = 0;
     PathLocks lock;
-    if(max.size() < 2)
-     throw Error(format("invalid number of generations ‘%1%’") % max);
-    string str_max = string(max, 1, max.size());
-    if (!string2Int(str_max, max_keep) || max_keep == 0)
-      throw Error(format("invalid number of generations to keep ‘%1%’") % max);
 
     lockProfile(lock, profile);
 
@@ -173,11 +168,11 @@ void deleteGenerationsGreaterThan(const Path & profile, const string & max, bool
     Generations gens = findGenerations(profile, curGen);
 
     for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
-      if (max_keep) {
-	max_keep--;
-	continue;
-      }
-      deleteGeneration2(profile, i->number, dryRun);
+        if (max) {
+            max--;
+            continue;
+        }
+        deleteGeneration2(profile, i->number, dryRun);
     }
 }
 
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 3cad08dc526..5fa1533de31 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -39,7 +39,7 @@ void deleteGeneration(const Path & profile, unsigned int gen);
 
 void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
 
-void deleteGenerationsGreaterThan(const Path & profile, const string & max, bool dryRun);
+void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
 
 void deleteOldGenerations(const Path & profile, bool dryRun);
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 1440b7744fd..0da31588d23 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1285,7 +1285,13 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
     } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) {
         deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) {
-        deleteGenerationsGreaterThan(globals.profile, opArgs.front(), globals.dryRun);
+        if(opArgs.front().size() < 2)
+            throw Error(format("invalid number of generations ‘%1%’") % opArgs.front());
+        string str_max = string(opArgs.front() 1, opArgs.front().size());
+        int max;
+        if (!string2Int(str_max, max) || max == 0)
+            throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front());
+        deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
     } else {
         std::set<unsigned int> gens;
         for (auto & i : opArgs) {

From 12fe2249e1b32dc9033e2680017a4c7f236edaac Mon Sep 17 00:00:00 2001
From: Matt O'Gorman <mog-lists@rldn.net>
Date: Thu, 21 Sep 2017 21:41:19 -0400
Subject: [PATCH 0878/2196] Update nix-env.cc

missing comma
---
 src/nix-env/nix-env.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 0da31588d23..f6a9d9e7b7c 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1287,7 +1287,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
     } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) {
         if(opArgs.front().size() < 2)
             throw Error(format("invalid number of generations ‘%1%’") % opArgs.front());
-        string str_max = string(opArgs.front() 1, opArgs.front().size());
+        string str_max = string(opArgs.front(), 1, opArgs.front().size());
         int max;
         if (!string2Int(str_max, max) || max == 0)
             throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front());

From 3c16044cb0acc87e128534cb3e2a2006b71059b6 Mon Sep 17 00:00:00 2001
From: Matthew O'Gorman <mog@rldn.net>
Date: Thu, 1 Mar 2018 22:22:02 -0500
Subject: [PATCH 0879/2196] remove unused variable and make sure to check that
 the current generation is not the one we are deleting

---
 src/libstore/profiles.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 44c3c6e0d47..b43ec66f60c 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -159,7 +159,6 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
 
 void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
 {
-    int max_keep = 0;
     PathLocks lock;
 
     lockProfile(lock, profile);
@@ -172,7 +171,8 @@ void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
             max--;
             continue;
         }
-        deleteGeneration2(profile, i->number, dryRun);
+        if (i->number != curGen)
+            deleteGeneration2(profile, i->number, dryRun);
     }
 }
 

From 467fdd8ca4d63972dbd94f0496918522c58916a0 Mon Sep 17 00:00:00 2001
From: Matthew O'Gorman <mog@rldn.net>
Date: Thu, 1 Mar 2018 22:59:00 -0500
Subject: [PATCH 0880/2196] only delete things older than current gen and
 update logic in doc as well

---
 doc/manual/command-ref/nix-env.xml | 10 +++++-----
 src/libstore/profiles.cc           | 13 +++++++++----
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index f0e70a41c02..7eb9cc855ba 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1349,9 +1349,9 @@ special value <literal>old</literal> to delete all non-current
 generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
 generation that was active at that point in time), or a value such as.
-<literal>+5</literal> to delete all but the number of items specified.
-Periodically deleting old generations is important to make garbage
-collection effective.</para>
+<literal>+5</literal> to only keep the specified items older than the
+current generation. Periodically deleting old generations is important
+to make garbage collection effective.</para>
 
 </refsection>
 
@@ -1461,7 +1461,7 @@ error: no generation older than the current (91) exists</screen>
 <refsection condition="manpage"><title>Environment variables</title>
 
 <variablelist>
-  
+
   <varlistentry><term><envar>NIX_PROFILE</envar></term>
 
     <listitem><para>Location of the Nix profile.  Defaults to the
@@ -1475,6 +1475,6 @@ error: no generation older than the current (91) exists</screen>
 </variablelist>
 
 </refsection>
-  
+
 
 </refentry>
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index b43ec66f60c..4c6af567ae6 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -160,19 +160,24 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
 void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
 {
     PathLocks lock;
-
     lockProfile(lock, profile);
 
     int curGen;
+    bool fromCurGen = false;
     Generations gens = findGenerations(profile, curGen);
-
     for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
-        if (max) {
+        if (i->number == curGen) {
+            fromCurGen = true;
             max--;
             continue;
         }
-        if (i->number != curGen)
+        if (fromCurGen) {
+            if (max) {
+                max--;
+                continue;
+            }
             deleteGeneration2(profile, i->number, dryRun);
+        }
     }
 }
 

From 939cf4ccebb199e61da648873fb078ae8833263f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Feb 2018 16:58:31 +0100
Subject: [PATCH 0881/2196] Fix error message

---
 src/buildenv/buildenv.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc
index eddb9fdaa8d..2afad913ac6 100644
--- a/src/buildenv/buildenv.cc
+++ b/src/buildenv/buildenv.cc
@@ -74,10 +74,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                     auto prevPriority = priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(format(
-                                "Packages '%1%' and '%2%' have the same priority '%3%'"
+                                "packages '%1%' and '%2%' have the same priority %3%; "
                                 "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
                                 "to change the priority of one of the conflicting packages"
-                                " ('0' being the highest priority)"
+                                " (0 being the highest priority)"
                                 ) % srcFile % target % priority);
                     if (prevPriority < priority)
                         continue;

From a0e38c16bc7773d0fc62771a9935e079c00899ef Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Sun, 25 Feb 2018 23:51:47 +0200
Subject: [PATCH 0882/2196] libexpr: Recognize newline in more places in lexer

Flex's regexes have an annoying feature: the dot matches everything
except a newline. This causes problems for expressions like:

"${0}\
"

where the backslash-newline combination matches this rule instead of the
intended one mentioned in the comment:

    <STRING>\$|\\|\$\\ {
                    /* This can only occur when we reach EOF, otherwise the above
                    (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
                    This is technically invalid, but we leave the problem to the
                    parser who fails with exact location. */
                    return STR;
                }
However, the parser actually accepts the resulting token sequence
('"' DOLLAR_CURLY 0 '}' STR '"'), which is a problem because the lexer
rule didn't assign anything to yylval. Ultimately this leads to a crash
when dereferencing a NULL pointer in ExprConcatStrings::bindVars().

The fix does change the syntax of the language in some corner cases
but I think it's only turning previously invalid (or crashing) syntax
to valid syntax. E.g.

"a\
b"

and

''a''\
b''

were previously syntax errors but now both result in "a\nb".

Found by afl-fuzz.
---
 src/libexpr/lexer.l                          | 9 +++++----
 tests/lang/eval-okay-backslash-newline-1.exp | 1 +
 tests/lang/eval-okay-backslash-newline-1.nix | 2 ++
 tests/lang/eval-okay-backslash-newline-2.exp | 1 +
 tests/lang/eval-okay-backslash-newline-2.nix | 2 ++
 5 files changed, 11 insertions(+), 4 deletions(-)
 create mode 100644 tests/lang/eval-okay-backslash-newline-1.exp
 create mode 100644 tests/lang/eval-okay-backslash-newline-1.nix
 create mode 100644 tests/lang/eval-okay-backslash-newline-2.exp
 create mode 100644 tests/lang/eval-okay-backslash-newline-2.nix

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index e5e01fb5831..1e9c29afa13 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -85,6 +85,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 %}
 
 
+ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
@@ -146,8 +147,8 @@ or          { return OR_KW; }
 <INITIAL,INSIDE_DOLLAR_CURLY>\" {
                 PUSH_STATE(STRING); return '"';
               }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ {
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
@@ -178,7 +179,7 @@ or          { return OR_KW; }
                    yylval->e = new ExprIndStr("''");
                    return IND_STR;
                  }
-<IND_STRING>\'\'\\. {
+<IND_STRING>\'\'\\{ANY} {
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
@@ -208,7 +209,7 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-.           return yytext[0];
+{ANY}           return yytext[0];
 
 }
 
diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/lang/eval-okay-backslash-newline-1.exp
new file mode 100644
index 00000000000..3e754364cc9
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-1.exp
@@ -0,0 +1 @@
+"a\nb"
diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/lang/eval-okay-backslash-newline-1.nix
new file mode 100644
index 00000000000..7fef3dddd4d
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-1.nix
@@ -0,0 +1,2 @@
+"a\
+b"
diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/lang/eval-okay-backslash-newline-2.exp
new file mode 100644
index 00000000000..3e754364cc9
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-2.exp
@@ -0,0 +1 @@
+"a\nb"
diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/lang/eval-okay-backslash-newline-2.nix
new file mode 100644
index 00000000000..35ddf495c63
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-2.nix
@@ -0,0 +1,2 @@
+''a''\
+b''

From c89a3d536891da84403b025c70f1ae225faa0eb2 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 1 Mar 2018 15:00:58 -0600
Subject: [PATCH 0883/2196] don't allocate large buffers on the stack

---
 src/libstore/build.cc        |  6 +++---
 src/libutil/archive.cc       | 18 +++++++++---------
 src/libutil/hash.cc          |  6 +++---
 src/libutil/util.cc          | 23 ++++++++++++-----------
 src/nix-daemon/nix-daemon.cc |  6 +++---
 5 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1d611ffbaba..b33649e6b4a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4156,8 +4156,8 @@ void Worker::waitForInput()
         set<int> fds2(j->fds);
         for (auto & k : fds2) {
             if (FD_ISSET(k, &fds)) {
-                unsigned char buffer[4096];
-                ssize_t rd = read(k, buffer, sizeof(buffer));
+                std::vector<unsigned char> buffer(4096);
+                ssize_t rd = read(k, buffer.data(), buffer.size());
                 if (rd == -1) {
                     if (errno != EINTR)
                         throw SysError(format("reading from %1%")
@@ -4169,7 +4169,7 @@ void Worker::waitForInput()
                 } else {
                     printMsg(lvlVomit, format("%1%: read %2% bytes")
                         % goal->getName() % rd);
-                    string data((char *) buffer, rd);
+                    string data((char *) buffer.data(), rd);
                     j->lastOutput = after;
                     goal->handleChildOutput(k, data);
                 }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index f71229d8fdd..59be91c5cfb 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -40,14 +40,14 @@ static void dumpContents(const Path & path, size_t size,
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd) throw SysError(format("opening file '%1%'") % path);
 
-    unsigned char buf[65536];
+    std::vector<unsigned char> buf(65536);
     size_t left = size;
 
     while (left > 0) {
-        size_t n = left > sizeof(buf) ? sizeof(buf) : left;
-        readFull(fd.get(), buf, n);
+        size_t n = left > buf.size() ? buf.size() : left;
+        readFull(fd.get(), buf.data(), n);
         left -= n;
-        sink(buf, n);
+        sink(buf.data(), n);
     }
 
     writePadding(size, sink);
@@ -146,14 +146,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
     sink.preallocateContents(size);
 
     unsigned long long left = size;
-    unsigned char buf[65536];
+    std::vector<unsigned char> buf(65536);
 
     while (left) {
         checkInterrupt();
-        unsigned int n = sizeof(buf);
-        if ((unsigned long long) n > left) n = left;
-        source(buf, n);
-        sink.receiveContents(buf, n);
+        auto n = buf.size();
+        if ((unsigned long long)n > left) n = left;
+        source(buf.data(), n);
+        sink.receiveContents(buf.data(), n);
         left -= n;
     }
 
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 75e4767550f..8267481b882 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -256,12 +256,12 @@ Hash hashFile(HashType ht, const Path & path)
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd) throw SysError(format("opening file '%1%'") % path);
 
-    unsigned char buf[8192];
+    std::vector<unsigned char> buf(8192);
     ssize_t n;
-    while ((n = read(fd.get(), buf, sizeof(buf)))) {
+    while ((n = read(fd.get(), buf.data(), buf.size()))) {
         checkInterrupt();
         if (n == -1) throw SysError(format("reading file '%1%'") % path);
-        update(ht, ctx, buf, n);
+        update(ht, ctx, buf.data(), n);
     }
 
     finish(ht, ctx, hash.hash);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 2391e14a94b..2c4705f6ba7 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -229,16 +229,17 @@ bool pathExists(const Path & path)
 Path readLink(const Path & path)
 {
     checkInterrupt();
+    std::vector<char> buf;
     for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
-        char buf[bufSize];
-        ssize_t rlSize = readlink(path.c_str(), buf, bufSize);
+        buf.resize(bufSize);
+        ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
         if (rlSize == -1)
             if (errno == EINVAL)
                 throw Error("'%1%' is not a symlink", path);
             else
                 throw SysError("reading symbolic link '%1%'", path);
         else if (rlSize < bufSize)
-            return string(buf, rlSize);
+            return string(buf.data(), rlSize);
     }
 }
 
@@ -293,10 +294,10 @@ string readFile(int fd)
     if (fstat(fd, &st) == -1)
         throw SysError("statting file");
 
-    auto buf = std::make_unique<unsigned char[]>(st.st_size);
-    readFull(fd, buf.get(), st.st_size);
+    std::vector<unsigned char> buf(st.st_size);
+    readFull(fd, buf.data(), st.st_size);
 
-    return string((char *) buf.get(), st.st_size);
+    return string((char *) buf.data(), st.st_size);
 }
 
 
@@ -438,10 +439,10 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 static Lazy<Path> getHome2([]() {
     Path homeDir = getEnv("HOME");
     if (homeDir.empty()) {
-        char buf[16384];
+        std::vector<char> buf(16384);
         struct passwd pwbuf;
         struct passwd * pw;
-        if (getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pw) != 0
+        if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
             || !pw || !pw->pw_dir || !pw->pw_dir[0])
             throw Error("cannot determine user's home directory");
         homeDir = pw->pw_dir;
@@ -569,16 +570,16 @@ void writeFull(int fd, const string & s, bool allowInterrupts)
 string drainFD(int fd)
 {
     string result;
-    unsigned char buffer[4096];
+    std::vector<unsigned char> buffer(4096);
     while (1) {
         checkInterrupt();
-        ssize_t rd = read(fd, buffer, sizeof buffer);
+        ssize_t rd = read(fd, buffer.data(), buffer.size());
         if (rd == -1) {
             if (errno != EINTR)
                 throw SysError("reading from file");
         }
         else if (rd == 0) break;
-        else result.append((char *) buffer, rd);
+        else result.append((char *) buffer.data(), rd);
     }
     return result;
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 890bffa19aa..c3b94614a87 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -37,13 +37,13 @@ using namespace nix;
 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags)
 {
     /* We ignore most parameters, we just have them for conformance with the linux syscall */
-    char buf[8192];
-    auto read_count = read(fd_in, buf, sizeof(buf));
+    std::vector<char> buf(8192);
+    auto read_count = read(fd_in, buf.data(), buf.size());
     if (read_count == -1)
         return read_count;
     auto write_count = decltype(read_count)(0);
     while (write_count < read_count) {
-        auto res = write(fd_out, buf + write_count, read_count - write_count);
+        auto res = write(fd_out, buf.data() + write_count, read_count - write_count);
         if (res == -1)
             return res;
         write_count += res;

From 6b9a03f5d878ae434b54bb883b51e28082dc30b3 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 1 Mar 2018 18:58:41 -0600
Subject: [PATCH 0884/2196] hoist vector out of loop just in case

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b33649e6b4a..a1654917dc3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4154,9 +4154,9 @@ void Worker::waitForInput()
         assert(goal);
 
         set<int> fds2(j->fds);
+        std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
             if (FD_ISSET(k, &fds)) {
-                std::vector<unsigned char> buffer(4096);
                 ssize_t rd = read(k, buffer.data(), buffer.size());
                 if (rd == -1) {
                     if (errno != EINTR)

From e9a5ce9b079f21a42b01bdee9a08953a2d34d901 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 3 Mar 2018 13:48:54 -0600
Subject: [PATCH 0885/2196] release.nix: don't try to use nix-2.0 branch, no
 longer exists

Probably should point at the 18.03 release branch once that's made.
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 3f8d5da4721..0bbd966fabd 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; }
+, nixpkgs ? builtins.fetchGit https://github.com/NixOS/nixpkgs.git
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:

From 088ef81759f22bf0115a52f183ba66b0be3b9ef2 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 5 Mar 2018 07:42:15 -0500
Subject: [PATCH 0886/2196] ssh-ng: Don't forward options to the daemon.

This can be iterated on and currently leaves out settings we know we
want to forward, but it fixes #1713 and fixes #1935 and isn't
fundamentally broken like the status quo. Future changes are suggested
in a comment.
---
 src/libstore/remote-store.hh |  3 ++-
 src/libstore/ssh-store.cc    | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 7f36e206416..0cc20bf9419 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -122,11 +122,12 @@ protected:
 
     ref<Pool<Connection>> connections;
 
+    virtual void setOptions(Connection & conn);
+
 private:
 
     std::atomic_bool failed{false};
 
-    void setOptions(Connection & conn);
 };
 
 class UDSRemoteStore : public LocalFSStore, public RemoteStore
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 107c6e1ecb4..398408ea8d7 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -51,6 +51,16 @@ class SSHStore : public RemoteStore
     std::string host;
 
     SSHMaster master;
+
+    void setOptions(RemoteStore::Connection & conn) override
+    {
+        /* TODO Add a way to explicitly ask for some options to be
+           forwarded. One option: A way to query the daemon for its
+           settings, and then a series of params to SSHStore like
+           forward-cores or forward-overridden-cores that only
+           override the requested settings.
+        */
+    };
 };
 
 

From e917c05628abd0ff429c429d0e814532180b9586 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 3 Mar 2018 14:39:04 -0600
Subject: [PATCH 0887/2196] logging: Don't throw from Activity destructor

Move definition of destructor to logging.cc for
access to util.hh's ignoreException.
---
 src/libutil/logging.cc | 8 ++++++++
 src/libutil/logging.hh | 3 +--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 27a631a37d1..c1cff5cf906 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -221,4 +221,12 @@ bool handleJSONLogMessage(const std::string & msg,
     return true;
 }
 
+Activity::~Activity() {
+    try {
+        logger.stopActivity(id);
+    } catch (...) {
+        ignoreException();
+    }
+}
+
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 677aa4daec4..ca768d555ad 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -94,8 +94,7 @@ struct Activity
 
     Activity(const Activity & act) = delete;
 
-    ~Activity()
-    { logger.stopActivity(id); }
+    ~Activity();
 
     void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
     { result(resProgress, done, expected, running, failed); }

From 56253bb08fef60f4768ebb2ca013135b0dbf5df3 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 6 Mar 2018 17:34:44 -0600
Subject: [PATCH 0888/2196] rand() -> random(), since we use srandom().

rand() requires we call srand() instead,
but might as well use random().
---
 src/libstore/build.cc          | 2 +-
 src/libstore/gc.cc             | 2 +-
 src/libstore/optimise-store.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1d611ffbaba..47a905c816c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1458,7 +1458,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
        tmpPath (the replacement), so we have to move it out of the
        way first.  We'd better not be interrupted here, because if
        we're repairing (say) Glibc, we end up with a broken system. */
-    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str();
+    Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
     if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 943b16c28fa..ba49749d830 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -59,7 +59,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
     /* Create the new symlink. */
     Path tempLink = (format("%1%.tmp-%2%-%3%")
-        % link % getpid() % rand()).str();
+        % link % getpid() % random()).str();
     createSymlink(target, tempLink);
 
     /* Atomically replace the old one. */
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 891540ae4c1..7840167d777 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -213,7 +213,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
 
     Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
-        % realStoreDir % getpid() % rand()).str();
+        % realStoreDir % getpid() % random()).str();
 
     if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
         if (errno == EMLINK) {

From 16551f54c94f2b551ebaf00a7bd0245dc3b0b9e4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Mar 2018 11:38:41 +0100
Subject: [PATCH 0889/2196] Fix #1957

---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4afe51ea91e..3441b2472fc 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1215,7 +1215,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
-                HashResult current = hashPath(info->narHash.type, i);
+                HashResult current = hashPath(info->narHash.type, toRealPath(i));
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError(format("path '%1%' was modified! "

From 383afab9ad0ebfcc17cb7155f66fc60a4bdcd9a8 Mon Sep 17 00:00:00 2001
From: Tim Engler <engler@gmail.com>
Date: Fri, 9 Mar 2018 03:16:33 +0000
Subject: [PATCH 0890/2196] Modified MakeBinOp to no longer produce its name
 using concatenation and "##". Doing so prevents emacs tags from working, as
 well as makes the code extremely confusing for a newbie.

In the prior state, if someone wants to find the definition of "ExprApp" for
example, a grep through the code reveals nothing. Since the definition could be
hiding in numerous ".h" files, it's really difficult to find. This personally
took me several hours to figure out.
---
 src/libexpr/nixexpr.hh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 30be79bb57a..eb9fac25c01 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -283,12 +283,12 @@ struct ExprOpNot : Expr
 };
 
 #define MakeBinOp(name, s) \
-    struct Expr##name : Expr \
+    struct name : Expr \
     { \
         Pos pos; \
         Expr * e1, * e2; \
-        Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
-        Expr##name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
+        name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
+        name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
         void show(std::ostream & str) \
         { \
             str << "(" << *e1 << " " s " " << *e2 << ")";   \
@@ -300,14 +300,14 @@ struct ExprOpNot : Expr
         void eval(EvalState & state, Env & env, Value & v); \
     };
 
-MakeBinOp(App, "")
-MakeBinOp(OpEq, "==")
-MakeBinOp(OpNEq, "!=")
-MakeBinOp(OpAnd, "&&")
-MakeBinOp(OpOr, "||")
-MakeBinOp(OpImpl, "->")
-MakeBinOp(OpUpdate, "//")
-MakeBinOp(OpConcatLists, "++")
+MakeBinOp(ExprApp, "")
+MakeBinOp(ExprOpEq, "==")
+MakeBinOp(ExprOpNEq, "!=")
+MakeBinOp(ExprOpAnd, "&&")
+MakeBinOp(ExprOpOr, "||")
+MakeBinOp(ExprOpImpl, "->")
+MakeBinOp(ExprOpUpdate, "//")
+MakeBinOp(ExprOpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {

From 24b739817fb719553dd4840308ee32ff60c88c1f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Mar 2018 14:19:51 +0100
Subject: [PATCH 0891/2196] Fix double free in Store::queryPathInfo()

It was holding on to a Value* (i.e. a std::shared_ptr<ValidPathInfo>*)
outside of the pathInfoCache lock, so the std::shared_ptr could be
destroyed between the release of the lock and the decrement of the
std::shared_ptr refcount. This can happen if more than
'path-info-cache-size' paths are added in the meantime, *or* if
clearPathInfoCache() is called. The hydra-queue-runner queue monitor
thread periodically calls the later, so is likely to trigger a crash.

Fixes https://github.com/NixOS/hydra/issues/542.
---
 src/libutil/lru-cache.hh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 3cb5d50889d..9b8290e634c 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -2,6 +2,7 @@
 
 #include <map>
 #include <list>
+#include <experimental/optional>
 
 namespace nix {
 
@@ -63,18 +64,17 @@ public:
 
     /* Look up an item in the cache. If it exists, it becomes the most
        recently used item. */
-    // FIXME: use boost::optional?
-    Value * get(const Key & key)
+    std::experimental::optional<Value> get(const Key & key)
     {
         auto i = data.find(key);
-        if (i == data.end()) return 0;
+        if (i == data.end()) return {};
 
         /* Move this item to the back of the LRU list. */
         lru.erase(i->second.first.it);
         auto j = lru.insert(lru.end(), i);
         i->second.first.it = j;
 
-        return &i->second.second;
+        return i->second.second;
     }
 
     size_t size()

From be54f4a0b6f160f8a2b0b9e9c988fb5251f70a97 Mon Sep 17 00:00:00 2001
From: Asad Saeeduddin <masaeedu@gmail.com>
Date: Mon, 12 Mar 2018 00:56:41 -0400
Subject: [PATCH 0892/2196] Wrap thread local in function for Cygwin

Fixes #1826. See #1352 for a previous instance of a similar change.
---
 src/libstore/download.hh |  2 +-
 src/libutil/logging.cc   | 11 ++++++++++-
 src/libutil/logging.hh   | 11 ++++++-----
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index d9d525d4e65..0b8d29b21df 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -22,7 +22,7 @@ struct DownloadRequest
     std::string mimeType;
 
     DownloadRequest(const std::string & uri)
-        : uri(uri), parentAct(curActivity) { }
+        : uri(uri), parentAct(getCurActivity()) { }
 };
 
 struct DownloadResult
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index c1cff5cf906..e860b27d3d0 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -6,7 +6,16 @@
 
 namespace nix {
 
-thread_local ActivityId curActivity = 0;
+static thread_local ActivityId curActivity = 0;
+
+ActivityId getCurActivity()
+{
+    return curActivity;
+}
+void setCurActivity(const ActivityId activityId)
+{
+    curActivity = activityId;
+}
 
 Logger * logger = makeDefaultLogger();
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index ca768d555ad..678703102e9 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -77,7 +77,8 @@ public:
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
 };
 
-extern thread_local ActivityId curActivity;
+ActivityId getCurActivity();
+void setCurActivity(const ActivityId activityId);
 
 struct Activity
 {
@@ -86,10 +87,10 @@ struct Activity
     const ActivityId id;
 
     Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "",
-        const Logger::Fields & fields = {}, ActivityId parent = curActivity);
+        const Logger::Fields & fields = {}, ActivityId parent = getCurActivity());
 
     Activity(Logger & logger, ActivityType type,
-        const Logger::Fields & fields = {}, ActivityId parent = curActivity)
+        const Logger::Fields & fields = {}, ActivityId parent = getCurActivity())
         : Activity(logger, lvlError, type, "", fields, parent) { };
 
     Activity(const Activity & act) = delete;
@@ -121,8 +122,8 @@ struct Activity
 struct PushActivity
 {
     const ActivityId prevAct;
-    PushActivity(ActivityId act) : prevAct(curActivity) { curActivity = act; }
-    ~PushActivity() { curActivity = prevAct; }
+    PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); }
+    ~PushActivity() { setCurActivity(prevAct); }
 };
 
 extern Logger * logger;

From 80735c4cc987d9db89bdc248010c8f47fd748c1b Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Tue, 13 Mar 2018 10:28:23 +0100
Subject: [PATCH 0893/2196] fetchGit: Fix debug message

---
 src/libexpr/primops/fetchGit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 9fc0d46626d..fd536c6a5e4 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -138,7 +138,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
     gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
-    printTalkative("using revision %s of repo '%s'", uri, gitInfo.rev);
+    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
 
     std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";

From 6e60141a80446de446906b4ffd2cd026daf6b9cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Mar 2018 17:48:09 +0100
Subject: [PATCH 0894/2196] Typo

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 47a905c816c..e162e81aab5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1189,7 +1189,7 @@ void DerivationGoal::outputsSubstituted()
     for (auto & i : drv->inputSrcs) {
         if (worker.store.isValidPath(i)) continue;
         if (!settings.useSubstitutes)
-            throw Error(format("dependency of '%1%' of '%2%' does not exist, and substitution is disabled")
+            throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled")
                 % i % drvPath);
         addWaitee(worker.makeSubstitutionGoal(i));
     }

From 7afdc8d4a100d26cd09f7d63c0fe915b4d6a53c9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Mar 2018 17:56:51 +0100
Subject: [PATCH 0895/2196] build-remote: Don't substitute the build result

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index dbf8fe1b8f8..6b294a4d2c2 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -244,7 +244,7 @@ int main (int argc, char * * argv)
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
+            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
         return;

From ca14b142001ab467e679d4ed0cc6236180bf34b2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Mar 2018 19:01:22 +0100
Subject: [PATCH 0896/2196] Use boost::format from the boost package

Note that this only requires headers from boost so it doesn't add a
runtime dependency.

Also, use Nixpkgs 18.03.
---
 Makefile                                  |   1 -
 local.mk                                  |   2 -
 release.nix                               |   5 +-
 shell.nix                                 |   3 +-
 src/boost/assert.hpp                      |  38 --
 src/boost/format.hpp                      |  64 --
 src/boost/format/exceptions.hpp           |  96 ---
 src/boost/format/feed_args.hpp            | 254 --------
 src/boost/format/format_class.hpp         | 135 -----
 src/boost/format/format_fwd.hpp           |  49 --
 src/boost/format/format_implementation.cc | 256 --------
 src/boost/format/free_funcs.cc            |  71 ---
 src/boost/format/group.hpp                | 680 ----------------------
 src/boost/format/internals.hpp            | 167 ------
 src/boost/format/internals_fwd.hpp        |  65 ---
 src/boost/format/local.mk                 |   7 -
 src/boost/format/macros_default.hpp       |  48 --
 src/boost/format/parsing.cc               | 454 ---------------
 src/boost/throw_exception.hpp             |  47 --
 src/libstore/build.cc                     |   6 +-
 src/libstore/sqlite.cc                    |   4 +-
 src/libstore/sqlite.hh                    |   2 +-
 tests/restricted.sh                       |   4 +-
 23 files changed, 13 insertions(+), 2445 deletions(-)
 delete mode 100644 src/boost/assert.hpp
 delete mode 100644 src/boost/format.hpp
 delete mode 100644 src/boost/format/exceptions.hpp
 delete mode 100644 src/boost/format/feed_args.hpp
 delete mode 100644 src/boost/format/format_class.hpp
 delete mode 100644 src/boost/format/format_fwd.hpp
 delete mode 100644 src/boost/format/format_implementation.cc
 delete mode 100644 src/boost/format/free_funcs.cc
 delete mode 100644 src/boost/format/group.hpp
 delete mode 100644 src/boost/format/internals.hpp
 delete mode 100644 src/boost/format/internals_fwd.hpp
 delete mode 100644 src/boost/format/local.mk
 delete mode 100644 src/boost/format/macros_default.hpp
 delete mode 100644 src/boost/format/parsing.cc
 delete mode 100644 src/boost/throw_exception.hpp

diff --git a/Makefile b/Makefile
index c867823fc48..c47603731dd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,5 @@
 makefiles = \
   local.mk \
-  src/boost/format/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \
diff --git a/local.mk b/local.mk
index 40a910991a4..5d7e0fb2e42 100644
--- a/local.mk
+++ b/local.mk
@@ -10,5 +10,3 @@ GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
-
-$(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644)))
diff --git a/release.nix b/release.nix
index 0bbd966fabd..2e9292955a2 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit https://github.com/NixOS/nixpkgs.git
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "release-18.03"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -71,6 +71,7 @@ let
           [ curl
             bzip2 xz brotli
             openssl pkgconfig sqlite boehmgc
+            boost
 
             # Tests
             git
@@ -192,7 +193,7 @@ let
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
+        lcovFilter = [ "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
diff --git a/shell.nix b/shell.nix
index d6ae5a55226..09318dfd078 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import <nixpkgs> {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "release-18.03"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
@@ -18,6 +18,7 @@ with import ./release-common.nix { inherit pkgs; };
         customMemoryManagement = false;
       })
       autoreconfHook
+      boost
 
       # For nix-perl
       perl
diff --git a/src/boost/assert.hpp b/src/boost/assert.hpp
deleted file mode 100644
index 754ebb954bc..00000000000
--- a/src/boost/assert.hpp
+++ /dev/null
@@ -1,38 +0,0 @@
-//
-//  boost/assert.hpp - BOOST_ASSERT(expr)
-//
-//  Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd.
-//
-//  Permission to copy, use, modify, sell and distribute this software
-//  is granted provided this copyright notice appears in all copies.
-//  This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-//
-//  Note: There are no include guards. This is intentional.
-//
-//  See http://www.boost.org/libs/utility/assert.html for documentation.
-//
-
-#undef BOOST_ASSERT
-
-#if defined(BOOST_DISABLE_ASSERTS)
-
-# define BOOST_ASSERT(expr) ((void)0)
-
-#elif defined(BOOST_ENABLE_ASSERT_HANDLER)
-
-#include <boost/current_function.hpp>
-
-namespace boost
-{
-
-void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined
-
-} // namespace boost
-
-#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__))
-
-#else
-# include <assert.h>
-# define BOOST_ASSERT(expr) assert(expr)
-#endif
diff --git a/src/boost/format.hpp b/src/boost/format.hpp
deleted file mode 100644
index f965f0f33e9..00000000000
--- a/src/boost/format.hpp
+++ /dev/null
@@ -1,64 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// format.hpp :  primary header
-// ----------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_HPP
-#define BOOST_FORMAT_HPP
-
-#include <vector>
-#include <string>
-#include <sstream>
-#include <cassert>
-
-#if HAVE_LOCALE
-#include <locale>
-#else
-#define BOOST_NO_STD_LOCALE
-#define BOOST_NO_LOCALE_ISIDIGIT
-#include <cctype>
-#endif
-
-#include <boost/format/macros_default.hpp>
-
-
-// ****  Forward declarations ----------------------------------
-#include <boost/format/format_fwd.hpp>           // basic_format<Ch,Tr>, and other frontends
-#include <boost/format/internals_fwd.hpp>        // misc forward declarations for internal use
-
-
-// ****  Auxiliary structs (stream_format_state<Ch,Tr> , and format_item<Ch,Tr> )
-#include <boost/format/internals.hpp>    
-
-// ****  Format  class  interface --------------------------------
-#include <boost/format/format_class.hpp>
-
-// **** Exceptions -----------------------------------------------
-#include <boost/format/exceptions.hpp>
-
-// **** Implementation -------------------------------------------
-//#include <boost/format/format_implementation.hpp>   // member functions
-
-#include <boost/format/group.hpp>                   // class for grouping arguments
-
-#include <boost/format/feed_args.hpp>               // argument-feeding functions
-//#include <boost/format/parsing.hpp>                 // format-string parsing (member-)functions
-
-// **** Implementation of the free functions ----------------------
-//#include <boost/format/free_funcs.hpp>
-
-
-#endif // BOOST_FORMAT_HPP
diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp
deleted file mode 100644
index a7641458c95..00000000000
--- a/src/boost/format/exceptions.hpp
+++ /dev/null
@@ -1,96 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// exceptions.hpp 
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_EXCEPTIONS_HPP
-#define BOOST_FORMAT_EXCEPTIONS_HPP
-
-
-#include <stdexcept>
-
-
-namespace boost {
-
-namespace io {
-
-// **** exceptions -----------------------------------------------
-
-class format_error : public std::exception
-{
-public:
-  format_error() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::format_error: "
-      "format generic failure";
-  }
-};
-
-class bad_format_string : public format_error
-{
-public:
-  bad_format_string() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::bad_format_string: "
-      "format-string is ill-formed";
-  }
-};
-
-class too_few_args : public format_error
-{
-public:
-  too_few_args() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::too_few_args: "
-      "format-string refered to more arguments than were passed";
-  }
-};
-
-class too_many_args : public format_error
-{
-public:
-  too_many_args() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::too_many_args: "
-      "format-string refered to less arguments than were passed";
-  }
-};
-
-
-class  out_of_range : public format_error
-{
-public:
-  out_of_range() { abort(); }
-  virtual const char *what() const throw()
-  {
-    return "boost::out_of_range: "
-      "tried to refer to an argument (or item) number which is out of range, "
-      "according to the format string.";
-  }
-};
-
-
-} // namespace io
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_EXCEPTIONS_HPP
diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp
deleted file mode 100644
index cdd57fdf2bf..00000000000
--- a/src/boost/format/feed_args.hpp
+++ /dev/null
@@ -1,254 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// feed_args.hpp :  functions for processing each argument 
-//                      (feed, feed_manip, and distribute)
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_FEED_ARGS_HPP
-#define BOOST_FORMAT_FEED_ARGS_HPP
-
-#include "boost/format/format_class.hpp"
-#include "boost/format/group.hpp"
-
-#include "boost/throw_exception.hpp"
-
-namespace boost {
-namespace io {
-namespace detail {
-namespace  { 
-
-  inline
-  void empty_buf(BOOST_IO_STD ostringstream & os) { 
-    static const std::string emptyStr;
-    os.str(emptyStr); 
-  }
-
-  void do_pad( std::string & s, 
-                std::streamsize w, 
-                const char c, 
-                std::ios::fmtflags f, 
-                bool center)
-    __attribute__ ((unused));
-
-  void do_pad( std::string & s, 
-                std::streamsize w, 
-                const char c, 
-                std::ios::fmtflags f, 
-                bool center) 
-    // applies centered / left / right  padding  to the string s.
-    // Effects : string s is padded.
-  {
-    std::streamsize n=w-s.size();
-    if(n<=0) {
-      return;
-    }
-    if(center) 
-      {
-        s.reserve(w); // allocate once for the 2 inserts
-        const std::streamsize n1 = n /2, n0 = n - n1; 
-        s.insert(s.begin(), n0, c);
-        s.append(n1, c);
-      } 
-    else 
-      {
-        if(f & std::ios::left) {
-          s.append(n, c);
-        }
-        else {
-          s.insert(s.begin(), n, c);
-        }
-      }
-  } // -do_pad(..) 
-
-
-  template<class T> inline
-  void put_head(BOOST_IO_STD ostream& , const T& ) {
-  }
-
-  template<class T> inline
-  void put_head( BOOST_IO_STD ostream& os, const group1<T>& x ) {
-    os << group_head(x.a1_); // send the first N-1 items, not the last
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, const T& x ) {
-    os << x ;
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, const group1<T>& x ) {
-    os << group_last(x.a1_); // this selects the last element
-  }
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST 
-  template<class T> inline
-  void put_head( BOOST_IO_STD ostream& , T& ) {
-  }
-
-  template<class T> inline
-  void put_last( BOOST_IO_STD ostream& os, T& x ) {
-    os << x ;
-  }
-#endif
-
-
-
-  
-template<class T> 
-void put( T x, 
-          const format_item& specs, 
-          std::string & res, 
-          BOOST_IO_STD ostringstream& oss_ )
-{
-  // does the actual conversion of x, with given params, into a string
-  // using the *supplied* strinstream. (the stream state is important)
-
-  typedef std::string string_t;
-  typedef format_item  format_item_t;
-
-  stream_format_state   prev_state(oss_);
-    
-  specs.state_.apply_on(oss_);
-
-  // in case x is a group, apply the manip part of it, 
-  // in order to find width
-  put_head( oss_, x );
-  empty_buf( oss_);
-
-  const std::streamsize w=oss_.width();
-  const std::ios::fmtflags fl=oss_.flags();
-  const bool internal = (fl & std::ios::internal) != 0;
-  const bool two_stepped_padding = internal
-    &&  ! ( specs.pad_scheme_ & format_item_t::spacepad ) 
-    && specs.truncate_ < 0 ;
-      
-
-  if(! two_stepped_padding) 
-    {
-      if(w>0) // handle simple padding via do_pad, not natively in stream 
-        oss_.width(0);
-      put_last( oss_, x);
-      res = oss_.str();
-
-      if (specs.truncate_ >= 0)
-        res.erase(specs.truncate_);
-
-      // complex pads :
-      if(specs.pad_scheme_ & format_item_t::spacepad)
-        {
-          if( res.size()==0 ||   ( res[0]!='+' && res[0]!='-'  ))
-            {
-              res.insert(res.begin(), 1, ' '); // insert 1 space at  pos 0
-            }
-        }
-      if(w > 0) // need do_pad
-        {
-          do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 );
-        }
-    } 
-  else  // 2-stepped padding
-    {
-      put_last( oss_, x); // oss_.width() may result in padding.
-      res = oss_.str();
-      
-      if (specs.truncate_ >= 0)
-        res.erase(specs.truncate_);
-
-      if( res.size() - w > 0)
-        { //   length w exceeded
-          // either it was multi-output with first output padding up all width..
-          // either it was one big arg and we are fine.
-          empty_buf( oss_);
-          oss_.width(0);
-          put_last(oss_, x );
-          string_t tmp = oss_.str();  // minimal-length output
-          std::streamsize d;
-          if( (d=w - tmp.size()) <=0 ) 
-            {
-              // minimal length is already >= w, so no padding  (cool!)
-              res.swap(tmp);
-            }
-          else
-            { // hum..  we need to pad (it was necessarily multi-output)
-              typedef typename string_t::size_type size_type;
-              size_type i = 0;
-              while( i<tmp.size() && tmp[i] == res[i] ) // find where we should pad.
-                ++i;
-              tmp.insert(i, static_cast<size_type>( d ), oss_.fill());
-              res.swap( tmp );
-            }
-        }
-      else 
-        { // okay, only one thing was printed and padded, so res is fine.
-        }
-    }
-
-  prev_state.apply_on(oss_);
-  empty_buf( oss_);
-  oss_.clear();
-} // end- put(..)
-
-
-}  // local namespace
-
-
-
-
-
-template<class T> 
-void distribute(basic_format& self, T x) 
-  // call put(x, ..) on every occurence of the current argument :
-{
-  if(self.cur_arg_ >= self.num_args_)
-    {
-      if( self.exceptions() & too_many_args_bit )
-        boost::throw_exception(too_many_args()); // too many variables have been supplied !
-      else return;
-    }
-  for(unsigned long i=0; i < self.items_.size(); ++i)
-    {
-      if(self.items_[i].argN_ == self.cur_arg_)
-        {
-          put<T> (x, self.items_[i], self.items_[i].res_, self.oss_ );
-        }
-    }
-}
-
-template<class T> 
-basic_format&  feed(basic_format& self, T x) 
-{
-  if(self.dumped_) self.clear();
-  distribute<T> (self, x);
-  ++self.cur_arg_;
-  if(self.bound_.size() != 0)
-    {
-      while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] )
-        ++self.cur_arg_;
-    }
-
-  // this arg is finished, reset the stream's format state
-  self.state0_.apply_on(self.oss_);
-  return self;
-}
-    
-
-} // namespace detail
-} // namespace io
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_FEED_ARGS_HPP
diff --git a/src/boost/format/format_class.hpp b/src/boost/format/format_class.hpp
deleted file mode 100644
index 6875623acb4..00000000000
--- a/src/boost/format/format_class.hpp
+++ /dev/null
@@ -1,135 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// format_class.hpp :  class interface
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_CLASS_HPP
-#define BOOST_FORMAT_CLASS_HPP
-
-#include <vector>
-#include <string>
-
-#include <boost/format/format_fwd.hpp>
-#include <boost/format/internals_fwd.hpp>
-
-#include <boost/format/internals.hpp>
-
-namespace boost {
-
-class basic_format 
-{
-public:
-  typedef std::string                string_t;
-  typedef BOOST_IO_STD ostringstream internal_stream_t;
-private:
-  typedef BOOST_IO_STD ostream       stream_t;
-  typedef io::detail::stream_format_state  stream_format_state;
-  typedef io::detail::format_item          format_item_t;
-
-public:
-  basic_format(const char* str);
-  basic_format(const string_t& s);
-#ifndef BOOST_NO_STD_LOCALE
-  basic_format(const char* str, const std::locale & loc);
-  basic_format(const string_t& s, const std::locale & loc);
-#endif // no locale
-  basic_format(const basic_format& x);
-  basic_format& operator= (const basic_format& x);
-
-  basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() )
-
-  // pass arguments through those operators :
-  template<class T>  basic_format&   operator%(const T& x) 
-  { 
-    return io::detail::feed<const T&>(*this,x);
-  }
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-  template<class T>  basic_format&   operator%(T& x) 
-  {
-    return io::detail::feed<T&>(*this,x);
-  }
-#endif
-
-
-  // system for binding arguments :
-  template<class T>  
-  basic_format&         bind_arg(int argN, const T& val) 
-  {
-    return io::detail::bind_arg_body(*this, argN, val); 
-  }
-  basic_format&         clear_bind(int argN);
-  basic_format&         clear_binds();
-
-  // modify the params of a directive, by applying a manipulator :
-  template<class T> 
-  basic_format&  modify_item(int itemN, const T& manipulator) 
-  {
-    return io::detail::modify_item_body(*this, itemN, manipulator) ;
-  }
-
-  // Choosing which errors will throw exceptions :
-  unsigned char exceptions() const;
-  unsigned char exceptions(unsigned char newexcept);
-
-  // final output
-  string_t str() const;
-  friend BOOST_IO_STD ostream& 
-  operator<< ( BOOST_IO_STD ostream& , const basic_format& ); 
-                      
-
-  template<class T>  friend basic_format&  
-  io::detail::feed(basic_format&, T);
-    
-  template<class T>  friend   
-  void io::detail::distribute(basic_format&, T);
-  
-  template<class T>  friend
-  basic_format&  io::detail::modify_item_body(basic_format&, int, const T&);
-
-  template<class T> friend
-  basic_format&  io::detail::bind_arg_body(basic_format&, int, const T&);
-
-// make the members private only if the friend templates are supported
-private:
-
-  // flag bits, used for style_
-  enum style_values  { ordered = 1,        // set only if all directives are  positional directives
-                       special_needs = 4 };     
-
-  // parse the format string :
-  void parse(const string_t&);
-
-  int                           style_;         // style of format-string :  positional or not, etc
-  int                           cur_arg_;       // keep track of wich argument will come
-  int                           num_args_;      // number of expected arguments
-  mutable bool                  dumped_;        // true only after call to str() or <<
-  std::vector<format_item_t>    items_;         // vector of directives (aka items)
-  string_t                      prefix_;        // piece of string to insert before first item
-
-  std::vector<bool>             bound_;         // stores which arguments were bound
-                                                //   size = num_args OR zero
-  internal_stream_t             oss_;           // the internal stream.
-  stream_format_state           state0_;        // reference state for oss_
-  unsigned char                 exceptions_;
-}; // class basic_format
-
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_CLASS_HPP
diff --git a/src/boost/format/format_fwd.hpp b/src/boost/format/format_fwd.hpp
deleted file mode 100644
index 97c55f6684c..00000000000
--- a/src/boost/format/format_fwd.hpp
+++ /dev/null
@@ -1,49 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// format_fwd.hpp :  forward declarations, for primary header format.hpp
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_FWD_HPP
-#define BOOST_FORMAT_FWD_HPP
-
-#include <string>
-#include <iosfwd>
-
-namespace boost {
-
-class basic_format;
-
-typedef basic_format    format;
-
-namespace io {
-enum format_error_bits { bad_format_string_bit = 1, 
-                         too_few_args_bit = 2, too_many_args_bit = 4,
-                         out_of_range_bit = 8,
-                         all_error_bits = 255, no_error_bits=0 };
-                  
-// Convertion:  format   to   string
-std::string     str(const basic_format& ) ;
-
-} // namespace io
-
-
-BOOST_IO_STD ostream& 
-operator<<( BOOST_IO_STD ostream&, const basic_format&);
-
-
-} // namespace boost
-
-#endif // BOOST_FORMAT_FWD_HPP
diff --git a/src/boost/format/format_implementation.cc b/src/boost/format/format_implementation.cc
deleted file mode 100644
index aa191afe113..00000000000
--- a/src/boost/format/format_implementation.cc
+++ /dev/null
@@ -1,256 +0,0 @@
-// -*- C++ -*-
-//  Boost general library format ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// format_implementation.hpp  Implementation of the basic_format class
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP
-#define BOOST_FORMAT_IMPLEMENTATION_HPP
-
-#include <boost/throw_exception.hpp>
-#include <boost/assert.hpp>
-#include <boost/format.hpp>
-
-namespace boost {
-
-// --------  format:: -------------------------------------------
-basic_format::basic_format(const char* str)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(), oss_(), exceptions_(io::all_error_bits)
-{
-    state0_.set_by_stream(oss_);
-    string_t emptyStr;
-    if( !str) str = emptyStr.c_str();
-    parse( str );
-}
-
-#ifndef BOOST_NO_STD_LOCALE
-basic_format::basic_format(const char* str, const std::locale & loc)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(), oss_(), exceptions_(io::all_error_bits)
-{
-    oss_.imbue( loc );
-    state0_.set_by_stream(oss_);
-    string_t emptyStr;
-    if( !str) str = emptyStr.c_str();
-    parse( str );
-}
-
-basic_format::basic_format(const string_t& s, const std::locale & loc)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(),  oss_(), exceptions_(io::all_error_bits)
-{
-    oss_.imbue( loc );
-    state0_.set_by_stream(oss_);
-    parse(s);  
-}
-#endif //BOOST_NO_STD_LOCALE
-
-basic_format::basic_format(const string_t& s)
-    : style_(0), cur_arg_(0), num_args_(0), dumped_(false),
-      items_(),  oss_(), exceptions_(io::all_error_bits)
-{
-    state0_.set_by_stream(oss_);
-    parse(s);  
-}
-
-basic_format:: basic_format(const basic_format& x)
-    : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), 
-      items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), 
-      oss_(),   // <- we obviously can't copy x.oss_
-      state0_(x.state0_), exceptions_(x.exceptions_)
-{ 
-    state0_.apply_on(oss_);
-} 
-
-basic_format& basic_format::operator= (const basic_format& x)
-{
-    if(this == &x)
-      return *this;
-    state0_ = x.state0_;
-    state0_.apply_on(oss_);
-
-    // plus all the other (trivial) assignments :
-    exceptions_ = x.exceptions_;
-    items_ = x.items_;
-    prefix_ = x.prefix_;
-    bound_=x.bound_;
-    style_=x.style_; 
-    cur_arg_=x.cur_arg_; 
-    num_args_=x.num_args_;
-    dumped_=x.dumped_;
-    return *this;
-}
-
-
-unsigned char basic_format::exceptions() const 
-{
-  return exceptions_; 
-}
-
-unsigned char basic_format::exceptions(unsigned char newexcept) 
-{ 
-  unsigned char swp = exceptions_; 
-  exceptions_ = newexcept; 
-  return swp; 
-}
-
-
-basic_format& basic_format ::clear()
-  // empty the string buffers (except bound arguments, see clear_binds() )
-  // and make the format object ready for formatting a new set of arguments
-{
-    BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast<int>(bound_.size()) );
-
-    for(unsigned long i=0; i<items_.size(); ++i){
-      items_[i].state_ = items_[i].ref_state_;
-      // clear converted strings only if the corresponding argument is not  bound :
-      if( bound_.size()==0 || !bound_[ items_[i].argN_ ] )  items_[i].res_.resize(0);
-    }
-    cur_arg_=0; dumped_=false;
-    // maybe first arg is bound:
-    if(bound_.size() != 0)
-      {
-        while(cur_arg_ < num_args_ && bound_[cur_arg_] )      ++cur_arg_;
-      }
-    return *this;
-}
-
-basic_format& basic_format ::clear_binds() 
-  // cancel all bindings, and clear()
-{
-    bound_.resize(0);
-    clear();
-    return *this;
-}
-
-basic_format& basic_format::clear_bind(int argN) 
-  // cancel the binding of ONE argument, and clear()
-{
-    if(argN<1 || argN > num_args_ || bound_.size()==0 || !bound_[argN-1] ) 
-      {
-        if( exceptions() & io::out_of_range_bit )
-          boost::throw_exception(io::out_of_range()); // arg not in range.
-        else return *this;
-      }
-    bound_[argN-1]=false;
-    clear();
-    return *this;
-}
-
-
-
-std::string basic_format::str() const
-{
-  dumped_=true;
-  if(items_.size()==0)
-    return prefix_;
-  if( cur_arg_ < num_args_)
-      if( exceptions() & io::too_few_args_bit )
-        boost::throw_exception(io::too_few_args()); // not enough variables have been supplied !
-
-  unsigned long sz = prefix_.size();
-  unsigned long i;
-  for(i=0; i < items_.size(); ++i) 
-    sz += items_[i].res_.size() + items_[i].appendix_.size();
-  string_t res;
-  res.reserve(sz);
-
-  res += prefix_;
-  for(i=0; i < items_.size(); ++i) 
-  {
-    const format_item_t& item = items_[i];
-    res += item.res_;
-    if( item.argN_ == format_item_t::argN_tabulation) 
-    { 
-      BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation);
-      std::streamsize  n = item.state_.width_ - res.size();
-      if( n > 0 )
-        res.append( n, item.state_.fill_ );
-    }
-    res += item.appendix_;
-  }
-  return res;
-}
-
-namespace io {
-namespace detail {
-
-template<class T>
-basic_format&  bind_arg_body( basic_format& self, 
-                                      int argN, 
-                                      const T& val)
-  // bind one argument to a fixed value
-  // this is persistent over clear() calls, thus also over str() and <<
-{
-    if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_..
-    if(argN<1 || argN > self.num_args_) 
-      {
-        if( self.exceptions() & io::out_of_range_bit )
-          boost::throw_exception(io::out_of_range()); // arg not in range.
-        else return self;
-      }
-    if(self.bound_.size()==0) 
-      self.bound_.assign(self.num_args_,false);
-    else 
-      BOOST_ASSERT( self.num_args_ == static_cast<signed int>(self.bound_.size()) );
-    int o_cur_arg = self.cur_arg_;
-    self.cur_arg_ = argN-1; // arrays begin at 0
-
-    self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets..
-    self.operator%(val); // put val at the right place, because cur_arg is set
-    
-
-    // Now re-position cur_arg before leaving :
-    self.cur_arg_ = o_cur_arg; 
-    self.bound_[argN-1]=true;
-    if(self.cur_arg_ == argN-1 )
-      // hum, now this arg is bound, so move to next free arg
-      {
-        while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_])   ++self.cur_arg_;
-      }
-    // In any case, we either have all args, or are on a non-binded arg :
-    BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]);
-    return self;
-}
-
-template<class T>
-basic_format&  modify_item_body( basic_format& self,
-                                      int itemN, 
-                                      const T& manipulator)
-  // applies a manipulator to the format_item describing a given directive.
-  // this is a permanent change, clear or clear_binds won't cancel that.
-{
-  if(itemN<1 || itemN >= static_cast<signed int>(self.items_.size() )) 
-    {
-      if( self.exceptions() & io::out_of_range_bit ) 
-        boost::throw_exception(io::out_of_range()); // item not in range.
-      else return self;
-    }
-  self.items_[itemN-1].ref_state_.apply_manip( manipulator );
-  self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_;
-  return self;
-}
-
-} // namespace detail
-
-} // namespace io
-
-} // namespace boost
-
-
-
-#endif  // BOOST_FORMAT_IMPLEMENTATION_HPP
diff --git a/src/boost/format/free_funcs.cc b/src/boost/format/free_funcs.cc
deleted file mode 100644
index 151db37a0ac..00000000000
--- a/src/boost/format/free_funcs.cc
+++ /dev/null
@@ -1,71 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// free_funcs.hpp :  implementation of the free functions declared in namespace format
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_FUNCS_HPP
-#define BOOST_FORMAT_FUNCS_HPP
-
-#include "boost/format.hpp"
-#include "boost/throw_exception.hpp"
-
-namespace boost {
-
-namespace io {
-  inline 
-  std::string str(const basic_format& f) 
-    // adds up all pieces of strings and converted items, and return the formatted string
-  {
-    return f.str();
-  }
-}   // - namespace io
-
-BOOST_IO_STD ostream& 
-operator<<( BOOST_IO_STD ostream& os, 
-            const boost::basic_format& f) 
-  // effect: "return os << str(f);" but we can try to do it faster
-{
-  typedef boost::basic_format   format_t;
-  if(f.items_.size()==0) 
-    os << f.prefix_;
-  else {
-    if(f.cur_arg_ < f.num_args_)
-      if( f.exceptions() & io::too_few_args_bit )
-        boost::throw_exception(io::too_few_args()); // not enough variables have been supplied !
-    if(f.style_ & format_t::special_needs) 
-        os << f.str();
-    else {
-    // else we dont have to count chars output, so we dump directly to os :
-      os << f.prefix_;
-      for(unsigned long i=0; i<f.items_.size(); ++i) 
-        {
-          const format_t::format_item_t& item = f.items_[i];
-          os << item.res_;
-          os << item.appendix_;
-
-        }
-    }
-  }
-  f.dumped_=true;
-  return os;
-}
-
-
-
-} // namespace boost
-
-
-#endif // BOOST_FORMAT_FUNCS_HPP
diff --git a/src/boost/format/group.hpp b/src/boost/format/group.hpp
deleted file mode 100644
index ac63f3f0bab..00000000000
--- a/src/boost/format/group.hpp
+++ /dev/null
@@ -1,680 +0,0 @@
-
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-
-// group.hpp :  encapsulates a group of manipulators along with an argument
-//                      
-// group_head : cut the last element of a group out.
-// (is overloaded below on each type of group)
-
-// group_last : returns the last element of a group
-// (is overloaded below on each type of group)
-
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_GROUP_HPP
-#define BOOST_FORMAT_GROUP_HPP
-
-
-namespace boost {
-namespace io {
-
-
-namespace detail {
-
-
-// empty group, but useful even though.
-struct group0 
-{
-    group0()      {}
-};
-
-template <class Ch, class Tr>
-inline
-BOOST_IO_STD ostream&
-operator << ( BOOST_IO_STD ostream& os,
-             const group0& )
-{ 
-   return os; 
-}
-
-template <class T1>
-struct group1
-{
-    T1 a1_;
-    group1(T1 a1)
-      : a1_(a1)
-      {}
-};
-
-template <class Ch, class Tr, class T1>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group1<T1>& x)
-{ 
-   os << x.a1_;  
-   return os; 
-}
-
-
-
-
-template <class T1,class T2>
-struct group2
-{
-    T1 a1_;
-    T2 a2_;
-    group2(T1 a1,T2 a2)
-      : a1_(a1),a2_(a2)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group2<T1,T2>& x)
-{ 
-   os << x.a1_<< x.a2_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3>
-struct group3
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    group3(T1 a1,T2 a2,T3 a3)
-      : a1_(a1),a2_(a2),a3_(a3)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group3<T1,T2,T3>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4>
-struct group4
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    group4(T1 a1,T2 a2,T3 a3,T4 a4)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group4<T1,T2,T3,T4>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5>
-struct group5
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group5<T1,T2,T3,T4,T5>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-struct group6
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group6<T1,T2,T3,T4,T5,T6>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-struct group7
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group7<T1,T2,T3,T4,T5,T6,T7>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-struct group8
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group8<T1,T2,T3,T4,T5,T6,T7,T8>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-struct group9
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    T9 a9_;
-    group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group9<T1,T2,T3,T4,T5,T6,T7,T8,T9>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_;  
-   return os; 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-struct group10
-{
-    T1 a1_;
-    T2 a2_;
-    T3 a3_;
-    T4 a4_;
-    T5 a5_;
-    T6 a6_;
-    T7 a7_;
-    T8 a8_;
-    T9 a9_;
-    T10 a10_;
-    group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10)
-      : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10)
-      {}
-};
-
-template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-BOOST_IO_STD ostream&
-operator << (BOOST_IO_STD ostream& os,
-             const group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10>& x)
-{ 
-   os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_;  
-   return os; 
-}
-
-
-
-
-template <class T1,class T2>
-inline
-group1<T1> 
-group_head( group2<T1,T2> const& x)
-{
-   return group1<T1> (x.a1_); 
-}
-
-template <class T1,class T2>
-inline
-group1<T2> 
-group_last( group2<T1,T2> const& x)
-{
-   return group1<T2> (x.a2_); 
-}
-
-
-
-template <class T1,class T2,class T3>
-inline
-group2<T1,T2> 
-group_head( group3<T1,T2,T3> const& x)
-{
-   return group2<T1,T2> (x.a1_,x.a2_); 
-}
-
-template <class T1,class T2,class T3>
-inline
-group1<T3> 
-group_last( group3<T1,T2,T3> const& x)
-{
-   return group1<T3> (x.a3_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4>
-inline
-group3<T1,T2,T3> 
-group_head( group4<T1,T2,T3,T4> const& x)
-{
-   return group3<T1,T2,T3> (x.a1_,x.a2_,x.a3_); 
-}
-
-template <class T1,class T2,class T3,class T4>
-inline
-group1<T4> 
-group_last( group4<T1,T2,T3,T4> const& x)
-{
-   return group1<T4> (x.a4_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5>
-inline
-group4<T1,T2,T3,T4> 
-group_head( group5<T1,T2,T3,T4,T5> const& x)
-{
-   return group4<T1,T2,T3,T4> (x.a1_,x.a2_,x.a3_,x.a4_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5>
-inline
-group1<T5> 
-group_last( group5<T1,T2,T3,T4,T5> const& x)
-{
-   return group1<T5> (x.a5_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-group5<T1,T2,T3,T4,T5> 
-group_head( group6<T1,T2,T3,T4,T5,T6> const& x)
-{
-   return group5<T1,T2,T3,T4,T5> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6>
-inline
-group1<T6> 
-group_last( group6<T1,T2,T3,T4,T5,T6> const& x)
-{
-   return group1<T6> (x.a6_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-group6<T1,T2,T3,T4,T5,T6> 
-group_head( group7<T1,T2,T3,T4,T5,T6,T7> const& x)
-{
-   return group6<T1,T2,T3,T4,T5,T6> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7>
-inline
-group1<T7> 
-group_last( group7<T1,T2,T3,T4,T5,T6,T7> const& x)
-{
-   return group1<T7> (x.a7_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-group7<T1,T2,T3,T4,T5,T6,T7> 
-group_head( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x)
-{
-   return group7<T1,T2,T3,T4,T5,T6,T7> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8>
-inline
-group1<T8> 
-group_last( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x)
-{
-   return group1<T8> (x.a8_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-group8<T1,T2,T3,T4,T5,T6,T7,T8> 
-group_head( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x)
-{
-   return group8<T1,T2,T3,T4,T5,T6,T7,T8> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9>
-inline
-group1<T9> 
-group_last( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x)
-{
-   return group1<T9> (x.a9_); 
-}
-
-
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> 
-group_head( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x)
-{
-   return group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); 
-}
-
-template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10>
-inline
-group1<T10> 
-group_last( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x)
-{
-   return group1<T10> (x.a10_); 
-}
-
-
-
-
-
-} // namespace detail
-
-
-
-// helper functions
-
-
-inline detail::group1< detail::group0 >  
-group() { return detail::group1< detail::group0 > ( detail::group0() ); }
-
-template  <class T1, class Var> 
-inline
-detail::group1< detail::group2<T1, Var const&> >
-  group(T1 a1, Var const& var)
-{ 
-   return detail::group1< detail::group2<T1, Var const&> >
-                   ( detail::group2<T1, Var const&> 
-                        (a1, var) 
-                  );
-}
-
-template  <class T1,class T2, class Var> 
-inline
-detail::group1< detail::group3<T1,T2, Var const&> >
-  group(T1 a1,T2 a2, Var const& var)
-{ 
-   return detail::group1< detail::group3<T1,T2, Var const&> >
-                   ( detail::group3<T1,T2, Var const&> 
-                        (a1,a2, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3, class Var> 
-inline
-detail::group1< detail::group4<T1,T2,T3, Var const&> >
-  group(T1 a1,T2 a2,T3 a3, Var const& var)
-{ 
-   return detail::group1< detail::group4<T1,T2,T3, Var const&> >
-                   ( detail::group4<T1,T2,T3, Var const&> 
-                        (a1,a2,a3, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4, class Var> 
-inline
-detail::group1< detail::group5<T1,T2,T3,T4, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var)
-{ 
-   return detail::group1< detail::group5<T1,T2,T3,T4, Var const&> >
-                   ( detail::group5<T1,T2,T3,T4, Var const&> 
-                        (a1,a2,a3,a4, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5, class Var> 
-inline
-detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var)
-{ 
-   return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> >
-                   ( detail::group6<T1,T2,T3,T4,T5, Var const&> 
-                        (a1,a2,a3,a4,a5, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6, class Var> 
-inline
-detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var)
-{ 
-   return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> >
-                   ( detail::group7<T1,T2,T3,T4,T5,T6, Var const&> 
-                        (a1,a2,a3,a4,a5,a6, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> 
-inline
-detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var)
-{ 
-   return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> >
-                   ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> 
-inline
-detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var)
-{ 
-   return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> >
-                   ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> 
-inline
-detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var)
-{ 
-   return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> >
-                   ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) 
-                  );
-}
-
-
-#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-
-template  <class T1, class Var> 
-inline
-detail::group1< detail::group2<T1, Var&> >
-  group(T1 a1, Var& var)
-{ 
-   return detail::group1< detail::group2<T1, Var&> >
-                   ( detail::group2<T1, Var&> 
-                        (a1, var) 
-                  );
-}
-
-template  <class T1,class T2, class Var> 
-inline
-detail::group1< detail::group3<T1,T2, Var&> >
-  group(T1 a1,T2 a2, Var& var)
-{ 
-   return detail::group1< detail::group3<T1,T2, Var&> >
-                   ( detail::group3<T1,T2, Var&> 
-                        (a1,a2, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3, class Var> 
-inline
-detail::group1< detail::group4<T1,T2,T3, Var&> >
-  group(T1 a1,T2 a2,T3 a3, Var& var)
-{ 
-   return detail::group1< detail::group4<T1,T2,T3, Var&> >
-                   ( detail::group4<T1,T2,T3, Var&> 
-                        (a1,a2,a3, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4, class Var> 
-inline
-detail::group1< detail::group5<T1,T2,T3,T4, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var)
-{ 
-   return detail::group1< detail::group5<T1,T2,T3,T4, Var&> >
-                   ( detail::group5<T1,T2,T3,T4, Var&> 
-                        (a1,a2,a3,a4, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5, class Var> 
-inline
-detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var)
-{ 
-   return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> >
-                   ( detail::group6<T1,T2,T3,T4,T5, Var&> 
-                        (a1,a2,a3,a4,a5, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6, class Var> 
-inline
-detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var)
-{ 
-   return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> >
-                   ( detail::group7<T1,T2,T3,T4,T5,T6, Var&> 
-                        (a1,a2,a3,a4,a5,a6, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> 
-inline
-detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var)
-{ 
-   return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> >
-                   ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> 
-inline
-detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var)
-{ 
-   return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> >
-                   ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8, var) 
-                  );
-}
-
-template  <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> 
-inline
-detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> >
-  group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var)
-{ 
-   return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> >
-                   ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> 
-                        (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) 
-                  );
-}
-
-
-#endif  //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST
-
-
-} // namespace io
-
-} // namespace boost
-
-
-#endif   // BOOST_FORMAT_GROUP_HPP
diff --git a/src/boost/format/internals.hpp b/src/boost/format/internals.hpp
deleted file mode 100644
index d25eb4c864c..00000000000
--- a/src/boost/format/internals.hpp
+++ /dev/null
@@ -1,167 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream
-
-// ----------------------------------------------------------------------------
-// internals.hpp :  internal structs. included by format.hpp
-//                              stream_format_state, and format_item
-// ----------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_INTERNALS_HPP
-#define BOOST_FORMAT_INTERNALS_HPP
-
-
-#include <string>
-#include <sstream>
-
-namespace boost {
-namespace io {
-namespace detail {
-
-
-// --------------
-// set of params that define the format state of a stream
-
-struct stream_format_state 
-{
-  typedef std::ios   basic_ios;
-
-  std::streamsize width_;
-  std::streamsize precision_;
-  char fill_; 
-  std::ios::fmtflags flags_;
-
-  stream_format_state()       : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec)  {}
-  stream_format_state(basic_ios& os)                  {set_by_stream(os); }
-
-  void apply_on(basic_ios & os) const;                //- applies format_state to the stream
-  template<class T> void apply_manip(T manipulator)   //- modifies state by applying manipulator.
-       { apply_manip_body<T>( *this, manipulator) ; }
-  void reset();                                       //- sets to default state.
-  void set_by_stream(const basic_ios& os);            //- sets to os's state.
-};  
-
-
-
-// --------------
-// format_item : stores all parameters that can be defined by directives in the format-string
-
-struct format_item 
-{     
-  enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 };
-
-  enum arg_values { argN_no_posit   = -1, // non-positional directive. argN will be set later.
-                    argN_tabulation = -2, // tabulation directive. (no argument read) 
-                    argN_ignored    = -3  // ignored directive. (no argument read)
-  };
-  typedef BOOST_IO_STD ios              basic_ios;
-  typedef detail::stream_format_state         stream_format_state;
-  typedef std::string           string_t;
-  typedef BOOST_IO_STD ostringstream    internal_stream_t;
-
-
-  int         argN_;           //- argument number (starts at 0,  eg : %1 => argN=0)
-                               //  negative values are used for items that don't process
-                               //  an argument
-  string_t    res_;            //- result of the formatting of this item
-  string_t    appendix_;       //- piece of string between this item and the next
-
-  stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item
-  stream_format_state state_;  // always same as ref_state, _unless_ modified by manipulators 'group(..)'
-
-  // non-stream format-state parameters
-  signed int truncate_;        //- is >=0 for directives like %.5s (take 5 chars from the string)
-  unsigned int pad_scheme_;    //- several possible padding schemes can mix. see pad_values
-
-  format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0)  {}
-
-  void compute_states();      // sets states  according to truncate and pad_scheme.
-}; 
-
-
-
-// -----------------------------------------------------------
-// Definitions
-// -----------------------------------------------------------
-
-// --- stream_format_state:: -------------------------------------------
-inline
-void stream_format_state::apply_on(basic_ios & os) const
-  // set the state of this stream according to our params
-{
-      if(width_ != -1)
-        os.width(width_);
-      if(precision_ != -1)
-        os.precision(precision_);
-      if(fill_ != 0)
-        os.fill(fill_);
-      os.flags(flags_);
-}
-
-inline
-void stream_format_state::set_by_stream(const basic_ios& os) 
-  // set our params according to the state of this stream
-{
-      flags_ = os.flags();
-      width_ = os.width();
-      precision_ = os.precision();
-      fill_ = os.fill();
-}
-
-template<class T>  inline
-void apply_manip_body( stream_format_state& self,
-                       T manipulator) 
-  // modify our params according to the manipulator
-{
-      BOOST_IO_STD stringstream  ss;
-      self.apply_on( ss );
-      ss << manipulator;
-      self.set_by_stream( ss );
-}
-
-inline
-void stream_format_state::reset() 
-  // set our params to standard's default state
-{
-      width_=-1; precision_=-1; fill_=0; 
-      flags_ = std::ios::dec; 
-}
-
-
-// --- format_items:: -------------------------------------------
-inline
-void format_item::compute_states() 
-  // reflect pad_scheme_   on  state_ and ref_state_ 
-  //   because some pad_schemes has complex consequences on several state params.
-{
-  if(pad_scheme_ & zeropad) 
-  {
-    if(ref_state_.flags_ & std::ios::left) 
-    {
-      pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment
-    }
-    else 
-    { 
-      ref_state_.fill_='0'; 
-      ref_state_.flags_ |= std::ios::internal;
-    }
-  }
-  state_ = ref_state_;
-}
-
-
-} } } // namespaces boost :: io :: detail
-
-
-#endif // BOOST_FORMAT_INTERNALS_HPP
diff --git a/src/boost/format/internals_fwd.hpp b/src/boost/format/internals_fwd.hpp
deleted file mode 100644
index a8ebf7c3abc..00000000000
--- a/src/boost/format/internals_fwd.hpp
+++ /dev/null
@@ -1,65 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// internals_fwd.hpp :  forward declarations, for internal headers
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP
-#define BOOST_FORMAT_INTERNAL_FWD_HPP
-
-#include "boost/format/format_fwd.hpp"
-
-
-namespace boost {
-namespace io {
-
-namespace detail {
-  struct stream_format_state;
-  struct format_item;
-}
-
-
-namespace detail {
-
-  // these functions were intended as methods, 
-  // but MSVC have problems with template member functions :
-
-  // defined in format_implementation.hpp :
-     template<class T> 
-     basic_format&  modify_item_body( basic_format& self, 
-                                          int itemN, const T& manipulator);
-
-     template<class T> 
-     basic_format&  bind_arg_body( basic_format& self,
-                                           int argN, const T& val);
-
-    template<class T> 
-    void apply_manip_body( stream_format_state& self,
-                           T manipulator);
-
-  // argument feeding (defined in feed_args.hpp ) :
-     template<class T> 
-     void distribute(basic_format& self, T x);
-
-     template<class T> 
-     basic_format& feed(basic_format& self, T x);
- 
-} // namespace detail
-
-} // namespace io
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_INTERNAL_FWD_HPP
diff --git a/src/boost/format/local.mk b/src/boost/format/local.mk
deleted file mode 100644
index 3776eff382f..00000000000
--- a/src/boost/format/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-libraries += libformat
-
-libformat_NAME = libnixformat
-
-libformat_DIR := $(d)
-
-libformat_SOURCES := $(wildcard $(d)/*.cc)
diff --git a/src/boost/format/macros_default.hpp b/src/boost/format/macros_default.hpp
deleted file mode 100644
index 4fd84a163fb..00000000000
--- a/src/boost/format/macros_default.hpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from R�diger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// macros_default.hpp : configuration for the format library
-//                       provides default values for the stl workaround macros
-// ------------------------------------------------------------------------------
-
-#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP
-#define BOOST_FORMAT_MACROS_DEFAULT_HPP
-
-// *** This should go to "boost/config/suffix.hpp".
-
-#ifndef BOOST_IO_STD
-#  define BOOST_IO_STD std::
-#endif
-
-// **** Workaround for io streams, stlport and msvc.
-#ifdef BOOST_IO_NEEDS_USING_DECLARATION
-namespace boost {
-  using std::char_traits;
-  using std::basic_ostream;
-  using std::basic_ostringstream;
-  namespace io {
-    using std::basic_ostream;
-    namespace detail {
-      using std::basic_ios;
-      using std::basic_ostream;
-      using std::basic_ostringstream;
-    }
-  }
-}
-#endif
-
-// ------------------------------------------------------------------------------
-
-#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP
diff --git a/src/boost/format/parsing.cc b/src/boost/format/parsing.cc
deleted file mode 100644
index 34c36adeb73..00000000000
--- a/src/boost/format/parsing.cc
+++ /dev/null
@@ -1,454 +0,0 @@
-// -*- C++ -*-
-//  Boost general library 'format'   ---------------------------
-//  See http://www.boost.org for updates, documentation, and revision history.
-
-//  (C) Samuel Krempp 2001
-//                  krempp@crans.ens-cachan.fr
-//  Permission to copy, use, modify, sell and
-//  distribute this software is granted provided this copyright notice appears
-//  in all copies. This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-
-// ideas taken from Rudiger Loos's format class
-// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing)
-
-// ------------------------------------------------------------------------------
-// parsing.hpp :  implementation of the parsing member functions
-//                      ( parse, parse_printf_directive)
-// ------------------------------------------------------------------------------
-
-
-#ifndef BOOST_FORMAT_PARSING_HPP
-#define BOOST_FORMAT_PARSING_HPP
-
-
-#include <boost/format.hpp>
-#include <boost/throw_exception.hpp>
-#include <boost/assert.hpp>
-
-
-namespace boost {
-namespace io {
-namespace detail {
-
-  template<class Stream> inline
-  bool wrap_isdigit(char c, Stream &os) 
-  {
-#ifndef BOOST_NO_LOCALE_ISIDIGIT
-    return std::isdigit(c, os.rdbuf()->getloc() );
-# else
-    using namespace std;
-    return isdigit(c); 
-#endif 
-  } //end- wrap_isdigit(..)
-
-  template<class Res> inline
-  Res str2int(const std::string& s, 
-              std::string::size_type start, 
-              BOOST_IO_STD ios &os,
-              const Res = Res(0)  ) 
-    // Input : char string, with starting index
-    //         a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    // Effects : reads s[start:] and converts digits into an integral n, of type Res
-    // Returns : n
-  {
-    Res n = 0;
-    while(start<s.size() && wrap_isdigit(s[start], os) ) {
-      char cur_ch = s[start];
-      BOOST_ASSERT(cur_ch != 0 ); // since we called isdigit, this should not happen.
-      n *= 10;
-      n += cur_ch - '0'; // 22.2.1.1.2 of the C++ standard
-      ++start;
-    }
-    return n;
-  }
-
-  void skip_asterisk(const std::string & buf, 
-                     std::string::size_type * pos_p,
-                     BOOST_IO_STD ios &os)
-    // skip printf's "asterisk-fields" directives in the format-string buf
-    // Input : char string, with starting index *pos_p
-    //         a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    // Effects : advance *pos_p by skipping printf's asterisk fields.
-    // Returns : nothing
-  {
-    using namespace std;
-    BOOST_ASSERT( pos_p != 0);
-    if(*pos_p >= buf.size() ) return;
-    if(buf[ *pos_p]=='*') {
-      ++ (*pos_p);
-      while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p);
-      if(buf[*pos_p]=='$') ++(*pos_p);
-    }
-  }
-
-
-  inline void maybe_throw_exception( unsigned char exceptions)
-    // auxiliary func called by parse_printf_directive
-    // for centralising error handling
-    // it either throws if user sets the corresponding flag, or does nothing.
-  {
-    if(exceptions & io::bad_format_string_bit)
-          boost::throw_exception(io::bad_format_string());
-  }
-    
-
-
-  bool parse_printf_directive(const std::string & buf,
-                              std::string::size_type * pos_p,
-                              detail::format_item * fpar,
-                              BOOST_IO_STD ios &os,
-                              unsigned char exceptions)
-    // Input   : a 'printf-directive' in the format-string, starting at buf[ *pos_p ]
-    //           a basic_ios& merely to call its widen/narrow member function in the desired locale.
-    //           a bitset'excpetions' telling whether to throw exceptions on errors.
-    // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) 
-    //           false if it failed so bad that the directive should be printed verbatim
-    // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive
-    //           - *fpar is set with the parameters read in the directive
-  {
-    typedef format_item  format_item_t;
-    BOOST_ASSERT( pos_p != 0);
-    std::string::size_type       &i1 = *pos_p,      
-                                                        i0; 
-    fpar->argN_ = format_item_t::argN_no_posit;  // if no positional-directive
-
-    bool in_brackets=false;
-    if(buf[i1]=='|')
-      {
-        in_brackets=true;
-        if( ++i1 >= buf.size() ) {
-          maybe_throw_exception(exceptions);
-          return false;
-        }
-      }
-
-    // the flag '0' would be picked as a digit for argument order, but here it's a flag :
-    if(buf[i1]=='0') 
-      goto parse_flags;
-
-    // handle argument order (%2$d)  or possibly width specification: %2d
-    i0 = i1;  // save position before digits
-    while (i1 < buf.size() && wrap_isdigit(buf[i1], os))
-      ++i1;
-    if (i1!=i0) 
-      {
-        if( i1 >= buf.size() ) {
-          maybe_throw_exception(exceptions);
-          return false;
-        }
-        int n=str2int(buf,i0, os, int(0) );
-        
-        // %N% case : this is already the end of the directive
-        if( buf[i1] == '%' ) 
-          {
-            fpar->argN_ = n-1;
-            ++i1;
-            if( in_brackets) 
-              maybe_throw_exception(exceptions); 
-              // but don't return.  maybe "%" was used in lieu of '$', so we go on.
-            else return true;
-          }
-
-        if ( buf[i1]=='$' ) 
-          {
-            fpar->argN_ = n-1;
-            ++i1;
-          } 
-        else  
-          {
-            // non-positionnal directive
-            fpar->ref_state_.width_ = n;
-            fpar->argN_  = format_item_t::argN_no_posit;
-            goto parse_precision;
-          }
-      }
-    
-  parse_flags: 
-    // handle flags
-    while ( i1 <buf.size()) // as long as char is one of + - = # 0 l h   or ' '
-      {  
-        // misc switches
-        switch (buf[i1]) 
-          {
-          case '\'' : break; // no effect yet. (painful to implement)
-          case 'l':
-          case 'h':  // short/long modifier : for printf-comaptibility (no action needed)
-             break;
-          case '-':
-            fpar->ref_state_.flags_ |= std::ios::left;
-            break;
-          case '=':
-            fpar->pad_scheme_ |= format_item_t::centered;
-            break;
-          case ' ':
-            fpar->pad_scheme_ |= format_item_t::spacepad;
-            break;
-          case '+':
-            fpar->ref_state_.flags_ |= std::ios::showpos;
-            break;
-          case '0':
-            fpar->pad_scheme_ |= format_item_t::zeropad; 
-            // need to know alignment before really setting flags,
-            // so just add 'zeropad' flag for now, it will be processed later.
-            break;
-          case '#':
-            fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase;
-            break;
-          default:
-            goto parse_width;
-          }
-        ++i1;
-      } // loop on flag.
-    if( i1>=buf.size()) {
-      maybe_throw_exception(exceptions);
-      return true; 
-    }
-
-  parse_width:
-    // handle width spec
-    skip_asterisk(buf, &i1, os); // skips 'asterisk fields' :  *, or *N$
-    i0 = i1;  // save position before digits
-    while (i1<buf.size() && wrap_isdigit(buf[i1], os))
-      i1++;
-    
-    if (i1!=i0) 
-      { fpar->ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); }
-
-  parse_precision:
-    if( i1>=buf.size()) { 
-      maybe_throw_exception(exceptions);
-      return true;
-    }
-    // handle precision spec
-    if (buf[i1]=='.')  
-      {
-        ++i1;
-        skip_asterisk(buf, &i1, os);
-        i0 = i1;  // save position before digits
-        while (i1<buf.size() && wrap_isdigit(buf[i1], os))
-          ++i1;
-
-        if(i1==i0)
-          fpar->ref_state_.precision_ = 0;
-        else 
-          fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) );
-      }
-    
-    // handle  formatting-type flags :
-    while( i1<buf.size() && 
-           ( buf[i1]=='l' || buf[i1]=='L' || buf[i1]=='h') )
-      ++i1;
-    if( i1>=buf.size()) {
-      maybe_throw_exception(exceptions);
-      return true;
-    }
-    
-    if( in_brackets && buf[i1]=='|' ) 
-      {
-        ++i1;
-        return true;
-      }
-    switch (buf[i1])  
-      {
-      case 'X':
-        fpar->ref_state_.flags_ |= std::ios::uppercase;
-      case 'p': // pointer => set hex.
-      case 'x':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |= std::ios::hex;
-        break;
-      
-      case 'o':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::oct;
-        break;
-
-      case 'E':
-        fpar->ref_state_.flags_ |=  std::ios::uppercase;
-      case 'e':
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield;
-        fpar->ref_state_.flags_ |=  std::ios::scientific;
-
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-        break;
-      
-      case 'f':
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield;
-        fpar->ref_state_.flags_ |=  std::ios::fixed;
-      case 'u':
-      case 'd':
-      case 'i':
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-        break;
-
-      case 'T':
-        ++i1;
-        if( i1 >= buf.size())
-          maybe_throw_exception(exceptions);
-        else
-          fpar->ref_state_.fill_ = buf[i1];
-        fpar->pad_scheme_ |= format_item_t::tabulation;
-        fpar->argN_ = format_item_t::argN_tabulation; 
-        break;
-      case 't': 
-        fpar->ref_state_.fill_ = ' ';
-        fpar->pad_scheme_ |= format_item_t::tabulation;
-        fpar->argN_ = format_item_t::argN_tabulation; 
-        break;
-
-      case 'G':
-        fpar->ref_state_.flags_ |= std::ios::uppercase;
-        break;
-      case 'g': // 'g' conversion is default for floats.
-        fpar->ref_state_.flags_ &= ~std::ios::basefield;
-        fpar->ref_state_.flags_ |=  std::ios::dec;
-
-        // CLEAR all floatield flags, so stream will CHOOSE
-        fpar->ref_state_.flags_ &= ~std::ios::floatfield; 
-        break;
-
-      case 'C':
-      case 'c': 
-        fpar->truncate_ = 1;
-        break;
-      case 'S':
-      case 's': 
-        fpar->truncate_ = fpar->ref_state_.precision_;
-        fpar->ref_state_.precision_ = -1;
-        break;
-      case 'n' :  
-        fpar->argN_ = format_item_t::argN_ignored;
-        break;
-      default: 
-        maybe_throw_exception(exceptions);
-      }
-    ++i1;
-
-    if( in_brackets )
-      {
-        if( i1<buf.size() && buf[i1]=='|' ) 
-          {
-            ++i1;
-            return true;
-          }
-        else  maybe_throw_exception(exceptions);
-      }
-    return true;
-  }
-
-} // detail namespace
-} // io namespace
-
-
-// -----------------------------------------------
-//  format :: parse(..)
-
-void basic_format::parse(const string_t & buf) 
-  // parse the format-string
-{
-    using namespace std;
-    const char arg_mark = '%';
-    bool ordered_args=true; 
-    int max_argN=-1;
-    string_t::size_type i1=0;
-    int num_items=0;
-    
-    // A: find upper_bound on num_items and allocates arrays
-    i1=0; 
-    while( (i1=buf.find(arg_mark,i1)) != string::npos ) 
-    {
-      if( i1+1 >= buf.size() ) {
-        if(exceptions() & io::bad_format_string_bit)
-          boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %"
-        else break; // stop there, ignore last '%'
-      }
-      if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##"
-      ++i1;
-      
-      // in case of %N% directives, dont count it double (wastes allocations..) :
-      while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1;
-      if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1;
-
-      ++num_items;
-    }
-    items_.assign( num_items, format_item_t() );
-    
-    // B: Now the real parsing of the format string :
-    num_items=0;
-    i1 = 0;
-    string_t::size_type i0 = i1;
-    bool special_things=false;
-    int cur_it=0;
-    while( (i1=buf.find(arg_mark,i1)) != string::npos ) 
-    {
-      string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_;
-
-      if( buf[i1+1] == buf[i1] ) // escaped mark, '%%'
-      {
-        piece += buf.substr(i0, i1-i0) + buf[i1]; 
-        i1+=2; i0=i1;
-        continue; 
-      }
-      BOOST_ASSERT(  static_cast<unsigned int>(cur_it) < items_.size() || cur_it==0);
-
-      if(i1!=i0) piece += buf.substr(i0, i1-i0);
-      ++i1;
-      
-      bool parse_ok;
-      parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions());
-      if( ! parse_ok ) continue; // the directive will be printed verbatim
-
-      i0=i1;
-      items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params.
-
-      int argN=items_[cur_it].argN_;
-      if(argN == format_item_t::argN_ignored)
-        continue;
-      if(argN ==format_item_t::argN_no_posit)
-        ordered_args=false;
-      else if(argN == format_item_t::argN_tabulation) special_things=true;
-      else if(argN > max_argN) max_argN = argN;
-      ++num_items;
-      ++cur_it;
-    } // loop on %'s
-    BOOST_ASSERT(cur_it == num_items);
-    
-    // store the final piece of string
-    string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_;
-    piece += buf.substr(i0);
-    
-    if( !ordered_args) 
-    {
-      if(max_argN >= 0 )  // dont mix positional with non-positionnal directives
-        {
-          if(exceptions() & io::bad_format_string_bit)
-            boost::throw_exception(io::bad_format_string());
-          // else do nothing. => positionnal arguments are processed as non-positionnal
-        }
-      // set things like it would have been with positional directives :
-      int non_ordered_items = 0;
-      for(int i=0; i< num_items; ++i)
-        if(items_[i].argN_ == format_item_t::argN_no_posit) 
-          {
-            items_[i].argN_ = non_ordered_items;
-            ++non_ordered_items;
-          }
-      max_argN = non_ordered_items-1;
-    }
-    
-    // C: set some member data :
-    items_.resize(num_items);
-
-    if(special_things) style_ |= special_needs;
-    num_args_ = max_argN + 1;
-    if(ordered_args) style_ |=  ordered;
-    else style_ &= ~ordered;
-}
-
-} // namespace boost
-
-
-#endif //  BOOST_FORMAT_PARSING_HPP
diff --git a/src/boost/throw_exception.hpp b/src/boost/throw_exception.hpp
deleted file mode 100644
index 07b4ae5ceae..00000000000
--- a/src/boost/throw_exception.hpp
+++ /dev/null
@@ -1,47 +0,0 @@
-#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED
-#define BOOST_THROW_EXCEPTION_HPP_INCLUDED
-
-// MS compatible compilers support #pragma once
-
-#if defined(_MSC_VER) && (_MSC_VER >= 1020)
-# pragma once
-#endif
-
-//
-//  boost/throw_exception.hpp
-//
-//  Copyright (c) 2002 Peter Dimov and Multi Media Ltd.
-//
-//  Permission to copy, use, modify, sell and distribute this software
-//  is granted provided this copyright notice appears in all copies.
-//  This software is provided "as is" without express or implied
-//  warranty, and with no claim as to its suitability for any purpose.
-//
-//  http://www.boost.org/libs/utility/throw_exception.html
-//
-
-//#include <boost/config.hpp>
-
-#ifdef BOOST_NO_EXCEPTIONS
-# include <exception>
-#endif
-
-namespace boost
-{
-
-#ifdef BOOST_NO_EXCEPTIONS
-
-void throw_exception(std::exception const & e); // user defined
-
-#else
-
-template<class E> void throw_exception(E const & e)
-{
-    throw e;
-}
-
-#endif
-
-} // namespace boost
-
-#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e162e81aab5..b8613427e0f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -156,7 +156,7 @@ class Goal : public std::enable_shared_from_this<Goal>
         abort();
     }
 
-    void trace(const format & f);
+    void trace(const FormatOrString & fs);
 
     string getName()
     {
@@ -417,9 +417,9 @@ void Goal::amDone(ExitCode result)
 }
 
 
-void Goal::trace(const format & f)
+void Goal::trace(const FormatOrString & fs)
 {
-    debug(format("%1%: %2%") % name % f);
+    debug("%1%: %2%", name, fs.s);
 }
 
 
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index b13001b06d5..42d40e71d8b 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -7,7 +7,7 @@
 
 namespace nix {
 
-[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f)
+[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs)
 {
     int err = sqlite3_errcode(db);
 
@@ -21,7 +21,7 @@ namespace nix {
             : fmt("SQLite database '%s' is busy", path));
     }
     else
-        throw SQLiteError("%s: %s (in '%s')", f.str(), sqlite3_errstr(err), path);
+        throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path);
 }
 
 SQLite::SQLite(const Path & path)
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 14a7a0dd899..115679b8415 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -93,7 +93,7 @@ struct SQLiteTxn
 MakeError(SQLiteError, Error);
 MakeError(SQLiteBusy, SQLiteError);
 
-[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f);
+[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs);
 
 void handleSQLiteBusy(const SQLiteBusy & e);
 
diff --git a/tests/restricted.sh b/tests/restricted.sh
index 0605383cc86..a87d8ec2c94 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -11,8 +11,8 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix')
 nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=..
 
-(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost')
-nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost' -I src=../src
+(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel')
+nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src
 
 (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.

From c04bca34016c41d1fcab222e2d61a26a285340c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Mar 2018 19:11:13 +0100
Subject: [PATCH 0897/2196] Factor out commonality between release.nix and
 shell.nix

---
 release-common.nix | 40 ++++++++++++++++++++++++++++++++++++++--
 release.nix        | 28 ++++------------------------
 shell.nix          | 23 +----------------------
 3 files changed, 43 insertions(+), 48 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index f9df6fab9ff..0c12bc7ceb3 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -1,9 +1,11 @@
 { pkgs }:
 
+with pkgs;
+
 rec {
   # Use "busybox-sandbox-shell" if present,
   # if not (legacy) fallback and hope it's sufficient.
-  sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
+  sh = pkgs.busybox-sandbox-shell or (busybox.override {
     useMusl = true;
     enableStatic = true;
     enableMinimal = true;
@@ -30,7 +32,41 @@ rec {
   configureFlags =
     [ "--disable-init-state"
       "--enable-gc"
-    ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
+    ] ++ lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
+
+  tarballDeps =
+    [ bison
+      flex
+      libxml2
+      libxslt
+      docbook5
+      docbook5_xsl
+      autoconf-archive
+      autoreconfHook
+    ];
+
+  buildDeps =
+    [ curl
+      bzip2 xz brotli
+      openssl pkgconfig sqlite boehmgc
+      boost
+
+      # Tests
+      git
+      mercurial
+    ]
+    ++ lib.optional stdenv.isLinux libseccomp
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+    ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+      (aws-sdk-cpp.override {
+        apis = ["s3"];
+        customMemoryManagement = false;
+      });
+
+  perlDeps =
+    [ perl
+      perlPackages.DBDSQLite
+    ];
 }
diff --git a/release.nix b/release.nix
index 2e9292955a2..7db5e20e1e1 100644
--- a/release.nix
+++ b/release.nix
@@ -14,6 +14,8 @@ let
     tarball =
       with pkgs;
 
+      with import ./release-common.nix { inherit pkgs; };
+
       releaseTools.sourceTarball {
         name = "nix-tarball";
         version = builtins.readFile ./version;
@@ -21,13 +23,7 @@ let
         src = nix;
         inherit officialRelease;
 
-        buildInputs =
-          [ curl bison flex libxml2 libxslt
-            bzip2 xz brotli
-            pkgconfig sqlite libsodium boehmgc
-            docbook5 docbook5_xsl
-            autoconf-archive
-          ] ++ lib.optional stdenv.isLinux libseccomp;
+        buildInputs = tarballDeps ++ buildDeps;
 
         configureFlags = "--enable-gc";
 
@@ -67,23 +63,7 @@ let
         name = "nix";
         src = tarball;
 
-        buildInputs =
-          [ curl
-            bzip2 xz brotli
-            openssl pkgconfig sqlite boehmgc
-            boost
-
-            # Tests
-            git
-            mercurial
-          ]
-          ++ lib.optional stdenv.isLinux libseccomp
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
-            (aws-sdk-cpp.override {
-              apis = ["s3"];
-              customMemoryManagement = false;
-            });
+        buildInputs = buildDeps;
 
         configureFlags = configureFlags ++
           [ "--sysconfdir=/etc" ];
diff --git a/shell.nix b/shell.nix
index 09318dfd078..20cf4133b3f 100644
--- a/shell.nix
+++ b/shell.nix
@@ -7,28 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs =
-    [ curl bison flex libxml2 libxslt
-      bzip2 xz brotli
-      pkgconfig sqlite libsodium boehmgc
-      docbook5 docbook5_xsl
-      autoconf-archive
-      (aws-sdk-cpp.override {
-        apis = ["s3"];
-        customMemoryManagement = false;
-      })
-      autoreconfHook
-      boost
-
-      # For nix-perl
-      perl
-      perlPackages.DBDSQLite
-
-      # Tests
-      git
-      mercurial
-    ]
-    ++ lib.optional stdenv.isLinux libseccomp;
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
 
   inherit configureFlags;
 

From 55aa622fb1ae01a8a939fa32f71e7eed50afbbf9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 14 Mar 2018 19:20:32 +0100
Subject: [PATCH 0898/2196] Fix compatibility with latest boost::format

---
 src/libexpr/nixexpr.cc | 38 +++++++++++++++++++-------------------
 src/libexpr/nixexpr.hh |  8 ++++----
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 7b0a127cd41..63cbef1ddf8 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -10,7 +10,7 @@ namespace nix {
 
 /* Displaying abstract syntax trees. */
 
-std::ostream & operator << (std::ostream & str, Expr & e)
+std::ostream & operator << (std::ostream & str, const Expr & e)
 {
     e.show(str);
     return str;
@@ -58,48 +58,48 @@ std::ostream & operator << (std::ostream & str, const Symbol & sym)
     return str;
 }
 
-void Expr::show(std::ostream & str)
+void Expr::show(std::ostream & str) const
 {
     abort();
 }
 
-void ExprInt::show(std::ostream & str)
+void ExprInt::show(std::ostream & str) const
 {
     str << n;
 }
 
-void ExprFloat::show(std::ostream & str)
+void ExprFloat::show(std::ostream & str) const
 {
     str << nf;
 }
 
-void ExprString::show(std::ostream & str)
+void ExprString::show(std::ostream & str) const
 {
     showString(str, s);
 }
 
-void ExprPath::show(std::ostream & str)
+void ExprPath::show(std::ostream & str) const
 {
     str << s;
 }
 
-void ExprVar::show(std::ostream & str)
+void ExprVar::show(std::ostream & str) const
 {
     str << name;
 }
 
-void ExprSelect::show(std::ostream & str)
+void ExprSelect::show(std::ostream & str) const
 {
     str << "(" << *e << ")." << showAttrPath(attrPath);
     if (def) str << " or (" << *def << ")";
 }
 
-void ExprOpHasAttr::show(std::ostream & str)
+void ExprOpHasAttr::show(std::ostream & str) const
 {
     str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
 }
 
-void ExprAttrs::show(std::ostream & str)
+void ExprAttrs::show(std::ostream & str) const
 {
     if (recursive) str << "rec ";
     str << "{ ";
@@ -113,7 +113,7 @@ void ExprAttrs::show(std::ostream & str)
     str << "}";
 }
 
-void ExprList::show(std::ostream & str)
+void ExprList::show(std::ostream & str) const
 {
     str << "[ ";
     for (auto & i : elems)
@@ -121,7 +121,7 @@ void ExprList::show(std::ostream & str)
     str << "]";
 }
 
-void ExprLambda::show(std::ostream & str)
+void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
     if (matchAttrs) {
@@ -143,7 +143,7 @@ void ExprLambda::show(std::ostream & str)
     str << ": " << *body << ")";
 }
 
-void ExprLet::show(std::ostream & str)
+void ExprLet::show(std::ostream & str) const
 {
     str << "(let ";
     for (auto & i : attrs->attrs)
@@ -155,27 +155,27 @@ void ExprLet::show(std::ostream & str)
     str << "in " << *body << ")";
 }
 
-void ExprWith::show(std::ostream & str)
+void ExprWith::show(std::ostream & str) const
 {
     str << "(with " << *attrs << "; " << *body << ")";
 }
 
-void ExprIf::show(std::ostream & str)
+void ExprIf::show(std::ostream & str) const
 {
     str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
 }
 
-void ExprAssert::show(std::ostream & str)
+void ExprAssert::show(std::ostream & str) const
 {
     str << "assert " << *cond << "; " << *body;
 }
 
-void ExprOpNot::show(std::ostream & str)
+void ExprOpNot::show(std::ostream & str) const
 {
     str << "(! " << *e << ")";
 }
 
-void ExprConcatStrings::show(std::ostream & str)
+void ExprConcatStrings::show(std::ostream & str) const
 {
     bool first = true;
     str << "(";
@@ -186,7 +186,7 @@ void ExprConcatStrings::show(std::ostream & str)
     str << ")";
 }
 
-void ExprPos::show(std::ostream & str)
+void ExprPos::show(std::ostream & str) const
 {
     str << "__curPos";
 }
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index eb9fac25c01..8c8f3964068 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -76,17 +76,17 @@ string showAttrPath(const AttrPath & attrPath);
 struct Expr
 {
     virtual ~Expr() { };
-    virtual void show(std::ostream & str);
+    virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
-std::ostream & operator << (std::ostream & str, Expr & e);
+std::ostream & operator << (std::ostream & str, const Expr & e);
 
 #define COMMON_METHODS \
-    void show(std::ostream & str); \
+    void show(std::ostream & str) const; \
     void eval(EvalState & state, Env & env, Value & v); \
     void bindVars(const StaticEnv & env);
 
@@ -289,7 +289,7 @@ struct ExprOpNot : Expr
         Expr * e1, * e2; \
         name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
         name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
-        void show(std::ostream & str) \
+        void show(std::ostream & str) const \
         { \
             str << "(" << *e1 << " " s " " << *e2 << ")";   \
         } \

From d98755b0c291fed59c71957ac8dbe4ee94e58b10 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 14 Mar 2018 21:39:01 -0500
Subject: [PATCH 0899/2196] nix-daemon: preserve errno in signal handler
 (thanks tsan)

---
 src/nix-daemon/nix-daemon.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 890bffa19aa..30e2b54552e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -816,8 +816,11 @@ static void processConnection(bool trusted)
 
 static void sigChldHandler(int sigNo)
 {
+    // Ensure we don't modify errno of whatever we've interrupted
+    auto saved_errno = errno;
     /* Reap all dead children. */
     while (waitpid(-1, 0, WNOHANG) > 0) ;
+    errno = saved_errno;
 }
 
 

From e2088febf3d0cb1138491c891b134887636e4220 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 14 Mar 2018 23:44:02 -0400
Subject: [PATCH 0900/2196] concatLists: Don't pass NULL pointers to memcpy.

This is UB, even if the size is 0. See #1976.

Fixes #1976.
---
 src/libexpr/eval.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f94c23ea72b..37b977736e2 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1316,7 +1316,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co
     auto out = v.listElems();
     for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
         unsigned int l = lists[n]->listSize();
-        memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
+        if (l)
+            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
     }
 }

From cfdbfa6b2cc27ae5b98c5f27599bbc0fc6a104c1 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 14 Mar 2018 23:53:43 -0400
Subject: [PATCH 0901/2196] Catch more possible instances of passing NULL to
 memcpy.

Actually fixes #1976.
---
 src/libexpr/primops.cc   | 12 ++++++++----
 src/libstore/download.cc |  1 +
 src/libutil/hash.cc      |  1 +
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6778023f506..c88f677da08 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1601,12 +1601,16 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
     state.mkAttrs(v, 2);
 
     Value * vRight = state.allocAttr(v, state.sRight);
-    state.mkList(*vRight, right.size());
-    memcpy(vRight->listElems(), right.data(), sizeof(Value *) * right.size());
+    auto rsize = right.size();
+    state.mkList(*vRight, rsize);
+    if (rsize)
+        memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize);
 
     Value * vWrong = state.allocAttr(v, state.sWrong);
-    state.mkList(*vWrong, wrong.size());
-    memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wrong.size());
+    auto wsize = wrong.size();
+    state.mkList(*vWrong, wsize);
+    if (wsize)
+        memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize);
 
     v.attrs->sort();
 }
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 5ab625f4228..9d9f1153f4d 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -195,6 +195,7 @@ struct CurlDownloader : public Downloader
             if (readOffset == request.data->length())
                 return 0;
             auto count = std::min(size * nitems, request.data->length() - readOffset);
+            assert(count);
             memcpy(buffer, request.data->data() + readOffset, count);
             readOffset += count;
             return count;
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 75e4767550f..150995f55f9 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -191,6 +191,7 @@ Hash::Hash(const std::string & s, HashType type)
         auto d = base64Decode(std::string(s, pos));
         if (d.size() != hashSize)
             throw BadHash("invalid base-64 hash '%s'", s);
+        assert(hashSize);
         memcpy(hash, d.data(), hashSize);
     }
 

From 486872150638d56483c2bc429ba9e137d9974ee8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 15 Mar 2018 16:08:07 +0100
Subject: [PATCH 0902/2196] Filter ANSI colors when not writing to a terminal

Fixes https://github.com/NixOS/nixpkgs/issues/37114.
---
 src/libutil/logging.cc  |  2 +-
 src/libutil/util.cc     |  4 ++--
 src/libutil/util.hh     | 10 ++++++----
 src/nix/progress-bar.cc |  2 +-
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index e860b27d3d0..799c6e1ae44 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -53,7 +53,7 @@ class SimpleLogger : public Logger
             prefix = std::string("<") + c + ">";
         }
 
-        writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
+        writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 2391e14a94b..a60ba8508e3 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1185,7 +1185,7 @@ void ignoreException()
 }
 
 
-std::string filterANSIEscapes(const std::string & s, unsigned int width)
+std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
 {
     std::string t, e;
     size_t w = 0;
@@ -1210,7 +1210,7 @@ std::string filterANSIEscapes(const std::string & s, unsigned int width)
                 if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
             }
 
-            if (last == 'm')
+            if (!filterAll && last == 'm')
                 t += e;
         }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index c5c537ee63d..500ab7811b9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -391,11 +391,13 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
-/* Truncate a string to 'width' printable characters. Certain ANSI
-   escape sequences (such as colour setting) are copied but not
-   included in the character count. Other ANSI escape sequences are
-   filtered. Also, tabs are expanded to spaces. */
+/* Truncate a string to 'width' printable characters. If 'filterAll'
+   is true, all ANSI escape sequences are filtered out. Otherwise,
+   some escape sequences (such as colour setting) are copied but not
+   included in the character count. Also, tabs are expanded to
+   spaces. */
 std::string filterANSIEscapes(const std::string & s,
+    bool filterAll = false,
     unsigned int width = std::numeric_limits<unsigned int>::max());
 
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index e6553c06f4a..40b905ba324 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -308,7 +308,7 @@ class ProgressBar : public Logger
         auto width = getWindowSize().second;
         if (width <= 0) std::numeric_limits<decltype(width)>::max();
 
-        writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
+        writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
     }
 
     std::string getStatus(State & state)

From eb75bc5afbaf2891a2db71e7932105223da94ac9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 16 Mar 2018 12:03:18 +0100
Subject: [PATCH 0903/2196] Don't retry on CURLE_SSL_CACERT_BADFILE

The certificates won't get any better if we retry.
---
 src/libstore/download.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 9d9f1153f4d..6eb87096692 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -340,6 +340,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_BAD_FUNCTION_ARGUMENT:
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
+                        case CURLE_SSL_CACERT_BADFILE:
                             err = Misc;
                             break;
                         default: // Shut up warnings

From 3e6b194d78024373c2320f31f4ba0de3d0658b83 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 16 Mar 2018 16:59:31 +0100
Subject: [PATCH 0904/2196] decompress(): Use a Source and Sink

This allows decompression to happen in O(1) memory.
---
 src/libstore/binary-cache-store.cc |  20 ++--
 src/libutil/compression.cc         | 166 +++++++++++++++++++----------
 src/libutil/compression.hh         |   6 +-
 src/libutil/serialise.hh           |  18 +++-
 src/libutil/util.cc                |  75 ++++++++++---
 src/libutil/util.hh                |   8 ++
 6 files changed, 206 insertions(+), 87 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index d1b278b8efb..2e9a13e564c 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -203,22 +203,18 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
     stats.narRead++;
     stats.narReadCompressedBytes += nar->size();
 
-    /* Decompress the NAR. FIXME: would be nice to have the remote
-       side do this. */
-    try {
-        nar = decompress(info->compression, *nar);
-    } catch (UnknownCompressionMethod &) {
-        throw Error(format("binary cache path '%s' uses unknown compression method '%s'")
-            % storePath % info->compression);
-    }
+    uint64_t narSize = 0;
 
-    stats.narReadBytes += nar->size();
+    StringSource source(*nar);
 
-    printMsg(lvlTalkative, format("exporting path '%1%' (%2% bytes)") % storePath % nar->size());
+    LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
+        sink(data, len);
+        narSize += len;
+    });
 
-    assert(nar->size() % 8 == 0);
+    decompress(info->compression, source, wrapperSink);
 
-    sink((unsigned char *) nar->c_str(), nar->size());
+    stats.narReadBytes += narSize;
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 470c925ed7a..e7fedcbdc52 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -17,7 +17,23 @@
 
 namespace nix {
 
-static ref<std::string> decompressXZ(const std::string & in)
+static const size_t bufSize = 32 * 1024;
+
+static void decompressNone(Source & source, Sink & sink)
+{
+    std::vector<unsigned char> buf(bufSize);
+    while (true) {
+        size_t n;
+        try {
+            n = source.read(buf.data(), buf.size());
+        } catch (EndOfFile &) {
+            break;
+        }
+        sink(buf.data(), n);
+    }
+}
+
+static void decompressXZ(Source & source, Sink & sink)
 {
     lzma_stream strm(LZMA_STREAM_INIT);
 
@@ -29,36 +45,44 @@ static ref<std::string> decompressXZ(const std::string & in)
     Finally free([&]() { lzma_end(&strm); });
 
     lzma_action action = LZMA_RUN;
-    uint8_t outbuf[BUFSIZ];
-    ref<std::string> res = make_ref<std::string>();
-    strm.next_in = (uint8_t *) in.c_str();
-    strm.avail_in = in.size();
-    strm.next_out = outbuf;
-    strm.avail_out = sizeof(outbuf);
+    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
+    strm.next_in = nullptr;
+    strm.avail_in = 0;
+    strm.next_out = outbuf.data();
+    strm.avail_out = outbuf.size();
+    bool eof = false;
 
     while (true) {
         checkInterrupt();
 
+        if (strm.avail_in == 0 && !eof) {
+            strm.next_in = inbuf.data();
+            try {
+                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
+            } catch (EndOfFile &) {
+                eof = true;
+            }
+        }
+
         if (strm.avail_in == 0)
             action = LZMA_FINISH;
 
         lzma_ret ret = lzma_code(&strm, action);
 
-        if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
-            res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out);
-            strm.next_out = outbuf;
-            strm.avail_out = sizeof(outbuf);
+        if (strm.avail_out < outbuf.size()) {
+            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
+            strm.next_out = outbuf.data();
+            strm.avail_out = outbuf.size();
         }
 
-        if (ret == LZMA_STREAM_END)
-            return res;
+        if (ret == LZMA_STREAM_END) return;
 
         if (ret != LZMA_OK)
             throw CompressionError("error %d while decompressing xz file", ret);
     }
 }
 
-static ref<std::string> decompressBzip2(const std::string & in)
+static void decompressBzip2(Source & source, Sink & sink)
 {
     bz_stream strm;
     memset(&strm, 0, sizeof(strm));
@@ -69,39 +93,50 @@ static ref<std::string> decompressBzip2(const std::string & in)
 
     Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
 
-    char outbuf[BUFSIZ];
-    ref<std::string> res = make_ref<std::string>();
-    strm.next_in = (char *) in.c_str();
-    strm.avail_in = in.size();
-    strm.next_out = outbuf;
-    strm.avail_out = sizeof(outbuf);
+    std::vector<char> inbuf(bufSize), outbuf(bufSize);
+    strm.next_in = nullptr;
+    strm.avail_in = 0;
+    strm.next_out = outbuf.data();
+    strm.avail_out = outbuf.size();
+    bool eof = false;
 
     while (true) {
         checkInterrupt();
 
+        if (strm.avail_in == 0 && !eof) {
+            strm.next_in = inbuf.data();
+            try {
+                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
+            } catch (EndOfFile &) {
+                eof = true;
+            }
+        }
+
         int ret = BZ2_bzDecompress(&strm);
 
-        if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
-            res->append(outbuf, sizeof(outbuf) - strm.avail_out);
-            strm.next_out = outbuf;
-            strm.avail_out = sizeof(outbuf);
+        if (strm.avail_in == 0 && strm.avail_out == outbuf.size() && eof)
+            throw CompressionError("bzip2 data ends prematurely");
+
+        if (strm.avail_out < outbuf.size()) {
+            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
+            strm.next_out = outbuf.data();
+            strm.avail_out = outbuf.size();
         }
 
-        if (ret == BZ_STREAM_END)
-            return res;
+        if (ret == BZ_STREAM_END) return;
 
         if (ret != BZ_OK)
             throw CompressionError("error while decompressing bzip2 file");
-
-        if (strm.avail_in == 0)
-            throw CompressionError("bzip2 data ends prematurely");
     }
 }
 
-static ref<std::string> decompressBrotli(const std::string & in)
+static void decompressBrotli(Source & source, Sink & sink)
 {
 #if !HAVE_BROTLI
-    return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in}));
+    RunOptions options(BROTLI, {"-d"});
+    options.stdin = &source;
+    options.stdout = &sink;
+    runProgram2(options);
 #else
     auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
     if (!s)
@@ -109,16 +144,26 @@ static ref<std::string> decompressBrotli(const std::string & in)
 
     Finally free([s]() { BrotliDecoderDestroyInstance(s); });
 
-    uint8_t outbuf[BUFSIZ];
-    ref<std::string> res = make_ref<std::string>();
-    const uint8_t *next_in = (uint8_t *)in.c_str();
-    size_t avail_in = in.size();
-    uint8_t *next_out = outbuf;
-    size_t avail_out = sizeof(outbuf);
+    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
+    const uint8_t * next_in = nullptr;
+    size_t avail_in = 0;
+    bool eof = false;
 
     while (true) {
         checkInterrupt();
 
+        if (avail_in == 0 && !eof) {
+            next_in = inbuf.data();
+            try {
+                avail_in = source.read((unsigned char *) next_in, inbuf.size());
+            } catch (EndOfFile &) {
+                eof = true;
+            }
+        }
+
+        uint8_t * next_out = outbuf.data();
+        size_t avail_out = outbuf.size();
+
         auto ret = BrotliDecoderDecompressStream(s,
                 &avail_in, &next_in,
                 &avail_out, &next_out,
@@ -128,51 +173,49 @@ static ref<std::string> decompressBrotli(const std::string & in)
         case BROTLI_DECODER_RESULT_ERROR:
             throw CompressionError("error while decompressing brotli file");
         case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
-            throw CompressionError("incomplete or corrupt brotli file");
+            if (eof)
+                throw CompressionError("incomplete or corrupt brotli file");
+            break;
         case BROTLI_DECODER_RESULT_SUCCESS:
             if (avail_in != 0)
                 throw CompressionError("unexpected input after brotli decompression");
             break;
         case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
             // I'm not sure if this can happen, but abort if this happens with empty buffer
-            if (avail_out == sizeof(outbuf))
+            if (avail_out == outbuf.size())
                 throw CompressionError("brotli decompression requires larger buffer");
             break;
         }
 
         // Always ensure we have full buffer for next invocation
-        if (avail_out < sizeof(outbuf)) {
-            res->append((char*)outbuf, sizeof(outbuf) - avail_out);
-            next_out = outbuf;
-            avail_out = sizeof(outbuf);
-        }
+        if (avail_out < outbuf.size())
+            sink((unsigned char *) outbuf.data(), outbuf.size() - avail_out);
 
-        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res;
+        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return;
     }
 #endif // HAVE_BROTLI
 }
 
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
+ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    StringSink ssink;
-    auto sink = makeCompressionSink(method, ssink, parallel);
-    (*sink)(in);
-    sink->finish();
-    return ssink.s;
+    StringSource source(in);
+    StringSink sink;
+    decompress(method, source, sink);
+    return sink.s;
 }
 
-ref<std::string> decompress(const std::string & method, const std::string & in)
+void decompress(const std::string & method, Source & source, Sink & sink)
 {
     if (method == "none")
-        return make_ref<std::string>(in);
+        return decompressNone(source, sink);
     else if (method == "xz")
-        return decompressXZ(in);
+        return decompressXZ(source, sink);
     else if (method == "bzip2")
-        return decompressBzip2(in);
+        return decompressBzip2(source, sink);
     else if (method == "br")
-        return decompressBrotli(in);
+        return decompressBrotli(source, sink);
     else
-        throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
+        throw UnknownCompressionMethod("unknown compression method '%s'", method);
 }
 
 struct NoneSink : CompressionSink
@@ -499,4 +542,13 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
 
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
+{
+    StringSink ssink;
+    auto sink = makeCompressionSink(method, ssink, parallel);
+    (*sink)(in);
+    sink->finish();
+    return ssink.s;
+}
+
 }
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index a0d7530d74f..f7a3e3fbd32 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,10 +8,12 @@
 
 namespace nix {
 
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
-
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
+void decompress(const std::string & method, Source & source, Sink & sink);
+
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
+
 struct CompressionSink : BufferedSink
 {
     virtual void finish() = 0;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 2ea5b6354ee..103b0576736 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -56,7 +56,7 @@ struct Source
     void operator () (unsigned char * data, size_t len);
 
     /* Store up to ‘len’ in the buffer pointed to by ‘data’, and
-       return the number of bytes stored.  If blocks until at least
+       return the number of bytes stored.  It blocks until at least
        one byte is available. */
     virtual size_t read(unsigned char * data, size_t len) = 0;
 
@@ -175,6 +175,22 @@ struct TeeSource : Source
 };
 
 
+/* Convert a function into a sink. */
+struct LambdaSink : Sink
+{
+    typedef std::function<void(const unsigned char *, size_t)> lambda_t;
+
+    lambda_t lambda;
+
+    LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
+
+    virtual void operator () (const unsigned char * data, size_t len)
+    {
+        lambda(data, len);
+    }
+};
+
+
 void writePadding(size_t len, Sink & sink);
 void writeString(const unsigned char * buf, size_t len, Sink & sink);
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a60ba8508e3..37a35ab233b 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -3,6 +3,7 @@
 #include "affinity.hh"
 #include "sync.hh"
 #include "finally.hh"
+#include "serialise.hh"
 
 #include <cctype>
 #include <cerrno>
@@ -568,19 +569,25 @@ void writeFull(int fd, const string & s, bool allowInterrupts)
 
 string drainFD(int fd)
 {
-    string result;
-    unsigned char buffer[4096];
+    StringSink sink;
+    drainFD(fd, sink);
+    return std::move(*sink.s);
+}
+
+
+void drainFD(int fd, Sink & sink)
+{
+    std::vector<unsigned char> buf(4096);
     while (1) {
         checkInterrupt();
-        ssize_t rd = read(fd, buffer, sizeof buffer);
+        ssize_t rd = read(fd, buf.data(), buf.size());
         if (rd == -1) {
             if (errno != EINTR)
                 throw SysError("reading from file");
         }
         else if (rd == 0) break;
-        else result.append((char *) buffer, rd);
+        else sink(buf.data(), rd);
     }
-    return result;
 }
 
 
@@ -920,20 +927,47 @@ string runProgram(Path program, bool searchPath, const Strings & args,
     return res.second;
 }
 
-std::pair<int, std::string> runProgram(const RunOptions & options)
+std::pair<int, std::string> runProgram(const RunOptions & options_)
+{
+    RunOptions options(options_);
+    StringSink sink;
+    options.stdout = &sink;
+
+    int status = 0;
+
+    try {
+        runProgram2(options);
+    } catch (ExecError & e) {
+        status = e.status;
+    }
+
+    return {status, std::move(*sink.s)};
+}
+
+void runProgram2(const RunOptions & options)
 {
     checkInterrupt();
 
+    assert(!(options.stdin && options.input));
+
+    std::unique_ptr<Source> source_;
+    Source * source = options.stdin;
+
+    if (options.input) {
+        source_ = std::make_unique<StringSource>(*options.input);
+        source = source_.get();
+    }
+
     /* Create a pipe. */
     Pipe out, in;
-    out.create();
-    if (options.input) in.create();
+    if (options.stdout) out.create();
+    if (source) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
-        if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+        if (options.stdout && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
-        if (options.input && dup2(in.readSide.get(), STDIN_FILENO) == -1)
+        if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
         Strings args_(options.args);
@@ -961,11 +995,20 @@ std::pair<int, std::string> runProgram(const RunOptions & options)
     });
 
 
-    if (options.input) {
+    if (source) {
         in.readSide = -1;
         writerThread = std::thread([&]() {
             try {
-                writeFull(in.writeSide.get(), *options.input);
+                std::vector<unsigned char> buf(8 * 1024);
+                while (true) {
+                    size_t n;
+                    try {
+                        n = source->read(buf.data(), buf.size());
+                    } catch (EndOfFile &) {
+                        break;
+                    }
+                    writeFull(in.writeSide.get(), buf.data(), n);
+                }
                 promise.set_value();
             } catch (...) {
                 promise.set_exception(std::current_exception());
@@ -974,15 +1017,17 @@ std::pair<int, std::string> runProgram(const RunOptions & options)
         });
     }
 
-    string result = drainFD(out.readSide.get());
+    if (options.stdout)
+        drainFD(out.readSide.get(), *options.stdout);
 
     /* Wait for the child to finish. */
     int status = pid.wait();
 
     /* Wait for the writer thread to finish. */
-    if (options.input) promise.get_future().get();
+    if (source) promise.get_future().get();
 
-    return {status, result};
+    if (status)
+        throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status)));
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 500ab7811b9..1ea1027ac0a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -25,6 +25,9 @@
 
 namespace nix {
 
+struct Sink;
+struct Source;
+
 
 /* Return an environment variable. */
 string getEnv(const string & key, const string & def = "");
@@ -150,6 +153,7 @@ MakeError(EndOfFile, Error)
 /* Read a file descriptor until EOF occurs. */
 string drainFD(int fd);
 
+void drainFD(int fd, Sink & sink);
 
 
 /* Automatic cleanup of resources. */
@@ -256,6 +260,8 @@ struct RunOptions
     bool searchPath = true;
     Strings args;
     std::experimental::optional<std::string> input;
+    Source * stdin = nullptr;
+    Sink * stdout = nullptr;
     bool _killStderr = false;
 
     RunOptions(const Path & program, const Strings & args)
@@ -266,6 +272,8 @@ struct RunOptions
 
 std::pair<int, std::string> runProgram(const RunOptions & options);
 
+void runProgram2(const RunOptions & options);
+
 
 class ExecError : public Error
 {

From 48662d151bdf4a38670897beacea9d1bd750376a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 16 Mar 2018 20:22:34 +0100
Subject: [PATCH 0905/2196] Reduce substitution memory consumption

copyStorePath() now pipes the output of srcStore->narFromPath()
directly into dstStore->addToStore(). The sink used by the former is
converted into a source usable by the latter using
boost::coroutine2. This is based on [1].

This reduces the maximum resident size of

  $ nix build --store ~/my-nix/ /nix/store/b0zlxla7dmy1iwc3g459rjznx59797xy-binutils-2.28.1 --substituters file:///tmp/binary-cache-xz/ --no-require-sigs

from 418592 KiB to 53416 KiB. (The previous commit also reduced the
runtime from ~4.2s to ~3.4s, not sure why.) A further improvement will
be to download files into a Sink.

[1] https://github.com/NixOS/nix/compare/master...Mathnerd314:dump-fix-coroutine#diff-dcbcac55a634031f9cc73707da6e4b18

Issue #1969.
---
 src/libstore/local-store.cc  | 34 ++++++++++++-------
 src/libstore/local-store.hh  |  2 +-
 src/libstore/store-api.cc    | 49 ++++++++++++++++------------
 src/libstore/store-api.hh    |  7 +++-
 src/libutil/local.mk         |  2 +-
 src/libutil/serialise.cc     | 63 ++++++++++++++++++++++++++++++++++++
 src/libutil/serialise.hh     | 23 +++++++++++++
 src/nix-daemon/nix-daemon.cc |  2 +-
 8 files changed, 145 insertions(+), 37 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3441b2472fc..acc0002acee 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -964,20 +964,11 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 }
 
 
-void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     assert(info.narHash);
 
-    Hash h = hashString(htSHA256, *nar);
-    if (h != info.narHash)
-        throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
-            info.path, info.narHash.to_string(), h.to_string());
-
-    if (nar->size() != info.narSize)
-        throw Error("size mismatch importing path '%s'; expected %s, got %s",
-            info.path, info.narSize, nar->size());
-
     if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys))
         throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
 
@@ -999,8 +990,27 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
 
             deletePath(realPath);
 
-            StringSource source(*nar);
-            restorePath(realPath, source);
+            /* While restoring the path from the NAR, compute the hash
+               of the NAR. */
+            HashSink hashSink(htSHA256);
+
+            LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
+                size_t n = source.read(data, len);
+                hashSink(data, n);
+                return n;
+            });
+
+            restorePath(realPath, wrapperSource);
+
+            auto hashResult = hashSink.finish();
+
+            if (hashResult.first != info.narHash)
+                throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
+                    info.path, info.narHash.to_string(), hashResult.first.to_string());
+
+            if (hashResult.second != info.narSize)
+                throw Error("size mismatch importing path '%s'; expected %s, got %s",
+                    info.path, info.narSize, hashResult.second);
 
             autoGC();
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index bbd50e1c145..0d6c176595c 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -143,7 +143,7 @@ public:
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) override;
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8830edcc344..64f9b8d68b0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -590,32 +590,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
-    auto progress = [&](size_t len) {
-        total += len;
-        act.progress(total, info->narSize);
-    };
-
-    struct MyStringSink : StringSink
-    {
-        typedef std::function<void(size_t)> Callback;
-        Callback callback;
-        MyStringSink(Callback callback) : callback(callback) { }
-        void operator () (const unsigned char * data, size_t len) override
-        {
-            StringSink::operator ()(data, len);
-            callback(len);
-        };
-    };
-
-    MyStringSink sink(progress);
-    srcStore->narFromPath({storePath}, sink);
-
+    // FIXME
+#if 0
     if (!info->narHash) {
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
         info = info2;
     }
+#endif
 
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);
@@ -623,7 +606,16 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
         info = info2;
     }
 
-    dstStore->addToStore(*info, sink.s, repair, checkSigs);
+    auto source = sinkToSource([&](Sink & sink) {
+        LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
+            sink(data, len);
+            total += len;
+            act.progress(total, info->narSize);
+        });
+        srcStore->narFromPath({storePath}, wrapperSink);
+    });
+
+    dstStore->addToStore(*info, *source, repair, checkSigs);
 }
 
 
@@ -808,6 +800,21 @@ std::string makeFixedOutputCA(bool recursive, const Hash & hash)
 }
 
 
+void Store::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs,
+    std::shared_ptr<FSAccessor> accessor)
+{
+    addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
+}
+
+void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    RepairFlag repair, CheckSigsFlag checkSigs,
+    std::shared_ptr<FSAccessor> accessor)
+{
+    StringSource source(*nar);
+    addToStore(info, source, repair, checkSigs, accessor);
+}
+
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 563aa566bd3..ea259f07e8a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -399,9 +399,14 @@ public:
     virtual bool wantMassQuery() { return false; }
 
     /* Import a path into the store. */
+    virtual void addToStore(const ValidPathInfo & info, Source & narSource,
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
+        std::shared_ptr<FSAccessor> accessor = 0);
+
+    // FIXME: remove
     virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0) = 0;
+        std::shared_ptr<FSAccessor> accessor = 0);
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 5fc2aab569d..824f48fbfc9 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,7 +6,7 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS)
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
 
 libutil_LIBS = libformat
 
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 9e2a502afaf..6b793922511 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -5,6 +5,8 @@
 #include <cerrno>
 #include <memory>
 
+#include <boost/coroutine2/coroutine.hpp>
+
 
 namespace nix {
 
@@ -88,6 +90,23 @@ void Source::operator () (unsigned char * data, size_t len)
 }
 
 
+std::string Source::drain()
+{
+    std::string s;
+    std::vector<unsigned char> buf(8192);
+    while (true) {
+        size_t n;
+        try {
+            n = read(buf.data(), buf.size());
+            s.append((char *) buf.data(), n);
+        } catch (EndOfFile &) {
+            break;
+        }
+    }
+    return s;
+}
+
+
 size_t BufferedSource::read(unsigned char * data, size_t len)
 {
     if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
@@ -138,6 +157,50 @@ size_t StringSource::read(unsigned char * data, size_t len)
 }
 
 
+std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
+{
+    struct SinkToSource : Source
+    {
+        typedef boost::coroutines2::coroutine<std::string> coro_t;
+
+        coro_t::pull_type coro;
+
+        SinkToSource(std::function<void(Sink &)> fun)
+            : coro([&](coro_t::push_type & yield) {
+                LambdaSink sink([&](const unsigned char * data, size_t len) {
+                    if (len) yield(std::string((const char *) data, len));
+                });
+                fun(sink);
+            })
+        {
+        }
+
+        std::string cur;
+        size_t pos = 0;
+
+        size_t read(unsigned char * data, size_t len) override
+        {
+            if (!coro)
+                throw EndOfFile("coroutine has finished");
+
+            if (pos == cur.size()) {
+                if (!cur.empty()) coro();
+                cur = std::move(coro.get());
+                pos = 0;
+            }
+
+            auto n = std::min(cur.size() - pos, len);
+            memcpy(data, (unsigned char *) cur.data() + pos, n);
+            pos += n;
+
+            return n;
+        }
+    };
+
+    return std::make_unique<SinkToSource>(fun);
+}
+
+
 void writePadding(size_t len, Sink & sink)
 {
     if (len % 8) {
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 103b0576736..d0b4552e339 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -61,6 +61,8 @@ struct Source
     virtual size_t read(unsigned char * data, size_t len) = 0;
 
     virtual bool good() { return true; }
+
+    std::string drain();
 };
 
 
@@ -191,6 +193,27 @@ struct LambdaSink : Sink
 };
 
 
+/* Convert a function into a source. */
+struct LambdaSource : Source
+{
+    typedef std::function<size_t(unsigned char *, size_t)> lambda_t;
+
+    lambda_t lambda;
+
+    LambdaSource(const lambda_t & lambda) : lambda(lambda) { }
+
+    size_t read(unsigned char * data, size_t len) override
+    {
+        return lambda(data, len);
+    }
+};
+
+
+/* Convert a function that feeds data into a Sink into a Source. The
+   Source executes the function as a coroutine. */
+std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun);
+
+
 void writePadding(size_t len, Sink & sink);
 void writeString(const unsigned char * buf, size_t len, Sink & sink);
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 30e2b54552e..50258018aef 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -695,7 +695,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
         parseDump(tee, tee.source);
 
         logger->startWork();
-        store->addToStore(info, tee.source.data, (RepairFlag) repair,
+        store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
         logger->stopWork();
         break;

From 12913ccf454c1bc82a51c59872f8185b40c533c5 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Fri, 16 Mar 2018 22:15:24 +0000
Subject: [PATCH 0906/2196] nix-shell: allow symlinks to .drvs

This makes persistent shell environments easier to use.
---
 src/nix-build/nix-build.cc             |  8 ++++++--
 src/nix-instantiate/nix-instantiate.cc |  2 +-
 tests/nix-shell.sh                     | 12 ++++++++++++
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 99f773451ff..cf628519c6e 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -271,10 +271,14 @@ void mainWrapped(int argc, char * * argv)
         exprs = {state.parseStdin()};
     else
         for (auto i : left) {
+            auto absolute = i;
+            try {
+                absolute = canonPath(absPath(i), true);
+            } catch (Error e) {};
             if (fromArgs)
                 exprs.push_back(state.parseExprFromString(i, absPath(".")));
-            else if (store->isStorePath(i) && std::regex_match(i, std::regex(".*\\.drv(!.*)?")))
-                drvs.push_back(DrvInfo(state, store, i));
+            else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+                drvs.push_back(DrvInfo(state, store, absolute));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index dd262bea091..5049460c754 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -70,7 +70,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 if (gcRoot == "")
                     printGCWarning();
                 else {
-                    Path rootName = gcRoot;
+                    Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot;
                     if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
                     if (store2)
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index f7cac6ecf8e..063e97ce2c7 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -17,6 +17,18 @@ output=$(nix-shell --pure shell.nix -A shellDrv --run \
 [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
+# Test nix-shell on a .drv symlink
+
+# Legacy: absolute path and .drv extension required
+nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv
+[[ $(nix-shell --pure $PWD/shell.drv --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+
+# New behaviour: just needs to resolve to a derivation in the store
+nix-instantiate shell.nix -A shellDrv --indirect --add-root shell
+[[ $(nix-shell --pure shell --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+
 # Test nix-shell -p
 output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"')
 [ "$output" = "foo bar" ]

From 637701b6047bff23a7dea4e02bebf94660b8cb4c Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Fri, 16 Mar 2018 22:50:27 +0000
Subject: [PATCH 0907/2196] rename build-extra-platforms -> extra-platforms

also document it
---
 doc/manual/command-ref/conf-file.xml | 19 +++++++++++++++++++
 src/libstore/globals.hh              |  8 ++++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c76640c97e7..5c21a40a11e 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -254,6 +254,25 @@ false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>extra-platforms</literal></term>
+
+    <listitem><para>Platforms other than the native one which
+    this machine is capable of building for. This can be useful for
+    supporting additional architectures on compatible machines:
+    i686-linux can be built on x86_64-linux machines (and the default
+    for this setting reflects this); armv7 is backwards-compatible with
+    armv6 and armv5tel; some aarch64 machines can also natively run
+    32-bit ARM code; and qemu-user may be used to support non-native
+    platforms (though this may be slow and buggy). Most values for this
+    are not enabled by default because build systems will often
+    misdetect the target platform and generate incompatible code, so you
+    may wish to cross-check the results of using this option against
+    proper natively-built versions of your
+    derivations.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>extra-substituters</literal></term>
 
     <listitem><para>Additional binary caches appended to those
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b382da01e97..897d53b7b94 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -295,8 +295,12 @@ public:
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};
 
-    Setting<StringSet> extraPlatforms{this, StringSet{}, "build-extra-platforms",
-        "Additional platforms that can be built on the local system, e.g. using qemu-user."};
+    Setting<StringSet> extraPlatforms{this,
+        SYSTEM == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        "extra-platforms",
+        "Additional platforms that can be built on the local system. "
+        "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
+        "or using hacks like qemu-user."};
 
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),

From 639c166647ae733e927ae589864d996bb2d95b88 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Fri, 16 Mar 2018 22:50:57 +0000
Subject: [PATCH 0908/2196] build-remote: take extra-platforms into account

---
 src/build-remote/build-remote.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index dbf8fe1b8f8..5ec49b5252d 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -98,7 +98,9 @@ int main (int argc, char * * argv)
             source >> drvPath;
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem);
+            auto canBuildLocally = amWilling
+                &&  (  neededSystem == settings.thisSystem
+                    || settings.extraPlatforms.get().count(neededSystem) > 0);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);

From fef8ebf51d8f190932a134186f775d57d1ccf054 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Sat, 17 Mar 2018 21:17:28 +0000
Subject: [PATCH 0909/2196] doc: add xml:id to all the config options

---
 doc/manual/command-ref/conf-file.xml | 36 ++++++++++++++--------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c76640c97e7..ab83f671a74 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -123,7 +123,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>auto-optimise-store</literal></term>
+  <varlistentry xmls:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix
     automatically detects files in the store that have identical
@@ -146,7 +146,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>builders-use-substitutes</literal></term>
+  <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will instruct
     remote build machines to use their own binary substitutes if available. In
@@ -201,7 +201,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>compress-build-log</literal></term>
+  <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     build logs written to <filename>/nix/var/log/nix/drvs</filename>
@@ -254,7 +254,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>extra-substituters</literal></term>
+  <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term>
 
     <listitem><para>Additional binary caches appended to those
     specified in <option>substituters</option>.  When used by
@@ -265,7 +265,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>fallback</literal></term>
+  <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
     back to building from source if a binary substitute fails.  This
@@ -275,7 +275,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><literal>fsync-metadata</literal></term>
+  <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, changes to the
     Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
@@ -314,7 +314,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>http-connections</literal></term>
+  <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
 
     <listitem><para>The maximum number of parallel TCP connections
     used to fetch files from binary caches and by other downloads. It
@@ -323,7 +323,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>keep-build-log</literal></term>
+  <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     Nix will write the build log of a derivation (i.e. the standard
@@ -352,7 +352,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>keep-env-derivations</literal></term>
+  <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
     are not stored in Nix user environments.  That is, the derivation
@@ -441,7 +441,7 @@ builtins.fetchurl {
   </varlistentry>
 
 
-  <varlistentry><term><literal>netrc-file</literal></term>
+  <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
 
     <listitem><para>If set to an absolute path to a <filename>netrc</filename>
     file, Nix will use the HTTP authentication credentials in this file when
@@ -544,7 +544,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>require-sigs</literal></term>
+  <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
     any non-content-addressed path added or copied to the Nix store
@@ -573,7 +573,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>sandbox</literal></term>
+  <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, builds will be
     performed in a <emphasis>sandboxed environment</emphasis>, i.e.,
@@ -644,7 +644,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>secret-key-files</literal></term>
+  <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term>
 
     <listitem><para>A whitespace-separated list of files containing
     secret (private) keys. These are used to sign locally-built
@@ -665,7 +665,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>substitute</literal></term>
+  <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
     will use binary substitutes if available.  This option can be
@@ -674,7 +674,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>substituters</literal></term>
+  <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
     whitespace.  The default is
@@ -683,7 +683,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>system</literal></term>
+  <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system
     name of the current installation, such as
@@ -724,7 +724,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>trusted-public-keys</literal></term>
+  <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When
     paths are copied from another Nix store (such as a binary cache),
@@ -735,7 +735,7 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
-  <varlistentry><term><literal>trusted-substituters</literal></term>
+  <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
     whitespace.  These are not used by default, but can be enabled by

From 43bef1b82c4a06a916b99643adaf398b04e18870 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Sat, 17 Mar 2018 21:25:50 +0000
Subject: [PATCH 0910/2196] doc: document the min-free and max-free options

---
 doc/manual/command-ref/conf-file.xml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index ab83f671a74..4a94b935558 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -406,6 +406,12 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
+
+    <listitem><para>This option defines after how many free bytes to stop collecting
+    garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term>
 
@@ -440,6 +446,17 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
+
+    <listitem>
+      <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix
+        will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk.
+        A value of <literal>0</literal> (the default) means that this feature is disabled.</para>
+      </para>
+    </listitem>
+
+  </varlistentry>
+
 
   <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
 

From b2074f0892cefff24646c9c346d7454fe92a0c27 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 18 Mar 2018 13:27:30 +0100
Subject: [PATCH 0911/2196] Fix bad XML

---
 doc/manual/command-ref/conf-file.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 4a94b935558..6638bf61e45 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -123,7 +123,7 @@ false</literal>.</para>
   </varlistentry>
 
 
-  <varlistentry xmls:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
+  <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix
     automatically detects files in the store that have identical
@@ -452,7 +452,6 @@ builtins.fetchurl {
       <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix
         will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk.
         A value of <literal>0</literal> (the default) means that this feature is disabled.</para>
-      </para>
     </listitem>
 
   </varlistentry>

From ec6a328fe8d4070669a6e5116cbc36e1909c4612 Mon Sep 17 00:00:00 2001
From: Ben Challenor <ben@challenor.org>
Date: Fri, 16 Mar 2018 03:17:49 +0000
Subject: [PATCH 0912/2196] Dockerfile: 1.11.14 -> 2.0

---
 misc/docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index d6b88c7e91a..0646d30e5c4 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,7 +4,7 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
+RUN wget -O- https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
   && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \

From a74288b9439153d17df8486230185c7e6b2b588b Mon Sep 17 00:00:00 2001
From: Ben Challenor <ben@challenor.org>
Date: Mon, 19 Mar 2018 04:56:13 +0000
Subject: [PATCH 0913/2196] Dockerfile: verify tar file with sha256sum

---
 misc/docker/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 0646d30e5c4..2f8e3dd7a67 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,7 +4,9 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget -O- https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 | bzcat - | tar xf - \
+RUN wget https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 \
+  && echo "6312837aee33306cdbb351b75ba1638b89d21b30f0caf0346f9a742425f197ee  nix-2.0-x86_64-linux.tar.bz2" | sha256sum -c \
+  && tar xjf nix-*-x86_64-linux.tar.bz2 \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
   && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \

From 138af2e554871b83e23f13939314d7b749de2013 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Mar 2018 11:57:15 +0100
Subject: [PATCH 0914/2196] Shut up signedness warning

---
 src/libexpr/primops/fetchGit.cc       | 2 +-
 src/libexpr/primops/fetchMercurial.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index fd536c6a5e4..8bb74dad639 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -114,7 +114,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
            git fetch to update the local ref to the remote ref. */
         struct stat st;
         doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            st.st_mtime <= now - settings.tarballTtl;
+            st.st_mtime + settings.tarballTtl <= now;
     }
     if (doFetch)
     {
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 5517d83df82..a75c5fc2ddf 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     time_t now = time(0);
     struct stat st;
     if (stat(stampFile.c_str(), &st) != 0 ||
-        st.st_mtime <= now - settings.tarballTtl)
+        st.st_mtime + settings.tarballTtl <= now)
     {
         /* Except that if this is a commit hash that we already have,
            we don't have to pull again. */

From 44a1b6c0264556d2bc7ac312728421342d92490a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Mar 2018 11:57:34 +0100
Subject: [PATCH 0915/2196] Fix build on non-x86_64-linux

---
 release.nix | 6 ++++--
 shell.nix   | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/release.nix b/release.nix
index 7db5e20e1e1..0dac174accd 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "release-18.03"; }
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -55,7 +55,9 @@ let
 
     build = pkgs.lib.genAttrs systems (system:
 
-      with import nixpkgs { inherit system; };
+      let pkgs = import nixpkgs { inherit system; }; in
+
+      with pkgs;
 
       with import ./release-common.nix { inherit pkgs; };
 
diff --git a/shell.nix b/shell.nix
index 20cf4133b3f..c04bcd15130 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "release-18.03"; }) {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From 9b33201e727b8c1ac0eef08624ebff5b60c9ffd7 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 19 Mar 2018 11:09:52 -0500
Subject: [PATCH 0916/2196] util: rename stdout/stdin members to avoid
 conflicts w/standard macro

(cherry picked from commit c389a7fb617ed7bcd617efa68c6a48c00405310d)
---
 src/libutil/util.cc | 14 +++++++-------
 src/libutil/util.hh |  4 ++--
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 37a35ab233b..764b4e44207 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -931,7 +931,7 @@ std::pair<int, std::string> runProgram(const RunOptions & options_)
 {
     RunOptions options(options_);
     StringSink sink;
-    options.stdout = &sink;
+    options.standardOut = &sink;
 
     int status = 0;
 
@@ -948,10 +948,10 @@ void runProgram2(const RunOptions & options)
 {
     checkInterrupt();
 
-    assert(!(options.stdin && options.input));
+    assert(!(options.standardIn && options.input));
 
     std::unique_ptr<Source> source_;
-    Source * source = options.stdin;
+    Source * source = options.standardIn;
 
     if (options.input) {
         source_ = std::make_unique<StringSource>(*options.input);
@@ -960,12 +960,12 @@ void runProgram2(const RunOptions & options)
 
     /* Create a pipe. */
     Pipe out, in;
-    if (options.stdout) out.create();
+    if (options.standardOut) out.create();
     if (source) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
-        if (options.stdout && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+        if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
@@ -1017,8 +1017,8 @@ void runProgram2(const RunOptions & options)
         });
     }
 
-    if (options.stdout)
-        drainFD(out.readSide.get(), *options.stdout);
+    if (options.standardOut)
+        drainFD(out.readSide.get(), *options.standardOut);
 
     /* Wait for the child to finish. */
     int status = pid.wait();
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 1ea1027ac0a..85952535db2 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -260,8 +260,8 @@ struct RunOptions
     bool searchPath = true;
     Strings args;
     std::experimental::optional<std::string> input;
-    Source * stdin = nullptr;
-    Sink * stdout = nullptr;
+    Source * standardIn = nullptr;
+    Sink * standardOut = nullptr;
     bool _killStderr = false;
 
     RunOptions(const Path & program, const Strings & args)

From f5a2136a404553ce3462be0541123a3532bc0ab0 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 19 Mar 2018 13:56:17 -0500
Subject: [PATCH 0917/2196] serialise.cc: remove pessimising move

from clang6:

src/libutil/serialise.cc:189:23: warning: moving a temporary object prevents copy elision [-Wpessimizing-move]
---
 src/libutil/serialise.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 6b793922511..33ae1ea389d 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -185,7 +185,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
 
             if (pos == cur.size()) {
                 if (!cur.empty()) coro();
-                cur = std::move(coro.get());
+                cur = coro.get();
                 pos = 0;
             }
 

From 78d0c72b52835161fd15058b111aa05d69ce062b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 14:55:47 +0100
Subject: [PATCH 0918/2196] Fix coverage job

---
 release.nix | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/release.nix b/release.nix
index 0dac174accd..bdcb6f75966 100644
--- a/release.nix
+++ b/release.nix
@@ -155,17 +155,15 @@ let
 
 
     coverage =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with pkgs;
+
+      with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
         src = tarball;
 
-        buildInputs =
-          [ curl bzip2 openssl pkgconfig sqlite xz libsodium libseccomp
-            # These are for "make check" only:
-            graphviz libxml2 libxslt git mercurial
-          ];
+        buildInputs = buildDeps;
 
         configureFlags = ''
           --disable-init-state
@@ -175,7 +173,7 @@ let
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
+        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some

From 1aca195e52d4fbf76908e6588b55ed8b5b25cfc8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 15:17:59 +0100
Subject: [PATCH 0919/2196] Hack to get SSH error messages from build-remote

E.g.

  cannot build on 'ssh://mac1': cannot connect to 'mac1': bash: nix-store: command not found
  cannot build on 'ssh://mac2': cannot connect to 'mac2': Host key verification failed.
  cannot build on 'ssh://mac3': cannot connect to 'mac3': Received disconnect from 213... port 6001:2: Too many authentication failures
  Authentication failed.
---
 src/build-remote/build-remote.cc |  8 ++++++--
 src/libstore/build.cc            |  5 +++++
 src/libutil/util.cc              | 23 ++++++++++++++++++++---
 src/libutil/util.hh              |  4 ++--
 4 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 6b294a4d2c2..9cd01bb61bf 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -191,8 +191,10 @@ int main (int argc, char * * argv)
                     storeUri = bestMachine->storeUri;
 
                 } catch (std::exception & e) {
-                    printError("unable to open SSH connection to '%s': %s; trying other available machines...",
-                        bestMachine->storeUri, e.what());
+                    auto msg = chomp(drainFD(5, false));
+                    printError("cannot build on '%s': %s%s",
+                        bestMachine->storeUri, e.what(),
+                        (msg.empty() ? "" : ": " + msg));
                     bestMachine->enabled = false;
                     continue;
                 }
@@ -202,6 +204,8 @@ int main (int argc, char * * argv)
         }
 
 connected:
+        close(5);
+
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b8613427e0f..5c548755c65 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -652,6 +652,11 @@ HookInstance::HookInstance()
         if (dup2(builderOut.writeSide.get(), 4) == -1)
             throw SysError("dupping builder's stdout/stderr");
 
+        /* Hack: pass the read side of that fd to allow build-remote
+           to read SSH error messages. */
+        if (dup2(builderOut.readSide.get(), 5) == -1)
+            throw SysError("dupping builder's stdout/stderr");
+
         Strings args = {
             baseNameOf(settings.buildHook),
             std::to_string(verbosity),
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 764b4e44207..ab2d097f8b6 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -567,21 +567,38 @@ void writeFull(int fd, const string & s, bool allowInterrupts)
 }
 
 
-string drainFD(int fd)
+string drainFD(int fd, bool block)
 {
     StringSink sink;
-    drainFD(fd, sink);
+    drainFD(fd, sink, block);
     return std::move(*sink.s);
 }
 
 
-void drainFD(int fd, Sink & sink)
+void drainFD(int fd, Sink & sink, bool block)
 {
+    int saved;
+
+    Finally finally([&]() {
+        if (!block) {
+            if (fcntl(fd, F_SETFL, saved) == -1)
+                throw SysError("making file descriptor blocking");
+        }
+    });
+
+    if (!block) {
+        saved = fcntl(fd, F_GETFL);
+        if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
+            throw SysError("making file descriptor non-blocking");
+    }
+
     std::vector<unsigned char> buf(4096);
     while (1) {
         checkInterrupt();
         ssize_t rd = read(fd, buf.data(), buf.size());
         if (rd == -1) {
+            if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
+                break;
             if (errno != EINTR)
                 throw SysError("reading from file");
         }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 85952535db2..743d238611f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -151,9 +151,9 @@ MakeError(EndOfFile, Error)
 
 
 /* Read a file descriptor until EOF occurs. */
-string drainFD(int fd);
+string drainFD(int fd, bool block = true);
 
-void drainFD(int fd, Sink & sink);
+void drainFD(int fd, Sink & sink, bool block = true);
 
 
 /* Automatic cleanup of resources. */

From 21ebf6a16028291914bd14b0e5463b020dc7affa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 15:30:55 +0100
Subject: [PATCH 0920/2196] SSHMaster: Optionally pass -v to ssh

---
 src/libstore/ssh.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 7ff7a9bffc4..033c580936a 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -49,6 +49,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         addCommonSSHOpts(args);
         if (socketPath != "")
             args.insert(args.end(), {"-S", socketPath});
+        if (verbosity >= lvlChatty)
+            args.push_back("-v");
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
@@ -93,6 +95,8 @@ Path SSHMaster::startMaster()
             , "-o", "LocalCommand=echo started"
             , "-o", "PermitLocalCommand=yes"
             };
+        if (verbosity >= lvlChatty)
+            args.push_back("-v");
         addCommonSSHOpts(args);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 

From 5df427f35b77a87caf280a556b97a35d821499d3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 16:32:59 +0100
Subject: [PATCH 0921/2196] Slight simplification

---
 src/libstore/nar-info-disk-cache.cc | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 6e155e87780..3c52303f0ea 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -260,11 +260,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
 ref<NarInfoDiskCache> getNarInfoDiskCache()
 {
-    static Sync<std::shared_ptr<NarInfoDiskCache>> cache;
-
-    auto cache_(cache.lock());
-    if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>();
-    return ref<NarInfoDiskCache>(*cache_);
+    static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>();
+    return cache;
 }
 
 }

From 9d40787938f38d19edbabad03586d0948236aefd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 17:01:36 +0100
Subject: [PATCH 0922/2196] Move builtinFetchurl to its own file

---
 src/libstore/{builtins.cc => builtins/fetchurl.cc} | 0
 src/libstore/local.mk                              | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename src/libstore/{builtins.cc => builtins/fetchurl.cc} (100%)

diff --git a/src/libstore/builtins.cc b/src/libstore/builtins/fetchurl.cc
similarity index 100%
rename from src/libstore/builtins.cc
rename to src/libstore/builtins/fetchurl.cc
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index e11efa5c2b5..a7279aa3939 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -4,7 +4,7 @@ libstore_NAME = libnixstore
 
 libstore_DIR := $(d)
 
-libstore_SOURCES := $(wildcard $(d)/*.cc)
+libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
 libstore_LIBS = libutil libformat
 

From 668ac3ea2c4c7390761dfbc5738c2aa85fda9751 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 17:28:09 +0100
Subject: [PATCH 0923/2196] Make <nix/buildenv.nix> a builtin builder

This avoids sandbox annoyances.
---
 .gitignore                                    |   6 -
 Makefile                                      |   1 -
 corepkgs/buildenv.nix                         |  23 +---
 src/buildenv/local.mk                         |   9 --
 src/libstore/build.cc                         |   2 +
 src/libstore/builtins.hh                      |   2 +
 .../builtins}/buildenv.cc                     | 120 +++++++++---------
 7 files changed, 69 insertions(+), 94 deletions(-)
 delete mode 100644 src/buildenv/local.mk
 rename src/{buildenv => libstore/builtins}/buildenv.cc (64%)

diff --git a/.gitignore b/.gitignore
index 0a959937856..0f2f3ddeec1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,9 +13,6 @@ perl/Makefile.config
 
 /corepkgs/config.nix
 
-# /corepkgs/buildenv/
-/corepkgs/buildenv/builder.pl
-
 # /corepkgs/channels/
 /corepkgs/channels/unpack.sh
 
@@ -72,9 +69,6 @@ perl/Makefile.config
 # /src/nix-channel/
 /src/nix-channel/nix-channel
 
-# /src/buildenv/
-/src/buildenv/buildenv
-
 # /src/nix-build/
 /src/nix-build/nix-build
 
diff --git a/Makefile b/Makefile
index c47603731dd..dbe864ec9bc 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,6 @@ makefiles = \
   src/nix-collect-garbage/local.mk \
   src/nix-copy-closure/local.mk \
   src/nix-prefetch-url/local.mk \
-  src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \
   src/nix-channel/local.mk \
   src/nix-build/local.mk \
diff --git a/corepkgs/buildenv.nix b/corepkgs/buildenv.nix
index 5e7b40eaa0c..0bac4c44b48 100644
--- a/corepkgs/buildenv.nix
+++ b/corepkgs/buildenv.nix
@@ -1,11 +1,9 @@
-with import <nix/config.nix>;
-
 { derivations, manifest }:
 
 derivation {
   name = "user-environment";
-  system = builtins.currentSystem;
-  builder = nixLibexecDir + "/nix/buildenv";
+  system = "builtin";
+  builder = "builtin:buildenv";
 
   inherit manifest;
 
@@ -24,21 +22,4 @@ derivation {
 
   # Also don't bother substituting.
   allowSubstitutes = false;
-
-  __sandboxProfile = ''
-    (allow sysctl-read)
-    (allow file-read*
-           (literal "/usr/lib/libSystem.dylib")
-           (literal "/usr/lib/libSystem.B.dylib")
-           (literal "/usr/lib/libobjc.A.dylib")
-           (literal "/usr/lib/libobjc.dylib")
-           (literal "/usr/lib/libauto.dylib")
-           (literal "/usr/lib/libc++abi.dylib")
-           (literal "/usr/lib/libc++.1.dylib")
-           (literal "/usr/lib/libDiagnosticMessagesClient.dylib")
-           (subpath "/usr/lib/system")
-           (subpath "/dev"))
-  '';
-
-  inherit chrootDeps;
 }
diff --git a/src/buildenv/local.mk b/src/buildenv/local.mk
deleted file mode 100644
index 17ec13b235f..00000000000
--- a/src/buildenv/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += buildenv
-
-buildenv_DIR := $(d)
-
-buildenv_INSTALL_DIR := $(libexecdir)/nix
-
-buildenv_LIBS = libmain libstore libutil libformat
-
-buildenv_SOURCES := $(d)/buildenv.cc
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5c548755c65..082cd7db0f8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2949,6 +2949,8 @@ void DerivationGoal::runChild()
 
                 if (drv->builder == "builtin:fetchurl")
                     builtinFetchurl(drv2, netrcData);
+                else if (drv->builder == "builtin:buildenv")
+                    builtinBuildenv(drv2);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh
index 0cc6ba31f65..0d2da873ece 100644
--- a/src/libstore/builtins.hh
+++ b/src/libstore/builtins.hh
@@ -4,6 +4,8 @@
 
 namespace nix {
 
+// TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
+void builtinBuildenv(const BasicDerivation & drv);
 
 }
diff --git a/src/buildenv/buildenv.cc b/src/libstore/builtins/buildenv.cc
similarity index 64%
rename from src/buildenv/buildenv.cc
rename to src/libstore/builtins/buildenv.cc
index 2afad913ac6..938d02c35a0 100644
--- a/src/buildenv/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -1,14 +1,15 @@
-#include "shared.hh"
+#include "builtins.hh"
+
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <fcntl.h>
 #include <algorithm>
 
-using namespace nix;
+namespace nix {
 
 typedef std::map<Path,int> Priorities;
 
-static bool isDirectory (const Path & path)
+static bool isDirectory(const Path & path)
 {
     struct stat st;
     if (stat(path.c_str(), &st) == -1)
@@ -16,9 +17,11 @@ static bool isDirectory (const Path & path)
     return S_ISDIR(st.st_mode);
 }
 
-static auto priorities = Priorities{};
+// FIXME: change into local variables.
+
+static Priorities priorities;
 
-static auto symlinks = 0;
+static unsigned long symlinks;
 
 /* For each activated package, create symlinks */
 static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
@@ -95,10 +98,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
 
 typedef std::set<Path> FileProp;
 
-static auto done = FileProp{};
-static auto postponed = FileProp{};
+static FileProp done;
+static FileProp postponed = FileProp{};
 
-static auto out = string{};
+static Path out;
 
 static void addPkg(const Path & pkgDir, int priority)
 {
@@ -107,7 +110,7 @@ static void addPkg(const Path & pkgDir, int priority)
     done.insert(pkgDir);
     createLinks(pkgDir, out, priority);
     auto propagatedFN = pkgDir + "/nix-support/propagated-user-env-packages";
-    auto propagated = string{};
+    std::string propagated;
     {
         AutoCloseFD fd = open(propagatedFN.c_str(), O_RDONLY | O_CLOEXEC);
         if (!fd) {
@@ -126,62 +129,65 @@ struct Package {
     Path path;
     bool active;
     int priority;
-    Package(Path path, bool active, int priority) : path{std::move(path)}, active{active}, priority{priority} {}
+    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
 };
 
 typedef std::vector<Package> Packages;
 
-int main(int argc, char ** argv)
+void builtinBuildenv(const BasicDerivation & drv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        out = getEnv("out");
-        if (mkdir(out.c_str(), 0755) == -1)
-            throw SysError(format("creating %1%") % out);
-
-        /* Convert the stuff we get from the environment back into a coherent
-         * data type.
-         */
-        auto pkgs = Packages{};
-        auto derivations = tokenizeString<Strings>(getEnv("derivations"));
-        while (!derivations.empty()) {
-            /* !!! We're trusting the caller to structure derivations env var correctly */
-            auto active = derivations.front(); derivations.pop_front();
-            auto priority = stoi(derivations.front()); derivations.pop_front();
-            auto outputs = stoi(derivations.front()); derivations.pop_front();
-            for (auto n = 0; n < outputs; n++) {
-                auto path = derivations.front(); derivations.pop_front();
-                pkgs.emplace_back(path, active != "false", priority);
-            }
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        return i->second;
+    };
+
+    out = getAttr("out");
+    createDirs(out);
+
+    /* Convert the stuff we get from the environment back into a
+     * coherent data type. */
+    Packages pkgs;
+    auto derivations = tokenizeString<Strings>(getAttr("derivations"));
+    while (!derivations.empty()) {
+        /* !!! We're trusting the caller to structure derivations env var correctly */
+        auto active = derivations.front(); derivations.pop_front();
+        auto priority = stoi(derivations.front()); derivations.pop_front();
+        auto outputs = stoi(derivations.front()); derivations.pop_front();
+        for (auto n = 0; n < outputs; n++) {
+            auto path = derivations.front(); derivations.pop_front();
+            pkgs.emplace_back(path, active != "false", priority);
         }
+    }
 
-        /* Symlink to the packages that have been installed explicitly by the
-         * user. Process in priority order to reduce unnecessary
-         * symlink/unlink steps.
-         */
-        std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
-            return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
-        });
-        for (const auto & pkg : pkgs)
-            if (pkg.active)
-                addPkg(pkg.path, pkg.priority);
-
-        /* Symlink to the packages that have been "propagated" by packages
-         * installed by the user (i.e., package X declares that it wants Y
-         * installed as well). We do these later because they have a lower
-         * priority in case of collisions.
-         */
-        auto priorityCounter = 1000;
-        while (!postponed.empty()) {
-            auto pkgDirs = postponed;
-            postponed = FileProp{};
-            for (const auto & pkgDir : pkgDirs)
-                addPkg(pkgDir, priorityCounter++);
-        }
+    /* Symlink to the packages that have been installed explicitly by the
+     * user. Process in priority order to reduce unnecessary
+     * symlink/unlink steps.
+     */
+    std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
+        return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
+    });
+    for (const auto & pkg : pkgs)
+        if (pkg.active)
+            addPkg(pkg.path, pkg.priority);
+
+    /* Symlink to the packages that have been "propagated" by packages
+     * installed by the user (i.e., package X declares that it wants Y
+     * installed as well). We do these later because they have a lower
+     * priority in case of collisions.
+     */
+    auto priorityCounter = 1000;
+    while (!postponed.empty()) {
+        auto pkgDirs = postponed;
+        postponed = FileProp{};
+        for (const auto & pkgDir : pkgDirs)
+            addPkg(pkgDir, priorityCounter++);
+    }
 
-        std::cerr << "created " << symlinks << " symlinks in user environment\n";
+    printError("created %d symlinks in user environment", symlinks);
+
+    createSymlink(getAttr("manifest"), out + "/manifest.nix");
+}
 
-        createSymlink(getEnv("manifest"), out + "/manifest.nix");
-    });
 }
 

From dc99ea4483e8500d8ffe3015abf3a891d3d19559 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 20 Mar 2018 11:25:13 -0500
Subject: [PATCH 0924/2196] ask autotools for c++14 support flags, not c++11;
 don't override later

---
 Makefile      | 2 +-
 configure.ac  | 2 +-
 perl/Makefile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index c47603731dd..bf709abe4a6 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,7 @@ makefiles = \
   tests/local.mk \
   tests/plugins/local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
+GLOBAL_CXXFLAGS += -g -Wall -include config.h
 
 -include Makefile.config
 
diff --git a/configure.ac b/configure.ac
index 54322d463ae..7177c891443 100644
--- a/configure.ac
+++ b/configure.ac
@@ -62,7 +62,7 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_11
+AX_CXX_COMPILE_STDCXX_14
 
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
diff --git a/perl/Makefile b/perl/Makefile
index 684a37e8121..284c7502249 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -std=c++14 -g -Wall
+GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 

From e0c15979106c670e8022a3e21f792cb91d59da30 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 17:33:45 +0100
Subject: [PATCH 0925/2196] Style fix

---
 src/nix-channel/nix-channel.cc | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index ec9a7174ecb..c34f1d4b4c9 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -10,8 +10,8 @@ using namespace nix;
 
 typedef std::map<string,string> Channels;
 
-static auto channels = Channels{};
-static auto channelsList = Path{};
+static Channels channels;
+static Path channelsList;
 
 // Reads the list of channels.
 static void readChannels()
@@ -52,7 +52,7 @@ static void addChannel(const string & url, const string & name)
     writeChannels();
 }
 
-static auto profile = Path{};
+static Path profile;
 
 // Remove a channel.
 static void removeChannel(const string & name)
@@ -64,7 +64,7 @@ static void removeChannel(const string & name)
     runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name });
 }
 
-static auto nixDefExpr = Path{};
+static Path nixDefExpr;
 
 // Fetch Nix expressions and binary cache URLs from the subscribed channels.
 static void update(const StringSet & channelNames)
@@ -74,7 +74,7 @@ static void update(const StringSet & channelNames)
     auto store = openStore();
 
     // Download each channel.
-    auto exprs = Strings{};
+    Strings exprs;
     for (const auto & channel : channels) {
         auto name = channel.first;
         auto url = channel.second;
@@ -84,7 +84,7 @@ static void update(const StringSet & channelNames)
         // We want to download the url to a file to see if it's a tarball while also checking if we
         // got redirected in the process, so that we can grab the various parts of a nix channel
         // definition from a consistent location if the redirect changes mid-download.
-        auto effectiveUrl = string{};
+        std::string effectiveUrl;
         auto dl = getDownloader();
         auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl);
         url = chomp(std::move(effectiveUrl));
@@ -99,9 +99,9 @@ static void update(const StringSet & channelNames)
             cname = cname + (string) match[1];
         }
 
-        auto extraAttrs = string{};
+        std::string extraAttrs;
 
-        auto unpacked = false;
+        bool unpacked = false;
         if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
             runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
                         "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
@@ -136,7 +136,7 @@ static void update(const StringSet & channelNames)
     // Unpack the channel tarballs into the Nix store and install them
     // into the channels profile.
     std::cerr << "unpacking channels...\n";
-    auto envArgs = Strings{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
+    Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));
     envArgs.push_back("--quiet");
@@ -173,12 +173,9 @@ int main(int argc, char ** argv)
         nixDefExpr = home + "/.nix-defexpr";
 
         // Figure out the name of the channels profile.
-        auto name = string{};
+        ;
         auto pw = getpwuid(getuid());
-        if (!pw)
-            name = getEnv("USER", "");
-        else
-            name = pw->pw_name;
+        std::string name = pw ? pw->pw_name : getEnv("USER", "");
         if (name.empty())
             throw Error("cannot figure out user name");
         profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels";
@@ -192,7 +189,7 @@ int main(int argc, char ** argv)
             cUpdate,
             cRollback
         } cmd = cNone;
-        auto args = std::vector<string>{};
+        std::vector<string> args;
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help") {
                 showManPage("nix-channel");
@@ -224,7 +221,7 @@ int main(int argc, char ** argv)
                     throw UsageError("'--add' requires one or two arguments");
                 {
                 auto url = args[0];
-                auto name = string{};
+                std::string name;
                 if (args.size() == 2) {
                     name = args[1];
                 } else {
@@ -253,7 +250,7 @@ int main(int argc, char ** argv)
             case cRollback:
                 if (args.size() > 1)
                     throw UsageError("'--rollback' has at most one argument");
-                auto envArgs = Strings{"--profile", profile};
+                Strings envArgs{"--profile", profile};
                 if (args.size() == 1) {
                     envArgs.push_back("--switch-generation");
                     envArgs.push_back(args[0]);

From 03d8136b02e8c5929dfc9db91214e34bdc8089be Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Mar 2018 18:22:49 +0100
Subject: [PATCH 0926/2196] Remove unused channel-cache directory

---
 src/nix-channel/nix-channel.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index c34f1d4b4c9..d1b47ede803 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -162,11 +162,6 @@ int main(int argc, char ** argv)
     return handleExceptions(argv[0], [&]() {
         initNix();
 
-        // Turn on caching in nix-prefetch-url.
-        auto channelCache = settings.nixStateDir + "/channel-cache";
-        createDirs(channelCache);
-        setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1);
-
         // Figure out the name of the `.nix-channels' file to use
         auto home = getHome();
         channelsList = home + "/.nix-channels";

From 02d36fdab7415cd57dc09ca36f20e195977b7641 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 20 Mar 2018 13:29:29 -0500
Subject: [PATCH 0927/2196] execl: cast NULL sentinel to (char *), per man page
 and compiler warning

From exec(3):

> The list of arguments must be terminated by a null pointer, and, since these
> are variadic functions, this pointer must be cast (char *) NULL
---
 src/libmain/shared.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 7d888202bbf..1445c8f0301 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -263,7 +263,7 @@ void showManPage(const string & name)
 {
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
-    execlp("man", "man", name.c_str(), NULL);
+    execlp("man", "man", name.c_str(), (char *)NULL);
     throw SysError(format("command 'man %1%' failed") % name.c_str());
 }
 
@@ -325,10 +325,10 @@ RunPager::RunPager()
             setenv("LESS", "FRSXMK", 1);
         restoreSignals();
         if (pager)
-            execl("/bin/sh", "sh", "-c", pager, NULL);
-        execlp("pager", "pager", NULL);
-        execlp("less", "less", NULL);
-        execlp("more", "more", NULL);
+            execl("/bin/sh", "sh", "-c", pager, (char *)NULL);
+        execlp("pager", "pager", (char *)NULL);
+        execlp("less", "less", (char *)NULL);
+        execlp("more", "more", (char *)NULL);
         throw SysError(format("executing '%1%'") % pager);
     });
 

From 22b144fea697742c46a74fef06b0a3da36916f55 Mon Sep 17 00:00:00 2001
From: Corey O'Connor <coreyoconnor@gmail.com>
Date: Tue, 20 Mar 2018 17:58:25 -0700
Subject: [PATCH 0928/2196] manual: correct repeated "--deriver". Add missing
 single char option aliases.

---
 doc/manual/command-ref/nix-store.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index a5f615b0c26..f2dc6ed8540 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -501,10 +501,11 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--referrers</option></arg>
     <arg choice='plain'><option>--referrers-closure</option></arg>
     <arg choice='plain'><option>--deriver</option></arg>
-    <arg choice='plain'><option>--deriver</option></arg>
+    <arg choice='plain'><option>-d</option></arg>
     <arg choice='plain'><option>--graph</option></arg>
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
+    <arg choice='plain'><option>-b</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
     <arg choice='plain'><option>--size</option></arg>
     <arg choice='plain'><option>--roots</option></arg>
@@ -642,6 +643,7 @@ query is applied to the target of the symlink.</para>
   </varlistentry>
 
   <varlistentry><term><option>--deriver</option></term>
+    <term><option>-d</option></term>
 
     <listitem><para>Prints the <link
     linkend="gloss-deriver">deriver</link> of the store paths
@@ -678,6 +680,7 @@ query is applied to the target of the symlink.</para>
   </varlistentry>
 
   <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term>
+    <term><option>-b</option> <replaceable>name</replaceable></term>
 
     <listitem><para>Prints the value of the attribute
     <replaceable>name</replaceable> (i.e., environment variable) of

From 8615cfb1301d9402495a7558f8c66f63978f1c6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Mar 2018 15:10:46 +0100
Subject: [PATCH 0929/2196] LegacySSHStore: Allow overriding the path to
 nix-store

---
 src/libstore/legacy-ssh-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index dfefdb9bc87..2a2e3d91481 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -16,6 +16,7 @@ struct LegacySSHStore : public Store
     const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
     const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
 
     // Hack for getting remote build log output.
     const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
@@ -55,7 +56,7 @@ struct LegacySSHStore : public Store
     ref<Connection> openConnection()
     {
         auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand("nix-store --serve --write");
+        conn->sshConn = master.startCommand(fmt("%s --serve --write", remoteProgram));
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
 

From 47f7e5585bbb9a6acfaddda7327ba4eeb5b662c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Mar 2018 22:56:02 +0100
Subject: [PATCH 0930/2196] Make 'nix copy --from ssh://...' run in constant
 memory

For instance, this reduced the memory consumption of

  $ nix copy --from ssh://localhost --to ~/my-nix /nix/store/1n7x0yv8vq6zi90hfmian84vdhd04bgp-blender-2.79a

from 632 MiB to 16 MiB.
---
 src/libstore/legacy-ssh-store.cc |  7 +------
 src/libstore/ssh-store.cc        | 19 +------------------
 src/libutil/archive.cc           | 17 +++++++++++++++++
 src/libutil/archive.hh           |  3 +++
 4 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 2a2e3d91481..b52de543415 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -151,12 +151,7 @@ struct LegacySSHStore : public Store
 
         conn->to << cmdDumpStorePath << path;
         conn->to.flush();
-
-        /* FIXME: inefficient. */
-        ParseSink parseSink; /* null sink; just parse the NAR */
-        TeeSource savedNAR(conn->from);
-        parseDump(parseSink, savedNAR);
-        sink(*savedNAR.data);
+        copyNAR(conn->from, sink);
     }
 
     PathSet queryAllValidPaths() override { unsupported(); }
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 398408ea8d7..39205ae2ce1 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -63,29 +63,12 @@ class SSHStore : public RemoteStore
     };
 };
 
-
-class ForwardSource : public Source
-{
-    Source & readSource;
-    Sink & writeSink;
-public:
-    ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {}
-    size_t read(unsigned char * data, size_t len) override
-    {
-        auto n = readSource.read(data, len);
-        writeSink(data, n);
-        return n;
-    }
-};
-
 void SSHStore::narFromPath(const Path & path, Sink & sink)
 {
     auto conn(connections->get());
     conn->to << wopNarFromPath << path;
     conn->processStderr();
-    ParseSink ps;
-    auto fwd = ForwardSource(conn->from, sink);
-    parseDump(ps, fwd);
+    copyNAR(conn->from, sink);
 }
 
 ref<FSAccessor> SSHStore::getFSAccessor()
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index f71229d8fdd..a1d9d3233a0 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -350,4 +350,21 @@ void restorePath(const Path & path, Source & source)
 }
 
 
+void copyNAR(Source & source, Sink & sink)
+{
+    // FIXME: if 'source' is the output of dumpPath() followed by EOF,
+    // we should just forward all data directly without parsing.
+
+    ParseSink parseSink; /* null sink; just parse the NAR */
+
+    LambdaSource wrapper([&](unsigned char * data, size_t len) {
+        auto n = source.read(data, len);
+        sink(data, n);
+        return n;
+    });
+
+    parseDump(parseSink, wrapper);
+}
+
+
 }
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 8a15e849c7b..7a0e688e420 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -74,6 +74,9 @@ void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
 
+/* Read a NAR from 'source' and write it to 'sink'. */
+void copyNAR(Source & source, Sink & sink);
+
 
 // FIXME: global variables are bad m'kay.
 extern bool useCaseHack;

From 11898d6a37c80d4e78fc2df44617335669dfa95e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Mar 2018 23:12:22 +0100
Subject: [PATCH 0931/2196] Make 'nix copy --to ssh://...' run in constant
 memory

---
 src/libstore/legacy-ssh-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index b52de543415..5dee25308f7 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -120,7 +120,7 @@ struct LegacySSHStore : public Store
         });
     }
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override
     {
@@ -131,7 +131,7 @@ struct LegacySSHStore : public Store
         conn->to
             << cmdImportPaths
             << 1;
-        conn->to(*nar);
+        copyNAR(source, conn->to);
         conn->to
             << exportMagic
             << info.path

From f66ae2228880ba757af7dfcd216408ff95abaeba Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 21 Mar 2018 17:17:37 -0500
Subject: [PATCH 0932/2196] shared.cc: style: prefer "nullptr" over '(char
 *)NULL'

---
 src/libmain/shared.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 1445c8f0301..91a4eaf922a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -263,7 +263,7 @@ void showManPage(const string & name)
 {
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
-    execlp("man", "man", name.c_str(), (char *)NULL);
+    execlp("man", "man", name.c_str(), nullptr);
     throw SysError(format("command 'man %1%' failed") % name.c_str());
 }
 
@@ -325,10 +325,10 @@ RunPager::RunPager()
             setenv("LESS", "FRSXMK", 1);
         restoreSignals();
         if (pager)
-            execl("/bin/sh", "sh", "-c", pager, (char *)NULL);
-        execlp("pager", "pager", (char *)NULL);
-        execlp("less", "less", (char *)NULL);
-        execlp("more", "more", (char *)NULL);
+            execl("/bin/sh", "sh", "-c", pager, nullptr);
+        execlp("pager", "pager", nullptr);
+        execlp("less", "less", nullptr);
+        execlp("more", "more", nullptr);
         throw SysError(format("executing '%1%'") % pager);
     });
 

From 97002b684c902dadcd351a67208f9c2a88ff8f8f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Mar 2018 23:42:21 +0100
Subject: [PATCH 0933/2196] Make 'nix copy --to daemon' run in constant memory

---
 src/libstore/remote-store.cc | 7 +++----
 src/libstore/remote-store.hh | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8f0b65557ac..1fd33c5a9bd 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -377,7 +377,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 }
 
 
-void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(connections->get());
@@ -388,8 +388,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
         StringSink sink;
         sink << 1 // == path follows
             ;
-        assert(nar->size() % 8 == 0);
-        sink((unsigned char *) nar->data(), nar->size());
+        copyNAR(source, sink);
         sink
             << exportMagic
             << info.path
@@ -412,7 +411,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
                  << info.references << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
                  << repair << !checkSigs;
-        conn->to(*nar);
+        copyNAR(source, conn->to);
         conn->processStderr();
     }
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 0cc20bf9419..95fa59a2069 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -58,7 +58,7 @@ public:
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) override;
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 

From ec91840e70a7dbe59669e08f83ddad149bfba906 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Mar 2018 23:46:03 +0100
Subject: [PATCH 0934/2196] Also make the backwards compatible case of
 RemoteStore::addToStore run in constant memory

Just because we can.
---
 src/libstore/remote-store.cc | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1fd33c5a9bd..080cef93d21 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -385,21 +385,21 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
         conn->to << wopImportPaths;
 
-        StringSink sink;
-        sink << 1 // == path follows
-            ;
-        copyNAR(source, sink);
-        sink
-            << exportMagic
-            << info.path
-            << info.references
-            << info.deriver
-            << 0 // == no legacy signature
-            << 0 // == no path follows
-            ;
-
-        StringSource source(*sink.s);
-        conn->processStderr(0, &source);
+        auto source2 = sinkToSource([&](Sink & sink) {
+            sink << 1 // == path follows
+                ;
+            copyNAR(source, sink);
+            sink
+                << exportMagic
+                << info.path
+                << info.references
+                << info.deriver
+                << 0 // == no legacy signature
+                << 0 // == no path follows
+                ;
+        });
+
+        conn->processStderr(0, source2.get());
 
         auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
         assert(importedPaths.size() <= 1);

From 92aee1b7d69adc9552dc0efae9d030e02aa2f353 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Mar 2018 13:03:56 +0100
Subject: [PATCH 0935/2196] Remove some release-critical jobs

The release does not in fact depend on Ubuntu/Fedora builds (we don't
publish the build results).
---
 release.nix | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/release.nix b/release.nix
index bdcb6f75966..8befc15221a 100644
--- a/release.nix
+++ b/release.nix
@@ -265,12 +265,6 @@ let
           binaryTarball.i686-linux
           binaryTarball.x86_64-darwin
           binaryTarball.x86_64-linux
-          #deb_debian8i386
-          #deb_debian8x86_64
-          deb_ubuntu1604i386
-          deb_ubuntu1604x86_64
-          rpm_fedora25i386
-          rpm_fedora25x86_64
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball

From dce8af59a6eec2fbca4033853533277f0be165ee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Mar 2018 13:20:55 +0100
Subject: [PATCH 0936/2196] Tweak

---
 src/libutil/archive.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index b2459336a88..154e2d20430 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -44,7 +44,7 @@ static void dumpContents(const Path & path, size_t size,
     size_t left = size;
 
     while (left > 0) {
-        size_t n = left > buf.size() ? buf.size() : left;
+        auto n = std::min(left, buf.size());
         readFull(fd.get(), buf.data(), n);
         left -= n;
         sink(buf.data(), n);

From ed87fd17dde1cb3366739e0eecc5716fc7bd7c3f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Mar 2018 13:30:36 +0100
Subject: [PATCH 0937/2196] Remove Fedora / Ubuntu releases that are no longer
 in Nixpkgs

---
 release.nix | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/release.nix b/release.nix
index 8befc15221a..92db8636fce 100644
--- a/release.nix
+++ b/release.nix
@@ -182,8 +182,8 @@ let
       };
 
 
-    rpm_fedora25i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora25i386) [ "libsodium-devel" ];
-    rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ];
+    rpm_fedora27i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora27i386) [ "libsodium-devel" ];
+    rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ "libsodium-devel" ];
 
 
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
@@ -191,8 +191,6 @@ let
 
     deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
     deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
 
 
     # System tests.

From 28eb9621cc0c6715c99111de6990726f38d0341f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Mar 2018 13:46:19 +0100
Subject: [PATCH 0938/2196] Fix build

---
 src/libutil/compression.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index e7fedcbdc52..81cb5e98c76 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -134,8 +134,8 @@ static void decompressBrotli(Source & source, Sink & sink)
 {
 #if !HAVE_BROTLI
     RunOptions options(BROTLI, {"-d"});
-    options.stdin = &source;
-    options.stdout = &sink;
+    options.standardIn = &source;
+    options.standardOut = &sink;
     runProgram2(options);
 #else
     auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);

From 2bc6cfe1adc89673626a173ee38ef300588be3d1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Mar 2018 14:15:46 +0100
Subject: [PATCH 0939/2196] Add Boost dependency to Fedora/Ubuntu builds

---
 nix.spec.in |  2 ++
 release.nix | 16 ++++++++--------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 4e7dd90ea68..d962bcc857b 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -21,10 +21,12 @@ Requires: bzip2
 Requires: gzip
 Requires: xz
 Requires: libseccomp
+Requires: boost-context
 BuildRequires: bzip2-devel
 BuildRequires: sqlite-devel
 BuildRequires: libcurl-devel
 BuildRequires: libseccomp-devel
+BuildRequires: boost-devel
 
 # Hack to make that shitty RPM scanning hack shut up.
 Provides: perl(Nix::SSH)
diff --git a/release.nix b/release.nix
index 92db8636fce..5fae5468bf5 100644
--- a/release.nix
+++ b/release.nix
@@ -92,7 +92,7 @@ let
         src = tarball;
 
         buildInputs =
-          [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ]
+          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -182,15 +182,15 @@ let
       };
 
 
-    rpm_fedora27i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora27i386) [ "libsodium-devel" ];
-    rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ "libsodium-devel" ];
+    rpm_fedora27i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora27i386) [ ];
+    rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
 
 
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
     #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
 
-    deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ];
-    deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ];
+    deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
+    deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
 
 
     # System tests.
@@ -287,7 +287,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
       memSize = 2047;
@@ -310,9 +310,9 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
             ++ extraPackages; };
-      memSize = 1024;
+      memSize = 2047;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";

From 59b32403f2320e1979e5707b59d1dafcaf6ae910 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 22 Mar 2018 12:37:20 -0500
Subject: [PATCH 0940/2196] download.cc: fix path for cached files

---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 6eb87096692..2954b221d80 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -632,7 +632,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     if (expectedHash) {
         expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
         if (store->isValidPath(expectedStorePath))
-            return expectedStorePath;
+            return store->toRealPath(expectedStorePath);
     }
 
     Path cacheDir = getCacheDir() + "/nix/tarballs";

From 74da813912895015cf0f26da7aa520b278cb8071 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 22 Mar 2018 09:56:54 -0500
Subject: [PATCH 0941/2196] download: improve error for hash mismatch ("store
 mismatch")

Fixes #1905
---
 src/libstore/download.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 2954b221d80..4d7f5690192 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -726,8 +726,13 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         storePath = unpackedStorePath;
     }
 
-    if (expectedStorePath != "" && storePath != expectedStorePath)
-        throw nix::Error("store path mismatch in file downloaded from '%s'", url);
+    if (expectedStorePath != "" && storePath != expectedStorePath) {
+        Hash gotHash = unpack
+            ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
+            : hashFile(expectedHash.type, store->toRealPath(storePath));
+        throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'",
+            url, expectedHash.to_string(), gotHash.to_string());
+    }
 
     return store->toRealPath(storePath);
 }

From e2f56c1333d542b8022205215b637a30e71314c7 Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Thu, 22 Mar 2018 19:17:45 -0400
Subject: [PATCH 0942/2196] doc: don't mention obsolete ssh-substituter-hosts

#1840
---
 doc/manual/packages/ssh-substituter.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/packages/ssh-substituter.xml b/doc/manual/packages/ssh-substituter.xml
index f24f354c4c3..8db3f96625d 100644
--- a/doc/manual/packages/ssh-substituter.xml
+++ b/doc/manual/packages/ssh-substituter.xml
@@ -12,7 +12,7 @@ automatically fetching any store paths in Firefox’s closure if they
 are available on the server <literal>avalon</literal>:
 
 <screen>
-$ nix-env -i firefox --option ssh-substituter-hosts alice@avalon
+$ nix-env -i firefox --substituters ssh://alice@avalon
 </screen>
 
 This works similar to the binary cache substituter that Nix usually
@@ -31,7 +31,7 @@ an SSH passphrase interactively. Therefore, you should use
 installing it into your profile, e.g.
 
 <screen>
-$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --option ssh-substituter-hosts alice@avalon
+$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon
 </screen>
 
 This is essentially equivalent to doing

From 6856fe62b0a81a948709498ea53d333b69cfeca9 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 22 Mar 2018 19:12:11 -0400
Subject: [PATCH 0943/2196] Document fetchGit.

Fixes #1981.
---
 doc/manual/expressions/builtins.xml | 55 +++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 8a32ed8b5c9..47b98460adf 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -307,6 +307,61 @@ stdenv.mkDerivation { … }
 
   </varlistentry>
 
+  <varlistentry>
+    <term>
+      <function>builtins.fetchGit</function>
+      <replaceable>args</replaceable>
+    </term>
+
+    <listitem>
+      <para>
+        Fetch a path from git. <replaceable>args</replaceable> can be
+        a URL, in which case the HEAD of the repo at that URL is
+        fetched. Otherwise, it can be an attribute with the following
+        attributes (all except <varname>url</varname> optional):
+      </para>
+
+      <variablelist>
+        <varlistentry>
+          <term>url</term>
+          <listitem>
+            <para>
+              The URL of the repo.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>name</term>
+          <listitem>
+            <para>
+              The name of the directory the repo should be exported to
+              in the store. Defaults to the basename of the URL.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>rev</term>
+          <listitem>
+            <para>
+              The git revision to fetch. Defaults to the tip of
+              <varname>ref</varname>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>ref</term>
+          <listitem>
+            <para>
+              The git ref to look for the requested revision under.
+              This is often a branch or tag name. Defaults to
+              <literal>HEAD</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+    </listitem>
+  </varlistentry>
+
 
   <varlistentry><term><function>builtins.filter</function>
   <replaceable>f</replaceable> <replaceable>xs</replaceable></term>

From cc6712ae90fb0783c51f1b3f86433adcd11a4ccb Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Mon, 26 Mar 2018 15:14:53 +0300
Subject: [PATCH 0944/2196] release.nix: Use pkgs.closureInfo for binaryTarball

pathsFromGraph is legacy since Nix 2.0.
---
 release.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/release.nix b/release.nix
index 5fae5468bf5..91c782c5191 100644
--- a/release.nix
+++ b/release.nix
@@ -115,16 +115,15 @@ let
       let
         toplevel = builtins.getAttr system jobs.build;
         version = toplevel.src.version;
+        installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ];
-          buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck;
+        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
-          storePaths=$(perl ${pathsFromGraph} ./closure1 ./closure2)
-          printRegistration=1 perl ${pathsFromGraph} ./closure1 ./closure2 > $TMPDIR/reginfo
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
@@ -150,7 +149,8 @@ let
             --transform "s,$TMPDIR/install,$dir/install," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo $storePaths
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
         '');
 
 

From 9d1220a01d662c1089e77f16d5d3e1ead8f0ed4d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Mar 2018 21:49:09 +0200
Subject: [PATCH 0945/2196] ValidPathInfo::isContentAddressed(): Ensure there
 are no references

---
 src/libstore/store-api.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 64f9b8d68b0..1a0d12ca78c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -757,7 +757,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     else if (hasPrefix(ca, "fixed:")) {
         bool recursive = ca.compare(6, 2, "r:") == 0;
         Hash hash(std::string(ca, recursive ? 8 : 6));
-        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
+        if (references.empty() &&
+            store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
             return true;
         else
             warn();

From 2b61c749227fcfc05e34bb9ea568602868ec11cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 15:34:37 +0200
Subject: [PATCH 0946/2196] Process --option use-case-hack properly

Fixes https://github.com/NixOS/nix/issues/2009.
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index dd01f832df0..117404ec14c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -29,7 +29,7 @@ struct CaseHackSetting : public BaseSetting<bool>
     void set(const std::string & str) override
     {
         BaseSetting<bool>::set(str);
-        nix::useCaseHack = true;
+        nix::useCaseHack = value;
     }
 };
 

From 446bb88f13080eed131a1b0bd304c45a860a3fab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 19:03:03 +0200
Subject: [PATCH 0947/2196] Remove reference to non-existent manpages

---
 src/nix/command.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 1e6f0d2bb75..3d7d582d6f5 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -57,8 +57,10 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out)
     }
     printTable(out, table);
 
+#if 0
     out << "\n";
     out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
+#endif
 }
 
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)

From 2921165a9d14b80fd4a7e350e72ec2b83791682c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 28 Mar 2018 09:16:13 -0400
Subject: [PATCH 0948/2196] Expand the multi-user installer to support Linuxes
 with systemd

 - darwin installer: delete hardware report, not necessary
 - moves os-specific code from the darwin installer to to `poly_*`
   functions
 - adds profile.d support to the profile targets, which automatically
   handles many distros which don't have a /etc/bashrc but do have an
   /etc/profile.d
 - /bin/bash -> /usr/bin/env bash
 - document why each excluded shellcheck check is excluded
 - rename the multi-user to Daemon-based
---
 release.nix                           |  34 +-
 scripts/install-darwin-multi-user.sh  | 831 +++-----------------------
 scripts/install-multi-user.sh         | 793 ++++++++++++++++++++++++
 scripts/install-nix-from-closure.sh   |  11 +-
 scripts/install-systemd-multi-user.sh | 154 +++++
 5 files changed, 1063 insertions(+), 760 deletions(-)
 create mode 100644 scripts/install-multi-user.sh
 create mode 100644 scripts/install-systemd-multi-user.sh

diff --git a/release.nix b/release.nix
index 91c782c5191..492065df985 100644
--- a/release.nix
+++ b/release.nix
@@ -127,17 +127,39 @@ let
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user \
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${toplevel} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
 
           if type -p shellcheck; then
-            shellcheck -e SC1090 $TMPDIR/install
-            shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
           fi
 
           chmod +x $TMPDIR/install
-          chmod +x $TMPDIR/install-darwin-multi-user
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
           dir=nix-${version}-${system}
           fn=$out/$dir.tar.bz2
           mkdir -p $out/nix-support
@@ -149,7 +171,9 @@ let
             --transform "s,$TMPDIR/install,$dir/install," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo \
+            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user $TMPDIR/reginfo \
             $(cat ${installerClosureInfo}/store-paths)
         '');
 
diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 716b6e9bc9a..87c4c2b0582 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -1,819 +1,144 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -eu
 set -o pipefail
 
-# Sourced from:
-# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh
-# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0
-# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh
-# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh
-# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a
-#
-# however tracking which bits came from which would be impossible.
-
-readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
-readonly RED_UL='\033[38;4;31m'
-readonly YELLOW='\033[38;33m'
-readonly YELLOW_UL='\033[38;4;33m'
-
-readonly CORES=$(sysctl -n hw.ncpu)
-readonly NIX_USER_COUNT="32"
-readonly NIX_BUILD_GROUP_ID="30000"
-readonly NIX_BUILD_GROUP_NAME="nixbld"
-readonly NIX_FIRST_BUILD_UID="30001"
-# Please don't change this. We don't support it, because the
-# default shell profile that comes with Nix doesn't support it.
-readonly NIX_ROOT="/nix"
 readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc")
-readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
-readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
-
-readonly NIX_INSTALLED_NIX="@nix@"
-readonly NIX_INSTALLED_CACERT="@cacert@"
-readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
-
-readonly ROOT_HOME="/var/root"
-
-if [ -t 0 ]; then
-    readonly IS_HEADLESS='no'
-else
-    readonly IS_HEADLESS='yes'
-fi
+dsclattr() {
+    /usr/bin/dscl . -read "$1" \
+        | awk "/$2/ { print \$2 }"
+}
 
-headless() {
-    if [ "$IS_HEADLESS" = "yes" ]; then
-        return 0
-    else
-        return 1
+poly_validate_assumptions() {
+    if [ "$(uname -s)" != "Darwin" ]; then
+        failure "This script is for use with macOS!"
     fi
 }
 
-contactme() {
-    echo "We'd love to help if you need it."
-    echo ""
-    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
-    echo ""
-    echo "Or feel free to contact the team,"
-    echo " - on IRC #nixos on irc.freenode.net"
-    echo " - on twitter @nixos_org"
+poly_service_installed_check() {
+    [ -e "$PLIST_DEST" ]
 }
 
-uninstall_directions() {
-    subheader "Uninstalling nix:"
-    local step=0
-
-    if [ -e "$PLIST_DEST" ]; then
-        step=$((step + 1))
+poly_service_uninstall_directions() {
         cat <<EOF
-$step. Delete $PLIST_DEST
+$1. Delete $PLIST_DEST
 
   sudo launchctl unload $PLIST_DEST
   sudo rm $PLIST_DEST
 
 EOF
-    fi
-
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-            step=$((step + 1))
-            cat <<EOF
-$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target
-
-  sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-(after this one, you may need to re-open any terminals that were
-opened while it existed.)
-
-EOF
-        fi
-    done
+}
 
-    step=$((step + 1))
+poly_service_setup_note() {
     cat <<EOF
-$step. Delete the files Nix added to your system:
-
-  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
-
-and that is it.
+ - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
 
 EOF
-
-}
-
-nix_user_for_core() {
-    printf "nixbld%d" "$1"
-}
-
-nix_uid_for_core() {
-    echo $((NIX_FIRST_BUILD_UID + $1 - 1))
 }
 
-dsclattr() {
-    /usr/bin/dscl . -read "$1" \
-        | awk "/$2/ { print \$2 }"
-}
-
-_textout() {
-    echo -en "$1"
-    shift
-    if [ "$*" = "" ]; then
-        cat
-    else
-        echo "$@"
-    fi
-    echo -en "$ESC"
-}
-
-header() {
-    follow="---------------------------------------------------------"
-    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
-    echo ""
-    _textout "$BLUE" "$header"
-}
-
-warningheader() {
-    follow="---------------------------------------------------------"
-    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
-    echo ""
-    _textout "$RED" "$header"
-}
-
-subheader() {
-    echo ""
-    _textout "$BLUE_UL" "$*"
-}
-
-row() {
-    printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2"
-}
-
-task() {
-    echo ""
-    ok "~~> $1"
-}
-
-bold() {
-    echo "$BOLD$*$ESC"
-}
-
-ok() {
-    _textout "$GREEN" "$@"
-}
-
-warning() {
-    warningheader "warning!"
-    cat
-    echo ""
-}
-
-failure() {
-    header "oh no!"
-    _textout "$RED" "$@"
-    echo ""
-    _textout "$RED" "$(contactme)"
-    trap finish_cleanup EXIT
-    exit 1
-}
-
-ui_confirm() {
-    _textout "$GREEN$GREEN_UL" "$1"
-
-    if headless; then
-        echo "No TTY, assuming you would say yes :)"
-        return 0
-    fi
-
-    local prompt="[y/n] "
-    echo -n "$prompt"
-    while read -r y; do
-        if [ "$y" = "y" ]; then
-            echo ""
-            return 0
-        elif [ "$y" = "n" ]; then
-            echo ""
-            return 1
-        else
-            _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n"
-            echo -n "$prompt"
-        fi
-    done
-    echo ""
-    return 1
-}
-
-__sudo() {
-    local expl="$1"
-    local cmd="$2"
-    shift
-    header "sudo execution"
-
-    echo "I am executing:"
-    echo ""
-    printf "    $ sudo %s\\n" "$cmd"
-    echo ""
-    echo "$expl"
-    echo ""
-
-    return 0
-}
+poly_configure_nix_daemon_service() {
+    _sudo "to set up the nix-daemon as a LaunchDaemon" \
+          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
 
-_sudo() {
-    local expl="$1"
-    shift
-    if ! headless; then
-        __sudo "$expl" "$*"
-    fi
-    sudo "$@"
-}
+    _sudo "to load the LaunchDaemon plist for nix-daemon" \
+          launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
+    _sudo "to start the nix-daemon" \
+          launchctl start org.nixos.nix-daemon
 
-readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
-function finish_cleanup {
-    rm -rf "$SCRATCH"
 }
 
-function finish_fail {
-    finish_cleanup
-
-    failure <<EOF
-Jeeze, something went wrong. If you can take all the output and open
-an issue, we'd love to fix the problem so nobody else has this issue.
-
-:(
-EOF
+poly_group_exists() {
+    /usr/bin/dscl . -read "/Groups/$1" > /dev/null 2>&1
 }
-trap finish_fail EXIT
-
-function finish_success {
-    finish_cleanup
 
-    ok "Alright! We're done!"
-    cat <<EOF
-
-Before Nix will work in your existing shells, you'll need to close
-them and open them again. Other than that, you should be ready to go.
-
-Try it! Open a new terminal, and type:
-
-  $ nix-shell -p nix-info --run "nix-info -m"
-
-Thank you for using this installer. If you have any feedback, don't
-hesitate:
-
-$(contactme)
-EOF
+poly_group_id_get() {
+    dsclattr "/Groups/$1" "PrimaryGroupID"
 }
 
-
-validate_starting_assumptions() {
-    if [ "$(uname -s)" != "Darwin" ]; then
-        failure "This script is for use with macOS!"
-    fi
-
-    if [ $EUID -eq 0 ]; then
-        failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
-EOF
-    fi
-
-    if type nix-env 2> /dev/null >&2; then
-        failure <<EOF
-Nix already appears to be installed, and this tool assumes it is
-_not_ yet installed.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ "${NIX_REMOTE:-}" != "" ]; then
-        failure <<EOF
-For some reason, \$NIX_REMOTE is set. It really should not be set
-before this installer runs, and it hints that Nix is currently
-installed. Please delete the old Nix installation and start again.
-
-Note: You might need to close your shell window and open a new shell
-to clear the variable.
-EOF
-    fi
-
-    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
-        failure <<EOF
-It looks like \$SSL_CERT_FILE is set to a path that used to be part of
-the old Nix installation. Please unset that variable and try again:
-
-  $ unset SSL_CERT_FILE
-
-EOF
-    fi
-
-    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        if [ -f "$file" ]; then
-            if grep -l "^[^#].*.nix-profile" "$file"; then
-                failure <<EOF
-I found a reference to a ".nix-profile" in $file.
-This has a high chance of breaking a new nix installation. It was most
-likely put there by a previous Nix installer.
-
-Please remove this reference and try running this again. You should
-also look for similar references in:
-
- - ~/.bash_profile
- - ~/.bash_login
- - ~/.profile
-
-or other shell init files that you may have.
-
-$(uninstall_directions)
-EOF
-            fi
-        fi
-    done
-
-    if [ -d /nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ -d /etc/nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /etc/nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-        failure <<EOF
-When this script runs, it backs up the current $profile_target to
-$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
-
-Please follow these instructions to clean up the old backup file:
-
-1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
-in case.
-
-2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
-it has anything nix-related in it. If it does, something is probably
-quite wrong. Please open an issue or get in touch immediately.
-
-3. Take care to make sure that $profile_target doesn't look like it has
-anything nix-related in it. If it does, and $profile_target _did not_,
-run:
-
-  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-and try again.
-EOF
-        fi
-
-        if grep -qi "nix" "$profile_target"; then
-            failure <<EOF
-It looks like $profile_target already has some Nix configuration in
-there. There should be no reason to run this again. If you're having
-trouble, please open an issue.
-EOF
-        fi
-    done
-
-    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
-    for danger_path in "${danger_paths[@]}"; do
-        if _sudo "making sure that $danger_path doesn't exist" \
-           test -e "$danger_path"; then
-            failure <<EOF
-I found a file at $danger_path, which is a relic of a previous
-installation. You must first delete this file before continuing.
-
-$(uninstall_directions)
-EOF
-        fi
-    done
+poly_create_build_group() {
+    _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
+          /usr/sbin/dseditgroup -o create \
+          -r "Nix build group for nix-daemon" \
+          -i "$NIX_BUILD_GROUP_ID" \
+          "$NIX_BUILD_GROUP_NAME" >&2
 }
 
-setup_report() {
-    header "hardware report"
-    row "           Cores" "$CORES"
-
-    header "Nix config report"
-    row "        Temp Dir" "$SCRATCH"
-    row "        Nix Root" "$NIX_ROOT"
-    row "     Build Users" "$NIX_USER_COUNT"
-    row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
-    row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then
-        row "Preexisting Install" "Allowed"
-    fi
-
-    subheader "build users:"
-
-    row "    Username" "UID"
-    for i in $(seq 1 "$NIX_USER_COUNT"); do
-        row "     $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")"
-    done
-    echo ""
+poly_user_exists() {
+    /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1
 }
 
-create_build_group() {
-    local primary_group_id
-
-    task "Setting up the build group $NIX_BUILD_GROUP_NAME"
-    if ! /usr/bin/dscl . -read "/Groups/$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1; then
-        _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
-              /usr/sbin/dseditgroup -o create \
-              -r "Nix build group for nix-daemon" \
-              -i "$NIX_BUILD_GROUP_ID" \
-              "$NIX_BUILD_GROUP_NAME" >&2
-        row "            Created" "Yes"
-    else
-        primary_group_id=$(dsclattr "/Groups/$NIX_BUILD_GROUP_NAME" "PrimaryGroupID")
-        if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then
-            failure <<EOF
-It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
-with the UID $primary_group_id. This script can't really handle
-that right now, so I'm going to give up.
-
-You can fix this by editing this script and changing the
-NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
-to $primary_group_id and re-run.
-EOF
-        else
-            row "            Exists" "Yes"
-        fi
-    fi
+poly_user_id_get() {
+    dsclattr "/Users/$1" "UniqueID"
 }
 
-create_build_user_for_core() {
-    local coreid
-    local username
-    local uid
-
-    coreid="$1"
-    username=$(nix_user_for_core "$coreid")
-    uid=$(nix_uid_for_core "$coreid")
-    dsclpath="/Users/$username"
-
-    task "Setting up the build user $username"
-
-    if ! /usr/bin/dscl . -read "$dsclpath" > /dev/null 2>&1; then
-        _sudo "Creating the Nix build user, $username" \
-              /usr/bin/dscl . create "$dsclpath" \
-              UniqueID "${uid}"
-        row "           Created" "Yes"
-    else
-        actual_uid=$(dsclattr "$dsclpath" "UniqueID")
-        if [ "$actual_uid" -ne "$uid" ]; then
-            failure <<EOF
-It seems the build user $username already exists, but with the UID
-with the UID $actual_uid. This script can't really handle that right
-now, so I'm going to give up.
-
-If you already created the users and you know they start from
-$actual_uid and go up from there, you can edit this script and change
-NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try
-again.
-EOF
-        else
-            row "            Exists" "Yes"
-        fi
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "IsHidden")" = "1" ]; then
-        row "          IsHidden" "Yes"
-    else
-        _sudo "in order to make $username a hidden user" \
-              /usr/bin/dscl . -create "$dsclpath" "IsHidden" "1"
-        row "          IsHidden" "Yes"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "NFSHomeDirectory")" = "/var/empty" ]; then
-        row "          NFSHomeDirectory" "/var/empty"
-    else
-        _sudo "in order to give $username a safe home directory" \
-              /usr/bin/dscl . -create "$dsclpath" "NFSHomeDirectory" "/var/empty"
-        row "          NFSHomeDirectory" "/var/empty"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "RealName")" = "Nix build user $coreid" ]; then
-        row "          RealName" "Nix build user $coreid"
-    else
-        _sudo "in order to give $username a useful name" \
-              /usr/bin/dscl . -create "$dsclpath" "RealName" "Nix build user $coreid"
-        row "          RealName" "Nix build user $coreid"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "UserShell")" = "/sbin/nologin" ]; then
-        row "   Logins Disabled" "Yes"
-    else
-        _sudo "in order to prevent $username from logging in" \
-              /usr/bin/dscl . -create "$dsclpath" "UserShell" "/sbin/nologin"
-        row "   Logins Disabled" "Yes"
-    fi
-
-    if dseditgroup -o checkmember -m "$username" "$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1 ; then
-        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
-    else
-        _sudo "Add $username to the $NIX_BUILD_GROUP_NAME group"\
-            /usr/sbin/dseditgroup -o edit -t user \
-            -a "$username" "$NIX_BUILD_GROUP_NAME"
-        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
-    fi
-
-    if [ "$(dsclattr "$dsclpath" "PrimaryGroupID")" = "$NIX_BUILD_GROUP_ID" ]; then
-        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-    else
-        _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
-              /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
-
-    fi
+poly_user_hidden_get() {
+    dsclattr "/Users/$1" "IsHidden"
 }
 
-create_build_users() {
-    for i in $(seq 1 "$NIX_USER_COUNT"); do
-        create_build_user_for_core "$i"
-    done
+poly_user_hidden_set() {
+    _sudo "in order to make $1 a hidden user" \
+          /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1"
 }
 
-create_directories() {
-    _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
-
-    _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
-          mkdir -pv -m 1775 /nix/store
-
-    _sudo "to make the basic directory structure of Nix (part 4)" \
-          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
-
-    _sudo "to set up the root user's profile (part 1)" \
-          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
-
-    _sudo "to set up the root user's profile (part 2)" \
-          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
-
-    _sudo "to place the default nix daemon configuration (part 1)" \
-          mkdir -pv -m 0555 /etc/nix
+poly_user_home_get() {
+    dsclattr "/Users/$1" "NFSHomeDirectory"
 }
 
-place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+poly_user_home_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2"
 }
 
-welcome_to_nix() {
-    ok "Welcome to the Multi-User Nix Installation"
-
-    cat <<EOF
-
-This installation tool will set up your computer with the Nix package
-manager. This will happen in a few stages:
-
-1. Make sure your computer doesn't already have Nix. If it does, I
-   will show you instructions on how to clean up your old one.
-
-2. Show you what we are going to install and where. Then we will ask
-   if you are ready to continue.
-
-3. Create the system users and groups that the Nix daemon uses to run
-   builds.
-
-4. Perform the basic installation of the Nix files daemon.
-
-5. Configure your shell to import special Nix Profile files, so you
-   can use Nix.
-
-6. Start the Nix daemon.
-
-EOF
-
-    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
-        cat <<EOF
-
-We will:
-
- - make sure your computer doesn't already have Nix files
-   (if it does, I  will tell you how to clean them up.)
- - create local users (see the list above for the users we'll make)
- - create a local group ($NIX_BUILD_GROUP_NAME)
- - install Nix in to $NIX_ROOT
- - create a configuration file in /etc/nix
- - set up the "default profile" by creating some Nix-related files in
-   $ROOT_HOME
-EOF
-        for profile_target in "${PROFILE_TARGETS[@]}"; do
-            if [ -e "$profile_target" ]; then
-                cat <<EOF
- - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
- - update $profile_target to include some Nix configuration
-EOF
-            fi
-        done
-        cat <<EOF
- - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon
-
-EOF
-        if ! ui_confirm "Ready to continue?"; then
-            failure <<EOF
-Okay, maybe you would like to talk to the team.
-EOF
-        fi
-    fi
+poly_user_note_get() {
+    dsclattr "/Users/$1" "RealName"
 }
 
-chat_about_sudo() {
-    header "let's talk about sudo"
-
-    if headless; then
-        cat <<EOF
-This script is going to call sudo a lot. Normally, it would show you
-exactly what commands it is running and why. However, the script is
-run in a headless fashion, like this:
-
-  $ curl https://nixos.org/nix/install | sh
-
-or maybe in a CI pipeline. Because of that, we're going to skip the
-verbose output in the interest of brevity.
-
-If you would like to
-see the output, try like this:
-
-  $ curl -o install-nix https://nixos.org/nix/install
-  $ sh ./install-nix
-
-EOF
-        return 0
-    fi
-
-    cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
-output exactly what it'll do, and why.
-
-Just like this:
-EOF
-
-    __sudo "to demonstrate how our sudo prompts look" \
-           echo "this is a sudo prompt"
-
-    cat <<EOF
-
-This might look scary, but everything can be undone by running just a
-few commands. We used to ask you to confirm each time sudo ran, but it
-was too many times. Instead, I'll just ask you this one time:
-
-EOF
-    if ui_confirm "Can we use sudo?"; then
-        ok "Yay! Thanks! Let's get going!"
-    else
-        failure <<EOF
-That is okay, but we can't install.
-EOF
-    fi
+poly_user_note_set() {
+    _sudo "in order to give $username a useful note" \
+          /usr/bin/dscl . -create "/Users/$1" "RealName" "$2"
 }
 
-install_from_extracted_nix() {
-    (
-        cd "$EXTRACTED_NIX_PATH"
-
-        _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
-
-        if [ -d "$NIX_INSTALLED_NIX" ]; then
-            echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
-        else
-            failure <<EOF
-Something went wrong, and I didn't find Nix installed at
-$NIX_INSTALLED_NIX.
-EOF
-        fi
-
-        _sudo "to initialize the Nix Database" \
-              $NIX_INSTALLED_NIX/bin/nix-store --init
-
-        cat ./.reginfo \
-            | _sudo "to load data for the first time in to the Nix Database" \
-                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
-
-        echo "      Just finished getting the nix database ready."
-    )
+poly_user_shell_get() {
+    dsclattr "/Users/$1" "UserShell"
 }
 
-shell_source_lines() {
-    cat <<EOF
-
-# Nix
-if [ -e '$PROFILE_NIX_FILE' ]; then
-  . '$PROFILE_NIX_FILE'
-fi
-# End Nix
-
-EOF
+poly_user_shell_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2"
 }
-configure_shell_profile() {
-    for profile_target in "${PROFILE_TARGETS[@]}"; do
-        if [ -e "$profile_target" ]; then
-            _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
-                  cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
-
-            shell_source_lines \
-                | _sudo "extend your $profile_target with nix-daemon settings" \
-                        tee -a "$profile_target"
-        fi
-    done
 
+poly_user_in_group_check() {
+    username=$1
+    group=$2
+    dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1
 }
 
-setup_default_profile() {
-    _sudo "to installing a bootstrapping Nix in to the default Profile" \
-          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
-
-    _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
-          HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+poly_user_in_group_set() {
+    username=$1
+    group=$2
 
-    _sudo "to update the default channel in the default profile" \
-          HOME=$ROOT_HOME NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+    _sudo "Add $username to the $group group"\
+          /usr/sbin/dseditgroup -o edit -t user \
+          -a "$username" "$group"
 }
 
-
-place_nix_configuration() {
-    cat <<EOF > "$SCRATCH/nix.conf"
-build-users-group = $NIX_BUILD_GROUP_NAME
-
-max-jobs = $NIX_USER_COUNT
-cores = 1
-sandbox = false
-EOF
-    _sudo "to place the default nix daemon configuration (part 2)" \
-          install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
+poly_user_primary_group_get() {
+    dsclattr "/Users/$1" "PrimaryGroupID"
 }
 
-configure_nix_daemon_plist() {
-    _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
-
-    _sudo "to load the LaunchDaemon plist for nix-daemon" \
-          launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-
-    _sudo "to start the nix-daemon" \
-          launchctl start org.nixos.nix-daemon
-
+poly_user_primary_group_set() {
+    _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
+          /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2"
 }
 
+poly_create_build_user() {
+    username=$1
+    uid=$2
+    builder_num=$3
 
-main() {
-    welcome_to_nix
-    chat_about_sudo
-
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
-        validate_starting_assumptions
-    fi
-
-    setup_report
-
-    if ! ui_confirm "Ready to continue?"; then
-        ok "Alright, no changes have been made :)"
-        contactme
-        trap finish_cleanup EXIT
-        exit 1
-    fi
-
-    create_build_group
-    create_build_users
-    create_directories
-    place_channel_configuration
-    install_from_extracted_nix
-
-    configure_shell_profile
-
-    set +eu
-    . /etc/profile
-    set -eu
-
-    setup_default_profile
-    place_nix_configuration
-    configure_nix_daemon_plist
-
-    trap finish_success EXIT
+    _sudo "Creating the Nix build user (#$builder_num), $username" \
+          /usr/bin/dscl . create "/Users/$username" \
+          UniqueID "${uid}"
 }
-
-
-main
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
new file mode 100644
index 00000000000..0e9500a98eb
--- /dev/null
+++ b/scripts/install-multi-user.sh
@@ -0,0 +1,793 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+# Sourced from:
+# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh
+# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0
+# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh
+# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh
+# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a
+#
+# however tracking which bits came from which would be impossible.
+
+readonly ESC='\033[0m'
+readonly BOLD='\033[38;1m'
+readonly BLUE='\033[38;34m'
+readonly BLUE_UL='\033[38;4;34m'
+readonly GREEN='\033[38;32m'
+readonly GREEN_UL='\033[38;4;32m'
+readonly RED='\033[38;31m'
+readonly RED_UL='\033[38;4;31m'
+readonly YELLOW='\033[38;33m'
+readonly YELLOW_UL='\033[38;4;33m'
+
+readonly NIX_USER_COUNT="32"
+readonly NIX_BUILD_GROUP_ID="30000"
+readonly NIX_BUILD_GROUP_NAME="nixbld"
+readonly NIX_FIRST_BUILD_UID="30001"
+# Please don't change this. We don't support it, because the
+# default shell profile that comes with Nix doesn't support it.
+readonly NIX_ROOT="/nix"
+
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
+readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
+
+readonly NIX_INSTALLED_NIX="@nix@"
+readonly NIX_INSTALLED_CACERT="@cacert@"
+readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
+
+readonly ROOT_HOME=$(echo ~root)
+
+if [ -t 0 ]; then
+    readonly IS_HEADLESS='no'
+else
+    readonly IS_HEADLESS='yes'
+fi
+
+headless() {
+    if [ "$IS_HEADLESS" = "yes" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+contactme() {
+    echo "We'd love to help if you need it."
+    echo ""
+    echo "If you can, open an issue at https://github.com/nixos/nix/issues"
+    echo ""
+    echo "Or feel free to contact the team,"
+    echo " - on IRC #nixos on irc.freenode.net"
+    echo " - on twitter @nixos_org"
+}
+
+uninstall_directions() {
+    subheader "Uninstalling nix:"
+    local step=0
+
+    if poly_service_installed_check; then
+        step=$((step + 1))
+        poly_service_uninstall_directions "$step"
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+            step=$((step + 1))
+            cat <<EOF
+$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target
+
+  sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+(after this one, you may need to re-open any terminals that were
+opened while it existed.)
+
+EOF
+        fi
+    done
+
+    step=$((step + 1))
+    cat <<EOF
+$step. Delete the files Nix added to your system:
+
+  sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels
+
+and that is it.
+
+EOF
+
+}
+
+nix_user_for_core() {
+    printf "nixbld%d" "$1"
+}
+
+nix_uid_for_core() {
+    echo $((NIX_FIRST_BUILD_UID + $1 - 1))
+}
+
+_textout() {
+    echo -en "$1"
+    shift
+    if [ "$*" = "" ]; then
+        cat
+    else
+        echo "$@"
+    fi
+    echo -en "$ESC"
+}
+
+header() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$BLUE" "$header"
+}
+
+warningheader() {
+    follow="---------------------------------------------------------"
+    header=$(echo "---- $* $follow$follow$follow" | head -c 80)
+    echo ""
+    _textout "$RED" "$header"
+}
+
+subheader() {
+    echo ""
+    _textout "$BLUE_UL" "$*"
+}
+
+row() {
+    printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2"
+}
+
+task() {
+    echo ""
+    ok "~~> $1"
+}
+
+bold() {
+    echo "$BOLD$*$ESC"
+}
+
+ok() {
+    _textout "$GREEN" "$@"
+}
+
+warning() {
+    warningheader "warning!"
+    cat
+    echo ""
+}
+
+failure() {
+    header "oh no!"
+    _textout "$RED" "$@"
+    echo ""
+    _textout "$RED" "$(contactme)"
+    trap finish_cleanup EXIT
+    exit 1
+}
+
+ui_confirm() {
+    _textout "$GREEN$GREEN_UL" "$1"
+
+    if headless; then
+        echo "No TTY, assuming you would say yes :)"
+        return 0
+    fi
+
+    local prompt="[y/n] "
+    echo -n "$prompt"
+    while read -r y; do
+        if [ "$y" = "y" ]; then
+            echo ""
+            return 0
+        elif [ "$y" = "n" ]; then
+            echo ""
+            return 1
+        else
+            _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n"
+            echo -n "$prompt"
+        fi
+    done
+    echo ""
+    return 1
+}
+
+__sudo() {
+    local expl="$1"
+    local cmd="$2"
+    shift
+    header "sudo execution"
+
+    echo "I am executing:"
+    echo ""
+    printf "    $ sudo %s\\n" "$cmd"
+    echo ""
+    echo "$expl"
+    echo ""
+
+    return 0
+}
+
+_sudo() {
+    local expl="$1"
+    shift
+    if ! headless; then
+        __sudo "$expl" "$*"
+    fi
+    sudo "$@"
+}
+
+
+readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX)
+function finish_cleanup {
+    rm -rf "$SCRATCH"
+}
+
+function finish_fail {
+    finish_cleanup
+
+    failure <<EOF
+Jeeze, something went wrong. If you can take all the output and open
+an issue, we'd love to fix the problem so nobody else has this issue.
+
+:(
+EOF
+}
+trap finish_fail EXIT
+
+function finish_success {
+    finish_cleanup
+
+    ok "Alright! We're done!"
+    cat <<EOF
+
+Before Nix will work in your existing shells, you'll need to close
+them and open them again. Other than that, you should be ready to go.
+
+Try it! Open a new terminal, and type:
+
+  $ nix-shell -p nix-info --run "nix-info -m"
+
+Thank you for using this installer. If you have any feedback, don't
+hesitate:
+
+$(contactme)
+EOF
+}
+
+
+validate_starting_assumptions() {
+    poly_validate_assumptions
+
+    if [ $EUID -eq 0 ]; then
+        failure <<EOF
+Please do not run this script with root privileges. We will call sudo
+when we need to.
+EOF
+    fi
+
+    if type nix-env 2> /dev/null >&2; then
+        failure <<EOF
+Nix already appears to be installed, and this tool assumes it is
+_not_ yet installed.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ "${NIX_REMOTE:-}" != "" ]; then
+        failure <<EOF
+For some reason, \$NIX_REMOTE is set. It really should not be set
+before this installer runs, and it hints that Nix is currently
+installed. Please delete the old Nix installation and start again.
+
+Note: You might need to close your shell window and open a new shell
+to clear the variable.
+EOF
+    fi
+
+    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
+        failure <<EOF
+It looks like \$SSL_CERT_FILE is set to a path that used to be part of
+the old Nix installation. Please unset that variable and try again:
+
+  $ unset SSL_CERT_FILE
+
+EOF
+    fi
+
+    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
+        if [ -f "$file" ]; then
+            if grep -l "^[^#].*.nix-profile" "$file"; then
+                failure <<EOF
+I found a reference to a ".nix-profile" in $file.
+This has a high chance of breaking a new nix installation. It was most
+likely put there by a previous Nix installer.
+
+Please remove this reference and try running this again. You should
+also look for similar references in:
+
+ - ~/.bash_profile
+ - ~/.bash_login
+ - ~/.profile
+
+or other shell init files that you may have.
+
+$(uninstall_directions)
+EOF
+            fi
+        fi
+    done
+
+    if [ -d /nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    if [ -d /etc/nix ]; then
+        failure <<EOF
+There are some relics of a previous installation of Nix at /etc/nix, and
+this scripts assumes Nix is _not_ yet installed. Please delete the old
+Nix installation and start again.
+
+$(uninstall_directions)
+EOF
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+        failure <<EOF
+When this script runs, it backs up the current $profile_target to
+$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
+
+Please follow these instructions to clean up the old backup file:
+
+1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
+in case.
+
+2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
+it has anything nix-related in it. If it does, something is probably
+quite wrong. Please open an issue or get in touch immediately.
+
+3. Take care to make sure that $profile_target doesn't look like it has
+anything nix-related in it. If it does, and $profile_target _did not_,
+run:
+
+  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
+
+and try again.
+EOF
+        fi
+
+        if grep -qi "nix" "$profile_target"; then
+            failure <<EOF
+It looks like $profile_target already has some Nix configuration in
+there. There should be no reason to run this again. If you're having
+trouble, please open an issue.
+EOF
+        fi
+    done
+
+    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
+    for danger_path in "${danger_paths[@]}"; do
+        if _sudo "making sure that $danger_path doesn't exist" \
+           test -e "$danger_path"; then
+            failure <<EOF
+I found a file at $danger_path, which is a relic of a previous
+installation. You must first delete this file before continuing.
+
+$(uninstall_directions)
+EOF
+        fi
+    done
+}
+
+setup_report() {
+    header "Nix config report"
+    row "        Temp Dir" "$SCRATCH"
+    row "        Nix Root" "$NIX_ROOT"
+    row "     Build Users" "$NIX_USER_COUNT"
+    row "  Build Group ID" "$NIX_BUILD_GROUP_ID"
+    row "Build Group Name" "$NIX_BUILD_GROUP_NAME"
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then
+        row "Preexisting Install" "Allowed"
+    fi
+
+    subheader "build users:"
+
+    row "    Username" "UID"
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        row "     $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")"
+    done
+    echo ""
+}
+
+create_build_group() {
+    local primary_group_id
+
+    task "Setting up the build group $NIX_BUILD_GROUP_NAME"
+    if ! poly_group_exists "$NIX_BUILD_GROUP_NAME"; then
+        poly_create_build_group
+        row "            Created" "Yes"
+    else
+        primary_group_id=$(poly_group_id_get "$NIX_BUILD_GROUP_NAME")
+        if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then
+            failure <<EOF
+It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
+with the UID $primary_group_id. This script can't really handle
+that right now, so I'm going to give up.
+
+You can fix this by editing this script and changing the
+NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
+to $primary_group_id and re-run.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+}
+
+create_build_user_for_core() {
+    local coreid
+    local username
+    local uid
+
+    coreid="$1"
+    username=$(nix_user_for_core "$coreid")
+    uid=$(nix_uid_for_core "$coreid")
+
+    task "Setting up the build user $username"
+
+    if ! poly_user_exists "$username"; then
+        poly_create_build_user "$username" "$uid" "$coreid"
+        row "           Created" "Yes"
+    else
+        actual_uid=$(poly_user_id_get "$username")
+        if [ "$actual_uid" -ne "$uid" ]; then
+            failure <<EOF
+It seems the build user $username already exists, but with the UID
+with the UID $actual_uid. This script can't really handle that right
+now, so I'm going to give up.
+
+If you already created the users and you know they start from
+$actual_uid and go up from there, you can edit this script and change
+NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try
+again.
+EOF
+        else
+            row "            Exists" "Yes"
+        fi
+    fi
+
+    if [ "$(poly_user_hidden_get "$username")" = "1" ]; then
+        row "            Hidden" "Yes"
+    else
+        poly_user_hidden_set "$username"
+        row "            Hidden" "Yes"
+    fi
+
+    if [ "$(poly_user_home_get "$username")" = "/var/empty" ]; then
+        row "    Home Directory" "/var/empty"
+    else
+        poly_user_home_set "$username" "/var/empty"
+        row "    Home Directory" "/var/empty"
+    fi
+
+    if [ "$(poly_user_note_get "$username")" = "Nix build user $coreid" ]; then
+        row "              Note" "Nix build user $coreid"
+    else
+        poly_user_note_set "$username" "Nix build user $coreid"
+        row "              Note" "Nix build user $coreid"
+    fi
+
+    if [ "$(poly_user_shell_get "$username")" = "/sbin/nologin" ]; then
+        row "   Logins Disabled" "Yes"
+    else
+        poly_user_shell_set "$username" "/sbin/nologin"
+        row "   Logins Disabled" "Yes"
+    fi
+
+    if poly_user_in_group_check "$username" "$NIX_BUILD_GROUP_NAME"; then
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    else
+        poly_user_in_group_set "$username" "$NIX_BUILD_GROUP_NAME"
+        row "  Member of $NIX_BUILD_GROUP_NAME" "Yes"
+    fi
+
+    if [ "$(poly_user_primary_group_get "$username")" = "$NIX_BUILD_GROUP_ID" ]; then
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+    else
+        poly_user_primary_group_set "$username" "$NIX_BUILD_GROUP_ID"
+        row "    PrimaryGroupID" "$NIX_BUILD_GROUP_ID"
+    fi
+}
+
+create_build_users() {
+    for i in $(seq 1 "$NIX_USER_COUNT"); do
+        create_build_user_for_core "$i"
+    done
+}
+
+create_directories() {
+    _sudo "to make the basic directory structure of Nix (part 1)" \
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+
+    _sudo "to make the basic directory structure of Nix (part 2)" \
+          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
+
+    _sudo "to make the basic directory structure of Nix (part 3)" \
+          mkdir -pv -m 1775 /nix/store
+
+    _sudo "to make the basic directory structure of Nix (part 4)" \
+          chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
+
+    _sudo "to set up the root user's profile (part 1)" \
+          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
+
+    _sudo "to set up the root user's profile (part 2)" \
+          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
+
+    _sudo "to place the default nix daemon configuration (part 1)" \
+          mkdir -pv -m 0555 /etc/nix
+}
+
+place_channel_configuration() {
+    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+    _sudo "to set up the default system channel (part 1)" \
+          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+}
+
+welcome_to_nix() {
+    ok "Welcome to the Multi-User Nix Installation"
+
+    cat <<EOF
+
+This installation tool will set up your computer with the Nix package
+manager. This will happen in a few stages:
+
+1. Make sure your computer doesn't already have Nix. If it does, I
+   will show you instructions on how to clean up your old one.
+
+2. Show you what we are going to install and where. Then we will ask
+   if you are ready to continue.
+
+3. Create the system users and groups that the Nix daemon uses to run
+   builds.
+
+4. Perform the basic installation of the Nix files daemon.
+
+5. Configure your shell to import special Nix Profile files, so you
+   can use Nix.
+
+6. Start the Nix daemon.
+
+EOF
+
+    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+        cat <<EOF
+
+We will:
+
+ - make sure your computer doesn't already have Nix files
+   (if it does, I  will tell you how to clean them up.)
+ - create local users (see the list above for the users we'll make)
+ - create a local group ($NIX_BUILD_GROUP_NAME)
+ - install Nix in to $NIX_ROOT
+ - create a configuration file in /etc/nix
+ - set up the "default profile" by creating some Nix-related files in
+   $ROOT_HOME
+EOF
+        for profile_target in "${PROFILE_TARGETS[@]}"; do
+            if [ -e "$profile_target" ]; then
+                cat <<EOF
+ - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX
+ - update $profile_target to include some Nix configuration
+EOF
+            fi
+        done
+        poly_service_setup_note
+        if ! ui_confirm "Ready to continue?"; then
+            failure <<EOF
+Okay, maybe you would like to talk to the team.
+EOF
+        fi
+    fi
+}
+
+chat_about_sudo() {
+    header "let's talk about sudo"
+
+    if headless; then
+        cat <<EOF
+This script is going to call sudo a lot. Normally, it would show you
+exactly what commands it is running and why. However, the script is
+run in a headless fashion, like this:
+
+  $ curl https://nixos.org/nix/install | sh
+
+or maybe in a CI pipeline. Because of that, we're going to skip the
+verbose output in the interest of brevity.
+
+If you would like to
+see the output, try like this:
+
+  $ curl -o install-nix https://nixos.org/nix/install
+  $ sh ./install-nix
+
+EOF
+        return 0
+    fi
+
+    cat <<EOF
+This script is going to call sudo a lot. Every time we do, it'll
+output exactly what it'll do, and why.
+
+Just like this:
+EOF
+
+    __sudo "to demonstrate how our sudo prompts look" \
+           echo "this is a sudo prompt"
+
+    cat <<EOF
+
+This might look scary, but everything can be undone by running just a
+few commands. We used to ask you to confirm each time sudo ran, but it
+was too many times. Instead, I'll just ask you this one time:
+
+EOF
+    if ui_confirm "Can we use sudo?"; then
+        ok "Yay! Thanks! Let's get going!"
+    else
+        failure <<EOF
+That is okay, but we can't install.
+EOF
+    fi
+}
+
+install_from_extracted_nix() {
+    (
+        cd "$EXTRACTED_NIX_PATH"
+
+        _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
+              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+
+        if [ -d "$NIX_INSTALLED_NIX" ]; then
+            echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
+        else
+            failure <<EOF
+Something went wrong, and I didn't find Nix installed at
+$NIX_INSTALLED_NIX.
+EOF
+        fi
+
+        _sudo "to initialize the Nix Database" \
+              $NIX_INSTALLED_NIX/bin/nix-store --init
+
+        cat ./.reginfo \
+            | _sudo "to load data for the first time in to the Nix Database" \
+                   "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
+
+        echo "      Just finished getting the nix database ready."
+    )
+}
+
+shell_source_lines() {
+    cat <<EOF
+
+# Nix
+if [ -e '$PROFILE_NIX_FILE' ]; then
+  . '$PROFILE_NIX_FILE'
+fi
+# End Nix
+
+EOF
+}
+
+configure_shell_profile() {
+    # If there is an /etc/profile.d directory, we want to ensure there
+    # is a nix.sh within it, so we can use the following loop to add
+    # the source lines to it. Note that I'm _not_ adding the source
+    # lines here, because we want to be using the regular machinery.
+    #
+    # If we go around that machinery, it becomes more complicated and
+    # adds complications to the uninstall instruction generator and
+    # old instruction sniffer as well.
+    if [ -d /etc/profile.d ]; then
+        _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \
+              touch /etc/profile.d/nix.sh
+    fi
+
+    for profile_target in "${PROFILE_TARGETS[@]}"; do
+        if [ -e "$profile_target" ]; then
+            _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
+                  cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
+
+            shell_source_lines \
+                | _sudo "extend your $profile_target with nix-daemon settings" \
+                        tee -a "$profile_target"
+        fi
+    done
+}
+
+setup_default_profile() {
+    _sudo "to installing a bootstrapping Nix in to the default Profile" \
+          HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
+
+    _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
+          HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+
+    _sudo "to update the default channel in the default profile" \
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+}
+
+
+place_nix_configuration() {
+    cat <<EOF > "$SCRATCH/nix.conf"
+build-users-group = $NIX_BUILD_GROUP_NAME
+
+max-jobs = $NIX_USER_COUNT
+cores = 1
+sandbox = false
+EOF
+    _sudo "to place the default nix daemon configuration (part 2)" \
+          install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
+}
+
+main() {
+    if [ "$(uname -s)" = "Darwin" ]; then
+        # shellcheck source=./install-darwin-multi-user.sh
+        . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+    elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+        # shellcheck source=./install-systemd-multi-user.sh
+        . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+    else
+        failure "Sorry, I don't know what to do on $(uname)"
+    fi
+
+    welcome_to_nix
+    chat_about_sudo
+
+    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
+        validate_starting_assumptions
+    fi
+
+    setup_report
+
+    if ! ui_confirm "Ready to continue?"; then
+        ok "Alright, no changes have been made :)"
+        contactme
+        trap finish_cleanup EXIT
+        exit 1
+    fi
+
+    create_build_group
+    create_build_users
+    create_directories
+    place_channel_configuration
+    install_from_extracted_nix
+
+    configure_shell_profile
+
+    set +eu
+    . /etc/profile
+    set -eu
+
+    setup_default_profile
+    place_nix_configuration
+    poly_configure_nix_daemon_service
+
+    trap finish_success EXIT
+}
+
+
+main
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 3e5676f419b..8416fb9673b 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -29,8 +29,15 @@ if [ "$(uname -s)" = "Darwin" ]; then
         exit 1
     fi
 
-    printf '\e[1;31mSwitching to the Multi-User Darwin Installer\e[0m\n'
-    exec "$self/install-darwin-multi-user"
+    printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+    exec "$self/install-multi-user"
+    exit 0
+fi
+
+# Linux & Systemd support
+if [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+    printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+    exec "$self/install-multi-user"
     exit 0
 fi
 
diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh
new file mode 100644
index 00000000000..04bc539a109
--- /dev/null
+++ b/scripts/install-systemd-multi-user.sh
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
+readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
+
+readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
+readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
+
+poly_validate_assumptions() {
+    if [ "$(uname -s)" != "Linux" ]; then
+        failure "This script is for use with Linux!"
+    fi
+}
+
+poly_service_installed_check() {
+    [ "$(systemctl is-enabled nix-daemon.service)" = "linked" ] \
+        || [ "$(systemctl is-enabled nix-daemon.socket)" = "enabled" ]
+}
+
+poly_service_uninstall_directions() {
+        cat <<EOF
+$1. Delete the systemd service and socket units
+
+  sudo systemctl stop nix-daemon.socket
+  sudo systemctl stop nix-daemon.service
+  sudo systemctl disable nix-daemon.socket
+  sudo systemctl disable nix-daemon.service
+  sudo systemctl daemon-reload
+EOF
+}
+
+poly_service_setup_note() {
+    cat <<EOF
+ - load and start a service (at $SERVICE_DEST
+   and $SOCKET_DEST) for nix-daemon
+
+EOF
+}
+
+poly_configure_nix_daemon_service() {
+    _sudo "to set up the nix-daemon service" \
+          systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
+
+    _sudo "to set up the nix-daemon socket service" \
+          systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
+
+    _sudo "to load the systemd unit for nix-daemon" \
+          systemctl daemon-reload
+
+    _sudo "to start the nix-daemon.socket" \
+          systemctl start nix-daemon.socket
+
+    _sudo "to start the nix-daemon.service" \
+          systemctl start nix-daemon.service
+
+}
+
+poly_group_exists() {
+    getent group "$1" > /dev/null 2>&1
+}
+
+poly_group_id_get() {
+    getent group "$1" | cut -d: -f3
+}
+
+poly_create_build_group() {
+    _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \
+          groupadd -g "$NIX_BUILD_GROUP_ID" --system \
+          "$NIX_BUILD_GROUP_NAME" >&2
+}
+
+poly_user_exists() {
+    getent passwd "$1" > /dev/null 2>&1
+}
+
+poly_user_id_get() {
+    getent passwd "$1" | cut -d: -f3
+}
+
+poly_user_hidden_get() {
+    echo "1"
+}
+
+poly_user_hidden_set() {
+    true
+}
+
+poly_user_home_get() {
+    getent passwd "$1" | cut -d: -f6
+}
+
+poly_user_home_set() {
+    _sudo "in order to give $1 a safe home directory" \
+          usermod --home "$2" "$1"
+}
+
+poly_user_note_get() {
+    getent passwd "$1" | cut -d: -f5
+}
+
+poly_user_note_set() {
+    _sudo "in order to give $1 a useful comment" \
+          usermod --comment "$2" "$1"
+}
+
+poly_user_shell_get() {
+    getent passwd "$1" | cut -d: -f7
+}
+
+poly_user_shell_set() {
+    _sudo "in order to prevent $1 from logging in" \
+          usermod --shell "$2" "$1"
+}
+
+poly_user_in_group_check() {
+    groups "$1" | grep -q "$2" > /dev/null 2>&1
+}
+
+poly_user_in_group_set() {
+    _sudo "Add $1 to the $2 group"\
+          usermod --append --groups "$2" "$1"
+}
+
+poly_user_primary_group_get() {
+    getent passwd "$1" | cut -d: -f4
+}
+
+poly_user_primary_group_set() {
+    _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \
+          usermod --gid "$2" "$1"
+
+}
+
+poly_create_build_user() {
+    username=$1
+    uid=$2
+    builder_num=$3
+
+    _sudo "Creating the Nix build user, $username" \
+          useradd \
+          --home-dir /var/empty \
+          --comment "Nix build user $builder_num" \
+          --gid "$NIX_BUILD_GROUP_ID" \
+          --groups "$NIX_BUILD_GROUP_NAME" \
+          --no-user-group \
+          --system \
+          --shell /sbin/nologin \
+          --uid "$uid" \
+          --password "!" \
+          "$username"
+}

From 865ca2402f411f0e5819c61203148de54ec5eda1 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Thu, 29 Mar 2018 11:52:08 +0100
Subject: [PATCH 0949/2196] nix copy: add an example with S3

I couldn't find a good example how to use it with non-us-east-1 buckets.
---
 src/nix/copy.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index f29429c1ac4..e4e6c3e303e 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -67,6 +67,12 @@ struct CmdCopy : StorePathsCommand
                 "To copy a closure from another machine via SSH:",
                 "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
             },
+#ifdef ENABLE_S3
+            Example{
+                "To populate the current folder build output to a S3 binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1"
+            },
+#endif
         };
     }
 

From 4ba91f5baefb3d6f187a72505a4a763c35db7b96 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 30 Mar 2018 11:37:32 -0400
Subject: [PATCH 0950/2196] Check for the existence of a profile target before
 seeing if it mentions Nix

Grep would ignore files that didn't exist, but would complain
about files in a directory if the directory didn't exist. Simply check
for the directory first, prior to grepping it.
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 0e9500a98eb..411de31f4cd 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -369,7 +369,7 @@ and try again.
 EOF
         fi
 
-        if grep -qi "nix" "$profile_target"; then
+        if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then
             failure <<EOF
 It looks like $profile_target already has some Nix configuration in
 there. There should be no reason to run this again. If you're having

From f06f8102bdb7c486d1b4723865988dded5575dad Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 30 Mar 2018 11:38:08 -0400
Subject: [PATCH 0951/2196] Use a looser comparison for the 'user note' check

We use grep instead of an equality check because it is difficult
to extract _just_ the user's note, instead it is prefixed with
some plist junk. This was causing the user note to always be set,
even if there was no reason for it.
---
 scripts/install-multi-user.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 411de31f4cd..8a1548a3ce2 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -483,7 +483,11 @@ EOF
         row "    Home Directory" "/var/empty"
     fi
 
-    if [ "$(poly_user_note_get "$username")" = "Nix build user $coreid" ]; then
+    # We use grep instead of an equality check because it is difficult
+    # to extract _just_ the user's note, instead it is prefixed with
+    # some plist junk. This was causing the user note to always be set,
+    # even if there was no reason for it.
+    if ! poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then
         row "              Note" "Nix build user $coreid"
     else
         poly_user_note_set "$username" "Nix build user $coreid"

From 4eb40c72ed21aa4b118811a1c099a5c0dd231145 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 30 Mar 2018 13:57:00 -0400
Subject: [PATCH 0952/2196] macos: Handle when a build user doesn't have a user
 ID

---
 scripts/install-multi-user.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 8a1548a3ce2..5f6542355e0 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -453,10 +453,10 @@ create_build_user_for_core() {
         row "           Created" "Yes"
     else
         actual_uid=$(poly_user_id_get "$username")
-        if [ "$actual_uid" -ne "$uid" ]; then
+        if [ "$actual_uid" != "$uid" ]; then
             failure <<EOF
 It seems the build user $username already exists, but with the UID
-with the UID $actual_uid. This script can't really handle that right
+with the UID '$actual_uid'. This script can't really handle that right
 now, so I'm going to give up.
 
 If you already created the users and you know they start from

From f2273b11f5aec82fd4546aff6ff621aeb397f794 Mon Sep 17 00:00:00 2001
From: Ryan Trinkle <ryan.trinkle@gmail.com>
Date: Fri, 30 Mar 2018 15:41:16 -0400
Subject: [PATCH 0953/2196] Add test eval-okay-nested-with

---
 tests/lang/eval-okay-nested-with.exp | 1 +
 tests/lang/eval-okay-nested-with.nix | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/lang/eval-okay-nested-with.exp
 create mode 100644 tests/lang/eval-okay-nested-with.nix

diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/lang/eval-okay-nested-with.exp
new file mode 100644
index 00000000000..0cfbf08886f
--- /dev/null
+++ b/tests/lang/eval-okay-nested-with.exp
@@ -0,0 +1 @@
+2
diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/lang/eval-okay-nested-with.nix
new file mode 100644
index 00000000000..ba9d79aa79b
--- /dev/null
+++ b/tests/lang/eval-okay-nested-with.nix
@@ -0,0 +1,3 @@
+with { x = 1; };
+with { x = 2; };
+x

From af86132e1afa65b4b466af3ea9c7084836c91ee0 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Tue, 3 Apr 2018 15:27:17 +0300
Subject: [PATCH 0954/2196] libexpr: Make unsafeGetAttrPos not crash on noPos

Currently e.g. `builtins.unsafeGetAttrPos "abort" builtins` will
eventually segfault because pos->file is an unset Symbol.

Found by afl-fuzz.
---
 src/libexpr/eval.cc                           | 2 +-
 tests/lang/eval-okay-getattrpos-undefined.exp | 1 +
 tests/lang/eval-okay-getattrpos-undefined.nix | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 tests/lang/eval-okay-getattrpos-undefined.exp
 create mode 100644 tests/lang/eval-okay-getattrpos-undefined.nix

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 37b977736e2..8548990795a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -628,7 +628,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 
 void EvalState::mkPos(Value & v, Pos * pos)
 {
-    if (pos) {
+    if (pos && pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/lang/eval-okay-getattrpos-undefined.exp
new file mode 100644
index 00000000000..19765bd501b
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-undefined.exp
@@ -0,0 +1 @@
+null
diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/lang/eval-okay-getattrpos-undefined.nix
new file mode 100644
index 00000000000..14dd38f7734
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-undefined.nix
@@ -0,0 +1 @@
+builtins.unsafeGetAttrPos "abort" builtins

From 045eb8440964ab6dbd61c51e831edc9c7bb9ef2a Mon Sep 17 00:00:00 2001
From: Justin Humm <justin.humm@posteo.de>
Date: Tue, 3 Apr 2018 23:26:47 +0200
Subject: [PATCH 0955/2196] document that writeTextFile from Nixpkgs is an
 alternative to builtins.toFile

---
 doc/manual/expressions/builtins.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 47b98460adf..ac1fe7e2faf 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1203,7 +1203,10 @@ in foo</programlisting>
 
     This is not allowed because it would cause a cyclic dependency in
     the computation of the cryptographic hashes for
-    <varname>foo</varname> and <varname>bar</varname>.</para></listitem>
+    <varname>foo</varname> and <varname>bar</varname>.</para>
+    <para>It is also not possible to reference the result of a derivation. 
+    If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to 
+    do that.</para></listitem>
 
   </varlistentry>
 

From d7a84d330cc677d3c9e2c5553a861ab9f82245f6 Mon Sep 17 00:00:00 2001
From: Matthew Justin Bauer <mjbauer95@gmail.com>
Date: Sat, 4 Nov 2017 21:17:09 -0500
Subject: [PATCH 0956/2196] Setup nix_path correctly in nix-profile-daemon

We need nixpkgs to be set in NIX_PATH for Nix 1.12 to work correctly
---
 scripts/nix-profile-daemon.sh.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 43c7156062d..4632099e3c8 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -50,5 +50,5 @@ if test -w $HOME; then
 fi
 
 export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels"
+export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
 export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"

From 5c904e10eb650f5ddd5d27a7259861e102b979e3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Apr 2018 19:32:13 +0200
Subject: [PATCH 0957/2196] <nix/buildenv.nix>: Skip non-directory top-level
 paths

Fixes https://github.com/NixOS/nix/issues/1934 (at least the "error:
opening directory '/nix/store/...-linux-config-4.4.14': Not a
directory" issue).
---
 src/libstore/builtins/buildenv.cc | 50 ++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 938d02c35a0..4f6758c56cc 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -26,12 +26,23 @@ static unsigned long symlinks;
 /* For each activated package, create symlinks */
 static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
 {
-    auto srcFiles = readDirectory(srcDir);
+    DirEntries srcFiles;
+
+    try {
+        srcFiles = readDirectory(srcDir);
+    } catch (SysError & e) {
+        if (e.errNo == ENOTDIR) {
+            printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
+            return;
+        }
+        throw;
+    }
+
     for (const auto & ent : srcFiles) {
         if (ent.name[0] == '.')
             /* not matched by glob */
             continue;
-        const auto & srcFile = srcDir + "/" + ent.name;
+        auto srcFile = srcDir + "/" + ent.name;
         auto dstFile = dstDir + "/" + ent.name;
 
         /* The files below are special-cased to that they don't show up
@@ -44,9 +55,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             hasSuffix(srcFile, "/nix-support") ||
             hasSuffix(srcFile, "/perllocal.pod") ||
             hasSuffix(srcFile, "/info/dir") ||
-            hasSuffix(srcFile, "/log")) {
+            hasSuffix(srcFile, "/log"))
             continue;
-        } else if (isDirectory(srcFile)) {
+
+        else if (isDirectory(srcFile)) {
             struct stat dstSt;
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
@@ -68,7 +80,9 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                 }
             } else if (errno != ENOENT)
                 throw SysError(format("getting status of '%1%'") % dstFile);
-        } else {
+        }
+
+        else {
             struct stat dstSt;
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
@@ -90,6 +104,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             } else if (errno != ENOENT)
                 throw SysError(format("getting status of '%1%'") % dstFile);
         }
+
         createSymlink(srcFile, dstFile);
         priorities[dstFile] = priority;
         symlinks++;
@@ -105,24 +120,18 @@ static Path out;
 
 static void addPkg(const Path & pkgDir, int priority)
 {
-    if (done.find(pkgDir) != done.end())
-        return;
+    if (done.count(pkgDir)) return;
     done.insert(pkgDir);
     createLinks(pkgDir, out, priority);
-    auto propagatedFN = pkgDir + "/nix-support/propagated-user-env-packages";
-    std::string propagated;
-    {
-        AutoCloseFD fd = open(propagatedFN.c_str(), O_RDONLY | O_CLOEXEC);
-        if (!fd) {
-            if (errno == ENOENT)
-                return;
-            throw SysError(format("opening '%1%'") % propagatedFN);
-        }
-        propagated = readFile(fd.get());
+
+    try {
+        for (const auto & p : tokenizeString<std::vector<string>>(
+                readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
+            if (!done.count(p))
+                postponed.insert(p);
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
     }
-    for (const auto & p : tokenizeString<std::vector<string>>(propagated, " \n"))
-        if (done.find(p) == done.end())
-            postponed.insert(p);
 }
 
 struct Package {
@@ -190,4 +199,3 @@ void builtinBuildenv(const BasicDerivation & drv)
 }
 
 }
-

From 1d9742b95f1546cc9fa7bc2fe300443c699f198f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Apr 2018 20:04:42 +0200
Subject: [PATCH 0958/2196] <nix/buildenv.nix>: Ignore dangling symlinks

Fixes

  error: getting status of '/nix/store/j8p0vv89k1pf0cn7kmfsdcs7bshwga1i-firefox-52.7.2esr/share/icons/hicolor/48x48/apps/firefox.png': No such file or directory

https://github.com/NixOS/nix/issues/1934

Also improve error message on directory/non-directory collisions.
---
 src/libstore/builtins/buildenv.cc | 39 +++++++++++++++++--------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 4f6758c56cc..74e70666469 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -9,14 +9,6 @@ namespace nix {
 
 typedef std::map<Path,int> Priorities;
 
-static bool isDirectory(const Path & path)
-{
-    struct stat st;
-    if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting status of '%1%'") % path);
-    return S_ISDIR(st.st_mode);
-}
-
 // FIXME: change into local variables.
 
 static Priorities priorities;
@@ -45,6 +37,18 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
         auto srcFile = srcDir + "/" + ent.name;
         auto dstFile = dstDir + "/" + ent.name;
 
+        struct stat srcSt;
+        try {
+            if (stat(srcFile.c_str(), &srcSt) == -1)
+                throw SysError("getting status of '%1%'", srcFile);
+        } catch (SysError & e) {
+            if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
+                printError("warning: skipping dangling symlink '%s'", dstFile);
+                continue;
+            }
+            throw;
+        }
+
         /* The files below are special-cased to that they don't show up
          * in user profiles, either because they are useless, or
          * because they would cauase pointless collisions (e.g., each
@@ -58,7 +62,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             hasSuffix(srcFile, "/log"))
             continue;
 
-        else if (isDirectory(srcFile)) {
+        else if (S_ISDIR(srcSt.st_mode)) {
             struct stat dstSt;
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
@@ -66,10 +70,9 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                     createLinks(srcFile, dstFile, priority);
                     continue;
                 } else if (S_ISLNK(dstSt.st_mode)) {
-                    auto target = readLink(dstFile);
-                    if (!isDirectory(target))
-                        throw Error(format("collision between '%1%' and non-directory '%2%'")
-                            % srcFile % target);
+                    auto target = canonPath(dstFile, true);
+                    if (!S_ISDIR(lstat(target).st_mode))
+                        throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
                     if (unlink(dstFile.c_str()) == -1)
                         throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
@@ -87,20 +90,20 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISLNK(dstSt.st_mode)) {
-                    auto target = readLink(dstFile);
                     auto prevPriority = priorities[dstFile];
                     if (prevPriority == priority)
-                        throw Error(format(
+                        throw Error(
                                 "packages '%1%' and '%2%' have the same priority %3%; "
                                 "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
                                 "to change the priority of one of the conflicting packages"
-                                " (0 being the highest priority)"
-                                ) % srcFile % target % priority);
+                                " (0 being the highest priority)",
+                                srcFile, readLink(dstFile), priority);
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
                         throw SysError(format("unlinking '%1%'") % dstFile);
-                }
+                } else if (S_ISDIR(dstSt.st_mode))
+                    throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
             } else if (errno != ENOENT)
                 throw SysError(format("getting status of '%1%'") % dstFile);
         }

From 0081a1eac221c996d287240bfe496db26587816b Mon Sep 17 00:00:00 2001
From: Andrew Dunham <andrew@du.nham.ca>
Date: Fri, 6 Apr 2018 00:25:34 -0700
Subject: [PATCH 0959/2196] Use $CPPFLAGS when detecting version of aws-sdk-cpp

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 7177c891443..c41a83c9764 100644
--- a/configure.ac
+++ b/configure.ac
@@ -215,7 +215,7 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
   AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
   AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
 fi

From 2855c3d965bab5e88c4cd08dc3a92e705ff5ac9f Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Mon, 2 Apr 2018 12:45:18 +0100
Subject: [PATCH 0960/2196] Make the TTL for disk cache configurable, we can
 now completely disable disk cache lookup for example by doing:   nix copy
 --from <binary-cahe> <store-path> --option \   positive-disk-cache-ttl 0

Issues: #1885 #2035
---
 src/libstore/globals.hh             |  6 ++++++
 src/libstore/nar-info-disk-cache.cc | 13 +++++--------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 117404ec14c..36c8b735790 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -310,6 +310,12 @@ public:
         "Disabled substituters that may be enabled via the substituters option by untrusted users.",
         {"trusted-binary-caches"}};
 
+    Setting<int> ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl",
+        "The TTL in seconds for negative lookups in the disk cache."};
+
+    Setting<int> ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl",
+        "The TTL in seconds for positive lookups in the disk cache."};
+
     Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
         "Which users or groups are trusted to ask the daemon to do unsafe things."};
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 3c52303f0ea..1e143736f8e 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -1,6 +1,7 @@
 #include "nar-info-disk-cache.hh"
 #include "sync.hh"
 #include "sqlite.hh"
+#include "globals.hh"
 
 #include <sqlite3.h>
 
@@ -47,10 +48,6 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 {
 public:
 
-    /* How long negative and positive lookups are valid. */
-    const int ttlNegative = 3600;
-    const int ttlPositive = 30 * 24 * 3600;
-
     /* How often to purge expired entries from the cache. */
     const int purgeInterval = 24 * 3600;
 
@@ -116,8 +113,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                 SQLiteStmt(state->db,
                     "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
                     .use()
-                    (now - ttlNegative)
-                    (now - ttlPositive)
+                    (now - settings.ttlNegativeDiskCache)
+                    (now - settings.ttlPositiveDiskCache)
                     .exec();
 
                 debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
@@ -186,8 +183,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             auto queryNAR(state->queryNAR.use()
                 (cache.id)
                 (hashPart)
-                (now - ttlNegative)
-                (now - ttlPositive));
+                (now - settings.ttlNegativeDiskCache)
+                (now - settings.ttlPositiveDiskCache));
 
             if (!queryNAR.next())
                 return {oUnknown, 0};

From 86930ed414f9243ca75679b924bf26f5940eb8c9 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Mon, 2 Apr 2018 17:41:49 +0100
Subject: [PATCH 0961/2196] add documentation for the local disk cache TTL
 config

---
 doc/manual/command-ref/conf-file.xml | 24 ++++++++++++++++++++++++
 src/libstore/globals.hh              | 14 ++++++++------
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 6638bf61e45..1033d385e79 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -788,6 +788,30 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-negative-disk-cache-ttl"><term><literal>negative-disk-cache-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for negative lookups. If a store path is queried from a substituer but
+      was not found, there will be a negative lookup cached in the local disk cache database for the specified
+      duration.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+  <varlistentry xml:id="conf-positive-disk-cache-ttl"><term><literal>positive-disk-cache-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for positive lookups. If a store path is queried from a substituer, the result of
+      the query will be cached in the local disk cache database including some of the NAR metadata. Setting a TTL
+      for positive lookups can be useful in case of builds that aren't reproducible, in which case having a more
+      frequent cache invalidation would prevent hash mismatch issues.</para>
+
+    </listitem>
+
+  </varlistentry>
 
 </variablelist>
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 36c8b735790..00147f615f4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -310,15 +310,17 @@ public:
         "Disabled substituters that may be enabled via the substituters option by untrusted users.",
         {"trusted-binary-caches"}};
 
-    Setting<int> ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl",
-        "The TTL in seconds for negative lookups in the disk cache."};
-
-    Setting<int> ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl",
-        "The TTL in seconds for positive lookups in the disk cache."};
-
     Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
         "Which users or groups are trusted to ask the daemon to do unsafe things."};
 
+    Setting<unsigned int> ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl",
+        "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
+        "return an invalid path result"};
+
+    Setting<unsigned int> ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl",
+        "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
+        "return a valid path result."};
+
     /* ?Who we trust to use the daemon in safe ways */
     Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
         "Which users or groups are allowed to connect to the daemon."};

From 33b08899d591125ad7900e7005e2b8862f31bba0 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Tue, 3 Apr 2018 11:33:39 +0100
Subject: [PATCH 0962/2196] re-order the options using the alphabetical order
 and improve the example in the positive lookup case

---
 doc/manual/command-ref/conf-file.xml | 54 +++++++++++++++-------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1033d385e79..a4d411d7075 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -456,6 +456,18 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-negative-disk-cache-ttl"><term><literal>negative-disk-cache-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for negative lookups. If a store path is
+      queried from a substituter but was not found, there will be a
+      negative lookup cached in the local disk cache database for the
+      specified duration.</para>
+
+    </listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
 
@@ -511,6 +523,23 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-positive-disk-cache-ttl"><term><literal>positive-disk-cache-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for positive lookups. If a store path is
+      queried from a substituter, the result of the query will be cached
+      in the local disk cache database including some of the NAR
+      metadata. The default TTL is a month, setting a shorter TTL for
+      positive lookups can be useful for binary caches that have
+      frequent garbage collection, in which case having a more frequent
+      cache invalidation would prevent trying to pull the path again and
+      failing with a hash mismatch if the build isn't reproducible.
+      </para>
+
+    </listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
 
@@ -788,31 +817,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-  <varlistentry xml:id="conf-negative-disk-cache-ttl"><term><literal>negative-disk-cache-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for negative lookups. If a store path is queried from a substituer but
-      was not found, there will be a negative lookup cached in the local disk cache database for the specified
-      duration.</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  <varlistentry xml:id="conf-positive-disk-cache-ttl"><term><literal>positive-disk-cache-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for positive lookups. If a store path is queried from a substituer, the result of
-      the query will be cached in the local disk cache database including some of the NAR metadata. Setting a TTL
-      for positive lookups can be useful in case of builds that aren't reproducible, in which case having a more
-      frequent cache invalidation would prevent hash mismatch issues.</para>
-
-    </listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 </para>

From 62d75ad3e18a6ff8973fb95e7fbc58802c477eac Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Fri, 6 Apr 2018 11:05:15 +0100
Subject: [PATCH 0963/2196] rename the options to mention it's a narinfo TTL as
 disk cache is used all over the place for other operations

---
 src/libstore/globals.hh             | 4 ++--
 src/libstore/nar-info-disk-cache.cc | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 00147f615f4..7430bbedbe4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -313,11 +313,11 @@ public:
     Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
         "Which users or groups are trusted to ask the daemon to do unsafe things."};
 
-    Setting<unsigned int> ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl",
+    Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
         "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
         "return an invalid path result"};
 
-    Setting<unsigned int> ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl",
+    Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
         "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
         "return a valid path result."};
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 1e143736f8e..35403e5df56 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -113,8 +113,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                 SQLiteStmt(state->db,
                     "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
                     .use()
-                    (now - settings.ttlNegativeDiskCache)
-                    (now - settings.ttlPositiveDiskCache)
+                    (now - settings.ttlNegativeNarInfoCache)
+                    (now - settings.ttlPositiveNarInfoCache)
                     .exec();
 
                 debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
@@ -183,8 +183,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             auto queryNAR(state->queryNAR.use()
                 (cache.id)
                 (hashPart)
-                (now - settings.ttlNegativeDiskCache)
-                (now - settings.ttlPositiveDiskCache));
+                (now - settings.ttlNegativeNarInfoCache)
+                (now - settings.ttlPositiveNarInfoCache));
 
             if (!queryNAR.next())
                 return {oUnknown, 0};

From e01b01c5799046a030f24d913ca4f4cbfa997a9c Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Fri, 6 Apr 2018 11:09:52 +0100
Subject: [PATCH 0964/2196] update/re-order the options docs

---
 doc/manual/command-ref/conf-file.xml | 38 ++++++++++++++--------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index a4d411d7075..431c0e6d357 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -456,7 +456,7 @@ builtins.fetchurl {
 
   </varlistentry>
 
-  <varlistentry xml:id="conf-negative-disk-cache-ttl"><term><literal>negative-disk-cache-ttl</literal></term>
+  <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term>
 
     <listitem>
 
@@ -469,6 +469,24 @@ builtins.fetchurl {
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term>
+
+    <listitem>
+
+      <para>The TTL in seconds for positive lookups. If a store path is
+      queried from a substituter, the result of the query will be cached
+      in the local disk cache database including some of the NAR
+      metadata. The default TTL is a month, setting a shorter TTL for
+      positive lookups can be useful for binary caches that have
+      frequent garbage collection, in which case having a more frequent
+      cache invalidation would prevent trying to pull the path again and
+      failing with a hash mismatch if the build isn't reproducible.
+      </para>
+
+    </listitem>
+
+  </varlistentry>
+
   <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term>
 
     <listitem><para>If set to an absolute path to a <filename>netrc</filename>
@@ -523,24 +541,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-  <varlistentry xml:id="conf-positive-disk-cache-ttl"><term><literal>positive-disk-cache-ttl</literal></term>
-
-    <listitem>
-
-      <para>The TTL in seconds for positive lookups. If a store path is
-      queried from a substituter, the result of the query will be cached
-      in the local disk cache database including some of the NAR
-      metadata. The default TTL is a month, setting a shorter TTL for
-      positive lookups can be useful for binary caches that have
-      frequent garbage collection, in which case having a more frequent
-      cache invalidation would prevent trying to pull the path again and
-      failing with a hash mismatch if the build isn't reproducible.
-      </para>
-
-    </listitem>
-
-  </varlistentry>
-
   <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
 
     <listitem>

From 72902ec39959b4adca0161fe8c82ff45bccb4212 Mon Sep 17 00:00:00 2001
From: Doug Beardsley <mightybyte@gmail.com>
Date: Fri, 6 Apr 2018 12:18:59 -0600
Subject: [PATCH 0965/2196] Add missing eval-okay-regex-split.exp test file

---
 tests/lang/eval-okay-regex-split.exp | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tests/lang/eval-okay-regex-split.exp

diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/lang/eval-okay-regex-split.exp
new file mode 100644
index 00000000000..27ba77ddaf6
--- /dev/null
+++ b/tests/lang/eval-okay-regex-split.exp
@@ -0,0 +1 @@
+true

From b828051659cf7cac21577982afd3f9314dc2e68c Mon Sep 17 00:00:00 2001
From: Bogdan Seniuc <bogdan@bogdan.site>
Date: Sat, 7 Apr 2018 10:15:41 +0300
Subject: [PATCH 0966/2196] release.nix: Use $sourceRoot instead of hardcoded
 source directory

---
 release.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 492065df985..b62ca93a3e3 100644
--- a/release.nix
+++ b/release.nix
@@ -28,8 +28,8 @@ let
         configureFlags = "--enable-gc";
 
         postUnpack = ''
-          (cd source && find . -type f) | cut -c3- > source/.dist-files
-          cat source/.dist-files
+          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
+          cat $sourceRoot/.dist-files
         '';
 
         preConfigure = ''

From d8a1c278060db32b9802546c47f4899941eca4ad Mon Sep 17 00:00:00 2001
From: Nicolas Dudebout <nicolas.dudebout@gmail.com>
Date: Sat, 7 Apr 2018 21:10:46 -0400
Subject: [PATCH 0967/2196] fix typo in nix-env man page

---
 doc/manual/command-ref/nix-env.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index d4563ac4755..eac7739558b 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -456,7 +456,7 @@ $ nix-env -f ~/foo.nix -i '.*'</screen>
 from another profile:
 
 <screen>
-$ nix-env -i --from-profile /nix/var/nix/profiles/foo -i gcc</screen>
+$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc</screen>
 
 </para>
 

From 399f43c3d5fe91db3a2d6968d8ec30a4d6fbf285 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Sat, 7 Apr 2018 22:38:45 -0400
Subject: [PATCH 0968/2196] nix-shell: Fixes use with ruby shebangs.

The ported code in 80ebc553eca19dafc64c47420cd49ddd506bc9b7 was incorrectly ported.

```
-            $envCommand = "exec $execArgs $interpreter -e 'load(\"$script\")' -- ${\(join ' ', (map shellEscape, @savedArgs))}";
...
+                    envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str();
```

The single-quote finishing the small ruby snippet was lost in
translation.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index cf628519c6e..a63b3e07ae7 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -212,7 +212,7 @@ void mainWrapped(int argc, char * * argv)
                 // read the shebang to understand which packages to read from. Since
                 // this is handled via nix-shell -p, we wrap our ruby script execution
                 // in ruby -e 'load' which ignores the shebangs.
-                envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str();
+                envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str();
             } else {
                 envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
             }

From 9478f886810b8db87b7ebfe14343f77159c86a24 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Sun, 8 Apr 2018 00:45:28 -0400
Subject: [PATCH 0969/2196] Adds `nix-shell` test for special-cased ruby
 interpreter.

The test fakes the interpreter only to verify the arguments it would be
given.
---
 tests/nix-shell.sh     | 9 +++++++++
 tests/shell.nix        | 7 +++++++
 tests/shell.shebang.rb | 7 +++++++
 3 files changed, 23 insertions(+)
 create mode 100644 tests/shell.shebang.rb

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 063e97ce2c7..7f14b6c31db 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -39,3 +39,12 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh
 
 output=$($TEST_ROOT/shell.shebang.sh abc def)
 [ "$output" = "foo bar abc def" ]
+
+# Test nix-shell shebang mode for ruby
+# This uses a fake interpreter that returns the arguments passed
+# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
+sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+chmod a+rx $TEST_ROOT/shell.shebang.rb
+
+output=$($TEST_ROOT/shell.shebang.rb abc ruby)
+[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
diff --git a/tests/shell.nix b/tests/shell.nix
index 5845d36fc16..eb39f9039a8 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -45,5 +45,12 @@ let pkgs = rec {
 
   bash = shell;
 
+  # ruby "interpreter" that outputs "$@"
+  ruby = runCommand "ruby" {} ''
+    mkdir -p $out/bin
+    echo 'printf -- "$*"' > $out/bin/ruby
+    chmod a+rx $out/bin/ruby
+  '';
+
   inherit pkgs;
 }; in pkgs
diff --git a/tests/shell.shebang.rb b/tests/shell.shebang.rb
new file mode 100644
index 00000000000..ea3953ae21b
--- /dev/null
+++ b/tests/shell.shebang.rb
@@ -0,0 +1,7 @@
+#! @ENV_PROG@ nix-shell
+#! ruby
+#! nix-shell -I nixpkgs=shell.nix --no-substitute
+#! nix-shell --pure -p ruby -i ruby
+
+# Contents doesn't matter.
+abort("This shouldn't be executed.")

From f8ab9cef6ce9f116682f0dab81243cb61294c887 Mon Sep 17 00:00:00 2001
From: Andrew Dunham <andrew@du.nham.ca>
Date: Sun, 8 Apr 2018 18:22:10 -0700
Subject: [PATCH 0970/2196] Fix missing $DESTDIR when installing programs

---
 mk/programs.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mk/programs.mk b/mk/programs.mk
index 3ac64494e3a..2fbda12bd15 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -51,7 +51,7 @@ define build-program
   else
 
     $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	install -t $$($(1)_INSTALL_DIR) $$<
+	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
   endif
 

From 346c0ac3619b8de3d5fc1d76c14d4be9ee20c353 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 9 Apr 2018 08:05:54 -0400
Subject: [PATCH 0971/2196] nix-daemon: Exit successfully when interrupted.

Fixes #2058.
---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index f28a52b73ff..35603af7082 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -1035,7 +1035,7 @@ static void daemonLoop(char * * argv)
             }, options);
 
         } catch (Interrupted & e) {
-            throw;
+            return;
         } catch (Error & e) {
             printError(format("error processing connection: %1%") % e.msg());
         }

From a4c9b2595e6e22250e4e744a35fff3c7f9cd85d9 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Mon, 9 Apr 2018 09:36:54 -0400
Subject: [PATCH 0972/2196] Hardcodes `nix-shell` instead of `/usr/bin/env
 nix-shell`...

... in the ruby shebang test.
---
 tests/nix-shell.sh     | 2 +-
 tests/shell.shebang.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 7f14b6c31db..d25c456cedf 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -43,7 +43,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
diff --git a/tests/shell.shebang.rb b/tests/shell.shebang.rb
index ea3953ae21b..ea67eb09c1c 100644
--- a/tests/shell.shebang.rb
+++ b/tests/shell.shebang.rb
@@ -1,4 +1,4 @@
-#! @ENV_PROG@ nix-shell
+#! @SHELL_PROG@
 #! ruby
 #! nix-shell -I nixpkgs=shell.nix --no-substitute
 #! nix-shell --pure -p ruby -i ruby

From fd98fca7bbcea393957e63f25e36d82ada43e255 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 9 Apr 2018 10:26:50 -0400
Subject: [PATCH 0973/2196] Make prim_exec and prim_importNative available to
 plugins

---
 src/libexpr/primops.cc | 4 ++--
 src/libexpr/primops.hh | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c88f677da08..9e7651da725 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -155,7 +155,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
-static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[0], context);
@@ -193,7 +193,7 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
 
 
 /* Execute a program and parse its output */
-static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceList(*args[0], pos);
     auto elems = args[0]->listElems();
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 31bf3f84f6c..c790b30f6d0 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -15,4 +15,12 @@ struct RegisterPrimOp
     RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
 };
 
+/* These primops are disabled without enableNativeCode, but plugins
+   may wish to use them in limited contexts without globally enabling
+   them. */
+/* Load a ValueInitializer from a DSO and return whatever it initializes */
+void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+/* Execute a program and parse its output */
+void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 }

From a38fe5c1a8ce2481f063ba173eea5226952c803b Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 9 Apr 2018 11:32:43 -0400
Subject: [PATCH 0974/2196] Export required C++ version in pkgconfig.

---
 src/libexpr/nix-expr.pc.in   | 2 +-
 src/libmain/nix-main.pc.in   | 2 +-
 src/libstore/nix-store.pc.in | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in
index 21b6c38dd13..79f3e2f4506 100644
--- a/src/libexpr/nix-expr.pc.in
+++ b/src/libexpr/nix-expr.pc.in
@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix
+Cflags: -I${includedir}/nix -std=c++14
diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in
index de1bdf706f7..38bc85c484e 100644
--- a/src/libmain/nix-main.pc.in
+++ b/src/libmain/nix-main.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix
+Cflags: -I${includedir}/nix -std=c++14
diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in
index 3f1a2d83d2f..4a6b96da200 100644
--- a/src/libstore/nix-store.pc.in
+++ b/src/libstore/nix-store.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil -lnixformat
-Cflags: -I${includedir}/nix
+Cflags: -I${includedir}/nix -std=c++14

From a405d25fa38a3bf342f932100c5077d9d8a16de5 Mon Sep 17 00:00:00 2001
From: John Arnold <IohannesArnold@users.noreply.github.com>
Date: Mon, 9 Apr 2018 16:31:39 -0500
Subject: [PATCH 0975/2196] manual: Add all dependencies to source
 prerequisites (#2060)

---
 doc/manual/installation/prerequisites-source.xml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index 49660c36e39..01e9688d635 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -9,6 +9,9 @@
 <itemizedlist>
 
   <listitem><para>GNU Make.</para></listitem>
+  
+  <listitem><para>Bash Shell. The <literal>./configure</literal> script
+  relies on bashisms, so Bash is required.</para></listitem>
 
   <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem>
 
@@ -28,6 +31,14 @@
   distribution does not provide these, you can obtain bzip2 from <link
   xlink:href="http://www.bzip.org/"/>.</para></listitem>
 
+  <listitem><para><literal>liblzma</literal>, which is provided by
+  XZ Utils. If your distribution does not provide this, you can
+  get it from <link xlink:href="https://tukaani.org/xz/"/>.</para></listitem>
+  
+  <listitem><para>cURL and its library. If your distribution does not
+  provide it, you can get it from <link
+  xlink:href="https://curl.haxx.se/"/>.</para></listitem>
+      
   <listitem><para>The SQLite embedded database library, version 3.6.19
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>

From d6aaa354788a72f792b7bf909b69a216b8634408 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Apr 2018 13:11:43 +0200
Subject: [PATCH 0976/2196] Fix perl-bindings

https://hydra.nixos.org/build/72636781
---
 src/libstore/nix-store.pc.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in
index 4a6b96da200..5cf22faadcb 100644
--- a/src/libstore/nix-store.pc.in
+++ b/src/libstore/nix-store.pc.in
@@ -5,5 +5,5 @@ includedir=@includedir@
 Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
-Libs: -L${libdir} -lnixstore -lnixutil -lnixformat
+Libs: -L${libdir} -lnixstore -lnixutil
 Cflags: -I${includedir}/nix -std=c++14

From f524bcb43d5035eca05b325d4cf7375ca816d619 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Tue, 10 Apr 2018 22:58:25 +0200
Subject: [PATCH 0977/2196] bump version to 2.1

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 415b19fc362..42f7d2336ea 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.0
\ No newline at end of file
+2.1
\ No newline at end of file

From 1839a5542acf4544633e104ecc0a99cbf6b8ba5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Apr 2018 11:42:17 +0200
Subject: [PATCH 0978/2196] Fix #2057

---
 src/nix/repl.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 9216209173d..9782e835aea 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -197,6 +197,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
       }
     }
     input += s;
+    input += '\n';
     return true;
 }
 

From e3cdcf89b0ef42f81c9df5899776ea225f1ecae0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Apr 2018 11:48:42 +0200
Subject: [PATCH 0979/2196] Remove non-existant fedora27i386

---
 release.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/release.nix b/release.nix
index b62ca93a3e3..9778b2db68a 100644
--- a/release.nix
+++ b/release.nix
@@ -206,7 +206,6 @@ let
       };
 
 
-    rpm_fedora27i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora27i386) [ ];
     rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
 
 

From dc0a542c9f8c839b8437b54fcf1c4b47b0cfba10 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 11 Apr 2018 21:02:50 -0400
Subject: [PATCH 0980/2196] initPlugins: Fix dlopen error message.

---
 src/libstore/globals.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index f46e8326235..544566e0b57 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -159,7 +159,7 @@ void initPlugins()
             void *handle =
                 dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
             if (!handle)
-                throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror());
+                throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
         }
     }
     /* We handle settings registrations here, since plugins can add settings */

From 2ef8f0608caa27d1a92d827f54b54155c89092a8 Mon Sep 17 00:00:00 2001
From: Sean Seefried <sean.seefried@gmail.com>
Date: Thu, 12 Apr 2018 19:23:24 +1000
Subject: [PATCH 0981/2196] Add to glossary and clarify garbage collection

While trying to understand garbage collection it was not immediately
clear that only the runtime dependency closure of output paths
would be kept (instead of the build-time dependency closure).
This commit attempts to clarify this by expanding some of the
glossary definitions and extending the Garbage Collection
section.
---
 doc/manual/glossary/glossary.xml           | 49 +++++++++++++++-------
 doc/manual/introduction/about-nix.xml      |  3 +-
 doc/manual/packages/garbage-collection.xml |  8 ++++
 3 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml
index e0636044cc2..a13630d2a4e 100644
--- a/doc/manual/glossary/glossary.xml
+++ b/doc/manual/glossary/glossary.xml
@@ -85,29 +85,48 @@
 
 <glossentry xml:id="gloss-reference"><glossterm>reference</glossterm>
 
-  <glossdef><para>A store path <varname>P</varname> is said to have a
-  reference to a store path <varname>Q</varname> if the store object
-  at <varname>P</varname> contains the path <varname>Q</varname>
-  somewhere.  This implies than an execution involving
-  <varname>P</varname> potentially needs <varname>Q</varname> to be
-  present.  The <emphasis>references</emphasis> of a store path are
-  the set of store paths to which it has a reference.</para></glossdef>
+  <glossdef>
+    <para>A store path <varname>P</varname> is said to have a
+    reference to a store path <varname>Q</varname> if the store object
+    at <varname>P</varname> contains the path <varname>Q</varname>
+    somewhere. The <emphasis>references</emphasis> of a store path are
+    the set of store paths to which it has a reference.
+    </para>
+    <para>A derivation can reference other derivations and sources
+    (but not output paths), whereas an output path only references other
+    output paths.
+    </para>
+  </glossdef>
 
 </glossentry>
 
+<glossentry xml:id="gloss-reachable"><glossterm>reachable</glossterm>
+
+  <glossdef><para>A store path <varname>Q</varname> is reachable from
+  another store path <varname>P</varname> if <varname>Q</varname> is in the
+  <link linked="gloss-closure">closure</link> of the
+  <link linkend="gloss-reference">references</link> relation.
+  </para></glossdef>
+</glossentry>
 
 <glossentry xml:id="gloss-closure"><glossterm>closure</glossterm>
 
   <glossdef><para>The closure of a store path is the set of store
   paths that are directly or indirectly “reachable” from that store
   path; that is, it’s the closure of the path under the <link
-  linkend="gloss-reference">references</link> relation.  For instance,
-  if the store object at path <varname>P</varname> contains a
-  reference to path <varname>Q</varname>, then <varname>Q</varname> is
-  in the closure of <varname>P</varname>.  For correct deployment it
-  is necessary to deploy whole closures, since otherwise at runtime
-  files could be missing.  The command <command>nix-store
-  -qR</command> prints out closures of store paths.</para></glossdef>
+  linkend="gloss-reference">references</link> relation. For a package, the
+  closure of its derivation is equivalent to the build-time
+  dependencies, while the closure of its output path is equivalent to its
+  runtime dependencies. For correct deployment it is necessary to deploy whole
+  closures, since otherwise at runtime files could be missing. The command
+  <command>nix-store -qR</command> prints out closures of store paths.
+  </para>
+  <para>As an example, if the store object at path <varname>P</varname> contains
+  a reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>. Further, if <varname>Q</varname>
+  references <varname>R</varname> then <varname>R</varname> is also in
+  the closure of <varname>P</varname>.
+  </para></glossdef>
 
 </glossentry>
 
@@ -147,7 +166,7 @@
   linkend="sec-profiles" />.</para>
 
   </glossdef>
-  
+
 </glossentry>
 
 
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index 83a2b6786ac..e8c0a29753a 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -60,7 +60,8 @@ This is because tools such as compilers don’t search in per-packages
 directories such as
 <filename>/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include</filename>,
 so if a package builds correctly on your system, this is because you
-specified the dependency explicitly.</para>
+specified the dependency explicitly. This takes care of the build-time
+dependencies.</para>
 
 <para>Once a package is built, runtime dependencies are found by
 scanning binaries for the hash parts of Nix store paths (such as
diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml
index 03b8e4c976c..a1b0ef22a11 100644
--- a/doc/manual/packages/garbage-collection.xml
+++ b/doc/manual/packages/garbage-collection.xml
@@ -52,6 +52,14 @@ garbage collector as follows:
 <screen>
 $ nix-store --gc</screen>
 
+The behaviour of the gargage collector is affected by the <literal>keep-
+derivations</literal> (default: true) and <literal>keep-outputs</literal>
+(default: false) options in the Nix configuration file. The defaults will ensure
+that all derivations that are not build-time dependencies of garbage collector roots
+will be collected but that all output paths that are not runtime dependencies
+will be collected. (This is usually what you want, but while you are developing
+it may make sense to keep outputs to ensure that rebuild times are quick.)
+
 If you are feeling uncertain, you can also first view what files would
 be deleted:
 

From 4fd28bee89a91230ed00fbe842ed51c289f89bd0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Apr 2018 15:12:58 +0200
Subject: [PATCH 0982/2196] Fix broken DocBook

---
 doc/manual/glossary/glossary.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml
index a13630d2a4e..4977825578f 100644
--- a/doc/manual/glossary/glossary.xml
+++ b/doc/manual/glossary/glossary.xml
@@ -104,7 +104,7 @@
 
   <glossdef><para>A store path <varname>Q</varname> is reachable from
   another store path <varname>P</varname> if <varname>Q</varname> is in the
-  <link linked="gloss-closure">closure</link> of the
+  <link linkend="gloss-closure">closure</link> of the
   <link linkend="gloss-reference">references</link> relation.
   </para></glossdef>
 </glossentry>

From d34fa2bcc3572fafc893755cee19d97aed7ec649 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Apr 2018 15:42:35 +0200
Subject: [PATCH 0983/2196] Fix #1921

---
 src/libstore/build.cc       |  2 +-
 src/libstore/local-store.cc | 12 ++++++++++--
 src/libstore/local-store.hh |  4 +++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 73139d6d551..122a754e1e1 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3687,7 +3687,7 @@ void SubstitutionGoal::tryNext()
        only after we've downloaded the path. */
     if (worker.store.requireSigs
         && !sub->isTrusted
-        && !info->checkSignatures(worker.store, worker.store.publicKeys))
+        && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
     {
         printError("warning: substituter '%s' does not have a valid signature for path '%s'",
             sub->getUri(), storePath);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index acc0002acee..b63584f28a3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -53,7 +53,6 @@ LocalStore::LocalStore(const Params & params)
     , trashDir(realStoreDir + "/trash")
     , tempRootsDir(stateDir + "/temproots")
     , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
-    , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());
 
@@ -964,12 +963,21 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 }
 
 
+const PublicKeys & LocalStore::getPublicKeys()
+{
+    auto state(_state.lock());
+    if (!state->publicKeys)
+        state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
+    return *state->publicKeys;
+}
+
+
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     assert(info.narHash);
 
-    if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys))
+    if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
         throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
 
     addTempRoot(info.path);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 0d6c176595c..1209a06356f 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -77,6 +77,8 @@ private:
            minFree but not much below availAfterGC, then there is no
            point in starting a new GC. */
         uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
+
+        std::unique_ptr<PublicKeys> publicKeys;
     };
 
     Sync<State, std::recursive_mutex> _state;
@@ -100,7 +102,7 @@ private:
         settings.requireSigs,
         "require-sigs", "whether store paths should have a trusted signature on import"};
 
-    PublicKeys publicKeys;
+    const PublicKeys & getPublicKeys();
 
 public:
 

From a91fb422fed800227a78b2fe213d8aa2a39b7529 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Mon, 16 Apr 2018 16:09:30 +0200
Subject: [PATCH 0984/2196] nix repl: Fix multiline SIGINT handling.

Fixes #2076
---
 src/nix/repl.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 9782e835aea..f84774a5336 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -189,6 +189,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
     if (!s) {
       switch (auto type = linenoiseKeyType()) {
         case 1: // ctrl-C
+          input = "";
           return true;
         case 2: // ctrl-D
           return false;

From 10d33452e289ded93e192c7d99c1da08a52448e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Tue, 17 Apr 2018 09:13:01 +0200
Subject: [PATCH 0985/2196] =?UTF-8?q?nix-lang=20parser:=C2=A0Add=20mixed?=
 =?UTF-8?q?=20nested=20attrs=20tests.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Part of issue #2077
---
 tests/lang/parse-fail-mixed-nested-attrs.nix   | 4 ++++
 tests/lang/parse-okay-mixed-nested-attrs-1.nix | 4 ++++
 tests/lang/parse-okay-mixed-nested-attrs-2.nix | 4 ++++
 tests/lang/parse-okay-nested-attrs.nix         | 4 ++++
 4 files changed, 16 insertions(+)
 create mode 100644 tests/lang/parse-fail-mixed-nested-attrs.nix
 create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-1.nix
 create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-2.nix
 create mode 100644 tests/lang/parse-okay-nested-attrs.nix

diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs.nix
new file mode 100644
index 00000000000..458a7c545b9
--- /dev/null
+++ b/tests/lang/parse-fail-mixed-nested-attrs.nix
@@ -0,0 +1,4 @@
+{ 
+  x = { y = 3; z = 3; }; 
+  x.z = 3; 
+}
diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix
new file mode 100644
index 00000000000..fd1001c8caf
--- /dev/null
+++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix
@@ -0,0 +1,4 @@
+{ 
+  x = { y = 3; z = 3; }; 
+  x.q = 3; 
+}
diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix
new file mode 100644
index 00000000000..ad066b68038
--- /dev/null
+++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix
@@ -0,0 +1,4 @@
+{ 
+  x.q = 3; 
+  x = { y = 3; z = 3; }; 
+}
diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix
new file mode 100644
index 00000000000..fd1001c8caf
--- /dev/null
+++ b/tests/lang/parse-okay-nested-attrs.nix
@@ -0,0 +1,4 @@
+{ 
+  x = { y = 3; z = 3; }; 
+  x.q = 3; 
+}

From c74f838620053dd914b948b775d4e7b168099348 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Apr 2018 12:03:27 +0200
Subject: [PATCH 0986/2196] exportReferences: Check whether the path is in the
 input closure

Fixes #2082.
---
 src/libstore/build.cc | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 122a754e1e1..e5285ad9a93 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -964,6 +964,8 @@ class DerivationGoal : public Goal
     }
 
     void done(BuildResult::Status status, const string & msg = "");
+
+    PathSet exportReferences(PathSet storePaths);
 };
 
 
@@ -1730,22 +1732,23 @@ int childEntry(void * arg)
 }
 
 
-PathSet exportReferences(Store & store, PathSet storePaths)
+PathSet DerivationGoal::exportReferences(PathSet storePaths)
 {
     PathSet paths;
 
     for (auto storePath : storePaths) {
 
         /* Check that the store path is valid. */
-        if (!store.isInStore(storePath))
+        if (!worker.store.isInStore(storePath))
             throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
                 % storePath);
-        storePath = store.toStorePath(storePath);
-        if (!store.isValidPath(storePath))
-            throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'")
-                % storePath);
 
-        store.computeFSClosure(storePath, paths);
+        storePath = worker.store.toStorePath(storePath);
+
+        if (!inputPaths.count(storePath))
+            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath);
+
+        worker.store.computeFSClosure(storePath, paths);
     }
 
     /* If there are derivations in the graph, then include their
@@ -1756,9 +1759,9 @@ PathSet exportReferences(Store & store, PathSet storePaths)
 
     for (auto & j : paths2) {
         if (isDerivation(j)) {
-            Derivation drv = store.derivationFromPath(j);
+            Derivation drv = worker.store.derivationFromPath(j);
             for (auto & k : drv.outputs)
-                store.computeFSClosure(k.second.path, paths);
+                worker.store.computeFSClosure(k.second.path, paths);
         }
     }
 
@@ -1882,7 +1885,7 @@ void DerivationGoal::startBuilder()
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences(worker.store, {storePath}), false, false));
+                    exportReferences({storePath}), false, false));
         }
     }
 
@@ -2384,7 +2387,7 @@ void DerivationGoal::writeStructuredAttrs()
                     for (auto & p : *i)
                         storePaths.insert(p.get<std::string>());
                     worker.store.pathInfoToJSON(jsonRoot,
-                        exportReferences(worker.store, storePaths), false, true);
+                        exportReferences(storePaths), false, true);
                 }
                 json[i.key()] = nlohmann::json::parse(str.str()); // urgh
             }

From a4aac7f88c59c97299027c9668461c637bbc6a72 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Apr 2018 12:16:04 +0200
Subject: [PATCH 0987/2196] Handle arguments in $EDITOR

Fixes #2079.
---
 src/nix/edit.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 7eaa86e2f91..c9671f76d0f 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -61,7 +61,7 @@ struct CmdEdit : InstallableCommand
 
         auto editor = getEnv("EDITOR", "cat");
 
-        Strings args{editor};
+        auto args = tokenizeString<Strings>(editor);
 
         if (editor.find("emacs") != std::string::npos ||
             editor.find("nano") != std::string::npos ||
@@ -72,7 +72,7 @@ struct CmdEdit : InstallableCommand
 
         stopProgressBar();
 
-        execvp(editor.c_str(), stringsToCharPtrs(args).data());
+        execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("cannot run editor '%s'", editor);
     }

From b37f5ae31dc0a0cbfa7e5bbf6b199e7e39c287ac Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Tue, 17 Apr 2018 14:33:12 -0400
Subject: [PATCH 0988/2196] isFunction: True on primops.

Fixes #2073
---
 src/libexpr/primops.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9e7651da725..57dc7bd1279 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->type == tLambda);
+    bool res;
+    switch (args[0]->type) {
+        case tLambda:
+        case tPrimOp:
+        case tPrimOpApp:
+            res = true;
+            break;
+        default:
+            res = false;
+            break;
+    }
+    mkBool(v, res);
 }
 
 

From 00584bb0915ee21fb01e9390a901161bdd988197 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Wed, 18 Apr 2018 18:39:40 +0200
Subject: [PATCH 0989/2196] parser: Allow mixed nested and top-level attrpaths

Fixes #2077.
---
 src/libexpr/parser.y                          | 26 ++++++++++++++++---
 ...nix => parse-fail-mixed-nested-attrs1.nix} |  2 +-
 tests/lang/parse-fail-mixed-nested-attrs2.nix |  4 +++
 ...attrs-6.nix => parse-okay-dup-attrs-6.nix} |  0
 tests/lang/parse-okay-nested-attrs.nix        |  4 ---
 5 files changed, 28 insertions(+), 8 deletions(-)
 rename tests/lang/{parse-fail-mixed-nested-attrs.nix => parse-fail-mixed-nested-attrs1.nix} (100%)
 create mode 100644 tests/lang/parse-fail-mixed-nested-attrs2.nix
 rename tests/lang/{parse-fail-dup-attrs-6.nix => parse-okay-dup-attrs-6.nix} (100%)
 delete mode 100644 tests/lang/parse-okay-nested-attrs.nix

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index ef11dd60921..df4fdf03247 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -83,7 +83,9 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
     AttrPath::iterator i;
     // All attrpaths have at least one attr
     assert(!attrPath.empty());
-    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
+    // Checking attrPath validity.
+    // ===========================
+    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { 
         if (i->symbol.set()) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
@@ -104,11 +106,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
             attrs = nested;
         }
     }
-    if (i->symbol.set()) {
+    // Expr insertion.
+    // ==========================
+    if (i->symbol.set()) { 
         ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
         if (j != attrs->attrs.end()) {
-            dupAttr(attrPath, pos, j->second.pos);
+            // This attr path is already defined. However, if both
+            // e and the expr pointed by the attr path are two attribute sets,
+            // we want to merge them.
+            // Otherwise, throw an error.
+            ExprAttrs* ae = dynamic_cast<ExprAttrs *>(e);
+            ExprAttrs* jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            if (jAttrs && ae) {
+                for (auto ad: ae->attrs) {
+                    ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first);
+                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
+                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
+                    jAttrs->attrs[ad.first] = ad.second;
+                }
+            } else {
+                dupAttr(attrPath, pos, j->second.pos);
+            }
         } else {
+            // This attr path is not defined. Let's create it.
             attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
             e->setName(i->symbol);
         }
diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix
similarity index 100%
rename from tests/lang/parse-fail-mixed-nested-attrs.nix
rename to tests/lang/parse-fail-mixed-nested-attrs1.nix
index 458a7c545b9..11e40e66fd1 100644
--- a/tests/lang/parse-fail-mixed-nested-attrs.nix
+++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix
@@ -1,4 +1,4 @@
 { 
-  x = { y = 3; z = 3; }; 
   x.z = 3; 
+  x = { y = 3; z = 3; }; 
 }
diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix
new file mode 100644
index 00000000000..17da82e5f0c
--- /dev/null
+++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix
@@ -0,0 +1,4 @@
+{ 
+  x.y.y = 3; 
+  x = { y.y= 3; z = 3; }; 
+}
diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix
similarity index 100%
rename from tests/lang/parse-fail-dup-attrs-6.nix
rename to tests/lang/parse-okay-dup-attrs-6.nix
diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix
deleted file mode 100644
index fd1001c8caf..00000000000
--- a/tests/lang/parse-okay-nested-attrs.nix
+++ /dev/null
@@ -1,4 +0,0 @@
-{ 
-  x = { y = 3; z = 3; }; 
-  x.q = 3; 
-}

From b712d4674bb18dd8c449c97b02e99b3e78bd6dd0 Mon Sep 17 00:00:00 2001
From: Daniel Poelzleithner <poelzleithner@b1-systems.de>
Date: Wed, 18 Apr 2018 21:08:35 +0200
Subject: [PATCH 0990/2196] Allow multiple search experssions in nix search

The common use case is to search for packages containing multiple words
like a "git" "frontend". Having only one expressions makes this simple regular
use case very complicated. Instead, search accepts multiple regular epressions
which all need to match.

nix search git 'gui|frontend'

returns a list of all git uis for example
---
 src/nix/search.cc | 56 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 5ccf1b7cf52..53967669808 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -25,14 +25,14 @@ std::string hilite(const std::string & s, const std::smatch & m)
 
 struct CmdSearch : SourceExprCommand, MixJSON
 {
-    std::string re;
+    std::vector<std::string> res;
 
     bool writeCache = true;
     bool useCache = true;
 
     CmdSearch()
     {
-        expectArg("regex", &re, true);
+        expectArgs("regex", &res);
 
         mkFlag()
             .longName("update-cache")
@@ -68,9 +68,13 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 "nix search blender"
             },
             Example{
-                "To search for Firefox and Chromium:",
+                "To search for Firefox or Chromium:",
                 "nix search 'firefox|chromium'"
             },
+            Example{
+                "To search for git and frontend or gui:",
+                "nix search git 'frontend|gui'"
+            },
         };
     }
 
@@ -81,9 +85,16 @@ struct CmdSearch : SourceExprCommand, MixJSON
         // Empty search string should match all packages
         // Use "^" here instead of ".*" due to differences in resulting highlighting
         // (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
-        if (re.empty()) re = "^";
+        if (res.empty()) {
+            res.push_back("^");
+        }
 
-        std::regex regex(re, std::regex::extended | std::regex::icase);
+        std::vector<std::regex> regexes;
+        regexes.reserve(res.size());
+
+        for (auto &re : res) {
+            regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase));
+        }
 
         auto state = getEvalState();
 
@@ -102,6 +113,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             debug("at attribute '%s'", attrPath);
 
             try {
+                uint found = 0;
 
                 state->forceValue(*v);
 
@@ -115,25 +127,33 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 if (state->isDerivation(*v)) {
 
                     DrvInfo drv(*state, attrPath, v->attrs);
+                    std::string description;
+                    std::smatch attrPathMatch;
+                    std::smatch descriptionMatch;
+                    std::smatch nameMatch;
+                    std::string name;
 
                     DrvName parsed(drv.queryName());
 
-                    std::smatch attrPathMatch;
-                    std::regex_search(attrPath, attrPathMatch, regex);
+                    for (auto &regex : regexes) {
+                        std::regex_search(attrPath, attrPathMatch, regex);
 
-                    auto name = parsed.name;
-                    std::smatch nameMatch;
-                    std::regex_search(name, nameMatch, regex);
+                        name = parsed.name;
+                        std::regex_search(name, nameMatch, regex);
 
-                    std::string description = drv.queryMetaString("description");
-                    std::replace(description.begin(), description.end(), '\n', ' ');
-                    std::smatch descriptionMatch;
-                    std::regex_search(description, descriptionMatch, regex);
+                        description = drv.queryMetaString("description");
+                        std::replace(description.begin(), description.end(), '\n', ' ');
+                        std::regex_search(description, descriptionMatch, regex);
+
+                        if (!attrPathMatch.empty()
+                            || !nameMatch.empty()
+                            || !descriptionMatch.empty())
+                        {
+                            found++;
+                        }
+                    }
 
-                    if (!attrPathMatch.empty()
-                        || !nameMatch.empty()
-                        || !descriptionMatch.empty())
-                    {
+                    if (found == res.size()) {
                         if (json) {
 
                             auto jsonElem = jsonOut->object(attrPath);

From f6e8ceafa68adcdd1f3683beaabff31d55c4d58e Mon Sep 17 00:00:00 2001
From: Daniel Poelzleithner <poelzleithner@b1-systems.de>
Date: Wed, 18 Apr 2018 21:45:46 +0200
Subject: [PATCH 0991/2196] add tests for multi search

---
 tests/search.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/search.sh b/tests/search.sh
index d83427247d3..0b26a125120 100644
--- a/tests/search.sh
+++ b/tests/search.sh
@@ -29,6 +29,11 @@ clearCache
 # Check search that matches nothing
 (( $(nix search nosuchpackageexists | wc -l) == 0 ))
 
+# Search for multiple arguments
+(( $(nix search hello empty | wc -l) == 5 ))
+
+# Multiple arguments will not exist
+(( $(nix search hello broken | wc -l) == 0 ))
 
 ## Search expressions
 

From a99027d5870a772db550f741cb62aee612fcd5da Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Apr 2018 15:28:40 +0200
Subject: [PATCH 0992/2196] Fix tests.evalNixOS

This failed because NixOS' release.nix calls builtins.fetchGit.
---
 release.nix | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 9778b2db68a..37deb8e7ee3 100644
--- a/release.nix
+++ b/release.nix
@@ -268,7 +268,8 @@ let
           export NIX_STATE_DIR=$TMPDIR
           nix-store --init
 
-          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run
+          nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
+            --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
 
           touch $out
         '';

From 17b158af85296c1c815af237cf5626e79b6b0a70 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 19 Apr 2018 13:37:54 -0400
Subject: [PATCH 0993/2196] installer: allow opting in / out to the daemon
 installer

By passing --daemon or --no-daemon, the installer can be forced to
select one or the other installation options, despite what the
automatic detection can provide.

This commit can be backported to 2.0-maintenance because it explicitly
turns off the daemon installation for Linux under systemd.
---
 scripts/install-nix-from-closure.sh | 37 ++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 8416fb9673b..4d01eb589c8 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -28,14 +28,39 @@ if [ "$(uname -s)" = "Darwin" ]; then
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
         exit 1
     fi
-
-    printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
-    exec "$self/install-multi-user"
-    exit 0
 fi
 
-# Linux & Systemd support
-if [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+# Determine if we should punt to the single-user installer or not
+if [ "$(uname -s)" = "Darwin" ]; then
+    INSTALL_MODE=daemon
+elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+    INSTALL_MODE=no-daemon
+else
+    INSTALL_MODE=no-daemon
+fi
+
+# Trivially handle the --daemon / --no-daemon options
+if [ "x${1:-}" = "x--no-daemon" ]; then
+    INSTALL_MODE=no-daemon
+elif [ "x${1:-}" = "x--daemon" ]; then
+    INSTALL_MODE=daemon
+elif [ "x${1:-}" != "x" ]; then
+    (
+        echo "Nix Installer [--daemon|--no-daemon]"
+        echo ""
+        echo " --daemon:    Force the installer to use the Daemon"
+        echo "              based installer, even though it may not"
+        echo "              work."
+        echo ""
+        echo " --no-daemon: Force a no-daemon, single-user"
+        echo "              installation even when the preferred"
+        echo "              method is with the daemon."
+        echo ""
+    ) >&2
+    exit
+fi
+
+if [ "$INSTALL_MODE" = "daemon" ]; then
     printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
     exec "$self/install-multi-user"
     exit 0

From 51cbe99104f45dd688c4c7f1257ac24aa9fe6afb Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 19 Apr 2018 13:40:27 -0400
Subject: [PATCH 0994/2196] installer: default to the daemon installor for
 Linux with systemd

Note: don't backport to 2.0-maintenance
---
 scripts/install-nix-from-closure.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 4d01eb589c8..cd71d7947d7 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -34,7 +34,7 @@ fi
 if [ "$(uname -s)" = "Darwin" ]; then
     INSTALL_MODE=daemon
 elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
-    INSTALL_MODE=no-daemon
+    INSTALL_MODE=daemon
 else
     INSTALL_MODE=no-daemon
 fi

From 6d9129014d443bcec23c6665f00fc020e1a9405b Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 20 Apr 2018 11:55:04 -0500
Subject: [PATCH 0995/2196] build.cc: fix bind-mount of /dev/{pts,ptmx}
 fallback

Don't bind-mount these to themselves,
mount them into the chroot directory.

Fixes pty issues when using sandbox on CentOS 7.4.
(build of perlPackages.IOTty fails before this change)
---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e5285ad9a93..416c775a35d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2699,8 +2699,8 @@ void DerivationGoal::runChild()
                 } else {
                     if (errno != EINVAL)
                         throw SysError("mounting /dev/pts");
-                    doBind("/dev/pts", "/dev/pts");
-                    doBind("/dev/ptmx", "/dev/ptmx");
+                    doBind("/dev/pts", chrootRootDir + "/dev/pts");
+                    doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
                 }
             }
 

From 13d4d1c0a2489a5fab506da57149863daf7f01b1 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 20 Apr 2018 21:07:32 +0200
Subject: [PATCH 0996/2196] download: make hash mismatch error consistent with
 fetchurl

---
 src/libstore/download.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4d7f5690192..538c558f550 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -730,8 +730,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         Hash gotHash = unpack
             ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
             : hashFile(expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'",
-            url, expectedHash.to_string(), gotHash.to_string());
+        throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'",
+            url, gotHash.to_string(), expectedHash.to_string());
     }
 
     return store->toRealPath(storePath);

From 4b3a7f93a58f6f8284af54a6c037c16a9d2204b3 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Fri, 20 Apr 2018 16:34:06 -0400
Subject: [PATCH 0997/2196] doc: Adds `--quiet` to the common options.

Fixes #1298
---
 doc/manual/command-ref/opt-common-syn.xml |  3 +++
 doc/manual/command-ref/opt-common.xml     | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
index 168bef080f4..b610b54b962 100644
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ b/doc/manual/command-ref/opt-common-syn.xml
@@ -8,6 +8,9 @@
     <arg choice='plain'><option>-v</option></arg>
   </group>
 </arg>
+<arg>
+  <arg choice='plain'><option>--quiet</option></arg>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index bcb60b30125..4c572e12944 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -75,6 +75,23 @@
 </varlistentry>
 
 
+<varlistentry><term><option>--quiet</option></term>
+
+  <listitem>
+
+  <para>Decreases the level of verbosity of diagnostic messages
+  printed on standard error.  This is the inverse option to
+  <option>-v</option> / <option>--verbose</option>.
+  </para>
+
+  <para>This option may be specified repeatedly.  See the previous
+  verbosity levels list.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

From 3a918014b248a41fe6307228d258b6113e84138b Mon Sep 17 00:00:00 2001
From: Andrew Dunham <andrew@du.nham.ca>
Date: Sat, 21 Apr 2018 21:10:52 -0700
Subject: [PATCH 0998/2196] Fix library ordering in Makefiles

The existing ordering linked `libutil` before `libstore`, which causes
link failures when building statically. This is due to `libstore` using
functions from `libutil`, and the fact that symbol resolution works
"forward" - i.e. if you pass `-lfoo -lbar -lbaz`, any symbols that
`libbar` uses from `libbaz` will be resolved, but symbols from `libfoo`
will not since it comes first in the command line.

All this to say: this commit reorders the libraries which fixes the link
errors.
---
 src/build-remote/local.mk     | 2 +-
 src/nix-channel/local.mk      | 2 +-
 src/nix-copy-closure/local.mk | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
index 64368a43ff7..50b0409d188 100644
--- a/src/build-remote/local.mk
+++ b/src/build-remote/local.mk
@@ -4,6 +4,6 @@ build-remote_DIR := $(d)
 
 build-remote_INSTALL_DIR := $(libexecdir)/nix
 
-build-remote_LIBS = libmain libutil libformat libstore
+build-remote_LIBS = libmain libformat libstore libutil
 
 build-remote_SOURCES := $(d)/build-remote.cc
diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk
index 49fc105c6f7..c14e8c359ca 100644
--- a/src/nix-channel/local.mk
+++ b/src/nix-channel/local.mk
@@ -2,6 +2,6 @@ programs += nix-channel
 
 nix-channel_DIR := $(d)
 
-nix-channel_LIBS = libmain libutil libformat libstore
+nix-channel_LIBS = libmain libformat libstore libutil
 
 nix-channel_SOURCES := $(d)/nix-channel.cc
diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk
index 42bb34dd820..5018ab975b4 100644
--- a/src/nix-copy-closure/local.mk
+++ b/src/nix-copy-closure/local.mk
@@ -2,6 +2,6 @@ programs += nix-copy-closure
 
 nix-copy-closure_DIR := $(d)
 
-nix-copy-closure_LIBS = libmain libutil libformat libstore
+nix-copy-closure_LIBS = libmain libformat libstore libutil
 
 nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

From 8e7d77d4944d7903f64dc8d61a43f81221cd4aea Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 23 Apr 2018 20:57:05 -0500
Subject: [PATCH 0999/2196] globals.hh: don't use '==' to compare string
 literals

Saw this in logs, also reported here:

https://github.com/NixOS/nix/commit/8e6108ff71caae180d764ab9e9bff5593724655c#r28707288
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0ae69242a59..9360096aae8 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -296,7 +296,7 @@ public:
         "listed in 'trusted-public-keys'."};
 
     Setting<StringSet> extraPlatforms{this,
-        SYSTEM == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
         "Additional platforms that can be built on the local system. "
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "

From 9a714f75af89da5e090f9d3b402b7266d5ec0144 Mon Sep 17 00:00:00 2001
From: Tim Sears <tim@timsears.com>
Date: Sat, 28 Apr 2018 15:34:37 -0700
Subject: [PATCH 1000/2196] add tests for builtins.add

---
 tests/lang/eval-okay-builtins-add.exp | 1 +
 tests/lang/eval-okay-builtins-add.nix | 8 ++++++++
 2 files changed, 9 insertions(+)
 create mode 100644 tests/lang/eval-okay-builtins-add.exp
 create mode 100644 tests/lang/eval-okay-builtins-add.nix

diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/lang/eval-okay-builtins-add.exp
new file mode 100644
index 00000000000..b55e8bce030
--- /dev/null
+++ b/tests/lang/eval-okay-builtins-add.exp
@@ -0,0 +1 @@
+[ 5 4 "int" "tt" "float" 4.0 ]
diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/lang/eval-okay-builtins-add.nix
new file mode 100644
index 00000000000..c841816222a
--- /dev/null
+++ b/tests/lang/eval-okay-builtins-add.nix
@@ -0,0 +1,8 @@
+[
+(builtins.add 2 3)
+(builtins.add 2 2)
+(builtins.typeOf (builtins.add 2  2))
+("t" + "t")
+(builtins.typeOf (builtins.add 2.0 2))
+(builtins.add 2.0 2)
+]

From ac22d77fd16f38a9ce1e3e72e27128e60aa32f81 Mon Sep 17 00:00:00 2001
From: Danylo Hlynskyi <abcz2.uprola@gmail.com>
Date: Tue, 31 Oct 2017 05:12:08 +0200
Subject: [PATCH 1001/2196] nix-collect-garbage: little doc fix

This removes confusing documentation. It's better to remove doc than add implementation, because Nix 1.12 will surely have new GC interface anyway.

Fixes https://github.com/NixOS/nix/issues/641
---
 doc/manual/command-ref/nix-collect-garbage.xml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml
index 35a78c5b201..43e06879691 100644
--- a/doc/manual/command-ref/nix-collect-garbage.xml
+++ b/doc/manual/command-ref/nix-collect-garbage.xml
@@ -22,12 +22,6 @@
     <arg><option>--delete-old</option></arg>
     <arg><option>-d</option></arg>
     <arg><option>--delete-older-than</option> <replaceable>period</replaceable></arg>
-    <group choice='opt'>
-      <arg choice='plain'><option>--print-roots</option></arg>
-      <arg choice='plain'><option>--print-live</option></arg>
-      <arg choice='plain'><option>--print-dead</option></arg>
-      <arg choice='plain'><option>--delete</option></arg>
-    </group>
     <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
     <arg><option>--dry-run</option></arg>
   </cmdsynopsis>

From b2f3a7411a509d5df5a189066870a0f02b6523a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Tue, 1 May 2018 14:42:34 +0200
Subject: [PATCH 1002/2196] nix-lang: Add deep nested mixed attrs test case.

---
 tests/lang/parse-okay-mixed-nested-attrs-3.nix | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-3.nix

diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix
new file mode 100644
index 00000000000..45a33e48037
--- /dev/null
+++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix
@@ -0,0 +1,7 @@
+{
+    services.ssh.enable = true;
+    services.ssh = { port = 123; };
+    services = {
+        httpd.enable = true;
+    };
+}

From 4a2c948943e02af7829a758104044c72b49f9b64 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 May 2018 12:54:30 +0200
Subject: [PATCH 1003/2196] Fix bzip2 compression of files > 4 GiB

Bzip2's 'avail_in' parameter is declared as an unsigned int, so
assigning a size_t length to it led to silent truncation.

Fixes #2111.
---
 src/libutil/compression.cc | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 81cb5e98c76..e1782f8c4bd 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -368,8 +368,21 @@ struct BzipSink : CompressionSink
     }
 
     void write(const unsigned char * data, size_t len) override
+    {
+        /* Bzip2's 'avail_in' parameter is an unsigned int, so we need
+           to split the input into chunks of at most 4 GiB. */
+        while (len) {
+            auto n = std::min((size_t) std::numeric_limits<decltype(strm.avail_in)>::max(), len);
+            writeInternal(data, n);
+            data += n;
+            len -= n;
+        }
+    }
+
+    void writeInternal(const unsigned char * data, size_t len)
     {
         assert(!finished);
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
         strm.next_in = (char *) data;
         strm.avail_in = len;
@@ -475,8 +488,6 @@ struct BrotliSink : CompressionSink
 
     void write(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
         // Don't feed brotli too much at once
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (len) {
@@ -486,7 +497,7 @@ struct BrotliSink : CompressionSink
           len -= n;
         }
     }
-  private:
+
     void writeInternal(const unsigned char * data, size_t len)
     {
         assert(!finished);

From 548ad391d96d8922d0f8c35b5dd72bf4fb7e3ed1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 May 2018 13:49:32 +0200
Subject: [PATCH 1004/2196] Fix builtins.add test

Nix prints the floating point number 4.0 as "4".
---
 tests/lang/eval-okay-builtins-add.exp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/lang/eval-okay-builtins-add.exp
index b55e8bce030..0350b518a7e 100644
--- a/tests/lang/eval-okay-builtins-add.exp
+++ b/tests/lang/eval-okay-builtins-add.exp
@@ -1 +1 @@
-[ 5 4 "int" "tt" "float" 4.0 ]
+[ 5 4 "int" "tt" "float" 4 ]

From 53ec5ac69f59ac435b18297a594933e83e236766 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 May 2018 13:56:34 +0200
Subject: [PATCH 1005/2196] Fix some random -Wconversion warnings

---
 src/libexpr/attr-set.cc     |  8 +++--
 src/libexpr/eval.cc         | 71 +++++++++++++++++++------------------
 src/libexpr/eval.hh         | 18 +++++-----
 src/libexpr/nixexpr.hh      |  2 +-
 src/libexpr/parser.y        | 12 +++----
 src/libexpr/symbol-table.hh |  2 +-
 src/libexpr/value.hh        |  4 +--
 src/libmain/stack.cc        |  2 +-
 src/libstore/build.cc       |  6 ++--
 src/libstore/references.cc  |  6 ++--
 src/libutil/hash.cc         |  2 +-
 src/libutil/serialise.hh    |  4 +--
 src/libutil/xml-writer.cc   |  4 +--
 src/libutil/xml-writer.hh   |  2 +-
 src/nix-env/nix-env.cc      | 16 ++++-----
 src/nix-store/dotgraph.cc   |  3 +-
 16 files changed, 82 insertions(+), 80 deletions(-)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index b284daa3c2f..0474865c6d7 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -24,13 +24,15 @@ static void * allocBytes(size_t n)
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(Bindings::size_t capacity)
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+        throw Error("attribute set of size %d is too big", capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
-void EvalState::mkAttrs(Value & v, unsigned int capacity)
+void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
         v = vEmptySet;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 8548990795a..a2cce162b90 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -422,7 +422,7 @@ Value * EvalState::addConstant(const string & name, Value & v)
 
 
 Value * EvalState::addPrimOp(const string & name,
-    unsigned int arity, PrimOpFun primOp)
+    size_t arity, PrimOpFun primOp)
 {
     if (arity == 0) {
         Value v;
@@ -528,7 +528,7 @@ Value & mkString(Value & v, const string & s, const PathSet & context)
 {
     mkString(v, s.c_str());
     if (!context.empty()) {
-        unsigned int n = 0;
+        size_t n = 0;
         v.string.context = (const char * *)
             allocBytes((context.size() + 1) * sizeof(char *));
         for (auto & i : context)
@@ -547,7 +547,7 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (unsigned int l = var.level; l; --l, env = env->up) ;
+    for (size_t l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
@@ -566,7 +566,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         if (!env->prevWith)
             throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
-        for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
 
@@ -578,14 +578,15 @@ Value * EvalState::allocValue()
 }
 
 
-Env & EvalState::allocEnv(unsigned int size)
+Env & EvalState::allocEnv(size_t size)
 {
-    assert(size <= std::numeric_limits<decltype(Env::size)>::max());
+    if (size > std::numeric_limits<decltype(Env::size)>::max())
+        throw Error("environment size %d is too big", size);
 
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = size;
+    env->size = (decltype(Env::size)) size;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -593,7 +594,7 @@ Env & EvalState::allocEnv(unsigned int size)
 }
 
 
-void EvalState::mkList(Value & v, unsigned int size)
+void EvalState::mkList(Value & v, size_t size)
 {
     clearValue(v);
     if (size == 1)
@@ -805,7 +806,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        unsigned int displ = 0;
+        size_t displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -879,7 +880,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    unsigned int displ = 0;
+    size_t displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -890,7 +891,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (unsigned int n = 0; n < elems.size(); ++n)
+    for (size_t n = 0; n < elems.size(); ++n)
         v.listElems()[n] = elems[n]->maybeThunk(state, env);
 }
 
@@ -1012,22 +1013,22 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v)
 void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
     /* Figure out the number of arguments still needed. */
-    unsigned int argsDone = 0;
+    size_t argsDone = 0;
     Value * primOp = &fun;
     while (primOp->type == tPrimOpApp) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
     assert(primOp->type == tPrimOp);
-    unsigned int arity = primOp->primOp->arity;
-    unsigned int argsLeft = arity - argsDone;
+    auto arity = primOp->primOp->arity;
+    auto argsLeft = arity - argsDone;
 
     if (argsLeft == 1) {
         /* We have all the arguments, so call the primop. */
 
         /* Put all the arguments in an array. */
         Value * vArgs[arity];
-        unsigned int n = arity - 1;
+        auto n = arity - 1;
         vArgs[n--] = &arg;
         for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
@@ -1076,13 +1077,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     ExprLambda & lambda(*fun.lambda.fun);
 
-    unsigned int size =
+    auto size =
         (lambda.arg.empty() ? 0 : 1) +
         (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
-    unsigned int displ = 0;
+    size_t displ = 0;
 
     if (!lambda.matchAttrs)
         env2.values[displ++] = &arg;
@@ -1096,7 +1097,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         /* For each formal argument, get the actual argument.  If
            there is no matching actual argument but the formal
            argument has a default, use the default. */
-        unsigned int attrsUsed = 0;
+        size_t attrsUsed = 0;
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
@@ -1294,15 +1295,15 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos)
+void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos)
 {
     nrListConcats++;
 
     Value * nonEmpty = 0;
-    unsigned int len = 0;
-    for (unsigned int n = 0; n < nrLists; ++n) {
+    size_t len = 0;
+    for (size_t n = 0; n < nrLists; ++n) {
         forceList(*lists[n], pos);
-        unsigned int l = lists[n]->listSize();
+        auto l = lists[n]->listSize();
         len += l;
         if (l) nonEmpty = lists[n];
     }
@@ -1314,8 +1315,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co
 
     mkList(v, len);
     auto out = v.listElems();
-    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        unsigned int l = lists[n]->listSize();
+    for (size_t n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n]->listSize();
         if (l)
             memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
@@ -1410,7 +1411,7 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (unsigned int n = 0; n < v.listSize(); ++n)
+            for (size_t n = 0; n < v.listSize(); ++n)
                 recurse(*v.listElems()[n]);
         }
     };
@@ -1562,7 +1563,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (size_t n = 0; n < v.listSize(); ++n) {
                 result += coerceToString(pos, *v.listElems()[n],
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
@@ -1649,7 +1650,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         case tList2:
         case tListN:
             if (v1.listSize() != v2.listSize()) return false;
-            for (unsigned int n = 0; n < v1.listSize(); ++n)
+            for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
@@ -1740,26 +1741,26 @@ void EvalState::printStats()
         v = lvlInfo;
 
         printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
-        typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
+        typedef std::multimap<size_t, Symbol> PrimOpCalls_;
         PrimOpCalls_ primOpCalls_;
         for (auto & i : primOpCalls)
-            primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first));
+            primOpCalls_.insert(std::pair<size_t, Symbol>(i.second, i.first));
         for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
             printMsg(v, format("%1$10d %2%") % i->first % i->second);
 
         printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_;
+        typedef std::multimap<size_t, ExprLambda *> FunctionCalls_;
         FunctionCalls_ functionCalls_;
         for (auto & i : functionCalls)
-            functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first));
+            functionCalls_.insert(std::pair<size_t, ExprLambda *>(i.second, i.first));
         for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
             printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
 
         printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
-        typedef std::multimap<unsigned int, Pos> AttrSelects_;
+        typedef std::multimap<size_t, Pos> AttrSelects_;
         AttrSelects_ attrSelects_;
         for (auto & i : attrSelects)
-            attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first));
+            attrSelects_.insert(std::pair<size_t, Pos>(i.second, i.first));
         for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
             printMsg(v, format("%1$10d %2%") % i->first % i->second);
 
@@ -1810,7 +1811,7 @@ size_t valueSize(Value & v)
             if (seen.find(v.listElems()) == seen.end()) {
                 seen.insert(v.listElems());
                 sz += v.listSize() * sizeof(Value *);
-                for (unsigned int n = 0; n < v.listSize(); ++n)
+                for (size_t n = 0; n < v.listSize(); ++n)
                     sz += doValue(*v.listElems()[n]);
             }
             break;
@@ -1846,7 +1847,7 @@ size_t valueSize(Value & v)
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
-        for (unsigned int i = 0; i < env.size; ++i)
+        for (size_t i = 0; i < env.size; ++i)
             if (env.values[i])
                 sz += doValue(*env.values[i]);
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9d8799b7906..86e93a5ac9c 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -214,7 +214,7 @@ private:
     Value * addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
-        unsigned int arity, PrimOpFun primOp);
+        size_t arity, PrimOpFun primOp);
 
 public:
 
@@ -248,18 +248,18 @@ public:
 
     /* Allocation primitives. */
     Value * allocValue();
-    Env & allocEnv(unsigned int size);
+    Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
 
-    Bindings * allocBindings(Bindings::size_t capacity);
+    Bindings * allocBindings(size_t capacity);
 
-    void mkList(Value & v, unsigned int length);
-    void mkAttrs(Value & v, unsigned int capacity);
+    void mkList(Value & v, size_t length);
+    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    void concatLists(Value & v, unsigned int nrLists, Value * * lists, const Pos & pos);
+    void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
     void printStats();
@@ -282,15 +282,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<Symbol, unsigned int> PrimOpCalls;
+    typedef std::map<Symbol, size_t> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, unsigned int> FunctionCalls;
+    typedef std::map<ExprLambda *, size_t> FunctionCalls;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<Pos, unsigned int> AttrSelects;
+    typedef std::map<Pos, size_t> AttrSelects;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 8c8f3964068..b486595f07a 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -255,7 +255,7 @@ struct ExprWith : Expr
 {
     Pos pos;
     Expr * attrs, * body;
-    unsigned int prevWith;
+    size_t prevWith;
     ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     COMMON_METHODS
 };
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index ef11dd60921..e3f4521844e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -136,8 +136,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
        whitespace-only final lines are not taken into account.  (So
        the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
-    unsigned int minIndent = 1000000;
-    unsigned int curIndent = 0;
+    size_t minIndent = 1000000;
+    size_t curIndent = 0;
     for (auto & i : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
@@ -148,7 +148,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
             }
             continue;
         }
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ')
                     curIndent++;
@@ -170,8 +170,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     /* Strip spaces from each line. */
     vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
-    unsigned int curDropped = 0;
-    unsigned int n = es.size();
+    size_t curDropped = 0;
+    size_t n = es.size();
     for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
         if (!e) {
@@ -182,7 +182,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
         }
 
         string s2;
-        for (unsigned int j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
                 if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index c2ee49dd32f..44929f7eea0 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -69,7 +69,7 @@ public:
         return Symbol(&*res.first);
     }
 
-    unsigned int size() const
+    size_t size() const
     {
         return symbols.size();
     }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 9df516f062e..66b41a15840 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -128,7 +128,7 @@ struct Value
         const char * path;
         Bindings * attrs;
         struct {
-            unsigned int size;
+            size_t size;
             Value * * elems;
         } bigList;
         Value * smallList[2];
@@ -166,7 +166,7 @@ struct Value
         return type == tList1 || type == tList2 ? smallList : bigList.elems;
     }
 
-    unsigned int listSize() const
+    size_t listSize() const
     {
         return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index cc0eea68fca..13896aeecb6 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -30,7 +30,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
         if (diff < 0) diff = -diff;
         if (diff < 4096) {
             char msg[] = "error: stack overflow (possible infinite recursion)\n";
-            [[gnu::unused]] int res = write(2, msg, strlen(msg));
+            [[gnu::unused]] auto res = write(2, msg, strlen(msg));
             _exit(1); // maybe abort instead?
         }
     }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6108785447a..a2b24524028 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -842,9 +842,9 @@ class DerivationGoal : public Goal
     BuildResult result;
 
     /* The current round, if we're building multiple times. */
-    unsigned int curRound = 1;
+    size_t curRound = 1;
 
-    unsigned int nrRounds;
+    size_t nrRounds;
 
     /* Path registration info from the previous round, if we're
        building multiple times. Since this contains the hash, it
@@ -1169,7 +1169,7 @@ void DerivationGoal::outputsSubstituted()
         return;
     }
 
-    unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
+    auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
     if (buildMode == bmNormal && nrInvalid == 0) {
         done(BuildResult::Substituted);
         return;
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index ba9f18b9ca5..5b7eb1f846a 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -13,7 +13,7 @@ namespace nix {
 static unsigned int refLength = 32; /* characters */
 
 
-static void search(const unsigned char * s, unsigned int len,
+static void search(const unsigned char * s, size_t len,
     StringSet & hashes, StringSet & seen)
 {
     static bool initialised = false;
@@ -25,7 +25,7 @@ static void search(const unsigned char * s, unsigned int len,
         initialised = true;
     }
 
-    for (unsigned int i = 0; i + refLength <= len; ) {
+    for (size_t i = 0; i + refLength <= len; ) {
         int j;
         bool match = true;
         for (j = refLength - 1; j >= 0; --j)
@@ -73,7 +73,7 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
 
     search(data, len, hashes, seen);
 
-    unsigned int tailLen = len <= refLength ? len : refLength;
+    size_t tailLen = len <= refLength ? len : refLength;
     tail =
         string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
         string((const char *) data + len - tailLen, tailLen);
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index a01d651e1ef..9d82f13a5e3 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -82,7 +82,7 @@ static string printHash32(const Hash & hash)
     string s;
     s.reserve(len);
 
-    for (int n = len - 1; n >= 0; n--) {
+    for (int n = (int) len - 1; n >= 0; n--) {
         unsigned int b = n * 5;
         unsigned int i = b / 8;
         unsigned int j = b % 8;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index d0b4552e339..6e703c52a1e 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -227,7 +227,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
     buf[4] = (n >> 32) & 0xff;
     buf[5] = (n >> 40) & 0xff;
     buf[6] = (n >> 48) & 0xff;
-    buf[7] = (n >> 56) & 0xff;
+    buf[7] = (unsigned char) (n >> 56) & 0xff;
     sink(buf, sizeof(buf));
     return sink;
 }
@@ -259,7 +259,7 @@ T readNum(Source & source)
     if (n > std::numeric_limits<T>::max())
         throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
 
-    return n;
+    return (T) n;
 }
 
 
diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc
index 98bd058d18b..e5cc2e9fc71 100644
--- a/src/libutil/xml-writer.cc
+++ b/src/libutil/xml-writer.cc
@@ -28,7 +28,7 @@ void XMLWriter::close()
 }
 
 
-void XMLWriter::indent_(unsigned int depth)
+void XMLWriter::indent_(size_t depth)
 {
     if (!indent) return;
     output << string(depth * 2, ' ');
@@ -75,7 +75,7 @@ void XMLWriter::writeAttrs(const XMLAttrs & attrs)
 {
     for (auto & i : attrs) {
         output << " " << i.first << "=\"";
-        for (unsigned int j = 0; j < i.second.size(); ++j) {
+        for (size_t j = 0; j < i.second.size(); ++j) {
             char c = i.second[j];
             if (c == '"') output << "&quot;";
             else if (c == '<') output << "&lt;";
diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh
index 3cefe3712c0..b98b445265a 100644
--- a/src/libutil/xml-writer.hh
+++ b/src/libutil/xml-writer.hh
@@ -44,7 +44,7 @@ public:
 private:
     void writeAttrs(const XMLAttrs & attrs);
 
-    void indent_(unsigned int depth);
+    void indent_(size_t depth);
 };
 
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 97e66cbd937..f60ff9e0718 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -198,13 +198,13 @@ static Path getDefNixExprPath()
 }
 
 
-static int getPriority(EvalState & state, DrvInfo & drv)
+static long getPriority(EvalState & state, DrvInfo & drv)
 {
     return drv.queryMetaInt("priority", 0);
 }
 
 
-static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
+static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
 {
     return getPriority(state, drv2) - getPriority(state, drv1);
 }
@@ -270,7 +270,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
 
             for (auto & j : matches) {
                 DrvName drvName(j.first.queryName());
-                int d = 1;
+                long d = 1;
 
                 Newest::iterator k = newest.find(drvName.name);
 
@@ -578,7 +578,7 @@ static void upgradeDerivations(Globals & globals,
                             (upgradeType == utEq && d == 0) ||
                             upgradeType == utAlways)
                         {
-                            int d2 = -1;
+                            long d2 = -1;
                             if (bestElem != availElems.end()) {
                                 d2 = comparePriorities(*globals.state, *bestElem, *j);
                                 if (d2 == 0) d2 = compareVersions(bestVersion, newName.version);
@@ -784,22 +784,22 @@ typedef list<Strings> Table;
 
 void printTable(Table & table)
 {
-    unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0;
+    auto nrColumns = table.size() > 0 ? table.front().size() : 0;
 
-    vector<unsigned int> widths;
+    vector<size_t> widths;
     widths.resize(nrColumns);
 
     for (auto & i : table) {
         assert(i.size() == nrColumns);
         Strings::iterator j;
-        unsigned int column;
+        size_t column;
         for (j = i.begin(), column = 0; j != i.end(); ++j, ++column)
             if (j->size() > widths[column]) widths[column] = j->size();
     }
 
     for (auto & i : table) {
         Strings::iterator j;
-        unsigned int column;
+        size_t column;
         for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) {
             string s = *j;
             replace(s.begin(), s.end(), '\n', ' ');
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index 51dedcf0a09..abdfa5e58f9 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -47,8 +47,7 @@ static string makeNode(const string & id, const string & label,
 static string symbolicName(const string & path)
 {
     string p = baseNameOf(path);
-    int dash = p.find('-');
-    return string(p, dash + 1);
+    return string(p, p.find('-') + 1);
 }
 
 

From 5ba6395378e6e92266bba0dd46e3cf3ab6e2fce8 Mon Sep 17 00:00:00 2001
From: Gleb Peregud <gleber.p@gmail.com>
Date: Wed, 2 May 2018 22:07:23 +0200
Subject: [PATCH 1006/2196] docs: mention source of env variables used by
 `impureEnvVars`.

---
 doc/manual/expressions/advanced-attributes.xml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index f3cf9837130..dfd013b5cf3 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -112,7 +112,13 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
     linkend="fixed-output-drvs">fixed-output derivations</link>, where
     impurities such as these are okay since (the hash of) the output
     is known in advance.  It is ignored for all other
-    derivations.</para></listitem>
+    derivations.</para>
+
+    <warning><para><varname>impureEnvVars</varname> implementation takes
+    environment variables from the current builder process. When a daemon is
+    building its environmental variables are used. Without the daemon, the
+    environmental variables come from the environment of the
+    <command>nix-build</command>.</para></warning></listitem>
 
   </varlistentry>
 

From 80a7b16593951791c4a85053a7cc14fc45260cee Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 May 2018 11:02:40 +0200
Subject: [PATCH 1007/2196] LocalStore::addValidPath(): Throw error when the CA
 assertion is wrong

Closes #2133.
---
 src/libstore/local-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b63584f28a3..ef8c2811bd8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -581,7 +581,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    assert(info.ca == "" || info.isContentAddressed(*this));
+    if (info.ca != "" && !info.isContentAddressed(*this))
+        throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path);
 
     state.stmtRegisterValidPath.use()
         (info.path)

From c717d8e3bd96db9a597e11f067585cf1aee50a40 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 3 May 2018 14:48:28 +0200
Subject: [PATCH 1008/2196] Don't set a CA assertion for paths with references

Really fixes #2133.
---
 src/libstore/build.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a2b24524028..f70ab8108fd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3249,6 +3249,8 @@ void DerivationGoal::registerOutputs()
         info.ultimate = true;
         worker.store.signPathInfo(info);
 
+        if (!info.references.empty()) info.ca.clear();
+
         infos.push_back(info);
     }
 

From a1adcdf0878501b512d7ab98d713167f035a330d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 7 May 2018 14:58:25 +0200
Subject: [PATCH 1009/2196] nix-shell: Only download requested outputs

Fixes #2116.
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index a63b3e07ae7..21d99878a51 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -354,7 +354,7 @@ void mainWrapped(int argc, char * * argv)
         // Build or fetch all dependencies of the derivation.
         for (const auto & input : drv.inputDrvs)
             if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); }))
-                pathsToBuild.insert(input.first);
+                pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second));
         for (const auto & src : drv.inputSrcs)
             pathsToBuild.insert(src);
 

From 129394fb953475888173de5d8f864de3a2569e0e Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Mon, 7 May 2018 14:23:51 +0100
Subject: [PATCH 1010/2196] Support multi-part uploads for large NARs that
 exceed the size of 5Gb.

---
 release-common.nix                    |  2 +-
 src/libstore/local.mk                 |  2 +-
 src/libstore/s3-binary-cache-store.cc | 80 +++++++++++++++++++++------
 3 files changed, 66 insertions(+), 18 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index 0c12bc7ceb3..d7fb8125f25 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -61,7 +61,7 @@ rec {
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {
-        apis = ["s3"];
+        apis = ["s3" "transfer"];
         customMemoryManagement = false;
       });
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index a7279aa3939..3799257f83f 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -18,7 +18,7 @@ libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
 
 ifeq ($(OS), SunOS)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 23af452094c..b55ff5c6d5d 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -17,6 +17,7 @@
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/FormattedLogSystem.h>
 #include <aws/core/utils/logging/LogMacros.h>
+#include <aws/core/utils/threading/Executor.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetBucketLocationRequest.h>
@@ -24,6 +25,9 @@
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <aws/transfer/TransferManager.h>
+
+using namespace Aws::Transfer;
 
 namespace nix {
 
@@ -169,6 +173,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    const Setting<uint64_t> bufferSize{
+        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads. defaults to 5Mb"};
 
     std::string bucketName;
 
@@ -271,34 +277,76 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
-        auto request =
-            Aws::S3::Model::PutObjectRequest()
-            .WithBucket(bucketName)
-            .WithKey(path);
+        auto stream = std::make_shared<istringstream_nocopy>(data);
 
-        request.SetContentType(mimeType);
+        auto maxThreads = std::thread::hardware_concurrency();
 
-        if (contentEncoding != "")
-            request.SetContentEncoding(contentEncoding);
+        auto executor =
+            std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
-        auto stream = std::make_shared<istringstream_nocopy>(data);
+        TransferManagerConfiguration transferConfig(executor.get());
 
-        request.SetBody(stream);
+        transferConfig.s3Client = s3Helper.client;
+        transferConfig.bufferSize = bufferSize;
 
-        stats.put++;
-        stats.putBytes += data.size();
+        if (contentEncoding != "")
+            transferConfig.createMultipartUploadTemplate.SetContentEncoding(
+                contentEncoding);
+
+        transferConfig.uploadProgressCallback =
+            [&](const TransferManager *transferManager,
+                const std::shared_ptr<const TransferHandle>
+                    &transferHandle)
+            {
+              checkInterrupt();
+              printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
+                             path,
+                             transferHandle->GetBytesTransferred(),
+                             transferHandle->GetBytesTotalSize());
+            };
+
+        transferConfig.transferStatusUpdatedCallback =
+            [&](const TransferManager *,
+                const std::shared_ptr<const TransferHandle>
+                    &transferHandle) {
+              switch (transferHandle->GetStatus()) {
+                  case TransferStatus::COMPLETED:
+                      printTalkative("upload of '%s' completed", path);
+                      stats.put++;
+                      stats.putBytes += data.size();
+                      break;
+                  case TransferStatus::IN_PROGRESS:
+                      break;
+                  case TransferStatus::FAILED:
+                      throw Error("AWS error: failed to upload 's3://%s/%s'",
+                                  bucketName, path);
+                      break;
+                  default:
+                      throw Error("AWS error: transfer status of 's3://%s/%s' "
+                                  "in unexpected state",
+                                  bucketName, path);
+              };
+            };
+
+        std::shared_ptr<TransferManager> transferManager =
+            TransferManager::Create(transferConfig);
 
         auto now1 = std::chrono::steady_clock::now();
 
-        auto result = checkAws(format("AWS error uploading '%s'") % path,
-            s3Helper.client->PutObject(request));
+        std::shared_ptr<TransferHandle> transferHandle =
+            transferManager->UploadFile(stream, bucketName, path, mimeType,
+                                        Aws::Map<Aws::String, Aws::String>());
+
+        transferHandle->WaitUntilFinished();
 
         auto now2 = std::chrono::steady_clock::now();
 
-        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+        auto duration =
+            std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
+                .count();
 
-        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms")
-            % bucketName % path % data.size() % duration);
+        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
+                  bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
     }

From 591e75cd01f1650aab5538432a8637683ad1a9d0 Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Mon, 7 May 2018 14:27:53 +0100
Subject: [PATCH 1011/2196] add a FIXME note to find a way to abort the
 multipart uploads in case the nix command is interrupted.

---
 src/libstore/s3-binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index b55ff5c6d5d..37da44da873 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -296,8 +296,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         transferConfig.uploadProgressCallback =
             [&](const TransferManager *transferManager,
                 const std::shared_ptr<const TransferHandle>
-                    &transferHandle)
-            {
+                    &transferHandle) {
+              //FIXME: find a way to properly abort the multipart upload.
               checkInterrupt();
               printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
                              path,

From 854c0860f4ec68e81869473cf0708529afd0486e Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Mon, 7 May 2018 15:07:00 +0100
Subject: [PATCH 1012/2196] share the executor between multiple copy threads.

---
 src/libstore/s3-binary-cache-store.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 37da44da873..96673a5b0cc 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -281,8 +281,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
         auto maxThreads = std::thread::hardware_concurrency();
 
-        auto executor =
-            std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
+        static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
+            executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
         TransferManagerConfiguration transferConfig(executor.get());
 

From 1e20ed5a0c00623409d25534dd00ea707b2c7dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 8 May 2018 12:28:06 +0200
Subject: [PATCH 1013/2196] Dockerfile: update to 2.0.1

---
 misc/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 2f8e3dd7a67..f655f1a5639 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,8 +4,8 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 \
-  && echo "6312837aee33306cdbb351b75ba1638b89d21b30f0caf0346f9a742425f197ee  nix-2.0-x86_64-linux.tar.bz2" | sha256sum -c \
+RUN wget https://nixos.org/releases/nix/nix-2.0.1/nix-2.0.1-x86_64-linux.tar.bz2 \
+  && echo "8b8f0b8d3912273b037dcc51497df7bbc9529dbad48aeadf322ca27c4c4c7a90  nix-2.0.1-x86_64-linux.tar.bz2" | sha256sum -c \
   && tar xjf nix-*-x86_64-linux.tar.bz2 \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \

From 257c3c763a1cc2d400f12d9d7e2895df8aa2d894 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 8 May 2018 12:28:29 +0200
Subject: [PATCH 1014/2196] Add README for docker

---
 misc/docker/README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 misc/docker/README.md

diff --git a/misc/docker/README.md b/misc/docker/README.md
new file mode 100644
index 00000000000..87dd8713fcd
--- /dev/null
+++ b/misc/docker/README.md
@@ -0,0 +1,10 @@
+To update https://hub.docker.com/r/nixos/nix/
+
+    $ docker build .
+    $ docker image ls
+    $ docker tag <hash> nixos/nix:2.0
+    $ docker tag <hash> nixos/nix:latest
+    $ docker push nixos/nix:latest
+    $ docker push nixos/nix:2.0
+
+Write access: @domenkozar

From 85f9650322e762db8a14daa15aee508cfc5e2d4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 8 May 2018 12:37:43 +0200
Subject: [PATCH 1015/2196] Dockerfile: 2.0.2

---
 misc/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index f655f1a5639..0f69d02df25 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,8 +4,8 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget https://nixos.org/releases/nix/nix-2.0.1/nix-2.0.1-x86_64-linux.tar.bz2 \
-  && echo "8b8f0b8d3912273b037dcc51497df7bbc9529dbad48aeadf322ca27c4c4c7a90  nix-2.0.1-x86_64-linux.tar.bz2" | sha256sum -c \
+RUN wget https://nixos.org/releases/nix/nix-2.0.2/nix-2.0.2-x86_64-linux.tar.bz2 \
+  && echo "d0c2492d7d8f824e3b1ace15a1a58f64a0a8faacc59936ebedfe18905d982d7c  nix-2.0.2-x86_64-linux.tar.bz2" | sha256sum -c \
   && tar xjf nix-*-x86_64-linux.tar.bz2 \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \

From 2228891260ad9f35b5f8267f9f12aca3509e5d73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 8 May 2018 12:37:52 +0200
Subject: [PATCH 1016/2196] docker readme: simplify

---
 misc/docker/README.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/misc/docker/README.md b/misc/docker/README.md
index 87dd8713fcd..491be740896 100644
--- a/misc/docker/README.md
+++ b/misc/docker/README.md
@@ -1,9 +1,7 @@
 To update https://hub.docker.com/r/nixos/nix/
 
-    $ docker build .
-    $ docker image ls
-    $ docker tag <hash> nixos/nix:2.0
-    $ docker tag <hash> nixos/nix:latest
+    $ docker build . -t nixos/nix:2.0
+    $ docker tag nixos/nix:2.0 nixos/nix:latest
     $ docker push nixos/nix:latest
     $ docker push nixos/nix:2.0
 

From a91c4ca01f720711d80a6f7d16ead70ce8f5f741 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 May 2018 15:45:05 +0200
Subject: [PATCH 1017/2196] In restricted eval mode, allow access to the
 closure of store paths

E.g. this makes

  nix eval --restrict-eval -I /nix/store/foo '(builtins.readFile "/nix/store/foo/symlink/bla")'

(where /nix/store/foo/symlink is a symlink to another path in the
closure of /nix/store/foo) succeed.

This fixes a regression in Hydra compared to Nix 1.x (where there were
no restrictions at all on access to the Nix store).
---
 src/libexpr/eval.cc | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index a2cce162b90..353097f8971 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -317,10 +317,20 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     if (settings.restrictEval || settings.pureEval) {
         allowedPaths = PathSet();
+
         for (auto & i : searchPath) {
             auto r = resolveSearchPathElem(i);
             if (!r.first) continue;
-            allowedPaths->insert(r.second);
+
+            auto path = r.second;
+
+            if (store->isInStore(r.second)) {
+                PathSet closure;
+                store->computeFSClosure(store->toStorePath(r.second), closure);
+                for (auto & path : closure)
+                    allowedPaths->insert(path);
+            } else
+                allowedPaths->insert(r.second);
         }
     }
 

From b9289e4855d0221825c6f673d64f69802daa6b42 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Wed, 9 May 2018 15:23:08 +0200
Subject: [PATCH 1018/2196] make sure not to use cached channels for
 nix-channel --update

fixes #1964
---
 src/libstore/download.cc       | 3 +--
 src/libstore/download.hh       | 3 ++-
 src/nix-channel/nix-channel.cc | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 18f9094f82e..54f4dd21800 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -623,7 +623,7 @@ DownloadResult Downloader::download(const DownloadRequest & request)
     return enqueueDownload(request).get();
 }
 
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl)
+Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
 {
     auto url = resolveUri(url_);
 
@@ -653,7 +653,6 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     string expectedETag;
 
-    int ttl = settings.tarballTtl;
     bool skip = false;
 
     if (pathExists(fileLink) && pathExists(dataFile)) {
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 0b8d29b21df..7ade756fc35 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "hash.hh"
+#include "globals.hh"
 
 #include <string>
 #include <future>
@@ -54,7 +55,7 @@ struct Downloader
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
     Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "",
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr);
+        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index d1b47ede803..55ebda43896 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -86,7 +86,7 @@ static void update(const StringSet & channelNames)
         // definition from a consistent location if the redirect changes mid-download.
         std::string effectiveUrl;
         auto dl = getDownloader();
-        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl);
+        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0);
         url = chomp(std::move(effectiveUrl));
 
         // If the URL contains a version number, append it to the name

From d1f8822a43ba5fcd0486e6e3dd24e4d95a9a097d Mon Sep 17 00:00:00 2001
From: Adrien Devresse <adrien.devresse@epfl.ch>
Date: Wed, 9 May 2018 18:16:39 +0200
Subject: [PATCH 1019/2196] Cleanup of nix.spec file to allow build on EL7 and
 beyond

- Clean deprecated spec file macros ( < EL6 )
- Allow to build Nix on EL7 using Software collection and Copr backports
- Add conditional build to build Nix RPM without documentation for dev builds
- Add missing BuildRequires (openssl, and others )
---
 nix.spec.in | 61 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index d962bcc857b..cd053dbfce5 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -3,33 +3,47 @@
 %global nixbld_user "nix-builder-"
 %global nixbld_group "nixbld"
 
+# NOTE: BUILD on EL7 requires
+# - Centos / RHEL7 software collection repository 
+#   yum install centos-release-scl
+#
+# - Recent boost backport
+#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
+#
+
+# Disable documentation generation
+# necessary on some platforms
+%bcond_without docgen
+
 Summary: The Nix software deployment system
 Name: nix
 Version: @PACKAGE_VERSION@
 Release: 2%{?dist}
 License: LGPLv2+
-%if 0%{?rhel} && 0%{?rhel} < 7
 Group: Applications/System
-%endif
 URL: http://nixos.org/
 Source0: %{name}-%{version}.tar.bz2
-%if 0%{?el5}
-BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
-%endif
+
 Requires: curl
 Requires: bzip2
 Requires: gzip
 Requires: xz
-Requires: libseccomp
-Requires: boost-context
+BuildRequires: bison
+BuildRequires: boost-devel >= 1.60
 BuildRequires: bzip2-devel
-BuildRequires: sqlite-devel
+
+# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
+%if 0%{?rhel}
+BuildRequires: devtoolset-7-gcc
+BuildRequires: devtoolset-7-gcc-c++
+%endif
+
+BuildRequires: flex
 BuildRequires: libcurl-devel
 BuildRequires: libseccomp-devel
-BuildRequires: boost-devel
-
-# Hack to make that shitty RPM scanning hack shut up.
-Provides: perl(Nix::SSH)
+BuildRequires: openssl-devel
+BuildRequires: sqlite-devel
+BuildRequires: xz-devel
 
 %description
 Nix is a purely functional package manager. It allows multiple
@@ -41,9 +55,6 @@ it can be used equally well under other Unix systems.
 
 %package        devel
 Summary:        Development files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Development/Libraries
-%endif
 Requires:       %{name}%{?_isa} = %{version}-%{release}
 
 %description   devel
@@ -53,9 +64,6 @@ developing applications that use %{name}.
 
 %package doc
 Summary:        Documentation files for %{name}
-%if 0%{?rhel} && 0%{?rhel} < 7
-Group:          Documentation
-%endif
 BuildArch:      noarch
 Requires:       %{name} = %{version}-%{release}
 
@@ -67,20 +75,25 @@ The %{name}-doc package contains documentation files for %{name}.
 
 
 %build
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
+%endif
 extraFlags=
 # - override docdir so large documentation files are owned by the
 #   -doc subpackage
 # - set localstatedir by hand to the preferred nix value
 %configure --localstatedir=/nix/var \
+	   %{!?without_docgen:--disable-doc-gen} \
            --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
            $extraFlags
-make -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+make V=1 %{?_smp_mflags}
 
 
 %install
-%if 0%{?el5}
-rm -rf $RPM_BUILD_ROOT
+%if 0%{?rhel}
+source /opt/rh/devtoolset-7/enable 
 %endif
+
 make DESTDIR=$RPM_BUILD_ROOT install
 
 find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
@@ -130,6 +143,7 @@ systemctl start  nix-daemon.socket
 %endif
 
 %files
+%license COPYING
 %{_bindir}/nix*
 %{_libdir}/*.so
 %{_prefix}/libexec/*
@@ -138,9 +152,11 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
 %{_datadir}/nix
+%if ! %{without docgen}
 %{_mandir}/man1/*.1*
 %{_mandir}/man5/*.5*
 %{_mandir}/man8/*.8*
+%endif
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
 %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
@@ -149,6 +165,9 @@ systemctl start  nix-daemon.socket
 %{_includedir}/nix
 %{_prefix}/lib/pkgconfig/*.pc
 
+
+%if ! %{without docgen}
 %files doc
 %docdir %{_defaultdocdir}/%{name}-doc-%{version}
 %{_defaultdocdir}/%{name}-doc-%{version}
+%endif

From 2b3c1b3a889cf8f2c210b71b208b9e14fc8fb6a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 May 2018 12:06:28 +0200
Subject: [PATCH 1020/2196] Remove mention of default since we already know the
 default

---
 src/libstore/s3-binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 96673a5b0cc..103f141a1a1 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -174,7 +174,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
     const Setting<uint64_t> bufferSize{
-        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads. defaults to 5Mb"};
+        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
     std::string bucketName;
 

From f3c85f9eb395b13790d1fced05665125e2ec81ca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 May 2018 11:39:25 +0200
Subject: [PATCH 1021/2196] Revert "Throw a specific error for incomplete parse
 errors."

This reverts commit 6498adb002bcf7e715afe46c23b8635d4592c156. We don't
actually use IncompleteParseError in 'nix repl'.
---
 src/libexpr/lexer.l    | 2 --
 src/libexpr/nixexpr.hh | 1 -
 src/libexpr/parser.y   | 9 +--------
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 1e9c29afa13..8855d794ea9 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -213,7 +213,5 @@ or          { return OR_KW; }
 
 }
 
-<<EOF>> { data->atEnd = true; return 0; }
-
 %%
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index b486595f07a..665a42987dc 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -11,7 +11,6 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(ParseError, Error)
-MakeError(IncompleteParseError, ParseError)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index e3f4521844e..eee48887dc2 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -31,12 +31,10 @@ namespace nix {
         Path basePath;
         Symbol path;
         string error;
-        bool atEnd;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , atEnd(false)
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
@@ -541,12 +539,7 @@ Expr * EvalState::parse(const char * text,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
-    if (res) {
-      if (data.atEnd)
-        throw IncompleteParseError(data.error);
-      else
-        throw ParseError(data.error);
-    }
+    if (res) throw ParseError(data.error);
 
     data.result->bindVars(staticEnv);
 

From 1ad19232c4bbecb06e6acbb2a3a538544a28e1f2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 May 2018 12:02:19 +0200
Subject: [PATCH 1022/2196] Don't return negative numbers from the flex
 tokenizer

Fixes #1374.
Closes #2129.
---
 src/libexpr/lexer.l            | 6 +++++-
 tests/lang/parse-fail-uft8.nix | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 tests/lang/parse-fail-uft8.nix

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 8855d794ea9..29ca327c1e4 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -209,7 +209,11 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-{ANY}           return yytext[0];
+{ANY}       {
+              /* Don't return a negative number, as this will cause
+                 Bison to stop parsing without an error. */
+              return (unsigned char) yytext[0];
+            }
 
 }
 
diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-uft8.nix
new file mode 100644
index 00000000000..34948d48aed
--- /dev/null
+++ b/tests/lang/parse-fail-uft8.nix
@@ -0,0 +1 @@
+123 é 4

From 8a6a14e1f5006543f63117fb1f0a81a0b1024ebd Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Sat, 12 May 2018 16:50:39 +0000
Subject: [PATCH 1023/2196] add `mod' and bitwise builtins

---
 src/libexpr/parser.y   |  7 ++++++
 src/libexpr/primops.cc | 53 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index eee48887dc2..6beb28868e8 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -338,6 +338,13 @@ expr_op
   | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
   | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
   | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
+  | expr_op '%' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mod")), $1), $3); }
+  | expr_op '&' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_and")), $1), $3); }
+  | expr_op '|' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_or")), $1), $3); }
+  | expr_op '^' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_xor")), $1), $3); }
+  | expr_op '<<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_shl")), $1), $3); }
+  | expr_op '>>>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_unsigned_shr")), $1), $3); }
+  | expr_op '>>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_signed_shr")), $1), $3); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
   | expr_app
   ;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 57dc7bd1279..fa17f06b4c7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1676,6 +1676,52 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
+static void prim_mod(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    NixFloat f2 = state.forceFloat(*args[1], pos);
+    if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
+
+    if (args[0]->type == tFloat || args[1]->type == tFloat) {
+        mkFloat(v, state.forceFloat(*args[0], pos) % state.forceFloat(*args[1], pos));
+    } else {
+        NixInt i1 = state.forceInt(*args[0], pos);
+        NixInt i2 = state.forceInt(*args[1], pos);
+        /* Avoid division overflow as it might raise SIGFPE. */
+        if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
+            throw EvalError(format("overflow in integer division, at %1%") % pos);
+        mkInt(v, i1 % i2);
+    }
+}
+
+static void prim_bin_and(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
+}
+
+static void prim_bin_or(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
+}
+
+static void prim_bin_xor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
+}
+
+static void prim_bin_shl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, state.forceInt(*args[0], pos) << (state.forceInt(*args[1], pos) & 63));
+}
+
+static void prim_bin_unsigned_shr(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, (unsigned long)state.forceInt(*args[0], pos) >> (state.forceInt(*args[1], pos) & 63));
+}
+
+static void prim_bin_signed_shr(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    mkInt(v, (signed long)state.forceInt(*args[0], pos) >> (state.forceInt(*args[1], pos) & 63));
+}
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2221,6 +2267,13 @@ void EvalState::createBaseEnv()
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
+    addPrimOp("__mod", 2, prim_mod);
+    addPrimOp("__bin_and", 2, prim_bin_and);
+    addPrimOp("__bin_or", 2, prim_bin_or);
+    addPrimOp("__bin_xor", 2, prim_bin_xor);
+    addPrimOp("__bin_shl", 2, prim_bin_shl);
+    addPrimOp("__bin_unsigned_shr", 2, prim_bin_unsigned_shr);
+    addPrimOp("__bin_signed_shr", 2, prim_bin_signed_shr);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation

From 49b7cf1813d71b2c9d09f7c69408061774c01c51 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Wed, 16 May 2018 06:55:24 +0000
Subject: [PATCH 1024/2196] add `mod' and bitwise builtins: remove infix
 functions

---
 src/libexpr/parser.y | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 6beb28868e8..eee48887dc2 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -338,13 +338,6 @@ expr_op
   | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
   | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
   | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
-  | expr_op '%' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mod")), $1), $3); }
-  | expr_op '&' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_and")), $1), $3); }
-  | expr_op '|' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_or")), $1), $3); }
-  | expr_op '^' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_xor")), $1), $3); }
-  | expr_op '<<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_shl")), $1), $3); }
-  | expr_op '>>>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_unsigned_shr")), $1), $3); }
-  | expr_op '>>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__bin_signed_shr")), $1), $3); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
   | expr_app
   ;

From f3c090f91ce5e188151d887e3b5806239f5e6c0a Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Wed, 16 May 2018 06:57:11 +0000
Subject: [PATCH 1025/2196] add `mod' and bitwise builtins: remove `mod' and
 shifts

---
 src/libexpr/primops.cc | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index fa17f06b4c7..8514e804f11 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1676,23 +1676,6 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
-static void prim_mod(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
-
-    if (args[0]->type == tFloat || args[1]->type == tFloat) {
-        mkFloat(v, state.forceFloat(*args[0], pos) % state.forceFloat(*args[1], pos));
-    } else {
-        NixInt i1 = state.forceInt(*args[0], pos);
-        NixInt i2 = state.forceInt(*args[1], pos);
-        /* Avoid division overflow as it might raise SIGFPE. */
-        if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError(format("overflow in integer division, at %1%") % pos);
-        mkInt(v, i1 % i2);
-    }
-}
-
 static void prim_bin_and(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
@@ -1708,21 +1691,6 @@ static void prim_bin_xor(EvalState & state, const Pos & pos, Value * * args, Val
     mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
 }
 
-static void prim_bin_shl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    mkInt(v, state.forceInt(*args[0], pos) << (state.forceInt(*args[1], pos) & 63));
-}
-
-static void prim_bin_unsigned_shr(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    mkInt(v, (unsigned long)state.forceInt(*args[0], pos) >> (state.forceInt(*args[1], pos) & 63));
-}
-
-static void prim_bin_signed_shr(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    mkInt(v, (signed long)state.forceInt(*args[0], pos) >> (state.forceInt(*args[1], pos) & 63));
-}
-
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
@@ -2267,13 +2235,9 @@ void EvalState::createBaseEnv()
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
-    addPrimOp("__mod", 2, prim_mod);
     addPrimOp("__bin_and", 2, prim_bin_and);
     addPrimOp("__bin_or", 2, prim_bin_or);
     addPrimOp("__bin_xor", 2, prim_bin_xor);
-    addPrimOp("__bin_shl", 2, prim_bin_shl);
-    addPrimOp("__bin_unsigned_shr", 2, prim_bin_unsigned_shr);
-    addPrimOp("__bin_signed_shr", 2, prim_bin_signed_shr);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation

From 6cc28c0589a0072e94128976ddbf8b2fbfd9f496 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Wed, 16 May 2018 10:52:19 +0000
Subject: [PATCH 1026/2196] add `mod' and bitwise builtins: camel-case function
 names

---
 src/libexpr/primops.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8514e804f11..9dab8ecb046 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1676,17 +1676,17 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     }
 }
 
-static void prim_bin_and(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos));
 }
 
-static void prim_bin_or(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos));
 }
 
-static void prim_bin_xor(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos));
 }
@@ -2235,9 +2235,9 @@ void EvalState::createBaseEnv()
     addPrimOp("__sub", 2, prim_sub);
     addPrimOp("__mul", 2, prim_mul);
     addPrimOp("__div", 2, prim_div);
-    addPrimOp("__bin_and", 2, prim_bin_and);
-    addPrimOp("__bin_or", 2, prim_bin_or);
-    addPrimOp("__bin_xor", 2, prim_bin_xor);
+    addPrimOp("__bitAnd", 2, prim_bitAnd);
+    addPrimOp("__bitOr", 2, prim_bitOr);
+    addPrimOp("__bitXor", 2, prim_bitXor);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation

From 33712fed389976a40ca48631266e5604e8f7e955 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 18 May 2018 14:19:11 -0400
Subject: [PATCH 1027/2196] manual: document why ~/.netrc doesn't work

Maybe there is a better place to document this, which is more generic?
---
 doc/manual/command-ref/conf-file.xml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1865bb37c86..5045f85d392 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -524,7 +524,12 @@ password <replaceable>my-password</replaceable>
 
     For the exact syntax, see <link
     xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the
-    <literal>curl</literal> documentation.</link></para></listitem>
+    <literal>curl</literal> documentation.</link></para>
+
+    <note><para>This must be an absolute path, and <literal>~</literal>
+    is not resolved. For example, <filename>~/.netrc</filename> won't
+    resolve to your home directory's <filename>.netrc</filename>.</para></note>
+    </listitem>
 
   </varlistentry>
 

From b08923b4a06cf585b038e73cc277fccff8ce9b58 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 21 May 2018 17:26:41 -0500
Subject: [PATCH 1028/2196] serialise: fix buffer size used, hide method for
 internal use only

Fixes #2169.
---
 src/libutil/serialise.cc     | 2 +-
 src/libutil/serialise.hh     | 9 ++++++---
 src/nix-daemon/nix-daemon.cc | 3 ++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 33ae1ea389d..21803edd056 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -133,7 +133,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
     ssize_t n;
     do {
         checkInterrupt();
-        n = ::read(fd, (char *) data, bufSize);
+        n = ::read(fd, (char *) data, len);
     } while (n == -1 && errno == EINTR);
     if (n == -1) { _good = false; throw SysError("reading from file"); }
     if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 6e703c52a1e..14b62fdb677 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -77,10 +77,12 @@ struct BufferedSource : Source
 
     size_t read(unsigned char * data, size_t len) override;
 
-    /* Underlying read call, to be overridden. */
-    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
 
     bool hasData();
+
+protected:
+    /* Underlying read call, to be overridden. */
+    virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
 };
 
 
@@ -134,8 +136,9 @@ struct FdSource : BufferedSource
         return *this;
     }
 
-    size_t readUnbuffered(unsigned char * data, size_t len) override;
     bool good() override;
+protected:
+    size_t readUnbuffered(unsigned char * data, size_t len) override;
 private:
     bool _good = true;
 };
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 35603af7082..3dd8cec290c 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -197,7 +197,8 @@ struct TunnelSource : BufferedSource
 {
     Source & from;
     TunnelSource(Source & from) : from(from) { }
-    size_t readUnbuffered(unsigned char * data, size_t len)
+protected:
+    size_t readUnbuffered(unsigned char * data, size_t len) override
     {
         to << STDERR_READ << len;
         to.flush();

From 9fd7cf98dbec047343baefa65cd26ce4493bee53 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 May 2018 13:02:14 +0200
Subject: [PATCH 1029/2196] Memoise checkSourcePath()

This prevents hydra-eval-jobs from statting the same files over and
over again.
---
 src/libexpr/eval.cc | 8 +++++++-
 src/libexpr/eval.hh | 4 ++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 353097f8971..b2df1496871 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -352,6 +352,10 @@ Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
 
+    auto i = resolvedPaths.find(path_);
+    if (i != resolvedPaths.end())
+        return i->second;
+
     bool found = false;
 
     for (auto & i : *allowedPaths) {
@@ -369,8 +373,10 @@ Path EvalState::checkSourcePath(const Path & path_)
     Path path = canonPath(path_, true);
 
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path, i))
+        if (isDirOrInDir(path, i)) {
+            resolvedPaths[path_] = path;
             return path;
+        }
     }
 
     throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 86e93a5ac9c..8594a2707d9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -7,6 +7,7 @@
 #include "hash.hh"
 
 #include <map>
+#include <unordered_map>
 
 
 namespace nix {
@@ -100,6 +101,9 @@ private:
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
 
+    /* Cache used by checkSourcePath(). */
+    std::unordered_map<Path, Path> resolvedPaths;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);

From 4bb8741b9839162b42d998a1511a3b9080a0cc36 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 May 2018 16:02:32 +0200
Subject: [PATCH 1030/2196] Make Env self-describing

If the Env denotes a 'with', then values[0] may be an Expr* cast to a
Value*. For code that generically traverses Values/Envs, it's useful
to know this.
---
 src/libexpr/eval.cc | 16 ++++++++++------
 src/libexpr/eval.hh |  4 ++--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b2df1496871..2473157b3f6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -307,6 +307,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
+    static_assert(sizeof(Env) == 16);
+
     /* Initialise the Nix expression search path. */
     if (!settings.pureEval) {
         Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
@@ -568,12 +570,12 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     if (!var.fromWith) return env->values[var.displ];
 
     while (1) {
-        if (!env->haveWithAttrs) {
+        if (env->type == Env::HasWithExpr) {
             if (noEval) return 0;
             Value * v = allocValue();
             evalAttrs(*env->up, (Expr *) env->values[0], *v);
             env->values[0] = v;
-            env->haveWithAttrs = true;
+            env->type = Env::HasWithAttrs;
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
@@ -603,6 +605,7 @@ Env & EvalState::allocEnv(size_t size)
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
     env->size = (decltype(Env::size)) size;
+    env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -1205,7 +1208,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(1));
     env2.up = &env;
     env2.prevWith = prevWith;
-    env2.haveWithAttrs = false;
+    env2.type = Env::HasWithExpr;
     env2.values[0] = (Value *) attrs;
 
     body->eval(state, env2, v);
@@ -1863,9 +1866,10 @@ size_t valueSize(Value & v)
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
-        for (size_t i = 0; i < env.size; ++i)
-            if (env.values[i])
-                sz += doValue(*env.values[i]);
+        if (env.type != Env::HasWithExpr)
+            for (size_t i = 0; i < env.size; ++i)
+                if (env.values[i])
+                    sz += doValue(*env.values[i]);
 
         if (env.up) sz += doEnv(*env.up);
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 8594a2707d9..3725e45dad9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -35,8 +35,8 @@ struct Env
 {
     Env * up;
     unsigned short size; // used by ‘valueSize’
-    unsigned short prevWith:15; // nr of levels up to next `with' environment
-    unsigned short haveWithAttrs:1;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 

From 579f3895b49f53d32905bc9172ed6adcff40ce5f Mon Sep 17 00:00:00 2001
From: Alexandre Esteves <alexandre.fmp.esteves@gmail.com>
Date: Wed, 23 May 2018 22:26:06 +0100
Subject: [PATCH 1031/2196] Fix docs on --check suffix

---
 doc/manual/command-ref/nix-store.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index f2dc6ed8540..5fff64a18f9 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -204,7 +204,7 @@ printed.)</para>
     with <option>-K</option>, if an output path is not identical to
     the corresponding output from the previous build, the new output
     path is left in
-    <filename>/nix/store/<replaceable>name</replaceable>-check.</filename></para>
+    <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para>
 
     <para>See also the <option>build-repeat</option> configuration
     option, which repeats a derivation a number of times and prevents

From 88c1ea30e461092db713e58fe7521c5c43c52212 Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Thu, 24 May 2018 12:48:48 +0000
Subject: [PATCH 1032/2196] add docs and tests

---
 doc/manual/expressions/builtins.xml | 30 +++++++++++++++++++++++++++++
 tests/lang/eval-okay-arithmetic.exp |  2 +-
 tests/lang/eval-okay-arithmetic.nix |  4 ++++
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index ac1fe7e2faf..5489fab0026 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -92,6 +92,36 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.bitAnd</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the bitwise AND of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><function>builtins.bitOr</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the bitwise OR of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><function>builtins.bitXor</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the bitwise XOR of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><varname>builtins</varname></term>
 
     <listitem><para>The set <varname>builtins</varname> contains all
diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/lang/eval-okay-arithmetic.exp
index b195055b7a0..5c54d10b7b4 100644
--- a/tests/lang/eval-okay-arithmetic.exp
+++ b/tests/lang/eval-okay-arithmetic.exp
@@ -1 +1 @@
-2188
+2216
diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/lang/eval-okay-arithmetic.nix
index bbbbc4691d7..7e9e6a0b666 100644
--- a/tests/lang/eval-okay-arithmetic.nix
+++ b/tests/lang/eval-okay-arithmetic.nix
@@ -26,6 +26,10 @@ let {
       (56088 / 123 / 2)
       (3 + 4 * const 5 0 - 6 / id 2)
 
+      (builtins.bitAnd 12 10) # 0b1100 & 0b1010 =  8
+      (builtins.bitOr  12 10) # 0b1100 | 0b1010 = 14
+      (builtins.bitXor 12 10) # 0b1100 ^ 0b1010 =  6
+
       (if 3 < 7 then 1 else err)
       (if 7 < 3 then err else 1)
       (if 3 < 3 then err else 1)

From c4b9486f9b20d319a7ac3ac2553bdc0fa63c3899 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 25 May 2018 13:57:03 -0400
Subject: [PATCH 1033/2196] install-multi-user: don't force NIX_SSL_CERT_FILE

Following the lead of the single user installer, if NIX_SSL_CERT_FILE is explicitly set prior to running, accept the user-provided version.
---
 scripts/install-multi-user.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 5f6542355e0..b4eec2fe94c 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -727,11 +727,17 @@ setup_default_profile() {
     _sudo "to installing a bootstrapping Nix in to the default Profile" \
           HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
 
-    _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
-          HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+    if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
+        _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
+              HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
+        export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+    fi
 
+    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+    # otherwise it will be lost in environments where sudo doesn't pass
+    # all the environment variables by default.
     _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
 }
 
 

From cad903b63438b7df6605ac35e79b28a7ce0744c2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 25 May 2018 15:28:26 -0400
Subject: [PATCH 1034/2196] multi-user profile: borrow single user profiles'
 NIX_SSL_CERT_FILE finding logic

---
 scripts/nix-profile-daemon.sh.in | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 4632099e3c8..52070e36a1e 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -49,6 +49,23 @@ if test -w $HOME; then
   fi
 fi
 
-export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+
+# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
+if [ ! -z "$NIX_SSL_CERT_FILE" ]; then
+    : # Allow users to override the NIX_SSL_CERT_FILE
+elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
+    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed
+    export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
+    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
+elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
+    export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+elif [ -e "$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile
+    export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt
+elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the default Nix profile
+    export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+fi
+
 export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
 export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"

From e6466c20b37b7dcbeaa5c6dd4a1fef24101a310c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 25 May 2018 15:28:36 -0400
Subject: [PATCH 1035/2196] Document NIX_SSL_CERT_FILE for installation

---
 doc/manual/installation/env-variables.xml | 49 ++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index fc39cdd9dfe..1fd6bafee7e 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -21,4 +21,51 @@ in your <filename>~/.profile</filename> (or similar), like this:</para>
 <screen>
 source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
 
-</chapter>
\ No newline at end of file
+<section xml:id="sec-nix-ssl-cert-file">
+
+<title><envar>NIX_SSL_CERT_FILE</envar></title>
+
+<para>If you need to specify a custom certificate bundle to account
+for an HTTPS-intercepting man in the middle proxy, you must specify
+the path to the certificate bundle in the environment variable
+<envar>NIX_SSL_CERT_FILE</envar>.</para>
+
+
+<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar>
+manually, Nix will install and use its own certificate
+bundle.</para>
+
+<procedure>
+  <step><para>Set the environment variable and install Nix</para>
+    <screen>
+$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+$ curl https://nixos.org/nix/install | sh
+</screen></step>
+
+  <step><para>In the shell profile and rc files (for example,
+  <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>),
+  add the following line:</para>
+<programlisting>
+export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
+</programlisting>
+</step>
+</procedure>
+
+<note><para>You must not add the export and then do the install, as
+the Nix installer will detect the presense of Nix configuration, and
+abort.</para></note>
+
+<section>
+<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title>
+
+<para>On macOS you must specify the environment variable for the Nix
+daemon service, then restart it:</para>
+
+<screen>
+$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt
+$ sudo launchctl kickstart -k system/org.nixos.nix-daemon
+</screen>
+</section>
+
+</section>
+</chapter>

From ca9c6cb95db03cfb46e044dd45f113a72ee13bd2 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 25 May 2018 19:49:45 -0500
Subject: [PATCH 1036/2196] libstore/download: remove unused function
 "getTime()"

---
 src/libstore/download.cc | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 54f4dd21800..1076a6d94a8 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -29,13 +29,6 @@ using namespace std::string_literals;
 
 namespace nix {
 
-double getTime()
-{
-    struct timeval tv;
-    gettimeofday(&tv, 0);
-    return tv.tv_sec + (tv.tv_usec / 1000000.0);
-}
-
 std::string resolveUri(const std::string & uri)
 {
     if (uri.compare(0, 8, "channel:") == 0)

From 14c464b6c8d9acec40298923a45a7e3860f70ebf Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 26 May 2018 13:09:55 -0500
Subject: [PATCH 1037/2196] nix-daemon: remove unused "pendingMsgs" variable

---
 src/nix-daemon/nix-daemon.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 3dd8cec290c..baf169804a5 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -120,8 +120,6 @@ struct TunnelLogger : public Logger
       want to send out stderr to the client. */
     void startWork()
     {
-        std::vector<std::string> pendingMsgs;
-
         auto state(state_.lock());
         state->canSendStderr = true;
 

From 93ae90de0f118417682ff532935675e4867df9db Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 26 May 2018 14:04:41 -0500
Subject: [PATCH 1038/2196] eval.cc: add message to static_assert, message can
 be omitted w/c++17

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2473157b3f6..7c897800091 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -307,7 +307,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
-    static_assert(sizeof(Env) == 16);
+    static_assert(sizeof(Env) == 16, "environment must be 16 bytes");
 
     /* Initialise the Nix expression search path. */
     if (!settings.pureEval) {

From 7f560b81eae47f876cb652b0014f55af0c711dfd Mon Sep 17 00:00:00 2001
From: Ben Gamari <ben@smart-cactus.org>
Date: Sat, 26 May 2018 18:43:46 -0400
Subject: [PATCH 1039/2196] json-to-value: Use strtol instead of strtoi

NixInt is long, so strtoi is too restrictive.
---
 src/libexpr/json-to-value.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 8b140459554..3f601795778 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
             if (number_type == tFloat)
                 mkFloat(v, stod(tmp_number));
             else
-                mkInt(v, stoi(tmp_number));
+                mkInt(v, stol(tmp_number));
         } catch (std::invalid_argument e) {
             throw JSONParseError("invalid JSON number");
         } catch (std::out_of_range e) {

From 3cab639e21fb39d6e0530910cf94b5bf60dbf5b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 16 Apr 2018 11:37:58 +0200
Subject: [PATCH 1040/2196] Use extended SQLite error codes

---
 src/libstore/sqlite.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 42d40e71d8b..a061d64f36d 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -10,6 +10,7 @@ namespace nix {
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs)
 {
     int err = sqlite3_errcode(db);
+    int exterr = sqlite3_extended_errcode(db);
 
     auto path = sqlite3_db_filename(db, nullptr);
     if (!path) path = "(in-memory)";
@@ -21,7 +22,7 @@ namespace nix {
             : fmt("SQLite database '%s' is busy", path));
     }
     else
-        throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path);
+        throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
 }
 
 SQLite::SQLite(const Path & path)

From 23d6bb583afb2b98cc9152911c12a37ba56d1e39 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Mar 2018 23:34:25 +0200
Subject: [PATCH 1041/2196] LocalStore::addToStore: Throw an exception if no
 hash is given

---
 src/libstore/local-store.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ef8c2811bd8..681abafef23 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -976,7 +976,8 @@ const PublicKeys & LocalStore::getPublicKeys()
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    assert(info.narHash);
+    if (!info.narHash)
+        throw Error("cannot add path '%s' because it lacks a hash", info.path);
 
     if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
         throw Error("cannot add path '%s' because it lacks a valid signature", info.path);

From 6185d25e523a3cd223dd6f6aca10cf6ff15b4823 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Mar 2018 23:36:30 +0200
Subject: [PATCH 1042/2196] Make 'nix copy --to daemon' run in constant memory
 (daemon side)

Continuation of 97002b684c902dadcd351a67208f9c2a88ff8f8f. This makes
the daemon use constant memory. For example, it reduces the daemon's
maximum RSS on

  $ nix copy --from ~/my-nix --to daemon /nix/store/1n7x0yv8vq6zi90hfmian84vdhd04bgp-blender-2.79a

from 264 MiB to 7 MiB.

We now use a TunnelSource to prevent the connection from ending up in
an undefined state if an exception is thrown while the NAR is being
sent.

Issue https://github.com/NixOS/nix/issues/1681.
---
 src/libstore/remote-store.cc    |  5 +++--
 src/libstore/worker-protocol.hh |  2 +-
 src/nix-daemon/nix-daemon.cc    | 16 +++++++++++++---
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 080cef93d21..d5303755855 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -411,8 +411,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << info.references << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
                  << repair << !checkSigs;
-        copyNAR(source, conn->to);
-        conn->processStderr();
+        bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
+        if (!tunnel) copyNAR(source, conn->to);
+        conn->processStderr(0, tunnel ? &source : nullptr);
     }
 }
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 996e1d25355..5ebdfaf134d 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x114
+#define PROTOCOL_VERSION 0x115
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index baf169804a5..590f9621142 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -690,12 +690,22 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
         if (!trusted)
             info.ultimate = false;
 
-        TeeSink tee(from);
-        parseDump(tee, tee.source);
+        std::string saved;
+        std::unique_ptr<Source> source;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+            source = std::make_unique<TunnelSource>(from);
+        else {
+            TeeSink tee(from);
+            parseDump(tee, tee.source);
+            saved = std::move(*tee.source.data);
+            source = std::make_unique<StringSource>(saved);
+        }
 
         logger->startWork();
-        store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair,
+
+        store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+
         logger->stopWork();
         break;
     }

From e606cd412f6ad0622feff55dc2a023dc4b2fe238 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 16:12:00 +0200
Subject: [PATCH 1043/2196] Fix assertion failure in storePathToHash()

Fixes https://github.com/NixOS/nix/issues/2015.
---
 src/libstore/store-api.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1a0d12ca78c..a8f3ae1ea6c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -253,6 +253,8 @@ std::string Store::getUri()
 
 bool Store::isValidPath(const Path & storePath)
 {
+    assertStorePath(storePath);
+
     auto hashPart = storePathToHash(storePath);
 
     {

From 737ed88f35ffddb2cb0d5e4b192e20a7b9439682 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 18:41:31 +0200
Subject: [PATCH 1044/2196] Modularize config settings

Allow global config settings to be defined in multiple Config
classes. For example, this means that libutil can have settings and
evaluator settings can be moved out of libstore. The Config classes
are registered in a new GlobalConfig class to which config files
etc. are applied.

Relevant to https://github.com/NixOS/nix/issues/2009 in that it
removes the need for ad hoc handling of useCaseHack, which was the
underlying cause of that issue.
---
 perl/lib/Nix/Store.xs        |  2 +-
 src/libmain/common-args.cc   |  4 +-
 src/libmain/shared.cc        |  2 +-
 src/libstore/build.cc        |  6 ++-
 src/libstore/globals.cc      | 38 +++++-----------
 src/libstore/globals.hh      | 36 +--------------
 src/libstore/remote-store.cc |  5 +-
 src/libstore/store-api.cc    |  2 +-
 src/libutil/archive.cc       | 26 +++++++----
 src/libutil/archive.hh       |  4 --
 src/libutil/config.cc        | 88 ++++++++++++++++++++++++++++--------
 src/libutil/config.hh        | 81 ++++++++++++++++++++++++++-------
 src/nix/main.cc              |  7 +--
 src/nix/show-config.cc       |  8 ++--
 tests/plugins/plugintest.cc  | 13 ++++--
 15 files changed, 192 insertions(+), 130 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index bbfb2934315..ce553bb53eb 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -27,7 +27,7 @@ static ref<Store> store()
     static std::shared_ptr<Store> _store;
     if (!_store) {
         try {
-            settings.loadConfFile();
+            loadConfFile();
             settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index bcc05c2cdad..4c35a419959 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -29,14 +29,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         .arity(2)
         .handler([](std::vector<std::string> ss) {
             try {
-                settings.set(ss[0], ss[1]);
+                globalConfig.set(ss[0], ss[1]);
             } catch (UsageError & e) {
                 warn(e.what());
             }
         });
 
     std::string cat = "config";
-    settings.convertToArgs(*this, cat);
+    globalConfig.convertToArgs(*this, cat);
 
     // Backward compatibility hack: nix-env already had a --system flag.
     if (programName == "nix-env") longFlags.erase("system");
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 91a4eaf922a..4ed34e54dc5 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -109,7 +109,7 @@ void initNix()
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
 
-    settings.loadConfFile();
+    loadConfFile();
 
     startSignalHandlerThread();
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f70ab8108fd..07b53378393 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -672,8 +672,10 @@ HookInstance::HookInstance()
     toHook.readSide = -1;
 
     sink = FdSink(toHook.writeSide.get());
-    for (auto & setting : settings.getSettings())
-        sink << 1 << setting.first << setting.second;
+    std::map<std::string, Config::SettingInfo> settings;
+    globalConfig.getSettings(settings);
+    for (auto & setting : settings)
+        sink << 1 << setting.first << setting.second.value;
     sink << 0;
 }
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 544566e0b57..d95db56726c 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -28,9 +28,10 @@ namespace nix {
 
 Settings settings;
 
+static GlobalConfig::Register r1(&settings);
+
 Settings::Settings()
-    : Config({})
-    , nixPrefix(NIX_PREFIX)
+    : nixPrefix(NIX_PREFIX)
     , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
     , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
     , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
@@ -69,20 +70,15 @@ Settings::Settings()
     allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
 }
 
-void Settings::loadConfFile()
+void loadConfFile()
 {
-    applyConfigFile(nixConfDir + "/nix.conf");
+    globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
-    resetOverriden();
+    globalConfig.resetOverriden();
 
-    applyConfigFile(getConfigDir() + "/nix/nix.conf");
-}
-
-void Settings::set(const string & name, const string & value)
-{
-    Config::set(name, value);
+    globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf");
 }
 
 unsigned int Settings::getDefaultCores()
@@ -162,23 +158,11 @@ void initPlugins()
                 throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
         }
     }
-    /* We handle settings registrations here, since plugins can add settings */
-    if (RegisterSetting::settingRegistrations) {
-        for (auto & registration : *RegisterSetting::settingRegistrations)
-            settings.addSetting(registration);
-        delete RegisterSetting::settingRegistrations;
-    }
-    settings.handleUnknownSettings();
-}
-
-RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations;
 
-RegisterSetting::RegisterSetting(AbstractSetting * s)
-{
-    if (!settingRegistrations)
-        settingRegistrations = new SettingRegistrations;
-    settingRegistrations->emplace_back(s);
+    /* Since plugins can add settings, try to re-apply previously
+       unknown settings. */
+    globalConfig.reapplyUnknownSettings();
+    globalConfig.warnUnknownSettings();
 }
 
-
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 9360096aae8..e8d95aed605 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -13,26 +13,6 @@ namespace nix {
 
 typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode;
 
-extern bool useCaseHack; // FIXME
-
-struct CaseHackSetting : public BaseSetting<bool>
-{
-    CaseHackSetting(Config * options,
-        const std::string & name,
-        const std::string & description,
-        const std::set<std::string> & aliases = {})
-        : BaseSetting<bool>(useCaseHack, name, description, aliases)
-    {
-        options->addSetting(this);
-    }
-
-    void set(const std::string & str) override
-    {
-        BaseSetting<bool>::set(str);
-        nix::useCaseHack = value;
-    }
-};
-
 struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
 {
     MaxBuildJobsSetting(Config * options,
@@ -56,10 +36,6 @@ public:
 
     Settings();
 
-    void loadConfFile();
-
-    void set(const string & name, const string & value);
-
     Path nixPrefix;
 
     /* The directory where we store sources and derived files. */
@@ -353,9 +329,6 @@ public:
     Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
         "Whether the evaluator allows importing the result of a derivation."};
 
-    CaseHackSetting useCaseHack{this, "use-case-hack",
-        "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
-
     Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
         "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
 
@@ -398,15 +371,8 @@ extern Settings settings;
    anything else */
 void initPlugins();
 
+void loadConfFile();
 
 extern const string nixVersion;
 
-struct RegisterSetting
-{
-    typedef std::vector<AbstractSetting *> SettingRegistrations;
-    static SettingRegistrations * settingRegistrations;
-    RegisterSetting(AbstractSetting * s);
-};
-
-
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index d5303755855..a157c6c4858 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -187,10 +187,11 @@ void RemoteStore::setOptions(Connection & conn)
        << settings.useSubstitutes;
 
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
-        auto overrides = settings.getSettings(true);
+        std::map<std::string, Config::SettingInfo> overrides;
+        globalConfig.getSettings(overrides, true);
         conn.to << overrides.size();
         for (auto & i : overrides)
-            conn.to << i.first << i.second;
+            conn.to << i.first << i.second.value;
     }
 
     conn.processStderr();
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a8f3ae1ea6c..49b32d11584 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -849,7 +849,7 @@ ref<Store> openStore(const std::string & uri_,
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->handleUnknownSettings();
+            store->warnUnknownSettings();
             return ref<Store>(store);
         }
     }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 154e2d20430..1be8934a2eb 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -13,17 +13,25 @@
 
 #include "archive.hh"
 #include "util.hh"
-
+#include "config.hh"
 
 namespace nix {
 
+struct ArchiveSettings : Config
+{
+    Setting<bool> useCaseHack{this,
+        #if __APPLE__
+            true,
+        #else
+            false,
+        #endif
+        "use-case-hack",
+        "Whether to enable a Darwin-specific hack for dealing with file name collisions."};
+};
 
-bool useCaseHack =
-#if __APPLE__
-    true;
-#else
-    false;
-#endif
+static ArchiveSettings archiveSettings;
+
+static GlobalConfig::Register r1(&archiveSettings);
 
 const std::string narVersionMagic1 = "nix-archive-1";
 
@@ -78,7 +86,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
            the case hack applied by restorePath(). */
         std::map<string, string> unhacked;
         for (auto & i : readDirectory(path))
-            if (useCaseHack) {
+            if (archiveSettings.useCaseHack) {
                 string name(i.name);
                 size_t pos = i.name.find(caseHackSuffix);
                 if (pos != string::npos) {
@@ -243,7 +251,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
-                    if (useCaseHack) {
+                    if (archiveSettings.useCaseHack) {
                         auto i = names.find(name);
                         if (i != names.end()) {
                             debug(format("case collision between '%1%' and '%2%'") % i->first % name);
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 7a0e688e420..25be426c1a4 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -78,10 +78,6 @@ void restorePath(const Path & path, Source & source);
 void copyNAR(Source & source, Sink & sink);
 
 
-// FIXME: global variables are bad m'kay.
-extern bool useCaseHack;
-
-
 extern const std::string narVersionMagic1;
 
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index ce6858f0d65..9023cb1bb6d 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -4,15 +4,13 @@
 
 namespace nix {
 
-void Config::set(const std::string & name, const std::string & value)
+bool Config::set(const std::string & name, const std::string & value)
 {
     auto i = _settings.find(name);
-    if (i == _settings.end()) {
-        extras.emplace(name, value);
-    } else {
-        i->second.setting->set(value);
-        i->second.setting->overriden = true;
-    }
+    if (i == _settings.end()) return false;
+    i->second.setting->set(value);
+    i->second.setting->overriden = true;
+    return true;
 }
 
 void Config::addSetting(AbstractSetting * setting)
@@ -23,46 +21,51 @@ void Config::addSetting(AbstractSetting * setting)
 
     bool set = false;
 
-    auto i = extras.find(setting->name);
-    if (i != extras.end()) {
+    auto i = unknownSettings.find(setting->name);
+    if (i != unknownSettings.end()) {
         setting->set(i->second);
         setting->overriden = true;
-        extras.erase(i);
+        unknownSettings.erase(i);
         set = true;
     }
 
     for (auto & alias : setting->aliases) {
-        auto i = extras.find(alias);
-        if (i != extras.end()) {
+        auto i = unknownSettings.find(alias);
+        if (i != unknownSettings.end()) {
             if (set)
                 warn("setting '%s' is set, but it's an alias of '%s' which is also set",
                     alias, setting->name);
             else {
                 setting->set(i->second);
                 setting->overriden = true;
-                extras.erase(i);
+                unknownSettings.erase(i);
                 set = true;
             }
         }
     }
 }
 
-void Config::handleUnknownSettings()
+void AbstractConfig::warnUnknownSettings()
 {
-    for (auto & s : extras)
+    for (auto & s : unknownSettings)
         warn("unknown setting '%s'", s.first);
 }
 
-StringMap Config::getSettings(bool overridenOnly)
+void AbstractConfig::reapplyUnknownSettings()
+{
+    auto unknownSettings2 = std::move(unknownSettings);
+    for (auto & s : unknownSettings2)
+        set(s.first, s.second);
+}
+
+void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
 {
-    StringMap res;
     for (auto & opt : _settings)
         if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
-            res.emplace(opt.first, opt.second.setting->to_string());
-    return res;
+            res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void Config::applyConfigFile(const Path & path)
+void AbstractConfig::applyConfigFile(const Path & path)
 {
     try {
         string contents = readFile(path);
@@ -287,4 +290,49 @@ void PathSetting::set(const std::string & str)
         value = canonPath(str);
 }
 
+bool GlobalConfig::set(const std::string & name, const std::string & value)
+{
+    for (auto & config : *configRegistrations)
+        if (config->set(name, value)) return true;
+
+    unknownSettings.emplace(name, value);
+
+    return false;
+}
+
+void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+{
+    for (auto & config : *configRegistrations)
+        config->getSettings(res, overridenOnly);
+}
+
+void GlobalConfig::resetOverriden()
+{
+    for (auto & config : *configRegistrations)
+        config->resetOverriden();
+}
+
+void GlobalConfig::toJSON(JSONObject & out)
+{
+    for (auto & config : *configRegistrations)
+        config->toJSON(out);
+}
+
+void GlobalConfig::convertToArgs(Args & args, const std::string & category)
+{
+    for (auto & config : *configRegistrations)
+        config->convertToArgs(args, category);
+}
+
+GlobalConfig globalConfig;
+
+GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations;
+
+GlobalConfig::Register::Register(Config * config)
+{
+    if (!configRegistrations)
+        configRegistrations = new ConfigRegistrations;
+    configRegistrations->emplace_back(config);
+}
+
 }
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index d2e7faf1743..d86c65ff033 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -12,6 +12,40 @@ class AbstractSetting;
 class JSONPlaceholder;
 class JSONObject;
 
+class AbstractConfig
+{
+protected:
+    StringMap unknownSettings;
+
+    AbstractConfig(const StringMap & initials = {})
+        : unknownSettings(initials)
+    { }
+
+public:
+
+    virtual bool set(const std::string & name, const std::string & value) = 0;
+
+    struct SettingInfo
+    {
+        std::string value;
+        std::string description;
+    };
+
+    virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
+
+    void applyConfigFile(const Path & path);
+
+    virtual void resetOverriden() = 0;
+
+    virtual void toJSON(JSONObject & out) = 0;
+
+    virtual void convertToArgs(Args & args, const std::string & category) = 0;
+
+    void warnUnknownSettings();
+
+    void reapplyUnknownSettings();
+};
+
 /* A class to simplify providing configuration settings. The typical
    use is to inherit Config and add Setting<T> members:
 
@@ -27,7 +61,7 @@ class JSONObject;
    };
 */
 
-class Config
+class Config : public AbstractConfig
 {
     friend class AbstractSetting;
 
@@ -48,31 +82,23 @@ private:
 
     Settings _settings;
 
-    StringMap extras;
-
 public:
 
-    Config(const StringMap & initials)
-        : extras(initials)
+    Config(const StringMap & initials = {})
+        : AbstractConfig(initials)
     { }
 
-    void set(const std::string & name, const std::string & value);
+    bool set(const std::string & name, const std::string & value) override;
 
     void addSetting(AbstractSetting * setting);
 
-    void handleUnknownSettings();
-
-    StringMap getSettings(bool overridenOnly = false);
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
 
-    const Settings & _getSettings() { return _settings; }
-
-    void applyConfigFile(const Path & path);
+    void resetOverriden() override;
 
-    void resetOverriden();
+    void toJSON(JSONObject & out) override;
 
-    void toJSON(JSONObject & out);
-
-    void convertToArgs(Args & args, const std::string & category);
+    void convertToArgs(Args & args, const std::string & category) override;
 };
 
 class AbstractSetting
@@ -209,4 +235,27 @@ public:
     void operator =(const Path & v) { this->assign(v); }
 };
 
+struct GlobalConfig : public AbstractConfig
+{
+    typedef std::vector<Config*> ConfigRegistrations;
+    static ConfigRegistrations * configRegistrations;
+
+    bool set(const std::string & name, const std::string & value) override;
+
+    void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+
+    void resetOverriden() override;
+
+    void toJSON(JSONObject & out) override;
+
+    void convertToArgs(Args & args, const std::string & category) override;
+
+    struct Register
+    {
+        Register(Config * config);
+    };
+};
+
+extern GlobalConfig globalConfig;
+
 }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index bb107ec7d3f..9cd5d21c84b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -34,9 +34,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .handler([&]() {
                 std::cout << "The following configuration options are available:\n\n";
                 Table2 tbl;
-                for (const auto & s : settings._getSettings())
-                    if (!s.second.isAlias)
-                        tbl.emplace_back(s.first, s.second.setting->description);
+                std::map<std::string, Config::SettingInfo> settings;
+                globalConfig.getSettings(settings);
+                for (const auto & s : settings)
+                    tbl.emplace_back(s.first, s.second.description);
                 printTable(std::cout, tbl);
                 throw Exit();
             });
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index c64b12c8dd6..86638b50d2c 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -27,10 +27,12 @@ struct CmdShowConfig : Command, MixJSON
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
             JSONObject jsonObj(std::cout);
-            settings.toJSON(jsonObj);
+            globalConfig.toJSON(jsonObj);
         } else {
-            for (auto & s : settings.getSettings())
-                std::cout << s.first + " = " + s.second + "\n";
+            std::map<std::string, Config::SettingInfo> settings;
+            globalConfig.getSettings(settings);
+            for (auto & s : settings)
+                std::cout << s.first + " = " + s.second.value + "\n";
         }
     }
 };
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc
index 8da15ebabd7..c085d33295b 100644
--- a/tests/plugins/plugintest.cc
+++ b/tests/plugins/plugintest.cc
@@ -1,16 +1,21 @@
-#include "globals.hh"
+#include "config.hh"
 #include "primops.hh"
 
 using namespace nix;
 
-static BaseSetting<bool> settingSet{false, "setting-set",
+struct MySettings : Config
+{
+    Setting<bool> settingSet{this, false, "setting-set",
         "Whether the plugin-defined setting was set"};
+};
+
+MySettings mySettings;
 
-static RegisterSetting rs(&settingSet);
+static GlobalConfig::Register rs(&mySettings);
 
 static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v)
 {
-    if (settingSet)
+    if (mySettings.settingSet)
         mkNull(v);
     else
         mkBool(v, false);

From c1d445ececa4ea17bd5b5bff3d5fb3515be723cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 18:54:02 +0200
Subject: [PATCH 1045/2196] Move some Download-specific settings to download.cc

---
 src/libstore/download.cc     | 32 ++++++++++++++++++++++++--------
 src/libstore/globals.hh      | 13 -------------
 src/nix-daemon/nix-daemon.cc |  2 +-
 3 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 1076a6d94a8..fce701a15a0 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -29,6 +29,26 @@ using namespace std::string_literals;
 
 namespace nix {
 
+struct DownloadSettings : Config
+{
+    Setting<bool> enableHttp2{this, true, "http2",
+        "Whether to enable HTTP/2 support."};
+
+    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
+        "String appended to the user agent in HTTP requests."};
+
+    Setting<size_t> httpConnections{this, 25, "http-connections",
+        "Number of parallel HTTP connections.",
+        {"binary-caches-parallel-connections"}};
+
+    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
+        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+};
+
+static DownloadSettings downloadSettings;
+
+static GlobalConfig::Register r1(&downloadSettings);
+
 std::string resolveUri(const std::string & uri)
 {
     if (uri.compare(0, 8, "channel:") == 0)
@@ -54,8 +74,6 @@ struct CurlDownloader : public Downloader
     std::random_device rd;
     std::mt19937 mt19937;
 
-    bool enableHttp2;
-
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
         CurlDownloader & downloader;
@@ -221,12 +239,12 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str());
+                    (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (downloader.enableHttp2)
+            if (downloadSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             #endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
@@ -258,7 +276,7 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
@@ -401,11 +419,9 @@ struct CurlDownloader : public Downloader
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            settings.binaryCachesParallelConnections.get());
+            downloadSettings.httpConnections.get());
         #endif
 
-        enableHttp2 = settings.enableHttp2;
-
         wakeupPipe.create();
         fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index e8d95aed605..881032cf6fa 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -256,13 +256,6 @@ public:
     Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
         "Secret keys with which to sign local builds."};
 
-    Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
-        {"binary-caches-parallel-connections"}};
-
-    Setting<bool> enableHttp2{this, true, "http2",
-        "Whether to enable HTTP/2 support."};
-
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
         "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
@@ -329,12 +322,6 @@ public:
     Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
         "Whether the evaluator allows importing the result of a derivation."};
 
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
-
-    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
-        "String appended to the user agent in HTTP requests."};
-
 #if __linux__
     Setting<bool> filterSyscalls{this, true, "filter-syscalls",
             "Whether to prevent certain dangerous system calls, such as "
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 590f9621142..423e6bb6789 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -553,7 +553,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
                     ;
                 else if (trusted
                     || name == settings.buildTimeout.name
-                    || name == settings.connectTimeout.name)
+                    || name == "connect-timeout")
                     settings.set(name, value);
                 else if (setSubstituters(settings.substituters))
                     ;

From 1672bcd230447f1ce0c3291950bdd9a662cee974 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 19:02:22 +0200
Subject: [PATCH 1046/2196] Move evaluator-specific settings out of libstore

---
 src/libexpr/eval.cc                   | 13 +++++++++----
 src/libexpr/eval.hh                   | 22 ++++++++++++++++++++++
 src/libexpr/primops.cc                | 16 ++++++++--------
 src/libexpr/primops/fetchGit.cc       |  2 +-
 src/libexpr/primops/fetchMercurial.cc |  2 +-
 src/libstore/globals.hh               | 16 ----------------
 src/nix/upgrade-nix.cc                |  2 +-
 7 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7c897800091..a3f411a5fb1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -310,14 +310,14 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     static_assert(sizeof(Env) == 16, "environment must be 16 bytes");
 
     /* Initialise the Nix expression search path. */
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
     addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
-    if (settings.restrictEval || settings.pureEval) {
+    if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
 
         for (auto & i : searchPath) {
@@ -387,13 +387,13 @@ Path EvalState::checkSourcePath(const Path & path_)
 
 void EvalState::checkURI(const std::string & uri)
 {
-    if (!settings.restrictEval) return;
+    if (!evalSettings.restrictEval) return;
 
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. Note: this allows 'http://' and
        'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : settings.allowedUris.get())
+    for (auto & prefix : evalSettings.allowedUris.get())
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
@@ -1898,4 +1898,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings evalSettings;
+
+static GlobalConfig::Register r1(&evalSettings);
+
+
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 3725e45dad9..146f2125503 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -5,6 +5,7 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "hash.hh"
+#include "config.hh"
 
 #include <map>
 #include <unordered_map>
@@ -320,4 +321,25 @@ struct InvalidPathError : EvalError
 #endif
 };
 
+struct EvalSettings : Config
+{
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
+
+    Setting<bool> restrictEval{this, false, "restrict-eval",
+        "Whether to restrict file system access to paths in $NIX_PATH, "
+        "and network access to the URI prefixes listed in 'allowed-uris'."};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
+    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
+        "Whether the evaluator allows importing the result of a derivation."};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+};
+
+extern EvalSettings evalSettings;
+
 }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9dab8ecb046..3a6c4035b8b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context)
 
     if (drvs.empty()) return;
 
-    if (!settings.enableImportFromDerivation)
+    if (!evalSettings.enableImportFromDerivation)
         throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
 
     /* For performance, prefetch all substitute info. */
@@ -464,7 +464,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
 }
 
 
@@ -1031,7 +1031,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
     Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    const auto path = settings.pureEval && expectedHash ?
+    const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
         state.checkSourcePath(path_);
     PathFilter filter = filterFun ? ([&](const Path & path) {
@@ -2056,7 +2056,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.checkURI(url);
 
-    if (settings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
     Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
@@ -2124,12 +2124,12 @@ void EvalState::createBaseEnv()
         addConstant(name, v);
     };
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
     }
 
-    if (!settings.pureEval) {
+    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem);
         addConstant("__currentSystem", v);
     }
@@ -2154,7 +2154,7 @@ void EvalState::createBaseEnv()
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
     addConstant("import", v);
-    if (settings.enableNativeCode) {
+    if (evalSettings.enableNativeCode) {
         addPrimOp("__importNative", 2, prim_importNative);
         addPrimOp("__exec", 1, prim_exec);
     }
@@ -2181,7 +2181,7 @@ void EvalState::createBaseEnv()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
-    if (settings.pureEval)
+    if (evalSettings.pureEval)
         addPurityError("__storePath");
     else
         addPrimOp("__storePath", 1, prim_storePath);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 8bb74dad639..7aa98e0bfab 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -28,7 +28,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::experimental::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
     if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a75c5fc2ddf..9d35f6d0d6d 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -27,7 +27,7 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
 HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     std::string rev, const std::string & name)
 {
-    if (settings.pureEval && rev == "")
+    if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
     if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 881032cf6fa..f589078dbb9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -193,9 +193,6 @@ public:
     Setting<bool> showTrace{this, false, "show-trace",
         "Whether to show a stack trace on evaluation errors."};
 
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
-
     Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
@@ -208,13 +205,6 @@ public:
         "Additional paths to make available inside the build sandbox.",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
-    Setting<bool> restrictEval{this, false, "restrict-eval",
-        "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and network access to the URI prefixes listed in 'allowed-uris'."};
-
-    Setting<bool> pureEval{this, false, "pure-eval",
-        "Whether to restrict file system and network access to files specified by cryptographic hash."};
-
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
         {"build-repeat"}};
@@ -319,9 +309,6 @@ public:
     /* Path to the SSL CA file used */
     Path caFile;
 
-    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
-        "Whether the evaluator allows importing the result of a derivation."};
-
 #if __linux__
     Setting<bool> filterSyscalls{this, true, "filter-syscalls",
             "Whether to prevent certain dangerous system calls, such as "
@@ -343,9 +330,6 @@ public:
     Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
         "Stop deleting garbage when free disk space is above the specified amount."};
 
-    Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
-
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 };
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 758bbbc688b..21892c31a89 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -46,7 +46,7 @@ struct CmdUpgradeNix : StoreCommand
 
     void run(ref<Store> store) override
     {
-        settings.pureEval = true;
+        evalSettings.pureEval = true;
 
         if (profileDir == "")
             profileDir = getProfileDir(store);

From 81ea8bd5ceb3dcae6af0b79c81a39ecbf2ba97a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 22:16:01 +0200
Subject: [PATCH 1047/2196] Simplify the callback mechanism

---
 src/libstore/binary-cache-store.cc       | 37 +++++-----
 src/libstore/binary-cache-store.hh       |  6 +-
 src/libstore/download.cc                 | 37 +++++-----
 src/libstore/download.hh                 |  3 +-
 src/libstore/http-binary-cache-store.cc  | 18 ++---
 src/libstore/legacy-ssh-store.cc         | 11 ++-
 src/libstore/local-binary-cache-store.cc | 17 ++---
 src/libstore/local-store.cc              | 13 ++--
 src/libstore/local-store.hh              |  3 +-
 src/libstore/misc.cc                     | 17 ++---
 src/libstore/remote-store.cc             |  9 ++-
 src/libstore/remote-store.hh             |  3 +-
 src/libstore/s3-binary-cache-store.cc    |  9 ++-
 src/libstore/store-api.cc                | 88 +++++++++++-------------
 src/libstore/store-api.hh                |  6 +-
 src/libutil/util.hh                      | 53 ++++++--------
 16 files changed, 151 insertions(+), 179 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 2e9a13e564c..45be490765a 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -58,12 +58,13 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
     std::promise<std::shared_ptr<std::string>> promise;
     getFile(path,
-        [&](std::shared_ptr<std::string> result) {
-            promise.set_value(result);
-        },
-        [&](std::exception_ptr exc) {
-            promise.set_exception(exc);
-        });
+        {[&](std::future<std::shared_ptr<std::string>> result) {
+            try {
+                promise.set_value(result.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        }});
     return promise.get_future().get();
 }
 
@@ -218,8 +219,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure)
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
     auto uri = getUri();
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
@@ -229,17 +229,22 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
     auto narInfoFile = narInfoFileFor(storePath);
 
     getFile(narInfoFile,
-        [=](std::shared_ptr<std::string> data) {
-            if (!data) return success(0);
+        {[=](std::future<std::shared_ptr<std::string>> fut) {
+            try {
+                auto data = fut.get();
+
+                if (!data) return callback(nullptr);
 
-            stats.narInfoRead++;
+                stats.narInfoRead++;
 
-            callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>)
-                std::make_shared<NarInfo>(*this, *data, narInfoFile));
+                callback((std::shared_ptr<ValidPathInfo>)
+                    std::make_shared<NarInfo>(*this, *data, narInfoFile));
 
-            (void) act; // force Activity into this lambda to ensure it stays alive
-        },
-        failure);
+                (void) act; // force Activity into this lambda to ensure it stays alive
+            } catch (...) {
+                callback.rethrow();
+            }
+        }});
 }
 
 Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index e20b968442b..fcde666beec 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -41,8 +41,7 @@ public:
     /* Return the contents of the specified file, or null if it
        doesn't exist. */
     virtual void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<std::shared_ptr<std::string>> callback) = 0;
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
@@ -71,8 +70,7 @@ public:
     { unsupported(); }
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     void queryReferrers(const Path & path,
         PathSet & referrers) override
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index fce701a15a0..afb066e1468 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -81,8 +81,7 @@ struct CurlDownloader : public Downloader
         DownloadResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        std::function<void(const DownloadResult &)> success;
-        std::function<void(std::exception_ptr exc)> failure;
+        Callback<DownloadResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -97,10 +96,13 @@ struct CurlDownloader : public Downloader
 
         std::string encoding;
 
-        DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
+        DownloadItem(CurlDownloader & downloader,
+            const DownloadRequest & request,
+            Callback<DownloadResult> callback)
             : downloader(downloader)
             , request(request)
             , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct)
+            , callback(callback)
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@@ -129,7 +131,7 @@ struct CurlDownloader : public Downloader
         {
             assert(!done);
             done = true;
-            callFailure(failure, std::make_exception_ptr(e));
+            callback.rethrow(std::make_exception_ptr(e));
         }
 
         size_t writeCallback(void * contents, size_t size, size_t nmemb)
@@ -316,11 +318,11 @@ struct CurlDownloader : public Downloader
                 try {
                     if (request.decompress)
                         result.data = decodeContent(encoding, ref<std::string>(result.data));
-                    callSuccess(success, failure, const_cast<const DownloadResult &>(result));
                     act.progress(result.data->size(), result.data->size());
+                    callback(std::move(result));
                 } catch (...) {
                     done = true;
-                    callFailure(failure, std::current_exception());
+                    callback.rethrow();
                 }
             } else {
                 // We treat most errors as transient, but won't retry when hopeless
@@ -570,13 +572,12 @@ struct CurlDownloader : public Downloader
     }
 
     void enqueueDownload(const DownloadRequest & request,
-        std::function<void(const DownloadResult &)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<DownloadResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
             // FIXME: do this on a worker thread
-            sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
+            try {
 #ifdef ENABLE_S3
                 S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
@@ -590,18 +591,15 @@ struct CurlDownloader : public Downloader
                 if (!s3Res.data)
                     throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
-                return res;
+                callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
 #endif
-            });
+            } catch (...) { callback.rethrow(); }
             return;
         }
 
-        auto item = std::make_shared<DownloadItem>(*this, request);
-        item->success = success;
-        item->failure = failure;
-        enqueueItem(item);
+        enqueueItem(std::make_shared<DownloadItem>(*this, request, callback));
     }
 };
 
@@ -622,8 +620,13 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
 {
     auto promise = std::make_shared<std::promise<DownloadResult>>();
     enqueueDownload(request,
-        [promise](const DownloadResult & result) { promise->set_value(result); },
-        [promise](std::exception_ptr exc) { promise->set_exception(exc); });
+        {[promise](std::future<DownloadResult> fut) {
+            try {
+                promise->set_value(fut.get());
+            } catch (...) {
+                promise->set_exception(std::current_exception());
+            }
+        }});
     return promise->get_future();
 }
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 7ade756fc35..01940f5447f 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -42,8 +42,7 @@ struct Downloader
        the download. The future may throw a DownloadError
        exception. */
     virtual void enqueueDownload(const DownloadRequest & request,
-        std::function<void(const DownloadResult &)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<DownloadResult> callback) = 0;
 
     std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
 
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index b9e9cd5daba..b8d670417aa 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -78,27 +78,23 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     }
 
     void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<std::string>> callback) override
     {
         DownloadRequest request(cacheUri + "/" + path);
         request.tries = 8;
 
         getDownloader()->enqueueDownload(request,
-            [success](const DownloadResult & result) {
-                success(result.data);
-            },
-            [success, failure](std::exception_ptr exc) {
+            {[callback](std::future<DownloadResult> result) {
                 try {
-                    std::rethrow_exception(exc);
+                    callback(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                        return success(0);
-                    failure(exc);
+                        return callback(std::shared_ptr<std::string>());
+                    callback.rethrow();
                 } catch (...) {
-                    failure(exc);
+                    callback.rethrow();
                 }
-            });
+            }});
     }
 
 };
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5dee25308f7..02d91ded04c 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -84,10 +84,9 @@ struct LegacySSHStore : public Store
     }
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override
     {
-        sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
+        try {
             auto conn(connections->get());
 
             debug("querying remote host '%s' for info on '%s'", host, path);
@@ -97,7 +96,7 @@ struct LegacySSHStore : public Store
 
             auto info = std::make_shared<ValidPathInfo>();
             conn->from >> info->path;
-            if (info->path.empty()) return nullptr;
+            if (info->path.empty()) return callback(nullptr);
             assert(path == info->path);
 
             PathSet references;
@@ -116,8 +115,8 @@ struct LegacySSHStore : public Store
             auto s = readString(conn->from);
             assert(s == "");
 
-            return info;
-        });
+            callback(std::move(info));
+        } catch (...) { callback.rethrow(); }
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 2577e90aef2..ae0ffa6a56e 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -35,17 +35,14 @@ class LocalBinaryCacheStore : public BinaryCacheStore
         const std::string & mimeType) override;
 
     void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<std::string>> callback) override
     {
-        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
-            try {
-                return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path));
-            } catch (SysError & e) {
-                if (e.errNo == ENOENT) return std::shared_ptr<std::string>();
-                throw;
-            }
-        });
+        try {
+            // FIXME: O(n) space
+            callback(std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)));
+        } catch (SysError & e) {
+            if (e.errNo == ENOENT) callback(nullptr); else callback.rethrow();
+        } catch (...) { callback.rethrow(); }
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 681abafef23..3b2ba65f3b4 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -629,17 +629,15 @@ uint64_t LocalStore::addValidPath(State & state,
 
 
 void LocalStore::queryPathInfoUncached(const Path & path,
-    std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
-    sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
-
+    try {
         auto info = std::make_shared<ValidPathInfo>();
         info->path = path;
 
         assertStorePath(path);
 
-        return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
+        callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
 
             /* Get the path info. */
@@ -679,8 +677,9 @@ void LocalStore::queryPathInfoUncached(const Path & path,
                 info->references.insert(useQueryReferences.getStr(0));
 
             return info;
-        });
-    });
+        }));
+
+    } catch (...) { callback.rethrow(); }
 }
 
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 1209a06356f..746bdbeed79 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,8 +127,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index a82aa4e9cfa..adcce026fa1 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -33,9 +33,11 @@ void Store::computeFSClosure(const PathSet & startPaths,
             state->pending++;
         }
 
-        queryPathInfo(path,
-            [&, path](ref<ValidPathInfo> info) {
-                // FIXME: calls to isValidPath() should be async
+        queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
+            // FIXME: calls to isValidPath() should be async
+
+            try {
+                auto info = fut.get();
 
                 if (flipDirection) {
 
@@ -75,14 +77,13 @@ void Store::computeFSClosure(const PathSet & startPaths,
                     if (!--state->pending) done.notify_one();
                 }
 
-            },
-
-            [&, path](std::exception_ptr exc) {
+            } catch (...) {
                 auto state(state_.lock());
-                if (!state->exc) state->exc = exc;
+                if (!state->exc) state->exc = std::current_exception();
                 assert(state->pending);
                 if (!--state->pending) done.notify_one();
-            });
+            };
+        }});
     };
 
     for (auto & startPath : startPaths)
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a157c6c4858..b72c940cdcf 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -294,10 +294,9 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
 
 void RemoteStore::queryPathInfoUncached(const Path & path,
-    std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
+    Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
-    sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() {
+    try {
         auto conn(connections->get());
         conn->to << wopQueryPathInfo << path;
         try {
@@ -324,8 +323,8 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
             info->sigs = readStrings<StringSet>(conn->from);
             conn->from >> info->ca;
         }
-        return info;
-    });
+        callback(std::move(info));
+    } catch (...) { callback.rethrow(); }
 }
 
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 95fa59a2069..b488e34ce26 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -40,8 +40,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 103f141a1a1..f2e8efc16e6 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -365,10 +365,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     }
 
     void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<std::string>> callback) override
     {
-        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
+        try {
             stats.get++;
 
             auto res = s3Helper.getObject(bucketName, path);
@@ -380,8 +379,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                     bucketName, path, res.data->size(), res.durationMs);
 
-            return res.data;
-        });
+            callback(std::move(res.data));
+        } catch (...) { callback.rethrow(); }
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 49b32d11584..9b0b7d6327e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -305,20 +305,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
     std::promise<ref<ValidPathInfo>> promise;
 
     queryPathInfo(storePath,
-        [&](ref<ValidPathInfo> info) {
-            promise.set_value(info);
-        },
-        [&](std::exception_ptr exc) {
-            promise.set_exception(exc);
-        });
+        {[&](std::future<ref<ValidPathInfo>> result) {
+            try {
+                promise.set_value(result.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        }});
 
     return promise.get_future().get();
 }
 
 
 void Store::queryPathInfo(const Path & storePath,
-    std::function<void(ref<ValidPathInfo>)> success,
-    std::function<void(std::exception_ptr exc)> failure)
+    Callback<ref<ValidPathInfo>> callback)
 {
     auto hashPart = storePathToHash(storePath);
 
@@ -330,7 +330,7 @@ void Store::queryPathInfo(const Path & storePath,
                 stats.narInfoReadAverted++;
                 if (!*res)
                     throw InvalidPath(format("path '%s' is not valid") % storePath);
-                return success(ref<ValidPathInfo>(*res));
+                return callback(ref<ValidPathInfo>(*res));
             }
         }
 
@@ -346,35 +346,36 @@ void Store::queryPathInfo(const Path & storePath,
                         (res.second->path != storePath && storePathToName(storePath) != ""))
                         throw InvalidPath(format("path '%s' is not valid") % storePath);
                 }
-                return success(ref<ValidPathInfo>(res.second));
+                return callback(ref<ValidPathInfo>(res.second));
             }
         }
 
-    } catch (std::exception & e) {
-        return callFailure(failure);
-    }
+    } catch (...) { return callback.rethrow(); }
 
     queryPathInfoUncached(storePath,
-        [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
+        {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) {
 
-            if (diskCache)
-                diskCache->upsertNarInfo(getUri(), hashPart, info);
+            try {
+                auto info = fut.get();
 
-            {
-                auto state_(state.lock());
-                state_->pathInfoCache.upsert(hashPart, info);
-            }
+                if (diskCache)
+                    diskCache->upsertNarInfo(getUri(), hashPart, info);
 
-            if (!info
-                || (info->path != storePath && storePathToName(storePath) != ""))
-            {
-                stats.narInfoMissing++;
-                return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath)));
-            }
+                {
+                    auto state_(state.lock());
+                    state_->pathInfoCache.upsert(hashPart, info);
+                }
 
-            callSuccess(success, failure, ref<ValidPathInfo>(info));
+                if (!info
+                    || (info->path != storePath && storePathToName(storePath) != ""))
+                {
+                    stats.narInfoMissing++;
+                    throw InvalidPath("path '%s' is not valid", storePath);
+                }
 
-        }, failure);
+                callback(ref<ValidPathInfo>(info));
+            } catch (...) { callback.rethrow(); }
+        }});
 }
 
 
@@ -394,26 +395,19 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
 
     auto doQuery = [&](const Path & path ) {
         checkInterrupt();
-        queryPathInfo(path,
-            [path, &state_, &wakeup](ref<ValidPathInfo> info) {
-                auto state(state_.lock());
+        queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
+            auto state(state_.lock());
+            try {
+                auto info = fut.get();
                 state->valid.insert(path);
-                assert(state->left);
-                if (!--state->left)
-                    wakeup.notify_one();
-            },
-            [path, &state_, &wakeup](std::exception_ptr exc) {
-                auto state(state_.lock());
-                try {
-                    std::rethrow_exception(exc);
-                } catch (InvalidPath &) {
-                } catch (...) {
-                    state->exc = exc;
-                }
-                assert(state->left);
-                if (!--state->left)
-                    wakeup.notify_one();
-            });
+            } catch (InvalidPath &) {
+            } catch (...) {
+                state->exc = std::current_exception();
+            }
+            assert(state->left);
+            if (!--state->left)
+                wakeup.notify_one();
+        }});
     };
 
     for (auto & path : paths)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ea259f07e8a..6ee2d550679 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -355,14 +355,12 @@ public:
 
     /* Asynchronous version of queryPathInfo(). */
     void queryPathInfo(const Path & path,
-        std::function<void(ref<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure);
+        Callback<ref<ValidPathInfo>> callback);
 
 protected:
 
     virtual void queryPathInfoUncached(const Path & path,
-        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
-        std::function<void(std::exception_ptr exc)> failure) = 0;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) = 0;
 
 public:
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 743d238611f..215c7cecafe 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -15,6 +15,7 @@
 #include <map>
 #include <sstream>
 #include <experimental/optional>
+#include <future>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -424,44 +425,30 @@ string get(const T & map, const string & key, const string & def = "")
 }
 
 
-/* Call ‘failure’ with the current exception as argument. If ‘failure’
-   throws an exception, abort the program. */
-void callFailure(const std::function<void(std::exception_ptr exc)> & failure,
-    std::exception_ptr exc = std::current_exception());
+/* A callback is a wrapper around a lambda that accepts a valid of
+   type T or an exception. (We abuse std::future<T> to pass the value or
+   exception.) */
+template<typename T>
+struct Callback
+{
+    std::function<void(std::future<T>)> fun;
 
+    Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
 
-/* Evaluate the function ‘f’. If it returns a value, call ‘success’
-   with that value as its argument. If it or ‘success’ throws an
-   exception, call ‘failure’. If ‘failure’ throws an exception, abort
-   the program. */
-template<class T>
-void sync2async(
-    const std::function<void(T)> & success,
-    const std::function<void(std::exception_ptr exc)> & failure,
-    const std::function<T()> & f)
-{
-    try {
-        success(f());
-    } catch (...) {
-        callFailure(failure);
+    void operator()(T && t) const
+    {
+        std::promise<T> promise;
+        promise.set_value(std::move(t));
+        fun(promise.get_future());
     }
-}
 
-
-/* Call the function ‘success’. If it throws an exception, call
-   ‘failure’. If that throws an exception, abort the program. */
-template<class T>
-void callSuccess(
-    const std::function<void(T)> & success,
-    const std::function<void(std::exception_ptr exc)> & failure,
-    T && arg)
-{
-    try {
-        success(arg);
-    } catch (...) {
-        callFailure(failure);
+    void rethrow(const std::exception_ptr & exc = std::current_exception()) const
+    {
+        std::promise<T> promise;
+        promise.set_exception(exc);
+        fun(promise.get_future());
     }
-}
+};
 
 
 /* Start a thread that handles various signals. Also block those signals

From 08ec757726e5ef47e71bf16ed0b252b288bcf0f3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Mar 2018 23:12:31 +0200
Subject: [PATCH 1048/2196] Make LocalBinaryCacheStore::narFromPath() run in
 constant memory

This reduces memory consumption of

  nix copy --from file://... --to ~/my-nix /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 514 MiB to 18 MiB for an uncompressed binary cache, and from 192
MiB to 53 MiB for a bzipped binary cache. It may also be faster
because fetching can happen concurrently with decompression/writing.

Continuation of 48662d151bdf4a38670897beacea9d1bd750376a.

Issue https://github.com/NixOS/nix/issues/1681.
---
 src/libstore/binary-cache-store.cc       | 36 ++++++++++++++++++------
 src/libstore/binary-cache-store.hh       | 14 +++++++--
 src/libstore/local-binary-cache-store.cc | 11 ++++----
 src/libstore/s3-binary-cache-store.cc    | 24 ++++++++--------
 src/libutil/util.cc                      | 10 ++++++-
 src/libutil/util.hh                      |  1 +
 6 files changed, 65 insertions(+), 31 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 45be490765a..11fa3cae27a 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -54,7 +54,15 @@ void BinaryCacheStore::init()
     }
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+void BinaryCacheStore::getFile(const std::string & path,
+    Callback<std::shared_ptr<std::string>> callback)
+{
+    try {
+        callback(getFile(path));
+    } catch (...) { callback.rethrow(); }
+}
+
+void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 {
     std::promise<std::shared_ptr<std::string>> promise;
     getFile(path,
@@ -65,7 +73,19 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
                 promise.set_exception(std::current_exception());
             }
         }});
-    return promise.get_future().get();
+    auto data = promise.get_future().get();
+    sink((unsigned char *) data->data(), data->size());
+}
+
+std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+{
+    StringSink sink;
+    try {
+        getFile(path, sink);
+    } catch (NoSuchBinaryCacheFile &) {
+        return nullptr;
+    }
+    return sink.s;
 }
 
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
@@ -197,23 +217,21 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    auto nar = getFile(info->url);
-
-    if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url);
+    auto source = sinkToSource([this, url{info->url}](Sink & sink) {
+        getFile(url, sink);
+    });
 
     stats.narRead++;
-    stats.narReadCompressedBytes += nar->size();
+    //stats.narReadCompressedBytes += nar->size(); // FIXME
 
     uint64_t narSize = 0;
 
-    StringSource source(*nar);
-
     LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
         sink(data, len);
         narSize += len;
     });
 
-    decompress(info->compression, source, wrapperSink);
+    decompress(info->compression, *source, wrapperSink);
 
     stats.narReadBytes += narSize;
 }
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index fcde666beec..6bc83fc50ca 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -38,10 +38,16 @@ public:
         const std::string & data,
         const std::string & mimeType) = 0;
 
-    /* Return the contents of the specified file, or null if it
-       doesn't exist. */
+    /* Note: subclasses must implement at least one of the two
+       following getFile() methods. */
+
+    /* Dump the contents of the specified file to a sink. */
+    virtual void getFile(const std::string & path, Sink & sink);
+
+    /* Fetch the specified file and call the specified callback with
+       the result. A subclass may implement this asynchronously. */
     virtual void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) = 0;
+        Callback<std::shared_ptr<std::string>> callback);
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
@@ -129,4 +135,6 @@ public:
 
 };
 
+MakeError(NoSuchBinaryCacheFile, Error);
+
 }
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index ae0ffa6a56e..b7001795be4 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -34,15 +34,14 @@ class LocalBinaryCacheStore : public BinaryCacheStore
         const std::string & data,
         const std::string & mimeType) override;
 
-    void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) override
+    void getFile(const std::string & path, Sink & sink) override
     {
         try {
-            // FIXME: O(n) space
-            callback(std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)));
+            readFile(binaryCacheDir + "/" + path, sink);
         } catch (SysError & e) {
-            if (e.errNo == ENOENT) callback(nullptr); else callback.rethrow();
-        } catch (...) { callback.rethrow(); }
+            if (e.errNo == ENOENT)
+                throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
+        }
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f2e8efc16e6..239739bae83 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -364,23 +364,23 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             uploadFile(path, data, mimeType, "");
     }
 
-    void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) override
+    void getFile(const std::string & path, Sink & sink) override
     {
-        try {
-            stats.get++;
+        stats.get++;
 
-            auto res = s3Helper.getObject(bucketName, path);
+        // FIXME: stream output to sink.
+        auto res = s3Helper.getObject(bucketName, path);
 
-            stats.getBytes += res.data ? res.data->size() : 0;
-            stats.getTimeMs += res.durationMs;
+        stats.getBytes += res.data ? res.data->size() : 0;
+        stats.getTimeMs += res.durationMs;
 
-            if (res.data)
-                printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
-                    bucketName, path, res.data->size(), res.durationMs);
+        if (res.data) {
+            printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
+                bucketName, path, res.data->size(), res.durationMs);
 
-            callback(std::move(res.data));
-        } catch (...) { callback.rethrow(); }
+            sink((unsigned char *) res.data->data(), res.data->size());
+        } else
+            throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
     }
 
     PathSet queryAllValidPaths() override
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 15962236ec6..22753f4b7ec 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -311,6 +311,14 @@ string readFile(const Path & path, bool drain)
 }
 
 
+void readFile(const Path & path, Sink & sink)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd) throw SysError("opening file '%s'", path);
+    drainFD(fd.get(), sink);
+}
+
+
 void writeFile(const Path & path, const string & s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
@@ -593,7 +601,7 @@ void drainFD(int fd, Sink & sink, bool block)
             throw SysError("making file descriptor non-blocking");
     }
 
-    std::vector<unsigned char> buf(4096);
+    std::vector<unsigned char> buf(64 * 1024);
     while (1) {
         checkInterrupt();
         ssize_t rd = read(fd, buf.data(), buf.size());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 215c7cecafe..af97b76d90b 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -98,6 +98,7 @@ unsigned char getFileType(const Path & path);
 /* Read the contents of a file into a string. */
 string readFile(int fd);
 string readFile(const Path & path, bool drain = false);
+void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
 void writeFile(const Path & path, const string & s, mode_t mode = 0666);

From e87e4a60d617bffadfedf23032254130cdb4d54d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Mar 2018 00:01:47 +0200
Subject: [PATCH 1049/2196] Make HttpBinaryCacheStore::narFromPath() run in
 constant memory

This reduces memory consumption of

  nix copy --from https://cache.nixos.org --to ~/my-nix /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 176 MiB to 82 MiB. (The remaining memory is probably due to xz
decompression overhead.)

Issue https://github.com/NixOS/nix/issues/1681.
Issue https://github.com/NixOS/nix/issues/1969.
---
 src/libstore/download.cc                | 92 ++++++++++++++++++++++++-
 src/libstore/download.hh                |  5 ++
 src/libstore/http-binary-cache-store.cc | 22 +++++-
 3 files changed, 116 insertions(+), 3 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index afb066e1468..d450714ca50 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -7,6 +7,7 @@
 #include "s3.hh"
 #include "compression.hh"
 #include "pathlocks.hh"
+#include "finally.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -137,7 +138,10 @@ struct CurlDownloader : public Downloader
         size_t writeCallback(void * contents, size_t size, size_t nmemb)
         {
             size_t realSize = size * nmemb;
-            result.data->append((char *) contents, realSize);
+            if (request.dataCallback)
+                request.dataCallback((char *) contents, realSize);
+            else
+                result.data->append((char *) contents, realSize);
             return realSize;
         }
 
@@ -635,6 +639,92 @@ DownloadResult Downloader::download(const DownloadRequest & request)
     return enqueueDownload(request).get();
 }
 
+void Downloader::download(DownloadRequest && request, Sink & sink)
+{
+    /* Note: we can't call 'sink' via request.dataCallback, because
+       that would cause the sink to execute on the downloader
+       thread. If 'sink' is a coroutine, this will fail. Also, if the
+       sink is expensive (e.g. one that does decompression and writing
+       to the Nix store), it would stall the download thread too much.
+       Therefore we use a buffer to communicate data between the
+       download thread and the calling thread. */
+
+    struct State {
+        bool quit = false;
+        std::exception_ptr exc;
+        std::string data;
+        std::condition_variable avail, request;
+    };
+
+    auto _state = std::make_shared<Sync<State>>();
+
+    /* In case of an exception, wake up the download thread. FIXME:
+       abort the download request. */
+    Finally finally([&]() {
+        auto state(_state->lock());
+        state->quit = true;
+        state->request.notify_one();
+    });
+
+    request.dataCallback = [_state](char * buf, size_t len) {
+
+        auto state(_state->lock());
+
+        if (state->quit) return;
+
+        /* If the buffer is full, then go to sleep until the calling
+           thread wakes us up (i.e. when it has removed data from the
+           buffer). Note: this does stall the download thread. */
+        while (state->data.size() > 1024 * 1024) {
+            if (state->quit) return;
+            debug("download buffer is full; going to sleep");
+            state.wait(state->request);
+        }
+
+        /* Append data to the buffer and wake up the calling
+           thread. */
+        state->data.append(buf, len);
+        state->avail.notify_one();
+    };
+
+    enqueueDownload(request,
+        {[_state](std::future<DownloadResult> fut) {
+            auto state(_state->lock());
+            state->quit = true;
+            try {
+                fut.get();
+            } catch (...) {
+                state->exc = std::current_exception();
+            }
+            state->avail.notify_one();
+            state->request.notify_one();
+        }});
+
+    auto state(_state->lock());
+
+    while (true) {
+        checkInterrupt();
+
+        if (state->quit) {
+            if (state->exc) std::rethrow_exception(state->exc);
+            break;
+        }
+
+        /* If no data is available, then wait for the download thread
+           to wake us up. */
+        if (state->data.empty())
+            state.wait(state->avail);
+
+        /* If data is available, then flush it to the sink and wake up
+           the download thread if it's blocked on a full buffer. */
+        if (!state->data.empty()) {
+            sink((unsigned char *) state->data.data(), state->data.size());
+            state->data.clear();
+            state->request.notify_one();
+        }
+    }
+}
+
 Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
 {
     auto url = resolveUri(url_);
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 01940f5447f..f56274b2353 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -21,6 +21,7 @@ struct DownloadRequest
     bool decompress = true;
     std::shared_ptr<std::string> data;
     std::string mimeType;
+    std::function<void(char *, size_t)> dataCallback;
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
@@ -49,6 +50,10 @@ struct Downloader
     /* Synchronously download a file. */
     DownloadResult download(const DownloadRequest & request);
 
+    /* Download a file, writing its data to a sink. The sink will be
+       invoked on the thread of the caller. */
+    void download(DownloadRequest && request, Sink & sink);
+
     /* Check if the specified file is already in ~/.cache/nix/tarballs
        and is more recent than ‘tarball-ttl’ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index b8d670417aa..6fdae40e360 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -77,11 +77,29 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         }
     }
 
-    void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) override
+    DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
         request.tries = 8;
+        return request;
+    }
+
+    void getFile(const std::string & path, Sink & sink) override
+    {
+        auto request(makeRequest(path));
+        try {
+            getDownloader()->download(std::move(request), sink);
+        } catch (DownloadError & e) {
+            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
+            throw;
+        }
+    }
+
+    void getFile(const std::string & path,
+        Callback<std::shared_ptr<std::string>> callback) override
+    {
+        auto request(makeRequest(path));
 
         getDownloader()->enqueueDownload(request,
             {[callback](std::future<DownloadResult> result) {

From 5a654fd7dd0bf58470e0a15d451cf36ce0c62beb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Mar 2018 13:16:27 +0200
Subject: [PATCH 1050/2196] getDownloader(): Simplify

---
 src/libstore/download.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index d450714ca50..72a08ef0089 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -609,10 +609,8 @@ struct CurlDownloader : public Downloader
 
 ref<Downloader> getDownloader()
 {
-    static std::shared_ptr<Downloader> downloader;
-    static std::once_flag downloaderCreated;
-    std::call_once(downloaderCreated, [&]() { downloader = makeDownloader(); });
-    return ref<Downloader>(downloader);
+    static ref<Downloader> downloader = makeDownloader();
+    return downloader;
 }
 
 ref<Downloader> makeDownloader()

From 7d21863bb3f3d4c42b273a8648a65bec83fe0b60 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Mar 2018 13:32:44 +0200
Subject: [PATCH 1051/2196] Make <nix/fetchurl.nix> run in constant memory

E.g.

  nix-build --store ~/my-nix/ -E 'import <nix/fetchurl.nix> { url = https://cache.nixos.org/nar/0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv.nar.xz; sha256 = "0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv"; }'

now runs in 17 MiB (was 70 MiB), while

  nix-build --store ~/my-nix/ -E 'import <nix/fetchurl.nix> { url = https://cache.nixos.org/nar/0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv.nar.xz; sha256 = "0d2fxljdih3nc5dqx41hjzic3141ajil94m8kdbpryq569dpsbvb"; unpack = true; }'

runs in 17 MiB (was 346 MiB).
---
 src/libstore/builtins/fetchurl.cc | 78 +++++++++++++++++--------------
 src/libutil/util.cc               | 17 +++++++
 src/libutil/util.hh               |  2 +
 3 files changed, 62 insertions(+), 35 deletions(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 4ca4a838e3c..1f4abd374f5 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -22,52 +22,60 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         return i->second;
     };
 
-    auto fetch = [&](const string & url) {
-        /* No need to do TLS verification, because we check the hash of
-           the result anyway. */
-        DownloadRequest request(url);
-        request.verifyTLS = false;
-        request.decompress = false;
-
-        /* Note: have to use a fresh downloader here because we're in
-           a forked process. */
-        auto data = makeDownloader()->download(request);
-        assert(data.data);
-
-        return data.data;
-    };
+    Path storePath = getAttr("out");
+    auto mainUrl = getAttr("url");
+
+    /* Note: have to use a fresh downloader here because we're in
+       a forked process. */
+    auto downloader = makeDownloader();
+
+    auto fetch = [&](const std::string & url) {
+
+        auto source = sinkToSource([&](Sink & sink) {
+
+            /* No need to do TLS verification, because we check the hash of
+               the result anyway. */
+            DownloadRequest request(url);
+            request.verifyTLS = false;
+            request.decompress = false;
+
+            downloader->download(std::move(request), sink);
+        });
+
+        if (get(drv.env, "unpack", "") == "1") {
 
-    std::shared_ptr<std::string> data;
+            if (hasSuffix(mainUrl, ".xz")) {
+                auto source2 = sinkToSource([&](Sink & sink) {
+                    decompress("xz", *source, sink);
+                });
+                restorePath(storePath, *source2);
+            } else
+                restorePath(storePath, *source);
 
+        } else
+              writeFile(storePath, *source);
+
+        auto executable = drv.env.find("executable");
+        if (executable != drv.env.end() && executable->second == "1") {
+            if (chmod(storePath.c_str(), 0755) == -1)
+                throw SysError(format("making '%1%' executable") % storePath);
+        }
+    };
+
+    /* Try the hashed mirrors first. */
     if (getAttr("outputHashMode") == "flat")
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
                 auto ht = parseHashType(getAttr("outputHashAlgo"));
-                data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
-                break;
+                fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
+                return;
             } catch (Error & e) {
                 debug(e.what());
             }
 
-    if (!data) data = fetch(getAttr("url"));
-
-    Path storePath = getAttr("out");
-
-    auto unpack = drv.env.find("unpack");
-    if (unpack != drv.env.end() && unpack->second == "1") {
-        if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6))
-            data = decompress("xz", *data);
-        StringSource source(*data);
-        restorePath(storePath, source);
-    } else
-        writeFile(storePath, *data);
-
-    auto executable = drv.env.find("executable");
-    if (executable != drv.env.end() && executable->second == "1") {
-        if (chmod(storePath.c_str(), 0755) == -1)
-            throw SysError(format("making '%1%' executable") % storePath);
-    }
+    /* Otherwise try the specified URL. */
+    fetch(mainUrl);
 }
 
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 22753f4b7ec..6bc64ae75a4 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -328,6 +328,23 @@ void writeFile(const Path & path, const string & s, mode_t mode)
 }
 
 
+void writeFile(const Path & path, Source & source, mode_t mode)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError(format("opening file '%1%'") % path);
+
+    std::vector<unsigned char> buf(64 * 1024);
+
+    while (true) {
+        try {
+            auto n = source.read(buf.data(), buf.size());
+            writeFull(fd.get(), (unsigned char *) buf.data(), n);
+        } catch (EndOfFile &) { break; }
+    }
+}
+
+
 string readLine(int fd)
 {
     string s;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index af97b76d90b..fc25d27758c 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -103,6 +103,8 @@ void readFile(const Path & path, Sink & sink);
 /* Write a string to a file. */
 void writeFile(const Path & path, const string & s, mode_t mode = 0666);
 
+void writeFile(const Path & path, Source & source, mode_t mode = 0666);
+
 /* Read a line from a file descriptor. */
 string readLine(int fd);
 

From a4c16188760fde2122df534f5db6b9ef26c9bc22 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Apr 2018 21:26:16 +0200
Subject: [PATCH 1052/2196] Fix deadlock in
 RemoteStore::queryPathInfoUncached()

---
 src/libstore/remote-store.cc | 53 +++++++++++++++++++-----------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b72c940cdcf..76b5660355e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -297,31 +297,34 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
     Callback<std::shared_ptr<ValidPathInfo>> callback)
 {
     try {
-        auto conn(connections->get());
-        conn->to << wopQueryPathInfo << path;
-        try {
-            conn->processStderr();
-        } catch (Error & e) {
-            // Ugly backwards compatibility hack.
-            if (e.msg().find("is not valid") != std::string::npos)
-                throw InvalidPath(e.what());
-            throw;
-        }
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-            bool valid; conn->from >> valid;
-            if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
-        }
-        auto info = std::make_shared<ValidPathInfo>();
-        info->path = path;
-        info->deriver = readString(conn->from);
-        if (info->deriver != "") assertStorePath(info->deriver);
-        info->narHash = Hash(readString(conn->from), htSHA256);
-        info->references = readStorePaths<PathSet>(*this, conn->from);
-        conn->from >> info->registrationTime >> info->narSize;
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-            conn->from >> info->ultimate;
-            info->sigs = readStrings<StringSet>(conn->from);
-            conn->from >> info->ca;
+        std::shared_ptr<ValidPathInfo> info;
+        {
+            auto conn(connections->get());
+            conn->to << wopQueryPathInfo << path;
+            try {
+                conn->processStderr();
+            } catch (Error & e) {
+                // Ugly backwards compatibility hack.
+                if (e.msg().find("is not valid") != std::string::npos)
+                    throw InvalidPath(e.what());
+                throw;
+            }
+            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
+                bool valid; conn->from >> valid;
+                if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
+            }
+            info = std::make_shared<ValidPathInfo>();
+            info->path = path;
+            info->deriver = readString(conn->from);
+            if (info->deriver != "") assertStorePath(info->deriver);
+            info->narHash = Hash(readString(conn->from), htSHA256);
+            info->references = readStorePaths<PathSet>(*this, conn->from);
+            conn->from >> info->registrationTime >> info->narSize;
+            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+                conn->from >> info->ultimate;
+                info->sigs = readStrings<StringSet>(conn->from);
+                conn->from >> info->ca;
+            }
         }
         callback(std::move(info));
     } catch (...) { callback.rethrow(); }

From 3c2de9830dfa9e41927c666b7a7083c53886fb89 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 16 Apr 2018 11:14:39 +0200
Subject: [PATCH 1053/2196] RemoteStore::addToStore(): Ensure capacity is
 decreased on exceptions

---
 src/libstore/remote-store.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 76b5660355e..ea86ef052f5 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -7,6 +7,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "pool.hh"
+#include "finally.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -439,8 +440,10 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
         conn->to.written = 0;
         conn->to.warn = true;
         connections->incCapacity();
-        dumpPath(srcPath, conn->to, filter);
-        connections->decCapacity();
+        {
+            Finally cleanup([&]() { connections->decCapacity(); });
+            dumpPath(srcPath, conn->to, filter);
+        }
         conn->to.warn = false;
         conn->processStderr();
     } catch (SysError & e) {

From 8dd2e28374b6b2f995d1c408b24ec9b838f62626 Mon Sep 17 00:00:00 2001
From: Michael Mercier <mickours@libr.fr>
Date: Wed, 16 May 2018 17:48:36 +0200
Subject: [PATCH 1054/2196] Explain fetchTarball timeout behavior in the doc

---
 doc/manual/expressions/builtins.xml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index ac1fe7e2faf..7bc0d266724 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -280,8 +280,17 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo
 
 stdenv.mkDerivation { … }
 </programlisting>
+    </para>
+
+    <para>The fetched tarball is cached for a certain amount of time
+    (1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>.
+    You can change the cache timeout either on the command line with
+    <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
+    in the Nix configuration file with this option:
+    <literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>.
+    </para>
 
-    Note that when obtaining the hash with <varname>nix-prefetch-url
+    <para>Note that when obtaining the hash with <varname>nix-prefetch-url
     </varname> the option <varname>--unpack</varname> is required.
     </para>
 

From d459d3307c7b1b10f6489ed048fff192e7834928 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 30 May 2018 09:11:09 -0400
Subject: [PATCH 1055/2196] nix-daemon.sh profile script: operate under `set
 -u`

If the profile is sourced inside a script with `set -u`, the check for
__ETC_PROFILE_NIX_SOURCED and NIX_SSL_CERT_FILE would raise an error.
A simple guard around this check allows the script to operate under
standard environments (where it is fairly reasonable to assume USER
and HOME are set.)
---
 scripts/nix-profile-daemon.sh.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 52070e36a1e..1be9a0755d8 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -1,5 +1,5 @@
 # Only execute this file once per shell.
-if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
+if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
 # Set up secure multi-user builds: non-root users build through the
@@ -51,7 +51,7 @@ fi
 
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if [ ! -z "$NIX_SSL_CERT_FILE" ]; then
+if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
 elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt

From 6ba1726eeb58cd8e7d69e2a090346f26fdf634a5 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 30 May 2018 09:35:21 -0400
Subject: [PATCH 1056/2196] install-multi-user: support 'set -u' runs, closes
 #2193

---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index b4eec2fe94c..6ee8dd48582 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -727,7 +727,7 @@ setup_default_profile() {
     _sudo "to installing a bootstrapping Nix in to the default Profile" \
           HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
 
-    if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
+    if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then
         _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
               HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt

From 4caaa4c5fe889be65932183e11556ee4d1fcfcac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 May 2018 17:17:50 +0200
Subject: [PATCH 1057/2196] Move installer script from nixos-homepage

---
 scripts/install.in | 67 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 scripts/install.in

diff --git a/scripts/install.in b/scripts/install.in
new file mode 100644
index 00000000000..32cd2494225
--- /dev/null
+++ b/scripts/install.in
@@ -0,0 +1,67 @@
+#!/bin/sh
+
+# This script installs the Nix package manager on your system by
+# downloading a binary distribution and running its installer script
+# (which in turn creates and populates /nix).
+
+{ # Prevent execution if this script was only partially downloaded
+oops() {
+    echo "$0:" "$@" >&2
+    exit 1
+}
+
+tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
+          oops "Can\'t create temporary directory for downloading the Nix binary tarball")"
+cleanup() {
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT INT QUIT TERM
+
+require_util() {
+    type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 ||
+        oops "you do not have '$1' installed, which I need to $2"
+}
+
+case "$(uname -s).$(uname -m)" in
+    Linux.x86_64) system=x86_64-linux; hash=[%nix_hash_x86_64_linux%];;
+    Linux.i?86) system=i686-linux; hash=[%nix_hash_i686_linux%];;
+    Linux.aarch64) system=aarch64-linux; hash=[%nix_hash_aarch64_linux%];;
+    Darwin.x86_64) system=x86_64-darwin; hash=[%nix_hash_x86_64_darwin%];;
+    *) oops "sorry, there is no binary distribution of Nix for your platform";;
+esac
+
+url="https://nixos.org/releases/nix/nix-[%latestNixVersion%]/nix-[%latestNixVersion%]-$system.tar.bz2"
+
+tarball="$tmpDir/$(basename "$tmpDir/nix-[%latestNixVersion%]-$system.tar.bz2")"
+
+require_util curl "download the binary tarball"
+require_util bzcat "decompress the binary tarball"
+require_util tar "unpack the binary tarball"
+
+echo "downloading Nix [%latestNixVersion%] binary tarball for $system from '$url' to '$tmpDir'..."
+curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
+
+if type sha256sum > /dev/null 2>&1; then
+    hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
+elif type shasum > /dev/null 2>&1; then
+    hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
+elif type openssl > /dev/null 2>&1; then
+    hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
+else
+    oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"
+fi
+
+if [ "$hash" != "$hash2" ]; then
+    oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2"
+fi
+
+unpack=$tmpDir/unpack
+mkdir -p "$unpack"
+< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
+
+script=$(echo "$unpack"/*/install)
+
+[ -e "$script" ] || oops "installation script is missing from the binary tarball!"
+"$script" "$@"
+
+} # End of wrapping

From 7c3c635d4fa27a36fcba239b7cb980bab06ce482 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 May 2018 17:40:08 +0200
Subject: [PATCH 1058/2196] release.nix: Generate the installer script

---
 release.nix        | 21 +++++++++++++++++++++
 scripts/install.in | 14 +++++++-------
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/release.nix b/release.nix
index 37deb8e7ee3..321f1688367 100644
--- a/release.nix
+++ b/release.nix
@@ -275,6 +275,24 @@ let
         '';
 
 
+    installerScript =
+      pkgs.runCommand "installer-script"
+        { buildInputs = [ build.x86_64-linux ];
+        }
+        ''
+          mkdir -p $out/nix-support
+
+          substitute ${./scripts/install.in} $out/install \
+            ${pkgs.lib.concatMapStrings
+              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
+              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+            } \
+            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+
+          echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+        '';
+
+
     # Aggregate job containing the release-critical jobs.
     release = pkgs.releaseTools.aggregate {
       name = "nix-${tarball.version}";
@@ -284,14 +302,17 @@ let
           build.i686-linux
           build.x86_64-darwin
           build.x86_64-linux
+          build.aarch64-linux
           binaryTarball.i686-linux
           binaryTarball.x86_64-darwin
           binaryTarball.x86_64-linux
+          binaryTarball.aarch64-linux
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
           tests.evalNixpkgs
           tests.evalNixOS
+          installerScript
         ];
     };
 
diff --git a/scripts/install.in b/scripts/install.in
index 32cd2494225..26ab85ba099 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -23,22 +23,22 @@ require_util() {
 }
 
 case "$(uname -s).$(uname -m)" in
-    Linux.x86_64) system=x86_64-linux; hash=[%nix_hash_x86_64_linux%];;
-    Linux.i?86) system=i686-linux; hash=[%nix_hash_i686_linux%];;
-    Linux.aarch64) system=aarch64-linux; hash=[%nix_hash_aarch64_linux%];;
-    Darwin.x86_64) system=x86_64-darwin; hash=[%nix_hash_x86_64_darwin%];;
+    Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
+    Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
+    Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
+    Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-[%latestNixVersion%]/nix-[%latestNixVersion%]-$system.tar.bz2"
+url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
 
-tarball="$tmpDir/$(basename "$tmpDir/nix-[%latestNixVersion%]-$system.tar.bz2")"
+tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
 
 require_util curl "download the binary tarball"
 require_util bzcat "decompress the binary tarball"
 require_util tar "unpack the binary tarball"
 
-echo "downloading Nix [%latestNixVersion%] binary tarball for $system from '$url' to '$tmpDir'..."
+echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
 
 if type sha256sum > /dev/null 2>&1; then

From 5d4a9d56777d8139705a920c79c77142b40a6d07 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 May 2018 17:47:30 +0200
Subject: [PATCH 1059/2196] Fix static assertion failure on 32-bit systems

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index a3f411a5fb1..517952164a9 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -307,7 +307,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     assert(gcInitialised);
 
-    static_assert(sizeof(Env) == 16, "environment must be 16 bytes");
+    static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {

From bbbfc180d9024991b9879e6d3c3c91d078bb499a Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Wed, 30 May 2018 12:34:41 -0400
Subject: [PATCH 1060/2196] Explicitly describe the effects of filterSource
 excluding a directory.

---
 doc/manual/expressions/builtins.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 54f697d852e..c7a7b49be9f 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -459,7 +459,9 @@ stdenv.mkDerivation {
       <literal>"unknown"</literal> (for other kinds of files such as
       device nodes or fifos — but note that those cannot be copied to
       the Nix store, so if the predicate returns
-      <literal>true</literal> for them, the copy will fail).</para>
+      <literal>true</literal> for them, the copy will fail). If you
+      exclude a directory, the entire corresponding subtree of
+      <replaceable>e2</replaceable> will be excluded.</para>
 
     </listitem>
 

From f0d9909f108b2f8bffaf65743e283e773204b62f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 31 May 2018 11:54:32 +0200
Subject: [PATCH 1061/2196] upload-release.pl: Copy the install script and sign
 everything

Note: this means that for those doing 'curl | sh', you can now pin a
specific version, e.g.

  curl https://nixos.org/releases/nix/nix-2.0.4/install | sh

https://nixos.org/nix/{install,install.sig} are now just symlinks to
the corresponding files in the latest release directory.
---
 maintainers/upload-release.pl | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index aa7633a709c..9b0a09e6c83 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -76,15 +76,20 @@ sub downloadFile {
 
     write_file("$dstFile.sha256", $sha256_expected);
 
+    if (! -e "$dstFile.asc") {
+        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    }
+
     return ($dstFile, $sha256_expected);
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
 my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
-my ($tarball_i686_linux, $tarball_i686_linux_hash) = downloadFile("binaryTarball.i686-linux", "1");
-my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTarball.x86_64-linux", "1");
-my ($tarball_aarch64_linux, $tarball_aarch64_linux_hash) = downloadFile("binaryTarball.aarch64-linux", "1");
-my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("binaryTarball.i686-linux", "1");
+downloadFile("binaryTarball.x86_64-linux", "1");
+downloadFile("binaryTarball.aarch64-linux", "1");
+downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("installerScript", "1");
 
 # Update Nixpkgs in a very hacky way.
 system("cd $nixpkgsDir && git pull") == 0 or die;
@@ -144,17 +149,6 @@ sub getStorePath {
 write_file("$siteDir/nix-release.tt",
            "[%-\n" .
            "latestNixVersion = \"$version\"\n" .
-           "nix_hash_i686_linux = \"$tarball_i686_linux_hash\"\n" .
-           "nix_hash_x86_64_linux = \"$tarball_x86_64_linux_hash\"\n" .
-           "nix_hash_aarch64_linux = \"$tarball_aarch64_linux_hash\"\n" .
-           "nix_hash_x86_64_darwin = \"$tarball_x86_64_darwin_hash\"\n" .
            "-%]\n");
 
-system("cd $siteDir && nix-shell --run 'make nix/install nix/install.sig'") == 0 or die;
-
-copy("$siteDir/nix/install", "$siteDir/nix/install-$version") or die;
-copy("$siteDir/nix/install.sig", "$siteDir/nix/install-$version.sig") or die;
-
-system("cd $siteDir && git add nix/install-$version nix/install-$version.sig") == 0 or die;
-
 system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

From 73357500aca2ad4a2d82111993fbd5aaf0af4ec6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 Jun 2018 14:14:22 +0200
Subject: [PATCH 1062/2196] Improve binary cache upload messages

Don't say "download" when we mean "upload".
---
 src/libstore/download.cc                | 18 ++++++++++++++----
 src/libstore/http-binary-cache-store.cc |  2 +-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 72a08ef0089..0b768b59178 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -102,7 +102,9 @@ struct CurlDownloader : public Downloader
             Callback<DownloadResult> callback)
             : downloader(downloader)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct)
+            , act(*logger, lvlTalkative, actDownload,
+                fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
+                {request.uri}, request.parentAct)
             , callback(callback)
         {
             if (!request.expectedETag.empty())
@@ -371,12 +373,20 @@ struct CurlDownloader : public Downloader
 
                 attempt++;
 
+                auto verb = request.data ? "upload" : "download";
+
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)
+                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", verb, request.uri))
                     : httpStatus != 0
-                      ? DownloadError(err, format("unable to download '%s': HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code))
-                      : DownloadError(err, format("unable to download '%s': %s (%d)") % request.uri % curl_easy_strerror(code) % code);
+                    ? DownloadError(err,
+                        fmt("unable to %s '%s': HTTP error %d",
+                            verb, request.uri, httpStatus)
+                        + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
+                        )
+                    : DownloadError(err,
+                        fmt("unable to %s '%s': %s (%d)",
+                            verb, request.uri, curl_easy_strerror(code), code));
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. */
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 6fdae40e360..ab524d523cf 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -73,7 +73,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         try {
             getDownloader()->download(req);
         } catch (DownloadError & e) {
-            throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg());
+            throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
 

From 54b1c596435b0aaf3a2557652ad4bf74d5756514 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 1 Jun 2018 10:53:28 -0500
Subject: [PATCH 1063/2196] download.cc: fix readCallback return type (int ->
 size_t)

I'm not sure if curl ever asks for enough data at once
for truncation to occur but better safe than sorry.
---
 src/libstore/download.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 0b768b59178..48d9a42e590 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -213,7 +213,7 @@ struct CurlDownloader : public Downloader
         }
 
         size_t readOffset = 0;
-        int readCallback(char *buffer, size_t size, size_t nitems)
+        size_t readCallback(char *buffer, size_t size, size_t nitems)
         {
             if (readOffset == request.data->length())
                 return 0;
@@ -224,7 +224,7 @@ struct CurlDownloader : public Downloader
             return count;
         }
 
-        static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
+        static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
         {
             return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
         }

From a936a19da3c338ae55a31eda8ce446cbbc37de5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jun 2018 13:40:45 +0200
Subject: [PATCH 1064/2196] Remove non-existent NIX_DEBUG_SUBST

---
 tests/binary-cache.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 7365a550e57..28b26568693 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -88,7 +88,7 @@ nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get inf
 mkdir $cacheDir/tmp
 mv $cacheDir/*.nar* $cacheDir/tmp/
 
-NIX_DEBUG_SUBST=1 nix-build --substituters "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
+nix-build --substituters "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
 
 mv $cacheDir/tmp/* $cacheDir/
 

From a2ec7a3bfd33a4792185a74e2ae20b48f7ac2de9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jun 2018 14:37:26 +0200
Subject: [PATCH 1065/2196] Further improve upload messages

---
 src/libstore/download.cc | 14 ++++++--------
 src/libstore/download.hh |  5 +++++
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 48d9a42e590..9a2d0dafa51 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -307,8 +307,8 @@ struct CurlDownloader : public Downloader
             if (effectiveUrlCStr)
                 result.effectiveUrl = effectiveUrlCStr;
 
-            debug(format("finished download of '%s'; curl status = %d, HTTP status = %d, body = %d bytes")
-                % request.uri % code % httpStatus % (result.data ? result.data->size() : 0));
+            debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
+                request.verb(), request.uri, code, httpStatus, result.data ? result.data->size() : 0);
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;
@@ -373,20 +373,18 @@ struct CurlDownloader : public Downloader
 
                 attempt++;
 
-                auto verb = request.data ? "upload" : "download";
-
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", verb, request.uri))
+                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
                     ? DownloadError(err,
                         fmt("unable to %s '%s': HTTP error %d",
-                            verb, request.uri, httpStatus)
+                            request.verb(), request.uri, httpStatus)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
                     : DownloadError(err,
                         fmt("unable to %s '%s': %s (%d)",
-                            verb, request.uri, curl_easy_strerror(code), code));
+                            request.verb(), request.uri, curl_easy_strerror(code), code));
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. */
@@ -547,7 +545,7 @@ struct CurlDownloader : public Downloader
             }
 
             for (auto & item : incoming) {
-                debug(format("starting download of %s") % item->request.uri);
+                debug("starting %s of %s", item->request.verb(), item->request.uri);
                 item->init();
                 curl_multi_add_handle(curlm, item->req);
                 item->active = true;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index f56274b2353..da55df7a6e7 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -25,6 +25,11 @@ struct DownloadRequest
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
+
+    std::string verb()
+    {
+        return data ? "upload" : "download";
+    }
 };
 
 struct DownloadResult

From ca06a9cea706bf06374efeed276cf3a90d2eaa80 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jun 2018 14:44:26 +0200
Subject: [PATCH 1066/2196] Improve binary cache fallback test

---
 src/libstore/download.cc |  5 +++++
 tests/binary-cache.sh    | 14 +++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 9a2d0dafa51..c7cffedbadc 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -574,6 +574,11 @@ struct CurlDownloader : public Downloader
 
     void enqueueItem(std::shared_ptr<DownloadItem> item)
     {
+        if (item->request.data
+            && !hasPrefix(item->request.uri, "http://")
+            && !hasPrefix(item->request.uri, "https://"))
+            throw nix::Error("uploading to '%s' is not supported", item->request.uri);
+
         {
             auto state(state_.lock());
             if (state->quit)
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 28b26568693..cbcdca89b13 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -52,9 +52,6 @@ export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1
 basicTests
 
 
-unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE
-
-
 # Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
 clearStore
 
@@ -86,11 +83,13 @@ clearStore
 nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info
 
 mkdir $cacheDir/tmp
-mv $cacheDir/*.nar* $cacheDir/tmp/
+mv $cacheDir/nar $cacheDir/nar2
 
-nix-build --substituters "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
+(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result)
 
-mv $cacheDir/tmp/* $cacheDir/
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback
+
+mv $cacheDir/nar2 $cacheDir/nar
 
 
 # Test whether building works if the binary cache contains an
@@ -107,6 +106,7 @@ if [ -n "$HAVE_SODIUM" ]; then
 
 # Create a signed binary cache.
 clearCache
+clearCacheCache
 
 declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 ))
 publicKey="$(cat $TEST_ROOT/pk1)"
@@ -117,7 +117,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
 res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
 otherKey="$(cat $TEST_ROOT/pk3)"
 
-nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
+_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 
 
 # Downloading should fail if we don't provide a key.

From 691b7582c76e05774548e84aba92ff0eb19b2589 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jun 2018 16:03:32 +0200
Subject: [PATCH 1067/2196] Handle HTTP 410 Gone errors

---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index c7cffedbadc..7a5deed3214 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -334,7 +334,7 @@ struct CurlDownloader : public Downloader
                 // We treat most errors as transient, but won't retry when hopeless
                 Error err = Transient;
 
-                if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) {
+                if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
                     // The file is definitely not there
                     err = NotFound;
                 } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {

From 4ac4f675df3da01b6d814cd328dd3219dd472ac9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Jun 2018 16:04:41 +0200
Subject: [PATCH 1068/2196] Don't require --fallback to recover from
 disappeared binary cache NARs

---
 src/libstore/binary-cache-store.cc |  6 +++++-
 src/libstore/build.cc              | 32 ++++++++++++++++++++----------
 src/libstore/store-api.hh          |  1 +
 tests/binary-cache.sh              | 17 ++++++++++++----
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 11fa3cae27a..76c0a1a891b 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -218,7 +218,11 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
     auto source = sinkToSource([this, url{info->url}](Sink & sink) {
-        getFile(url, sink);
+        try {
+            getFile(url, sink);
+        } catch (NoSuchBinaryCacheFile & e) {
+            throw SubstituteGone(e.what());
+        }
     });
 
     stats.narRead++;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 07b53378393..8eb19205970 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -733,7 +733,7 @@ class DerivationGoal : public Goal
 
     /* Whether to retry substituting the outputs after building the
        inputs. */
-    bool retrySubstitution = false;
+    bool retrySubstitution;
 
     /* The derivation stored at drvPath. */
     std::unique_ptr<BasicDerivation> drv;
@@ -1123,6 +1123,8 @@ void DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
+    retrySubstitution = false;
+
     for (auto & i : drv->outputs)
         worker.store.addTempRoot(i.second.path);
 
@@ -1161,7 +1163,7 @@ void DerivationGoal::outputsSubstituted()
     /*  If the substitutes form an incomplete closure, then we should
         build the dependencies of this derivation, but after that, we
         can still use the substitutes for this derivation itself. */
-    if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true;
+    if (nrIncompleteClosure > 0) retrySubstitution = true;
 
     nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
 
@@ -3524,8 +3526,8 @@ class SubstitutionGoal : public Goal
     /* The current substituter. */
     std::shared_ptr<Store> sub;
 
-    /* Whether any substituter can realise this path. */
-    bool hasSubstitute;
+    /* Whether a substituter failed. */
+    bool substituterFailed = false;
 
     /* Path info returned by the substituter's query info operation. */
     std::shared_ptr<const ValidPathInfo> info;
@@ -3589,7 +3591,6 @@ class SubstitutionGoal : public Goal
 
 SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair)
     : Goal(worker)
-    , hasSubstitute(false)
     , repair(repair)
 {
     this->storePath = storePath;
@@ -3653,9 +3654,9 @@ void SubstitutionGoal::tryNext()
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
            build. */
-        amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
+        amDone(substituterFailed ? ecFailed : ecNoSubstituters);
 
-        if (hasSubstitute) {
+        if (substituterFailed) {
             worker.failedSubstitutions++;
             worker.updateProgress();
         }
@@ -3691,8 +3692,6 @@ void SubstitutionGoal::tryNext()
 
     worker.updateProgress();
 
-    hasSubstitute = true;
-
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
@@ -3807,8 +3806,19 @@ void SubstitutionGoal::finished()
         state = &SubstitutionGoal::init;
         worker.waitForAWhile(shared_from_this());
         return;
-    } catch (Error & e) {
-        printError(e.msg());
+    } catch (std::exception & e) {
+        printError(e.what());
+
+        /* Cause the parent build to fail unless --fallback is given,
+           or the substitute has disappeared. The latter case behaves
+           the same as the substitute never having existed in the
+           first place. */
+        try {
+            throw;
+        } catch (SubstituteGone &) {
+        } catch (...) {
+            substituterFailed = true;
+        }
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6ee2d550679..7c5b495a448 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -22,6 +22,7 @@ MakeError(SubstError, Error)
 MakeError(BuildError, Error) /* denotes a permanent build failure */
 MakeError(InvalidPath, Error)
 MakeError(Unsupported, Error)
+MakeError(SubstituteGone, Error)
 
 
 struct BasicDerivation;
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index cbcdca89b13..eb58ae7c12a 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -76,19 +76,28 @@ if nix-store --substituters "file://$cacheDir" -r $outPath; then
 fi
 
 
-# Test whether fallback works if we have cached info but the
-# corresponding NAR has disappeared.
+# Test whether fallback works if a NAR has disappeared. This does not require --fallback.
 clearStore
 
-nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info
+mv $cacheDir/nar $cacheDir/nar2
+
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result
+
+mv $cacheDir/nar2 $cacheDir/nar
+
+
+# Test whether fallback works if a NAR is corrupted. This does require --fallback.
+clearStore
 
-mkdir $cacheDir/tmp
 mv $cacheDir/nar $cacheDir/nar2
+mkdir $cacheDir/nar
+for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done
 
 (! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result)
 
 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback
 
+rm -rf $cacheDir/nar
 mv $cacheDir/nar2 $cacheDir/nar
 
 

From 5ee4472b8bd5a3bda25599eb710f2c979e9c8876 Mon Sep 17 00:00:00 2001
From: Aleksandr Pashkov <pashkov.alex+github@gmail.com>
Date: Tue, 5 Jun 2018 23:13:51 +0300
Subject: [PATCH 1069/2196] tests: more robust check for user namespaces
 availability (canUseSandbox)

Issue https://github.com/NixOS/nix/issues/2165
---
 release-common.nix | 2 +-
 tests/common.sh.in | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index d7fb8125f25..fbdb8aca1fd 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -57,7 +57,7 @@ rec {
       git
       mercurial
     ]
-    ++ lib.optional stdenv.isLinux libseccomp
+    ++ lib.optional stdenv.isLinux [libseccomp pkgs.utillinux]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 195205988af..fddd25b366b 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -94,11 +94,9 @@ canUseSandbox() {
         return 1
     fi
 
-    if [ -e /proc/sys/kernel/unprivileged_userns_clone ]; then
-        if [ "$(cat /proc/sys/kernel/unprivileged_userns_clone)" != 1 ]; then
-            echo "Unprivileged user namespaces disabled by sysctl, skipping this test..."
-            return 1
-        fi
+    if ! unshare --user true ; then
+        echo "Unprivileged user namespaces disabled by sysctl, skipping this test..."
+        return 1
     fi
 
     return 0

From b600d46412892d7192d3a032354dbab4a5c6390e Mon Sep 17 00:00:00 2001
From: Lorenzo Manacorda <lorenzo@mailbox.org>
Date: Wed, 6 Jun 2018 14:50:47 +0200
Subject: [PATCH 1070/2196] Dockerfile: 2.0.4

---
 misc/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
index 0f69d02df25..eeca12353d9 100644
--- a/misc/docker/Dockerfile
+++ b/misc/docker/Dockerfile
@@ -4,8 +4,8 @@ FROM alpine
 RUN apk add --update openssl
 
 # Download Nix and install it into the system.
-RUN wget https://nixos.org/releases/nix/nix-2.0.2/nix-2.0.2-x86_64-linux.tar.bz2 \
-  && echo "d0c2492d7d8f824e3b1ace15a1a58f64a0a8faacc59936ebedfe18905d982d7c  nix-2.0.2-x86_64-linux.tar.bz2" | sha256sum -c \
+RUN wget https://nixos.org/releases/nix/nix-2.0.4/nix-2.0.4-x86_64-linux.tar.bz2 \
+  && echo "d6db178007014ed47ad5460c1bd5b2cb9403b1ec543a0d6507cb27e15358341f  nix-2.0.4-x86_64-linux.tar.bz2" | sha256sum -c \
   && tar xjf nix-*-x86_64-linux.tar.bz2 \
   && addgroup -g 30000 -S nixbld \
   && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \

From b333e6d6fee471977c2713ae369da8a48d518f14 Mon Sep 17 00:00:00 2001
From: Aleksandr Pashkov <pashkov.alex+github@gmail.com>
Date: Thu, 7 Jun 2018 23:08:43 +0300
Subject: [PATCH 1071/2196] pr-2211: cleanup

---
 release-common.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release-common.nix b/release-common.nix
index fbdb8aca1fd..a50cc5cf506 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -57,7 +57,7 @@ rec {
       git
       mercurial
     ]
-    ++ lib.optional stdenv.isLinux [libseccomp pkgs.utillinux]
+    ++ lib.optional stdenv.isLinux [libseccomp utillinux]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {

From 170165ee5adea35707cdb635f5116432ded916aa Mon Sep 17 00:00:00 2001
From: AmineChikhaoui <amine.chikhaoui91@gmail.com>
Date: Fri, 8 Jun 2018 11:43:32 +0100
Subject: [PATCH 1072/2196] use debug instead of printTalkative to avoid
 spamming hydra queue runner logs.

---
 src/libstore/s3-binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 239739bae83..26144ccb40c 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -299,7 +299,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                     &transferHandle) {
               //FIXME: find a way to properly abort the multipart upload.
               checkInterrupt();
-              printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
+              debug("upload progress ('%s'): '%d' of '%d' bytes",
                              path,
                              transferHandle->GetBytesTransferred(),
                              transferHandle->GetBytesTotalSize());

From a717ad7484e098839017a876bba5705975563d08 Mon Sep 17 00:00:00 2001
From: Aleksandr Pashkov <pashkov.alex+github@gmail.com>
Date: Fri, 8 Jun 2018 23:31:42 +0300
Subject: [PATCH 1073/2196] pr-2211: cleanup

---
 release-common.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release-common.nix b/release-common.nix
index a50cc5cf506..9138c112784 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -57,7 +57,7 @@ rec {
       git
       mercurial
     ]
-    ++ lib.optional stdenv.isLinux [libseccomp utillinux]
+    ++ lib.optionals stdenv.isLinux [libseccomp utillinux]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {

From 6bf7dc1baf272b37aa2731544f00491402d323ff Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 24 May 2018 16:23:05 -0500
Subject: [PATCH 1074/2196] download: if there are active requests, never sleep
 for 10s

In some versions/configurations libcurl doesn't handle timeouts
(especially DNS timeouts) in a way that wakes curl_multi_wait.

This doesn't appear to be a problem if using c-ares, FWIW.
---
 src/libstore/download.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 7a5deed3214..29bb56ea127 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -503,10 +503,11 @@ struct CurlDownloader : public Downloader
             extraFDs[0].fd = wakeupPipe.readSide.get();
             extraFDs[0].events = CURL_WAIT_POLLIN;
             extraFDs[0].revents = 0;
+            long maxSleepTimeMs = items.empty() ? 10000 : 100;
             auto sleepTimeMs =
                 nextWakeup != std::chrono::steady_clock::time_point()
                 ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count())
-                : 10000;
+                : maxSleepTimeMs;
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)

From 17bc7579809e2afa3870ac1d85f7027d3681f506 Mon Sep 17 00:00:00 2001
From: Michael Raskin <7c6f434c@mail.ru>
Date: Sun, 10 Jun 2018 14:20:18 +0200
Subject: [PATCH 1075/2196] manual: builtins.fromJSON: remove the claim that
 floats are not allowed

floating-point numbers are supported now, including the fromJSON
builtin. Reported on IRC by inquisitiv3
---
 doc/manual/expressions/builtins.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index c7a7b49be9f..07d8357b40b 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -509,8 +509,7 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
 </programlisting>
 
     returns the value <literal>{ x = [ 1 2 3 ]; y = null;
-    }</literal>. Floating point numbers are not
-    supported.</para></listitem>
+    }</literal>.</para></listitem>
 
   </varlistentry>
 

From 8c567afe355dff8cb86dd15c94755eb6500f48cd Mon Sep 17 00:00:00 2001
From: Symphorien Gibol <symphorien+git@xlumurb.eu>
Date: Mon, 11 Jun 2018 15:59:32 +0200
Subject: [PATCH 1076/2196] libstore/gc.cc: ignore ESRCH when reading /proc

If a process disappears between the time /proc/[pid]/maps is opened and
the time it is read, the read() syscall will return ESRCH. This should be ignored.
---
 src/libstore/gc.cc | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ba49749d830..233a70bd2d2 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -425,25 +425,28 @@ PathSet LocalStore::findRuntimeRoots()
                         readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
                     }
                 }
-                if (errno)
+                if (errno) {
+                    if (errno == ESRCH)
+                        continue;
                     throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
-                fdDir.reset();
-
-                auto mapLines =
-                    tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
-                for (const auto& line : mapLines) {
-                    auto match = std::smatch{};
-                    if (std::regex_match(line, match, mapRegex))
-                        paths.emplace(match[1]);
                 }
+                fdDir.reset();
 
                 try {
+                    auto mapLines =
+                        tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
+                    for (const auto& line : mapLines) {
+                        auto match = std::smatch{};
+                        if (std::regex_match(line, match, mapRegex))
+                            paths.emplace(match[1]);
+                    }
+
                     auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         paths.emplace(i->str());
                 } catch (SysError & e) {
-                    if (errno == ENOENT || errno == EACCES)
+                    if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
                     throw;
                 }

From 72a78beb34c7ce6cd88d2801f3fcf2d8aa83a5aa Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Tue, 12 Jun 2018 13:05:14 +0200
Subject: [PATCH 1077/2196] Fix #2162: use getaddrinfo instead of curl to
 preload NSS

---
 src/libstore/build.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8eb19205970..d75ca0be86e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -29,7 +29,9 @@
 #include <sys/utsname.h>
 #include <sys/select.h>
 #include <sys/resource.h>
+#include <sys/socket.h>
 #include <fcntl.h>
+#include <netdb.h>
 #include <unistd.h>
 #include <errno.h>
 #include <cstring>
@@ -1777,12 +1779,14 @@ static std::once_flag dns_resolve_flag;
 static void preloadNSS() {
     /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
        one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
        load its lookup libraries in the parent before any child gets a chance to. */
     std::call_once(dns_resolve_flag, []() {
-        DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid");
-        request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning
-        try { getDownloader()->download(request); } catch (...) {}
+        struct addrinfo *res = NULL;
+
+        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
+            if (res) freeaddrinfo(res);
+        }
     });
 }
 

From 6ad0a2f749b4db6b24e8b86625883b0b43ad7ba1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jun 2018 15:58:19 +0200
Subject: [PATCH 1078/2196] Remove duplicate definition of allocBytes()

---
 src/libexpr/attr-set.cc    | 16 +---------------
 src/libexpr/eval-inline.hh | 14 ++++++++++++++
 src/libexpr/eval.cc        | 14 --------------
 3 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index 0474865c6d7..0785897d251 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -1,5 +1,5 @@
 #include "attr-set.hh"
-#include "eval.hh"
+#include "eval-inline.hh"
 
 #include <algorithm>
 
@@ -7,20 +7,6 @@
 namespace nix {
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 8cc50e56135..ad172c37c7b 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -78,4 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_malloc(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 517952164a9..01658aa25f2 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -43,20 +43,6 @@ static char * dupString(const char * s)
 }
 
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-static void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_malloc(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();

From 24c68069948681366ff2351dd7400af6e69006d9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jun 2018 16:06:01 +0200
Subject: [PATCH 1079/2196] Cache parse trees

This prevents EvalState::resetFileCache() from parsing everything all
over again.
---
 src/libexpr/eval.cc | 13 +++++++++++--
 src/libexpr/eval.hh |  8 ++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 01658aa25f2..facfddbb530 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -332,7 +332,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    fileEvalCache.clear();
 }
 
 
@@ -711,7 +710,17 @@ void EvalState::evalFile(const Path & path_, Value & v)
     }
 
     printTalkative("evaluating file '%1%'", path2);
-    Expr * e = parseExprFromFile(checkSourcePath(path2));
+    Expr * e = nullptr;
+
+    auto j = fileParseCache.find(path2);
+    if (j != fileParseCache.end())
+        e = j->second;
+
+    if (!e)
+        e = parseExprFromFile(checkSourcePath(path2));
+
+    fileParseCache[path2] = e;
+
     try {
         eval(e, v);
     } catch (Error & e) {
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 146f2125503..d0f298e168e 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -90,6 +90,14 @@ public:
 private:
     SrcToStore srcToStore;
 
+    /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
+    typedef std::map<Path, Expr *> FileParseCache;
+#endif
+    FileParseCache fileParseCache;
+
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;

From 30964103dc31c32ccb69912fdd73474cd65447e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Jun 2018 16:10:50 +0200
Subject: [PATCH 1080/2196] Add temporary stats

---
 src/libexpr/eval.cc | 28 +++++++++++++++++++++++++++-
 src/libexpr/eval.hh |  1 +
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index facfddbb530..1ab2b0ac188 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -13,6 +13,9 @@
 #include <sys/time.h>
 #include <sys/resource.h>
 
+#include <sys/time.h>
+#include <sys/resource.h>
+
 #if HAVE_BOEHMGC
 
 #include <gc/gc.h>
@@ -574,10 +577,19 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 }
 
 
+std::atomic<uint64_t> nrValuesFreed{0};
+
+void finalizeValue(void * obj, void * data)
+{
+    nrValuesFreed++;
+}
+
 Value * EvalState::allocValue()
 {
     nrValues++;
-    return (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) allocBytes(sizeof(Value));
+    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
+    return v;
 }
 
 
@@ -1707,6 +1719,20 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 }
 
 
+void EvalState::printStats2()
+{
+    struct rusage ru;
+    getrusage(RUSAGE_SELF, &ru);
+
+    GC_prof_stats_s gc;
+    GC_get_prof_stats(&gc, sizeof(gc));
+
+    printError("STATS %d %d %d %d %d %d",
+        nrValues, nrValuesFreed.load(), nrValues - nrValuesFreed,
+        ru.ru_maxrss,
+        gc.heapsize_full, gc.free_bytes_full);
+}
+
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index d0f298e168e..46bda86d084 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -276,6 +276,7 @@ public:
 
     /* Print statistics. */
     void printStats();
+    void printStats2();
 
     void realiseContext(const PathSet & context);
 

From 455d1f01d041baf58abc3c9baf5e59ee054d9c9b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jun 2018 15:41:37 +0200
Subject: [PATCH 1081/2196] Don't scan for roots in dynamic libraries

This reduces the risk of object liveness misdetection. For example,
Glibc has an internal variable "mp_" that often points to a Boehm
object, keeping it alive unnecessarily. Since we don't store any
actual roots in global variables, we can just disable data segment
scanning.

With this, the max RSS doing 100 evaluations of
nixos.tests.firefox.x86_64-linux.drvPath went from 718 MiB to 455 MiB.
---
 src/libexpr/eval.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1ab2b0ac188..113850bff1f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -188,8 +188,15 @@ void initGC()
 
 #if HAVE_BOEHMGC
     /* Initialise the Boehm garbage collector. */
+
+    /* Don't look for interior pointers. This reduces the odds of
+       misdetection a bit. */
     GC_set_all_interior_pointers(0);
 
+    /* We don't have any roots in data segments, so don't scan from
+       there. */
+    GC_set_no_dls(1);
+
     GC_INIT();
 
     GC_set_oom_fn(oomHandler);

From c905d8b0a84877d3cab10d4a8b8ad81895a6500e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jun 2018 16:16:03 +0200
Subject: [PATCH 1082/2196] GC_malloc -> GC_MALLOC

This makes it possible to build with -DGC_DEBUG.
---
 src/libexpr/eval-inline.hh |  2 +-
 src/libexpr/eval.cc        | 10 ++--------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index ad172c37c7b..c27116e3b44 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -83,7 +83,7 @@ inline void * allocBytes(size_t n)
 {
     void * p;
 #if HAVE_BOEHMGC
-    p = GC_malloc(n);
+    p = GC_MALLOC(n);
 #else
     p = calloc(n, 1);
 #endif
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 113850bff1f..e09297546c9 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -21,12 +21,6 @@
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#define NEW new (UseGC)
-
-#else
-
-#define NEW new
-
 #endif
 
 
@@ -37,7 +31,7 @@ static char * dupString(const char * s)
 {
     char * t;
 #if HAVE_BOEHMGC
-    t = GC_strdup(s);
+    t = GC_STRDUP(s);
 #else
     t = strdup(s);
 #endif
@@ -446,7 +440,7 @@ Value * EvalState::addPrimOp(const string & name,
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
     Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
-    v->primOp = NEW PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));

From 0629601da1d163a059fa19004256961f8ecdeb78 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Jun 2018 17:26:36 +0200
Subject: [PATCH 1083/2196] Move EvalState from the stack to the heap

EvalState contains a few counters (e.g. nrValues) that increase
quickly enough that they end up being interpreted as pointers by the
garbage collector. Moving it to the heap makes them invisible to the
garbage collector.

This reduces the max RSS doing 100 evaluations of
nixos.tests.firefox.x86_64-linux.drvPath from 455 MiB to 292 MiB.

Note: ideally, allocations would be much further up in the 64-bit
address space to reduce the odds of an integer being misinterpreted as
a pointer. Maybe we can use some linker magic to move the .bss segment
to a higher address.
---
 src/nix-build/nix-build.cc               | 28 ++++++++++++------------
 src/nix-instantiate/nix-instantiate.cc   | 20 ++++++++---------
 src/nix-prefetch-url/nix-prefetch-url.cc | 28 ++++++++++++------------
 src/nix/repl.cc                          |  4 ++--
 src/nix/upgrade-nix.cc                   | 12 +++++-----
 5 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 21d99878a51..de0e9118fd2 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -239,10 +239,10 @@ void mainWrapped(int argc, char * * argv)
 
     auto store = openStore();
 
-    EvalState state(myArgs.searchPath, store);
-    state.repair = repair;
+    auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+    state->repair = repair;
 
-    Bindings & autoArgs = *myArgs.getAutoArgs(state);
+    Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
     if (packages) {
         std::ostringstream joined;
@@ -268,7 +268,7 @@ void mainWrapped(int argc, char * * argv)
     std::vector<Expr *> exprs;
 
     if (readStdin)
-        exprs = {state.parseStdin()};
+        exprs = {state->parseStdin()};
     else
         for (auto i : left) {
             auto absolute = i;
@@ -276,13 +276,13 @@ void mainWrapped(int argc, char * * argv)
                 absolute = canonPath(absPath(i), true);
             } catch (Error e) {};
             if (fromArgs)
-                exprs.push_back(state.parseExprFromString(i, absPath(".")));
+                exprs.push_back(state->parseExprFromString(i, absPath(".")));
             else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
-                drvs.push_back(DrvInfo(state, store, absolute));
+                drvs.push_back(DrvInfo(*state, store, absolute));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
-                exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state,
+                exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
                     inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
         }
 
@@ -291,12 +291,12 @@ void mainWrapped(int argc, char * * argv)
 
     for (auto e : exprs) {
         Value vRoot;
-        state.eval(e, vRoot);
+        state->eval(e, vRoot);
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot));
-            state.forceValue(v);
-            getDerivations(state, v, "", autoArgs, drvs, false);
+            Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot));
+            state->forceValue(v);
+            getDerivations(*state, v, "", autoArgs, drvs, false);
         }
     }
 
@@ -332,12 +332,12 @@ void mainWrapped(int argc, char * * argv)
         if (shell == "") {
 
             try {
-                auto expr = state.parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
+                auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
 
                 Value v;
-                state.eval(expr, v);
+                state->eval(expr, v);
 
-                auto drv = getDerivation(state, v, false);
+                auto drv = getDerivation(*state, v, false);
                 if (!drv)
                     throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation");
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 5049460c754..eb6d34dd821 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -158,16 +158,16 @@ int main(int argc, char * * argv)
 
         auto store = openStore();
 
-        EvalState state(myArgs.searchPath, store);
-        state.repair = repair;
+        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+        state->repair = repair;
 
-        Bindings & autoArgs = *myArgs.getAutoArgs(state);
+        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
         if (attrPaths.empty()) attrPaths = {""};
 
         if (findFile) {
             for (auto & i : files) {
-                Path p = state.findFile(i);
+                Path p = state->findFile(i);
                 if (p == "") throw Error(format("unable to find '%1%'") % i);
                 std::cout << p << std::endl;
             }
@@ -175,20 +175,20 @@ int main(int argc, char * * argv)
         }
 
         if (readStdin) {
-            Expr * e = state.parseStdin();
-            processExpr(state, attrPaths, parseOnly, strict, autoArgs,
+            Expr * e = state->parseStdin();
+            processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         } else if (files.empty() && !fromArgs)
             files.push_back("./default.nix");
 
         for (auto & i : files) {
             Expr * e = fromArgs
-                ? state.parseExprFromString(i, absPath("."))
-                : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i))));
-            processExpr(state, attrPaths, parseOnly, strict, autoArgs,
+                ? state->parseExprFromString(i, absPath("."))
+                : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i))));
+            processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
 
-        state.printStats();
+        state->printStats();
     });
 }
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index fa7ee254500..50b2c2803ec 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -95,9 +95,9 @@ int main(int argc, char * * argv)
             throw UsageError("too many arguments");
 
         auto store = openStore();
-        EvalState state(myArgs.searchPath, store);
+        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
 
-        Bindings & autoArgs = *myArgs.getAutoArgs(state);
+        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
 
         /* If -A is given, get the URI from the specified Nix
            expression. */
@@ -107,33 +107,33 @@ int main(int argc, char * * argv)
                 throw UsageError("you must specify a URI");
             uri = args[0];
         } else {
-            Path path = resolveExprPath(lookupFileArg(state, args.empty() ? "." : args[0]));
+            Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
             Value vRoot;
-            state.evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(state, attrPath, autoArgs, vRoot));
-            state.forceAttrs(v);
+            state->evalFile(path, vRoot);
+            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot));
+            state->forceAttrs(v);
 
             /* Extract the URI. */
-            auto attr = v.attrs->find(state.symbols.create("urls"));
+            auto attr = v.attrs->find(state->symbols.create("urls"));
             if (attr == v.attrs->end())
                 throw Error("attribute set does not contain a 'urls' attribute");
-            state.forceList(*attr->value);
+            state->forceList(*attr->value);
             if (attr->value->listSize() < 1)
                 throw Error("'urls' list is empty");
-            uri = state.forceString(*attr->value->listElems()[0]);
+            uri = state->forceString(*attr->value->listElems()[0]);
 
             /* Extract the hash mode. */
-            attr = v.attrs->find(state.symbols.create("outputHashMode"));
+            attr = v.attrs->find(state->symbols.create("outputHashMode"));
             if (attr == v.attrs->end())
                 printInfo("warning: this does not look like a fetchurl call");
             else
-                unpack = state.forceString(*attr->value) == "recursive";
+                unpack = state->forceString(*attr->value) == "recursive";
 
             /* Extract the name. */
             if (name.empty()) {
-                attr = v.attrs->find(state.symbols.create("name"));
+                attr = v.attrs->find(state->symbols.create("name"));
                 if (attr != v.attrs->end())
-                    name = state.forceString(*attr->value);
+                    name = state->forceString(*attr->value);
             }
         }
 
@@ -158,7 +158,7 @@ int main(int argc, char * * argv)
 
         if (storePath.empty()) {
 
-            auto actualUri = resolveMirrorUri(state, uri);
+            auto actualUri = resolveMirrorUri(*state, uri);
 
             /* Download the file. */
             DownloadRequest req(actualUri);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index f84774a5336..4723a1974b7 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -693,8 +693,8 @@ struct CmdRepl : StoreCommand, MixEvalArgs
 
     void run(ref<Store> store) override
     {
-        NixRepl repl(searchPath, openStore());
-        repl.mainLoop(files);
+        auto repl = std::make_unique<NixRepl>(searchPath, openStore());
+        repl->mainLoop(files);
     }
 };
 
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 21892c31a89..e23ae792369 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -118,13 +118,13 @@ struct CmdUpgradeNix : StoreCommand
         auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix");
         auto res = getDownloader()->download(req);
 
-        EvalState state(Strings(), store);
-        auto v = state.allocValue();
-        state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v);
-        Bindings & bindings(*state.allocBindings(0));
-        auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v);
+        auto state = std::make_unique<EvalState>(Strings(), store);
+        auto v = state->allocValue();
+        state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v);
+        Bindings & bindings(*state->allocBindings(0));
+        auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v);
 
-        return state.forceString(*v2);
+        return state->forceString(*v2);
     }
 };
 

From 3c5d9f478d0c0b70ec3e3100a8dacbd7437bda3e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jun 2018 16:56:19 +0200
Subject: [PATCH 1084/2196] std::random_shuffle -> std::shuffle

The former is removed in C++17.
---
 src/libstore/gc.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 233a70bd2d2..b5020a506be 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -7,6 +7,7 @@
 #include <queue>
 #include <algorithm>
 #include <regex>
+#include <random>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -832,7 +833,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                alphabetically first (e.g. /nix/store/000...).  This
                matters when using --max-freed etc. */
             vector<Path> entries_(entries.begin(), entries.end());
-            random_shuffle(entries_.begin(), entries_.end());
+            std::mt19937 gen(1);
+            std::shuffle(entries_.begin(), entries_.end(), gen);
 
             for (auto & i : entries_)
                 tryToDelete(state, i);

From 7902fccf6fa903414703a4eccd2c3514809a0772 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jun 2018 18:00:54 +0200
Subject: [PATCH 1085/2196] hashDerivationModulo(): Handle chroot stores

This makes 'nix copy --to /mnt /nix/store/bla.drv' work.
---
 src/libstore/derivations.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 74b861281ee..1e187ec5e95 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -342,7 +342,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
         Hash h = drvHashes[i.first];
         if (!h) {
             assert(store.isValidPath(i.first));
-            Derivation drv2 = readDerivation(i.first);
+            Derivation drv2 = readDerivation(store.toRealPath(i.first));
             h = hashDerivationModulo(store, drv2);
             drvHashes[i.first] = h;
         }

From 85530bcc0b4c61499f90fa6125c19beeb0f4485e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Jun 2018 18:04:22 +0200
Subject: [PATCH 1086/2196] nix: Remove special handling of .drv files

This makes 'nix copy' and 'nix path-info' work on .drv store
paths. Removing special treatment of .drv files seems the most
future-proof approach given the possible removal of .drv files in the
future.

Note that 'nix build' will still build (rather than substitute) .drv
paths due to the unfortunate overloading in Store::buildPaths().
---
 src/nix/installables.cc | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index a3fdd8a2808..0be992b03c5 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -86,22 +86,6 @@ Buildable Installable::toBuildable()
     return std::move(buildables[0]);
 }
 
-struct InstallableStoreDrv : Installable
-{
-    Path drvPath;
-
-    InstallableStoreDrv(const Path & drvPath) : drvPath(drvPath) { }
-
-    std::string what() override { return drvPath; }
-
-    Buildables toBuildables() override
-    {
-        Buildable b = {drvPath};
-        // FIXME: add outputs?
-        return {b};
-    }
-};
-
 struct InstallableStorePath : Installable
 {
     Path storePath;
@@ -226,12 +210,8 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
 
             auto path = store->toStorePath(store->followLinksToStore(s));
 
-            if (store->isStorePath(path)) {
-                if (isDerivation(path))
-                    result.push_back(std::make_shared<InstallableStoreDrv>(path));
-                else
-                    result.push_back(std::make_shared<InstallableStorePath>(path));
-            }
+            if (store->isStorePath(path))
+                result.push_back(std::make_shared<InstallableStorePath>(path));
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))

From 6d09e4400ce8a2f8ec24b0c934268c6fdb42d482 Mon Sep 17 00:00:00 2001
From: Matthew Justin Bauer <mjbauer95@gmail.com>
Date: Thu, 14 Jun 2018 12:34:55 -0400
Subject: [PATCH 1087/2196] Fix optimise-store.cc to skip more paths on macOS

I hate to make this such a large check but the lack of documentation means we really have no idea what's allowed. All of them reported so far have been within ".app/Contents" directories. That appears to be a safe starting point. However, I would not be surprised to also find more paths that are disallowed for instance in .framework or .bundle directories.

Fixes #2031
Fixes #2229
---
 src/libstore/optimise-store.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 7840167d777..991512f2179 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -104,8 +104,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
        https://github.com/NixOS/nix/issues/1443 for more discussion. */
 
-    if (std::regex_search(path, std::regex("\\.app/Contents/PkgInfo$")) ||
-        std::regex_search(path, std::regex("\\.app/Contents/Resources/.+\\.lproj$")))
+    if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
     {
         debug(format("'%1%' is not allowed to be linked in macOS") % path);
         return;

From ef8122064baf54c37a8fe15e78b9da7c5180bbe6 Mon Sep 17 00:00:00 2001
From: Symphorien Gibol <symphorien+git@xlumurb.eu>
Date: Sat, 16 Jun 2018 18:04:09 +0200
Subject: [PATCH 1088/2196] libstore/gc.cc: ignore ESRCH when reading symlinks
 in /proc

readlink is also affected by the problem fixed for regular files in
c567afe355df
---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 233a70bd2d2..40b4f965f25 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -365,7 +365,7 @@ static void readProcLink(const string & file, StringSet & paths)
     char buf[bufsiz];
     auto res = readlink(file.c_str(), buf, bufsiz);
     if (res == -1) {
-        if (errno == ENOENT || errno == EACCES)
+        if (errno == ENOENT || errno == EACCES || errno == ESRCH)
             return;
         throw SysError("reading symlink");
     }

From 10ebcf867043c95e585b5e30d6413572dbc0b665 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sun, 17 Jun 2018 19:01:07 +0200
Subject: [PATCH 1089/2196] manual: distributed-builds: Mention - as default

---
 doc/manual/advanced-topics/distributed-builds.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index 20fd6a0cfb0..bbb573e3540 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -81,6 +81,7 @@ or a newline, e.g.
 
 <para>Each machine specification consists of the following elements,
 separated by spaces. Only the first element is required.
+To leave a field at its default, set it to <literal>-</literal>.
 
 <orderedlist>
 

From b920b908578d68c7c80f1c1e89c42784693e18d5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 Jun 2018 10:36:19 +0200
Subject: [PATCH 1090/2196] Detect redirect loops

Fixes #2225.
---
 src/libstore/download.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 7a5deed3214..07acd5d0e00 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -244,6 +244,7 @@ struct CurlDownloader : public Downloader
 
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
             curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
+            curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
@@ -364,6 +365,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
                         case CURLE_SSL_CACERT_BADFILE:
+                        case CURLE_TOO_MANY_REDIRECTS:
                             err = Misc;
                             break;
                         default: // Shut up warnings

From 25cb1a63438efca539a85647031d3bb84b3bd14d Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 18 Jun 2018 15:50:37 -0500
Subject: [PATCH 1091/2196] release-common: prefer utillinuxMinimal to reduce
 compile-time deps

Honestly could probably use busybox here instead, but at least
with utillinuxMinimal there's no build-time dependency on systemd.
---
 release-common.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release-common.nix b/release-common.nix
index 9138c112784..f98e86a1b44 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -57,7 +57,7 @@ rec {
       git
       mercurial
     ]
-    ++ lib.optionals stdenv.isLinux [libseccomp utillinux]
+    ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
       (aws-sdk-cpp.override {

From f601bc049258ed438aa6f8199c5be11770d44aaf Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 18 Jun 2018 17:31:00 -0500
Subject: [PATCH 1092/2196] sync.hh: return cv_status instead of dropping it,
 in case useful.

---
 src/libutil/sync.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh
index 3b2710f6fd3..e1d591d77a8 100644
--- a/src/libutil/sync.hh
+++ b/src/libutil/sync.hh
@@ -57,11 +57,11 @@ public:
         }
 
         template<class Rep, class Period>
-        void wait_for(std::condition_variable & cv,
+        std::cv_status wait_for(std::condition_variable & cv,
             const std::chrono::duration<Rep, Period> & duration)
         {
             assert(s);
-            cv.wait_for(lk, duration);
+            return cv.wait_for(lk, duration);
         }
 
         template<class Rep, class Period, class Predicate>

From 44de71a39624d86d6744062ee36f57170024c9a0 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 18 Jun 2018 17:33:35 -0500
Subject: [PATCH 1093/2196] progress-bar: re-draw last update if nothing new
 for 1sec.

Slightly nicer behavior when updates are somewhat far apart
(during a long linking step, perhaps) ensuring things
don't appear unresponsive.

If we wait the maximum amount for the update,
don't bother waiting another 50ms (for rate-limiting purposes)
and just check if we should quit.

This also ensures we'll notice the request to quit within 1s
if quit is signalled but there is not an udpate.
(I'm not sure if this happens or not)
---
 src/nix/progress-bar.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 40b905ba324..8093d8761c4 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -75,9 +75,10 @@ class ProgressBar : public Logger
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
-                state.wait(updateCV);
+                auto r = state.wait_for(updateCV, std::chrono::seconds(1));
                 draw(*state);
-                state.wait_for(quitCV, std::chrono::milliseconds(50));
+                if (r == std::cv_status::no_timeout)
+                  state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
         });
     }

From 81261ca5381eb503b27a529bb9d35383e79f98b8 Mon Sep 17 00:00:00 2001
From: Peter Simons <simons@cryp.to>
Date: Thu, 21 Jun 2018 19:13:04 +0200
Subject: [PATCH 1094/2196] docker: move the docker file to
 https://github.com/NixOS/docker

We have automated builds at https://hub.docker.com/r/nixos/nix/ now. The master
branch of the "docker" repository is available as "latest". Branches that match
the regular expression "^[0-9.]+$" are pushed to the tag that corresponds to
their branch name. Other branches are ignored.
---
 misc/docker/Dockerfile | 28 ----------------------------
 misc/docker/README.md  |  8 --------
 2 files changed, 36 deletions(-)
 delete mode 100644 misc/docker/Dockerfile
 delete mode 100644 misc/docker/README.md

diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile
deleted file mode 100644
index eeca12353d9..00000000000
--- a/misc/docker/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-FROM alpine
-
-# Enable HTTPS support in wget.
-RUN apk add --update openssl
-
-# Download Nix and install it into the system.
-RUN wget https://nixos.org/releases/nix/nix-2.0.4/nix-2.0.4-x86_64-linux.tar.bz2 \
-  && echo "d6db178007014ed47ad5460c1bd5b2cb9403b1ec543a0d6507cb27e15358341f  nix-2.0.4-x86_64-linux.tar.bz2" | sha256sum -c \
-  && tar xjf nix-*-x86_64-linux.tar.bz2 \
-  && addgroup -g 30000 -S nixbld \
-  && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \
-  && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \
-  && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \
-  && rm -r /nix-*-x86_64-linux \
-  && rm -r /var/cache/apk/*
-
-ONBUILD ENV \
-    ENV=/etc/profile \
-    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
-
-ENV \
-    ENV=/etc/profile \
-    PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \
-    GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt \
-    NIX_PATH=/nix/var/nix/profiles/per-user/root/channels
diff --git a/misc/docker/README.md b/misc/docker/README.md
deleted file mode 100644
index 491be740896..00000000000
--- a/misc/docker/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-To update https://hub.docker.com/r/nixos/nix/
-
-    $ docker build . -t nixos/nix:2.0
-    $ docker tag nixos/nix:2.0 nixos/nix:latest
-    $ docker push nixos/nix:latest
-    $ docker push nixos/nix:2.0
-
-Write access: @domenkozar

From 6a24e49ba82553b5efcc49c627424bd4e26e16a5 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 23 Jun 2018 07:49:24 -0500
Subject: [PATCH 1095/2196] repl: use `nix build` for building instead of
 `nix-store -r`

progress bar!
---
 src/nix/repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 4723a1974b7..1eb71600637 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -385,7 +385,7 @@ bool NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) {
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", drvPath}) == 0) {
                 Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)

From 6b74fdac271cb12f1e9a56b13c77c027404b9a00 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sat, 12 May 2018 20:00:59 +0200
Subject: [PATCH 1096/2196] search.cc: improve UX for `nix search`

As proposed in #1634 the `nix search` command could use some
improvements. Initially 0413aeb35d6ee869a98b6565781c1cf47dc80080 added
some basic sorting behavior using `std::map`, a next step would be an
improvement of the output.

This patch includes the following changes:

* Use `$PAGER` for outputs with `RunPager` from `shared.hh`:
  The same behavior is defined for `nix-env --query`, furthermore it
  makes searching huge results way easier.

* Simplified result blocks:
  The new output is heavily inspired by the output from `nox`, the first
  line shows the attribute path and the derivaiton name
  (`attribute path (derivation name)`) and the description in the second
  line.
---
 src/nix/search.cc | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 53967669808..4cb1efa7955 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -7,19 +7,25 @@
 #include "common-args.hh"
 #include "json.hh"
 #include "json-to-value.hh"
+#include "shared.hh"
 
 #include <regex>
 #include <fstream>
 
 using namespace nix;
 
-std::string hilite(const std::string & s, const std::smatch & m)
+std::string wrap(std::string prefix, std::string s)
+{
+    return prefix + s + ANSI_NORMAL;
+}
+
+std::string hilite(const std::string & s, const std::smatch & m, std::string postfix)
 {
     return
         m.empty()
         ? s
         : std::string(m.prefix())
-          + ANSI_RED + std::string(m.str()) + ANSI_NORMAL
+          + ANSI_RED + std::string(m.str()) + postfix
           + std::string(m.suffix());
 }
 
@@ -75,6 +81,10 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 "To search for git and frontend or gui:",
                 "nix search git 'frontend|gui'"
             },
+            Example{
+                "To display the description of the found packages:",
+                "nix search git --verbose"
+            }
         };
     }
 
@@ -164,14 +174,10 @@ struct CmdSearch : SourceExprCommand, MixJSON
 
                         } else {
                             results[attrPath] = fmt(
-                                "Attribute name: %s\n"
-                                "Package name: %s\n"
-                                "Version: %s\n"
-                                "Description: %s\n",
-                                hilite(attrPath, attrPathMatch),
-                                hilite(name, nameMatch),
-                                parsed.version,
-                                hilite(description, descriptionMatch));
+                                "* %s (%s)\n  %s\n",
+                                wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
+                                wrap("\e[0;2m", hilite(parsed.fullName, nameMatch, "\e[0;2m")),
+                                hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }
 
@@ -263,6 +269,10 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
 
+        if (results.size() == 0)
+            throw Error("no results for the given search term(s)!");
+
+        RunPager pager;
         for (auto el : results) std::cout << el.second << "\n";
 
     }

From 8a98ffc276989700f55847e212837d93d6898add Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Tue, 3 Jul 2018 12:49:12 +0200
Subject: [PATCH 1097/2196] nix why-depends: render output into $PAGER

---
 src/nix/why-depends.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 17e0595ae88..325a2be0a79 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "progress-bar.hh"
 #include "fs-accessor.hh"
+#include "shared.hh"
 
 #include <queue>
 
@@ -237,6 +238,7 @@ struct CmdWhyDepends : SourceExprCommand
 
             visitPath(node.path);
 
+            RunPager pager;
             for (auto & ref : refs) {
                 auto hash = storePathToHash(ref.second->path);
 

From a92ed973e569324b55cddb128f6f638f3ea1985c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Jul 2018 18:04:51 +0200
Subject: [PATCH 1098/2196] Store floating point numbers in double precision

Even on 32-bit systems, Value has enough space to hold a double.
---
 src/libexpr/value.hh | 2 +-
 src/libutil/json.cc  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 66b41a15840..809772f7c08 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -44,7 +44,7 @@ class JSONPlaceholder;
 
 
 typedef long NixInt;
-typedef float NixFloat;
+typedef double NixFloat;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
diff --git a/src/libutil/json.cc b/src/libutil/json.cc
index 813b257016e..0a6fb65f060 100644
--- a/src/libutil/json.cc
+++ b/src/libutil/json.cc
@@ -31,6 +31,7 @@ template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long &
 template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
 template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
 template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
+template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
 
 template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
 {

From 3b1f54cf06b6127a14479d18db8f08dc4c05d57a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Jul 2018 18:15:59 +0200
Subject: [PATCH 1099/2196] Add a fromTOML primop

This is primarily useful for processing Cargo.lock files.
---
 release-common.nix                |  18 +++
 src/libexpr/primops/fromTOML.cc   |  77 +++++++++++++
 tests/lang/eval-okay-fromTOML.exp |   1 +
 tests/lang/eval-okay-fromTOML.nix | 184 ++++++++++++++++++++++++++++++
 4 files changed, 280 insertions(+)
 create mode 100644 src/libexpr/primops/fromTOML.cc
 create mode 100644 tests/lang/eval-okay-fromTOML.exp
 create mode 100644 tests/lang/eval-okay-fromTOML.nix

diff --git a/release-common.nix b/release-common.nix
index f98e86a1b44..4aa18d4ee02 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -2,6 +2,23 @@
 
 with pkgs;
 
+let
+
+  cpptoml = runCommand "cpptoml"
+    { src = fetchFromGitHub {
+        owner = "skystrife";
+        repo = "cpptoml";
+        rev = "43d7d8e52de149fd84aedf7eb71814ff9e6b2f7e";
+        sha256 = "0gdxk1mj5hpj93df1kwfl903w06nihbb1ayr3x336775jm2d2cw6";
+      };
+    }
+    ''
+      mkdir -p $out/include
+      cp $src/include/cpptoml.h $out/include/
+    '';
+
+in
+
 rec {
   # Use "busybox-sandbox-shell" if present,
   # if not (legacy) fallback and hope it's sufficient.
@@ -52,6 +69,7 @@ rec {
       bzip2 xz brotli
       openssl pkgconfig sqlite boehmgc
       boost
+      cpptoml
 
       # Tests
       git
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
new file mode 100644
index 00000000000..b04d7c40710
--- /dev/null
+++ b/src/libexpr/primops/fromTOML.cc
@@ -0,0 +1,77 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+
+#include <cpptoml.h>
+
+namespace nix {
+
+static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    using namespace cpptoml;
+
+    auto toml = state.forceStringNoCtx(*args[0], pos);
+
+    std::istringstream tomlStream(toml);
+
+    std::function<void(Value &, std::shared_ptr<base>)> visit;
+
+    visit = [&](Value & v, std::shared_ptr<base> t) {
+
+        if (auto t2 = t->as_table()) {
+
+            size_t size = 0;
+            for (auto & i : *t2) size++;
+
+            state.mkAttrs(v, size);
+
+            for (auto & i : *t2) {
+                auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
+
+                if (auto i2 = i.second->as_table_array()) {
+                    size_t size2 = i2->get().size();
+                    state.mkList(v2, size2);
+                    for (size_t j = 0; j < size2; ++j)
+                        visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
+                }
+                else
+                    visit(v2, i.second);
+            }
+
+            v.attrs->sort();
+        }
+
+        else if (auto t2 = t->as_array()) {
+            size_t size = t2->get().size();
+
+            state.mkList(v, size);
+
+            for (size_t i = 0; i < size; ++i)
+                visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
+        }
+
+        else if (t->is_value()) {
+            if (auto val = t->as<NixInt>())
+                mkInt(v, val->get());
+            else if (auto val = t->as<NixFloat>())
+                mkFloat(v, val->get());
+            else if (auto val = t->as<bool>())
+                mkBool(v, val->get());
+            else if (auto val = t->as<std::string>())
+                mkString(v, val->get());
+            else
+                throw EvalError("unsupported value type in TOML");
+        }
+
+        else abort();
+    };
+
+    try {
+        visit(v, parser(tomlStream).parse());
+    } catch (std::runtime_error & e) {
+        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+    }
+}
+
+static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
+
+}
diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp
new file mode 100644
index 00000000000..5b9d47122cc
--- /dev/null
+++ b/tests/lang/eval-okay-fromTOML.exp
@@ -0,0 +1 @@
+[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix
new file mode 100644
index 00000000000..8e7cbd1c61e
--- /dev/null
+++ b/tests/lang/eval-okay-fromTOML.nix
@@ -0,0 +1,184 @@
+[
+
+  (builtins.fromTOML ''
+    # This is a TOML document.
+
+    title = "TOML Example"
+
+    [owner]
+    name = "Tom Preston-Werner"
+    #dob = 1979-05-27T07:32:00-08:00 # First class dates
+
+    [database]
+    server = "192.168.1.1"
+    ports = [ 8001, 8001, 8002 ]
+    connection_max = 5000
+    enabled = true
+
+    [servers]
+
+      # Indentation (tabs and/or spaces) is allowed but not required
+      [servers.alpha]
+      ip = "10.0.0.1"
+      dc = "eqdc10"
+
+      [servers.beta]
+      ip = "10.0.0.2"
+      dc = "eqdc10"
+
+    [clients]
+    data = [ ["gamma", "delta"], [1, 2] ]
+
+    # Line breaks are OK when inside arrays
+    hosts = [
+      "alpha",
+      "omega"
+    ]
+  '')
+
+  (builtins.fromTOML ''
+    key = "value"
+    bare_key = "value"
+    bare-key = "value"
+    1234 = "value"
+
+    "127.0.0.1" = "value"
+    "character encoding" = "value"
+    "ʎǝʞ" = "value"
+    'key2' = "value"
+    #'quoted "value"' = "value"
+
+    name = "Orange"
+
+    # FIXME: cpptoml doesn't handle dotted keys properly yet.
+    #physical.color = "orange"
+    #physical.shape = "round"
+    #site."google.com" = true
+
+    #a.b.c = 1
+    #a.d = 2
+
+    str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+
+    int1 = +99
+    int2 = 42
+    int3 = 0
+    int4 = -17
+    int5 = 1_000
+    int6 = 5_349_221
+    int7 = 1_2_3_4_5
+
+    # FIXME: cpptoml doesn't support these yet:
+
+    #hex1 = 0xDEADBEEF
+    #hex2 = 0xdeadbeef
+    #hex3 = 0xdead_beef
+
+    #oct1 = 0o01234567
+    #oct2 = 0o755
+
+    #bin1 = 0b11010110
+
+    flt1 = +1.0
+    flt2 = 3.1415
+    flt3 = -0.01
+    flt4 = 5e+22
+    flt5 = 1e6
+    flt6 = -2E-2
+    flt7 = 6.626e-34
+    flt8 = 9_224_617.445_991_228_313
+
+    bool1 = true
+    bool2 = false
+
+    # FIXME: not supported because Nix doesn't have a date/time type.
+    #odt1 = 1979-05-27T07:32:00Z
+    #odt2 = 1979-05-27T00:32:00-07:00
+    #odt3 = 1979-05-27T00:32:00.999999-07:00
+    #odt4 = 1979-05-27 07:32:00Z
+    #ldt1 = 1979-05-27T07:32:00
+    #ldt2 = 1979-05-27T00:32:00.999999
+    #ld1 = 1979-05-27
+    #lt1 = 07:32:00
+    #lt2 = 00:32:00.999999
+
+    arr1 = [ 1, 2, 3 ]
+    arr2 = [ "red", "yellow", "green" ]
+    arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+    arr4 = [ "all", 'strings', """are the same""", ''''type'''']
+    arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+
+    arr7 = [
+      1, 2, 3
+    ]
+
+    arr8 = [
+      1,
+      2, # this is ok
+    ]
+
+    [table-1]
+    key1 = "some string"
+    key2 = 123
+
+
+    [table-2]
+    key1 = "another string"
+    key2 = 456
+
+    #[dog."tater.man"]
+    #type.name = "pug"
+
+    [a.b.c]
+    [ d.e.f ]
+    [ g .  h  . i ]
+    [ j . "ʞ" . 'l' ]
+    [x.y.z.w]
+
+    name = { first = "Tom", last = "Preston-Werner" }
+    point = { x = 1, y = 2 }
+    #animal = { type.name = "pug" }
+
+    [[products]]
+    name = "Hammer"
+    sku = 738594937
+
+    [[products]]
+
+    [[products]]
+    name = "Nail"
+    sku = 284758393
+    color = "gray"
+  '')
+
+  (builtins.fromTOML ''
+    [[package]]
+    name = "aho-corasick"
+    version = "0.6.4"
+    source = "registry+https://github.com/rust-lang/crates.io-index"
+    dependencies = [
+     "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+    ]
+
+    [[package]]
+    name = "ansi_term"
+    version = "0.9.0"
+    source = "registry+https://github.com/rust-lang/crates.io-index"
+
+    [[package]]
+    name = "atty"
+    version = "0.2.10"
+    source = "registry+https://github.com/rust-lang/crates.io-index"
+    dependencies = [
+     "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
+     "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+     "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+    ]
+
+    [metadata]
+    "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
+    "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+    "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
+    "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
+  '')
+]

From ea3c9dab5fa6c9ba0e1c441ac0c27a73cc7632bb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Jul 2018 18:21:12 +0200
Subject: [PATCH 1100/2196] Include cpptoml for build simplicity

---
 release-common.nix              |   18 -
 src/cpptoml/LICENSE             |   18 +
 src/cpptoml/cpptoml.h           | 3457 +++++++++++++++++++++++++++++++
 src/libexpr/primops/fromTOML.cc |    2 +-
 4 files changed, 3476 insertions(+), 19 deletions(-)
 create mode 100644 src/cpptoml/LICENSE
 create mode 100644 src/cpptoml/cpptoml.h

diff --git a/release-common.nix b/release-common.nix
index 4aa18d4ee02..f98e86a1b44 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -2,23 +2,6 @@
 
 with pkgs;
 
-let
-
-  cpptoml = runCommand "cpptoml"
-    { src = fetchFromGitHub {
-        owner = "skystrife";
-        repo = "cpptoml";
-        rev = "43d7d8e52de149fd84aedf7eb71814ff9e6b2f7e";
-        sha256 = "0gdxk1mj5hpj93df1kwfl903w06nihbb1ayr3x336775jm2d2cw6";
-      };
-    }
-    ''
-      mkdir -p $out/include
-      cp $src/include/cpptoml.h $out/include/
-    '';
-
-in
-
 rec {
   # Use "busybox-sandbox-shell" if present,
   # if not (legacy) fallback and hope it's sufficient.
@@ -69,7 +52,6 @@ rec {
       bzip2 xz brotli
       openssl pkgconfig sqlite boehmgc
       boost
-      cpptoml
 
       # Tests
       git
diff --git a/src/cpptoml/LICENSE b/src/cpptoml/LICENSE
new file mode 100644
index 00000000000..8802c4fa5a3
--- /dev/null
+++ b/src/cpptoml/LICENSE
@@ -0,0 +1,18 @@
+Copyright (c) 2014 Chase Geigle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h
new file mode 100644
index 00000000000..07de010b152
--- /dev/null
+++ b/src/cpptoml/cpptoml.h
@@ -0,0 +1,3457 @@
+/**
+ * @file cpptoml.h
+ * @author Chase Geigle
+ * @date May 2013
+ */
+
+#ifndef _CPPTOML_H_
+#define _CPPTOML_H_
+
+#include <algorithm>
+#include <cassert>
+#include <clocale>
+#include <cstdint>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#if __cplusplus > 201103L
+#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]]
+#elif defined(__clang__)
+#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason)))
+#elif defined(__GNUG__)
+#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated))
+#elif defined(_MSC_VER)
+#if _MSC_VER < 1910
+#define CPPTOML_DEPRECATED(reason) __declspec(deprecated)
+#else
+#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]]
+#endif
+#endif
+
+namespace cpptoml
+{
+class writer; // forward declaration
+class base;   // forward declaration
+#if defined(CPPTOML_USE_MAP)
+// a std::map will ensure that entries a sorted, albeit at a slight
+// performance penalty relative to the (default) unordered_map
+using string_to_base_map = std::map<std::string, std::shared_ptr<base>>;
+#else
+// by default an unordered_map is used for best performance as the
+// toml specification does not require entries to be sorted
+using string_to_base_map
+    = std::unordered_map<std::string, std::shared_ptr<base>>;
+#endif
+
+// if defined, `base` will retain type information in form of an enum class
+// such that static_cast can be used instead of dynamic_cast
+// #define CPPTOML_NO_RTTI
+
+template <class T>
+class option
+{
+  public:
+    option() : empty_{true}
+    {
+        // nothing
+    }
+
+    option(T value) : empty_{false}, value_(std::move(value))
+    {
+        // nothing
+    }
+
+    explicit operator bool() const
+    {
+        return !empty_;
+    }
+
+    const T& operator*() const
+    {
+        return value_;
+    }
+
+    const T* operator->() const
+    {
+        return &value_;
+    }
+
+    const T& value_or(const T& alternative) const
+    {
+        if (!empty_)
+            return value_;
+        return alternative;
+    }
+
+  private:
+    bool empty_;
+    T value_;
+};
+
+struct local_date
+{
+    int year = 0;
+    int month = 0;
+    int day = 0;
+};
+
+struct local_time
+{
+    int hour = 0;
+    int minute = 0;
+    int second = 0;
+    int microsecond = 0;
+};
+
+struct zone_offset
+{
+    int hour_offset = 0;
+    int minute_offset = 0;
+};
+
+struct local_datetime : local_date, local_time
+{
+};
+
+struct offset_datetime : local_datetime, zone_offset
+{
+    static inline struct offset_datetime from_zoned(const struct tm& t)
+    {
+        offset_datetime dt;
+        dt.year = t.tm_year + 1900;
+        dt.month = t.tm_mon + 1;
+        dt.day = t.tm_mday;
+        dt.hour = t.tm_hour;
+        dt.minute = t.tm_min;
+        dt.second = t.tm_sec;
+
+        char buf[16];
+        strftime(buf, 16, "%z", &t);
+
+        int offset = std::stoi(buf);
+        dt.hour_offset = offset / 100;
+        dt.minute_offset = offset % 100;
+        return dt;
+    }
+
+    CPPTOML_DEPRECATED("from_local has been renamed to from_zoned")
+    static inline struct offset_datetime from_local(const struct tm& t)
+    {
+        return from_zoned(t);
+    }
+
+    static inline struct offset_datetime from_utc(const struct tm& t)
+    {
+        offset_datetime dt;
+        dt.year = t.tm_year + 1900;
+        dt.month = t.tm_mon + 1;
+        dt.day = t.tm_mday;
+        dt.hour = t.tm_hour;
+        dt.minute = t.tm_min;
+        dt.second = t.tm_sec;
+        return dt;
+    }
+};
+
+CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime")
+typedef offset_datetime datetime;
+
+class fill_guard
+{
+  public:
+    fill_guard(std::ostream& os) : os_(os), fill_{os.fill()}
+    {
+        // nothing
+    }
+
+    ~fill_guard()
+    {
+        os_.fill(fill_);
+    }
+
+  private:
+    std::ostream& os_;
+    std::ostream::char_type fill_;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const local_date& dt)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+    os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2)
+       << dt.day;
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const local_time& ltime)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+    os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":"
+       << setw(2) << ltime.second;
+
+    if (ltime.microsecond > 0)
+    {
+        os << ".";
+        int power = 100000;
+        for (int curr_us = ltime.microsecond; curr_us; power /= 10)
+        {
+            auto num = curr_us / power;
+            os << num;
+            curr_us -= num * power;
+        }
+    }
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo)
+{
+    fill_guard g{os};
+    os.fill('0');
+
+    using std::setw;
+
+    if (zo.hour_offset != 0 || zo.minute_offset != 0)
+    {
+        if (zo.hour_offset > 0)
+        {
+            os << "+";
+        }
+        else
+        {
+            os << "-";
+        }
+        os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2)
+           << std::abs(zo.minute_offset);
+    }
+    else
+    {
+        os << "Z";
+    }
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt)
+{
+    return os << static_cast<const local_date&>(dt) << "T"
+              << static_cast<const local_time&>(dt);
+}
+
+inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt)
+{
+    return os << static_cast<const local_datetime&>(dt)
+              << static_cast<const zone_offset&>(dt);
+}
+
+template <class T, class... Ts>
+struct is_one_of;
+
+template <class T, class V>
+struct is_one_of<T, V> : std::is_same<T, V>
+{
+};
+
+template <class T, class V, class... Ts>
+struct is_one_of<T, V, Ts...>
+{
+    const static bool value
+        = std::is_same<T, V>::value || is_one_of<T, Ts...>::value;
+};
+
+template <class T>
+class value;
+
+template <class T>
+struct valid_value
+    : is_one_of<T, std::string, int64_t, double, bool, local_date, local_time,
+                local_datetime, offset_datetime>
+{
+};
+
+template <class T, class Enable = void>
+struct value_traits;
+
+template <class T>
+struct valid_value_or_string_convertible
+{
+
+    const static bool value = valid_value<typename std::decay<T>::type>::value
+                              || std::is_convertible<T, std::string>::value;
+};
+
+template <class T>
+struct value_traits<T, typename std::
+                           enable_if<valid_value_or_string_convertible<T>::
+                                         value>::type>
+{
+    using value_type = typename std::
+        conditional<valid_value<typename std::decay<T>::type>::value,
+                    typename std::decay<T>::type, std::string>::type;
+
+    using type = value<value_type>;
+
+    static value_type construct(T&& val)
+    {
+        return value_type(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_floating_point<
+                                         typename std::decay<T>::type>::value>::
+                            type>
+{
+    using value_type = typename std::decay<T>::type;
+
+    using type = value<double>;
+
+    static value_type construct(T&& val)
+    {
+        return value_type(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_signed<typename std::decay<T>::
+                                                        type>::value>::type>
+{
+    using value_type = int64_t;
+
+    using type = value<int64_t>;
+
+    static value_type construct(T&& val)
+    {
+        if (val < (std::numeric_limits<int64_t>::min)())
+            throw std::underflow_error{"constructed value cannot be "
+                                       "represented by a 64-bit signed "
+                                       "integer"};
+
+        if (val > (std::numeric_limits<int64_t>::max)())
+            throw std::overflow_error{"constructed value cannot be represented "
+                                      "by a 64-bit signed integer"};
+
+        return static_cast<int64_t>(val);
+    }
+};
+
+template <class T>
+struct value_traits<T,
+                    typename std::
+                        enable_if<!valid_value_or_string_convertible<T>::value
+                                  && std::is_unsigned<typename std::decay<T>::
+                                                          type>::value>::type>
+{
+    using value_type = int64_t;
+
+    using type = value<int64_t>;
+
+    static value_type construct(T&& val)
+    {
+        if (val > static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
+            throw std::overflow_error{"constructed value cannot be represented "
+                                      "by a 64-bit signed integer"};
+
+        return static_cast<int64_t>(val);
+    }
+};
+
+class array;
+class table;
+class table_array;
+
+template <class T>
+struct array_of_trait
+{
+    using return_type = option<std::vector<T>>;
+};
+
+template <>
+struct array_of_trait<array>
+{
+    using return_type = option<std::vector<std::shared_ptr<array>>>;
+};
+
+template <class T>
+inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val);
+inline std::shared_ptr<array> make_array();
+template <class T>
+inline std::shared_ptr<T> make_element();
+inline std::shared_ptr<table> make_table();
+inline std::shared_ptr<table_array> make_table_array();
+
+#if defined(CPPTOML_NO_RTTI)
+/// Base type used to store underlying data type explicitly if RTTI is disabled
+enum class base_type
+{
+    NONE,
+    STRING,
+    LOCAL_TIME,
+    LOCAL_DATE,
+    LOCAL_DATETIME,
+    OFFSET_DATETIME,
+    INT,
+    FLOAT,
+    BOOL,
+    TABLE,
+    ARRAY,
+    TABLE_ARRAY
+};
+
+/// Type traits class to convert C++ types to enum member
+template <class T>
+struct base_type_traits;
+
+template <>
+struct base_type_traits<std::string>
+{
+    static const base_type type = base_type::STRING;
+};
+
+template <>
+struct base_type_traits<local_time>
+{
+    static const base_type type = base_type::LOCAL_TIME;
+};
+
+template <>
+struct base_type_traits<local_date>
+{
+    static const base_type type = base_type::LOCAL_DATE;
+};
+
+template <>
+struct base_type_traits<local_datetime>
+{
+    static const base_type type = base_type::LOCAL_DATETIME;
+};
+
+template <>
+struct base_type_traits<offset_datetime>
+{
+    static const base_type type = base_type::OFFSET_DATETIME;
+};
+
+template <>
+struct base_type_traits<int64_t>
+{
+    static const base_type type = base_type::INT;
+};
+
+template <>
+struct base_type_traits<double>
+{
+    static const base_type type = base_type::FLOAT;
+};
+
+template <>
+struct base_type_traits<bool>
+{
+    static const base_type type = base_type::BOOL;
+};
+
+template <>
+struct base_type_traits<table>
+{
+    static const base_type type = base_type::TABLE;
+};
+
+template <>
+struct base_type_traits<array>
+{
+    static const base_type type = base_type::ARRAY;
+};
+
+template <>
+struct base_type_traits<table_array>
+{
+    static const base_type type = base_type::TABLE_ARRAY;
+};
+#endif
+
+/**
+ * A generic base TOML value used for type erasure.
+ */
+class base : public std::enable_shared_from_this<base>
+{
+  public:
+    virtual ~base() = default;
+
+    virtual std::shared_ptr<base> clone() const = 0;
+
+    /**
+     * Determines if the given TOML element is a value.
+     */
+    virtual bool is_value() const
+    {
+        return false;
+    }
+
+    /**
+     * Determines if the given TOML element is a table.
+     */
+    virtual bool is_table() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element into a table.
+     */
+    std::shared_ptr<table> as_table()
+    {
+        if (is_table())
+            return std::static_pointer_cast<table>(shared_from_this());
+        return nullptr;
+    }
+    /**
+     * Determines if the TOML element is an array of "leaf" elements.
+     */
+    virtual bool is_array() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element to an array.
+     */
+    std::shared_ptr<array> as_array()
+    {
+        if (is_array())
+            return std::static_pointer_cast<array>(shared_from_this());
+        return nullptr;
+    }
+
+    /**
+     * Determines if the given TOML element is an array of tables.
+     */
+    virtual bool is_table_array() const
+    {
+        return false;
+    }
+
+    /**
+     * Converts the TOML element into a table array.
+     */
+    std::shared_ptr<table_array> as_table_array()
+    {
+        if (is_table_array())
+            return std::static_pointer_cast<table_array>(shared_from_this());
+        return nullptr;
+    }
+
+    /**
+     * Attempts to coerce the TOML element into a concrete TOML value
+     * of type T.
+     */
+    template <class T>
+    std::shared_ptr<value<T>> as();
+
+    template <class T>
+    std::shared_ptr<const value<T>> as() const;
+
+    template <class Visitor, class... Args>
+    void accept(Visitor&& visitor, Args&&... args) const;
+
+#if defined(CPPTOML_NO_RTTI)
+    base_type type() const
+    {
+      return type_;
+    }
+
+  protected:
+    base(const base_type t) : type_(t)
+    {
+        // nothing
+    }
+
+  private:
+    const base_type type_ = base_type::NONE;
+
+#else
+  protected:
+    base()
+    {
+        // nothing
+    }
+#endif
+};
+
+/**
+ * A concrete TOML value representing the "leaves" of the "tree".
+ */
+template <class T>
+class value : public base
+{
+    struct make_shared_enabler
+    {
+        // nothing; this is a private key accessible only to friends
+    };
+
+    template <class U>
+    friend std::shared_ptr<typename value_traits<U>::type>
+    cpptoml::make_value(U&& val);
+
+  public:
+    static_assert(valid_value<T>::value, "invalid value type");
+
+    std::shared_ptr<base> clone() const override;
+
+    value(const make_shared_enabler&, const T& val) : value(val)
+    {
+        // nothing; note that users cannot actually invoke this function
+        // because they lack access to the make_shared_enabler.
+    }
+
+    bool is_value() const override
+    {
+        return true;
+    }
+
+    /**
+     * Gets the data associated with this value.
+     */
+    T& get()
+    {
+        return data_;
+    }
+
+    /**
+     * Gets the data associated with this value. Const version.
+     */
+    const T& get() const
+    {
+        return data_;
+    }
+
+  private:
+    T data_;
+
+    /**
+     * Constructs a value from the given data.
+     */
+#if defined(CPPTOML_NO_RTTI)
+    value(const T& val) : base(base_type_traits<T>::type), data_(val)
+    {
+    }
+#else
+    value(const T& val) : data_(val)
+    {
+    }
+#endif
+
+    value(const value& val) = delete;
+    value& operator=(const value& val) = delete;
+};
+
+template <class T>
+std::shared_ptr<typename value_traits<T>::type> make_value(T&& val)
+{
+    using value_type = typename value_traits<T>::type;
+    using enabler = typename value_type::make_shared_enabler;
+    return std::make_shared<value_type>(
+        enabler{}, value_traits<T>::construct(std::forward<T>(val)));
+}
+
+template <class T>
+inline std::shared_ptr<value<T>> base::as()
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type_traits<T>::type)
+        return std::static_pointer_cast<value<T>>(shared_from_this());
+    else
+        return nullptr;
+#else
+    return std::dynamic_pointer_cast<value<T>>(shared_from_this());
+#endif
+}
+
+// special case value<double> to allow getting an integer parameter as a
+// double value
+template <>
+inline std::shared_ptr<value<double>> base::as()
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type::FLOAT)
+        return std::static_pointer_cast<value<double>>(shared_from_this());
+
+    if (type() == base_type::INT)
+    {
+        auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this());
+        return make_value<double>(static_cast<double>(v->get()));;
+    }
+#else
+    if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this()))
+        return v;
+
+    if (auto v = std::dynamic_pointer_cast<value<int64_t>>(shared_from_this()))
+        return make_value<double>(static_cast<double>(v->get()));
+#endif
+
+    return nullptr;
+}
+
+template <class T>
+inline std::shared_ptr<const value<T>> base::as() const
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type_traits<T>::type)
+        return std::static_pointer_cast<const value<T>>(shared_from_this());
+    else
+        return nullptr;
+#else
+    return std::dynamic_pointer_cast<const value<T>>(shared_from_this());
+#endif
+}
+
+// special case value<double> to allow getting an integer parameter as a
+// double value
+template <>
+inline std::shared_ptr<const value<double>> base::as() const
+{
+#if defined(CPPTOML_NO_RTTI)
+    if (type() == base_type::FLOAT)
+        return std::static_pointer_cast<const value<double>>(shared_from_this());
+
+    if (type() == base_type::INT)
+    {
+        auto v = as<int64_t>();
+        // the below has to be a non-const value<double> due to a bug in
+        // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843
+        return make_value<double>(static_cast<double>(v->get()));
+    }
+#else
+    if (auto v
+        = std::dynamic_pointer_cast<const value<double>>(shared_from_this()))
+        return v;
+
+    if (auto v = as<int64_t>())
+    {
+        // the below has to be a non-const value<double> due to a bug in
+        // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843
+        return make_value<double>(static_cast<double>(v->get()));
+    }
+#endif
+
+    return nullptr;
+}
+
+/**
+ * Exception class for array insertion errors.
+ */
+class array_exception : public std::runtime_error
+{
+  public:
+    array_exception(const std::string& err) : std::runtime_error{err}
+    {
+    }
+};
+
+class array : public base
+{
+  public:
+    friend std::shared_ptr<array> make_array();
+
+    std::shared_ptr<base> clone() const override;
+
+    virtual bool is_array() const override
+    {
+        return true;
+    }
+
+    using size_type = std::size_t;
+
+    /**
+     * arrays can be iterated over
+     */
+    using iterator = std::vector<std::shared_ptr<base>>::iterator;
+
+    /**
+     * arrays can be iterated over.  Const version.
+     */
+    using const_iterator = std::vector<std::shared_ptr<base>>::const_iterator;
+
+    iterator begin()
+    {
+        return values_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return values_.begin();
+    }
+
+    iterator end()
+    {
+        return values_.end();
+    }
+
+    const_iterator end() const
+    {
+        return values_.end();
+    }
+
+    /**
+     * Obtains the array (vector) of base values.
+     */
+    std::vector<std::shared_ptr<base>>& get()
+    {
+        return values_;
+    }
+
+    /**
+     * Obtains the array (vector) of base values. Const version.
+     */
+    const std::vector<std::shared_ptr<base>>& get() const
+    {
+        return values_;
+    }
+
+    std::shared_ptr<base> at(size_t idx) const
+    {
+        return values_.at(idx);
+    }
+
+    /**
+     * Obtains an array of value<T>s. Note that elements may be
+     * nullptr if they cannot be converted to a value<T>.
+     */
+    template <class T>
+    std::vector<std::shared_ptr<value<T>>> array_of() const
+    {
+        std::vector<std::shared_ptr<value<T>>> result(values_.size());
+
+        std::transform(values_.begin(), values_.end(), result.begin(),
+                       [&](std::shared_ptr<base> v) { return v->as<T>(); });
+
+        return result;
+    }
+
+    /**
+     * Obtains a option<vector<T>>. The option will be empty if the array
+     * contains values that are not of type T.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type get_array_of() const
+    {
+        std::vector<T> result;
+        result.reserve(values_.size());
+
+        for (const auto& val : values_)
+        {
+            if (auto v = val->as<T>())
+                result.push_back(v->get());
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    /**
+     * Obtains an array of arrays. Note that elements may be nullptr
+     * if they cannot be converted to a array.
+     */
+    std::vector<std::shared_ptr<array>> nested_array() const
+    {
+        std::vector<std::shared_ptr<array>> result(values_.size());
+
+        std::transform(values_.begin(), values_.end(), result.begin(),
+                       [&](std::shared_ptr<base> v) -> std::shared_ptr<array> {
+                           if (v->is_array())
+                               return std::static_pointer_cast<array>(v);
+                           return std::shared_ptr<array>{};
+                       });
+
+        return result;
+    }
+
+    /**
+     * Add a value to the end of the array
+     */
+    template <class T>
+    void push_back(const std::shared_ptr<value<T>>& val)
+    {
+        if (values_.empty() || values_[0]->as<T>())
+        {
+            values_.push_back(val);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Add an array to the end of the array
+     */
+    void push_back(const std::shared_ptr<array>& val)
+    {
+        if (values_.empty() || values_[0]->is_array())
+        {
+            values_.push_back(val);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Convenience function for adding a simple element to the end
+     * of the array.
+     */
+    template <class T>
+    void push_back(T&& val, typename value_traits<T>::type* = 0)
+    {
+        push_back(make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Insert a value into the array
+     */
+    template <class T>
+    iterator insert(iterator position, const std::shared_ptr<value<T>>& value)
+    {
+        if (values_.empty() || values_[0]->as<T>())
+        {
+            return values_.insert(position, value);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Insert an array into the array
+     */
+    iterator insert(iterator position, const std::shared_ptr<array>& value)
+    {
+        if (values_.empty() || values_[0]->is_array())
+        {
+            return values_.insert(position, value);
+        }
+        else
+        {
+            throw array_exception{"Arrays must be homogenous."};
+        }
+    }
+
+    /**
+     * Convenience function for inserting a simple element in the array
+     */
+    template <class T>
+    iterator insert(iterator position, T&& val,
+                    typename value_traits<T>::type* = 0)
+    {
+        return insert(position, make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Erase an element from the array
+     */
+    iterator erase(iterator position)
+    {
+        return values_.erase(position);
+    }
+
+    /**
+     * Clear the array
+     */
+    void clear()
+    {
+        values_.clear();
+    }
+
+    /**
+     * Reserve space for n values.
+     */
+    void reserve(size_type n)
+    {
+        values_.reserve(n);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    array() : base(base_type::ARRAY)
+    {
+        // empty
+    }
+#else
+    array() = default;
+#endif
+
+    template <class InputIterator>
+    array(InputIterator begin, InputIterator end) : values_{begin, end}
+    {
+        // nothing
+    }
+
+    array(const array& obj) = delete;
+    array& operator=(const array& obj) = delete;
+
+    std::vector<std::shared_ptr<base>> values_;
+};
+
+inline std::shared_ptr<array> make_array()
+{
+    struct make_shared_enabler : public array
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<array> make_element<array>()
+{
+    return make_array();
+}
+
+/**
+ * Obtains a option<vector<T>>. The option will be empty if the array
+ * contains values that are not of type T.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+array::get_array_of<array>() const
+{
+    std::vector<std::shared_ptr<array>> result;
+    result.reserve(values_.size());
+
+    for (const auto& val : values_)
+    {
+        if (auto v = val->as_array())
+            result.push_back(v);
+        else
+            return {};
+    }
+
+    return {std::move(result)};
+}
+
+class table;
+
+class table_array : public base
+{
+    friend class table;
+    friend std::shared_ptr<table_array> make_table_array();
+
+  public:
+    std::shared_ptr<base> clone() const override;
+
+    using size_type = std::size_t;
+
+    /**
+     * arrays can be iterated over
+     */
+    using iterator = std::vector<std::shared_ptr<table>>::iterator;
+
+    /**
+     * arrays can be iterated over.  Const version.
+     */
+    using const_iterator = std::vector<std::shared_ptr<table>>::const_iterator;
+
+    iterator begin()
+    {
+        return array_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return array_.begin();
+    }
+
+    iterator end()
+    {
+        return array_.end();
+    }
+
+    const_iterator end() const
+    {
+        return array_.end();
+    }
+
+    virtual bool is_table_array() const override
+    {
+        return true;
+    }
+
+    std::vector<std::shared_ptr<table>>& get()
+    {
+        return array_;
+    }
+
+    const std::vector<std::shared_ptr<table>>& get() const
+    {
+        return array_;
+    }
+
+    /**
+     * Add a table to the end of the array
+     */
+    void push_back(const std::shared_ptr<table>& val)
+    {
+        array_.push_back(val);
+    }
+
+    /**
+     * Insert a table into the array
+     */
+    iterator insert(iterator position, const std::shared_ptr<table>& value)
+    {
+        return array_.insert(position, value);
+    }
+
+    /**
+     * Erase an element from the array
+     */
+    iterator erase(iterator position)
+    {
+        return array_.erase(position);
+    }
+
+    /**
+     * Clear the array
+     */
+    void clear()
+    {
+        array_.clear();
+    }
+
+    /**
+     * Reserve space for n tables.
+     */
+    void reserve(size_type n)
+    {
+        array_.reserve(n);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    table_array() : base(base_type::TABLE_ARRAY)
+    {
+        // nothing
+    }
+#else
+    table_array()
+    {
+        // nothing
+    }
+#endif
+
+    table_array(const table_array& obj) = delete;
+    table_array& operator=(const table_array& rhs) = delete;
+
+    std::vector<std::shared_ptr<table>> array_;
+};
+
+inline std::shared_ptr<table_array> make_table_array()
+{
+    struct make_shared_enabler : public table_array
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<table_array> make_element<table_array>()
+{
+    return make_table_array();
+}
+
+// The below are overloads for fetching specific value types out of a value
+// where special casting behavior (like bounds checking) is desired
+
+template <class T>
+typename std::enable_if<!std::is_floating_point<T>::value
+                            && std::is_signed<T>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<int64_t>())
+    {
+        if (v->get() < (std::numeric_limits<T>::min)())
+            throw std::underflow_error{
+                "T cannot represent the value requested in get"};
+
+        if (v->get() > (std::numeric_limits<T>::max)())
+            throw std::overflow_error{
+                "T cannot represent the value requested in get"};
+
+        return {static_cast<T>(v->get())};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+template <class T>
+typename std::enable_if<!std::is_same<T, bool>::value
+                            && std::is_unsigned<T>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<int64_t>())
+    {
+        if (v->get() < 0)
+            throw std::underflow_error{"T cannot store negative value in get"};
+
+        if (static_cast<uint64_t>(v->get()) > (std::numeric_limits<T>::max)())
+            throw std::overflow_error{
+                "T cannot represent the value requested in get"};
+
+        return {static_cast<T>(v->get())};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+template <class T>
+typename std::enable_if<!std::is_integral<T>::value
+                            || std::is_same<T, bool>::value,
+                        option<T>>::type
+get_impl(const std::shared_ptr<base>& elem)
+{
+    if (auto v = elem->as<T>())
+    {
+        return {v->get()};
+    }
+    else
+    {
+        return {};
+    }
+}
+
+/**
+ * Represents a TOML keytable.
+ */
+class table : public base
+{
+  public:
+    friend class table_array;
+    friend std::shared_ptr<table> make_table();
+
+    std::shared_ptr<base> clone() const override;
+
+    /**
+     * tables can be iterated over.
+     */
+    using iterator = string_to_base_map::iterator;
+
+    /**
+     * tables can be iterated over. Const version.
+     */
+    using const_iterator = string_to_base_map::const_iterator;
+
+    iterator begin()
+    {
+        return map_.begin();
+    }
+
+    const_iterator begin() const
+    {
+        return map_.begin();
+    }
+
+    iterator end()
+    {
+        return map_.end();
+    }
+
+    const_iterator end() const
+    {
+        return map_.end();
+    }
+
+    bool is_table() const override
+    {
+        return true;
+    }
+
+    bool empty() const
+    {
+        return map_.empty();
+    }
+
+    /**
+     * Determines if this key table contains the given key.
+     */
+    bool contains(const std::string& key) const
+    {
+        return map_.find(key) != map_.end();
+    }
+
+    /**
+     * Determines if this key table contains the given key. Will
+     * resolve "qualified keys". Qualified keys are the full access
+     * path separated with dots like "grandparent.parent.child".
+     */
+    bool contains_qualified(const std::string& key) const
+    {
+        return resolve_qualified(key);
+    }
+
+    /**
+     * Obtains the base for a given key.
+     * @throw std::out_of_range if the key does not exist
+     */
+    std::shared_ptr<base> get(const std::string& key) const
+    {
+        return map_.at(key);
+    }
+
+    /**
+     * Obtains the base for a given key. Will resolve "qualified
+     * keys". Qualified keys are the full access path separated with
+     * dots like "grandparent.parent.child".
+     *
+     * @throw std::out_of_range if the key does not exist
+     */
+    std::shared_ptr<base> get_qualified(const std::string& key) const
+    {
+        std::shared_ptr<base> p;
+        resolve_qualified(key, &p);
+        return p;
+    }
+
+    /**
+     * Obtains a table for a given key, if possible.
+     */
+    std::shared_ptr<table> get_table(const std::string& key) const
+    {
+        if (contains(key) && get(key)->is_table())
+            return std::static_pointer_cast<table>(get(key));
+        return nullptr;
+    }
+
+    /**
+     * Obtains a table for a given key, if possible. Will resolve
+     * "qualified keys".
+     */
+    std::shared_ptr<table> get_table_qualified(const std::string& key) const
+    {
+        if (contains_qualified(key) && get_qualified(key)->is_table())
+            return std::static_pointer_cast<table>(get_qualified(key));
+        return nullptr;
+    }
+
+    /**
+     * Obtains an array for a given key.
+     */
+    std::shared_ptr<array> get_array(const std::string& key) const
+    {
+        if (!contains(key))
+            return nullptr;
+        return get(key)->as_array();
+    }
+
+    /**
+     * Obtains an array for a given key. Will resolve "qualified keys".
+     */
+    std::shared_ptr<array> get_array_qualified(const std::string& key) const
+    {
+        if (!contains_qualified(key))
+            return nullptr;
+        return get_qualified(key)->as_array();
+    }
+
+    /**
+     * Obtains a table_array for a given key, if possible.
+     */
+    std::shared_ptr<table_array> get_table_array(const std::string& key) const
+    {
+        if (!contains(key))
+            return nullptr;
+        return get(key)->as_table_array();
+    }
+
+    /**
+     * Obtains a table_array for a given key, if possible. Will resolve
+     * "qualified keys".
+     */
+    std::shared_ptr<table_array>
+    get_table_array_qualified(const std::string& key) const
+    {
+        if (!contains_qualified(key))
+            return nullptr;
+        return get_qualified(key)->as_table_array();
+    }
+
+    /**
+     * Helper function that attempts to get a value corresponding
+     * to the template parameter from a given key.
+     */
+    template <class T>
+    option<T> get_as(const std::string& key) const
+    {
+        try
+        {
+            return get_impl<T>(get(key));
+        }
+        catch (const std::out_of_range&)
+        {
+            return {};
+        }
+    }
+
+    /**
+     * Helper function that attempts to get a value corresponding
+     * to the template parameter from a given key. Will resolve "qualified
+     * keys".
+     */
+    template <class T>
+    option<T> get_qualified_as(const std::string& key) const
+    {
+        try
+        {
+            return get_impl<T>(get_qualified(key));
+        }
+        catch (const std::out_of_range&)
+        {
+            return {};
+        }
+    }
+
+    /**
+     * Helper function that attempts to get an array of values of a given
+     * type corresponding to the template parameter for a given key.
+     *
+     * If the key doesn't exist, doesn't exist as an array type, or one or
+     * more keys inside the array type are not of type T, an empty option
+     * is returned. Otherwise, an option containing a vector of the values
+     * is returned.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type
+    get_array_of(const std::string& key) const
+    {
+        if (auto v = get_array(key))
+        {
+            std::vector<T> result;
+            result.reserve(v->get().size());
+
+            for (const auto& b : v->get())
+            {
+                if (auto val = b->as<T>())
+                    result.push_back(val->get());
+                else
+                    return {};
+            }
+            return {std::move(result)};
+        }
+
+        return {};
+    }
+
+    /**
+     * Helper function that attempts to get an array of values of a given
+     * type corresponding to the template parameter for a given key. Will
+     * resolve "qualified keys".
+     *
+     * If the key doesn't exist, doesn't exist as an array type, or one or
+     * more keys inside the array type are not of type T, an empty option
+     * is returned. Otherwise, an option containing a vector of the values
+     * is returned.
+     */
+    template <class T>
+    inline typename array_of_trait<T>::return_type
+    get_qualified_array_of(const std::string& key) const
+    {
+        if (auto v = get_array_qualified(key))
+        {
+            std::vector<T> result;
+            result.reserve(v->get().size());
+
+            for (const auto& b : v->get())
+            {
+                if (auto val = b->as<T>())
+                    result.push_back(val->get());
+                else
+                    return {};
+            }
+            return {std::move(result)};
+        }
+
+        return {};
+    }
+
+    /**
+     * Adds an element to the keytable.
+     */
+    void insert(const std::string& key, const std::shared_ptr<base>& value)
+    {
+        map_[key] = value;
+    }
+
+    /**
+     * Convenience shorthand for adding a simple element to the
+     * keytable.
+     */
+    template <class T>
+    void insert(const std::string& key, T&& val,
+                typename value_traits<T>::type* = 0)
+    {
+        insert(key, make_value(std::forward<T>(val)));
+    }
+
+    /**
+     * Removes an element from the table.
+     */
+    void erase(const std::string& key)
+    {
+        map_.erase(key);
+    }
+
+  private:
+#if defined(CPPTOML_NO_RTTI)
+    table() : base(base_type::TABLE)
+    {
+        // nothing
+    }
+#else
+    table()
+    {
+        // nothing
+    }
+#endif
+
+    table(const table& obj) = delete;
+    table& operator=(const table& rhs) = delete;
+
+    std::vector<std::string> split(const std::string& value,
+                                   char separator) const
+    {
+        std::vector<std::string> result;
+        std::string::size_type p = 0;
+        std::string::size_type q;
+        while ((q = value.find(separator, p)) != std::string::npos)
+        {
+            result.emplace_back(value, p, q - p);
+            p = q + 1;
+        }
+        result.emplace_back(value, p);
+        return result;
+    }
+
+    // If output parameter p is specified, fill it with the pointer to the
+    // specified entry and throw std::out_of_range if it couldn't be found.
+    //
+    // Otherwise, just return true if the entry could be found or false
+    // otherwise and do not throw.
+    bool resolve_qualified(const std::string& key,
+                           std::shared_ptr<base>* p = nullptr) const
+    {
+        auto parts = split(key, '.');
+        auto last_key = parts.back();
+        parts.pop_back();
+
+        auto cur_table = this;
+        for (const auto& part : parts)
+        {
+            cur_table = cur_table->get_table(part).get();
+            if (!cur_table)
+            {
+                if (!p)
+                    return false;
+
+                throw std::out_of_range{key + " is not a valid key"};
+            }
+        }
+
+        if (!p)
+            return cur_table->map_.count(last_key) != 0;
+
+        *p = cur_table->map_.at(last_key);
+        return true;
+    }
+
+    string_to_base_map map_;
+};
+
+/**
+ * Helper function that attempts to get an array of arrays for a given
+ * key.
+ *
+ * If the key doesn't exist, doesn't exist as an array type, or one or
+ * more keys inside the array type are not of type T, an empty option
+ * is returned. Otherwise, an option containing a vector of the values
+ * is returned.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+table::get_array_of<array>(const std::string& key) const
+{
+    if (auto v = get_array(key))
+    {
+        std::vector<std::shared_ptr<array>> result;
+        result.reserve(v->get().size());
+
+        for (const auto& b : v->get())
+        {
+            if (auto val = b->as_array())
+                result.push_back(val);
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    return {};
+}
+
+/**
+ * Helper function that attempts to get an array of arrays for a given
+ * key. Will resolve "qualified keys".
+ *
+ * If the key doesn't exist, doesn't exist as an array type, or one or
+ * more keys inside the array type are not of type T, an empty option
+ * is returned. Otherwise, an option containing a vector of the values
+ * is returned.
+ */
+template <>
+inline typename array_of_trait<array>::return_type
+table::get_qualified_array_of<array>(const std::string& key) const
+{
+    if (auto v = get_array_qualified(key))
+    {
+        std::vector<std::shared_ptr<array>> result;
+        result.reserve(v->get().size());
+
+        for (const auto& b : v->get())
+        {
+            if (auto val = b->as_array())
+                result.push_back(val);
+            else
+                return {};
+        }
+
+        return {std::move(result)};
+    }
+
+    return {};
+}
+
+std::shared_ptr<table> make_table()
+{
+    struct make_shared_enabler : public table
+    {
+        make_shared_enabler()
+        {
+            // nothing
+        }
+    };
+
+    return std::make_shared<make_shared_enabler>();
+}
+
+template <>
+inline std::shared_ptr<table> make_element<table>()
+{
+    return make_table();
+}
+
+template <class T>
+std::shared_ptr<base> value<T>::clone() const
+{
+    return make_value(data_);
+}
+
+inline std::shared_ptr<base> array::clone() const
+{
+    auto result = make_array();
+    result->reserve(values_.size());
+    for (const auto& ptr : values_)
+        result->values_.push_back(ptr->clone());
+    return result;
+}
+
+inline std::shared_ptr<base> table_array::clone() const
+{
+    auto result = make_table_array();
+    result->reserve(array_.size());
+    for (const auto& ptr : array_)
+        result->array_.push_back(ptr->clone()->as_table());
+    return result;
+}
+
+inline std::shared_ptr<base> table::clone() const
+{
+    auto result = make_table();
+    for (const auto& pr : map_)
+        result->insert(pr.first, pr.second->clone());
+    return result;
+}
+
+/**
+ * Exception class for all TOML parsing errors.
+ */
+class parse_exception : public std::runtime_error
+{
+  public:
+    parse_exception(const std::string& err) : std::runtime_error{err}
+    {
+    }
+
+    parse_exception(const std::string& err, std::size_t line_number)
+        : std::runtime_error{err + " at line " + std::to_string(line_number)}
+    {
+    }
+};
+
+inline bool is_number(char c)
+{
+    return c >= '0' && c <= '9';
+}
+
+/**
+ * Helper object for consuming expected characters.
+ */
+template <class OnError>
+class consumer
+{
+  public:
+    consumer(std::string::iterator& it, const std::string::iterator& end,
+             OnError&& on_error)
+        : it_(it), end_(end), on_error_(std::forward<OnError>(on_error))
+    {
+        // nothing
+    }
+
+    void operator()(char c)
+    {
+        if (it_ == end_ || *it_ != c)
+            on_error_();
+        ++it_;
+    }
+
+    template <std::size_t N>
+    void operator()(const char (&str)[N])
+    {
+        std::for_each(std::begin(str), std::end(str) - 1,
+                      [&](char c) { (*this)(c); });
+    }
+
+    int eat_digits(int len)
+    {
+        int val = 0;
+        for (int i = 0; i < len; ++i)
+        {
+            if (!is_number(*it_) || it_ == end_)
+                on_error_();
+            val = 10 * val + (*it_++ - '0');
+        }
+        return val;
+    }
+
+    void error()
+    {
+        on_error_();
+    }
+
+  private:
+    std::string::iterator& it_;
+    const std::string::iterator& end_;
+    OnError on_error_;
+};
+
+template <class OnError>
+consumer<OnError> make_consumer(std::string::iterator& it,
+                                const std::string::iterator& end,
+                                OnError&& on_error)
+{
+    return consumer<OnError>(it, end, std::forward<OnError>(on_error));
+}
+
+// replacement for std::getline to handle incorrectly line-ended files
+// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf
+namespace detail
+{
+inline std::istream& getline(std::istream& input, std::string& line)
+{
+    line.clear();
+
+    std::istream::sentry sentry{input, true};
+    auto sb = input.rdbuf();
+
+    while (true)
+    {
+        auto c = sb->sbumpc();
+        if (c == '\r')
+        {
+            if (sb->sgetc() == '\n')
+                c = sb->sbumpc();
+        }
+
+        if (c == '\n')
+            return input;
+
+        if (c == std::istream::traits_type::eof())
+        {
+            if (line.empty())
+                input.setstate(std::ios::eofbit);
+            return input;
+        }
+
+        line.push_back(static_cast<char>(c));
+    }
+}
+}
+
+/**
+ * The parser class.
+ */
+class parser
+{
+  public:
+    /**
+     * Parsers are constructed from streams.
+     */
+    parser(std::istream& stream) : input_(stream)
+    {
+        // nothing
+    }
+
+    parser& operator=(const parser& parser) = delete;
+
+    /**
+     * Parses the stream this parser was created on until EOF.
+     * @throw parse_exception if there are errors in parsing
+     */
+    std::shared_ptr<table> parse()
+    {
+        std::shared_ptr<table> root = make_table();
+
+        table* curr_table = root.get();
+
+        while (detail::getline(input_, line_))
+        {
+            line_number_++;
+            auto it = line_.begin();
+            auto end = line_.end();
+            consume_whitespace(it, end);
+            if (it == end || *it == '#')
+                continue;
+            if (*it == '[')
+            {
+                curr_table = root.get();
+                parse_table(it, end, curr_table);
+            }
+            else
+            {
+                parse_key_value(it, end, curr_table);
+                consume_whitespace(it, end);
+                eol_or_comment(it, end);
+            }
+        }
+        return root;
+    }
+
+  private:
+#if defined _MSC_VER
+    __declspec(noreturn)
+#elif defined __GNUC__
+    __attribute__((noreturn))
+#endif
+        void throw_parse_exception(const std::string& err)
+    {
+        throw parse_exception{err, line_number_};
+    }
+
+    void parse_table(std::string::iterator& it,
+                     const std::string::iterator& end, table*& curr_table)
+    {
+        // remove the beginning keytable marker
+        ++it;
+        if (it == end)
+            throw_parse_exception("Unexpected end of table");
+        if (*it == '[')
+            parse_table_array(it, end, curr_table);
+        else
+            parse_single_table(it, end, curr_table);
+    }
+
+    void parse_single_table(std::string::iterator& it,
+                            const std::string::iterator& end,
+                            table*& curr_table)
+    {
+        if (it == end || *it == ']')
+            throw_parse_exception("Table name cannot be empty");
+
+        std::string full_table_name;
+        bool inserted = false;
+        while (it != end && *it != ']')
+        {
+            auto part = parse_key(it, end,
+                                  [](char c) { return c == '.' || c == ']'; });
+
+            if (part.empty())
+                throw_parse_exception("Empty component of table name");
+
+            if (!full_table_name.empty())
+                full_table_name += ".";
+            full_table_name += part;
+
+            if (curr_table->contains(part))
+            {
+                auto b = curr_table->get(part);
+                if (b->is_table())
+                    curr_table = static_cast<table*>(b.get());
+                else if (b->is_table_array())
+                    curr_table = std::static_pointer_cast<table_array>(b)
+                                     ->get()
+                                     .back()
+                                     .get();
+                else
+                    throw_parse_exception("Key " + full_table_name
+                                          + "already exists as a value");
+            }
+            else
+            {
+                inserted = true;
+                curr_table->insert(part, make_table());
+                curr_table = static_cast<table*>(curr_table->get(part).get());
+            }
+            consume_whitespace(it, end);
+            if (it != end && *it == '.')
+                ++it;
+            consume_whitespace(it, end);
+        }
+
+        if (it == end)
+            throw_parse_exception(
+                "Unterminated table declaration; did you forget a ']'?");
+
+        // table already existed
+        if (!inserted)
+        {
+            auto is_value
+                = [](const std::pair<const std::string&,
+                                     const std::shared_ptr<base>&>& p) {
+                      return p.second->is_value();
+                  };
+
+            // if there are any values, we can't add values to this table
+            // since it has already been defined. If there aren't any
+            // values, then it was implicitly created by something like
+            // [a.b]
+            if (curr_table->empty() || std::any_of(curr_table->begin(),
+                                                   curr_table->end(), is_value))
+            {
+                throw_parse_exception("Redefinition of table "
+                                      + full_table_name);
+            }
+        }
+
+        ++it;
+        consume_whitespace(it, end);
+        eol_or_comment(it, end);
+    }
+
+    void parse_table_array(std::string::iterator& it,
+                           const std::string::iterator& end, table*& curr_table)
+    {
+        ++it;
+        if (it == end || *it == ']')
+            throw_parse_exception("Table array name cannot be empty");
+
+        std::string full_ta_name;
+        while (it != end && *it != ']')
+        {
+            auto part = parse_key(it, end,
+                                  [](char c) { return c == '.' || c == ']'; });
+
+            if (part.empty())
+                throw_parse_exception("Empty component of table array name");
+
+            if (!full_ta_name.empty())
+                full_ta_name += ".";
+            full_ta_name += part;
+
+            consume_whitespace(it, end);
+            if (it != end && *it == '.')
+                ++it;
+            consume_whitespace(it, end);
+
+            if (curr_table->contains(part))
+            {
+                auto b = curr_table->get(part);
+
+                // if this is the end of the table array name, add an
+                // element to the table array that we just looked up
+                if (it != end && *it == ']')
+                {
+                    if (!b->is_table_array())
+                        throw_parse_exception("Key " + full_ta_name
+                                              + " is not a table array");
+                    auto v = b->as_table_array();
+                    v->get().push_back(make_table());
+                    curr_table = v->get().back().get();
+                }
+                // otherwise, just keep traversing down the key name
+                else
+                {
+                    if (b->is_table())
+                        curr_table = static_cast<table*>(b.get());
+                    else if (b->is_table_array())
+                        curr_table = std::static_pointer_cast<table_array>(b)
+                                         ->get()
+                                         .back()
+                                         .get();
+                    else
+                        throw_parse_exception("Key " + full_ta_name
+                                              + " already exists as a value");
+                }
+            }
+            else
+            {
+                // if this is the end of the table array name, add a new
+                // table array and a new table inside that array for us to
+                // add keys to next
+                if (it != end && *it == ']')
+                {
+                    curr_table->insert(part, make_table_array());
+                    auto arr = std::static_pointer_cast<table_array>(
+                        curr_table->get(part));
+                    arr->get().push_back(make_table());
+                    curr_table = arr->get().back().get();
+                }
+                // otherwise, create the implicitly defined table and move
+                // down to it
+                else
+                {
+                    curr_table->insert(part, make_table());
+                    curr_table
+                        = static_cast<table*>(curr_table->get(part).get());
+                }
+            }
+        }
+
+        // consume the last "]]"
+        if (it == end)
+            throw_parse_exception("Unterminated table array name");
+        ++it;
+        if (it == end)
+            throw_parse_exception("Unterminated table array name");
+        ++it;
+
+        consume_whitespace(it, end);
+        eol_or_comment(it, end);
+    }
+
+    void parse_key_value(std::string::iterator& it, std::string::iterator& end,
+                         table* curr_table)
+    {
+        auto key = parse_key(it, end, [](char c) { return c == '='; });
+        if (curr_table->contains(key))
+            throw_parse_exception("Key " + key + " already present");
+        if (it == end || *it != '=')
+            throw_parse_exception("Value must follow after a '='");
+        ++it;
+        consume_whitespace(it, end);
+        curr_table->insert(key, parse_value(it, end));
+        consume_whitespace(it, end);
+    }
+
+    template <class Function>
+    std::string parse_key(std::string::iterator& it,
+                          const std::string::iterator& end, Function&& fun)
+    {
+        consume_whitespace(it, end);
+        if (*it == '"')
+        {
+            return parse_quoted_key(it, end);
+        }
+        else
+        {
+            auto bke = std::find_if(it, end, std::forward<Function>(fun));
+            return parse_bare_key(it, bke);
+        }
+    }
+
+    std::string parse_bare_key(std::string::iterator& it,
+                               const std::string::iterator& end)
+    {
+        if (it == end)
+        {
+            throw_parse_exception("Bare key missing name");
+        }
+
+        auto key_end = end;
+        --key_end;
+        consume_backwards_whitespace(key_end, it);
+        ++key_end;
+        std::string key{it, key_end};
+
+        if (std::find(it, key_end, '#') != key_end)
+        {
+            throw_parse_exception("Bare key " + key + " cannot contain #");
+        }
+
+        if (std::find_if(it, key_end,
+                         [](char c) { return c == ' ' || c == '\t'; })
+            != key_end)
+        {
+            throw_parse_exception("Bare key " + key
+                                  + " cannot contain whitespace");
+        }
+
+        if (std::find_if(it, key_end,
+                         [](char c) { return c == '[' || c == ']'; })
+            != key_end)
+        {
+            throw_parse_exception("Bare key " + key
+                                  + " cannot contain '[' or ']'");
+        }
+
+        it = end;
+        return key;
+    }
+
+    std::string parse_quoted_key(std::string::iterator& it,
+                                 const std::string::iterator& end)
+    {
+        return string_literal(it, end, '"');
+    }
+
+    enum class parse_type
+    {
+        STRING = 1,
+        LOCAL_TIME,
+        LOCAL_DATE,
+        LOCAL_DATETIME,
+        OFFSET_DATETIME,
+        INT,
+        FLOAT,
+        BOOL,
+        ARRAY,
+        INLINE_TABLE
+    };
+
+    std::shared_ptr<base> parse_value(std::string::iterator& it,
+                                      std::string::iterator& end)
+    {
+        parse_type type = determine_value_type(it, end);
+        switch (type)
+        {
+            case parse_type::STRING:
+                return parse_string(it, end);
+            case parse_type::LOCAL_TIME:
+                return parse_time(it, end);
+            case parse_type::LOCAL_DATE:
+            case parse_type::LOCAL_DATETIME:
+            case parse_type::OFFSET_DATETIME:
+                return parse_date(it, end);
+            case parse_type::INT:
+            case parse_type::FLOAT:
+                return parse_number(it, end);
+            case parse_type::BOOL:
+                return parse_bool(it, end);
+            case parse_type::ARRAY:
+                return parse_array(it, end);
+            case parse_type::INLINE_TABLE:
+                return parse_inline_table(it, end);
+            default:
+                throw_parse_exception("Failed to parse value");
+        }
+    }
+
+    parse_type determine_value_type(const std::string::iterator& it,
+                                    const std::string::iterator& end)
+    {
+        if(it == end)
+        {
+            throw_parse_exception("Failed to parse value type");
+        }
+        if (*it == '"' || *it == '\'')
+        {
+            return parse_type::STRING;
+        }
+        else if (is_time(it, end))
+        {
+            return parse_type::LOCAL_TIME;
+        }
+        else if (auto dtype = date_type(it, end))
+        {
+            return *dtype;
+        }
+        else if (is_number(*it) || *it == '-' || *it == '+')
+        {
+            return determine_number_type(it, end);
+        }
+        else if (*it == 't' || *it == 'f')
+        {
+            return parse_type::BOOL;
+        }
+        else if (*it == '[')
+        {
+            return parse_type::ARRAY;
+        }
+        else if (*it == '{')
+        {
+            return parse_type::INLINE_TABLE;
+        }
+        throw_parse_exception("Failed to parse value type");
+    }
+
+    parse_type determine_number_type(const std::string::iterator& it,
+                                     const std::string::iterator& end)
+    {
+        // determine if we are an integer or a float
+        auto check_it = it;
+        if (*check_it == '-' || *check_it == '+')
+            ++check_it;
+        while (check_it != end && is_number(*check_it))
+            ++check_it;
+        if (check_it != end && *check_it == '.')
+        {
+            ++check_it;
+            while (check_it != end && is_number(*check_it))
+                ++check_it;
+            return parse_type::FLOAT;
+        }
+        else
+        {
+            return parse_type::INT;
+        }
+    }
+
+    std::shared_ptr<value<std::string>> parse_string(std::string::iterator& it,
+                                                     std::string::iterator& end)
+    {
+        auto delim = *it;
+        assert(delim == '"' || delim == '\'');
+
+        // end is non-const here because we have to be able to potentially
+        // parse multiple lines in a string, not just one
+        auto check_it = it;
+        ++check_it;
+        if (check_it != end && *check_it == delim)
+        {
+            ++check_it;
+            if (check_it != end && *check_it == delim)
+            {
+                it = ++check_it;
+                return parse_multiline_string(it, end, delim);
+            }
+        }
+        return make_value<std::string>(string_literal(it, end, delim));
+    }
+
+    std::shared_ptr<value<std::string>>
+    parse_multiline_string(std::string::iterator& it,
+                           std::string::iterator& end, char delim)
+    {
+        std::stringstream ss;
+
+        auto is_ws = [](char c) { return c == ' ' || c == '\t'; };
+
+        bool consuming = false;
+        std::shared_ptr<value<std::string>> ret;
+
+        auto handle_line
+            = [&](std::string::iterator& local_it,
+                  std::string::iterator& local_end) {
+                  if (consuming)
+                  {
+                      local_it = std::find_if_not(local_it, local_end, is_ws);
+
+                      // whole line is whitespace
+                      if (local_it == local_end)
+                          return;
+                  }
+
+                  consuming = false;
+
+                  while (local_it != local_end)
+                  {
+                      // handle escaped characters
+                      if (delim == '"' && *local_it == '\\')
+                      {
+                          auto check = local_it;
+                          // check if this is an actual escape sequence or a
+                          // whitespace escaping backslash
+                          ++check;
+                          consume_whitespace(check, local_end);
+                          if (check == local_end)
+                          {
+                              consuming = true;
+                              break;
+                          }
+
+                          ss << parse_escape_code(local_it, local_end);
+                          continue;
+                      }
+
+                      // if we can end the string
+                      if (std::distance(local_it, local_end) >= 3)
+                      {
+                          auto check = local_it;
+                          // check for """
+                          if (*check++ == delim && *check++ == delim
+                              && *check++ == delim)
+                          {
+                              local_it = check;
+                              ret = make_value<std::string>(ss.str());
+                              break;
+                          }
+                      }
+
+                      ss << *local_it++;
+                  }
+              };
+
+        // handle the remainder of the current line
+        handle_line(it, end);
+        if (ret)
+            return ret;
+
+        // start eating lines
+        while (detail::getline(input_, line_))
+        {
+            ++line_number_;
+
+            it = line_.begin();
+            end = line_.end();
+
+            handle_line(it, end);
+
+            if (ret)
+                return ret;
+
+            if (!consuming)
+                ss << std::endl;
+        }
+
+        throw_parse_exception("Unterminated multi-line basic string");
+    }
+
+    std::string string_literal(std::string::iterator& it,
+                               const std::string::iterator& end, char delim)
+    {
+        ++it;
+        std::string val;
+        while (it != end)
+        {
+            // handle escaped characters
+            if (delim == '"' && *it == '\\')
+            {
+                val += parse_escape_code(it, end);
+            }
+            else if (*it == delim)
+            {
+                ++it;
+                consume_whitespace(it, end);
+                return val;
+            }
+            else
+            {
+                val += *it++;
+            }
+        }
+        throw_parse_exception("Unterminated string literal");
+    }
+
+    std::string parse_escape_code(std::string::iterator& it,
+                                  const std::string::iterator& end)
+    {
+        ++it;
+        if (it == end)
+            throw_parse_exception("Invalid escape sequence");
+        char value;
+        if (*it == 'b')
+        {
+            value = '\b';
+        }
+        else if (*it == 't')
+        {
+            value = '\t';
+        }
+        else if (*it == 'n')
+        {
+            value = '\n';
+        }
+        else if (*it == 'f')
+        {
+            value = '\f';
+        }
+        else if (*it == 'r')
+        {
+            value = '\r';
+        }
+        else if (*it == '"')
+        {
+            value = '"';
+        }
+        else if (*it == '\\')
+        {
+            value = '\\';
+        }
+        else if (*it == 'u' || *it == 'U')
+        {
+            return parse_unicode(it, end);
+        }
+        else
+        {
+            throw_parse_exception("Invalid escape sequence");
+        }
+        ++it;
+        return std::string(1, value);
+    }
+
+    std::string parse_unicode(std::string::iterator& it,
+                              const std::string::iterator& end)
+    {
+        bool large = *it++ == 'U';
+        auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000);
+
+        if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff)
+        {
+            throw_parse_exception(
+                "Unicode escape sequence is not a Unicode scalar value");
+        }
+
+        std::string result;
+        // See Table 3-6 of the Unicode standard
+        if (codepoint <= 0x7f)
+        {
+            // 1-byte codepoints: 00000000 0xxxxxxx
+            // repr: 0xxxxxxx
+            result += static_cast<char>(codepoint & 0x7f);
+        }
+        else if (codepoint <= 0x7ff)
+        {
+            // 2-byte codepoints: 00000yyy yyxxxxxx
+            // repr: 110yyyyy 10xxxxxx
+            //
+            // 0x1f = 00011111
+            // 0xc0 = 11000000
+            //
+            result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f));
+            //
+            // 0x80 = 10000000
+            // 0x3f = 00111111
+            //
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        else if (codepoint <= 0xffff)
+        {
+            // 3-byte codepoints: zzzzyyyy yyxxxxxx
+            // repr: 1110zzzz 10yyyyyy 10xxxxxx
+            //
+            // 0xe0 = 11100000
+            // 0x0f = 00001111
+            //
+            result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f));
+            result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f));
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        else
+        {
+            // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx
+            // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx
+            //
+            // 0xf0 = 11110000
+            // 0x07 = 00000111
+            //
+            result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07));
+            result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f));
+            result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f));
+            result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        }
+        return result;
+    }
+
+    uint32_t parse_hex(std::string::iterator& it,
+                       const std::string::iterator& end, uint32_t place)
+    {
+        uint32_t value = 0;
+        while (place > 0)
+        {
+            if (it == end)
+                throw_parse_exception("Unexpected end of unicode sequence");
+
+            if (!is_hex(*it))
+                throw_parse_exception("Invalid unicode escape sequence");
+
+            value += place * hex_to_digit(*it++);
+            place /= 16;
+        }
+        return value;
+    }
+
+    bool is_hex(char c)
+    {
+        return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+    }
+
+    uint32_t hex_to_digit(char c)
+    {
+        if (is_number(c))
+            return static_cast<uint32_t>(c - '0');
+        return 10 + static_cast<uint32_t>(
+                        c - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
+    }
+
+    std::shared_ptr<base> parse_number(std::string::iterator& it,
+                                       const std::string::iterator& end)
+    {
+        auto check_it = it;
+        auto check_end = find_end_of_number(it, end);
+
+        auto eat_sign = [&]() {
+            if (check_it != end && (*check_it == '-' || *check_it == '+'))
+                ++check_it;
+        };
+
+        eat_sign();
+
+        auto eat_numbers = [&]() {
+            auto beg = check_it;
+            while (check_it != end && is_number(*check_it))
+            {
+                ++check_it;
+                if (check_it != end && *check_it == '_')
+                {
+                    ++check_it;
+                    if (check_it == end || !is_number(*check_it))
+                        throw_parse_exception("Malformed number");
+                }
+            }
+
+            if (check_it == beg)
+                throw_parse_exception("Malformed number");
+        };
+
+        auto check_no_leading_zero = [&]() {
+            if (check_it != end && *check_it == '0' && check_it + 1 != check_end
+                && check_it[1] != '.')
+            {
+                throw_parse_exception("Numbers may not have leading zeros");
+            }
+        };
+
+        check_no_leading_zero();
+        eat_numbers();
+
+        if (check_it != end
+            && (*check_it == '.' || *check_it == 'e' || *check_it == 'E'))
+        {
+            bool is_exp = *check_it == 'e' || *check_it == 'E';
+
+            ++check_it;
+            if (check_it == end)
+                throw_parse_exception("Floats must have trailing digits");
+
+            auto eat_exp = [&]() {
+                eat_sign();
+                check_no_leading_zero();
+                eat_numbers();
+            };
+
+            if (is_exp)
+                eat_exp();
+            else
+                eat_numbers();
+
+            if (!is_exp && check_it != end
+                && (*check_it == 'e' || *check_it == 'E'))
+            {
+                ++check_it;
+                eat_exp();
+            }
+
+            return parse_float(it, check_it);
+        }
+        else
+        {
+            return parse_int(it, check_it);
+        }
+    }
+
+    std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it,
+                                              const std::string::iterator& end)
+    {
+        std::string v{it, end};
+        v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
+        it = end;
+        try
+        {
+            return make_value<int64_t>(std::stoll(v));
+        }
+        catch (const std::invalid_argument& ex)
+        {
+            throw_parse_exception("Malformed number (invalid argument: "
+                                  + std::string{ex.what()} + ")");
+        }
+        catch (const std::out_of_range& ex)
+        {
+            throw_parse_exception("Malformed number (out of range: "
+                                  + std::string{ex.what()} + ")");
+        }
+    }
+
+    std::shared_ptr<value<double>> parse_float(std::string::iterator& it,
+                                               const std::string::iterator& end)
+    {
+        std::string v{it, end};
+        v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
+        it = end;
+        char decimal_point = std::localeconv()->decimal_point[0];
+        std::replace(v.begin(), v.end(), '.', decimal_point);
+        try
+        {
+            return make_value<double>(std::stod(v));
+        }
+        catch (const std::invalid_argument& ex)
+        {
+            throw_parse_exception("Malformed number (invalid argument: "
+                                  + std::string{ex.what()} + ")");
+        }
+        catch (const std::out_of_range& ex)
+        {
+            throw_parse_exception("Malformed number (out of range: "
+                                  + std::string{ex.what()} + ")");
+        }
+    }
+
+    std::shared_ptr<value<bool>> parse_bool(std::string::iterator& it,
+                                            const std::string::iterator& end)
+    {
+        auto eat = make_consumer(it, end, [this]() {
+            throw_parse_exception("Attempted to parse invalid boolean value");
+        });
+
+        if (*it == 't')
+        {
+            eat("true");
+            return make_value<bool>(true);
+        }
+        else if (*it == 'f')
+        {
+            eat("false");
+            return make_value<bool>(false);
+        }
+
+        eat.error();
+        return nullptr;
+    }
+
+    std::string::iterator find_end_of_number(std::string::iterator it,
+                                             std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E'
+                   && c != '-' && c != '+';
+        });
+    }
+
+    std::string::iterator find_end_of_date(std::string::iterator it,
+                                           std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-'
+                   && c != '+' && c != '.';
+        });
+    }
+
+    std::string::iterator find_end_of_time(std::string::iterator it,
+                                           std::string::iterator end)
+    {
+        return std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != ':' && c != '.';
+        });
+    }
+
+    local_time read_time(std::string::iterator& it,
+                         const std::string::iterator& end)
+    {
+        auto time_end = find_end_of_time(it, end);
+
+        auto eat = make_consumer(
+            it, time_end, [&]() { throw_parse_exception("Malformed time"); });
+
+        local_time ltime;
+
+        ltime.hour = eat.eat_digits(2);
+        eat(':');
+        ltime.minute = eat.eat_digits(2);
+        eat(':');
+        ltime.second = eat.eat_digits(2);
+
+        int power = 100000;
+        if (it != time_end && *it == '.')
+        {
+            ++it;
+            while (it != time_end && is_number(*it))
+            {
+                ltime.microsecond += power * (*it++ - '0');
+                power /= 10;
+            }
+        }
+
+        if (it != time_end)
+            throw_parse_exception("Malformed time");
+
+        return ltime;
+    }
+
+    std::shared_ptr<value<local_time>>
+    parse_time(std::string::iterator& it, const std::string::iterator& end)
+    {
+        return make_value(read_time(it, end));
+    }
+
+    std::shared_ptr<base> parse_date(std::string::iterator& it,
+                                     const std::string::iterator& end)
+    {
+        auto date_end = find_end_of_date(it, end);
+
+        auto eat = make_consumer(
+            it, date_end, [&]() { throw_parse_exception("Malformed date"); });
+
+        local_date ldate;
+        ldate.year = eat.eat_digits(4);
+        eat('-');
+        ldate.month = eat.eat_digits(2);
+        eat('-');
+        ldate.day = eat.eat_digits(2);
+
+        if (it == date_end)
+            return make_value(ldate);
+
+        eat('T');
+
+        local_datetime ldt;
+        static_cast<local_date&>(ldt) = ldate;
+        static_cast<local_time&>(ldt) = read_time(it, date_end);
+
+        if (it == date_end)
+            return make_value(ldt);
+
+        offset_datetime dt;
+        static_cast<local_datetime&>(dt) = ldt;
+
+        int hoff = 0;
+        int moff = 0;
+        if (*it == '+' || *it == '-')
+        {
+            auto plus = *it == '+';
+            ++it;
+
+            hoff = eat.eat_digits(2);
+            dt.hour_offset = (plus) ? hoff : -hoff;
+            eat(':');
+            moff = eat.eat_digits(2);
+            dt.minute_offset = (plus) ? moff : -moff;
+        }
+        else if (*it == 'Z')
+        {
+            ++it;
+        }
+
+        if (it != date_end)
+            throw_parse_exception("Malformed date");
+
+        return make_value(dt);
+    }
+
+    std::shared_ptr<base> parse_array(std::string::iterator& it,
+                                      std::string::iterator& end)
+    {
+        // this gets ugly because of the "homogeneity" restriction:
+        // arrays can either be of only one type, or contain arrays
+        // (each of those arrays could be of different types, though)
+        //
+        // because of the latter portion, we don't really have a choice
+        // but to represent them as arrays of base values...
+        ++it;
+
+        // ugh---have to read the first value to determine array type...
+        skip_whitespace_and_comments(it, end);
+
+        // edge case---empty array
+        if (*it == ']')
+        {
+            ++it;
+            return make_array();
+        }
+
+        auto val_end = std::find_if(
+            it, end, [](char c) { return c == ',' || c == ']' || c == '#'; });
+        parse_type type = determine_value_type(it, val_end);
+        switch (type)
+        {
+            case parse_type::STRING:
+                return parse_value_array<std::string>(it, end);
+            case parse_type::LOCAL_TIME:
+                return parse_value_array<local_time>(it, end);
+            case parse_type::LOCAL_DATE:
+                return parse_value_array<local_date>(it, end);
+            case parse_type::LOCAL_DATETIME:
+                return parse_value_array<local_datetime>(it, end);
+            case parse_type::OFFSET_DATETIME:
+                return parse_value_array<offset_datetime>(it, end);
+            case parse_type::INT:
+                return parse_value_array<int64_t>(it, end);
+            case parse_type::FLOAT:
+                return parse_value_array<double>(it, end);
+            case parse_type::BOOL:
+                return parse_value_array<bool>(it, end);
+            case parse_type::ARRAY:
+                return parse_object_array<array>(&parser::parse_array, '[', it,
+                                                 end);
+            case parse_type::INLINE_TABLE:
+                return parse_object_array<table_array>(
+                    &parser::parse_inline_table, '{', it, end);
+            default:
+                throw_parse_exception("Unable to parse array");
+        }
+    }
+
+    template <class Value>
+    std::shared_ptr<array> parse_value_array(std::string::iterator& it,
+                                             std::string::iterator& end)
+    {
+        auto arr = make_array();
+        while (it != end && *it != ']')
+        {
+            auto value = parse_value(it, end);
+            if (auto v = value->as<Value>())
+                arr->get().push_back(value);
+            else
+                throw_parse_exception("Arrays must be homogeneous");
+            skip_whitespace_and_comments(it, end);
+            if (*it != ',')
+                break;
+            ++it;
+            skip_whitespace_and_comments(it, end);
+        }
+        if (it != end)
+            ++it;
+        return arr;
+    }
+
+    template <class Object, class Function>
+    std::shared_ptr<Object> parse_object_array(Function&& fun, char delim,
+                                               std::string::iterator& it,
+                                               std::string::iterator& end)
+    {
+        auto arr = make_element<Object>();
+
+        while (it != end && *it != ']')
+        {
+            if (*it != delim)
+                throw_parse_exception("Unexpected character in array");
+
+            arr->get().push_back(((*this).*fun)(it, end));
+            skip_whitespace_and_comments(it, end);
+
+            if (*it != ',')
+                break;
+
+            ++it;
+            skip_whitespace_and_comments(it, end);
+        }
+
+        if (it == end || *it != ']')
+            throw_parse_exception("Unterminated array");
+
+        ++it;
+        return arr;
+    }
+
+    std::shared_ptr<table> parse_inline_table(std::string::iterator& it,
+                                              std::string::iterator& end)
+    {
+        auto tbl = make_table();
+        do
+        {
+            ++it;
+            if (it == end)
+                throw_parse_exception("Unterminated inline table");
+
+            consume_whitespace(it, end);
+            parse_key_value(it, end, tbl.get());
+            consume_whitespace(it, end);
+        } while (*it == ',');
+
+        if (it == end || *it != '}')
+            throw_parse_exception("Unterminated inline table");
+
+        ++it;
+        consume_whitespace(it, end);
+
+        return tbl;
+    }
+
+    void skip_whitespace_and_comments(std::string::iterator& start,
+                                      std::string::iterator& end)
+    {
+        consume_whitespace(start, end);
+        while (start == end || *start == '#')
+        {
+            if (!detail::getline(input_, line_))
+                throw_parse_exception("Unclosed array");
+            line_number_++;
+            start = line_.begin();
+            end = line_.end();
+            consume_whitespace(start, end);
+        }
+    }
+
+    void consume_whitespace(std::string::iterator& it,
+                            const std::string::iterator& end)
+    {
+        while (it != end && (*it == ' ' || *it == '\t'))
+            ++it;
+    }
+
+    void consume_backwards_whitespace(std::string::iterator& back,
+                                      const std::string::iterator& front)
+    {
+        while (back != front && (*back == ' ' || *back == '\t'))
+            --back;
+    }
+
+    void eol_or_comment(const std::string::iterator& it,
+                        const std::string::iterator& end)
+    {
+        if (it != end && *it != '#')
+            throw_parse_exception("Unidentified trailing character '"
+                                  + std::string{*it}
+                                  + "'---did you forget a '#'?");
+    }
+
+    bool is_time(const std::string::iterator& it,
+                 const std::string::iterator& end)
+    {
+        auto time_end = find_end_of_time(it, end);
+        auto len = std::distance(it, time_end);
+
+        if (len < 8)
+            return false;
+
+        if (it[2] != ':' || it[5] != ':')
+            return false;
+
+        if (len > 8)
+            return it[8] == '.' && len > 9;
+
+        return true;
+    }
+
+    option<parse_type> date_type(const std::string::iterator& it,
+                                 const std::string::iterator& end)
+    {
+        auto date_end = find_end_of_date(it, end);
+        auto len = std::distance(it, date_end);
+
+        if (len < 10)
+            return {};
+
+        if (it[4] != '-' || it[7] != '-')
+            return {};
+
+        if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end))
+        {
+            // datetime type
+            auto time_end = find_end_of_time(it + 11, date_end);
+            if (time_end == date_end)
+                return {parse_type::LOCAL_DATETIME};
+            else
+                return {parse_type::OFFSET_DATETIME};
+        }
+        else if (len == 10)
+        {
+            // just a regular date
+            return {parse_type::LOCAL_DATE};
+        }
+
+        return {};
+    }
+
+    std::istream& input_;
+    std::string line_;
+    std::size_t line_number_ = 0;
+};
+
+/**
+ * Utility function to parse a file as a TOML file. Returns the root table.
+ * Throws a parse_exception if the file cannot be opened.
+ */
+inline std::shared_ptr<table> parse_file(const std::string& filename)
+{
+#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP)
+    boost::nowide::ifstream file{filename.c_str()};
+#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP)
+    nowide::ifstream file{filename.c_str()};
+#else
+    std::ifstream file{filename};
+#endif
+    if (!file.is_open())
+        throw parse_exception{filename + " could not be opened for parsing"};
+    parser p{file};
+    return p.parse();
+}
+
+template <class... Ts>
+struct value_accept;
+
+template <>
+struct value_accept<>
+{
+    template <class Visitor, class... Args>
+    static void accept(const base&, Visitor&&, Args&&...)
+    {
+        // nothing
+    }
+};
+
+template <class T, class... Ts>
+struct value_accept<T, Ts...>
+{
+    template <class Visitor, class... Args>
+    static void accept(const base& b, Visitor&& visitor, Args&&... args)
+    {
+        if (auto v = b.as<T>())
+        {
+            visitor.visit(*v, std::forward<Args>(args)...);
+        }
+        else
+        {
+            value_accept<Ts...>::accept(b, std::forward<Visitor>(visitor),
+                                        std::forward<Args>(args)...);
+        }
+    }
+};
+
+/**
+ * base implementation of accept() that calls visitor.visit() on the concrete
+ * class.
+ */
+template <class Visitor, class... Args>
+void base::accept(Visitor&& visitor, Args&&... args) const
+{
+    if (is_value())
+    {
+        using value_acceptor
+            = value_accept<std::string, int64_t, double, bool, local_date,
+                           local_time, local_datetime, offset_datetime>;
+        value_acceptor::accept(*this, std::forward<Visitor>(visitor),
+                               std::forward<Args>(args)...);
+    }
+    else if (is_table())
+    {
+        visitor.visit(static_cast<const table&>(*this),
+                      std::forward<Args>(args)...);
+    }
+    else if (is_array())
+    {
+        visitor.visit(static_cast<const array&>(*this),
+                      std::forward<Args>(args)...);
+    }
+    else if (is_table_array())
+    {
+        visitor.visit(static_cast<const table_array&>(*this),
+                      std::forward<Args>(args)...);
+    }
+}
+
+/**
+ * Writer that can be passed to accept() functions of cpptoml objects and
+ * will output valid TOML to a stream.
+ */
+class toml_writer
+{
+  public:
+    /**
+     * Construct a toml_writer that will write to the given stream
+     */
+    toml_writer(std::ostream& s, const std::string& indent_space = "\t")
+        : stream_(s), indent_(indent_space), has_naked_endline_(false)
+    {
+        // nothing
+    }
+
+  public:
+    /**
+     * Output a base value of the TOML tree.
+     */
+    template <class T>
+    void visit(const value<T>& v, bool = false)
+    {
+        write(v);
+    }
+
+    /**
+     * Output a table element of the TOML tree
+     */
+    void visit(const table& t, bool in_array = false)
+    {
+        write_table_header(in_array);
+        std::vector<std::string> values;
+        std::vector<std::string> tables;
+
+        for (const auto& i : t)
+        {
+            if (i.second->is_table() || i.second->is_table_array())
+            {
+                tables.push_back(i.first);
+            }
+            else
+            {
+                values.push_back(i.first);
+            }
+        }
+
+        for (unsigned int i = 0; i < values.size(); ++i)
+        {
+            path_.push_back(values[i]);
+
+            if (i > 0)
+                endline();
+
+            write_table_item_header(*t.get(values[i]));
+            t.get(values[i])->accept(*this, false);
+            path_.pop_back();
+        }
+
+        for (unsigned int i = 0; i < tables.size(); ++i)
+        {
+            path_.push_back(tables[i]);
+
+            if (values.size() > 0 || i > 0)
+                endline();
+
+            write_table_item_header(*t.get(tables[i]));
+            t.get(tables[i])->accept(*this, false);
+            path_.pop_back();
+        }
+
+        endline();
+    }
+
+    /**
+     * Output an array element of the TOML tree
+     */
+    void visit(const array& a, bool = false)
+    {
+        write("[");
+
+        for (unsigned int i = 0; i < a.get().size(); ++i)
+        {
+            if (i > 0)
+                write(", ");
+
+            if (a.get()[i]->is_array())
+            {
+                a.get()[i]->as_array()->accept(*this, true);
+            }
+            else
+            {
+                a.get()[i]->accept(*this, true);
+            }
+        }
+
+        write("]");
+    }
+
+    /**
+     * Output a table_array element of the TOML tree
+     */
+    void visit(const table_array& t, bool = false)
+    {
+        for (unsigned int j = 0; j < t.get().size(); ++j)
+        {
+            if (j > 0)
+                endline();
+
+            t.get()[j]->accept(*this, true);
+        }
+
+        endline();
+    }
+
+    /**
+     * Escape a string for output.
+     */
+    static std::string escape_string(const std::string& str)
+    {
+        std::string res;
+        for (auto it = str.begin(); it != str.end(); ++it)
+        {
+            if (*it == '\b')
+            {
+                res += "\\b";
+            }
+            else if (*it == '\t')
+            {
+                res += "\\t";
+            }
+            else if (*it == '\n')
+            {
+                res += "\\n";
+            }
+            else if (*it == '\f')
+            {
+                res += "\\f";
+            }
+            else if (*it == '\r')
+            {
+                res += "\\r";
+            }
+            else if (*it == '"')
+            {
+                res += "\\\"";
+            }
+            else if (*it == '\\')
+            {
+                res += "\\\\";
+            }
+            else if ((const uint32_t)*it <= 0x001f)
+            {
+                res += "\\u";
+                std::stringstream ss;
+                ss << std::hex << static_cast<uint32_t>(*it);
+                res += ss.str();
+            }
+            else
+            {
+                res += *it;
+            }
+        }
+        return res;
+    }
+
+  protected:
+    /**
+     * Write out a string.
+     */
+    void write(const value<std::string>& v)
+    {
+        write("\"");
+        write(escape_string(v.get()));
+        write("\"");
+    }
+
+    /**
+     * Write out a double.
+     */
+    void write(const value<double>& v)
+    {
+        std::ios::fmtflags flags{stream_.flags()};
+
+        stream_ << std::showpoint;
+        write(v.get());
+
+        stream_.flags(flags);
+    }
+
+    /**
+     * Write out an integer, local_date, local_time, local_datetime, or
+     * offset_datetime.
+     */
+    template <class T>
+    typename std::enable_if<is_one_of<T, int64_t, local_date, local_time,
+                                      local_datetime,
+                                      offset_datetime>::value>::type
+    write(const value<T>& v)
+    {
+        write(v.get());
+    }
+
+    /**
+     * Write out a boolean.
+     */
+    void write(const value<bool>& v)
+    {
+        write((v.get() ? "true" : "false"));
+    }
+
+    /**
+     * Write out the header of a table.
+     */
+    void write_table_header(bool in_array = false)
+    {
+        if (!path_.empty())
+        {
+            indent();
+
+            write("[");
+
+            if (in_array)
+            {
+                write("[");
+            }
+
+            for (unsigned int i = 0; i < path_.size(); ++i)
+            {
+                if (i > 0)
+                {
+                    write(".");
+                }
+
+                if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde"
+                                               "fghijklmnopqrstuvwxyz0123456789"
+                                               "_-")
+                    == std::string::npos)
+                {
+                    write(path_[i]);
+                }
+                else
+                {
+                    write("\"");
+                    write(escape_string(path_[i]));
+                    write("\"");
+                }
+            }
+
+            if (in_array)
+            {
+                write("]");
+            }
+
+            write("]");
+            endline();
+        }
+    }
+
+    /**
+     * Write out the identifier for an item in a table.
+     */
+    void write_table_item_header(const base& b)
+    {
+        if (!b.is_table() && !b.is_table_array())
+        {
+            indent();
+
+            if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde"
+                                               "fghijklmnopqrstuvwxyz0123456789"
+                                               "_-")
+                == std::string::npos)
+            {
+                write(path_.back());
+            }
+            else
+            {
+                write("\"");
+                write(escape_string(path_.back()));
+                write("\"");
+            }
+
+            write(" = ");
+        }
+    }
+
+  private:
+    /**
+     * Indent the proper number of tabs given the size of
+     * the path.
+     */
+    void indent()
+    {
+        for (std::size_t i = 1; i < path_.size(); ++i)
+            write(indent_);
+    }
+
+    /**
+     * Write a value out to the stream.
+     */
+    template <class T>
+    void write(const T& v)
+    {
+        stream_ << v;
+        has_naked_endline_ = false;
+    }
+
+    /**
+     * Write an endline out to the stream
+     */
+    void endline()
+    {
+        if (!has_naked_endline_)
+        {
+            stream_ << "\n";
+            has_naked_endline_ = true;
+        }
+    }
+
+  private:
+    std::ostream& stream_;
+    const std::string indent_;
+    std::vector<std::string> path_;
+    bool has_naked_endline_;
+};
+
+inline std::ostream& operator<<(std::ostream& stream, const base& b)
+{
+    toml_writer writer{stream};
+    b.accept(writer);
+    return stream;
+}
+
+template <class T>
+std::ostream& operator<<(std::ostream& stream, const value<T>& v)
+{
+    toml_writer writer{stream};
+    v.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const table& t)
+{
+    toml_writer writer{stream};
+    t.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const table_array& t)
+{
+    toml_writer writer{stream};
+    t.accept(writer);
+    return stream;
+}
+
+inline std::ostream& operator<<(std::ostream& stream, const array& a)
+{
+    toml_writer writer{stream};
+    a.accept(writer);
+    return stream;
+}
+}
+#endif
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index b04d7c40710..ae8aba612bb 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include <cpptoml.h>
+#include "cpptoml/cpptoml.h"
 
 namespace nix {
 

From e388739098cfb14a166923cd24b6140674b176f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Jul 2018 19:52:32 +0200
Subject: [PATCH 1101/2196] nix-daemon: Respect --store

For example, this allows you to do run nix-daemon as a non-privileged
user:

  eelco$ NIX_STATE_DIR=~/my-nix/nix/var nix-daemon --store ~/my-nix/

The NIX_STATE_DIR is still needed because settings.nixDaemonSocketFile
is not derived from settings.storeUri (and we can't derive it from the
store's state directory because we don't want to open the store in the
parent process).
---
 src/nix-daemon/nix-daemon.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 423e6bb6789..a2e54b93c05 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -233,7 +233,7 @@ struct RetrieveRegularNARSink : ParseSink
 };
 
 
-static void performOp(TunnelLogger * logger, ref<LocalStore> store,
+static void performOp(TunnelLogger * logger, ref<Store> store,
     bool trusted, unsigned int clientVersion,
     Source & from, Sink & to, unsigned int op)
 {
@@ -362,7 +362,11 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store,
 
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
-        Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+        if (!store2) throw Error("operation is only supported by LocalStore");
+
+        Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         logger->stopWork();
 
         to << path;
@@ -776,7 +780,7 @@ static void processConnection(bool trusted)
         Store::Params params; // FIXME: get params from somewhere
         // Disable caching since the client already does that.
         params["path-info-cache-size"] = "0";
-        auto store = make_ref<LocalStore>(params);
+        auto store = openStore(settings.storeUri, params);
 
         tunnelLogger->stopWork();
         to.flush();

From cb9ef858529348df8dded2f0bface7d9612cafc6 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 3 Jul 2018 15:36:20 -0500
Subject: [PATCH 1102/2196] tests/search.sh: minor fix to unbreak tests after
 search ux merge

---
 tests/search.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/search.sh b/tests/search.sh
index 0b26a125120..14da3127b0d 100644
--- a/tests/search.sh
+++ b/tests/search.sh
@@ -30,7 +30,7 @@ clearCache
 (( $(nix search nosuchpackageexists | wc -l) == 0 ))
 
 # Search for multiple arguments
-(( $(nix search hello empty | wc -l) == 5 ))
+(( $(nix search hello empty | wc -l) == 3 ))
 
 # Multiple arguments will not exist
 (( $(nix search hello broken | wc -l) == 0 ))

From 403a76a18fbbbc7b1ad4b65ed8c66c9abbbffa6f Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Thu, 5 Jul 2018 02:52:02 +0000
Subject: [PATCH 1103/2196] lib.concatMap and lib.mapAttrs to be builtins

---
 src/libexpr/primops.cc             | 42 ++++++++++++++++++++++++++++++
 tests/lang/eval-okay-concatmap.exp |  1 +
 tests/lang/eval-okay-concatmap.nix |  5 ++++
 tests/lang/eval-okay-mapattrs.exp  |  1 +
 tests/lang/eval-okay-mapattrs.nix  |  3 +++
 5 files changed, 52 insertions(+)
 create mode 100644 tests/lang/eval-okay-concatmap.exp
 create mode 100644 tests/lang/eval-okay-concatmap.nix
 create mode 100644 tests/lang/eval-okay-mapattrs.exp
 create mode 100644 tests/lang/eval-okay-mapattrs.nix

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 3a6c4035b8b..7ff57f0b01c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1356,6 +1356,24 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 }
 
 
+/* Apply a function to every element of a list. */
+static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceAttrs(*args[1], pos);
+
+    state.mkAttrs(v, args[1]->attrs->size());
+
+    for (auto & i : *args[1]->attrs) {
+        Value vName, vFun2;
+        mkString(vName, i.name);
+        state.callFunction(*args[0], vName, vFun2, pos);
+        state.callFunction(vFun2, *i.value, *state.allocAttr(v, i.name), pos);
+    }
+}
+
+
+
 /*************************************************************
  * Lists
  *************************************************************/
@@ -1627,6 +1645,28 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 }
 
 
+/* concatMap = f: list: concatLists (map f list); */
+/* C++-version is to avoid allocating `mkApp', call `f' eagerly */
+static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+    auto len = args[1]->listSize();
+
+    Value vList;
+    state.mkList(vList, len);
+
+    for (unsigned int n = 0; n < len; ++n) {
+        Value * vElem = args[1]->listElems()[n];
+        state.forceValue(*vElem);
+        state.callFunction(*args[0], *vElem, *(vList.listElems()[n] = state.allocValue()), pos);
+    }
+
+    state.concatLists(v, len, vList.listElems(), pos);
+}
+
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -2212,6 +2252,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
     addPrimOp("__catAttrs", 2, prim_catAttrs);
     addPrimOp("__functionArgs", 1, prim_functionArgs);
+    addPrimOp("__mapAttrs", 2, prim_mapAttrs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -2229,6 +2270,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__genList", 2, prim_genList);
     addPrimOp("__sort", 2, prim_sort);
     addPrimOp("__partition", 2, prim_partition);
+    addPrimOp("__concatMap", 2, prim_concatMap);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/lang/eval-okay-concatmap.exp
new file mode 100644
index 00000000000..3b8be7739de
--- /dev/null
+++ b/tests/lang/eval-okay-concatmap.exp
@@ -0,0 +1 @@
+[ [ 1 3 5 7 9 ] [ "a" "z" "b" "z" ] ]
diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/lang/eval-okay-concatmap.nix
new file mode 100644
index 00000000000..97da5d37a41
--- /dev/null
+++ b/tests/lang/eval-okay-concatmap.nix
@@ -0,0 +1,5 @@
+with import ./lib.nix;
+
+[ (builtins.concatMap (x: if x / 2 * 2 == x then [] else [ x ]) (range 0 10))
+  (builtins.concatMap (x: [x] ++ ["z"]) ["a" "b"])
+]
diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/lang/eval-okay-mapattrs.exp
new file mode 100644
index 00000000000..3f113f17bab
--- /dev/null
+++ b/tests/lang/eval-okay-mapattrs.exp
@@ -0,0 +1 @@
+{ x = "x-foo"; y = "y-bar"; }
diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/lang/eval-okay-mapattrs.nix
new file mode 100644
index 00000000000..f075b6275e5
--- /dev/null
+++ b/tests/lang/eval-okay-mapattrs.nix
@@ -0,0 +1,3 @@
+with import ./lib.nix;
+
+builtins.mapAttrs (name: value: name + "-" + value) { x = "foo"; y = "bar"; }

From ee218f99cac96bfd48aeb8fa17c271bcc2515ab1 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Thu, 5 Jul 2018 11:58:15 +0000
Subject: [PATCH 1104/2196] primops.cc: fix comment

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7ff57f0b01c..aea2a34354b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1356,7 +1356,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-/* Apply a function to every element of a list. */
+/* Apply a function to every element of an attribute set. */
 static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceFunction(*args[0], pos);

From 841747b0e6d9c391c13159aaeee44599f9508868 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Thu, 5 Jul 2018 12:37:37 +0000
Subject: [PATCH 1105/2196] prim_concatMap: allocate intermediate list on stack

---
 src/libexpr/primops.cc | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index aea2a34354b..9b475197016 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1651,22 +1651,30 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
 {
     state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
-    auto len = args[1]->listSize();
+    auto nrLists = args[1]->listSize();
 
-    Value vList;
-    state.mkList(vList, len);
+    Value lists[nrLists];
+    size_t len = 0;
 
-    for (unsigned int n = 0; n < len; ++n) {
+    for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.forceValue(*vElem);
-        state.callFunction(*args[0], *vElem, *(vList.listElems()[n] = state.allocValue()), pos);
+        state.callFunction(*args[0], *vElem, lists[n], pos);
+        state.forceList(lists[n], pos);
+        len += lists[n].listSize();
     }
 
-    state.concatLists(v, len, vList.listElems(), pos);
+    state.mkList(v, len);
+    auto out = v.listElems();
+    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+        auto l = lists[n].listSize();
+        if (l)
+            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+        pos += l;
+    }
 }
 
 
-
 /*************************************************************
  * Integer arithmetic
  *************************************************************/

From e6bf1a79d70686ae48096f2c7227d8a363358ab4 Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Thu, 5 Jul 2018 15:33:12 +0000
Subject: [PATCH 1106/2196] prim_mapAttrs: must be lazy to avoid infinite
 recursion

---
 src/libexpr/primops.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9b475197016..ed4c0285b30 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1365,10 +1365,11 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     state.mkAttrs(v, args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
-        Value vName, vFun2;
-        mkString(vName, i.name);
-        state.callFunction(*args[0], vName, vFun2, pos);
-        state.callFunction(vFun2, *i.value, *state.allocAttr(v, i.name), pos);
+        Value * vName = state.allocValue();
+        Value * vFun2 = state.allocValue();
+        mkString(*vName, i.name);
+        state.callFunction(*args[0], *vName, *vFun2, pos);
+        mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
     }
 }
 

From 1515c6561688b216d27c732c8df43b65909fa03e Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Thu, 5 Jul 2018 15:33:33 +0000
Subject: [PATCH 1107/2196] prim_concatMap: no need to force value

---
 src/libexpr/primops.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ed4c0285b30..e71e3a6d46c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1659,7 +1659,6 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
 
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
-        state.forceValue(*vElem);
         state.callFunction(*args[0], *vElem, lists[n], pos);
         state.forceList(lists[n], pos);
         len += lists[n].listSize();

From 627e28ba332f21109d7eace8ba0a7932e4d98b2e Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Fri, 6 Jul 2018 21:49:51 +0000
Subject: [PATCH 1108/2196] prim_mapAttrs: `f' must be evaluated lazily to
 avoid infinite recursion

---
 src/libexpr/eval.cc    | 6 ++----
 src/libexpr/primops.cc | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e09297546c9..30dbb17fb67 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1076,6 +1076,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    forceValue(fun, pos);
+
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
         callPrimOp(fun, arg, v, pos);
         return;
@@ -1091,10 +1093,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         auto & fun2 = *allocValue();
         fun2 = fun;
         /* !!! Should we use the attr pos here? */
-        forceValue(*found->value, pos);
         Value v2;
         callFunction(*found->value, fun2, v2, pos);
-        forceValue(v2, pos);
         return callFunction(v2, arg, v, pos);
       }
     }
@@ -1181,7 +1181,6 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     if (fun.type == tAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
-            forceValue(*found->value);
             Value * v = allocValue();
             callFunction(*found->value, fun, *v, noPos);
             forceValue(*v);
@@ -1565,7 +1564,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     if (v.type == tAttrs) {
         auto i = v.attrs->find(sToString);
         if (i != v.attrs->end()) {
-            forceValue(*i->value, pos);
             Value v1;
             callFunction(*i->value, v, v1, pos);
             return coerceToString(pos, v1, context, coerceMore, copyToStore);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e71e3a6d46c..e7b319fea64 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1359,7 +1359,6 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 /* Apply a function to every element of an attribute set. */
 static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceAttrs(*args[1], pos);
 
     state.mkAttrs(v, args[1]->attrs->size());
@@ -1368,7 +1367,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
         Value * vName = state.allocValue();
         Value * vFun2 = state.allocValue();
         mkString(*vName, i.name);
-        state.callFunction(*args[0], *vName, *vFun2, pos);
+        mkApp(*vFun2, *args[0], *vName);
         mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
     }
 }

From c03d73c1cfbeb7c7d00a6b48111f152496e46c8f Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Sat, 7 Jul 2018 23:30:17 +0000
Subject: [PATCH 1109/2196] prim_map, prim_genList: no need to force function
 argument which will be stored in mkApp

---
 src/libexpr/primops.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e7b319fea64..09b01cae8f7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1428,7 +1428,6 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 /* Apply a function to every element of a list. */
 static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
     state.mkList(v, args[1]->listSize());
@@ -1556,7 +1555,6 @@ static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceFunction(*args[0], pos);
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)

From 875cd9da2b20a7228dd337aa007ae11ef02a220a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Jul 2018 20:29:18 +0200
Subject: [PATCH 1110/2196] Remove unused function printStats2()

Closes #2282.
---
 src/libexpr/eval.cc | 14 --------------
 src/libexpr/eval.hh |  1 -
 2 files changed, 15 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e09297546c9..2c8a0eb422f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1720,20 +1720,6 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 }
 
 
-void EvalState::printStats2()
-{
-    struct rusage ru;
-    getrusage(RUSAGE_SELF, &ru);
-
-    GC_prof_stats_s gc;
-    GC_get_prof_stats(&gc, sizeof(gc));
-
-    printError("STATS %d %d %d %d %d %d",
-        nrValues, nrValuesFreed.load(), nrValues - nrValuesFreed,
-        ru.ru_maxrss,
-        gc.heapsize_full, gc.free_bytes_full);
-}
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 46bda86d084..d0f298e168e 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -276,7 +276,6 @@ public:
 
     /* Print statistics. */
     void printStats();
-    void printStats2();
 
     void realiseContext(const PathSet & context);
 

From efd04888cad18b76729b39294224ca547d074846 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Jul 2018 21:05:09 +0200
Subject: [PATCH 1111/2196] Shup up a warning

---
 src/libexpr/primops/fromTOML.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index ae8aba612bb..12e5de08e51 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -20,7 +20,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
         if (auto t2 = t->as_table()) {
 
             size_t size = 0;
-            for (auto & i : *t2) size++;
+            for (auto & i : *t2) { (void) i; size++; }
 
             state.mkAttrs(v, size);
 

From c2de2ff385a7eed2cb5853d7fde203a6394698ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Jul 2018 21:12:18 +0200
Subject: [PATCH 1112/2196] Fix build on 32-bit systems and macOS

Apparently, on macOS, 'long' != 'int64_t'.

https://hydra.nixos.org/build/77100756
---
 src/libexpr/primops/fromTOML.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 12e5de08e51..4128de05d0c 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -50,7 +50,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
         }
 
         else if (t->is_value()) {
-            if (auto val = t->as<NixInt>())
+            if (auto val = t->as<int64_t>())
                 mkInt(v, val->get());
             else if (auto val = t->as<NixFloat>())
                 mkFloat(v, val->get());

From 1b34b69b45106e5bfdbdc0201d3ff4dcd36632f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 12 Jul 2018 18:44:37 +0200
Subject: [PATCH 1113/2196] nix-prefetch-url: Download file in constant memory

Before:

  $ command time nix-prefetch-url https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.6.tar.xz
  1.19user 1.02system 0:41.96elapsed 5%CPU (0avgtext+0avgdata 182720maxresident)k

After:

  1.38user 1.05system 0:39.73elapsed 6%CPU (0avgtext+0avgdata 16204maxresident)k

Note however that addToStore() can still take a lot of memory
(e.g. RemoteStore::addToStore() is constant space, but
LocalStore::addToStore() isn't; that's fixed by
https://github.com/edolstra/nix/commit/c94b4fc7ee0c7b322a5f3c7ee784063b47a11d98
though).

Fixes #1400.
---
 src/nix-prefetch-url/nix-prefetch-url.cc | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 50b2c2803ec..a3b025723cf 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -9,6 +9,10 @@
 
 #include <iostream>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
 using namespace nix;
 
 
@@ -160,14 +164,20 @@ int main(int argc, char * * argv)
 
             auto actualUri = resolveMirrorUri(*state, uri);
 
-            /* Download the file. */
-            DownloadRequest req(actualUri);
-            req.decompress = false;
-            auto result = getDownloader()->download(req);
-
             AutoDelete tmpDir(createTempDir(), true);
             Path tmpFile = (Path) tmpDir + "/tmp";
-            writeFile(tmpFile, *result.data);
+
+            /* Download the file. */
+            {
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+                FdSink sink(fd.get());
+
+                DownloadRequest req(actualUri);
+                req.decompress = false;
+                getDownloader()->download(std::move(req), sink);
+            }
 
             /* Optionally unpack the file. */
             if (unpack) {
@@ -191,7 +201,7 @@ int main(int argc, char * * argv)
 
             /* FIXME: inefficient; addToStore() will also hash
                this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
+            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
                 throw Error(format("hash mismatch for '%1%'") % uri);

From 41f38fbb4b89dc5df244a07410724beca2e14d96 Mon Sep 17 00:00:00 2001
From: Nathan van Doorn <nvd1234@gmail.com>
Date: Mon, 16 Jul 2018 10:00:42 +0100
Subject: [PATCH 1114/2196] nix-channel documentation: don't suggest deprecated
 function

Running `nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion` emits a warning
```
trace: `lib.nixpkgsVersion` is deprecated, use `lib.version` instead!
```
---
 doc/manual/command-ref/nix-channel.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
index ff4021a765e..f5159165a75 100644
--- a/doc/manual/command-ref/nix-channel.xml
+++ b/doc/manual/command-ref/nix-channel.xml
@@ -112,13 +112,13 @@ $ nix-env -iA nixpkgs.hello</screen>
 <para>You can revert channel updates using <option>--rollback</option>:</para>
 
 <screen>
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.527.0e935f1"
 
 $ nix-channel --rollback
 switching from generation 483 to 482
 
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.526.dbadfad"
 </screen>
 

From 4495560d6df5f89c3ae32b960f457fbc2f3fb831 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 19 Jul 2018 10:29:31 -0500
Subject: [PATCH 1115/2196] Revert "progress-bar: re-draw last update if
 nothing new for 1sec."

Not ready for this yet, causes the prompt to disappear in nix repl
and more generally can overwrite non-progress-bar messages.

This reverts commit 44de71a39624d86d6744062ee36f57170024c9a0.
---
 src/nix/progress-bar.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 8093d8761c4..40b905ba324 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -75,10 +75,9 @@ class ProgressBar : public Logger
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
-                auto r = state.wait_for(updateCV, std::chrono::seconds(1));
+                state.wait(updateCV);
                 draw(*state);
-                if (r == std::cv_status::no_timeout)
-                  state.wait_for(quitCV, std::chrono::milliseconds(50));
+                state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
         });
     }

From e2b114cfe19c28bfd9fe95e1aa97aa154b904002 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Sat, 21 Jul 2018 06:44:42 +0000
Subject: [PATCH 1116/2196] prim_foldlStrict: call forceValue() before value is
 copied

forceValue() were called after a value is copied effectively forcing only one of the copies keeping another copy not evaluated.
This resulted in its evaluation of the same lazy value more than once (the number of hits is not big though)
---
 src/libexpr/primops.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e71e3a6d46c..311a32b0b30 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1508,19 +1508,20 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceFunction(*args[0], pos);
     state.forceList(*args[2], pos);
 
-    Value * vCur = args[1];
+    if (args[2]->listSize()) {
+        Value * vCur = args[1];
 
-    if (args[2]->listSize())
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
             Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-    else
-        v = *vCur;
-
-    state.forceValue(v);
+        state.forceValue(v);
+    } else {
+        state.forceValue(*args[1]);
+        v = *args[1];
+    }
 }
 
 

From 85fe4a819ca52ffbe99dd0d7820b5cd6219c50f1 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Sat, 21 Jul 2018 15:24:51 +0000
Subject: [PATCH 1117/2196] parser.y: fix assoc of -> and < > <= >=

The parser allowed senseless `a > b > c` but disallowed `a -> b -> c` which seems valid
It might be a typo
---
 src/libexpr/parser.y | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index eee48887dc2..3ef896692e3 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -273,11 +273,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%nonassoc IMPL
+%left IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ
-%left '<' '>' LEQ GEQ
+%nonassoc '<' '>' LEQ GEQ
 %right UPDATE
 %left NOT
 %left '+' '-'

From deaa6e9a34f24c1fdfeed0b98054c28e50100c12 Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Mon, 23 Jul 2018 07:28:48 +0000
Subject: [PATCH 1118/2196] parser.y: right-associativity of ->

---
 src/libexpr/parser.y | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 3ef896692e3..cbd576d7d12 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -273,7 +273,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
 
-%left IMPL
+%right IMPL
 %left OR
 %left AND
 %nonassoc EQ NEQ

From 9ac1a7988216f1252a600f6f67b685b9cc01283a Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Tue, 24 Jul 2018 17:03:54 +0200
Subject: [PATCH 1119/2196] copyPathsToStore: honour keep-going

---
 src/libstore/store-api.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9b0b7d6327e..1854353828d 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -629,11 +629,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
     Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
 
     std::atomic<size_t> nrDone{0};
+    std::atomic<size_t> nrFailed{0};
     std::atomic<uint64_t> bytesExpected{0};
     std::atomic<uint64_t> nrRunning{0};
 
     auto showProgress = [&]() {
-        act.progress(nrDone, missing.size(), nrRunning);
+        act.progress(nrDone, missing.size(), nrRunning, nrFailed);
     };
 
     ThreadPool pool;
@@ -662,7 +663,16 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
             if (!dstStore->isValidPath(storePath)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
-                copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                try {
+                    copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                } catch (Error &e) {
+                    nrFailed++;
+                    if (!settings.keepGoing)
+                        throw e;
+                    logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
+                    showProgress();
+                    return;
+                }
             }
 
             nrDone++;

From 81d13854375cfae08bac742879bd18dcbd28c5da Mon Sep 17 00:00:00 2001
From: Symphorien Gibol <symphorien+git@xlumurb.eu>
Date: Thu, 26 Jul 2018 18:14:50 +0200
Subject: [PATCH 1120/2196] add manual entries for disallowedRequisites and
 disallowedReferences

---
 .../expressions/advanced-attributes.xml       | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index dfd013b5cf3..9422e82ff36 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -50,6 +50,40 @@ allowedRequisites = [ foobar ];
 
   </varlistentry>
 
+  <varlistentry><term><varname>disallowedReferences</varname></term>
+
+    <listitem><para>The optional attribute
+    <varname>disallowedReferences</varname> specifies a list of illegal
+    references (dependencies) of the output of the builder.  For
+    example,
+
+<programlisting>
+disallowedReferences = [ foo ];
+</programlisting>
+
+    enforces that the output of a derivation cannot have a direct runtime
+    dependencies on the derivation <varname>foo</varname>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><varname>disallowedRequisites</varname></term>
+
+    <listitem><para>This attribute is similar to
+    <varname>disallowedReferences</varname>, but it specifies illegal
+    requisites for the whole closure, so all the dependencies
+    recursively.  For example,
+
+<programlisting>
+disallowedRequisites = [ foobar ];
+</programlisting>
+
+    enforces that the output of a derivation cannot have any
+    runtime dependency on <varname>foobar</varname> or any other derivation
+    depending recursively on <varname>foobar</varname>.</para></listitem>
+
+  </varlistentry>
+
 
   <varlistentry><term><varname>exportReferencesGraph</varname></term>
 

From 0d72f4e0ca3e88f3ded66112a6b157e14c2724c4 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 27 Jul 2018 09:52:54 -0500
Subject: [PATCH 1121/2196] search: include version

---
 src/nix/search.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 4cb1efa7955..e086de2260a 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -173,10 +173,12 @@ struct CmdSearch : SourceExprCommand, MixJSON
                             jsonElem.attr("description", description);
 
                         } else {
+                            auto name = hilite(parsed.name, nameMatch, "\e[0;2m")
+                                + std::string(parsed.fullName, parsed.name.length());
                             results[attrPath] = fmt(
                                 "* %s (%s)\n  %s\n",
                                 wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
-                                wrap("\e[0;2m", hilite(parsed.fullName, nameMatch, "\e[0;2m")),
+                                wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")),
                                 hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }

From bd19959d9000ca0a952a0461339a384427632576 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 11:53:53 +0200
Subject: [PATCH 1122/2196] Fix RPM build

https://hydra.nixos.org/build/73991153
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 321f1688367..27d56e5f0c8 100644
--- a/release.nix
+++ b/release.nix
@@ -332,7 +332,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
       memSize = 2047;

From 607e75083af97b12df1cfacb82e2e1e69fdd5517 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 14:03:19 +0200
Subject: [PATCH 1123/2196] Attempt to fix the RPM build

---
 nix.spec.in | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index cd053dbfce5..477768c6a68 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -152,11 +152,11 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/systemd/system/nix-daemon.service
 %endif
 %{_datadir}/nix
-%if ! %{without docgen}
-%{_mandir}/man1/*.1*
-%{_mandir}/man5/*.5*
-%{_mandir}/man8/*.8*
-%endif
+#%if ! %{without docgen}
+#%{_mandir}/man1/*.1*
+#%{_mandir}/man5/*.5*
+#%{_mandir}/man8/*.8*
+#%endif
 %config(noreplace) %{_sysconfdir}/profile.d/nix.sh
 %config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
 /nix
@@ -166,8 +166,8 @@ systemctl start  nix-daemon.socket
 %{_prefix}/lib/pkgconfig/*.pc
 
 
-%if ! %{without docgen}
-%files doc
-%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-%{_defaultdocdir}/%{name}-doc-%{version}
-%endif
+#%if ! %{without docgen}
+#%files doc
+#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
+#%{_defaultdocdir}/%{name}-doc-%{version}
+#%endif

From d9e8ab0ff25979baa5b3c852f6a5aeb7b9d2ed36 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 16:35:05 +0200
Subject: [PATCH 1124/2196] Add a check for broken Boost versions

In some Boost versions, coroutines don't propagate exceptions
properly, causing Nix to fail with the exception 'coroutine has
finished'.
---
 src/libutil/serialise.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 21803edd056..b2c49d911b3 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -157,6 +157,10 @@ size_t StringSource::read(unsigned char * data, size_t len)
 }
 
 
+#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
+#error Coroutines are broken in this version of Boost!
+#endif
+
 std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
 {
     struct SinkToSource : Source

From 4b4adbc93dafcd020d9d79b2d9e376a80c6f5fef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 17:26:04 +0200
Subject: [PATCH 1125/2196] Fix race downloading into a sink

This fixes 'error 10 while decompressing xz file'.

https://hydra.nixos.org/build/78308551
---
 src/libstore/download.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 07acd5d0e00..ecec0f205b6 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -718,15 +718,17 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     while (true) {
         checkInterrupt();
 
-        if (state->quit) {
-            if (state->exc) std::rethrow_exception(state->exc);
-            break;
-        }
-
         /* If no data is available, then wait for the download thread
            to wake us up. */
-        if (state->data.empty())
+        if (state->data.empty()) {
+
+            if (state->quit) {
+                if (state->exc) std::rethrow_exception(state->exc);
+                break;
+            }
+
             state.wait(state->avail);
+        }
 
         /* If data is available, then flush it to the sink and wake up
            the download thread if it's blocked on a full buffer. */

From 94b2e4e1be4526f2f85efd29a52a63e1858ea414 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 17:34:53 +0200
Subject: [PATCH 1126/2196] Disable rpm_fedora27x86_64

Fedora 27 provides an incompatible version of Boost (1.64.0).
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 27d56e5f0c8..076f1de8dea 100644
--- a/release.nix
+++ b/release.nix
@@ -206,7 +206,7 @@ let
       };
 
 
-    rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
+    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
 
 
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];

From b27431b7cb918e6866f4bb4fa7ec6e23240cfeab Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Wed, 1 Aug 2018 11:03:32 -0400
Subject: [PATCH 1127/2196] Document s3 substitutions

---
 doc/manual/packages/s3-substituter.xml   | 142 +++++++++++++++++++++++
 doc/manual/packages/sharing-packages.xml |   1 +
 2 files changed, 143 insertions(+)
 create mode 100644 doc/manual/packages/s3-substituter.xml

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
new file mode 100644
index 00000000000..9fe137d7207
--- /dev/null
+++ b/doc/manual/packages/s3-substituter.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="utf-8"?>
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         version="5.0"
+         xml:id="ssec-s3-substituter">
+
+<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
+
+<para>Nix has built-in support for storing and fetching store paths
+from Amazon S3 and S3 compatible services. This uses the same
+<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
+fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
+
+<para>In this example we will use the bucket named
+<literal>example-bucket</literal>.</para>
+
+<section xml:id="ssec-s3-substituter-anonymous-reads">
+  <title>Anonymous Reads to your S3-compatible binary cache</title>
+
+  <para>If your binary cache is publicly accessible and does not
+  require authentication, the simplest and easiest way to use Nix with
+  your S3 compatible binary cache is to use the HTTP URL for that
+  cache.</para>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
+  compatible binary caches ago have to consult your software's
+  documentation.</para>
+
+  <para>Your bucket will need the following bucket policy:</para>
+
+  <programlisting>
+<![CDATA[
+{
+  "Id": "DirectReads",
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AlowDirectReads",
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::example-bucket/*",
+      "Principal": "*"
+    }
+  ]
+}
+]]>
+</programlisting>
+</section>
+
+<section xml:id="ssec-s3-substituter-authenticated-reads">
+  <title>Authenticated Reads to your S3 binary cache</title>
+
+  <para>For AWS S3 the binary cache URL for example bucket will be
+  exactly <uri>s3://example-bucket</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>Nix supports authenticated writes to S3 compatible binary
+  caches but only supports Authenticated reads from Amazon S3.
+  Additionally, the following limitations are in place for
+  authenticated reads:</para>
+
+  <itemizedlist>
+    <listitem><para>The bucket must actually be hosted by Amazon S3 and
+    <emphasis>not</emphasis> an S3 compatible
+    service.</para></listitem>
+
+    <listitem><para>The bucket must be within the
+    <literal>us-east-1</literal> region.</para></listitem>
+
+    <listitem><para>The Amazon credentials, if stored in a credential
+    profile, must be stored in the <literal>default</literal>
+    profile.</para></listitem>
+  </itemizedlist>
+
+  <para>Your bucket will need a bucket policy allowing the desired
+  users to perform the <literal>s3:GetObject</literal> action on all
+  objects in the bucket.</para>
+</section>
+
+
+<section xml:id="ssec-s3-substituter-authenticated-writes">
+  <title>Authenticated Writes to your S3-compatible binary cache</title>
+
+  <para>Nix support fully supports writing to Amazon S3 and S3
+  compatible buckets. The binary cache URL for our example bucket will
+  be <uri>s3://example-bucket</uri>.</para>
+
+  <para>Nix will use the <link
+  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  credential provider chain</link> for authenticating requests to
+  Amazon S3.</para>
+
+  <para>The following options can be specified as URL parameters to
+  the S3 URL:</para>
+  <variablelist>
+    <varlistentry><term><literal>profile</literal></term>
+    <listitem>
+      <para>
+        The name of the AWS configuration profile to use. By default
+        Nix will use the <literal>default</literal> profile.
+      </para>
+    </listitem>
+    </varlistentry>
+
+    <varlistentry><term><literal>region</literal></term>
+    <listitem>
+      <para>
+        The region of the S3 bucket. <literal>us–east-1</literal> by
+        default.
+      </para>
+    </listitem>
+    </varlistentry>
+  </variablelist>
+
+  <example><title>Uploading with non-default credential profile for Amazon S3</title>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
+  </example>
+
+  <para>The user writing to the bucket will need to perform the
+  following actions against the bucket:</para>
+
+  <itemizedlist>
+    <listitem><para><literal>s3:ListBucket</literal></para></listitem>
+    <listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
+    <listitem><para><literal>s3:ListObjects</literal></para></listitem>
+    <listitem><para><literal>s3:GetObject</literal></para></listitem>
+    <listitem><para><literal>s3:PutObject</literal></para></listitem>
+    <listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
+    <listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
+    <listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
+    <listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
+  </itemizedlist>
+</section>
+</section>
diff --git a/doc/manual/packages/sharing-packages.xml b/doc/manual/packages/sharing-packages.xml
index 8465c182ee7..bb6c52b8f8c 100644
--- a/doc/manual/packages/sharing-packages.xml
+++ b/doc/manual/packages/sharing-packages.xml
@@ -15,5 +15,6 @@ packages between machines.</para>
 <xi:include href="binary-cache-substituter.xml" />
 <xi:include href="copy-closure.xml" />
 <xi:include href="ssh-substituter.xml" />
+<xi:include href="s3-substituter.xml" />
 
 </chapter>

From fb6907e3d68f1373917b8c46f4cd1abb341f4bbd Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Wed, 1 Aug 2018 18:30:03 -0400
Subject: [PATCH 1128/2196] Removes unused variable from `nix-build/nix-shell`

This particular `shell` variable wasn't used, since a new one was
declared in the only side of the `if` branch that used a `shell`
variable.

It could realistically confuse developers thinking it could use `$SHELL`
under some situations.
---
 src/nix-build/nix-build.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index de0e9118fd2..35ad9873153 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -85,7 +85,6 @@ void mainWrapped(int argc, char * * argv)
     BuildMode buildMode = bmNormal;
     bool readStdin = false;
 
-    auto shell = getEnv("SHELL", "/bin/sh");
     std::string envCommand; // interactive shell
     Strings envExclude;
 

From 49a53c1d3f77ff33e14a95d004a6ce0720293997 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Tue, 31 Jul 2018 16:45:49 -0400
Subject: [PATCH 1129/2196] s3 binary cache: support specifying an endpoint

Works for uploading and not downloading.
---
 doc/manual/packages/s3-substituter.xml | 17 +++++++++++++++++
 src/libstore/download.cc               |  2 +-
 src/libstore/s3-binary-cache-store.cc  | 14 +++++++++-----
 src/libstore/s3.hh                     |  4 ++--
 src/nix/copy.cc                        |  4 ++++
 5 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index 9fe137d7207..bcd91cfdbcc 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -118,12 +118,29 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
       </para>
     </listitem>
     </varlistentry>
+
+    <varlistentry><term><literal>endpoint</literal></term>
+    <listitem>
+      <para>
+        The URL to your S3-compatible service, for when not using
+        Amazon S3. Do not specify this value if you're using Amazon
+        S3.
+      </para>
+      <note><para>This endpoint must support HTTPS and will use
+      path-based addressing instead of virtual host based
+      addressing.</para></note>
+    </listitem>
+    </varlistentry>
   </variablelist>
 
   <example><title>Uploading with non-default credential profile for Amazon S3</title>
     <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
   </example>
 
+  <example><title>Uploading to an S3-Compatible Binary Cache</title>
+    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&amp;endpoint=minio.example.com</command></para>
+  </example>
+
   <para>The user writing to the bucket will need to perform the
   following actions against the bucket:</para>
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index ecec0f205b6..f80c9e45b82 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -598,7 +598,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 26144ccb40c..2f18e3f38c6 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -84,8 +84,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const std::string & profile, const std::string & region)
-    : config(makeConfig(region))
+S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
+    : config(makeConfig(region, endpoint))
     , client(make_ref<Aws::S3::S3Client>(
             profile == ""
             ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@@ -99,7 +99,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
 #else
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
 #endif
-            false))
+            endpoint.empty()))
 {
 }
 
@@ -116,11 +116,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = region;
+    if (!endpoint.empty()) {
+        res->endpointOverride = endpoint;
+    }
     res->requestTimeoutMs = 600 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
@@ -170,6 +173,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
@@ -186,7 +190,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(profile, region)
+        , s3Helper(profile, region, endpoint)
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 4f996400343..95d612b6633 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & profile, const std::string & region);
+    S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
 
     struct DownloadResult
     {
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index e4e6c3e303e..91711c8b46d 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -72,6 +72,10 @@ struct CmdCopy : StorePathsCommand
                 "To populate the current folder build output to a S3 binary cache:",
                 "nix copy --to s3://my-bucket?region=eu-west-1"
             },
+            Example{
+                "To populate the current folder build output to an S3-compatible binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
+            },
 #endif
         };
     }

From 438e02529dfb4b26603e2a06992df977cd80ecd9 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Wed, 1 Aug 2018 21:28:49 -0400
Subject: [PATCH 1130/2196] Allows selectively adding environment variables to
 pure shells.

Includes documentation and test.
---
 doc/manual/command-ref/nix-shell.xml | 8 ++++++++
 src/nix-build/nix-build.cc           | 7 ++++++-
 tests/nix-shell.sh                   | 7 +++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 62d026ac238..5c44c4a8f44 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -32,6 +32,7 @@
     <arg><option>--run</option> <replaceable>cmd</replaceable></arg>
     <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
     <arg><option>--pure</option></arg>
+    <arg><option>--keep</option> <replaceable>name</replaceable></arg>
     <group choice='req'>
       <arg choice='plain'>
         <group choice='req'>
@@ -165,6 +166,13 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     </listitem></varlistentry>
 
+  <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term>
+
+    <listitem><para>When a <option>--pure</option> shell is started,
+    keep the listed environment variables.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 <para>The following common options are supported:</para>
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index de0e9118fd2..a7ec206aa87 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -99,6 +99,9 @@ void mainWrapped(int argc, char * * argv)
 
     std::string outLink = "./result";
 
+    // List of environment variables kept for --pure
+    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+
     Strings args;
     for (int i = 1; i < argc; ++i)
         args.push_back(argv[i]);
@@ -218,6 +221,9 @@ void mainWrapped(int argc, char * * argv)
             }
         }
 
+        else if (*arg == "--keep")
+            keepVars.insert(getArg(*arg, arg, end));
+
         else if (*arg == "-")
             readStdin = true;
 
@@ -368,7 +374,6 @@ void mainWrapped(int argc, char * * argv)
         auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
 
         if (pure) {
-            std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
             decltype(env) newEnv;
             for (auto & i : env)
                 if (keepVars.count(i.first))
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index d25c456cedf..6024ea39975 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -4,12 +4,19 @@ clearStore
 
 # Test nix-shell -A
 export IMPURE_VAR=foo
+export SELECTED_IMPURE_VAR=baz
 export NIX_BUILD_SHELL=$SHELL
 output=$(nix-shell --pure shell.nix -A shellDrv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"')
 
 [ "$output" = " - foo - bar" ]
 
+# Test --keep
+output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"')
+
+[ "$output" = " - foo - bar - baz" ]
+
 # Test nix-shell on a .drv
 [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]

From 43e28a1b756c2f7ee139c999e6169a71f555e9e5 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Fri, 3 Aug 2018 06:24:57 +0200
Subject: [PATCH 1131/2196] Fix symlink leak in restricted eval mode

In EvalState::checkSourcePath, the path is checked against the list of
allowed paths first and later it's checked again *after* resolving
symlinks.

The resolving of the symlinks is done via canonPath, which also strips
out "../" and "./". However after the canonicalisation the error message
pointing out that the path is not allowed prints the symlink target in
the error message.

Even if we'd suppress the message, symlink targets could still be leaked
if the symlink target doesn't exist (in this case the error is thrown in
canonPath).

So instead, we now do canonPath() without symlink resolving first before
even checking against the list of allowed paths and then later do the
symlink resolving and checking the allowed paths again.

The first call to canonPath() should get rid of all the "../" and "./",
so in theory the only way to leak a symlink if the attacker is able to
put a symlink in one of the paths allowed by restricted evaluation mode.

For the latter I don't think this is part of the threat model, because
if the attacker can write to that path, the attack vector is even
larger.

Signed-off-by: aszlig <aszlig@nix.build>
---
 src/libexpr/eval.cc | 14 ++++++++++----
 tests/restricted.sh | 11 +++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e09297546c9..3abde6c9296 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -349,19 +349,25 @@ Path EvalState::checkSourcePath(const Path & path_)
 
     bool found = false;
 
+    /* First canonicalize the path without symlinks, so we make sure an
+     * attacker can't append ../../... to a path that would be in allowedPaths
+     * and thus leak symlink targets.
+     */
+    Path abspath = canonPath(path_);
+
     for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path_, i)) {
+        if (isDirOrInDir(abspath, i)) {
             found = true;
             break;
         }
     }
 
     if (!found)
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_);
+        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
 
     /* Resolve symlinks. */
-    debug(format("checking access to '%s'") % path_);
-    Path path = canonPath(path_, true);
+    debug(format("checking access to '%s'") % abspath);
+    Path path = canonPath(abspath, true);
 
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(path, i)) {
diff --git a/tests/restricted.sh b/tests/restricted.sh
index a87d8ec2c94..e02becc60e3 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -38,3 +38,14 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix
 nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I .
 
 [[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]]
+
+# Check whether we can leak symlink information through directory traversal.
+traverseDir="$(pwd)/restricted-traverse-me"
+ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent"
+mkdir -p "$traverseDir"
+goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')"
+output="$(nix eval --raw --restrict-eval -I "$traverseDir" \
+    "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \
+    2>&1 || :)"
+echo "$output" | grep "is forbidden"
+! echo "$output" | grep -F restricted-secret

From 87356cc8a4a5fd827e92464fdb960efa93f3dfda Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Jul 2018 21:51:03 +0200
Subject: [PATCH 1132/2196] Add Nix 2.1 release notes

---
 doc/manual/release-notes/release-notes.xml |  1 +
 doc/manual/release-notes/rl-2.1.xml        | 47 ++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 doc/manual/release-notes/rl-2.1.xml

diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index b8392a647af..ff4085cb792 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />
 <xi:include href="rl-1.11.xml" />
diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
new file mode 100644
index 00000000000..5f37083eefc
--- /dev/null
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -0,0 +1,47 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.1">
+
+<title>Release 2.1 (2018-08-??)</title>
+
+<para>This is primarily a bug fix release. It also reduces memory
+consumption in certain situations. In addition, it has the following
+new features:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>New builtin functions:
+    <literal>builtins.bitAnd</literal>,
+    <literal>builtins.bitOr</literal>,
+    <literal>builtins.bitXor</literal>,
+    <literal>builtins.fromTOML</literal>,
+    <literal>builtins.concatMap</literal>,
+    <literal>builtins.mapAttrs</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The S3 binary cache store now supports uploading NARs larger
+    than 5 GiB.</para>
+  </listitem>
+
+  <listitem>
+    <para>The flag <option>--fallback</option> is no longer required
+    to recover from disappeared NARs in binary caches.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-daemon</command> now respects
+    <option>--store</option>, so it can be run as a non-root
+    user.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>This release has contributions from TODO.
+</para>
+
+</section>

From 06080e4abce5f8ffc4a658802692f7464ec359b5 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 3 Aug 2018 11:09:31 -0400
Subject: [PATCH 1133/2196] 2.1 release notes: Add note about s3-compatible
 stores

---
 doc/manual/release-notes/rl-2.1.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index 5f37083eefc..c628d04a707 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -28,6 +28,12 @@ new features:</para>
     than 5 GiB.</para>
   </listitem>
 
+  <listitem>
+    <para>The S3 binary cache store now supports uploading to
+    S3-compatible services with the <literal>endpoint</literal>
+    option.</para>
+  </listitem>
+
   <listitem>
     <para>The flag <option>--fallback</option> is no longer required
     to recover from disappeared NARs in binary caches.</para>

From e268bbc05435d8121275136934a594fc70a73da9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 18:07:46 +0200
Subject: [PATCH 1134/2196] LegacySSHStore: Add remote-store option

This is primarily useful for testing, e.g.

  $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' ...
---
 src/libstore/legacy-ssh-store.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 02d91ded04c..f9b0b42c83d 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -17,6 +17,7 @@ struct LegacySSHStore : public Store
     const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
     const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
+    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
 
     // Hack for getting remote build log output.
     const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
@@ -56,7 +57,9 @@ struct LegacySSHStore : public Store
     ref<Connection> openConnection()
     {
         auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand(fmt("%s --serve --write", remoteProgram));
+        conn->sshConn = master.startCommand(
+            fmt("%s --serve --write", remoteProgram)
+            + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
 

From 4e7d5f660cec6da3a432c726dfe7a579c7581c79 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 18:12:28 +0200
Subject: [PATCH 1135/2196] SSHMaster: Bypass SSH when connecting to localhost

This is primarily useful for testing since it removes the need to have
SSH working.
---
 src/libstore/ssh.cc | 22 +++++++++++++++-------
 src/libstore/ssh.hh |  1 +
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 033c580936a..5e0e44935cc 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -4,8 +4,9 @@ namespace nix {
 
 SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
     : host(host)
+    , fakeSSH(host == "localhost")
     , keyFile(keyFile)
-    , useMaster(useMaster)
+    , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
 {
@@ -45,12 +46,19 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1)
             throw SysError("duping over stderr");
 
-        Strings args = { "ssh", host.c_str(), "-x", "-a" };
-        addCommonSSHOpts(args);
-        if (socketPath != "")
-            args.insert(args.end(), {"-S", socketPath});
-        if (verbosity >= lvlChatty)
-            args.push_back("-v");
+        Strings args;
+
+        if (fakeSSH) {
+            args = { "bash", "-c" };
+        } else {
+            args = { "ssh", host.c_str(), "-x", "-a" };
+            addCommonSSHOpts(args);
+            if (socketPath != "")
+                args.insert(args.end(), {"-S", socketPath});
+            if (verbosity >= lvlChatty)
+                args.push_back("-v");
+        }
+
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 1268e6d0005..4f0f0bd29f9 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -10,6 +10,7 @@ class SSHMaster
 private:
 
     const std::string host;
+    bool fakeSSH;
     const std::string keyFile;
     const bool useMaster;
     const bool compress;

From 848a9375c3753202bf53defda469ca5d7538e135 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 20:36:25 +0200
Subject: [PATCH 1136/2196] Support escaping in store URIs

---
 src/libstore/store-api.cc | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1854353828d..7a4a5f5eb85 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -844,8 +844,24 @@ ref<Store> openStore(const std::string & uri_,
     if (q != std::string::npos) {
         for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
             auto e = s.find('=');
-            if (e != std::string::npos)
-                params[s.substr(0, e)] = s.substr(e + 1);
+            if (e != std::string::npos) {
+                auto value = s.substr(e + 1);
+                std::string decoded;
+                for (size_t i = 0; i < value.size(); ) {
+                    if (value[i] == '%') {
+                        if (i + 2 >= value.size())
+                            throw Error("invalid URI parameter '%s'", value);
+                        try {
+                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
+                            i += 3;
+                        } catch (...) {
+                            throw Error("invalid URI parameter '%s'", value);
+                        }
+                    } else
+                        decoded += value[i++];
+                }
+                params[s.substr(0, e)] = decoded;
+            }
         }
         uri = uri_.substr(0, q);
     }

From eeebe4cdc509aaaefb70d4af55e5174a75c0281f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 20:39:07 +0200
Subject: [PATCH 1137/2196] cmdDumpStorePath: Support chroot stores

---
 src/nix-store/nix-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e1e27ceef94..942fe50e86c 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -860,7 +860,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             }
 
             case cmdDumpStorePath:
-                dumpPath(readStorePath(*store, in), out);
+                store->narFromPath(readStorePath(*store, in), out);
                 break;
 
             case cmdImportPaths: {

From 34c17fdae535d0128d178dde676f56f1bc8439e3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 20:41:56 +0200
Subject: [PATCH 1138/2196] Add a test for LegacySSHStore that doesn't require
 a VM

---
 tests/local.mk        |  3 ++-
 tests/nix-copy-ssh.sh | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 tests/nix-copy-ssh.sh

diff --git a/tests/local.mk b/tests/local.mk
index 9df0adf1bfd..1ff68348b3c 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -25,7 +25,8 @@ nix_tests = \
   pure-eval.sh \
   check.sh \
   plugins.sh \
-  search.sh
+  search.sh \
+  nix-copy-ssh.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh
new file mode 100644
index 00000000000..6aba667a45a
--- /dev/null
+++ b/tests/nix-copy-ssh.sh
@@ -0,0 +1,20 @@
+source common.sh
+
+clearStore
+clearCache
+
+remoteRoot=$TEST_ROOT/store2
+chmod -R u+w "$remoteRoot" || true
+rm -rf "$remoteRoot"
+
+outPath=$(nix-build dependencies.nix)
+
+nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
+
+[ -f $remoteRoot$outPath/foobar ]
+
+clearStore
+
+nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
+
+[ -f $outPath/foobar ]

From 2825e05d21ecabc8b8524836baf0b9b05da993c6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Aug 2018 21:10:03 +0200
Subject: [PATCH 1139/2196] Make adding paths via nix-store --serve run in
 constant memory

It adds a new operation, cmdAddToStoreNar, that does the same thing as
the corresponding nix-daemon operation, i.e. call addToStore(). This
replaces cmdImportPaths, which has the major issue that it sends the
NAR first and the store path second, thus requiring us to store the
incoming NAR either in memory or on disk until we decide what to do
with it.

For example, this reduces the memory usage of

  $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 267 MiB to 12 MiB.

Probably fixes #1988.
---
 src/libstore/legacy-ssh-store.cc | 59 ++++++++++++++++++++++++--------
 src/libstore/serve-protocol.hh   |  3 +-
 src/nix-daemon/nix-daemon.cc     |  1 +
 src/nix-store/nix-store.cc       | 22 ++++++++++++
 4 files changed, 70 insertions(+), 15 deletions(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index f9b0b42c83d..ddfd17d641b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -28,6 +28,7 @@ struct LegacySSHStore : public Store
         FdSink to;
         FdSource from;
         int remoteVersion;
+        bool good = true;
     };
 
     std::string host;
@@ -42,7 +43,7 @@ struct LegacySSHStore : public Store
         , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() { return openConnection(); },
-            [](const ref<Connection> & r) { return true; }
+            [](const ref<Connection> & r) { return r->good; }
             ))
         , master(
             host,
@@ -58,7 +59,7 @@ struct LegacySSHStore : public Store
     {
         auto conn = make_ref<Connection>();
         conn->sshConn = master.startCommand(
-            fmt("%s --serve --write", remoteProgram)
+            fmt("command time %s --serve --write", remoteProgram)
             + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());
@@ -130,18 +131,48 @@ struct LegacySSHStore : public Store
 
         auto conn(connections->get());
 
-        conn->to
-            << cmdImportPaths
-            << 1;
-        copyNAR(source, conn->to);
-        conn->to
-            << exportMagic
-            << info.path
-            << info.references
-            << info.deriver
-            << 0
-            << 0;
-        conn->to.flush();
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
+
+            conn->to
+                << cmdAddToStoreNar
+                << info.path
+                << info.deriver
+                << info.narHash.to_string(Base16, false)
+                << info.references
+                << info.registrationTime
+                << info.narSize
+                << info.ultimate
+                << info.sigs
+                << info.ca;
+            try {
+                copyNAR(source, conn->to);
+            } catch (...) {
+                conn->good = false;
+                throw;
+            }
+            conn->to.flush();
+
+        } else {
+
+            conn->to
+                << cmdImportPaths
+                << 1;
+            try {
+                copyNAR(source, conn->to);
+            } catch (...) {
+                conn->good = false;
+                throw;
+            }
+            conn->to
+                << exportMagic
+                << info.path
+                << info.references
+                << info.deriver
+                << 0
+                << 0;
+            conn->to.flush();
+
+        }
 
         if (readInt(conn->from) != 1)
             throw Error("failed to add path '%s' to remote host '%s', info.path, host");
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index f67d1e2580a..9fae6d5349f 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
 #define SERVE_MAGIC_1 0x390c9deb
 #define SERVE_MAGIC_2 0x5452eecb
 
-#define SERVE_PROTOCOL_VERSION 0x204
+#define SERVE_PROTOCOL_VERSION 0x205
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -18,6 +18,7 @@ typedef enum {
     cmdBuildPaths = 6,
     cmdQueryClosure = 7,
     cmdBuildDerivation = 8,
+    cmdAddToStoreNar = 9,
 } ServeCommand;
 
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index a2e54b93c05..644fa6681de 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -707,6 +707,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         logger->startWork();
 
+        // FIXME: race if addToStore doesn't read source?
         store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 942fe50e86c..fe68f681ae2 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -924,6 +924,28 @@ static void opServe(Strings opFlags, Strings opArgs)
                 break;
             }
 
+            case cmdAddToStoreNar: {
+                if (!writeAllowed) throw Error("importing paths is not allowed");
+
+                ValidPathInfo info;
+                info.path = readStorePath(*store, in);
+                in >> info.deriver;
+                if (!info.deriver.empty())
+                    store->assertStorePath(info.deriver);
+                info.narHash = Hash(readString(in), htSHA256);
+                info.references = readStorePaths<PathSet>(*store, in);
+                in >> info.registrationTime >> info.narSize >> info.ultimate;
+                info.sigs = readStrings<StringSet>(in);
+                in >> info.ca;
+
+                // FIXME: race if addToStore doesn't read source?
+                store->addToStore(info, in, NoRepair, NoCheckSigs);
+
+                out << 1; // indicate success
+
+                break;
+            }
+
             default:
                 throw Error(format("unknown serve command %1%") % cmd);
         }

From 4361a4331f1b2eb3fcfd954c96de353c9d516508 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Aug 2018 11:31:14 +0200
Subject: [PATCH 1140/2196] Fix reporting of HTTP body size when a result
 callback is used

---
 src/libstore/download.cc | 5 ++++-
 src/libstore/download.hh | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index f80c9e45b82..f0ea1995ae7 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -140,6 +140,7 @@ struct CurlDownloader : public Downloader
         size_t writeCallback(void * contents, size_t size, size_t nmemb)
         {
             size_t realSize = size * nmemb;
+            result.bodySize += realSize;
             if (request.dataCallback)
                 request.dataCallback((char *) contents, realSize);
             else
@@ -162,6 +163,7 @@ struct CurlDownloader : public Downloader
                 auto ss = tokenizeString<vector<string>>(line, " ");
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
+                result.bodySize = 0;
                 encoding = "";
             } else {
                 auto i = line.find(':');
@@ -296,6 +298,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
             result.data = std::make_shared<std::string>();
+            result.bodySize = 0;
         }
 
         void finish(CURLcode code)
@@ -309,7 +312,7 @@ struct CurlDownloader : public Downloader
                 result.effectiveUrl = effectiveUrlCStr;
 
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
-                request.verb(), request.uri, code, httpStatus, result.data ? result.data->size() : 0);
+                request.verb(), request.uri, code, httpStatus, result.bodySize);
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index da55df7a6e7..ff38a2870cc 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -38,6 +38,7 @@ struct DownloadResult
     std::string etag;
     std::string effectiveUrl;
     std::shared_ptr<std::string> data;
+    uint64_t bodySize = 0;
 };
 
 class Store;

From fa4def3d4675c8b2d6aacb56959dbbf9e52df66a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Aug 2018 11:48:35 +0200
Subject: [PATCH 1141/2196] Require libbrotli

---
 Makefile.config.in         |  1 -
 configure.ac               |  9 +++----
 src/libutil/compression.cc | 50 --------------------------------------
 tests/brotli.sh            |  7 ------
 tests/common.sh.in         |  1 -
 5 files changed, 3 insertions(+), 65 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index a9785dc7395..08edcb863ef 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -6,7 +6,6 @@ CXXFLAGS = @CXXFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
-HAVE_BROTLI = @HAVE_BROTLI@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
diff --git a/configure.ac b/configure.ac
index c41a83c9764..6aeeacdba13 100644
--- a/configure.ac
+++ b/configure.ac
@@ -179,12 +179,9 @@ AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
 
-# Look for libbrotli{enc,dec}, optional dependencies
-PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec],
-  [AC_DEFINE([HAVE_BROTLI], [1], [Whether to use libbrotli.])
-   CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]
-   have_brotli=1], [have_brotli=])
-AC_SUBST(HAVE_BROTLI, [$have_brotli])
+# Look for libbrotli{enc,dec}.
+PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
+
 
 # Look for libseccomp, required for Linux sandboxing.
 if test "$sys_name" = linux; then
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index e1782f8c4bd..5de85ede150 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -8,10 +8,8 @@
 #include <cstdio>
 #include <cstring>
 
-#if HAVE_BROTLI
 #include <brotli/decode.h>
 #include <brotli/encode.h>
-#endif // HAVE_BROTLI
 
 #include <iostream>
 
@@ -132,12 +130,6 @@ static void decompressBzip2(Source & source, Sink & sink)
 
 static void decompressBrotli(Source & source, Sink & sink)
 {
-#if !HAVE_BROTLI
-    RunOptions options(BROTLI, {"-d"});
-    options.standardIn = &source;
-    options.standardOut = &sink;
-    runProgram2(options);
-#else
     auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
     if (!s)
         throw CompressionError("unable to initialize brotli decoder");
@@ -193,7 +185,6 @@ static void decompressBrotli(Source & source, Sink & sink)
 
         if (ret == BROTLI_DECODER_RESULT_SUCCESS) return;
     }
-#endif // HAVE_BROTLI
 }
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
@@ -403,42 +394,6 @@ struct BzipSink : CompressionSink
     }
 };
 
-struct LambdaCompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    std::string data;
-    using CompressFnTy = std::function<std::string(const std::string&)>;
-    CompressFnTy compressFn;
-    LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn)
-        : nextSink(nextSink)
-        , compressFn(std::move(compressFn))
-    {
-    };
-
-    void finish() override
-    {
-        flush();
-        nextSink(compressFn(data));
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        checkInterrupt();
-        this->data.append((const char *) data, len);
-    }
-};
-
-struct BrotliCmdSink : LambdaCompressionSink
-{
-    BrotliCmdSink(Sink& nextSink)
-        : LambdaCompressionSink(nextSink, [](const std::string& data) {
-            return runProgram(BROTLI, true, {}, data);
-        })
-    {
-    }
-};
-
-#if HAVE_BROTLI
 struct BrotliSink : CompressionSink
 {
     Sink & nextSink;
@@ -525,7 +480,6 @@ struct BrotliSink : CompressionSink
         }
     }
 };
-#endif // HAVE_BROTLI
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
@@ -544,11 +498,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "bzip2")
         return make_ref<BzipSink>(nextSink);
     else if (method == "br")
-#if HAVE_BROTLI
         return make_ref<BrotliSink>(nextSink);
-#else
-        return make_ref<BrotliCmdSink>(nextSink);
-#endif
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
diff --git a/tests/brotli.sh b/tests/brotli.sh
index 645dd4214ec..a3c6e55a8fa 100644
--- a/tests/brotli.sh
+++ b/tests/brotli.sh
@@ -1,10 +1,5 @@
 source common.sh
 
-
-# Only test if we found brotli libraries
-# (CLI tool is likely unavailable if libraries are missing)
-if [ -n "$HAVE_BROTLI" ]; then
-
 clearStore
 clearCache
 
@@ -24,5 +19,3 @@ nix copy --from $cacheURI $outPath --no-check-sigs
 HASH2=$(nix hash-path $outPath)
 
 [[ $HASH = $HASH2 ]]
-
-fi # HAVE_BROTLI
diff --git a/tests/common.sh.in b/tests/common.sh.in
index fddd25b366b..2ee2f589dae 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -31,7 +31,6 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
-export HAVE_BROTLI="@HAVE_BROTLI@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@

From d3761f5f8bce1e4c8dcfdff3fa77c173157c0346 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Aug 2018 15:40:29 +0200
Subject: [PATCH 1142/2196] Fix Brotli decompression in 'nix log'

This didn't work anymore since decompression was only done in the
non-coroutine case.

Decompressors are now sinks, just like compressors.

Also fixed a bug in bzip2 API handling (we have to handle BZ_RUN_OK
rather than BZ_OK), which we didn't notice because there was a missing
'throw':

  if (ret != BZ_OK)
      CompressionError("error while compressing bzip2 file");
---
 src/libstore/binary-cache-store.cc    |  23 +-
 src/libstore/builtins/fetchurl.cc     |  21 +-
 src/libstore/download.cc              |  68 ++--
 src/libstore/download.hh              |   3 -
 src/libstore/s3-binary-cache-store.cc |   6 +-
 src/libutil/compression.cc            | 557 +++++++++++---------------
 src/libutil/compression.hh            |  12 +-
 7 files changed, 309 insertions(+), 381 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 76c0a1a891b..9c75c85993f 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -217,17 +217,6 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    auto source = sinkToSource([this, url{info->url}](Sink & sink) {
-        try {
-            getFile(url, sink);
-        } catch (NoSuchBinaryCacheFile & e) {
-            throw SubstituteGone(e.what());
-        }
-    });
-
-    stats.narRead++;
-    //stats.narReadCompressedBytes += nar->size(); // FIXME
-
     uint64_t narSize = 0;
 
     LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
@@ -235,8 +224,18 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
         narSize += len;
     });
 
-    decompress(info->compression, *source, wrapperSink);
+    auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
 
+    try {
+        getFile(info->url, *decompressor);
+    } catch (NoSuchBinaryCacheFile & e) {
+        throw SubstituteGone(e.what());
+    }
+
+    decompressor->flush();
+
+    stats.narRead++;
+    //stats.narReadCompressedBytes += nar->size(); // FIXME
     stats.narReadBytes += narSize;
 }
 
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 1f4abd374f5..b4dcb35f951 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -39,21 +39,16 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
             request.verifyTLS = false;
             request.decompress = false;
 
-            downloader->download(std::move(request), sink);
+            auto decompressor = makeDecompressionSink(
+                hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
+            downloader->download(std::move(request), *decompressor);
+            decompressor->finish();
         });
 
-        if (get(drv.env, "unpack", "") == "1") {
-
-            if (hasSuffix(mainUrl, ".xz")) {
-                auto source2 = sinkToSource([&](Sink & sink) {
-                    decompress("xz", *source, sink);
-                });
-                restorePath(storePath, *source2);
-            } else
-                restorePath(storePath, *source);
-
-        } else
-              writeFile(storePath, *source);
+        if (get(drv.env, "unpack", "") == "1")
+            restorePath(storePath, *source);
+        else
+            writeFile(storePath, *source);
 
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index f0ea1995ae7..973fca0b130 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -58,16 +58,6 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data)
-{
-    if (encoding == "")
-        return data;
-    else if (encoding == "br")
-        return decompress(encoding, *data);
-    else
-        throw Error("unsupported Content-Encoding '%s'", encoding);
-}
-
 struct CurlDownloader : public Downloader
 {
     CURLM * curlm = 0;
@@ -106,6 +96,12 @@ struct CurlDownloader : public Downloader
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(callback)
+            , finalSink([this](const unsigned char * data, size_t len) {
+                if (this->request.dataCallback)
+                    this->request.dataCallback((char *) data, len);
+                else
+                    this->result.data->append((char *) data, len);
+              })
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@@ -129,23 +125,40 @@ struct CurlDownloader : public Downloader
             }
         }
 
-        template<class T>
-        void fail(const T & e)
+        void failEx(std::exception_ptr ex)
         {
             assert(!done);
             done = true;
-            callback.rethrow(std::make_exception_ptr(e));
+            callback.rethrow(ex);
         }
 
+        template<class T>
+        void fail(const T & e)
+        {
+            failEx(std::make_exception_ptr(e));
+        }
+
+        LambdaSink finalSink;
+        std::shared_ptr<CompressionSink> decompressionSink;
+
+        std::exception_ptr writeException;
+
         size_t writeCallback(void * contents, size_t size, size_t nmemb)
         {
-            size_t realSize = size * nmemb;
-            result.bodySize += realSize;
-            if (request.dataCallback)
-                request.dataCallback((char *) contents, realSize);
-            else
-                result.data->append((char *) contents, realSize);
-            return realSize;
+            try {
+                size_t realSize = size * nmemb;
+                result.bodySize += realSize;
+
+                if (!decompressionSink)
+                    decompressionSink = makeDecompressionSink(encoding, finalSink);
+
+                (*decompressionSink)((unsigned char *) contents, realSize);
+
+                return realSize;
+            } catch (...) {
+                writeException = std::current_exception();
+                return 0;
+            }
         }
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -314,27 +327,33 @@ struct CurlDownloader : public Downloader
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
                 request.verb(), request.uri, code, httpStatus, result.bodySize);
 
+            if (decompressionSink)
+                decompressionSink->finish();
+
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;
                 httpStatus = 304;
             }
 
-            if (code == CURLE_OK &&
+            if (writeException)
+                failEx(writeException);
+
+            else if (code == CURLE_OK &&
                 (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
 
                 try {
-                    if (request.decompress)
-                        result.data = decodeContent(encoding, ref<std::string>(result.data));
                     act.progress(result.data->size(), result.data->size());
                     callback(std::move(result));
                 } catch (...) {
                     done = true;
                     callback.rethrow();
                 }
-            } else {
+            }
+
+            else {
                 // We treat most errors as transient, but won't retry when hopeless
                 Error err = Transient;
 
@@ -369,6 +388,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_UNKNOWN_OPTION:
                         case CURLE_SSL_CACERT_BADFILE:
                         case CURLE_TOO_MANY_REDIRECTS:
+                        case CURLE_WRITE_ERROR:
                             err = Misc;
                             break;
                         default: // Shut up warnings
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index ff38a2870cc..f0228f7d053 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -88,7 +88,4 @@ public:
 
 bool isUri(const string & s);
 
-/* Decode data according to the Content-Encoding header. */
-ref<std::string> decodeContent(const std::string & encoding, ref<std::string> data);
-
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 2f18e3f38c6..660583d31fe 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -153,10 +153,8 @@ S3Helper::DownloadResult S3Helper::getObject(
         auto result = checkAws(fmt("AWS error fetching '%s'", key),
             client->GetObject(request));
 
-        res.data = decodeContent(
-            result.GetContentEncoding(),
-            make_ref<std::string>(
-                dynamic_cast<std::stringstream &>(result.GetBody()).str()));
+        res.data = decompress(result.GetContentEncoding(),
+            dynamic_cast<std::stringstream &>(result.GetBody()).str());
 
     } catch (S3Error & e) {
         if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5de85ede150..53b62f62a76 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -17,228 +17,258 @@ namespace nix {
 
 static const size_t bufSize = 32 * 1024;
 
-static void decompressNone(Source & source, Sink & sink)
+// Don't feed brotli too much at once.
+struct ChunkedCompressionSink : CompressionSink
 {
-    std::vector<unsigned char> buf(bufSize);
-    while (true) {
-        size_t n;
-        try {
-            n = source.read(buf.data(), buf.size());
-        } catch (EndOfFile &) {
-            break;
+    uint8_t outbuf[BUFSIZ];
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
+        while (len) {
+            size_t n = std::min(CHUNK_SIZE, len);
+            writeInternal(data, n);
+            data += n;
+            len -= n;
         }
-        sink(buf.data(), n);
     }
-}
 
-static void decompressXZ(Source & source, Sink & sink)
+    virtual void writeInternal(const unsigned char * data, size_t len) = 0;
+};
+
+struct NoneSink : CompressionSink
 {
-    lzma_stream strm(LZMA_STREAM_INIT);
-
-    lzma_ret ret = lzma_stream_decoder(
-        &strm, UINT64_MAX, LZMA_CONCATENATED);
-    if (ret != LZMA_OK)
-        throw CompressionError("unable to initialise lzma decoder");
-
-    Finally free([&]() { lzma_end(&strm); });
-
-    lzma_action action = LZMA_RUN;
-    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
-    strm.next_in = nullptr;
-    strm.avail_in = 0;
-    strm.next_out = outbuf.data();
-    strm.avail_out = outbuf.size();
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (strm.avail_in == 0 && !eof) {
-            strm.next_in = inbuf.data();
-            try {
-                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
-            }
-        }
+    Sink & nextSink;
+    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
+    void finish() override { flush(); }
+    void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
+};
 
-        if (strm.avail_in == 0)
-            action = LZMA_FINISH;
+struct XzDecompressionSink : CompressionSink
+{
+    Sink & nextSink;
+    uint8_t outbuf[BUFSIZ];
+    lzma_stream strm = LZMA_STREAM_INIT;
+    bool finished = false;
 
-        lzma_ret ret = lzma_code(&strm, action);
+    XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        lzma_ret ret = lzma_stream_decoder(
+            &strm, UINT64_MAX, LZMA_CONCATENATED);
+        if (ret != LZMA_OK)
+            throw CompressionError("unable to initialise lzma decoder");
 
-        if (strm.avail_out < outbuf.size()) {
-            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
-            strm.next_out = outbuf.data();
-            strm.avail_out = outbuf.size();
-        }
+        strm.next_out = outbuf;
+        strm.avail_out = sizeof(outbuf);
+    }
 
-        if (ret == LZMA_STREAM_END) return;
+    ~XzDecompressionSink()
+    {
+        lzma_end(&strm);
+    }
 
-        if (ret != LZMA_OK)
-            throw CompressionError("error %d while decompressing xz file", ret);
+    void finish() override
+    {
+        CompressionSink::flush();
+        write(nullptr, 0);
     }
-}
 
-static void decompressBzip2(Source & source, Sink & sink)
-{
-    bz_stream strm;
-    memset(&strm, 0, sizeof(strm));
-
-    int ret = BZ2_bzDecompressInit(&strm, 0, 0);
-    if (ret != BZ_OK)
-        throw CompressionError("unable to initialise bzip2 decoder");
-
-    Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
-
-    std::vector<char> inbuf(bufSize), outbuf(bufSize);
-    strm.next_in = nullptr;
-    strm.avail_in = 0;
-    strm.next_out = outbuf.data();
-    strm.avail_out = outbuf.size();
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (strm.avail_in == 0 && !eof) {
-            strm.next_in = inbuf.data();
-            try {
-                strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
-            }
-        }
+    void write(const unsigned char * data, size_t len) override
+    {
+        strm.next_in = data;
+        strm.avail_in = len;
 
-        int ret = BZ2_bzDecompress(&strm);
+        while (!finished && (!data || strm.avail_in)) {
+            checkInterrupt();
 
-        if (strm.avail_in == 0 && strm.avail_out == outbuf.size() && eof)
-            throw CompressionError("bzip2 data ends prematurely");
+            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+                throw CompressionError("error %d while decompressing xz file", ret);
+
+            finished = ret == LZMA_STREAM_END;
 
-        if (strm.avail_out < outbuf.size()) {
-            sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out);
-            strm.next_out = outbuf.data();
-            strm.avail_out = outbuf.size();
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = outbuf;
+                strm.avail_out = sizeof(outbuf);
+            }
         }
+    }
+};
 
-        if (ret == BZ_STREAM_END) return;
+struct BzipDecompressionSink : ChunkedCompressionSink
+{
+    Sink & nextSink;
+    bz_stream strm;
+    bool finished = false;
 
+    BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        memset(&strm, 0, sizeof(strm));
+        int ret = BZ2_bzDecompressInit(&strm, 0, 0);
         if (ret != BZ_OK)
-            throw CompressionError("error while decompressing bzip2 file");
+            throw CompressionError("unable to initialise bzip2 decoder");
+
+        strm.next_out = (char *) outbuf;
+        strm.avail_out = sizeof(outbuf);
     }
-}
 
-static void decompressBrotli(Source & source, Sink & sink)
-{
-    auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
-    if (!s)
-        throw CompressionError("unable to initialize brotli decoder");
-
-    Finally free([s]() { BrotliDecoderDestroyInstance(s); });
-
-    std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize);
-    const uint8_t * next_in = nullptr;
-    size_t avail_in = 0;
-    bool eof = false;
-
-    while (true) {
-        checkInterrupt();
-
-        if (avail_in == 0 && !eof) {
-            next_in = inbuf.data();
-            try {
-                avail_in = source.read((unsigned char *) next_in, inbuf.size());
-            } catch (EndOfFile &) {
-                eof = true;
+    ~BzipDecompressionSink()
+    {
+        BZ2_bzDecompressEnd(&strm);
+    }
+
+    void finish() override
+    {
+        flush();
+        write(nullptr, 0);
+    }
+
+    void writeInternal(const unsigned char * data, size_t len)
+    {
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+
+        strm.next_in = (char *) data;
+        strm.avail_in = len;
+
+        while (strm.avail_in) {
+            checkInterrupt();
+
+            int ret = BZ2_bzDecompress(&strm);
+            if (ret != BZ_OK && ret != BZ_STREAM_END)
+                throw CompressionError("error while decompressing bzip2 file");
+
+            finished = ret == BZ_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (char *) outbuf;
+                strm.avail_out = sizeof(outbuf);
             }
         }
+    }
+};
 
-        uint8_t * next_out = outbuf.data();
-        size_t avail_out = outbuf.size();
-
-        auto ret = BrotliDecoderDecompressStream(s,
-                &avail_in, &next_in,
-                &avail_out, &next_out,
-                nullptr);
-
-        switch (ret) {
-        case BROTLI_DECODER_RESULT_ERROR:
-            throw CompressionError("error while decompressing brotli file");
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
-            if (eof)
-                throw CompressionError("incomplete or corrupt brotli file");
-            break;
-        case BROTLI_DECODER_RESULT_SUCCESS:
-            if (avail_in != 0)
-                throw CompressionError("unexpected input after brotli decompression");
-            break;
-        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
-            // I'm not sure if this can happen, but abort if this happens with empty buffer
-            if (avail_out == outbuf.size())
-                throw CompressionError("brotli decompression requires larger buffer");
-            break;
-        }
+struct BrotliDecompressionSink : ChunkedCompressionSink
+{
+    Sink & nextSink;
+    BrotliDecoderState * state;
+    bool finished = false;
 
-        // Always ensure we have full buffer for next invocation
-        if (avail_out < outbuf.size())
-            sink((unsigned char *) outbuf.data(), outbuf.size() - avail_out);
+    BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+        if (!state)
+            throw CompressionError("unable to initialize brotli decoder");
+    }
 
-        if (ret == BROTLI_DECODER_RESULT_SUCCESS) return;
+    ~BrotliDecompressionSink()
+    {
+        BrotliDecoderDestroyInstance(state);
     }
-}
+
+    void finish() override
+    {
+        flush();
+        writeInternal(nullptr, 0);
+    }
+
+    void writeInternal(const unsigned char * data, size_t len)
+    {
+        const uint8_t * next_in = data;
+        size_t avail_in = len;
+        uint8_t * next_out = outbuf;
+        size_t avail_out = sizeof(outbuf);
+
+        while (!finished && (!data || avail_in)) {
+            checkInterrupt();
+
+            if (!BrotliDecoderDecompressStream(state,
+                    &avail_in, &next_in,
+                    &avail_out, &next_out,
+                    nullptr))
+                throw CompressionError("error while decompressing brotli file");
+
+            if (avail_out < sizeof(outbuf) || avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - avail_out);
+                next_out = outbuf;
+                avail_out = sizeof(outbuf);
+            }
+
+            finished = BrotliDecoderIsFinished(state);
+        }
+    }
+};
 
 ref<std::string> decompress(const std::string & method, const std::string & in)
 {
-    StringSource source(in);
-    StringSink sink;
-    decompress(method, source, sink);
-    return sink.s;
+    StringSink ssink;
+    auto sink = makeDecompressionSink(method, ssink);
+    (*sink)(in);
+    sink->finish();
+    return ssink.s;
 }
 
-void decompress(const std::string & method, Source & source, Sink & sink)
+ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
 {
-    if (method == "none")
-        return decompressNone(source, sink);
+    if (method == "none" || method == "")
+        return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return decompressXZ(source, sink);
+        return make_ref<XzDecompressionSink>(nextSink);
     else if (method == "bzip2")
-        return decompressBzip2(source, sink);
+        return make_ref<BzipDecompressionSink>(nextSink);
     else if (method == "br")
-        return decompressBrotli(source, sink);
+        return make_ref<BrotliDecompressionSink>(nextSink);
     else
         throw UnknownCompressionMethod("unknown compression method '%s'", method);
 }
 
-struct NoneSink : CompressionSink
-{
-    Sink & nextSink;
-    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
-    void finish() override { flush(); }
-    void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
-};
-
-struct XzSink : CompressionSink
+struct XzCompressionSink : CompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
     lzma_stream strm = LZMA_STREAM_INIT;
     bool finished = false;
 
-    template <typename F>
-    XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
-        lzma_ret ret = initEncoder();
+    XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
+    {
+        lzma_ret ret;
+        bool done = false;
+
+        if (parallel) {
+#ifdef HAVE_LZMA_MT
+            lzma_mt mt_options = {};
+            mt_options.flags = 0;
+            mt_options.timeout = 300; // Using the same setting as the xz cmd line
+            mt_options.preset = LZMA_PRESET_DEFAULT;
+            mt_options.filters = NULL;
+            mt_options.check = LZMA_CHECK_CRC64;
+            mt_options.threads = lzma_cputhreads();
+            mt_options.block_size = 0;
+            if (mt_options.threads == 0)
+                mt_options.threads = 1;
+            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+            // number of threads.
+            ret = lzma_stream_encoder_mt(&strm, &mt_options);
+            done = true;
+#else
+            printMsg(lvlError, "warning: parallel compression requested but not supported for metho  d '%1%', falling back to single-threaded compression", method);
+#endif
+        }
+
+        if (!done)
+            ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
+
         if (ret != LZMA_OK)
             throw CompressionError("unable to initialise lzma encoder");
+
         // FIXME: apply the x86 BCJ filter?
 
         strm.next_out = outbuf;
         strm.avail_out = sizeof(outbuf);
     }
-    XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
-        return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-    }) {}
 
-    ~XzSink()
+    ~XzCompressionSink()
     {
         lzma_end(&strm);
     }
@@ -246,43 +276,25 @@ struct XzSink : CompressionSink
     void finish() override
     {
         CompressionSink::flush();
-
-        assert(!finished);
-        finished = true;
-
-        while (true) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error while flushing xz file");
-
-            if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
-                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-
-            if (ret == LZMA_STREAM_END) break;
-        }
+        write(nullptr, 0);
     }
 
     void write(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
         strm.next_in = data;
         strm.avail_in = len;
 
-        while (strm.avail_in) {
+        while (!finished && (!data || strm.avail_in)) {
             checkInterrupt();
 
-            lzma_ret ret = lzma_code(&strm, LZMA_RUN);
-            if (ret != LZMA_OK)
-                throw CompressionError("error while compressing xz file");
+            lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+                throw CompressionError("error %d while compressing xz file", ret);
+
+            finished = ret == LZMA_STREAM_END;
 
-            if (strm.avail_out == 0) {
-                nextSink(outbuf, sizeof(outbuf));
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
                 strm.next_out = outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
@@ -290,46 +302,24 @@ struct XzSink : CompressionSink
     }
 };
 
-#ifdef HAVE_LZMA_MT
-struct ParallelXzSink : public XzSink
-{
-  ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
-        lzma_mt mt_options = {};
-        mt_options.flags = 0;
-        mt_options.timeout = 300; // Using the same setting as the xz cmd line
-        mt_options.preset = LZMA_PRESET_DEFAULT;
-        mt_options.filters = NULL;
-        mt_options.check = LZMA_CHECK_CRC64;
-        mt_options.threads = lzma_cputhreads();
-        mt_options.block_size = 0;
-        if (mt_options.threads == 0)
-            mt_options.threads = 1;
-        // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-        // number of threads.
-        return lzma_stream_encoder_mt(&strm, &mt_options);
-  }) {}
-};
-#endif
-
-struct BzipSink : CompressionSink
+struct BzipCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
-    char outbuf[BUFSIZ];
     bz_stream strm;
     bool finished = false;
 
-    BzipSink(Sink & nextSink) : nextSink(nextSink)
+    BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
     {
         memset(&strm, 0, sizeof(strm));
         int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
         if (ret != BZ_OK)
             throw CompressionError("unable to initialise bzip2 encoder");
 
-        strm.next_out = outbuf;
+        strm.next_out = (char *) outbuf;
         strm.avail_out = sizeof(outbuf);
     }
 
-    ~BzipSink()
+    ~BzipCompressionSink()
     {
         BZ2_bzCompressEnd(&strm);
     }
@@ -337,78 +327,49 @@ struct BzipSink : CompressionSink
     void finish() override
     {
         flush();
-
-        assert(!finished);
-        finished = true;
-
-        while (true) {
-            checkInterrupt();
-
-            int ret = BZ2_bzCompress(&strm, BZ_FINISH);
-            if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error while flushing bzip2 file");
-
-            if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
-                nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out);
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-
-            if (ret == BZ_STREAM_END) break;
-        }
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        /* Bzip2's 'avail_in' parameter is an unsigned int, so we need
-           to split the input into chunks of at most 4 GiB. */
-        while (len) {
-            auto n = std::min((size_t) std::numeric_limits<decltype(strm.avail_in)>::max(), len);
-            writeInternal(data, n);
-            data += n;
-            len -= n;
-        }
+        writeInternal(nullptr, 0);
     }
 
     void writeInternal(const unsigned char * data, size_t len)
     {
-        assert(!finished);
         assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
         strm.next_in = (char *) data;
         strm.avail_in = len;
 
-        while (strm.avail_in) {
+        while (!finished && (!data || strm.avail_in)) {
             checkInterrupt();
 
-            int ret = BZ2_bzCompress(&strm, BZ_RUN);
-            if (ret != BZ_OK)
-                CompressionError("error while compressing bzip2 file");
+            int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
+            if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
+                throw CompressionError("error %d while compressing bzip2 file", ret);
 
-            if (strm.avail_out == 0) {
-                nextSink((unsigned char *) outbuf, sizeof(outbuf));
-                strm.next_out = outbuf;
+            finished = ret == BZ_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (char *) outbuf;
                 strm.avail_out = sizeof(outbuf);
             }
         }
     }
 };
 
-struct BrotliSink : CompressionSink
+struct BrotliCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
     BrotliEncoderState *state;
     bool finished = false;
 
-    BrotliSink(Sink & nextSink) : nextSink(nextSink)
+    BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
     {
         state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
         if (!state)
             throw CompressionError("unable to initialise brotli encoder");
     }
 
-    ~BrotliSink()
+    ~BrotliCompressionSink()
     {
         BrotliEncoderDestroyInstance(state);
     }
@@ -416,89 +377,47 @@ struct BrotliSink : CompressionSink
     void finish() override
     {
         flush();
-        assert(!finished);
-
-        const uint8_t *next_in = nullptr;
-        size_t avail_in = 0;
-        uint8_t *next_out = outbuf;
-        size_t avail_out = sizeof(outbuf);
-        while (!finished) {
-            checkInterrupt();
-
-            if (!BrotliEncoderCompressStream(state,
-                        BROTLI_OPERATION_FINISH,
-                        &avail_in, &next_in,
-                        &avail_out, &next_out,
-                        nullptr))
-                throw CompressionError("error while finishing brotli file");
-
-            finished = BrotliEncoderIsFinished(state);
-            if (avail_out == 0 || finished) {
-                nextSink(outbuf, sizeof(outbuf) - avail_out);
-                next_out = outbuf;
-                avail_out = sizeof(outbuf);
-            }
-        }
-    }
-
-    void write(const unsigned char * data, size_t len) override
-    {
-        // Don't feed brotli too much at once
-        const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
-        while (len) {
-          size_t n = std::min(CHUNK_SIZE, len);
-          writeInternal(data, n);
-          data += n;
-          len -= n;
-        }
+        writeInternal(nullptr, 0);
     }
 
     void writeInternal(const unsigned char * data, size_t len)
     {
-        assert(!finished);
-
-        const uint8_t *next_in = data;
+        const uint8_t * next_in = data;
         size_t avail_in = len;
-        uint8_t *next_out = outbuf;
+        uint8_t * next_out = outbuf;
         size_t avail_out = sizeof(outbuf);
 
-        while (avail_in > 0) {
+        while (!finished && (!data || avail_in)) {
             checkInterrupt();
 
             if (!BrotliEncoderCompressStream(state,
-                      BROTLI_OPERATION_PROCESS,
-                      &avail_in, &next_in,
-                      &avail_out, &next_out,
-                      nullptr))
-                throw CompressionError("error while compressing brotli file");
+                    data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+                    &avail_in, &next_in,
+                    &avail_out, &next_out,
+                    nullptr))
+                throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
                 nextSink(outbuf, sizeof(outbuf) - avail_out);
                 next_out = outbuf;
                 avail_out = sizeof(outbuf);
             }
+
+            finished = BrotliEncoderIsFinished(state);
         }
     }
 };
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
-    if (parallel) {
-#ifdef HAVE_LZMA_MT
-        if (method == "xz")
-            return make_ref<ParallelXzSink>(nextSink);
-#endif
-        printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
-    }
-
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
     else if (method == "xz")
-        return make_ref<XzSink>(nextSink);
+        return make_ref<XzCompressionSink>(nextSink, parallel);
     else if (method == "bzip2")
-        return make_ref<BzipSink>(nextSink);
+        return make_ref<BzipCompressionSink>(nextSink);
     else if (method == "br")
-        return make_ref<BrotliSink>(nextSink);
+        return make_ref<BrotliCompressionSink>(nextSink);
     else
         throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
 }
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index f7a3e3fbd32..dd666a4e19f 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,17 +8,17 @@
 
 namespace nix {
 
-ref<std::string> decompress(const std::string & method, const std::string & in);
-
-void decompress(const std::string & method, Source & source, Sink & sink);
-
-ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
-
 struct CompressionSink : BufferedSink
 {
     virtual void finish() = 0;
 };
 
+ref<std::string> decompress(const std::string & method, const std::string & in);
+
+ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
 
 MakeError(UnknownCompressionMethod, Error);

From 7de3e00ad905bba85abadd86b83973fdba8d0dfd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Aug 2018 17:20:23 +0200
Subject: [PATCH 1143/2196] Fix setting Content-Encoding in S3 uploads

Fixes https://github.com/NixOS/nix/issues/2333 and https://github.com/NixOS/nixpkgs/issues/44337.
---
 release-common.nix                    |  9 +++++++--
 src/libstore/s3-binary-cache-store.cc | 10 ++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index f98e86a1b44..e23967bdbfd 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -60,10 +60,15 @@ rec {
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
-      (aws-sdk-cpp.override {
+      ((aws-sdk-cpp.override {
         apis = ["s3" "transfer"];
         customMemoryManagement = false;
-      });
+      }).overrideDerivation (args: {
+        patches = args.patches or [] ++ [ (fetchpatch {
+          url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
+          sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
+        }) ];
+      }));
 
   perlDeps =
     [ perl
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 660583d31fe..6d95c1fa8c6 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -291,10 +291,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         transferConfig.s3Client = s3Helper.client;
         transferConfig.bufferSize = bufferSize;
 
-        if (contentEncoding != "")
-            transferConfig.createMultipartUploadTemplate.SetContentEncoding(
-                contentEncoding);
-
         transferConfig.uploadProgressCallback =
             [&](const TransferManager *transferManager,
                 const std::shared_ptr<const TransferHandle>
@@ -336,8 +332,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         auto now1 = std::chrono::steady_clock::now();
 
         std::shared_ptr<TransferHandle> transferHandle =
-            transferManager->UploadFile(stream, bucketName, path, mimeType,
-                                        Aws::Map<Aws::String, Aws::String>());
+            transferManager->UploadFile(
+                stream, bucketName, path, mimeType,
+                Aws::Map<Aws::String, Aws::String>(),
+                nullptr, contentEncoding);
 
         transferHandle->WaitUntilFinished();
 

From ed6c646f44e5b9e0fcbc53058491e97875a263c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Aug 2018 17:27:08 +0200
Subject: [PATCH 1144/2196] Doh

---
 src/libstore/legacy-ssh-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index ddfd17d641b..7c214f09d6f 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -59,7 +59,7 @@ struct LegacySSHStore : public Store
     {
         auto conn = make_ref<Connection>();
         conn->sshConn = master.startCommand(
-            fmt("command time %s --serve --write", remoteProgram)
+            fmt("%s --serve --write", remoteProgram)
             + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
         conn->to = FdSink(conn->sshConn->in.get());
         conn->from = FdSource(conn->sshConn->out.get());

From ec49ea28dc5309c8905fc4385fb65ec7a88b33f6 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Tue, 7 Aug 2018 03:40:44 +0000
Subject: [PATCH 1145/2196] repl: don't add trailing spaces to history lines

---
 src/nix/repl.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 1eb71600637..b71e6f905f2 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -173,9 +173,14 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
             printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         }
 
+        if (input.size() > 0) {
+            // Remove trailing newline before adding to history
+            input.erase(input.size() - 1);
+            linenoiseHistoryAdd(input.c_str());
+        }
+
         // We handled the current input fully, so we should clear it
         // and read brand new input.
-        linenoiseHistoryAdd(input.c_str());
         input.clear();
         std::cout << std::endl;
     }

From 58a85fa4621faaa89286065f5583328783085722 Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Wed, 8 Aug 2018 21:21:21 +0200
Subject: [PATCH 1146/2196] mention nix-store --query --roots when a path
 cannot be deleted

---
 src/libstore/gc.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index b415d542147..65220a9f6ae 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -791,7 +791,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             assertStorePath(i);
             tryToDelete(state, i);
             if (state.dead.find(i) == state.dead.end())
-                throw Error(format("cannot delete path '%1%' since it is still alive") % i);
+                throw Error(format(
+                    "cannot delete path '%1%' since it is still alive. "
+                    "To find out why use: "
+                    "nix-store --query --roots"
+                    ) % i);
         }
 
     } else if (options.maxFreed > 0) {

From a0b971dd9c19819d4f7a3a8ab102be9d7101e3e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 8 Aug 2018 21:39:11 +0200
Subject: [PATCH 1147/2196] S3BinaryCacheStore: Don't use the transfer status
 callback

This callback is executed on a different thread, so exceptions thrown
from the callback are not caught:

  Aug 08 16:25:48 chef hydra-queue-runner[11967]: terminate called after throwing an instance of 'nix::Error'
  Aug 08 16:25:48 chef hydra-queue-runner[11967]:   what():  AWS error: failed to upload 's3://nix-cache/19dbddlfb0vp68g68y19p9fswrgl0bg7.ls'

Therefore, just check the transfer status after it completes. Also
include the S3 error message in the exception.
---
 src/libstore/s3-binary-cache-store.cc | 37 ++++++++++-----------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 6d95c1fa8c6..ef41e413fcf 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -296,36 +296,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 const std::shared_ptr<const TransferHandle>
                     &transferHandle) {
               //FIXME: find a way to properly abort the multipart upload.
-              checkInterrupt();
+              //checkInterrupt();
               debug("upload progress ('%s'): '%d' of '%d' bytes",
                              path,
                              transferHandle->GetBytesTransferred(),
                              transferHandle->GetBytesTotalSize());
             };
 
-        transferConfig.transferStatusUpdatedCallback =
-            [&](const TransferManager *,
-                const std::shared_ptr<const TransferHandle>
-                    &transferHandle) {
-              switch (transferHandle->GetStatus()) {
-                  case TransferStatus::COMPLETED:
-                      printTalkative("upload of '%s' completed", path);
-                      stats.put++;
-                      stats.putBytes += data.size();
-                      break;
-                  case TransferStatus::IN_PROGRESS:
-                      break;
-                  case TransferStatus::FAILED:
-                      throw Error("AWS error: failed to upload 's3://%s/%s'",
-                                  bucketName, path);
-                      break;
-                  default:
-                      throw Error("AWS error: transfer status of 's3://%s/%s' "
-                                  "in unexpected state",
-                                  bucketName, path);
-              };
-            };
-
         std::shared_ptr<TransferManager> transferManager =
             TransferManager::Create(transferConfig);
 
@@ -339,6 +316,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
         transferHandle->WaitUntilFinished();
 
+        if (transferHandle->GetStatus() == TransferStatus::FAILED)
+            throw Error("AWS error: failed to upload 's3://%s/%s': %s",
+                bucketName, path, transferHandle->GetLastError().GetMessage());
+
+        if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
+            throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
+                bucketName, path);
+
+        printTalkative("upload of '%s' completed", path);
+
         auto now2 = std::chrono::steady_clock::now();
 
         auto duration =
@@ -349,6 +336,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                   bucketName % path % data.size() % duration);
 
         stats.putTimeMs += duration;
+        stats.putBytes += data.size();
+        stats.put++;
     }
 
     void upsertFile(const std::string & path, const std::string & data,

From c87f4b9324b87a059cf760a477177f322bb8dc26 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 9 Aug 2018 13:01:03 +0200
Subject: [PATCH 1148/2196] nix run: Respect propagated-user-env-packages

Also, add $path/bin to $PATH even if it doesn't exist. This makes
'man' work properly (since it looks for ../share/man relative to $PATH
entries).
---
 src/nix/run.cc | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index d04e106e037..65ced34759b 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -12,6 +12,8 @@
 #include <sys/mount.h>
 #endif
 
+#include <queue>
+
 using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
@@ -121,10 +123,27 @@ struct CmdRun : InstallablesCommand
                 unsetenv(var.c_str());
         }
 
+        std::unordered_set<Path> done;
+        std::queue<Path> todo;
+        for (auto & path : outPaths) todo.push(path);
+
         auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
-        for (auto & path : outPaths)
-            if (accessor->stat(path + "/bin").type != FSAccessor::tMissing)
+
+        while (!todo.empty()) {
+            Path path = todo.front();
+            todo.pop();
+            if (!done.insert(path).second) continue;
+
+            if (true)
                 unixPath.push_front(path + "/bin");
+
+            auto propPath = path + "/nix-support/propagated-user-env-packages";
+            if (accessor->stat(propPath).type == FSAccessor::tRegular) {
+                for (auto & p : tokenizeString<Paths>(readFile(propPath)))
+                    todo.push(p);
+            }
+        }
+
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
         std::string cmd = *command.begin();

From ada4e902672e4d82f47fa260adcc450d9a85d41f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 9 Aug 2018 20:44:18 +0200
Subject: [PATCH 1149/2196] S3BinaryCacheStore: Share TransferManager

TransferManager allocates a lot of memory (50 MiB by default), and it
might leak but I'm not sure about that. In any case it was causing
OOMs in hydra-queue-runner. So allocate only one TransferManager per
S3BinaryCacheStore.

Hopefully fixes https://github.com/NixOS/hydra/issues/586.
---
 src/libstore/s3-binary-cache-store.cc | 42 +++++++++++++++------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ef41e413fcf..7711388f05a 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -275,6 +275,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return true;
     }
 
+    std::shared_ptr<TransferManager> transferManager;
+    std::once_flag transferManagerCreated;
+
     void uploadFile(const std::string & path, const std::string & data,
         const std::string & mimeType,
         const std::string & contentEncoding)
@@ -286,25 +289,28 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
             executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
-        TransferManagerConfiguration transferConfig(executor.get());
-
-        transferConfig.s3Client = s3Helper.client;
-        transferConfig.bufferSize = bufferSize;
-
-        transferConfig.uploadProgressCallback =
-            [&](const TransferManager *transferManager,
-                const std::shared_ptr<const TransferHandle>
-                    &transferHandle) {
-              //FIXME: find a way to properly abort the multipart upload.
-              //checkInterrupt();
-              debug("upload progress ('%s'): '%d' of '%d' bytes",
-                             path,
-                             transferHandle->GetBytesTransferred(),
-                             transferHandle->GetBytesTotalSize());
-            };
+        std::call_once(transferManagerCreated, [&]() {
+
+            TransferManagerConfiguration transferConfig(executor.get());
+
+            transferConfig.s3Client = s3Helper.client;
+            transferConfig.bufferSize = bufferSize;
+
+            transferConfig.uploadProgressCallback =
+                [&](const TransferManager *transferManager,
+                    const std::shared_ptr<const TransferHandle>
+                    &transferHandle)
+                {
+                    //FIXME: find a way to properly abort the multipart upload.
+                    //checkInterrupt();
+                    debug("upload progress ('%s'): '%d' of '%d' bytes",
+                        path,
+                        transferHandle->GetBytesTransferred(),
+                        transferHandle->GetBytesTotalSize());
+                };
 
-        std::shared_ptr<TransferManager> transferManager =
-            TransferManager::Create(transferConfig);
+            transferManager = TransferManager::Create(transferConfig);
+        });
 
         auto now1 = std::chrono::steady_clock::now();
 

From f72c907ad833fa26800ad1694e63f3cec952b444 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 10 Aug 2018 11:34:41 +0200
Subject: [PATCH 1150/2196] Slightly questionable workaround for #2342

---
 src/nix/installables.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 0be992b03c5..0c1ad3ab3db 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -96,7 +96,7 @@ struct InstallableStorePath : Installable
 
     Buildables toBuildables() override
     {
-        return {{"", {{"out", storePath}}}};
+        return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}};
     }
 };
 

From d7402c9cd5c17644d73b2a7a39e10be22fffeb00 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Mon, 13 Aug 2018 11:27:35 +0200
Subject: [PATCH 1151/2196] dirOf: allow use on non-absolute paths

---
 src/libexpr/primops.cc | 2 +-
 src/libutil/util.cc    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8ace6db4d11..32d6640fad2 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -866,7 +866,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args,
 static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path dir = dirOf(state.coerceToPath(pos, *args[0], context));
+    Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
     if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6bc64ae75a4..4cc7455be6c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -167,7 +167,7 @@ Path dirOf(const Path & path)
 {
     Path::size_type pos = path.rfind('/');
     if (pos == string::npos)
-        throw Error(format("invalid file name '%1%'") % path);
+        return ".";
     return pos == 0 ? "/" : Path(path, 0, pos);
 }
 

From 5b19a6663b06a158cb77d170e6740dce7ced6185 Mon Sep 17 00:00:00 2001
From: Symphorien Gibol <symphorien+git@xlumurb.eu>
Date: Mon, 13 Aug 2018 21:11:36 +0200
Subject: [PATCH 1152/2196] ignore when listxattr fails with ENODATA

This happens on CIFS and means the remote filesystem has no extended
attributes.
---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 3b2ba65f3b4..c91dbf241bc 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -450,7 +450,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 
     if (eaSize < 0) {
-        if (errno != ENOTSUP)
+        if (errno != ENOTSUP && errno != ENODATA)
             throw SysError("querying extended attributes of '%s'", path);
     } else if (eaSize > 0) {
         std::vector<char> eaBuf(eaSize);

From bad27dc4755b91b421a0df96dd1a11bc043a5c25 Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Mon, 13 Aug 2018 20:00:06 +0000
Subject: [PATCH 1153/2196] update config/config.{sub,guess}

Just
curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'   > config/config.sub
curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' > config/config.guess

Those files are 5 years old and failed to guess new archs ("ppc64-linux")
---
 config/config.guess |  983 +++++++++++-----------
 config/config.sub   | 1944 ++++++++++++++++++++++---------------------
 2 files changed, 1454 insertions(+), 1473 deletions(-)

diff --git a/config/config.guess b/config/config.guess
index 137bedf2e28..d4fb3213ec7 100755
--- a/config/config.guess
+++ b/config/config.guess
@@ -1,14 +1,12 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-#   2011, 2012 Free Software Foundation, Inc.
+#   Copyright 1992-2018 Free Software Foundation, Inc.
 
-timestamp='2012-08-14'
+timestamp='2018-08-02'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful, but
@@ -17,24 +15,22 @@ timestamp='2012-08-14'
 # General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
+# along with this program; if not, see <https://www.gnu.org/licenses/>.
 #
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-
-# Originally written by Per Bothner.  Please send patches (context
-# diff format) to <config-patches@gnu.org> and include a ChangeLog
-# entry.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 #
-# This script attempts to guess a canonical system name similar to
-# config.sub.  If it succeeds, it prints the system name on stdout, and
-# exits with 0.  Otherwise, it exits with 1.
+# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
 #
 # You can get the latest version of this script from:
-# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
+#
+# Please send patches to <config-patches@gnu.org>.
+
 
 me=`echo "$0" | sed -e 's,.*/,,'`
 
@@ -43,7 +39,7 @@ Usage: $0 [OPTION]
 
 Output the configuration name of the system \`$me' is run on.
 
-Operation modes:
+Options:
   -h, --help         print this help, then exit
   -t, --time-stamp   print date of last modification, then exit
   -v, --version      print version number, then exit
@@ -54,9 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
-Free Software Foundation, Inc.
+Copyright 1992-2018 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -90,8 +84,6 @@ if test $# != 0; then
   exit 1
 fi
 
-trap 'exit 1' 1 2 15
-
 # CC_FOR_BUILD -- compiler used by this script. Note that the use of a
 # compiler to aid in system detection is discouraged as it requires
 # temporary files to be created and, as you can see below, it is a
@@ -102,34 +94,39 @@ trap 'exit 1' 1 2 15
 
 # Portable tmp directory creation inspired by the Autoconf team.
 
-set_cc_for_build='
-trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ;
-trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ;
-: ${TMPDIR=/tmp} ;
- { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
- { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } ||
- { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } ||
- { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ;
-dummy=$tmp/dummy ;
-tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ;
-case $CC_FOR_BUILD,$HOST_CC,$CC in
- ,,)    echo "int x;" > $dummy.c ;
-	for c in cc gcc c89 c99 ; do
-	  if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then
-	     CC_FOR_BUILD="$c"; break ;
-	  fi ;
-	done ;
-	if test x"$CC_FOR_BUILD" = x ; then
-	  CC_FOR_BUILD=no_compiler_found ;
-	fi
-	;;
- ,,*)   CC_FOR_BUILD=$CC ;;
- ,*,*)  CC_FOR_BUILD=$HOST_CC ;;
-esac ; set_cc_for_build= ;'
+tmp=
+# shellcheck disable=SC2172
+trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15
+trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0
+
+set_cc_for_build() {
+    : "${TMPDIR=/tmp}"
+    # shellcheck disable=SC2039
+    { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
+	{ test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } ||
+	{ tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } ||
+	{ echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; }
+    dummy=$tmp/dummy
+    case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in
+	,,)    echo "int x;" > "$dummy.c"
+	       for driver in cc gcc c89 c99 ; do
+		   if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then
+		       CC_FOR_BUILD="$driver"
+		       break
+		   fi
+	       done
+	       if test x"$CC_FOR_BUILD" = x ; then
+		   CC_FOR_BUILD=no_compiler_found
+	       fi
+	       ;;
+	,,*)   CC_FOR_BUILD=$CC ;;
+	,*,*)  CC_FOR_BUILD=$HOST_CC ;;
+    esac
+}
 
 # This is needed to find uname on a Pyramid OSx when run in the BSD universe.
 # (ghazi@noc.rutgers.edu 1994-08-24)
-if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+if test -f /.attbin/uname ; then
 	PATH=$PATH:/.attbin ; export PATH
 fi
 
@@ -138,9 +135,37 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
 UNAME_SYSTEM=`(uname -s) 2>/dev/null`  || UNAME_SYSTEM=unknown
 UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
 
+case "$UNAME_SYSTEM" in
+Linux|GNU|GNU/*)
+	# If the system lacks a compiler, then just pick glibc.
+	# We could probably try harder.
+	LIBC=gnu
+
+	set_cc_for_build
+	cat <<-EOF > "$dummy.c"
+	#include <features.h>
+	#if defined(__UCLIBC__)
+	LIBC=uclibc
+	#elif defined(__dietlibc__)
+	LIBC=dietlibc
+	#else
+	LIBC=gnu
+	#endif
+	EOF
+	eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`"
+
+	# If ldd exists, use it to detect musl libc.
+	if command -v ldd >/dev/null && \
+		ldd --version 2>&1 | grep -q ^musl
+	then
+	    LIBC=musl
+	fi
+	;;
+esac
+
 # Note: order is significant - the case branches are not exclusive.
 
-case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
     *:NetBSD:*:*)
 	# NetBSD (nbsd) targets should (where applicable) match one or
 	# more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*,
@@ -153,21 +178,31 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	# Note: NetBSD doesn't particularly care about the vendor
 	# portion of the name.  We always set it to "unknown".
 	sysctl="sysctl -n hw.machine_arch"
-	UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
-	    /usr/sbin/$sysctl 2>/dev/null || echo unknown)`
-	case "${UNAME_MACHINE_ARCH}" in
+	UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
+	    "/sbin/$sysctl" 2>/dev/null || \
+	    "/usr/sbin/$sysctl" 2>/dev/null || \
+	    echo unknown)`
+	case "$UNAME_MACHINE_ARCH" in
 	    armeb) machine=armeb-unknown ;;
 	    arm*) machine=arm-unknown ;;
 	    sh3el) machine=shl-unknown ;;
 	    sh3eb) machine=sh-unknown ;;
 	    sh5el) machine=sh5le-unknown ;;
-	    *) machine=${UNAME_MACHINE_ARCH}-unknown ;;
+	    earmv*)
+		arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
+		endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'`
+		machine="${arch}${endian}"-unknown
+		;;
+	    *) machine="$UNAME_MACHINE_ARCH"-unknown ;;
 	esac
 	# The Operating System including object format, if it has switched
-	# to ELF recently, or will in the future.
-	case "${UNAME_MACHINE_ARCH}" in
+	# to ELF recently (or will in the future) and ABI.
+	case "$UNAME_MACHINE_ARCH" in
+	    earm*)
+		os=netbsdelf
+		;;
 	    arm*|i386|m68k|ns32k|sh3*|sparc|vax)
-		eval $set_cc_for_build
+		set_cc_for_build
 		if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
 			| grep -q __ELF__
 		then
@@ -182,44 +217,67 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		os=netbsd
 		;;
 	esac
+	# Determine ABI tags.
+	case "$UNAME_MACHINE_ARCH" in
+	    earm*)
+		expr='s/^earmv[0-9]/-eabi/;s/eb$//'
+		abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"`
+		;;
+	esac
 	# The OS release
 	# Debian GNU/NetBSD machines have a different userland, and
 	# thus, need a distinct triplet. However, they do not need
 	# kernel version information, so it can be replaced with a
 	# suitable tag, in the style of linux-gnu.
-	case "${UNAME_VERSION}" in
+	case "$UNAME_VERSION" in
 	    Debian*)
 		release='-gnu'
 		;;
 	    *)
-		release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+		release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2`
 		;;
 	esac
 	# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
 	# contains redundant information, the shorter form:
 	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
-	echo "${machine}-${os}${release}"
+	echo "$machine-${os}${release}${abi-}"
 	exit ;;
     *:Bitrig:*:*)
 	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
-	echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE}
+	echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE"
 	exit ;;
     *:OpenBSD:*:*)
 	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
-	echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
+	echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE"
+	exit ;;
+    *:LibertyBSD:*:*)
+	UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
+	echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE"
+	exit ;;
+    *:MidnightBSD:*:*)
+	echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE"
 	exit ;;
     *:ekkoBSD:*:*)
-	echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE"
 	exit ;;
     *:SolidBSD:*:*)
-	echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE"
 	exit ;;
     macppc:MirBSD:*:*)
-	echo powerpc-unknown-mirbsd${UNAME_RELEASE}
+	echo powerpc-unknown-mirbsd"$UNAME_RELEASE"
 	exit ;;
     *:MirBSD:*:*)
-	echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE"
+	exit ;;
+    *:Sortix:*:*)
+	echo "$UNAME_MACHINE"-unknown-sortix
+	exit ;;
+    *:Redox:*:*)
+	echo "$UNAME_MACHINE"-unknown-redox
 	exit ;;
+    mips:OSF1:*.*)
+        echo mips-dec-osf1
+        exit ;;
     alpha:OSF1:*:*)
 	case $UNAME_RELEASE in
 	*4.0)
@@ -236,63 +294,54 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^  The alpha \(.*\) processor.*$/\1/p' | head -n 1`
 	case "$ALPHA_CPU_TYPE" in
 	    "EV4 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV4.5 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "LCA4 (21066/21068)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV5 (21164)")
-		UNAME_MACHINE="alphaev5" ;;
+		UNAME_MACHINE=alphaev5 ;;
 	    "EV5.6 (21164A)")
-		UNAME_MACHINE="alphaev56" ;;
+		UNAME_MACHINE=alphaev56 ;;
 	    "EV5.6 (21164PC)")
-		UNAME_MACHINE="alphapca56" ;;
+		UNAME_MACHINE=alphapca56 ;;
 	    "EV5.7 (21164PC)")
-		UNAME_MACHINE="alphapca57" ;;
+		UNAME_MACHINE=alphapca57 ;;
 	    "EV6 (21264)")
-		UNAME_MACHINE="alphaev6" ;;
+		UNAME_MACHINE=alphaev6 ;;
 	    "EV6.7 (21264A)")
-		UNAME_MACHINE="alphaev67" ;;
+		UNAME_MACHINE=alphaev67 ;;
 	    "EV6.8CB (21264C)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8AL (21264B)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8CX (21264D)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.9A (21264/EV69A)")
-		UNAME_MACHINE="alphaev69" ;;
+		UNAME_MACHINE=alphaev69 ;;
 	    "EV7 (21364)")
-		UNAME_MACHINE="alphaev7" ;;
+		UNAME_MACHINE=alphaev7 ;;
 	    "EV7.9 (21364A)")
-		UNAME_MACHINE="alphaev79" ;;
+		UNAME_MACHINE=alphaev79 ;;
 	esac
 	# A Pn.n version is a patched version.
 	# A Vn.n version is a released version.
 	# A Tn.n version is a released field test version.
 	# A Xn.n version is an unreleased experimental baselevel.
 	# 1.2 uses "1.2" for uname -r.
-	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+	echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`"
 	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
 	exitcode=$?
 	trap '' 0
 	exit $exitcode ;;
-    Alpha\ *:Windows_NT*:*)
-	# How do we know it's Interix rather than the generic POSIX subsystem?
-	# Should we change UNAME_MACHINE based on the output of uname instead
-	# of the specific Alpha model?
-	echo alpha-pc-interix
-	exit ;;
-    21064:Windows_NT:50:3)
-	echo alpha-dec-winnt3.5
-	exit ;;
     Amiga*:UNIX_System_V:4.0:*)
 	echo m68k-unknown-sysv4
 	exit ;;
     *:[Aa]miga[Oo][Ss]:*:*)
-	echo ${UNAME_MACHINE}-unknown-amigaos
+	echo "$UNAME_MACHINE"-unknown-amigaos
 	exit ;;
     *:[Mm]orph[Oo][Ss]:*:*)
-	echo ${UNAME_MACHINE}-unknown-morphos
+	echo "$UNAME_MACHINE"-unknown-morphos
 	exit ;;
     *:OS/390:*:*)
 	echo i370-ibm-openedition
@@ -304,9 +353,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	echo powerpc-ibm-os400
 	exit ;;
     arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
-	echo arm-acorn-riscix${UNAME_RELEASE}
+	echo arm-acorn-riscix"$UNAME_RELEASE"
 	exit ;;
-    arm:riscos:*:*|arm:RISCOS:*:*)
+    arm*:riscos:*:*|arm*:RISCOS:*:*)
 	echo arm-unknown-riscos
 	exit ;;
     SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
@@ -331,38 +380,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	    sparc) echo sparc-icl-nx7; exit ;;
 	esac ;;
     s390x:SunOS:*:*)
-	echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
 	exit ;;
     sun4H:SunOS:5.*:*)
-	echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
 	exit ;;
     sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
-	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
 	exit ;;
     i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
-	echo i386-pc-auroraux${UNAME_RELEASE}
+	echo i386-pc-auroraux"$UNAME_RELEASE"
 	exit ;;
     i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
-	eval $set_cc_for_build
-	SUN_ARCH="i386"
-	# If there is a compiler, see if it is configured for 64-bit objects.
-	# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
-	# This test works for both compilers.
-	if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
-	    if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
-		(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
-		grep IS_64BIT_ARCH >/dev/null
-	    then
-		SUN_ARCH="x86_64"
-	    fi
-	fi
-	echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
+	case `isainfo -b` in
+	    32)
+		echo i386-pc-solaris2"$UNAME_REL"
+		;;
+	    64)
+		echo x86_64-pc-solaris2"$UNAME_REL"
+		;;
+	esac
 	exit ;;
     sun4*:SunOS:6*:*)
 	# According to config.sub, this is the proper way to canonicalize
 	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
 	# it's likely to be more like Solaris than SunOS4.
-	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
 	exit ;;
     sun4*:SunOS:*:*)
 	case "`/usr/bin/arch -k`" in
@@ -371,25 +415,25 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		;;
 	esac
 	# Japanese Language versions have a version number like `4.1.3-JL'.
-	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`"
 	exit ;;
     sun3*:SunOS:*:*)
-	echo m68k-sun-sunos${UNAME_RELEASE}
+	echo m68k-sun-sunos"$UNAME_RELEASE"
 	exit ;;
     sun*:*:4.2BSD:*)
 	UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
-	test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
+	test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3
 	case "`/bin/arch`" in
 	    sun3)
-		echo m68k-sun-sunos${UNAME_RELEASE}
+		echo m68k-sun-sunos"$UNAME_RELEASE"
 		;;
 	    sun4)
-		echo sparc-sun-sunos${UNAME_RELEASE}
+		echo sparc-sun-sunos"$UNAME_RELEASE"
 		;;
 	esac
 	exit ;;
     aushp:SunOS:*:*)
-	echo sparc-auspex-sunos${UNAME_RELEASE}
+	echo sparc-auspex-sunos"$UNAME_RELEASE"
 	exit ;;
     # The situation for MiNT is a little confusing.  The machine name
     # can be virtually everything (everything which is not
@@ -400,44 +444,44 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
     # MiNT.  But MiNT is downward compatible to TOS, so this should
     # be no problem.
     atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
-	echo m68k-atari-mint${UNAME_RELEASE}
+	echo m68k-atari-mint"$UNAME_RELEASE"
 	exit ;;
     atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
-	echo m68k-atari-mint${UNAME_RELEASE}
+	echo m68k-atari-mint"$UNAME_RELEASE"
 	exit ;;
     *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
-	echo m68k-atari-mint${UNAME_RELEASE}
+	echo m68k-atari-mint"$UNAME_RELEASE"
 	exit ;;
     milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
-	echo m68k-milan-mint${UNAME_RELEASE}
+	echo m68k-milan-mint"$UNAME_RELEASE"
 	exit ;;
     hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
-	echo m68k-hades-mint${UNAME_RELEASE}
+	echo m68k-hades-mint"$UNAME_RELEASE"
 	exit ;;
     *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
-	echo m68k-unknown-mint${UNAME_RELEASE}
+	echo m68k-unknown-mint"$UNAME_RELEASE"
 	exit ;;
     m68k:machten:*:*)
-	echo m68k-apple-machten${UNAME_RELEASE}
+	echo m68k-apple-machten"$UNAME_RELEASE"
 	exit ;;
     powerpc:machten:*:*)
-	echo powerpc-apple-machten${UNAME_RELEASE}
+	echo powerpc-apple-machten"$UNAME_RELEASE"
 	exit ;;
     RISC*:Mach:*:*)
 	echo mips-dec-mach_bsd4.3
 	exit ;;
     RISC*:ULTRIX:*:*)
-	echo mips-dec-ultrix${UNAME_RELEASE}
+	echo mips-dec-ultrix"$UNAME_RELEASE"
 	exit ;;
     VAX*:ULTRIX*:*:*)
-	echo vax-dec-ultrix${UNAME_RELEASE}
+	echo vax-dec-ultrix"$UNAME_RELEASE"
 	exit ;;
     2020:CLIX:*:* | 2430:CLIX:*:*)
-	echo clipper-intergraph-clix${UNAME_RELEASE}
+	echo clipper-intergraph-clix"$UNAME_RELEASE"
 	exit ;;
     mips:*:*:UMIPS | mips:*:*:RISCos)
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
+	set_cc_for_build
+	sed 's/^	//' << EOF > "$dummy.c"
 #ifdef __cplusplus
 #include <stdio.h>  /* for printf() prototype */
 	int main (int argc, char *argv[]) {
@@ -446,23 +490,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 #endif
 	#if defined (host_mips) && defined (MIPSEB)
 	#if defined (SYSTYPE_SYSV)
-	  printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0);
+	  printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0);
 	#endif
 	#if defined (SYSTYPE_SVR4)
-	  printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0);
+	  printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0);
 	#endif
 	#if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD)
-	  printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0);
+	  printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0);
 	#endif
 	#endif
 	  exit (-1);
 	}
 EOF
-	$CC_FOR_BUILD -o $dummy $dummy.c &&
-	  dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` &&
-	  SYSTEM_NAME=`$dummy $dummyarg` &&
+	$CC_FOR_BUILD -o "$dummy" "$dummy.c" &&
+	  dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` &&
+	  SYSTEM_NAME=`"$dummy" "$dummyarg"` &&
 	    { echo "$SYSTEM_NAME"; exit; }
-	echo mips-mips-riscos${UNAME_RELEASE}
+	echo mips-mips-riscos"$UNAME_RELEASE"
 	exit ;;
     Motorola:PowerMAX_OS:*:*)
 	echo powerpc-motorola-powermax
@@ -488,17 +532,17 @@ EOF
     AViiON:dgux:*:*)
 	# DG/UX returns AViiON for all architectures
 	UNAME_PROCESSOR=`/usr/bin/uname -p`
-	if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
+	if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ]
 	then
-	    if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
-	       [ ${TARGET_BINARY_INTERFACE}x = x ]
+	    if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \
+	       [ "$TARGET_BINARY_INTERFACE"x = x ]
 	    then
-		echo m88k-dg-dgux${UNAME_RELEASE}
+		echo m88k-dg-dgux"$UNAME_RELEASE"
 	    else
-		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+		echo m88k-dg-dguxbcs"$UNAME_RELEASE"
 	    fi
 	else
-	    echo i586-dg-dgux${UNAME_RELEASE}
+	    echo i586-dg-dgux"$UNAME_RELEASE"
 	fi
 	exit ;;
     M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
@@ -515,7 +559,7 @@ EOF
 	echo m68k-tektronix-bsd
 	exit ;;
     *:IRIX*:*:*)
-	echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'`
+	echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`"
 	exit ;;
     ????????:AIX?:[12].1:2)   # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX.
 	echo romp-ibm-aix     # uname -m gives an 8 hex-code CPU id
@@ -527,14 +571,14 @@ EOF
 	if [ -x /usr/bin/oslevel ] ; then
 		IBM_REV=`/usr/bin/oslevel`
 	else
-		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+		IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
 	fi
-	echo ${UNAME_MACHINE}-ibm-aix${IBM_REV}
+	echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV"
 	exit ;;
     *:AIX:2:3)
 	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
-		eval $set_cc_for_build
-		sed 's/^		//' << EOF >$dummy.c
+		set_cc_for_build
+		sed 's/^		//' << EOF > "$dummy.c"
 		#include <sys/systemcfg.h>
 
 		main()
@@ -545,7 +589,7 @@ EOF
 			exit(0);
 			}
 EOF
-		if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy`
+		if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"`
 		then
 			echo "$SYSTEM_NAME"
 		else
@@ -559,26 +603,27 @@ EOF
 	exit ;;
     *:AIX:*:[4567])
 	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
-	if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
+	if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then
 		IBM_ARCH=rs6000
 	else
 		IBM_ARCH=powerpc
 	fi
-	if [ -x /usr/bin/oslevel ] ; then
-		IBM_REV=`/usr/bin/oslevel`
+	if [ -x /usr/bin/lslpp ] ; then
+		IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
+			   awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
 	else
-		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+		IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
 	fi
-	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	echo "$IBM_ARCH"-ibm-aix"$IBM_REV"
 	exit ;;
     *:AIX:*:*)
 	echo rs6000-ibm-aix
 	exit ;;
-    ibmrt:4.4BSD:*|romp-ibm:BSD:*)
+    ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*)
 	echo romp-ibm-bsd4.4
 	exit ;;
     ibmrt:*BSD:*|romp-ibm:BSD:*)            # covers RT/PC BSD and
-	echo romp-ibm-bsd${UNAME_RELEASE}   # 4.3 with uname added to
+	echo romp-ibm-bsd"$UNAME_RELEASE"   # 4.3 with uname added to
 	exit ;;                             # report: romp-ibm BSD 4.3
     *:BOSX:*:*)
 	echo rs6000-bull-bosx
@@ -593,28 +638,28 @@ EOF
 	echo m68k-hp-bsd4.4
 	exit ;;
     9000/[34678]??:HP-UX:*:*)
-	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
-	case "${UNAME_MACHINE}" in
-	    9000/31? )            HP_ARCH=m68000 ;;
-	    9000/[34]?? )         HP_ARCH=m68k ;;
+	HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
+	case "$UNAME_MACHINE" in
+	    9000/31?)            HP_ARCH=m68000 ;;
+	    9000/[34]??)         HP_ARCH=m68k ;;
 	    9000/[678][0-9][0-9])
 		if [ -x /usr/bin/getconf ]; then
 		    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
 		    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
-		    case "${sc_cpu_version}" in
-		      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
-		      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+		    case "$sc_cpu_version" in
+		      523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
+		      528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
 		      532)                      # CPU_PA_RISC2_0
-			case "${sc_kernel_bits}" in
-			  32) HP_ARCH="hppa2.0n" ;;
-			  64) HP_ARCH="hppa2.0w" ;;
-			  '') HP_ARCH="hppa2.0" ;;   # HP-UX 10.20
+			case "$sc_kernel_bits" in
+			  32) HP_ARCH=hppa2.0n ;;
+			  64) HP_ARCH=hppa2.0w ;;
+			  '') HP_ARCH=hppa2.0 ;;   # HP-UX 10.20
 			esac ;;
 		    esac
 		fi
-		if [ "${HP_ARCH}" = "" ]; then
-		    eval $set_cc_for_build
-		    sed 's/^		//' << EOF >$dummy.c
+		if [ "$HP_ARCH" = "" ]; then
+		    set_cc_for_build
+		    sed 's/^		//' << EOF > "$dummy.c"
 
 		#define _HPUX_SOURCE
 		#include <stdlib.h>
@@ -647,13 +692,13 @@ EOF
 		    exit (0);
 		}
 EOF
-		    (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
+		    (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"`
 		    test -z "$HP_ARCH" && HP_ARCH=hppa
 		fi ;;
 	esac
-	if [ ${HP_ARCH} = "hppa2.0w" ]
+	if [ "$HP_ARCH" = hppa2.0w ]
 	then
-	    eval $set_cc_for_build
+	    set_cc_for_build
 
 	    # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating
 	    # 32-bit code.  hppa64-hp-hpux* has the same kernel and a compiler
@@ -664,23 +709,23 @@ EOF
 	    # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
 	    # => hppa64-hp-hpux11.23
 
-	    if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
+	    if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) |
 		grep -q __LP64__
 	    then
-		HP_ARCH="hppa2.0w"
+		HP_ARCH=hppa2.0w
 	    else
-		HP_ARCH="hppa64"
+		HP_ARCH=hppa64
 	    fi
 	fi
-	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	echo "$HP_ARCH"-hp-hpux"$HPUX_REV"
 	exit ;;
     ia64:HP-UX:*:*)
-	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
-	echo ia64-hp-hpux${HPUX_REV}
+	HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
+	echo ia64-hp-hpux"$HPUX_REV"
 	exit ;;
     3050*:HI-UX:*:*)
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
+	set_cc_for_build
+	sed 's/^	//' << EOF > "$dummy.c"
 	#include <unistd.h>
 	int
 	main ()
@@ -705,11 +750,11 @@ EOF
 	  exit (0);
 	}
 EOF
-	$CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` &&
+	$CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` &&
 		{ echo "$SYSTEM_NAME"; exit; }
 	echo unknown-hitachi-hiuxwe2
 	exit ;;
-    9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* )
+    9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*)
 	echo hppa1.1-hp-bsd
 	exit ;;
     9000/8??:4.3bsd:*:*)
@@ -718,7 +763,7 @@ EOF
     *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*)
 	echo hppa1.0-hp-mpeix
 	exit ;;
-    hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* )
+    hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*)
 	echo hppa1.1-hp-osf
 	exit ;;
     hp8??:OSF1:*:*)
@@ -726,9 +771,9 @@ EOF
 	exit ;;
     i*86:OSF1:*:*)
 	if [ -x /usr/sbin/sysversion ] ; then
-	    echo ${UNAME_MACHINE}-unknown-osf1mk
+	    echo "$UNAME_MACHINE"-unknown-osf1mk
 	else
-	    echo ${UNAME_MACHINE}-unknown-osf1
+	    echo "$UNAME_MACHINE"-unknown-osf1
 	fi
 	exit ;;
     parisc*:Lites*:*:*)
@@ -753,127 +798,120 @@ EOF
 	echo c4-convex-bsd
 	exit ;;
     CRAY*Y-MP:*:*:*)
-	echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     CRAY*[A-Z]90:*:*:*)
-	echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \
+	echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \
 	| sed -e 's/CRAY.*\([A-Z]90\)/\1/' \
 	      -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \
 	      -e 's/\.[^.]*$/.X/'
 	exit ;;
     CRAY*TS:*:*:*)
-	echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     CRAY*T3E:*:*:*)
-	echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     CRAY*SV1:*:*:*)
-	echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     *:UNICOS/mp:*:*)
-	echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
-	FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
-	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
-	FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+	FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
+	FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'`
 	echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     5000:UNIX_System_V:4.*:*)
-	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
-	FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
+	FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
 	echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
-	echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE"
 	exit ;;
     sparc*:BSD/OS:*:*)
-	echo sparc-unknown-bsdi${UNAME_RELEASE}
+	echo sparc-unknown-bsdi"$UNAME_RELEASE"
 	exit ;;
     *:BSD/OS:*:*)
-	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE"
+	exit ;;
+    arm*:FreeBSD:*:*)
+	UNAME_PROCESSOR=`uname -p`
+	set_cc_for_build
+	if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
+	    | grep -q __ARM_PCS_VFP
+	then
+	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi
+	else
+	    echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf
+	fi
 	exit ;;
     *:FreeBSD:*:*)
 	UNAME_PROCESSOR=`/usr/bin/uname -p`
-	case ${UNAME_PROCESSOR} in
+	case "$UNAME_PROCESSOR" in
 	    amd64)
-		echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
-	    *)
-		echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+		UNAME_PROCESSOR=x86_64 ;;
+	    i386)
+		UNAME_PROCESSOR=i586 ;;
 	esac
+	echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
 	exit ;;
     i*:CYGWIN*:*)
-	echo ${UNAME_MACHINE}-pc-cygwin
+	echo "$UNAME_MACHINE"-pc-cygwin
 	exit ;;
     *:MINGW64*:*)
-	echo ${UNAME_MACHINE}-pc-mingw64
+	echo "$UNAME_MACHINE"-pc-mingw64
 	exit ;;
     *:MINGW*:*)
-	echo ${UNAME_MACHINE}-pc-mingw32
+	echo "$UNAME_MACHINE"-pc-mingw32
 	exit ;;
-    i*:MSYS*:*)
-	echo ${UNAME_MACHINE}-pc-msys
-	exit ;;
-    i*:windows32*:*)
-	# uname -m includes "-pc" on this system.
-	echo ${UNAME_MACHINE}-mingw32
+    *:MSYS*:*)
+	echo "$UNAME_MACHINE"-pc-msys
 	exit ;;
     i*:PW*:*)
-	echo ${UNAME_MACHINE}-pc-pw32
+	echo "$UNAME_MACHINE"-pc-pw32
 	exit ;;
     *:Interix*:*)
-	case ${UNAME_MACHINE} in
+	case "$UNAME_MACHINE" in
 	    x86)
-		echo i586-pc-interix${UNAME_RELEASE}
+		echo i586-pc-interix"$UNAME_RELEASE"
 		exit ;;
 	    authenticamd | genuineintel | EM64T)
-		echo x86_64-unknown-interix${UNAME_RELEASE}
+		echo x86_64-unknown-interix"$UNAME_RELEASE"
 		exit ;;
 	    IA64)
-		echo ia64-unknown-interix${UNAME_RELEASE}
+		echo ia64-unknown-interix"$UNAME_RELEASE"
 		exit ;;
 	esac ;;
-    [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*)
-	echo i${UNAME_MACHINE}-pc-mks
-	exit ;;
-    8664:Windows_NT:*)
-	echo x86_64-pc-mks
-	exit ;;
-    i*:Windows_NT*:* | Pentium*:Windows_NT*:*)
-	# How do we know it's Interix rather than the generic POSIX subsystem?
-	# It also conflicts with pre-2.0 versions of AT&T UWIN. Should we
-	# UNAME_MACHINE based on the output of uname instead of i386?
-	echo i586-pc-interix
-	exit ;;
     i*:UWIN*:*)
-	echo ${UNAME_MACHINE}-pc-uwin
+	echo "$UNAME_MACHINE"-pc-uwin
 	exit ;;
     amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*)
 	echo x86_64-unknown-cygwin
 	exit ;;
-    p*:CYGWIN*:*)
-	echo powerpcle-unknown-cygwin
-	exit ;;
     prep*:SunOS:5.*:*)
-	echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
 	exit ;;
     *:GNU:*:*)
 	# the GNU system
-	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`"
 	exit ;;
     *:GNU/*:*:*)
 	# other systems with GNU libc and userland
-	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu
+	echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC"
 	exit ;;
-    i*86:Minix:*:*)
-	echo ${UNAME_MACHINE}-pc-minix
+    *:Minix:*:*)
+	echo "$UNAME_MACHINE"-unknown-minix
 	exit ;;
     aarch64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     aarch64_be:Linux:*:*)
 	UNAME_MACHINE=aarch64_be
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     alpha:Linux:*:*)
 	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
@@ -886,63 +924,64 @@ EOF
 	  EV68*) UNAME_MACHINE=alphaev68 ;;
 	esac
 	objdump --private-headers /bin/sh | grep -q ld.so.1
-	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
-	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
+	if test "$?" = 0 ; then LIBC=gnulibc1 ; fi
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
+	exit ;;
+    arc:Linux:*:* | arceb:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     arm*:Linux:*:*)
-	eval $set_cc_for_build
+	set_cc_for_build
 	if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
 	    | grep -q __ARM_EABI__
 	then
-	    echo ${UNAME_MACHINE}-unknown-linux-gnu
+	    echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	else
 	    if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
 		| grep -q __ARM_PCS_VFP
 	    then
-		echo ${UNAME_MACHINE}-unknown-linux-gnueabi
+		echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi
 	    else
-		echo ${UNAME_MACHINE}-unknown-linux-gnueabihf
+		echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf
 	    fi
 	fi
 	exit ;;
     avr32*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     cris:Linux:*:*)
-	echo ${UNAME_MACHINE}-axis-linux-gnu
+	echo "$UNAME_MACHINE"-axis-linux-"$LIBC"
 	exit ;;
     crisv32:Linux:*:*)
-	echo ${UNAME_MACHINE}-axis-linux-gnu
+	echo "$UNAME_MACHINE"-axis-linux-"$LIBC"
+	exit ;;
+    e2k:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     frv:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     hexagon:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     i*86:Linux:*:*)
-	LIBC=gnu
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
-	#ifdef __dietlibc__
-	LIBC=dietlibc
-	#endif
-EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
-	echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
+	echo "$UNAME_MACHINE"-pc-linux-"$LIBC"
 	exit ;;
     ia64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
+	exit ;;
+    k1om:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     m32r*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     m68*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     mips:Linux:*:* | mips64:Linux:*:*)
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
+	set_cc_for_build
+	sed 's/^	//' << EOF > "$dummy.c"
 	#undef CPU
 	#undef ${UNAME_MACHINE}
 	#undef ${UNAME_MACHINE}el
@@ -956,55 +995,70 @@ EOF
 	#endif
 	#endif
 EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
-	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+	eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`"
+	test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; }
 	;;
-    or32:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+    mips64el:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
+	exit ;;
+    openrisc*:Linux:*:*)
+	echo or1k-unknown-linux-"$LIBC"
+	exit ;;
+    or32:Linux:*:* | or1k*:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     padre:Linux:*:*)
-	echo sparc-unknown-linux-gnu
+	echo sparc-unknown-linux-"$LIBC"
 	exit ;;
     parisc64:Linux:*:* | hppa64:Linux:*:*)
-	echo hppa64-unknown-linux-gnu
+	echo hppa64-unknown-linux-"$LIBC"
 	exit ;;
     parisc:Linux:*:* | hppa:Linux:*:*)
 	# Look for CPU level
 	case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
-	  PA7*) echo hppa1.1-unknown-linux-gnu ;;
-	  PA8*) echo hppa2.0-unknown-linux-gnu ;;
-	  *)    echo hppa-unknown-linux-gnu ;;
+	  PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;;
+	  PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;;
+	  *)    echo hppa-unknown-linux-"$LIBC" ;;
 	esac
 	exit ;;
     ppc64:Linux:*:*)
-	echo powerpc64-unknown-linux-gnu
+	echo powerpc64-unknown-linux-"$LIBC"
 	exit ;;
     ppc:Linux:*:*)
-	echo powerpc-unknown-linux-gnu
+	echo powerpc-unknown-linux-"$LIBC"
+	exit ;;
+    ppc64le:Linux:*:*)
+	echo powerpc64le-unknown-linux-"$LIBC"
+	exit ;;
+    ppcle:Linux:*:*)
+	echo powerpcle-unknown-linux-"$LIBC"
+	exit ;;
+    riscv32:Linux:*:* | riscv64:Linux:*:*)
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
-	echo ${UNAME_MACHINE}-ibm-linux
+	echo "$UNAME_MACHINE"-ibm-linux-"$LIBC"
 	exit ;;
     sh64*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     sh*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     sparc:Linux:*:* | sparc64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     tile*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     vax:Linux:*:*)
-	echo ${UNAME_MACHINE}-dec-linux-gnu
+	echo "$UNAME_MACHINE"-dec-linux-"$LIBC"
 	exit ;;
     x86_64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-pc-linux-"$LIBC"
 	exit ;;
     xtensa*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
 	exit ;;
     i*86:DYNIX/ptx:4*:*)
 	# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
@@ -1018,34 +1072,34 @@ EOF
 	# I am not positive that other SVR4 systems won't match this,
 	# I just have to hope.  -- rms.
 	# Use sysv4.2uw... so that sysv4* matches it.
-	echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
+	echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION"
 	exit ;;
     i*86:OS/2:*:*)
 	# If we were able to find `uname', then EMX Unix compatibility
 	# is probably installed.
-	echo ${UNAME_MACHINE}-pc-os2-emx
+	echo "$UNAME_MACHINE"-pc-os2-emx
 	exit ;;
     i*86:XTS-300:*:STOP)
-	echo ${UNAME_MACHINE}-unknown-stop
+	echo "$UNAME_MACHINE"-unknown-stop
 	exit ;;
     i*86:atheos:*:*)
-	echo ${UNAME_MACHINE}-unknown-atheos
+	echo "$UNAME_MACHINE"-unknown-atheos
 	exit ;;
     i*86:syllable:*:*)
-	echo ${UNAME_MACHINE}-pc-syllable
+	echo "$UNAME_MACHINE"-pc-syllable
 	exit ;;
     i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*)
-	echo i386-unknown-lynxos${UNAME_RELEASE}
+	echo i386-unknown-lynxos"$UNAME_RELEASE"
 	exit ;;
     i*86:*DOS:*:*)
-	echo ${UNAME_MACHINE}-pc-msdosdjgpp
+	echo "$UNAME_MACHINE"-pc-msdosdjgpp
 	exit ;;
-    i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*)
-	UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'`
+    i*86:*:4.*:*)
+	UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'`
 	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
-		echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL}
+		echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL"
 	else
-		echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL}
+		echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL"
 	fi
 	exit ;;
     i*86:*:5:[678]*)
@@ -1055,12 +1109,12 @@ EOF
 	    *Pentium)	     UNAME_MACHINE=i586 ;;
 	    *Pent*|*Celeron) UNAME_MACHINE=i686 ;;
 	esac
-	echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}
+	echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}"
 	exit ;;
     i*86:*:3.2:*)
 	if test -f /usr/options/cb.name; then
 		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
-		echo ${UNAME_MACHINE}-pc-isc$UNAME_REL
+		echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL"
 	elif /bin/uname -X 2>/dev/null >/dev/null ; then
 		UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')`
 		(/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486
@@ -1070,9 +1124,9 @@ EOF
 			&& UNAME_MACHINE=i686
 		(/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \
 			&& UNAME_MACHINE=i686
-		echo ${UNAME_MACHINE}-pc-sco$UNAME_REL
+		echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL"
 	else
-		echo ${UNAME_MACHINE}-pc-sysv32
+		echo "$UNAME_MACHINE"-pc-sysv32
 	fi
 	exit ;;
     pc:*:*:*)
@@ -1080,7 +1134,7 @@ EOF
 	# uname -m prints for DJGPP always 'pc', but it prints nothing about
 	# the processor, so we play safe by assuming i586.
 	# Note: whatever this is, it MUST be the same as what config.sub
-	# prints for the "djgpp" host, or else GDB configury will decide that
+	# prints for the "djgpp" host, or else GDB configure will decide that
 	# this is a cross-build.
 	echo i586-pc-msdosdjgpp
 	exit ;;
@@ -1092,9 +1146,9 @@ EOF
 	exit ;;
     i860:*:4.*:*) # i860-SVR4
 	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
-	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	  echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4
 	else # Add other i860-SVR4 vendors below as they are discovered.
-	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	  echo i860-unknown-sysv"$UNAME_RELEASE"  # Unknown i860-SVR4
 	fi
 	exit ;;
     mini*:CTIX:SYS*5:*)
@@ -1114,9 +1168,9 @@ EOF
 	test -r /etc/.relid \
 	&& OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
 	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
-	  && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
+	  && { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
-	  && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+	  && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;;
     3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
 	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
 	  && { echo i486-ncr-sysv4; exit; } ;;
@@ -1125,28 +1179,28 @@ EOF
 	test -r /etc/.relid \
 	    && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
 	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
-	    && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
+	    && { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
-	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; }
+	    && { echo i586-ncr-sysv4.3"$OS_REL"; exit; }
 	/bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \
-	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+	    && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;;
     m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*)
-	echo m68k-unknown-lynxos${UNAME_RELEASE}
+	echo m68k-unknown-lynxos"$UNAME_RELEASE"
 	exit ;;
     mc68030:UNIX_System_V:4.*:*)
 	echo m68k-atari-sysv4
 	exit ;;
     TSUNAMI:LynxOS:2.*:*)
-	echo sparc-unknown-lynxos${UNAME_RELEASE}
+	echo sparc-unknown-lynxos"$UNAME_RELEASE"
 	exit ;;
     rs6000:LynxOS:2.*:*)
-	echo rs6000-unknown-lynxos${UNAME_RELEASE}
+	echo rs6000-unknown-lynxos"$UNAME_RELEASE"
 	exit ;;
     PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*)
-	echo powerpc-unknown-lynxos${UNAME_RELEASE}
+	echo powerpc-unknown-lynxos"$UNAME_RELEASE"
 	exit ;;
     SM[BE]S:UNIX_SV:*:*)
-	echo mips-dde-sysv${UNAME_RELEASE}
+	echo mips-dde-sysv"$UNAME_RELEASE"
 	exit ;;
     RM*:ReliantUNIX-*:*:*)
 	echo mips-sni-sysv4
@@ -1157,7 +1211,7 @@ EOF
     *:SINIX-*:*:*)
 	if uname -p 2>/dev/null >/dev/null ; then
 		UNAME_MACHINE=`(uname -p) 2>/dev/null`
-		echo ${UNAME_MACHINE}-sni-sysv4
+		echo "$UNAME_MACHINE"-sni-sysv4
 	else
 		echo ns32k-sni-sysv
 	fi
@@ -1177,23 +1231,23 @@ EOF
 	exit ;;
     i*86:VOS:*:*)
 	# From Paul.Green@stratus.com.
-	echo ${UNAME_MACHINE}-stratus-vos
+	echo "$UNAME_MACHINE"-stratus-vos
 	exit ;;
     *:VOS:*:*)
 	# From Paul.Green@stratus.com.
 	echo hppa1.1-stratus-vos
 	exit ;;
     mc68*:A/UX:*:*)
-	echo m68k-apple-aux${UNAME_RELEASE}
+	echo m68k-apple-aux"$UNAME_RELEASE"
 	exit ;;
     news*:NEWS-OS:6*:*)
 	echo mips-sony-newsos6
 	exit ;;
     R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
 	if [ -d /usr/nec ]; then
-		echo mips-nec-sysv${UNAME_RELEASE}
+		echo mips-nec-sysv"$UNAME_RELEASE"
 	else
-		echo mips-unknown-sysv${UNAME_RELEASE}
+		echo mips-unknown-sysv"$UNAME_RELEASE"
 	fi
 	exit ;;
     BeBox:BeOS:*:*)	# BeOS running on hardware made by Be, PPC only.
@@ -1212,65 +1266,93 @@ EOF
 	echo x86_64-unknown-haiku
 	exit ;;
     SX-4:SUPER-UX:*:*)
-	echo sx4-nec-superux${UNAME_RELEASE}
+	echo sx4-nec-superux"$UNAME_RELEASE"
 	exit ;;
     SX-5:SUPER-UX:*:*)
-	echo sx5-nec-superux${UNAME_RELEASE}
+	echo sx5-nec-superux"$UNAME_RELEASE"
 	exit ;;
     SX-6:SUPER-UX:*:*)
-	echo sx6-nec-superux${UNAME_RELEASE}
+	echo sx6-nec-superux"$UNAME_RELEASE"
 	exit ;;
     SX-7:SUPER-UX:*:*)
-	echo sx7-nec-superux${UNAME_RELEASE}
+	echo sx7-nec-superux"$UNAME_RELEASE"
 	exit ;;
     SX-8:SUPER-UX:*:*)
-	echo sx8-nec-superux${UNAME_RELEASE}
+	echo sx8-nec-superux"$UNAME_RELEASE"
 	exit ;;
     SX-8R:SUPER-UX:*:*)
-	echo sx8r-nec-superux${UNAME_RELEASE}
+	echo sx8r-nec-superux"$UNAME_RELEASE"
+	exit ;;
+    SX-ACE:SUPER-UX:*:*)
+	echo sxace-nec-superux"$UNAME_RELEASE"
 	exit ;;
     Power*:Rhapsody:*:*)
-	echo powerpc-apple-rhapsody${UNAME_RELEASE}
+	echo powerpc-apple-rhapsody"$UNAME_RELEASE"
 	exit ;;
     *:Rhapsody:*:*)
-	echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE"
 	exit ;;
     *:Darwin:*:*)
 	UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
-	case $UNAME_PROCESSOR in
-	    i386)
-		eval $set_cc_for_build
-		if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
-		  if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
-		      (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
-		      grep IS_64BIT_ARCH >/dev/null
-		  then
-		      UNAME_PROCESSOR="x86_64"
-		  fi
-		fi ;;
-	    unknown) UNAME_PROCESSOR=powerpc ;;
-	esac
-	echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
+	set_cc_for_build
+	if test "$UNAME_PROCESSOR" = unknown ; then
+	    UNAME_PROCESSOR=powerpc
+	fi
+	if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then
+	    if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
+		if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+		       (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		       grep IS_64BIT_ARCH >/dev/null
+		then
+		    case $UNAME_PROCESSOR in
+			i386) UNAME_PROCESSOR=x86_64 ;;
+			powerpc) UNAME_PROCESSOR=powerpc64 ;;
+		    esac
+		fi
+		# On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc
+		if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \
+		       (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		       grep IS_PPC >/dev/null
+		then
+		    UNAME_PROCESSOR=powerpc
+		fi
+	    fi
+	elif test "$UNAME_PROCESSOR" = i386 ; then
+	    # Avoid executing cc on OS X 10.9, as it ships with a stub
+	    # that puts up a graphical alert prompting to install
+	    # developer tools.  Any system running Mac OS X 10.7 or
+	    # later (Darwin 11 and later) is required to have a 64-bit
+	    # processor. This is not true of the ARM version of Darwin
+	    # that Apple uses in portable devices.
+	    UNAME_PROCESSOR=x86_64
+	fi
+	echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE"
 	exit ;;
     *:procnto*:*:* | *:QNX:[0123456789]*:*)
 	UNAME_PROCESSOR=`uname -p`
-	if test "$UNAME_PROCESSOR" = "x86"; then
+	if test "$UNAME_PROCESSOR" = x86; then
 		UNAME_PROCESSOR=i386
 		UNAME_MACHINE=pc
 	fi
-	echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE}
+	echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE"
 	exit ;;
     *:QNX:*:4*)
 	echo i386-pc-qnx
 	exit ;;
-    NEO-?:NONSTOP_KERNEL:*:*)
-	echo neo-tandem-nsk${UNAME_RELEASE}
+    NEO-*:NONSTOP_KERNEL:*:*)
+	echo neo-tandem-nsk"$UNAME_RELEASE"
 	exit ;;
     NSE-*:NONSTOP_KERNEL:*:*)
-	echo nse-tandem-nsk${UNAME_RELEASE}
+	echo nse-tandem-nsk"$UNAME_RELEASE"
+	exit ;;
+    NSR-*:NONSTOP_KERNEL:*:*)
+	echo nsr-tandem-nsk"$UNAME_RELEASE"
 	exit ;;
-    NSR-?:NONSTOP_KERNEL:*:*)
-	echo nsr-tandem-nsk${UNAME_RELEASE}
+    NSV-*:NONSTOP_KERNEL:*:*)
+	echo nsv-tandem-nsk"$UNAME_RELEASE"
+	exit ;;
+    NSX-*:NONSTOP_KERNEL:*:*)
+	echo nsx-tandem-nsk"$UNAME_RELEASE"
 	exit ;;
     *:NonStop-UX:*:*)
 	echo mips-compaq-nonstopux
@@ -1279,18 +1361,19 @@ EOF
 	echo bs2000-siemens-sysv
 	exit ;;
     DS/*:UNIX_System_V:*:*)
-	echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE}
+	echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE"
 	exit ;;
     *:Plan9:*:*)
 	# "uname -m" is not consistent, so use $cputype instead. 386
 	# is converted to i386 for consistency with other x86
 	# operating systems.
-	if test "$cputype" = "386"; then
+	# shellcheck disable=SC2154
+	if test "$cputype" = 386; then
 	    UNAME_MACHINE=i386
 	else
 	    UNAME_MACHINE="$cputype"
 	fi
-	echo ${UNAME_MACHINE}-unknown-plan9
+	echo "$UNAME_MACHINE"-unknown-plan9
 	exit ;;
     *:TOPS-10:*:*)
 	echo pdp10-unknown-tops10
@@ -1311,14 +1394,14 @@ EOF
 	echo pdp10-unknown-its
 	exit ;;
     SEI:*:*:SEIUX)
-	echo mips-sei-seiux${UNAME_RELEASE}
+	echo mips-sei-seiux"$UNAME_RELEASE"
 	exit ;;
     *:DragonFly:*:*)
-	echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
 	exit ;;
     *:*VMS:*:*)
 	UNAME_MACHINE=`(uname -p) 2>/dev/null`
-	case "${UNAME_MACHINE}" in
+	case "$UNAME_MACHINE" in
 	    A*) echo alpha-dec-vms ; exit ;;
 	    I*) echo ia64-dec-vms ; exit ;;
 	    V*) echo vax-dec-vms ; exit ;;
@@ -1327,182 +1410,48 @@ EOF
 	echo i386-pc-xenix
 	exit ;;
     i*86:skyos:*:*)
-	echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
+	echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`"
 	exit ;;
     i*86:rdos:*:*)
-	echo ${UNAME_MACHINE}-pc-rdos
+	echo "$UNAME_MACHINE"-pc-rdos
 	exit ;;
     i*86:AROS:*:*)
-	echo ${UNAME_MACHINE}-pc-aros
+	echo "$UNAME_MACHINE"-pc-aros
 	exit ;;
     x86_64:VMkernel:*:*)
-	echo ${UNAME_MACHINE}-unknown-esx
+	echo "$UNAME_MACHINE"-unknown-esx
+	exit ;;
+    amd64:Isilon\ OneFS:*:*)
+	echo x86_64-unknown-onefs
 	exit ;;
 esac
 
-eval $set_cc_for_build
-cat >$dummy.c <<EOF
-#ifdef _SEQUENT_
-# include <sys/types.h>
-# include <sys/utsname.h>
-#endif
-main ()
-{
-#if defined (sony)
-#if defined (MIPSEB)
-  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
-     I don't know....  */
-  printf ("mips-sony-bsd\n"); exit (0);
-#else
-#include <sys/param.h>
-  printf ("m68k-sony-newsos%s\n",
-#ifdef NEWSOS4
-	"4"
-#else
-	""
-#endif
-	); exit (0);
-#endif
-#endif
-
-#if defined (__arm) && defined (__acorn) && defined (__unix)
-  printf ("arm-acorn-riscix\n"); exit (0);
-#endif
+echo "$0: unable to guess system type" >&2
 
-#if defined (hp300) && !defined (hpux)
-  printf ("m68k-hp-bsd\n"); exit (0);
-#endif
+case "$UNAME_MACHINE:$UNAME_SYSTEM" in
+    mips:Linux | mips64:Linux)
+	# If we got here on MIPS GNU/Linux, output extra information.
+	cat >&2 <<EOF
 
-#if defined (NeXT)
-#if !defined (__ARCHITECTURE__)
-#define __ARCHITECTURE__ "m68k"
-#endif
-  int version;
-  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
-  if (version < 4)
-    printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
-  else
-    printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
-  exit (0);
-#endif
-
-#if defined (MULTIMAX) || defined (n16)
-#if defined (UMAXV)
-  printf ("ns32k-encore-sysv\n"); exit (0);
-#else
-#if defined (CMU)
-  printf ("ns32k-encore-mach\n"); exit (0);
-#else
-  printf ("ns32k-encore-bsd\n"); exit (0);
-#endif
-#endif
-#endif
-
-#if defined (__386BSD__)
-  printf ("i386-pc-bsd\n"); exit (0);
-#endif
-
-#if defined (sequent)
-#if defined (i386)
-  printf ("i386-sequent-dynix\n"); exit (0);
-#endif
-#if defined (ns32000)
-  printf ("ns32k-sequent-dynix\n"); exit (0);
-#endif
-#endif
-
-#if defined (_SEQUENT_)
-    struct utsname un;
-
-    uname(&un);
-
-    if (strncmp(un.version, "V2", 2) == 0) {
-	printf ("i386-sequent-ptx2\n"); exit (0);
-    }
-    if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
-	printf ("i386-sequent-ptx1\n"); exit (0);
-    }
-    printf ("i386-sequent-ptx\n"); exit (0);
-
-#endif
-
-#if defined (vax)
-# if !defined (ultrix)
-#  include <sys/param.h>
-#  if defined (BSD)
-#   if BSD == 43
-      printf ("vax-dec-bsd4.3\n"); exit (0);
-#   else
-#    if BSD == 199006
-      printf ("vax-dec-bsd4.3reno\n"); exit (0);
-#    else
-      printf ("vax-dec-bsd\n"); exit (0);
-#    endif
-#   endif
-#  else
-    printf ("vax-dec-bsd\n"); exit (0);
-#  endif
-# else
-    printf ("vax-dec-ultrix\n"); exit (0);
-# endif
-#endif
-
-#if defined (alliant) && defined (i860)
-  printf ("i860-alliant-bsd\n"); exit (0);
-#endif
-
-  exit (1);
-}
+NOTE: MIPS GNU/Linux systems require a C compiler to fully recognize
+the system type. Please install a C compiler and try again.
 EOF
-
-$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` &&
-	{ echo "$SYSTEM_NAME"; exit; }
-
-# Apollos put the system type in the environment.
-
-test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; }
-
-# Convex versions that predate uname can use getsysinfo(1)
-
-if [ -x /usr/convex/getsysinfo ]
-then
-    case `getsysinfo -f cpu_type` in
-    c1*)
-	echo c1-convex-bsd
-	exit ;;
-    c2*)
-	if getsysinfo -f scalar_acc
-	then echo c32-convex-bsd
-	else echo c2-convex-bsd
-	fi
-	exit ;;
-    c34*)
-	echo c34-convex-bsd
-	exit ;;
-    c38*)
-	echo c38-convex-bsd
-	exit ;;
-    c4*)
-	echo c4-convex-bsd
-	exit ;;
-    esac
-fi
+	;;
+esac
 
 cat >&2 <<EOF
-$0: unable to guess system type
 
-This script, last modified $timestamp, has failed to recognize
-the operating system you are using. It is advised that you
-download the most up to date version of the config scripts from
+This script (version $timestamp), has failed to recognize the
+operating system you are using. If your script is old, overwrite *all*
+copies of config.guess and config.sub with the latest versions from:
 
-  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+  https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
 and
-  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+  https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
 
-If the version you run ($0) is already up to date, please
-send the following data and any information you think might be
-pertinent to <config-patches@gnu.org> in order to provide the needed
-information to handle your system.
+If $0 has already been updated, send the following data and any
+information you think might be pertinent to config-patches@gnu.org to
+provide the necessary information to handle your system.
 
 config.guess timestamp = $timestamp
 
@@ -1521,16 +1470,16 @@ hostinfo               = `(hostinfo) 2>/dev/null`
 /usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null`
 /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null`
 
-UNAME_MACHINE = ${UNAME_MACHINE}
-UNAME_RELEASE = ${UNAME_RELEASE}
-UNAME_SYSTEM  = ${UNAME_SYSTEM}
-UNAME_VERSION = ${UNAME_VERSION}
+UNAME_MACHINE = "$UNAME_MACHINE"
+UNAME_RELEASE = "$UNAME_RELEASE"
+UNAME_SYSTEM  = "$UNAME_SYSTEM"
+UNAME_VERSION = "$UNAME_VERSION"
 EOF
 
 exit 1
 
 # Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
+# eval: (add-hook 'before-save-hook 'time-stamp)
 # time-stamp-start: "timestamp='"
 # time-stamp-format: "%:y-%02m-%02d"
 # time-stamp-end: "'"
diff --git a/config/config.sub b/config/config.sub
index bdda9e4a32c..c19e671805a 100755
--- a/config/config.sub
+++ b/config/config.sub
@@ -1,36 +1,31 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-#   2011, 2012 Free Software Foundation, Inc.
+#   Copyright 1992-2018 Free Software Foundation, Inc.
 
-timestamp='2012-08-18'
+timestamp='2018-08-13'
 
-# This file is (in principle) common to ALL GNU software.
-# The presence of a machine in this file suggests that SOME GNU software
-# can handle that machine.  It does not imply ALL GNU software can.
-#
-# This file is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
+# along with this program; if not, see <https://www.gnu.org/licenses/>.
 #
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 
 
-# Please send patches to <config-patches@gnu.org>.  Submit a context
-# diff and a properly formatted GNU ChangeLog entry.
+# Please send patches to <config-patches@gnu.org>.
 #
 # Configuration subroutine to validate and canonicalize a configuration type.
 # Supply the specified configuration type as an argument.
@@ -38,7 +33,7 @@ timestamp='2012-08-18'
 # Otherwise, we print the canonical config type on stdout and succeed.
 
 # You can get the latest version of this script from:
-# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
 
 # This file is supposed to be the same for all GNU packages
 # and recognize all the CPU types, system types and aliases
@@ -58,12 +53,11 @@ timestamp='2012-08-18'
 me=`echo "$0" | sed -e 's,.*/,,'`
 
 usage="\
-Usage: $0 [OPTION] CPU-MFR-OPSYS
-       $0 [OPTION] ALIAS
+Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
 
 Canonicalize a configuration name.
 
-Operation modes:
+Options:
   -h, --help         print this help, then exit
   -t, --time-stamp   print date of last modification, then exit
   -v, --version      print version number, then exit
@@ -73,9 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
-Free Software Foundation, Inc.
+Copyright 1992-2018 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -102,7 +94,7 @@ while test $# -gt 0 ; do
 
     *local*)
        # First pass through any local machine types.
-       echo $1
+       echo "$1"
        exit ;;
 
     * )
@@ -118,162 +110,596 @@ case $# in
     exit 1;;
 esac
 
-# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any).
-# Here we must recognize all the valid KERNEL-OS combinations.
-maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
-case $maybe_os in
-  nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
-  linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
-  knetbsd*-gnu* | netbsd*-gnu* | \
-  kopensolaris*-gnu* | \
-  storm-chaos* | os2-emx* | rtmk-nova*)
-    os=-$maybe_os
-    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
-    ;;
-  android-linux)
-    os=-linux-android
-    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown
-    ;;
-  *)
-    basic_machine=`echo $1 | sed 's/-[^-]*$//'`
-    if [ $basic_machine != $1 ]
-    then os=`echo $1 | sed 's/.*-/-/'`
-    else os=; fi
-    ;;
-esac
+# Split fields of configuration type
+IFS="-" read -r field1 field2 field3 field4 <<EOF
+$1
+EOF
 
-### Let's recognize common machines as not being operating systems so
-### that things like config.sub decstation-3100 work.  We also
-### recognize some manufacturers as not being operating systems, so we
-### can provide default operating systems below.
-case $os in
-	-sun*os*)
-		# Prevent following clause from handling this invalid input.
-		;;
-	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
-	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
-	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
-	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
-	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
-	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-	-apple | -axis | -knuth | -cray | -microblaze)
-		os=
-		basic_machine=$1
-		;;
-	-bluegene*)
-		os=-cnk
-		;;
-	-sim | -cisco | -oki | -wec | -winbond)
-		os=
-		basic_machine=$1
-		;;
-	-scout)
+# Separate into logical components for further validation
+case $1 in
+	*-*-*-*-*)
+		echo Invalid configuration \`"$1"\': more than four components >&2
+		exit 1
 		;;
-	-wrs)
-		os=-vxworks
-		basic_machine=$1
+	*-*-*-*)
+		basic_machine=$field1-$field2
+		os=$field3-$field4
 		;;
-	-chorusos*)
-		os=-chorusos
-		basic_machine=$1
+	*-*-*)
+		# Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two
+		# parts
+		maybe_os=$field2-$field3
+		case $maybe_os in
+			nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \
+			| linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \
+			| uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \
+			| netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \
+			| storm-chaos* | os2-emx* | rtmk-nova*)
+				basic_machine=$field1
+				os=$maybe_os
+				;;
+			android-linux)
+				basic_machine=$field1-unknown
+				os=linux-android
+				;;
+			*)
+				basic_machine=$field1-$field2
+				os=$field3
+				;;
+		esac
 		;;
-	-chorusrdb)
-		os=-chorusrdb
-		basic_machine=$1
+	*-*)
+		# Second component is usually, but not always the OS
+		case $field2 in
+			# Prevent following clause from handling this valid os
+			sun*os*)
+				basic_machine=$field1
+				os=$field2
+				;;
+			# Manufacturers
+			dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \
+			| att* | 7300* | 3300* | delta* | motorola* | sun[234]* \
+			| unicom* | ibm* | next | hp | isi* | apollo | altos* \
+			| convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \
+			| c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \
+			| harris | dolphin | highlevel | gould | cbm | ns | masscomp \
+			| apple | axis | knuth | cray | microblaze* \
+			| sim | cisco | oki | wec | wrs | winbond)
+				basic_machine=$field1-$field2
+				os=
+				;;
+			*)
+				basic_machine=$field1
+				os=$field2
+				;;
+		esac
 		;;
-	-hiux*)
-		os=-hiuxwe2
+	*)
+		# Convert single-component short-hands not valid as part of
+		# multi-component configurations.
+		case $field1 in
+			386bsd)
+				basic_machine=i386-pc
+				os=bsd
+				;;
+			a29khif)
+				basic_machine=a29k-amd
+				os=udi
+				;;
+			adobe68k)
+				basic_machine=m68010-adobe
+				os=scout
+				;;
+			alliant)
+				basic_machine=fx80-alliant
+				os=
+				;;
+			altos | altos3068)
+				basic_machine=m68k-altos
+				os=
+				;;
+			am29k)
+				basic_machine=a29k-none
+				os=bsd
+				;;
+			amdahl)
+				basic_machine=580-amdahl
+				os=sysv
+				;;
+			amigaos | amigados)
+				basic_machine=m68k-unknown
+				os=amigaos
+				;;
+			amigaunix | amix)
+				basic_machine=m68k-unknown
+				os=sysv4
+				;;
+			apollo68)
+				basic_machine=m68k-apollo
+				os=sysv
+				;;
+			apollo68bsd)
+				basic_machine=m68k-apollo
+				os=bsd
+				;;
+			aros)
+				basic_machine=i386-pc
+				os=aros
+				;;
+			aux)
+				basic_machine=m68k-apple
+				os=aux
+				;;
+			balance)
+				basic_machine=ns32k-sequent
+				os=dynix
+				;;
+			blackfin)
+				basic_machine=bfin-unknown
+				os=linux
+				;;
+			cegcc)
+				basic_machine=arm-unknown
+				os=cegcc
+				;;
+			convex-c1)
+				basic_machine=c1-convex
+				os=bsd
+				;;
+			convex-c2)
+				basic_machine=c2-convex
+				os=bsd
+				;;
+			convex-c32)
+				basic_machine=c32-convex
+				os=bsd
+				;;
+			convex-c34)
+				basic_machine=c34-convex
+				os=bsd
+				;;
+			convex-c38)
+				basic_machine=c38-convex
+				os=bsd
+				;;
+			cray)
+				basic_machine=j90-cray
+				os=unicos
+				;;
+			crds | unos)
+				basic_machine=m68k-crds
+				os=
+				;;
+			delta88)
+				basic_machine=m88k-motorola
+				os=sysv3
+				;;
+			dicos)
+				basic_machine=i686-pc
+				os=dicos
+				;;
+			djgpp)
+				basic_machine=i586-pc
+				os=msdosdjgpp
+				;;
+			ebmon29k)
+				basic_machine=a29k-amd
+				os=ebmon
+				;;
+			es1800 | OSE68k | ose68k | ose | OSE)
+				basic_machine=m68k-ericsson
+				os=ose
+				;;
+			gmicro)
+				basic_machine=tron-gmicro
+				os=sysv
+				;;
+			go32)
+				basic_machine=i386-pc
+				os=go32
+				;;
+			h8300hms)
+				basic_machine=h8300-hitachi
+				os=hms
+				;;
+			h8300xray)
+				basic_machine=h8300-hitachi
+				os=xray
+				;;
+			h8500hms)
+				basic_machine=h8500-hitachi
+				os=hms
+				;;
+			harris)
+				basic_machine=m88k-harris
+				os=sysv3
+				;;
+			hp300bsd)
+				basic_machine=m68k-hp
+				os=bsd
+				;;
+			hp300hpux)
+				basic_machine=m68k-hp
+				os=hpux
+				;;
+			hppaosf)
+				basic_machine=hppa1.1-hp
+				os=osf
+				;;
+			hppro)
+				basic_machine=hppa1.1-hp
+				os=proelf
+				;;
+			i386mach)
+				basic_machine=i386-mach
+				os=mach
+				;;
+			vsta)
+				basic_machine=i386-pc
+				os=vsta
+				;;
+			isi68 | isi)
+				basic_machine=m68k-isi
+				os=sysv
+				;;
+			m68knommu)
+				basic_machine=m68k-unknown
+				os=linux
+				;;
+			magnum | m3230)
+				basic_machine=mips-mips
+				os=sysv
+				;;
+			merlin)
+				basic_machine=ns32k-utek
+				os=sysv
+				;;
+			mingw64)
+				basic_machine=x86_64-pc
+				os=mingw64
+				;;
+			mingw32)
+				basic_machine=i686-pc
+				os=mingw32
+				;;
+			mingw32ce)
+				basic_machine=arm-unknown
+				os=mingw32ce
+				;;
+			monitor)
+				basic_machine=m68k-rom68k
+				os=coff
+				;;
+			morphos)
+				basic_machine=powerpc-unknown
+				os=morphos
+				;;
+			moxiebox)
+				basic_machine=moxie-unknown
+				os=moxiebox
+				;;
+			msdos)
+				basic_machine=i386-pc
+				os=msdos
+				;;
+			msys)
+				basic_machine=i686-pc
+				os=msys
+				;;
+			mvs)
+				basic_machine=i370-ibm
+				os=mvs
+				;;
+			nacl)
+				basic_machine=le32-unknown
+				os=nacl
+				;;
+			ncr3000)
+				basic_machine=i486-ncr
+				os=sysv4
+				;;
+			netbsd386)
+				basic_machine=i386-pc
+				os=netbsd
+				;;
+			netwinder)
+				basic_machine=armv4l-rebel
+				os=linux
+				;;
+			news | news700 | news800 | news900)
+				basic_machine=m68k-sony
+				os=newsos
+				;;
+			news1000)
+				basic_machine=m68030-sony
+				os=newsos
+				;;
+			necv70)
+				basic_machine=v70-nec
+				os=sysv
+				;;
+			nh3000)
+				basic_machine=m68k-harris
+				os=cxux
+				;;
+			nh[45]000)
+				basic_machine=m88k-harris
+				os=cxux
+				;;
+			nindy960)
+				basic_machine=i960-intel
+				os=nindy
+				;;
+			mon960)
+				basic_machine=i960-intel
+				os=mon960
+				;;
+			nonstopux)
+				basic_machine=mips-compaq
+				os=nonstopux
+				;;
+			os400)
+				basic_machine=powerpc-ibm
+				os=os400
+				;;
+			OSE68000 | ose68000)
+				basic_machine=m68000-ericsson
+				os=ose
+				;;
+			os68k)
+				basic_machine=m68k-none
+				os=os68k
+				;;
+			paragon)
+				basic_machine=i860-intel
+				os=osf
+				;;
+			parisc)
+				basic_machine=hppa-unknown
+				os=linux
+				;;
+			pw32)
+				basic_machine=i586-unknown
+				os=pw32
+				;;
+			rdos | rdos64)
+				basic_machine=x86_64-pc
+				os=rdos
+				;;
+			rdos32)
+				basic_machine=i386-pc
+				os=rdos
+				;;
+			rom68k)
+				basic_machine=m68k-rom68k
+				os=coff
+				;;
+			sa29200)
+				basic_machine=a29k-amd
+				os=udi
+				;;
+			sei)
+				basic_machine=mips-sei
+				os=seiux
+				;;
+			sps7)
+				basic_machine=m68k-bull
+				os=sysv2
+				;;
+			st2000)
+				basic_machine=m68k-tandem
+				os=
+				;;
+			stratus)
+				basic_machine=i860-stratus
+				os=sysv4
+				;;
+			sun2)
+				basic_machine=m68000-sun
+				os=
+				;;
+			sun2os3)
+				basic_machine=m68000-sun
+				os=sunos3
+				;;
+			sun2os4)
+				basic_machine=m68000-sun
+				os=sunos4
+				;;
+			sun3)
+				basic_machine=m68k-sun
+				os=
+				;;
+			sun3os3)
+				basic_machine=m68k-sun
+				os=sunos3
+				;;
+			sun3os4)
+				basic_machine=m68k-sun
+				os=sunos4
+				;;
+			sun4)
+				basic_machine=sparc-sun
+				os=
+				;;
+			sun4os3)
+				basic_machine=sparc-sun
+				os=sunos3
+				;;
+			sun4os4)
+				basic_machine=sparc-sun
+				os=sunos4
+				;;
+			sun4sol2)
+				basic_machine=sparc-sun
+				os=solaris2
+				;;
+			sun386 | sun386i | roadrunner)
+				basic_machine=i386-sun
+				os=
+				;;
+			sv1)
+				basic_machine=sv1-cray
+				os=unicos
+				;;
+			symmetry)
+				basic_machine=i386-sequent
+				os=dynix
+				;;
+			t3e)
+				basic_machine=alphaev5-cray
+				os=unicos
+				;;
+			t90)
+				basic_machine=t90-cray
+				os=unicos
+				;;
+			toad1)
+				basic_machine=pdp10-xkl
+				os=tops20
+				;;
+			tpf)
+				basic_machine=s390x-ibm
+				os=tpf
+				;;
+			udi29k)
+				basic_machine=a29k-amd
+				os=udi
+				;;
+			ultra3)
+				basic_machine=a29k-nyu
+				os=sym1
+				;;
+			v810 | necv810)
+				basic_machine=v810-nec
+				os=none
+				;;
+			vaxv)
+				basic_machine=vax-dec
+				os=sysv
+				;;
+			vms)
+				basic_machine=vax-dec
+				os=vms
+				;;
+			vxworks960)
+				basic_machine=i960-wrs
+				os=vxworks
+				;;
+			vxworks68)
+				basic_machine=m68k-wrs
+				os=vxworks
+				;;
+			vxworks29k)
+				basic_machine=a29k-wrs
+				os=vxworks
+				;;
+			xbox)
+				basic_machine=i686-pc
+				os=mingw32
+				;;
+			ymp)
+				basic_machine=ymp-cray
+				os=unicos
+				;;
+			*)
+				basic_machine=$1
+				os=
+				;;
+		esac
 		;;
-	-sco6)
-		os=-sco5v6
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Here we handle the default manufacturer of certain CPU types.  It is in
+	# some cases the only manufacturer, in others, it is the most popular.
+	craynv)
+		basic_machine=craynv-cray
+		os=${os:-unicosmp}
 		;;
-	-sco5)
-		os=-sco3.2v5
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	fx80)
+		basic_machine=fx80-alliant
 		;;
-	-sco4)
-		os=-sco3.2v4
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	w89k)
+		basic_machine=hppa1.1-winbond
 		;;
-	-sco3.2.[4-9]*)
-		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	op50n)
+		basic_machine=hppa1.1-oki
 		;;
-	-sco3.2v[4-9]*)
-		# Don't forget version if it is 3.2v4 or newer.
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	op60c)
+		basic_machine=hppa1.1-oki
 		;;
-	-sco5v6*)
-		# Don't forget version if it is 3.2v4 or newer.
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	romp)
+		basic_machine=romp-ibm
 		;;
-	-sco*)
-		os=-sco3.2v2
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	mmix)
+		basic_machine=mmix-knuth
 		;;
-	-udk*)
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	rs6000)
+		basic_machine=rs6000-ibm
 		;;
-	-isc)
-		os=-isc2.2
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	vax)
+		basic_machine=vax-dec
 		;;
-	-clix*)
-		basic_machine=clipper-intergraph
+	pdp11)
+		basic_machine=pdp11-dec
 		;;
-	-isc*)
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+	we32k)
+		basic_machine=we32k-att
 		;;
-	-lynx*178)
-		os=-lynxos178
+	cydra)
+		basic_machine=cydra-cydrome
 		;;
-	-lynx*5)
-		os=-lynxos5
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
 		;;
-	-lynx*)
-		os=-lynxos
+	orion)
+		basic_machine=orion-highlevel
 		;;
-	-ptx*)
-		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+	orion105)
+		basic_machine=clipper-highlevel
 		;;
-	-windowsnt*)
-		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+	mac | mpw | mac-mpw)
+		basic_machine=m68k-apple
 		;;
-	-psos*)
-		os=-psos
+	pmac | pmac-mpw)
+		basic_machine=powerpc-apple
 		;;
-	-mint | -mint[0-9]*)
-		basic_machine=m68k-atari
-		os=-mint
+	xps | xps100)
+		basic_machine=xps100-honeywell
 		;;
-esac
 
-# Decode aliases for certain CPU-COMPANY combinations.
-case $basic_machine in
 	# Recognize the basic CPU types without company name.
 	# Some are omitted here because they have special meanings below.
 	1750a | 580 \
 	| a29k \
 	| aarch64 | aarch64_be \
+	| abacus \
 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
 	| am33_2.0 \
-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \
-        | be32 | be64 \
+	| arc | arceb \
+	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \
+	| avr | avr32 \
+	| asmjs \
+	| ba \
+	| be32 | be64 \
 	| bfin \
-	| c4x | clipper \
+	| c4x | c8051 | clipper | csky \
 	| d10v | d30v | dlx | dsp16xx \
-	| epiphany \
-	| fido | fr30 | frv \
+	| e2k | epiphany \
+	| fido | fr30 | frv | ft32 \
 	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
 	| hexagon \
-	| i370 | i860 | i960 | ia64 \
+	| i370 | i860 | i960 | ia16 | ia64 \
 	| ip2k | iq2000 \
+	| k1om \
 	| le32 | le64 \
 	| lm32 \
 	| m32c | m32r | m32rle | m68000 | m68k | m88k \
-	| maxq | mb | microblaze | mcore | mep | metag \
+	| m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \
+	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
 	| mips | mipsbe | mipseb | mipsel | mipsle \
 	| mips16 \
 	| mips64 | mips64el \
@@ -287,26 +713,31 @@ case $basic_machine in
 	| mips64vr5900 | mips64vr5900el \
 	| mipsisa32 | mipsisa32el \
 	| mipsisa32r2 | mipsisa32r2el \
+	| mipsisa32r6 | mipsisa32r6el \
 	| mipsisa64 | mipsisa64el \
 	| mipsisa64r2 | mipsisa64r2el \
+	| mipsisa64r6 | mipsisa64r6el \
 	| mipsisa64sb1 | mipsisa64sb1el \
 	| mipsisa64sr71k | mipsisa64sr71kel \
+	| mipsr5900 | mipsr5900el \
 	| mipstx39 | mipstx39el \
 	| mn10200 | mn10300 \
 	| moxie \
 	| mt \
 	| msp430 \
 	| nds32 | nds32le | nds32be \
-	| nios | nios2 \
+	| nfp \
+	| nios | nios2 | nios2eb | nios2el \
 	| ns16k | ns32k \
-	| open8 \
-	| or32 \
-	| pdp10 | pdp11 | pj | pjl \
+	| open8 | or1k | or1knd | or32 \
+	| pdp10 | pj | pjl \
 	| powerpc | powerpc64 | powerpc64le | powerpcle \
+	| pru \
 	| pyramid \
+	| riscv | riscv32 | riscv64 \
 	| rl78 | rx \
 	| score \
-	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
+	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \
 	| sh64 | sh64le \
 	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
 	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
@@ -314,8 +745,9 @@ case $basic_machine in
 	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
 	| ubicom32 \
 	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
-	| we32k \
-	| x86 | xc16x | xstormy16 | xtensa \
+	| visium \
+	| wasm32 \
+	| x86 | xc16x | xstormy16 | xgate | xtensa \
 	| z8k | z80)
 		basic_machine=$basic_machine-unknown
 		;;
@@ -328,23 +760,23 @@ case $basic_machine in
 	c6x)
 		basic_machine=tic6x-unknown
 		;;
-	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip)
-		basic_machine=$basic_machine-unknown
-		os=-none
+	leon|leon[3-9])
+		basic_machine=sparc-$basic_machine
+		;;
+	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65)
+		;;
+	m9s12z | m68hcs12z | hcs12z | s12z)
+		basic_machine=s12z-unknown
 		;;
-	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
+	m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*)
+		basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	ms1)
 		basic_machine=mt-unknown
 		;;
-
 	strongarm | thumb | xscale)
 		basic_machine=arm-unknown
 		;;
-	xgate)
-		basic_machine=$basic_machine-unknown
-		os=-none
-		;;
 	xscaleeb)
 		basic_machine=armeb-unknown
 		;;
@@ -359,37 +791,40 @@ case $basic_machine in
 	i*86 | x86_64)
 	  basic_machine=$basic_machine-pc
 	  ;;
-	# Object if more than one company name word.
-	*-*-*)
-		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
-		exit 1
-		;;
 	# Recognize the basic CPU types with company name.
-	580-* \
+	1750a-* | 580-* \
 	| a29k-* \
 	| aarch64-* | aarch64_be-* \
+	| abacus-* \
 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
-	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
-	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
+	| alphapca5[67]-* | alpha64pca5[67]-* \
+	| am33_2.0-* \
+	| arc-* | arceb-* \
+	| arm-*  | arm[lb]e-* | arme[lb]-* | armv*-* \
 	| avr-* | avr32-* \
+	| asmjs-* \
+	| ba-* \
 	| be32-* | be64-* \
 	| bfin-* | bs2000-* \
 	| c[123]* | c30-* | [cjt]90-* | c4x-* \
-	| clipper-* | craynv-* | cydra-* \
-	| d10v-* | d30v-* | dlx-* \
-	| elxsi-* \
-	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
+	| c8051-* | clipper-* | craynv-* | csky-* | cydra-* \
+	| d10v-* | d30v-* | dlx-* | dsp16xx-* \
+	| e2k-* | elxsi-* | epiphany-* \
+	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \
 	| h8300-* | h8500-* \
 	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
 	| hexagon-* \
-	| i*86-* | i860-* | i960-* | ia64-* \
+	| i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \
 	| ip2k-* | iq2000-* \
+	| k1om-* \
 	| le32-* | le64-* \
 	| lm32-* \
 	| m32c-* | m32r-* | m32rle-* \
-	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
-	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \
+	| m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \
+	| m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \
+	| m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \
+	| microblaze-* | microblazeel-* \
 	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
 	| mips16-* \
 	| mips64-* | mips64el-* \
@@ -403,37 +838,50 @@ case $basic_machine in
 	| mips64vr5900-* | mips64vr5900el-* \
 	| mipsisa32-* | mipsisa32el-* \
 	| mipsisa32r2-* | mipsisa32r2el-* \
+	| mipsisa32r6-* | mipsisa32r6el-* \
 	| mipsisa64-* | mipsisa64el-* \
 	| mipsisa64r2-* | mipsisa64r2el-* \
+	| mipsisa64r6-* | mipsisa64r6el-* \
 	| mipsisa64sb1-* | mipsisa64sb1el-* \
 	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
+	| mipsr5900-* | mipsr5900el-* \
 	| mipstx39-* | mipstx39el-* \
 	| mmix-* \
+	| mn10200-* | mn10300-* \
+	| moxie-* \
 	| mt-* \
 	| msp430-* \
 	| nds32-* | nds32le-* | nds32be-* \
-	| nios-* | nios2-* \
+	| nfp-* \
+	| nios-* | nios2-* | nios2eb-* | nios2el-* \
 	| none-* | np1-* | ns16k-* | ns32k-* \
 	| open8-* \
+	| or1k*-* \
+	| or32-* \
 	| orion-* \
 	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
 	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
+	| pru-* \
 	| pyramid-* \
+	| riscv-* | riscv32-* | riscv64-* \
 	| rl78-* | romp-* | rs6000-* | rx-* \
-	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
-	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
+	| score-* \
+	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \
+	| sh[1234]e[lb]-* |  sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \
 	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
 	| sparclite-* \
-	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
+	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
+	| spu-* \
 	| tahoe-* \
 	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
-	| tile*-* \
 	| tron-* \
 	| ubicom32-* \
 	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
 	| vax-* \
+	| visium-* \
+	| wasm32-* \
 	| we32k-* \
-	| x86-* | x86_64-* | xc16x-* | xps100-* \
+	| x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \
 	| xstormy16-* | xtensa*-* \
 	| ymp-* \
 	| z8k-* | z80-*)
@@ -444,141 +892,45 @@ case $basic_machine in
 		;;
 	# Recognize the various machine names and aliases which stand
 	# for a CPU type and a company and sometimes even an OS.
-	386bsd)
-		basic_machine=i386-unknown
-		os=-bsd
-		;;
 	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
 		basic_machine=m68000-att
 		;;
 	3b*)
 		basic_machine=we32k-att
 		;;
-	a29khif)
-		basic_machine=a29k-amd
-		os=-udi
-		;;
-	abacus)
-		basic_machine=abacus-unknown
-		;;
-	adobe68k)
-		basic_machine=m68010-adobe
-		os=-scout
-		;;
-	alliant | fx80)
-		basic_machine=fx80-alliant
-		;;
-	altos | altos3068)
-		basic_machine=m68k-altos
-		;;
-	am29k)
-		basic_machine=a29k-none
-		os=-bsd
-		;;
 	amd64)
 		basic_machine=x86_64-pc
 		;;
 	amd64-*)
-		basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'`
-		;;
-	amdahl)
-		basic_machine=580-amdahl
-		os=-sysv
+		basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	amiga | amiga-*)
 		basic_machine=m68k-unknown
 		;;
-	amigaos | amigados)
-		basic_machine=m68k-unknown
-		os=-amigaos
-		;;
-	amigaunix | amix)
-		basic_machine=m68k-unknown
-		os=-sysv4
-		;;
-	apollo68)
-		basic_machine=m68k-apollo
-		os=-sysv
-		;;
-	apollo68bsd)
-		basic_machine=m68k-apollo
-		os=-bsd
-		;;
-	aros)
-		basic_machine=i386-pc
-		os=-aros
-		;;
-	aux)
-		basic_machine=m68k-apple
-		os=-aux
-		;;
-	balance)
-		basic_machine=ns32k-sequent
-		os=-dynix
-		;;
-	blackfin)
-		basic_machine=bfin-unknown
-		os=-linux
-		;;
 	blackfin-*)
-		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
-		os=-linux
+		basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+		os=linux
 		;;
 	bluegene*)
 		basic_machine=powerpc-ibm
-		os=-cnk
+		os=cnk
 		;;
 	c54x-*)
-		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	c55x-*)
-		basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	c6x-*)
-		basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	c90)
 		basic_machine=c90-cray
-		os=-unicos
-		;;
-	cegcc)
-		basic_machine=arm-unknown
-		os=-cegcc
-		;;
-	convex-c1)
-		basic_machine=c1-convex
-		os=-bsd
-		;;
-	convex-c2)
-		basic_machine=c2-convex
-		os=-bsd
-		;;
-	convex-c32)
-		basic_machine=c32-convex
-		os=-bsd
-		;;
-	convex-c34)
-		basic_machine=c34-convex
-		os=-bsd
-		;;
-	convex-c38)
-		basic_machine=c38-convex
-		os=-bsd
-		;;
-	cray | j90)
-		basic_machine=j90-cray
-		os=-unicos
-		;;
-	craynv)
-		basic_machine=craynv-cray
-		os=-unicosmp
+		os=${os:-unicos}
 		;;
 	cr16 | cr16-*)
 		basic_machine=cr16-unknown
-		os=-elf
-		;;
-	crds | unos)
-		basic_machine=m68k-crds
+		os=${os:-elf}
 		;;
 	crisv32 | crisv32-* | etraxfs*)
 		basic_machine=crisv32-axis
@@ -588,7 +940,7 @@ case $basic_machine in
 		;;
 	crx)
 		basic_machine=crx-unknown
-		os=-elf
+		os=${os:-elf}
 		;;
 	da30 | da30-*)
 		basic_machine=m68k-da30
@@ -598,50 +950,38 @@ case $basic_machine in
 		;;
 	decsystem10* | dec10*)
 		basic_machine=pdp10-dec
-		os=-tops10
+		os=tops10
 		;;
 	decsystem20* | dec20*)
 		basic_machine=pdp10-dec
-		os=-tops20
+		os=tops20
 		;;
 	delta | 3300 | motorola-3300 | motorola-delta \
 	      | 3300-motorola | delta-motorola)
 		basic_machine=m68k-motorola
 		;;
-	delta88)
-		basic_machine=m88k-motorola
-		os=-sysv3
-		;;
-	dicos)
-		basic_machine=i686-pc
-		os=-dicos
-		;;
-	djgpp)
-		basic_machine=i586-pc
-		os=-msdosdjgpp
-		;;
 	dpx20 | dpx20-*)
 		basic_machine=rs6000-bull
-		os=-bosx
+		os=${os:-bosx}
 		;;
-	dpx2* | dpx2*-bull)
+	dpx2*)
 		basic_machine=m68k-bull
-		os=-sysv3
+		os=sysv3
 		;;
-	ebmon29k)
-		basic_machine=a29k-amd
-		os=-ebmon
+	e500v[12])
+		basic_machine=powerpc-unknown
+		os=$os"spe"
 		;;
-	elxsi)
-		basic_machine=elxsi-elxsi
-		os=-bsd
+	e500v[12]-*)
+		basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+		os=$os"spe"
 		;;
 	encore | umax | mmax)
 		basic_machine=ns32k-encore
 		;;
-	es1800 | OSE68k | ose68k | ose | OSE)
-		basic_machine=m68k-ericsson
-		os=-ose
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=${os:-bsd}
 		;;
 	fx2800)
 		basic_machine=i860-alliant
@@ -649,45 +989,13 @@ case $basic_machine in
 	genix)
 		basic_machine=ns32k-ns
 		;;
-	gmicro)
-		basic_machine=tron-gmicro
-		os=-sysv
-		;;
-	go32)
-		basic_machine=i386-pc
-		os=-go32
-		;;
 	h3050r* | hiux*)
 		basic_machine=hppa1.1-hitachi
-		os=-hiuxwe2
-		;;
-	h8300hms)
-		basic_machine=h8300-hitachi
-		os=-hms
-		;;
-	h8300xray)
-		basic_machine=h8300-hitachi
-		os=-xray
-		;;
-	h8500hms)
-		basic_machine=h8500-hitachi
-		os=-hms
-		;;
-	harris)
-		basic_machine=m88k-harris
-		os=-sysv3
+		os=hiuxwe2
 		;;
 	hp300-*)
 		basic_machine=m68k-hp
 		;;
-	hp300bsd)
-		basic_machine=m68k-hp
-		os=-bsd
-		;;
-	hp300hpux)
-		basic_machine=m68k-hp
-		os=-hpux
-		;;
 	hp3k9[0-9][0-9] | hp9[0-9][0-9])
 		basic_machine=hppa1.0-hp
 		;;
@@ -717,193 +1025,79 @@ case $basic_machine in
 	hp9k8[0-9][0-9] | hp8[0-9][0-9])
 		basic_machine=hppa1.0-hp
 		;;
-	hppa-next)
-		os=-nextstep3
-		;;
-	hppaosf)
-		basic_machine=hppa1.1-hp
-		os=-osf
-		;;
-	hppro)
-		basic_machine=hppa1.1-hp
-		os=-proelf
-		;;
-	i370-ibm* | ibm*)
-		basic_machine=i370-ibm
-		;;
 	i*86v32)
-		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
-		os=-sysv32
+		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
+		os=sysv32
 		;;
 	i*86v4*)
-		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
-		os=-sysv4
+		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
+		os=sysv4
 		;;
 	i*86v)
-		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
-		os=-sysv
+		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
+		os=sysv
 		;;
 	i*86sol2)
-		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
-		os=-solaris2
-		;;
-	i386mach)
-		basic_machine=i386-mach
-		os=-mach
+		basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'`
+		os=solaris2
 		;;
-	i386-vsta | vsta)
-		basic_machine=i386-unknown
-		os=-vsta
+	j90 | j90-cray)
+		basic_machine=j90-cray
+		os=${os:-unicos}
 		;;
 	iris | iris4d)
 		basic_machine=mips-sgi
 		case $os in
-		    -irix*)
+		    irix*)
 			;;
 		    *)
-			os=-irix4
+			os=irix4
 			;;
 		esac
 		;;
-	isi68 | isi)
-		basic_machine=m68k-isi
-		os=-sysv
-		;;
-	m68knommu)
-		basic_machine=m68k-unknown
-		os=-linux
+	leon-*|leon[3-9]-*)
+		basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'`
 		;;
 	m68knommu-*)
-		basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'`
-		os=-linux
-		;;
-	m88k-omron*)
-		basic_machine=m88k-omron
-		;;
-	magnum | m3230)
-		basic_machine=mips-mips
-		os=-sysv
+		basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+		os=linux
 		;;
-	merlin)
-		basic_machine=ns32k-utek
-		os=-sysv
-		;;
-	microblaze)
+	microblaze*)
 		basic_machine=microblaze-xilinx
 		;;
-	mingw64)
-		basic_machine=x86_64-pc
-		os=-mingw64
-		;;
-	mingw32)
-		basic_machine=i386-pc
-		os=-mingw32
-		;;
-	mingw32ce)
-		basic_machine=arm-unknown
-		os=-mingw32ce
-		;;
 	miniframe)
 		basic_machine=m68000-convergent
 		;;
-	*mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*)
+	*mint | mint[0-9]* | *MiNT | *MiNT[0-9]*)
 		basic_machine=m68k-atari
-		os=-mint
+		os=mint
 		;;
 	mips3*-*)
-		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`
 		;;
 	mips3*)
-		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
-		;;
-	monitor)
-		basic_machine=m68k-rom68k
-		os=-coff
-		;;
-	morphos)
-		basic_machine=powerpc-unknown
-		os=-morphos
-		;;
-	msdos)
-		basic_machine=i386-pc
-		os=-msdos
+		basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown
 		;;
 	ms1-*)
-		basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
-		;;
-	msys)
-		basic_machine=i386-pc
-		os=-msys
-		;;
-	mvs)
-		basic_machine=i370-ibm
-		os=-mvs
-		;;
-	nacl)
-		basic_machine=le32-unknown
-		os=-nacl
-		;;
-	ncr3000)
-		basic_machine=i486-ncr
-		os=-sysv4
-		;;
-	netbsd386)
-		basic_machine=i386-unknown
-		os=-netbsd
-		;;
-	netwinder)
-		basic_machine=armv4l-rebel
-		os=-linux
-		;;
-	news | news700 | news800 | news900)
-		basic_machine=m68k-sony
-		os=-newsos
-		;;
-	news1000)
-		basic_machine=m68030-sony
-		os=-newsos
+		basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'`
 		;;
 	news-3600 | risc-news)
 		basic_machine=mips-sony
-		os=-newsos
-		;;
-	necv70)
-		basic_machine=v70-nec
-		os=-sysv
+		os=newsos
 		;;
-	next | m*-next )
+	next | m*-next)
 		basic_machine=m68k-next
 		case $os in
-		    -nextstep* )
+		    nextstep* )
 			;;
-		    -ns2*)
-		      os=-nextstep2
+		    ns2*)
+		      os=nextstep2
 			;;
 		    *)
-		      os=-nextstep3
+		      os=nextstep3
 			;;
 		esac
 		;;
-	nh3000)
-		basic_machine=m68k-harris
-		os=-cxux
-		;;
-	nh[45]000)
-		basic_machine=m88k-harris
-		os=-cxux
-		;;
-	nindy960)
-		basic_machine=i960-intel
-		os=-nindy
-		;;
-	mon960)
-		basic_machine=i960-intel
-		os=-mon960
-		;;
-	nonstopux)
-		basic_machine=mips-compaq
-		os=-nonstopux
-		;;
 	np1)
 		basic_machine=np1-gould
 		;;
@@ -916,40 +1110,26 @@ case $basic_machine in
 	nsr-tandem)
 		basic_machine=nsr-tandem
 		;;
+	nsv-tandem)
+		basic_machine=nsv-tandem
+		;;
+	nsx-tandem)
+		basic_machine=nsx-tandem
+		;;
 	op50n-* | op60c-*)
 		basic_machine=hppa1.1-oki
-		os=-proelf
+		os=proelf
 		;;
 	openrisc | openrisc-*)
 		basic_machine=or32-unknown
 		;;
-	os400)
-		basic_machine=powerpc-ibm
-		os=-os400
-		;;
-	OSE68000 | ose68000)
-		basic_machine=m68000-ericsson
-		os=-ose
-		;;
-	os68k)
-		basic_machine=m68k-none
-		os=-os68k
-		;;
 	pa-hitachi)
 		basic_machine=hppa1.1-hitachi
-		os=-hiuxwe2
-		;;
-	paragon)
-		basic_machine=i860-intel
-		os=-osf
-		;;
-	parisc)
-		basic_machine=hppa-unknown
-		os=-linux
+		os=hiuxwe2
 		;;
 	parisc-*)
-		basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'`
-		os=-linux
+		basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'`
+		os=linux
 		;;
 	pbd)
 		basic_machine=sparc-tti
@@ -964,7 +1144,7 @@ case $basic_machine in
 		basic_machine=i386-pc
 		;;
 	pc98-*)
-		basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	pentium | p5 | k5 | k6 | nexgen | viac3)
 		basic_machine=i586-pc
@@ -979,16 +1159,16 @@ case $basic_machine in
 		basic_machine=i786-pc
 		;;
 	pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*)
-		basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	pentiumpro-* | p6-* | 6x86-* | athlon-*)
-		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*)
-		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	pentium4-*)
-		basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	pn)
 		basic_machine=pn-gould
@@ -998,39 +1178,27 @@ case $basic_machine in
 	ppc | ppcbe)	basic_machine=powerpc-unknown
 		;;
 	ppc-* | ppcbe-*)
-		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
-	ppcle | powerpclittle | ppc-le | powerpc-little)
+	ppcle | powerpclittle)
 		basic_machine=powerpcle-unknown
 		;;
 	ppcle-* | powerpclittle-*)
-		basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	ppc64)	basic_machine=powerpc64-unknown
 		;;
-	ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
+	ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
-	ppc64le | powerpc64little | ppc64-le | powerpc64-little)
+	ppc64le | powerpc64little)
 		basic_machine=powerpc64le-unknown
 		;;
 	ppc64le-* | powerpc64little-*)
-		basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'`
+		basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
 	ps2)
 		basic_machine=i386-ibm
 		;;
-	pw32)
-		basic_machine=i586-unknown
-		os=-pw32
-		;;
-	rdos)
-		basic_machine=i386-pc
-		os=-rdos
-		;;
-	rom68k)
-		basic_machine=m68k-rom68k
-		os=-coff
-		;;
 	rm[46]00)
 		basic_machine=mips-siemens
 		;;
@@ -1043,10 +1211,6 @@ case $basic_machine in
 	s390x | s390x-*)
 		basic_machine=s390x-ibm
 		;;
-	sa29200)
-		basic_machine=a29k-amd
-		os=-udi
-		;;
 	sb1)
 		basic_machine=mipsisa64sb1-unknown
 		;;
@@ -1055,105 +1219,32 @@ case $basic_machine in
 		;;
 	sde)
 		basic_machine=mipsisa32-sde
-		os=-elf
-		;;
-	sei)
-		basic_machine=mips-sei
-		os=-seiux
+		os=${os:-elf}
 		;;
 	sequent)
 		basic_machine=i386-sequent
 		;;
-	sh)
-		basic_machine=sh-hitachi
-		os=-hms
-		;;
 	sh5el)
 		basic_machine=sh5le-unknown
 		;;
-	sh64)
-		basic_machine=sh64-unknown
+	sh5el-*)
+		basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
-	sparclite-wrs | simso-wrs)
+	simso-wrs)
 		basic_machine=sparclite-wrs
-		os=-vxworks
-		;;
-	sps7)
-		basic_machine=m68k-bull
-		os=-sysv2
+		os=vxworks
 		;;
 	spur)
 		basic_machine=spur-unknown
 		;;
-	st2000)
-		basic_machine=m68k-tandem
-		;;
-	stratus)
-		basic_machine=i860-stratus
-		os=-sysv4
-		;;
 	strongarm-* | thumb-*)
-		basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'`
-		;;
-	sun2)
-		basic_machine=m68000-sun
-		;;
-	sun2os3)
-		basic_machine=m68000-sun
-		os=-sunos3
-		;;
-	sun2os4)
-		basic_machine=m68000-sun
-		os=-sunos4
+		basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'`
 		;;
-	sun3os3)
-		basic_machine=m68k-sun
-		os=-sunos3
-		;;
-	sun3os4)
-		basic_machine=m68k-sun
-		os=-sunos4
-		;;
-	sun4os3)
-		basic_machine=sparc-sun
-		os=-sunos3
-		;;
-	sun4os4)
-		basic_machine=sparc-sun
-		os=-sunos4
-		;;
-	sun4sol2)
-		basic_machine=sparc-sun
-		os=-solaris2
-		;;
-	sun3 | sun3-*)
-		basic_machine=m68k-sun
-		;;
-	sun4)
-		basic_machine=sparc-sun
-		;;
-	sun386 | sun386i | roadrunner)
-		basic_machine=i386-sun
-		;;
-	sv1)
-		basic_machine=sv1-cray
-		os=-unicos
-		;;
-	symmetry)
-		basic_machine=i386-sequent
-		os=-dynix
-		;;
-	t3e)
-		basic_machine=alphaev5-cray
-		os=-unicos
-		;;
-	t90)
-		basic_machine=t90-cray
-		os=-unicos
+	tile*-*)
 		;;
 	tile*)
 		basic_machine=$basic_machine-unknown
-		os=-linux-gnu
+		os=${os:-linux-gnu}
 		;;
 	tx39)
 		basic_machine=mipstx39-unknown
@@ -1161,146 +1252,32 @@ case $basic_machine in
 	tx39el)
 		basic_machine=mipstx39el-unknown
 		;;
-	toad1)
-		basic_machine=pdp10-xkl
-		os=-tops20
-		;;
 	tower | tower-32)
 		basic_machine=m68k-ncr
 		;;
-	tpf)
-		basic_machine=s390x-ibm
-		os=-tpf
-		;;
-	udi29k)
-		basic_machine=a29k-amd
-		os=-udi
-		;;
-	ultra3)
-		basic_machine=a29k-nyu
-		os=-sym1
-		;;
-	v810 | necv810)
-		basic_machine=v810-nec
-		os=-none
-		;;
-	vaxv)
-		basic_machine=vax-dec
-		os=-sysv
-		;;
-	vms)
-		basic_machine=vax-dec
-		os=-vms
-		;;
 	vpp*|vx|vx-*)
 		basic_machine=f301-fujitsu
 		;;
-	vxworks960)
-		basic_machine=i960-wrs
-		os=-vxworks
-		;;
-	vxworks68)
-		basic_machine=m68k-wrs
-		os=-vxworks
-		;;
-	vxworks29k)
-		basic_machine=a29k-wrs
-		os=-vxworks
-		;;
 	w65*)
 		basic_machine=w65-wdc
-		os=-none
+		os=none
 		;;
 	w89k-*)
 		basic_machine=hppa1.1-winbond
-		os=-proelf
-		;;
-	xbox)
-		basic_machine=i686-pc
-		os=-mingw32
+		os=proelf
 		;;
-	xps | xps100)
-		basic_machine=xps100-honeywell
+	x64)
+		basic_machine=x86_64-pc
 		;;
 	xscale-* | xscalee[bl]-*)
-		basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'`
-		;;
-	ymp)
-		basic_machine=ymp-cray
-		os=-unicos
-		;;
-	z8k-*-coff)
-		basic_machine=z8k-unknown
-		os=-sim
-		;;
-	z80-*-coff)
-		basic_machine=z80-unknown
-		os=-sim
+		basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'`
 		;;
 	none)
 		basic_machine=none-none
-		os=-none
 		;;
 
-# Here we handle the default manufacturer of certain CPU types.  It is in
-# some cases the only manufacturer, in others, it is the most popular.
-	w89k)
-		basic_machine=hppa1.1-winbond
-		;;
-	op50n)
-		basic_machine=hppa1.1-oki
-		;;
-	op60c)
-		basic_machine=hppa1.1-oki
-		;;
-	romp)
-		basic_machine=romp-ibm
-		;;
-	mmix)
-		basic_machine=mmix-knuth
-		;;
-	rs6000)
-		basic_machine=rs6000-ibm
-		;;
-	vax)
-		basic_machine=vax-dec
-		;;
-	pdp10)
-		# there are many clones, so DEC is not a safe bet
-		basic_machine=pdp10-unknown
-		;;
-	pdp11)
-		basic_machine=pdp11-dec
-		;;
-	we32k)
-		basic_machine=we32k-att
-		;;
-	sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele)
-		basic_machine=sh-unknown
-		;;
-	sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
-		basic_machine=sparc-sun
-		;;
-	cydra)
-		basic_machine=cydra-cydrome
-		;;
-	orion)
-		basic_machine=orion-highlevel
-		;;
-	orion105)
-		basic_machine=clipper-highlevel
-		;;
-	mac | mpw | mac-mpw)
-		basic_machine=m68k-apple
-		;;
-	pmac | pmac-mpw)
-		basic_machine=powerpc-apple
-		;;
-	*-unknown)
-		# Make sure to match an already-canonicalized machine name.
-		;;
 	*)
-		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2
 		exit 1
 		;;
 esac
@@ -1308,10 +1285,10 @@ esac
 # Here we canonicalize certain aliases for manufacturers.
 case $basic_machine in
 	*-digital*)
-		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'`
 		;;
 	*-commodore*)
-		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'`
 		;;
 	*)
 		;;
@@ -1319,200 +1296,246 @@ esac
 
 # Decode manufacturer-specific aliases for certain operating systems.
 
-if [ x"$os" != x"" ]
+if [ x$os != x ]
 then
 case $os in
-	# First match some system type aliases
-	# that might get confused with valid system types.
-	# -solaris* is a basic system type, with this one exception.
-	-auroraux)
-		os=-auroraux
+	# First match some system type aliases that might get confused
+	# with valid system types.
+	# solaris* is a basic system type, with this one exception.
+	auroraux)
+		os=auroraux
 		;;
-	-solaris1 | -solaris1.*)
-		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+	bluegene*)
+		os=cnk
 		;;
-	-solaris)
-		os=-solaris2
+	solaris1 | solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
 		;;
-	-svr4*)
-		os=-sysv4
+	solaris)
+		os=solaris2
 		;;
-	-unixware*)
-		os=-sysv4.2uw
+	unixware*)
+		os=sysv4.2uw
 		;;
-	-gnu/linux*)
+	gnu/linux*)
 		os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'`
 		;;
-	# First accept the basic system types.
+	# es1800 is here to avoid being matched by es* (a different OS)
+	es1800*)
+		os=ose
+		;;
+	# Some version numbers need modification
+	chorusos*)
+		os=chorusos
+		;;
+	isc)
+		os=isc2.2
+		;;
+	sco6)
+		os=sco5v6
+		;;
+	sco5)
+		os=sco3.2v5
+		;;
+	sco4)
+		os=sco3.2v4
+		;;
+	sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		;;
+	sco3.2v[4-9]* | sco5v6*)
+		# Don't forget version if it is 3.2v4 or newer.
+		;;
+	scout)
+		# Don't match below
+		;;
+	sco*)
+		os=sco3.2v2
+		;;
+	psos*)
+		os=psos
+		;;
+	# Now accept the basic system types.
 	# The portable systems comes first.
-	# Each alternative MUST END IN A *, to match a version number.
-	# -sysv* is not here because it comes later, after sysvr4.
-	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
-	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
-	      | -sym* | -kopensolaris* \
-	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
-	      | -aos* | -aros* \
-	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
-	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
-	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
-	      | -bitrig* | -openbsd* | -solidbsd* \
-	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
-	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
-	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
-	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
-	      | -chorusos* | -chorusrdb* | -cegcc* \
-	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
-	      | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
-	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
-	      | -uxpv* | -beos* | -mpeix* | -udk* \
-	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
-	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
-	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
-	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
-	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
-	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
-	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
+	# Each alternative MUST end in a * to match a version number.
+	# sysv* is not here because it comes later, after sysvr4.
+	gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \
+	     | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\
+	     | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \
+	     | sym* | kopensolaris* | plan9* \
+	     | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \
+	     | aos* | aros* | cloudabi* | sortix* \
+	     | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \
+	     | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \
+	     | knetbsd* | mirbsd* | netbsd* \
+	     | bitrig* | openbsd* | solidbsd* | libertybsd* \
+	     | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \
+	     | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \
+	     | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \
+	     | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \
+	     | chorusrdb* | cegcc* | glidix* \
+	     | cygwin* | msys* | pe* | moss* | proelf* | rtems* \
+	     | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \
+	     | linux-newlib* | linux-musl* | linux-uclibc* \
+	     | uxpv* | beos* | mpeix* | udk* | moxiebox* \
+	     | interix* | uwin* | mks* | rhapsody* | darwin* \
+	     | openstep* | oskit* | conix* | pw32* | nonstopux* \
+	     | storm-chaos* | tops10* | tenex* | tops20* | its* \
+	     | os2* | vos* | palmos* | uclinux* | nucleus* \
+	     | morphos* | superux* | rtmk* | windiss* \
+	     | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \
+	     | skyos* | haiku* | rdos* | toppers* | drops* | es* \
+	     | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \
+	     | midnightbsd*)
 	# Remember, each alternative MUST END IN *, to match a version number.
 		;;
-	-qnx*)
+	qnx*)
 		case $basic_machine in
 		    x86-* | i*86-*)
 			;;
 		    *)
-			os=-nto$os
+			os=nto-$os
 			;;
 		esac
 		;;
-	-nto-qnx*)
+	hiux*)
+		os=hiuxwe2
 		;;
-	-nto*)
-		os=`echo $os | sed -e 's|nto|nto-qnx|'`
+	nto-qnx*)
 		;;
-	-sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
-	      | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \
-	      | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
+	nto*)
+		os=`echo $os | sed -e 's|nto|nto-qnx|'`
 		;;
-	-mac*)
-		os=`echo $os | sed -e 's|mac|macos|'`
+	sim | xray | os68k* | v88r* \
+	    | windows* | osx | abug | netware* | os9* \
+	    | macos* | mpw* | magic* | mmixware* | mon960* | lnews*)
 		;;
-	-linux-dietlibc)
-		os=-linux-dietlibc
+	linux-dietlibc)
+		os=linux-dietlibc
 		;;
-	-linux*)
+	linux*)
 		os=`echo $os | sed -e 's|linux|linux-gnu|'`
 		;;
-	-sunos5*)
-		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+	lynx*178)
+		os=lynxos178
 		;;
-	-sunos6*)
-		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+	lynx*5)
+		os=lynxos5
 		;;
-	-opened*)
-		os=-openedition
+	lynx*)
+		os=lynxos
 		;;
-	-os400*)
-		os=-os400
+	mac*)
+		os=`echo "$os" | sed -e 's|mac|macos|'`
 		;;
-	-wince*)
-		os=-wince
+	opened*)
+		os=openedition
 		;;
-	-osfrose*)
-		os=-osfrose
+	os400*)
+		os=os400
 		;;
-	-osf*)
-		os=-osf
+	sunos5*)
+		os=`echo "$os" | sed -e 's|sunos5|solaris2|'`
 		;;
-	-utek*)
-		os=-bsd
+	sunos6*)
+		os=`echo "$os" | sed -e 's|sunos6|solaris3|'`
 		;;
-	-dynix*)
-		os=-bsd
+	wince*)
+		os=wince
 		;;
-	-acis*)
-		os=-aos
+	utek*)
+		os=bsd
 		;;
-	-atheos*)
-		os=-atheos
+	dynix*)
+		os=bsd
 		;;
-	-syllable*)
-		os=-syllable
+	acis*)
+		os=aos
 		;;
-	-386bsd)
-		os=-bsd
+	atheos*)
+		os=atheos
 		;;
-	-ctix* | -uts*)
-		os=-sysv
+	syllable*)
+		os=syllable
 		;;
-	-nova*)
-		os=-rtmk-nova
+	386bsd)
+		os=bsd
+		;;
+	ctix* | uts*)
+		os=sysv
+		;;
+	nova*)
+		os=rtmk-nova
 		;;
-	-ns2 )
-		os=-nextstep2
+	ns2)
+		os=nextstep2
 		;;
-	-nsk*)
-		os=-nsk
+	nsk*)
+		os=nsk
 		;;
 	# Preserve the version number of sinix5.
-	-sinix5.*)
+	sinix5.*)
 		os=`echo $os | sed -e 's|sinix|sysv|'`
 		;;
-	-sinix*)
-		os=-sysv4
+	sinix*)
+		os=sysv4
 		;;
-	-tpf*)
-		os=-tpf
+	tpf*)
+		os=tpf
 		;;
-	-triton*)
-		os=-sysv3
+	triton*)
+		os=sysv3
 		;;
-	-oss*)
-		os=-sysv3
+	oss*)
+		os=sysv3
 		;;
-	-svr4)
-		os=-sysv4
+	svr4*)
+		os=sysv4
 		;;
-	-svr3)
-		os=-sysv3
+	svr3)
+		os=sysv3
 		;;
-	-sysvr4)
-		os=-sysv4
+	sysvr4)
+		os=sysv4
 		;;
-	# This must come after -sysvr4.
-	-sysv*)
+	# This must come after sysvr4.
+	sysv*)
 		;;
-	-ose*)
-		os=-ose
+	ose*)
+		os=ose
 		;;
-	-es1800*)
-		os=-ose
+	*mint | mint[0-9]* | *MiNT | MiNT[0-9]*)
+		os=mint
 		;;
-	-xenix)
-		os=-xenix
+	zvmoe)
+		os=zvmoe
 		;;
-	-*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
-		os=-mint
+	dicos*)
+		os=dicos
 		;;
-	-aros*)
-		os=-aros
-		;;
-	-kaos*)
-		os=-kaos
+	pikeos*)
+		# Until real need of OS specific support for
+		# particular features comes up, bare metal
+		# configurations are quite functional.
+		case $basic_machine in
+		    arm*)
+			os=eabi
+			;;
+		    *)
+			os=elf
+			;;
+		esac
 		;;
-	-zvmoe)
-		os=-zvmoe
+	nacl*)
 		;;
-	-dicos*)
-		os=-dicos
+	ios)
 		;;
-	-nacl*)
+	none)
 		;;
-	-none)
+	*-eabi)
 		;;
 	*)
-		# Get rid of the `-' at the beginning of $os.
-		os=`echo $os | sed 's/[^-]*-//'`
-		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2
 		exit 1
 		;;
 esac
@@ -1530,173 +1553,179 @@ else
 
 case $basic_machine in
 	score-*)
-		os=-elf
+		os=elf
 		;;
 	spu-*)
-		os=-elf
+		os=elf
 		;;
 	*-acorn)
-		os=-riscix1.2
+		os=riscix1.2
 		;;
 	arm*-rebel)
-		os=-linux
+		os=linux
 		;;
 	arm*-semi)
-		os=-aout
+		os=aout
 		;;
 	c4x-* | tic4x-*)
-		os=-coff
+		os=coff
+		;;
+	c8051-*)
+		os=elf
+		;;
+	clipper-intergraph)
+		os=clix
 		;;
 	hexagon-*)
-		os=-elf
+		os=elf
 		;;
 	tic54x-*)
-		os=-coff
+		os=coff
 		;;
 	tic55x-*)
-		os=-coff
+		os=coff
 		;;
 	tic6x-*)
-		os=-coff
+		os=coff
 		;;
 	# This must come before the *-dec entry.
 	pdp10-*)
-		os=-tops20
+		os=tops20
 		;;
 	pdp11-*)
-		os=-none
+		os=none
 		;;
 	*-dec | vax-*)
-		os=-ultrix4.2
+		os=ultrix4.2
 		;;
 	m68*-apollo)
-		os=-domain
+		os=domain
 		;;
 	i386-sun)
-		os=-sunos4.0.2
+		os=sunos4.0.2
 		;;
 	m68000-sun)
-		os=-sunos3
+		os=sunos3
 		;;
 	m68*-cisco)
-		os=-aout
+		os=aout
 		;;
 	mep-*)
-		os=-elf
+		os=elf
 		;;
 	mips*-cisco)
-		os=-elf
+		os=elf
 		;;
 	mips*-*)
-		os=-elf
+		os=elf
 		;;
 	or32-*)
-		os=-coff
+		os=coff
 		;;
 	*-tti)	# must be before sparc entry or we get the wrong os.
-		os=-sysv3
+		os=sysv3
 		;;
 	sparc-* | *-sun)
-		os=-sunos4.1.1
+		os=sunos4.1.1
 		;;
-	*-be)
-		os=-beos
+	pru-*)
+		os=elf
 		;;
-	*-haiku)
-		os=-haiku
+	*-be)
+		os=beos
 		;;
 	*-ibm)
-		os=-aix
+		os=aix
 		;;
 	*-knuth)
-		os=-mmixware
+		os=mmixware
 		;;
 	*-wec)
-		os=-proelf
+		os=proelf
 		;;
 	*-winbond)
-		os=-proelf
+		os=proelf
 		;;
 	*-oki)
-		os=-proelf
+		os=proelf
 		;;
 	*-hp)
-		os=-hpux
+		os=hpux
 		;;
 	*-hitachi)
-		os=-hiux
+		os=hiux
 		;;
 	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
-		os=-sysv
+		os=sysv
 		;;
 	*-cbm)
-		os=-amigaos
+		os=amigaos
 		;;
 	*-dg)
-		os=-dgux
+		os=dgux
 		;;
 	*-dolphin)
-		os=-sysv3
+		os=sysv3
 		;;
 	m68k-ccur)
-		os=-rtu
+		os=rtu
 		;;
 	m88k-omron*)
-		os=-luna
+		os=luna
 		;;
-	*-next )
-		os=-nextstep
+	*-next)
+		os=nextstep
 		;;
 	*-sequent)
-		os=-ptx
+		os=ptx
 		;;
 	*-crds)
-		os=-unos
+		os=unos
 		;;
 	*-ns)
-		os=-genix
+		os=genix
 		;;
 	i370-*)
-		os=-mvs
-		;;
-	*-next)
-		os=-nextstep3
+		os=mvs
 		;;
 	*-gould)
-		os=-sysv
+		os=sysv
 		;;
 	*-highlevel)
-		os=-bsd
+		os=bsd
 		;;
 	*-encore)
-		os=-bsd
+		os=bsd
 		;;
 	*-sgi)
-		os=-irix
+		os=irix
 		;;
 	*-siemens)
-		os=-sysv4
+		os=sysv4
 		;;
 	*-masscomp)
-		os=-rtu
+		os=rtu
 		;;
 	f30[01]-fujitsu | f700-fujitsu)
-		os=-uxpv
+		os=uxpv
 		;;
 	*-rom68k)
-		os=-coff
+		os=coff
 		;;
 	*-*bug)
-		os=-coff
+		os=coff
 		;;
 	*-apple)
-		os=-macos
+		os=macos
 		;;
 	*-atari*)
-		os=-mint
+		os=mint
+		;;
+	*-wrs)
+		os=vxworks
 		;;
 	*)
-		os=-none
+		os=none
 		;;
 esac
 fi
@@ -1707,79 +1736,82 @@ vendor=unknown
 case $basic_machine in
 	*-unknown)
 		case $os in
-			-riscix*)
+			riscix*)
 				vendor=acorn
 				;;
-			-sunos*)
+			sunos*)
 				vendor=sun
 				;;
-			-cnk*|-aix*)
+			cnk*|-aix*)
 				vendor=ibm
 				;;
-			-beos*)
+			beos*)
 				vendor=be
 				;;
-			-hpux*)
+			hpux*)
 				vendor=hp
 				;;
-			-mpeix*)
+			mpeix*)
 				vendor=hp
 				;;
-			-hiux*)
+			hiux*)
 				vendor=hitachi
 				;;
-			-unos*)
+			unos*)
 				vendor=crds
 				;;
-			-dgux*)
+			dgux*)
 				vendor=dg
 				;;
-			-luna*)
+			luna*)
 				vendor=omron
 				;;
-			-genix*)
+			genix*)
 				vendor=ns
 				;;
-			-mvs* | -opened*)
+			clix*)
+				vendor=intergraph
+				;;
+			mvs* | opened*)
 				vendor=ibm
 				;;
-			-os400*)
+			os400*)
 				vendor=ibm
 				;;
-			-ptx*)
+			ptx*)
 				vendor=sequent
 				;;
-			-tpf*)
+			tpf*)
 				vendor=ibm
 				;;
-			-vxsim* | -vxworks* | -windiss*)
+			vxsim* | vxworks* | windiss*)
 				vendor=wrs
 				;;
-			-aux*)
+			aux*)
 				vendor=apple
 				;;
-			-hms*)
+			hms*)
 				vendor=hitachi
 				;;
-			-mpw* | -macos*)
+			mpw* | macos*)
 				vendor=apple
 				;;
-			-*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+			*mint | mint[0-9]* | *MiNT | MiNT[0-9]*)
 				vendor=atari
 				;;
-			-vos*)
+			vos*)
 				vendor=stratus
 				;;
 		esac
-		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"`
 		;;
 esac
 
-echo $basic_machine$os
+echo "$basic_machine-$os"
 exit
 
 # Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
+# eval: (add-hook 'before-save-hook 'time-stamp)
 # time-stamp-start: "timestamp='"
 # time-stamp-format: "%:y-%02m-%02d"
 # time-stamp-end: "'"

From d277442df53a01343ba7c1df0bbd2a294058dcba Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 17 Aug 2018 12:59:23 +0200
Subject: [PATCH 1154/2196] Fix parser/lexer generation with parallel make

Fun fact: rules with multiple targets don't work properly with 'make
-j'. For example, a rule like

  a b: c
    touch a b

is equivalent to

  a: c
    touch a b

  b: c
    touch a b

so with 'make -j', the 'touch' command will be run twice. See
e.g. https://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file.
---
 src/libexpr/local.mk | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index daa3258f0d3..78c8b0deae6 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -20,10 +20,14 @@ libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
 
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
+$(d)/parser-tab.hh: $(d)/parser-tab.cc
+
+$(d)/parser-tab.cc: $(d)/parser.y
 	$(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d
 
-$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
+$(d)/lexer-tab.hh: $(d)/lexer-tab.cc
+
+$(d)/lexer-tab.cc: $(d)/lexer.l
 	$(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $<
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh

From 02098d2073e2f7c06b6d05c6749ae2b76b7f57d5 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 17 Aug 2018 10:29:32 -0400
Subject: [PATCH 1155/2196] fetchGit: use a better caching scheme

The current usage technically works by putting multiple different
repos in to the same git directory. However, it is very slow as
Git tries very hard to find common commits between the two
repositories. If the two repositories are large (like Nixpkgs and
another long-running project,) it is maddeningly slow.

This change busts the cache for existing deployments, but users
will be promptly repaid in per-repository performance.
---
 src/libexpr/primops/fetchGit.cc | 4 +++-
 tests/fetchGit.sh               | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 7aa98e0bfab..aeb2df5f8ae 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -3,6 +3,7 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
+#include "hash.hh"
 
 #include <sys/time.h>
 
@@ -84,9 +85,10 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
-    Path cacheDir = getCacheDir() + "/nix/git";
+    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
 
     if (!pathExists(cacheDir)) {
+        createDirs(dirOf(cacheDir));
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 530ac7bb813..4c46bdf0465 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -9,7 +9,7 @@ clearStore
 
 repo=$TEST_ROOT/git
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/git
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
 
 git init $repo
 git -C $repo config user.email "foobar@example.com"
@@ -129,7 +129,7 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP
 
 
 # Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/git
+rm -rf $TEST_HOME/.cache/nix/gitv2
 
 # Try again, but without 'git' on PATH
 NIX=$(command -v nix)

From 9b1bdf2db8798c024e6fbe84d4144c04f61d80c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 19 Aug 2018 11:59:49 +0200
Subject: [PATCH 1156/2196] FIx floating point evaluation

Fixes #2361.
---
 src/libexpr/primops.cc         | 9 +++++++++
 tests/lang/eval-okay-float.exp | 1 +
 tests/lang/eval-okay-float.nix | 6 ++++++
 3 files changed, 16 insertions(+)
 create mode 100644 tests/lang/eval-okay-float.exp
 create mode 100644 tests/lang/eval-okay-float.nix

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8ace6db4d11..6f82c6c404f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1680,6 +1680,8 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
 
 static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
@@ -1689,6 +1691,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
@@ -1698,6 +1702,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     if (args[0]->type == tFloat || args[1]->type == tFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
@@ -1707,6 +1713,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 
 static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
+
     NixFloat f2 = state.forceFloat(*args[1], pos);
     if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
 
diff --git a/tests/lang/eval-okay-float.exp b/tests/lang/eval-okay-float.exp
new file mode 100644
index 00000000000..3c50a8adce8
--- /dev/null
+++ b/tests/lang/eval-okay-float.exp
@@ -0,0 +1 @@
+[ 3.4 3.5 2.5 1.5 ]
diff --git a/tests/lang/eval-okay-float.nix b/tests/lang/eval-okay-float.nix
new file mode 100644
index 00000000000..b2702c7b166
--- /dev/null
+++ b/tests/lang/eval-okay-float.nix
@@ -0,0 +1,6 @@
+[
+  (1.1 + 2.3)
+  (builtins.add (0.5 + 0.5) (2.0 + 0.5))
+  ((0.5 + 0.5) * (2.0 + 0.5))
+  ((1.5 + 1.5) / (0.5 * 4.0))
+]

From cc7b4386b16885a22ccabb019381539fecb00230 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 19 Aug 2018 12:05:08 +0200
Subject: [PATCH 1157/2196] nix run: Restore CPU affinity

Fixes #2359.
---
 src/nix/run.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 65ced34759b..35b76334587 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -7,6 +7,7 @@
 #include "finally.hh"
 #include "fs-accessor.hh"
 #include "progress-bar.hh"
+#include "affinity.hh"
 
 #if __linux__
 #include <sys/mount.h>
@@ -154,6 +155,8 @@ struct CmdRun : InstallablesCommand
 
         restoreSignals();
 
+        restoreAffinity();
+
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location
            (e.g. /home/eelco/nix/store), then run the command in a

From 458282be5946318086b74dfa5aedb91e6f0477aa Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas.tynkkynen@iki.fi>
Date: Mon, 20 Aug 2018 01:51:23 +0300
Subject: [PATCH 1158/2196] Drop all references to --disable-init-state

It's all dead code since 2014 (commit 0c6d62cf27b3b2).
---
 configure.ac       | 6 ------
 release-common.nix | 2 +-
 release.nix        | 4 ----
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/configure.ac b/configure.ac
index 6aeeacdba13..cc354f6f302 100644
--- a/configure.ac
+++ b/configure.ac
@@ -229,12 +229,6 @@ if test "$gc" = yes; then
 fi
 
 
-AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
-  [do not initialise DB etc. in `make install']),
-  init_state=$enableval, init_state=yes)
-#AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes")
-
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),
diff --git a/release-common.nix b/release-common.nix
index e23967bdbfd..ace2a4f9b91 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -30,7 +30,7 @@ rec {
   });
 
   configureFlags =
-    [ "--disable-init-state"
+    [
       "--enable-gc"
     ] ++ lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
diff --git a/release.nix b/release.nix
index 076f1de8dea..5c3fdf55ca3 100644
--- a/release.nix
+++ b/release.nix
@@ -189,10 +189,6 @@ let
 
         buildInputs = buildDeps;
 
-        configureFlags = ''
-          --disable-init-state
-        '';
-
         dontInstall = false;
 
         doInstallCheck = true;

From 2894197de768bf5433595886d0248e77ffa3c693 Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas.tynkkynen@iki.fi>
Date: Mon, 20 Aug 2018 02:01:59 +0300
Subject: [PATCH 1159/2196] Drop all references to NIX_INDENT_MAKE

Dead code since https://github.com/NixOS/nixpkgs/commit/6669a3b47711dc967df0ea8ff93fa9857aad015d
---
 release.nix                | 4 ----
 src/nix-build/nix-build.cc | 1 -
 2 files changed, 5 deletions(-)

diff --git a/release.nix b/release.nix
index 5c3fdf55ca3..58809ba8ab0 100644
--- a/release.nix
+++ b/release.nix
@@ -74,8 +74,6 @@ let
 
         makeFlags = "profiledir=$(out)/etc/profile.d";
 
-        preBuild = "unset NIX_INDENT_MAKE";
-
         installFlags = "sysconfdir=$(out)/etc";
 
         doInstallCheck = true;
@@ -103,8 +101,6 @@ let
         enableParallelBuilding = true;
 
         postUnpack = "sourceRoot=$sourceRoot/perl";
-
-        preBuild = "unset NIX_INDENT_MAKE";
       });
 
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 34f1cba9dda..21a0756a200 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -419,7 +419,6 @@ void mainWrapped(int argc, char * * argv)
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
-                "unset NIX_INDENT_MAKE; "
                 "shopt -u nullglob; "
                 "unset TZ; %4%"
                 "%5%",

From 17a92dfb7d87499d7aed5d3b0504d4c724db18a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Aug 2018 15:19:20 +0200
Subject: [PATCH 1160/2196] Fix another 'coroutine has finished' during
 decompression

https://hydra.nixos.org/build/79867739
---
 src/libstore/binary-cache-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9c75c85993f..4527ee6ba66 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -232,7 +232,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
         throw SubstituteGone(e.what());
     }
 
-    decompressor->flush();
+    decompressor->finish();
 
     stats.narRead++;
     //stats.narReadCompressedBytes += nar->size(); // FIXME

From 6317c65937ad7d9c5a954c4dfc48de259e55f4f3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Aug 2018 15:20:23 +0200
Subject: [PATCH 1161/2196] Fix warnings in compression.cc

---
 src/libutil/compression.cc | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 53b62f62a76..204c63cd26f 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -15,12 +15,10 @@
 
 namespace nix {
 
-static const size_t bufSize = 32 * 1024;
-
 // Don't feed brotli too much at once.
 struct ChunkedCompressionSink : CompressionSink
 {
-    uint8_t outbuf[BUFSIZ];
+    uint8_t outbuf[32 * 1024];
 
     void write(const unsigned char * data, size_t len) override
     {
@@ -124,7 +122,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink
         write(nullptr, 0);
     }
 
-    void writeInternal(const unsigned char * data, size_t len)
+    void writeInternal(const unsigned char * data, size_t len) override
     {
         assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
@@ -173,7 +171,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
         writeInternal(nullptr, 0);
     }
 
-    void writeInternal(const unsigned char * data, size_t len)
+    void writeInternal(const unsigned char * data, size_t len) override
     {
         const uint8_t * next_in = data;
         size_t avail_in = len;
@@ -330,7 +328,7 @@ struct BzipCompressionSink : ChunkedCompressionSink
         writeInternal(nullptr, 0);
     }
 
-    void writeInternal(const unsigned char * data, size_t len)
+    void writeInternal(const unsigned char * data, size_t len) override
     {
         assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
@@ -380,7 +378,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink
         writeInternal(nullptr, 0);
     }
 
-    void writeInternal(const unsigned char * data, size_t len)
+    void writeInternal(const unsigned char * data, size_t len) override
     {
         const uint8_t * next_in = data;
         size_t avail_in = len;

From ebe3d2d3704aca001ff818295af014581dcf348f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Aug 2018 15:22:04 +0200
Subject: [PATCH 1162/2196] Improve 'coroutine has finished' error message

---
 src/libstore/store-api.cc |  2 ++
 src/libutil/serialise.cc  | 15 +++++++++------
 src/libutil/serialise.hh  |  6 +++++-
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7a4a5f5eb85..1f42097fccf 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -609,6 +609,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
             act.progress(total, info->narSize);
         });
         srcStore->narFromPath({storePath}, wrapperSink);
+    }, [&]() {
+	throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
     });
 
     dstStore->addToStore(*info, *source, repair, checkSigs);
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index b2c49d911b3..17448f70efb 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -161,16 +161,20 @@ size_t StringSource::read(unsigned char * data, size_t len)
 #error Coroutines are broken in this version of Boost!
 #endif
 
-std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
+std::unique_ptr<Source> sinkToSource(
+    std::function<void(Sink &)> fun,
+    std::function<void()> eof)
 {
     struct SinkToSource : Source
     {
         typedef boost::coroutines2::coroutine<std::string> coro_t;
 
+        std::function<void()> eof;
         coro_t::pull_type coro;
 
-        SinkToSource(std::function<void(Sink &)> fun)
-            : coro([&](coro_t::push_type & yield) {
+        SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
+            : eof(eof)
+            , coro([&](coro_t::push_type & yield) {
                 LambdaSink sink([&](const unsigned char * data, size_t len) {
                     if (len) yield(std::string((const char *) data, len));
                 });
@@ -184,8 +188,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
 
         size_t read(unsigned char * data, size_t len) override
         {
-            if (!coro)
-                throw EndOfFile("coroutine has finished");
+            if (!coro) { eof(); abort(); }
 
             if (pos == cur.size()) {
                 if (!cur.empty()) coro();
@@ -201,7 +204,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun)
         }
     };
 
-    return std::make_unique<SinkToSource>(fun);
+    return std::make_unique<SinkToSource>(fun, eof);
 }
 
 
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 14b62fdb677..4b6ad5da5b9 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -214,7 +214,11 @@ struct LambdaSource : Source
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */
-std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun);
+std::unique_ptr<Source> sinkToSource(
+    std::function<void(Sink &)> fun,
+    std::function<void()> eof = []() {
+        throw EndOfFile("coroutine has finished");
+    });
 
 
 void writePadding(size_t len, Sink & sink);

From 954d1f4d0a35063ff431b258beebadf753cb9efe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Aug 2018 15:33:29 +0200
Subject: [PATCH 1163/2196] Disable the Ubuntu 17.10 build

Ubuntu 17.10 doesn't have libbrotli.

https://hydra.nixos.org/build/79867741
---
 release.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 076f1de8dea..7ce84e445ab 100644
--- a/release.nix
+++ b/release.nix
@@ -212,8 +212,8 @@ let
     #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
     #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
 
-    deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
-    deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
+    #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
+    #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
 
 
     # System tests.

From 8ad2defdf0b33c34e48773b501e35bfdbbb936f7 Mon Sep 17 00:00:00 2001
From: Erik Arvstedt <erik.arvstedt@gmail.com>
Date: Tue, 21 Aug 2018 21:44:37 +0200
Subject: [PATCH 1164/2196] Docs: Fix install prefix

shell.nix defines the install prefix as $(pwd)/inst
---
 doc/manual/hacking.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml
index 183aed7adff..b671811d3a3 100644
--- a/doc/manual/hacking.xml
+++ b/doc/manual/hacking.xml
@@ -30,7 +30,7 @@ To build Nix itself in this shell:
 [nix-shell]$ configurePhase
 [nix-shell]$ make
 </screen>
-To install it in <literal>$(pwd)/nix</literal> and test it:
+To install it in <literal>$(pwd)/inst</literal> and test it:
 <screen>
 [nix-shell]$ make install
 [nix-shell]$ make installcheck

From c651b7bdc996a18688c5e5dd5dd84eeeb8d3376d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 23 Aug 2018 00:23:35 +0200
Subject: [PATCH 1165/2196] Revert "Fix parser/lexer generation with parallel
 make"

This reverts commit d277442df53a01343ba7c1df0bbd2a294058dcba.

Make sucks.
---
 src/libexpr/local.mk | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 78c8b0deae6..daa3258f0d3 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -20,14 +20,10 @@ libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
 
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(d)/parser-tab.hh: $(d)/parser-tab.cc
-
-$(d)/parser-tab.cc: $(d)/parser.y
+$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
 	$(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d
 
-$(d)/lexer-tab.hh: $(d)/lexer-tab.cc
-
-$(d)/lexer-tab.cc: $(d)/lexer.l
+$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 	$(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $<
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh

From 414397759a5390b54cee12c6834e6106bea12f5f Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 25 Aug 2018 20:25:43 +0200
Subject: [PATCH 1166/2196] upgrade-nix: add --dry-run

---
 src/nix/upgrade-nix.cc | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index e23ae792369..3417adb62c9 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "common-args.hh"
 #include "store-api.hh"
 #include "download.hh"
 #include "eval.hh"
@@ -6,7 +7,7 @@
 
 using namespace nix;
 
-struct CmdUpgradeNix : StoreCommand
+struct CmdUpgradeNix : MixDryRun, StoreCommand
 {
     Path profileDir;
 
@@ -61,21 +62,25 @@ struct CmdUpgradeNix : StoreCommand
 
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
-            store->ensurePath(storePath);
+            if (!dryRun)
+                store->ensurePath(storePath);
         }
 
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath));
-            auto program = storePath + "/bin/nix-env";
-            auto s = runProgram(program, false, {"--version"});
-            if (s.find("Nix") == std::string::npos)
-                throw Error("could not verify that '%s' works", program);
+            if (!dryRun) {
+                auto program = storePath + "/bin/nix-env";
+                auto s = runProgram(program, false, {"--version"});
+                if (s.find("Nix") == std::string::npos)
+                    throw Error("could not verify that '%s' works", program);
+            }
         }
 
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
-            runProgram(settings.nixBinDir + "/nix-env", false,
-                {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
+            if (!dryRun)
+                runProgram(settings.nixBinDir + "/nix-env", false,
+                    {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
         }
     }
 

From d85bb4814f4fa2ed6046bba6e8c6394adf9e5666 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 25 Aug 2018 20:33:01 +0200
Subject: [PATCH 1167/2196] upgrade-nix: resolve profile symlinks

The profile present in PATH is not necessarily the actual profile
location. User profiles are generally added as $HOME/.nix-profile
in which case the indirect profile link needs to be resolved first.

/home/user/.nix-profile -> /nix/var/nix/profiles/per-user/user/profile
/nix/var/nix/profiles/per-user/user/profile -> profile-15-link
/nix/var/nix/profiles/per-user/user/profile-14-link -> /nix/store/hyi4kkjh3bwi2z3wfljrkfymz9904h62-user-environment
/nix/var/nix/profiles/per-user/user/profile-15-link -> /nix/store/6njpl3qvihz46vj911pwx7hfcvwhifl9-user-environment

To upgrade nix here we want /nix/var/nix/profiles/per-user/user/profile-16-link
instead of /home/user/.nix-profile-1-link. The latter is not a gcroot
and would be garbage collected, resulting in a broken profile.

Fixes #2175
---
 src/nix/upgrade-nix.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 3417adb62c9..72b298283a6 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -103,11 +103,18 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         if (hasPrefix(where, "/run/current-system"))
             throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'");
 
-        Path profileDir;
-        Path userEnv;
+        Path profileDir = dirOf(where);
+
+        // Resolve profile to /nix/var/nix/profiles/<name> link.
+        while (baseNameOf(dirOf(canonPath(profileDir))) != "profiles")
+            profileDir = readLink(profileDir);
+
+        printInfo("found profile '%s'", profileDir);
+
+        Path userEnv = canonPath(profileDir, true);
 
         if (baseNameOf(where) != "bin" ||
-            !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment"))
+            !hasSuffix(userEnv, "user-environment"))
             throw Error("directory '%s' does not appear to be part of a Nix profile", where);
 
         if (!store->isValidPath(userEnv))

From 0ad643ed5c9642738a8e8b37b7e4b1835f160bc8 Mon Sep 17 00:00:00 2001
From: aszlig <aszlig@nix.build>
Date: Wed, 29 Aug 2018 00:23:51 +0200
Subject: [PATCH 1168/2196] libexpr: Use int64_t for NixInt

Using a 64bit integer on 32bit systems will come with a bit of a
performance overhead, but given that Nix doesn't use a lot of integers
compared to other types, I think the overhead is negligible also
considering that 32bit systems are in decline.

The biggest advantage however is that when we use a consistent integer
size across all platforms it's less likely that we miss things that we
break due to that. One example would be:

https://github.com/NixOS/nixpkgs/pull/44233

On Hydra it will evaluate, because the evaluator runs on a 64bit
machine, but when evaluating the same on a 32bit machine it will fail,
so using 64bit integers should make that consistent.

While the change of the type in value.hh is rather easy to do, we have a
few more options available for doing the conversion in the lexer:

  * Via an #ifdef on the architecture and using strtol() or strtoll()
    accordingly depending on which architecture we are. For the #ifdef
    we would need another AX_COMPILE_CHECK_SIZEOF in configure.ac.
  * Using istringstream, which would involve copying the value.
  * As we're already using boost, lexical_cast might be a good idea.

Spoiler: I went for the latter, first of all because lexical_cast does
have an overload for const char* and second of all, because it doesn't
involve copying around the input string. Also, because istringstream
seems to come with a bigger overhead than boost::lexical_cast:

https://www.boost.org/doc/libs/release/doc/html/boost_lexical_cast/performance.html

The first method (still using strtol/strtoll) also wasn't something I
pursued further, because it is also locale-aware which I doubt is what
we want, given that the regex for int is [0-9]+.

Signed-off-by: aszlig <aszlig@nix.build>
Fixes: #2339
---
 src/libexpr/lexer.l  | 8 ++++++--
 src/libexpr/value.hh | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 29ca327c1e4..a052447d3dc 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -12,6 +12,8 @@
 
 
 %{
+#include <boost/lexical_cast.hpp>
+
 #include "nixexpr.hh"
 #include "parser-tab.hh"
 
@@ -124,9 +126,11 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { errno = 0;
-              yylval->n = strtol(yytext, 0, 10);
-              if (errno != 0)
+              try {
+                  yylval->n = boost::lexical_cast<int64_t>(yytext);
+              } catch (const boost::bad_lexical_cast &) {
                   throw ParseError(format("invalid integer '%1%'") % yytext);
+              }
               return INT;
             }
 {FLOAT}     { errno = 0;
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 809772f7c08..e1ec87d3b84 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -43,7 +43,7 @@ class XMLWriter;
 class JSONPlaceholder;
 
 
-typedef long NixInt;
+typedef int64_t NixInt;
 typedef double NixFloat;
 
 /* External values must descend from ExternalValueBase, so that

From 54df4bb0b5d45a17945d8c4b0e38cd4ae167a732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 29 Aug 2018 11:51:51 +0100
Subject: [PATCH 1169/2196] nix.conf: mention -j0 is useful

---
 doc/manual/command-ref/conf-file.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1865bb37c86..3851cf18d7f 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -437,7 +437,8 @@ builtins.fetchurl {
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
     <literal>1</literal>. The special value <literal>auto</literal>
-    causes Nix to use the number of CPUs in your system.  It can be
+    causes Nix to use the number of CPUs in your system.  <literal>0</literal>
+    is useful when using remote builders to prevent any local builds.  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>

From d16ff76c69318c4ef04b1dcc873a72910b8eca49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 29 Aug 2018 14:33:14 +0100
Subject: [PATCH 1170/2196] nix.conf: clarify -j0 doesn't affect
 preferLocalBuild

---
 doc/manual/command-ref/conf-file.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 3851cf18d7f..6a23b8f1fda 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -438,7 +438,9 @@ builtins.fetchurl {
     that Nix will try to build in parallel.  The default is
     <literal>1</literal>. The special value <literal>auto</literal>
     causes Nix to use the number of CPUs in your system.  <literal>0</literal>
-    is useful when using remote builders to prevent any local builds.  It can be
+    is useful when using remote builders to prevent any local builds (except for
+    <literal>preferLocalBuild</literal> derivation attribute which executes locally
+    regardless).  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>

From 245d01701d421d5c6aafd2b377a7b14c254f788c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 15:21:58 +0200
Subject: [PATCH 1171/2196] upload-release.pl: Handle prerelease versions

---
 maintainers/upload-release.pl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 9b0a09e6c83..8432c95960c 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -91,6 +91,8 @@ sub downloadFile {
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
+exit if $version =~ /pre/;
+
 # Update Nixpkgs in a very hacky way.
 system("cd $nixpkgsDir && git pull") == 0 or die;
 my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName;

From 64d7d1a884838f6a15fd18d90f73a8061832af57 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 15:28:18 +0200
Subject: [PATCH 1172/2196] Update release notes

---
 doc/manual/release-notes/rl-2.1.xml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index c628d04a707..09b4f92e199 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -4,7 +4,7 @@
       version="5.0"
       xml:id="ssec-relnotes-2.1">
 
-<title>Release 2.1 (2018-08-??)</title>
+<title>Release 2.1 (2018-08-31)</title>
 
 <para>This is primarily a bug fix release. It also reduces memory
 consumption in certain situations. In addition, it has the following
@@ -41,8 +41,12 @@ new features:</para>
 
   <listitem>
     <para><command>nix-daemon</command> now respects
-    <option>--store</option>, so it can be run as a non-root
-    user.</para>
+    <option>--store</option>.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix run</command> now respects
+    <varname>nix-support/propagated-user-env-packages</varname>.</para>
   </listitem>
 
 </itemizedlist>

From 145db703e58fa8438413cb67a1361020850ce10e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 16:03:06 +0200
Subject: [PATCH 1173/2196] Remove boost from the closure

This reduces the size of the closure by 45 MiB.
---
 release.nix | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/release.nix b/release.nix
index beb8b5aa97c..f9ee07e3478 100644
--- a/release.nix
+++ b/release.nix
@@ -67,6 +67,14 @@ let
 
         buildInputs = buildDeps;
 
+        preConfigure =
+          # Copy libboost_context so we don't get all of Boost in our closure.
+          # https://github.com/NixOS/nixpkgs/issues/45462
+          ''
+            mkdir -p $out/lib
+            cp ${boost}/lib/libboost_context* $out/lib
+          '';
+
         configureFlags = configureFlags ++
           [ "--sysconfdir=/etc" ];
 

From 5e83b0227f024c63fd1adb7995d17a8a1604b10c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 16:14:19 +0200
Subject: [PATCH 1174/2196] nix: Remove the -h flag

---
 src/nix/main.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 9cd5d21c84b..69791e223c2 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -24,7 +24,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     {
         mkFlag()
             .longName("help")
-            .shortName('h')
             .description("show usage information")
             .handler([&]() { showHelpAndExit(); });
 

From 3407a5d9361611ea8b5da9e02a7a1e2f90c928cd Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sat, 25 Aug 2018 18:35:15 -0400
Subject: [PATCH 1175/2196] Add human readable closure sizes to nix path-info

Unfortunately, -h is already taken as a short option by --help, so we have to
use a different letter or the capitalized version.

Resolves #2363
---
 src/nix/path-info.cc | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 47caa401d3c..ee48d483c7e 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -13,12 +13,14 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
 {
     bool showSize = false;
     bool showClosureSize = false;
+    bool humanReadable = false;
     bool showSigs = false;
 
     CmdPathInfo()
     {
         mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
         mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
+        mkFlag('H', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
         mkFlag(0, "sigs", "show signatures", &showSigs);
     }
 
@@ -39,6 +41,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 "To show the closure sizes of every path in the current NixOS system closure, sorted by size:",
                 "nix path-info -rS /run/current-system | sort -nk2"
             },
+            Example{
+                "To show a package's closure size and all its dependencies with human readable sizes:",
+                "nix path-info -rsSH nixpkgs.rust"
+            },
             Example{
                 "To check the existence of a path in a binary cache:",
                 "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/"
@@ -58,6 +64,25 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         };
     }
 
+    void printSize(int value)
+    {
+        if (!humanReadable) {
+            std::cout << '\t' << std::setw(11) << value;
+            return;
+        }
+
+        static constexpr std::array<char, 6> idents = {
+            ' ', 'K', 'M', 'G', 'T', 'P'
+        };
+        size_t power = 0;
+        double res = value;
+        while (res > 1024) {
+            ++power;
+            res /= 1024;
+        }
+        std::cout << '\t' << std::setw(11) << std::setprecision(3) << res << idents[power];
+    }
+
     void run(ref<Store> store, Paths storePaths) override
     {
         size_t pathLen = 0;
@@ -81,10 +106,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
 
                 if (showSize)
-                    std::cout << '\t' << std::setw(11) << info->narSize;
+                    printSize(info->narSize);
 
                 if (showClosureSize)
-                    std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath).first;
+                    printSize(store->getClosureSize(storePath).first);
 
                 if (showSigs) {
                     std::cout << '\t';

From c908df881f105d6a1004ae6a0e39da2d0149f3e9 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sun, 26 Aug 2018 18:12:06 -0400
Subject: [PATCH 1176/2196] Avoid overflow and use boost::format

If the user has an object greater than 1024 yottabytes, it'll just display it as
N yottabytes instead of overflowing.

Swaps to use boost::format strings instead of std::setw and std::setprecision.
---
 src/nix/path-info.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index ee48d483c7e..fa4f45fb429 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -4,8 +4,9 @@
 #include "json.hh"
 #include "common-args.hh"
 
-#include <iomanip>
 #include <algorithm>
+#include <boost/format.hpp>
+#include <iomanip>
 
 using namespace nix;
 
@@ -67,20 +68,20 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
     void printSize(int value)
     {
         if (!humanReadable) {
-            std::cout << '\t' << std::setw(11) << value;
+            std::cout << '\t' << boost::format("%11d") % value;
             return;
         }
 
-        static constexpr std::array<char, 6> idents = {
-            ' ', 'K', 'M', 'G', 'T', 'P'
+        static constexpr std::array<char, 9> idents = {
+            ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'
         };
         size_t power = 0;
         double res = value;
-        while (res > 1024) {
+        while (res > 1024 && power < idents.size()) {
             ++power;
             res /= 1024;
         }
-        std::cout << '\t' << std::setw(11) << std::setprecision(3) << res << idents[power];
+        std::cout << '\t' << boost::format("%11.1f") % res << idents[power];
     }
 
     void run(ref<Store> store, Paths storePaths) override

From 99828245f8f03a8b3b54a04f67e6df31235838f4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 16:18:53 +0200
Subject: [PATCH 1177/2196] printSize() fixes

Fix a 32-bit overflow that resulted in negative numbers being printed;
use fmt() instead of boost::format(); change -H to -h for consistency
with 'ls' and 'du'; make the columns narrower (since they can't be
bigger than 1024.0).
---
 src/nix/path-info.cc | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index fa4f45fb429..916ed360e62 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -5,8 +5,6 @@
 #include "common-args.hh"
 
 #include <algorithm>
-#include <boost/format.hpp>
-#include <iomanip>
 
 using namespace nix;
 
@@ -21,7 +19,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
     {
         mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
         mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
-        mkFlag('H', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
+        mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
         mkFlag(0, "sigs", "show signatures", &showSigs);
     }
 
@@ -44,7 +42,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             },
             Example{
                 "To show a package's closure size and all its dependencies with human readable sizes:",
-                "nix path-info -rsSH nixpkgs.rust"
+                "nix path-info -rsSh nixpkgs.rust"
             },
             Example{
                 "To check the existence of a path in a binary cache:",
@@ -65,10 +63,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         };
     }
 
-    void printSize(int value)
+    void printSize(unsigned long long value)
     {
         if (!humanReadable) {
-            std::cout << '\t' << boost::format("%11d") % value;
+            std::cout << fmt("\t%11d", value);
             return;
         }
 
@@ -81,7 +79,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             ++power;
             res /= 1024;
         }
-        std::cout << '\t' << boost::format("%11.1f") % res << idents[power];
+        std::cout << fmt("\t%6.1f%c", res, idents.at(power));
     }
 
     void run(ref<Store> store, Paths storePaths) override

From 264e66f69699919f1040be2986b91fd6b188e9af Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 17:10:28 +0200
Subject: [PATCH 1178/2196] Add simple test for nix upgrade-nix

---
 release.nix            | 12 ++++++++++++
 src/nix/upgrade-nix.cc |  9 ++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index f9ee07e3478..fdf3fa946aa 100644
--- a/release.nix
+++ b/release.nix
@@ -249,6 +249,18 @@ let
           su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
           su - alice -c 'nix-store --verify'
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
+          # Check whether 'nix upgrade-nix' works.
+          (! [ -L /nix/var/nix/profiles/per-user/alice/profile-2-link ])
+          cat > /tmp/paths.nix <<EOF
+          {
+            x86_64-linux = "${build.x86_64-linux}";
+          }
+          EOF
+          su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          [ -L /nix/var/nix/profiles/per-user/alice/profile-2-link ]
+          su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
+
           mkdir -p $out/nix-support
           touch $out/nix-support/hydra-build-products
           umount /nix
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index e23ae792369..4f4d0ebc08a 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -9,6 +9,7 @@ using namespace nix;
 struct CmdUpgradeNix : StoreCommand
 {
     Path profileDir;
+    std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix";
 
     CmdUpgradeNix()
     {
@@ -18,6 +19,12 @@ struct CmdUpgradeNix : StoreCommand
             .labels({"profile-dir"})
             .description("the Nix profile to upgrade")
             .dest(&profileDir);
+
+        mkFlag()
+            .longName("nix-store-paths-url")
+            .labels({"url"})
+            .description("URL of the file that contains the store paths of the latest Nix release")
+            .dest(&storePathsUrl);
     }
 
     std::string name() override
@@ -115,7 +122,7 @@ struct CmdUpgradeNix : StoreCommand
     Path getLatestNix(ref<Store> store)
     {
         // FIXME: use nixos.org?
-        auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix");
+        auto req = DownloadRequest(storePathsUrl);
         auto res = getDownloader()->download(req);
 
         auto state = std::make_unique<EvalState>(Strings(), store);

From f08b14c9d0e0a1d24b0a5d231ba7d448ef000106 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 17:14:17 +0200
Subject: [PATCH 1179/2196] nix upgrade-nix: Improve error message if 'nix' is
 not in a profile

E.g.

  $ nix upgrade-nix
  error: directory '/home/eelco/Dev/nix/inst/bin' does not appear to be part of a Nix profile

instead of

  $ nix upgrade-nix
  error: '/home/eelco/Dev/nix/inst' is not a symlink
---
 src/nix/upgrade-nix.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 666ac68b0a7..2db60922a9c 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -113,7 +113,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Path profileDir = dirOf(where);
 
         // Resolve profile to /nix/var/nix/profiles/<name> link.
-        while (baseNameOf(dirOf(canonPath(profileDir))) != "profiles")
+        while (baseNameOf(dirOf(canonPath(profileDir))) != "profiles" && isLink(profileDir))
             profileDir = readLink(profileDir);
 
         printInfo("found profile '%s'", profileDir);

From 39f1722f364d7ce95717161cc283e96250c14643 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Aug 2018 21:18:56 +0200
Subject: [PATCH 1180/2196] nix upgrade-nix: Handle .nix-profile being a link
 to .../profiles/per-user/...

Also some cosmetic improvements.
---
 release.nix            |  4 ++--
 src/nix/upgrade-nix.cc | 34 ++++++++++++++++++++++------------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/release.nix b/release.nix
index fdf3fa946aa..e359ebcb2ce 100644
--- a/release.nix
+++ b/release.nix
@@ -241,6 +241,7 @@ let
         { diskImage = vmTools.diskImages.ubuntu1204x86_64;
         }
         ''
+          set -x
           useradd -m alice
           su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
           mkdir /dest-nix
@@ -251,14 +252,13 @@ let
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
 
           # Check whether 'nix upgrade-nix' works.
-          (! [ -L /nix/var/nix/profiles/per-user/alice/profile-2-link ])
           cat > /tmp/paths.nix <<EOF
           {
             x86_64-linux = "${build.x86_64-linux}";
           }
           EOF
           su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
-          [ -L /nix/var/nix/profiles/per-user/alice/profile-2-link ]
+          (! [ -L /home/alice/.profile-1-link ])
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
 
           mkdir -p $out/nix-support
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 2db60922a9c..35c44a70cf5 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -4,6 +4,8 @@
 #include "download.hh"
 #include "eval.hh"
 #include "attr-path.hh"
+#include "names.hh"
+#include "progress-bar.hh"
 
 using namespace nix;
 
@@ -67,28 +69,36 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
             storePath = getLatestNix(store);
         }
 
+        auto version = DrvName(storePathToName(storePath)).version;
+
+        if (dryRun) {
+            stopProgressBar();
+            printError("would upgrade to version %s", version);
+            return;
+        }
+
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
-            if (!dryRun)
-                store->ensurePath(storePath);
+            store->ensurePath(storePath);
         }
 
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath));
-            if (!dryRun) {
-                auto program = storePath + "/bin/nix-env";
-                auto s = runProgram(program, false, {"--version"});
-                if (s.find("Nix") == std::string::npos)
-                    throw Error("could not verify that '%s' works", program);
-            }
+            auto program = storePath + "/bin/nix-env";
+            auto s = runProgram(program, false, {"--version"});
+            if (s.find("Nix") == std::string::npos)
+                throw Error("could not verify that '%s' works", program);
         }
 
+        stopProgressBar();
+
         {
             Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
-            if (!dryRun)
-                runProgram(settings.nixBinDir + "/nix-env", false,
-                    {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
+            runProgram(settings.nixBinDir + "/nix-env", false,
+                {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
         }
+
+        printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
     }
 
     /* Return the profile in which Nix is installed. */
@@ -113,7 +123,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Path profileDir = dirOf(where);
 
         // Resolve profile to /nix/var/nix/profiles/<name> link.
-        while (baseNameOf(dirOf(canonPath(profileDir))) != "profiles" && isLink(profileDir))
+        while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir))
             profileDir = readLink(profileDir);
 
         printInfo("found profile '%s'", profileDir);

From 0767e402f1d166bdc25755422091bbd67eda52fe Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Thu, 30 Aug 2018 20:00:01 -0300
Subject: [PATCH 1181/2196] fix `error: unknown serve command 9`

---
 src/libstore/legacy-ssh-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 7c214f09d6f..88d2574e86e 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -131,7 +131,7 @@ struct LegacySSHStore : public Store
 
         auto conn(connections->get());
 
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
+        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
 
             conn->to
                 << cmdAddToStoreNar

From c0c31b58a43dc46c5f2a4f7f1880387db449711b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Aug 2018 11:12:51 +0200
Subject: [PATCH 1182/2196] Add localhost alias for ::1 to the sandbox

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d75ca0be86e..cd37f7a3fc0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2007,7 +2007,7 @@ void DerivationGoal::startBuilder()
 
         /* Create /etc/hosts with localhost entry. */
         if (!fixedOutput)
-            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
+            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access

From 2df21b78b98dcfcb1fd36e15b76e78cc2d8587a0 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 31 Aug 2018 09:32:59 -0400
Subject: [PATCH 1183/2196] docs: Add some examples to fetchGit

---
 doc/manual/expressions/builtins.xml | 89 ++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 07d8357b40b..6f3c4e3d0b6 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -398,6 +398,91 @@ stdenv.mkDerivation { … }
           </listitem>
         </varlistentry>
       </variablelist>
+
+      <example>
+        <title>Fetching a private repository over SSH</title>
+        <programlisting>builtins.fetchGit {
+  url = "ssh://git@github.com/my-secret/repository.git";
+  ref = "master";
+  rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+}</programlisting>
+
+      <note><para>
+        Note the URL format is not the same as <command>git
+        clone</command>. <function>builtins.fetchGit</function> uses
+        a <literal>/</literal> instead of a <literal>:</literal>
+        between <literal>github.com</literal> and
+        <literal>my-secret</literal>.</para></note>
+      </example>
+
+      <example>
+        <title>Fetching a repository's specific commit on an arbitrary branch</title>
+        <para>
+          If the revision you're looking for is in the default branch
+          of the gift repository you don't strictly need to specify
+          the branch name in the <varname>ref</varname> attribute.
+        </para>
+        <para>
+          However, if the revision you're looking for is in a future
+          branch for the non-default branch you will need to specify
+          the the <varname>ref</varname> attribute as well.
+        </para>
+        <programlisting>builtins.fetchGit {
+  url = "https://github.com/nixos/nix.git";
+  rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+  ref = "1.11-maintenance";
+}</programlisting>
+        <note>
+          <para>
+            It is nice to always specify the branch which a revision
+            belongs to. Without the branch being specified, the
+            fetcher might fail if the default branch changes.
+            Additionally, it can be confusing to try a commit from a
+            non-default branch and see the fetch fail. If the branch
+            is specified the fault is much more obvious.
+          </para>
+        </note>
+      </example>
+
+      <example>
+        <title>Fetching a repository's specific commit on the default branch</title>
+        <para>
+          If the revision you're looking for is in the default branch
+          of the gift repository you may omit the
+          <varname>ref</varname> attribute.
+        </para>
+        <programlisting>builtins.fetchGit {
+  url = "https://github.com/nixos/nix.git";
+  rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+}</programlisting>
+      </example>
+
+      <example>
+        <title>Fetching a tag</title>
+        <programlisting>builtins.fetchGit {
+  url = "https://github.com/nixos/nix.git";
+  ref = "tags/1.9";
+}</programlisting>
+        <note><para>Due to a bug (<link
+        xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
+        only non-annotated tags can be fetched.</para></note>
+      </example>
+
+      <example>
+        <title>Fetching the latest version of a remote branch</title>
+        <para>
+          <function>builtins.fetchGit</function> can behave impurely
+           fetch the latest version of a remote branch.
+        </para>
+        <note><para>Nix will refetch the branch in accordance to
+        <option>tarball-ttl</option>.</para></note>
+        <note><para>This behavior is disabled in
+        <emphasis>Pure evaluation mode</emphasis>.</para></note>
+        <programlisting>builtins.fetchGit {
+  url = "ssh://git@github.com/nixos/nix.git";
+  ref = "master";
+}</programlisting>
+      </example>
     </listitem>
   </varlistentry>
 
@@ -1244,8 +1329,8 @@ in foo</programlisting>
     This is not allowed because it would cause a cyclic dependency in
     the computation of the cryptographic hashes for
     <varname>foo</varname> and <varname>bar</varname>.</para>
-    <para>It is also not possible to reference the result of a derivation. 
-    If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to 
+    <para>It is also not possible to reference the result of a derivation.
+    If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to
     do that.</para></listitem>
 
   </varlistentry>

From 149d10c308090dafa852542419010b0caf4412a5 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham.christensen@target.com>
Date: Fri, 31 Aug 2018 10:06:33 -0400
Subject: [PATCH 1184/2196] docs: Add IDs to important sections

---
 doc/manual/advanced-topics/advanced-topics.xml | 1 +
 doc/manual/expressions/language-constructs.xml | 2 +-
 doc/manual/glossary/glossary.xml               | 3 ++-
 doc/manual/installation/env-variables.xml      | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml
index 338aa6f3a23..b710f9f2b51 100644
--- a/doc/manual/advanced-topics/advanced-topics.xml
+++ b/doc/manual/advanced-topics/advanced-topics.xml
@@ -1,6 +1,7 @@
 <part xmlns="http://docbook.org/ns/docbook"
       xmlns:xlink="http://www.w3.org/1999/xlink"
       xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="part-advanced-topics"
       version="5.0">
 
 <title>Advanced Topics</title>
diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 47d95f8a13e..f961ed921bc 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -41,7 +41,7 @@ encountered</quote>).</para></footnote>.</para>
 </simplesect>
 
 
-<simplesect><title>Let-expressions</title>
+<simplesect xml:id="sect-let-expressions"><title>Let-expressions</title>
 
 <para>A let-expression allows you define local variables for an
 expression.  For instance,
diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml
index 4977825578f..e3162ed8d46 100644
--- a/doc/manual/glossary/glossary.xml
+++ b/doc/manual/glossary/glossary.xml
@@ -1,5 +1,6 @@
 <appendix xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xml:id="part-glossary">
 
 <title>Glossary</title>
 
diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index 1fd6bafee7e..91ecd114f6d 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -55,7 +55,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
 the Nix installer will detect the presense of Nix configuration, and
 abort.</para></note>
 
-<section>
+<section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos">
 <title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title>
 
 <para>On macOS you must specify the environment variable for the Nix

From ef09da58f2cb592b10c0c3bffff01cf89a5fdb7d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Aug 2018 17:30:45 +0200
Subject: [PATCH 1185/2196] nix path-info: Hopefully fix macOS build failure

https://hydra.nixos.org/build/80480356
---
 src/nix/path-info.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 916ed360e62..fbe5f91698f 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -70,9 +70,9 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             return;
         }
 
-        static constexpr std::array<char, 9> idents = {
+        static const std::array<char, 9> idents{{
             ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'
-        };
+        }};
         size_t power = 0;
         double res = value;
         while (res > 1024 && power < idents.size()) {

From d1f36e8787d2aa2071ece097273975656f788f2a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 Aug 2018 19:20:08 +0200
Subject: [PATCH 1186/2196] Really fix the build

---
 src/nix/path-info.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index fbe5f91698f..11c47bc4b49 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -5,6 +5,7 @@
 #include "common-args.hh"
 
 #include <algorithm>
+#include <array>
 
 using namespace nix;
 

From 4095cd6438812a3f7871796604f1ae548b592aeb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 1 Sep 2018 00:01:05 +0200
Subject: [PATCH 1187/2196] Add contributors

---
 doc/manual/release-notes/rl-2.1.xml | 55 ++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index 09b4f92e199..9a5f37f6625 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -51,7 +51,60 @@ new features:</para>
 
 </itemizedlist>
 
-<para>This release has contributions from TODO.
+<para>This release has contributions from
+
+Adrien Devresse,
+Aleksandr Pashkov,
+Alexandre Esteves,
+Amine Chikhaoui,
+Andrew Dunham,
+Asad Saeeduddin,
+aszlig,
+Ben Challenor,
+Ben Gamari,
+Benjamin Hipple,
+Bogdan Seniuc,
+Corey O'Connor,
+Daiderd Jordan,
+Daniel Peebles,
+Daniel Poelzleithner,
+Danylo Hlynskyi,
+Dmitry Kalinkin,
+Domen Kožar,
+Doug Beardsley,
+Eelco Dolstra,
+Erik Arvstedt,
+Félix Baylac-Jacqué,
+Gleb Peregud,
+Graham Christensen,
+Guillaume Maudoux,
+Ivan Kozik,
+John Arnold,
+Justin Humm,
+Linus Heckemann,
+Lorenzo Manacorda,
+Matthew Justin Bauer,
+Matthew O'Gorman,
+Maximilian Bosch,
+Michael Bishop,
+Michael Fiano,
+Michael Mercier,
+Michael Raskin,
+Michael Weiss,
+Nicolas Dudebout,
+Peter Simons,
+Ryan Trinkle,
+Samuel Dionne-Riel,
+Sean Seefried,
+Shea Levy,
+Symphorien Gibol,
+Tim Engler,
+Tim Sears,
+Tuomas Tynkkynen,
+volth,
+Will Dietz,
+Yorick van Pelt and
+zimbatm.
 </para>
 
 </section>

From b7409c57544997e6c31a05b5ca024e752cb1a35d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 1 Sep 2018 00:04:57 +0200
Subject: [PATCH 1188/2196] nix path-info: Remove trailing spaces

Fixes #2390.
---
 src/nix/path-info.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 11c47bc4b49..dea5f0557b8 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -103,7 +103,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 auto info = store->queryPathInfo(storePath);
                 storePath = info->path; // FIXME: screws up padding
 
-                std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
+                std::cout << storePath;
+
+                if (showSize || showClosureSize || showSigs)
+                    std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
 
                 if (showSize)
                     printSize(info->narSize);

From 475a0a54a9c28c311f0c05d129bb165e7c215efb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 1 Sep 2018 00:19:49 +0200
Subject: [PATCH 1189/2196] fetchGit/fetchMercurial: Don't absolutize paths

This is already done by coerceToString(), provided that the argument
is a path (e.g. 'fetchGit ./bla'). It fixes the handling of URLs like
git@github.com:owner/repo.git. It breaks 'fetchGit "./bla"', but that
was never intended to work anyway and is inconsistent with other
builtin functions (e.g. 'readFile "./bla"' fails).
---
 src/libexpr/primops/fetchGit.cc       | 2 --
 src/libexpr/primops/fetchMercurial.cc | 2 --
 2 files changed, 4 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 7aa98e0bfab..0c6539959bf 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -219,8 +219,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 9d35f6d0d6d..97cda2458c9 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -184,8 +184,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
 
-    if (!isUri(url)) url = absPath(url);
-
     // FIXME: git externals probably can be used to bypass the URI
     // whitelist. Ah well.
     state.checkURI(url);

From 0b7568fb730439a2fbbf3c4b3b222319faa7e66a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 31 Aug 2018 19:49:56 -0400
Subject: [PATCH 1190/2196] Drop ssh://... as a required formatting for
 builtins.fetchGit

---
 doc/manual/expressions/builtins.xml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 6f3c4e3d0b6..873f30b062e 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -402,17 +402,10 @@ stdenv.mkDerivation { … }
       <example>
         <title>Fetching a private repository over SSH</title>
         <programlisting>builtins.fetchGit {
-  url = "ssh://git@github.com/my-secret/repository.git";
+  url = "git@github.com:my-secret/repository.git";
   ref = "master";
   rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
 }</programlisting>
-
-      <note><para>
-        Note the URL format is not the same as <command>git
-        clone</command>. <function>builtins.fetchGit</function> uses
-        a <literal>/</literal> instead of a <literal>:</literal>
-        between <literal>github.com</literal> and
-        <literal>my-secret</literal>.</para></note>
       </example>
 
       <example>

From b7bb627f674ce90d1a543bf3eab80b834bb592b2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 31 Aug 2018 20:57:07 -0400
Subject: [PATCH 1191/2196] nix-build: print stats on sucessful builds

---
 src/nix-build/nix-build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 21a0756a200..54d6b1db736 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -495,6 +495,7 @@ void mainWrapped(int argc, char * * argv)
 
         for (auto & path : outPaths)
             std::cout << path << '\n';
+        state->printStats();
     }
 }
 

From 87702532d2aa24394493ccd12e77811cc2d0893c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 31 Aug 2018 21:03:32 -0400
Subject: [PATCH 1192/2196] nix-build: Print stats even in failing builds

---
 src/nix-build/nix-build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 54d6b1db736..94d3a27560f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -305,6 +305,8 @@ void mainWrapped(int argc, char * * argv)
         }
     }
 
+    state->printStats();
+
     auto buildPaths = [&](const PathSet & paths) {
         /* Note: we do this even when !printMissing to efficiently
            fetch binary cache data. */
@@ -495,7 +497,6 @@ void mainWrapped(int argc, char * * argv)
 
         for (auto & path : outPaths)
             std::cout << path << '\n';
-        state->printStats();
     }
 }
 

From f66fa7cd20731ab08be72e016cc5b912940cd1db Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 1 Sep 2018 08:56:53 -0400
Subject: [PATCH 1193/2196] We support aarch64 now

---
 doc/manual/installation/supported-platforms.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml
index 6858573ff40..3e74be49d1f 100644
--- a/doc/manual/installation/supported-platforms.xml
+++ b/doc/manual/installation/supported-platforms.xml
@@ -10,7 +10,7 @@
 
 <itemizedlist>
 
-  <listitem><para>Linux (i686, x86_64).</para></listitem>
+  <listitem><para>Linux (i686, x86_64, aarch64).</para></listitem>
 
   <listitem><para>macOS (x86_64).</para></listitem>
 

From c3e508d9245f9439343d88d827c2d7c369ebfc88 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 31 Aug 2018 11:47:00 -0400
Subject: [PATCH 1194/2196] Document the multi-user installer some

Use sh <(...) syntax for installation to preserve stdin and prompting

also update installation docs to account for changes in multi-user selection
---
 doc/manual/installation/env-variables.xml     |   2 +-
 doc/manual/installation/installing-binary.xml | 178 ++++++++++++++----
 2 files changed, 140 insertions(+), 40 deletions(-)

diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index 91ecd114f6d..d1ee0bb2e09 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ curl https://nixos.org/nix/install | sh
+$ sh &lt;(curl https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,
diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 7e8dfb0db3d..394d8053b94 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -6,13 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install
-Nix is to run the following command:
+<para>If you are using Linux or macOS, the easiest way to install Nix
+is to run the following command:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl https://nixos.org/nix/install)
 </screen>
 
+As of Nix 2.1.0, the Nix installer will always default to creating a
+single-user installation, however opting in to the multi-user
+installation is highly recommended.
+</para>
+
+<section xml:id="sect-single-user-installation">
+  <title>Single User Installation</title>
+
+  <para>
+    To explicitly select a single-user installation on your system:
+
+    <screen>
+  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+</screen>
+  </para>
+
+<para>
 This will perform a single-user installation of Nix, meaning that
 <filename>/nix</filename> is owned by the invoking user.  You should
 run this under your usual user account, <emphasis>not</emphasis> as
@@ -33,58 +50,141 @@ and <filename>.profile</filename> to source
 the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
 variable before executing the install script to disable this
 behaviour.
-
 </para>
 
-<!--
-<para>You can also manually download and install a binary package.
-Binary packages of the latest stable release are available for Fedora,
-Debian, Ubuntu, macOS and various other systems from the <link
-xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
-You can also get builds of the latest development release from our
-<link
-xlink:href="http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents">continuous
-build system</link>.</para>
 
-<para>For Fedora, RPM packages are available.  These can be installed
-or upgraded using <command>rpm -U</command>.  For example,
+<para>You can uninstall Nix simply by running:
 
 <screen>
-$ rpm -U nix-1.8-1.i386.rpm</screen>
+$ rm -rf /nix
+</screen>
 
 </para>
-
-<para>For Debian and Ubuntu, you can download a Deb package and
-install it like this:
+</section>
+
+<section xml:id="sect-multi-user-installation">
+  <title>Multi User Installation</title>
+  <para>
+    The multi-user Nix installation creates system users, and a system
+    service for the Nix daemon.
+  </para>
+
+  <itemizedlist>
+    <title>Supported Systems</title>
+
+    <listitem>
+      <para>Linux running systemd, with SELinux disabled</para>
+    </listitem>
+    <listitem><para>macOS</para></listitem>
+  </itemizedlist>
+
+  <para>
+    You can instruct the installer to perform a multi-user
+    installation on your system:
+
+    <screen>
+  sh &lt;(curl https://nixos.org/nix/install) --daemon
+</screen>
+  </para>
+
+  <para>
+    The multi-user installation of Nix will create build users between
+    the user IDs 30001 and 30032, and a group with the group ID 30000.
+
+    You should run this under your usual user account,
+    <emphasis>not</emphasis> as root. The script will invoke
+    <command>sudo</command> as needed.
+  </para>
+
+  <note><para>
+    If you need Nix to use a different group ID or user ID set, you
+    will have to download the tarball manually and <link
+    linkend="sect-nix-install-binary-tarball">edit the install
+    script</link>.
+  </para></note>
+
+  <para>
+    The installer will modify <filename>/etc/bashrc</filename>, and
+    <filename>/etc/zshrc</filename> if they exist. The installer will
+    first back up these files with a
+    <literal>.backup-before-nix</literal> extension. The installer
+    will also create <filename>/etc/profile.d/nix.sh</filename>.
+  </para>
+
+  <para>You can uninstall Nix with the following commands:
 
 <screen>
-$ dpkg -i nix_1.8-1_amd64.deb</screen>
+sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+
+# If you are on Linux with systemd, you will need to run:
+sudo systemctl stop nix-daemon.socket
+sudo systemctl stop nix-daemon.service
+sudo systemctl disable nix-daemon.socket
+sudo systemctl disable nix-daemon.service
+sudo systemctl daemon-reload
+
+# If you are on macOS, you will need to run:
+sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+</screen>
 
-</para>
--->
+    There may also be references to Nix in
+    <filename>/etc/profile</filename>,
+    <filename>/etc/bashrc</filename>, and
+    <filename>/etc/zshrc</filename> which you may remove.
+  </para>
 
-<para>You can also download a binary tarball that contains Nix and all
-its dependencies.  (This is what the install script at
-<uri>https://nixos.org/nix/install</uri> does automatically.)  You
-should unpack it somewhere (e.g. in <filename>/tmp</filename>), and
-then run the script named <command>install</command> inside the binary
-tarball:
+</section>
 
-<screen>
-alice$ cd /tmp
-alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-alice$ cd nix-1.8-x86_64-darwin
-alice$ ./install
-</screen>
+<section xml:id="sect-nix-install-pinned-version-url">
+  <title>Installing a pinned Nix version from a URL</title>
 
-</para>
+  <para>
+    NixOS.org hosts version-specific installation URLs for all Nix
+    versions since 1.11.16, at
+    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+  </para>
 
-<para>You can uninstall Nix simply by running:
+  <para>
+    These install scripts can be used the same as the main
+  NixOS.org installation script:
 
-<screen>
-$ rm -rf /nix
+  <screen>
+  sh &lt;(curl https://nixos.org/nix/install)
 </screen>
+  </para>
 
-</para>
+  <para>
+    In the same directory of the install script are sha256 sums, and
+    gpg signature files.
+  </para>
+</section>
+
+<section xml:id="sect-nix-install-binary-tarball">
+  <title>Installing from a binary tarball</title>
 
+  <para>
+    You can also download a binary tarball that contains Nix and all
+    its dependencies.  (This is what the install script at
+    <uri>https://nixos.org/nix/install</uri> does automatically.)  You
+    should unpack it somewhere (e.g. in <filename>/tmp</filename>),
+    and then run the script named <command>install</command> inside
+    the binary tarball:
+
+
+<screen>
+alice$ cd /tmp
+alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
+alice$ cd nix-1.8-x86_64-darwin
+alice$ ./install
+</screen>
+  </para>
+
+  <para>
+    If you need to edit the multi-user installation script to use
+    different group ID or a different user ID range, modify the
+    variables set in the file named
+    <filename>install-multi-user</filename>.
+  </para>
+</section>
 </chapter>

From 51f9682a8b16c4f97856510c1cc99576326c9192 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 1 Sep 2018 08:51:32 -0400
Subject: [PATCH 1195/2196] Default to single-user install

---
 scripts/install-nix-from-closure.sh | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index cd71d7947d7..ab20774bbf0 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -30,15 +30,14 @@ if [ "$(uname -s)" = "Darwin" ]; then
     fi
 fi
 
-# Determine if we should punt to the single-user installer or not
+# Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Darwin" ]; then
-    INSTALL_MODE=daemon
+    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
 elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
-    INSTALL_MODE=daemon
-else
-    INSTALL_MODE=no-daemon
+    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
 fi
 
+INSTALL_MODE=no-daemon
 # Trivially handle the --daemon / --no-daemon options
 if [ "x${1:-}" = "x--no-daemon" ]; then
     INSTALL_MODE=no-daemon
@@ -47,14 +46,18 @@ elif [ "x${1:-}" = "x--daemon" ]; then
 elif [ "x${1:-}" != "x" ]; then
     (
         echo "Nix Installer [--daemon|--no-daemon]"
+
+        echo "Choose installation method."
         echo ""
-        echo " --daemon:    Force the installer to use the Daemon"
-        echo "              based installer, even though it may not"
-        echo "              work."
+        echo " --daemon:    Installs and configures a background daemon that manages the store,"
+        echo "              providing multi-user support and better isolation for local builds."
+        echo "              Both for security and reproducibility, this method is recommended if"
+        echo "              supported on your platform."
+        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
         echo ""
-        echo " --no-daemon: Force a no-daemon, single-user"
-        echo "              installation even when the preferred"
-        echo "              method is with the daemon."
+        echo " --no-daemon: Simple, single-user installation that does not require root and is"
+        echo "              trivial to uninstall."
+        echo "              (default)"
         echo ""
     ) >&2
     exit

From 4be7652dd3c577ce0890ee849c041e4cb59a3db7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 31 Aug 2018 11:47:42 -0400
Subject: [PATCH 1196/2196] release notes: note that the multi-user installer
 is available but not selected for Linux with systemd, and the bug about
 selinux

---
 doc/manual/release-notes/rl-2.1.xml | 30 +++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index 9a5f37f6625..20a8e4ea6ca 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -49,8 +49,38 @@ new features:</para>
     <varname>nix-support/propagated-user-env-packages</varname>.</para>
   </listitem>
 
+  <listitem>
+    <para>The Nix installer will no longer default to the Multi-User
+    installation for macOS. You can still <link
+    linkend="sect-multi-user-installation">instruct the installer to
+    run in multi-user mode</link>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The Nix installer now supports performing a Multi-User
+    installation for Linux computers which are running systemd. You
+    can <link
+    linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the
+    <option>--daemon</option> flag to the installer: <command>sh &lt;(curl
+    https://nixos.org/nix/install) --daemon</command>.
+    </para>
+
+    <para>The multi-user installer cannot handle systems with SELinux.
+    If your system has SELinux enabled, you can <link
+    linkend="sect-single-user-installation">force the installer to run
+    in single-user mode</link>.</para>
+  </listitem>
+
 </itemizedlist>
 
+<para>
+  Multi-user Nix users on macOS can upgrade Nix by running
+  <command>sudo -i sh -c 'nix-channel --update &amp;&amp; nix-env -iA
+  nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo
+  launchctl start org.nixos.nix-daemon</command>.
+</para>
+
 <para>This release has contributions from
 
 Adrien Devresse,

From c42eaaf684a6b1f7358d33cfaea1614885467d72 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 1 Sep 2018 15:35:46 -0400
Subject: [PATCH 1197/2196] Create upgrade notes

---
 doc/manual/installation/upgrading.xml | 21 +++++++++++++++++++++
 doc/manual/manual.xml                 |  1 +
 doc/manual/release-notes/rl-2.1.xml   |  7 -------
 3 files changed, 22 insertions(+), 7 deletions(-)
 create mode 100644 doc/manual/installation/upgrading.xml

diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml
new file mode 100644
index 00000000000..a3f86ade95c
--- /dev/null
+++ b/doc/manual/installation/upgrading.xml
@@ -0,0 +1,21 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ch-upgrading-nix">
+
+  <title>Upgrading Nix</title>
+
+  <para>
+    Multi-user Nix users on macOS can upgrade Nix by running
+    <command>sudo -i sh -c 'nix-channel --update &amp;&amp; nix-env
+    -iA nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo
+    launchctl start org.nixos.nix-daemon</command>.
+  </para>
+
+  <para>
+    Single-user installations of Nix should run <command>nix-channel
+    --update; nix-env -iA nixpkgs.nix</command>.
+  </para>
+
+</chapter>
diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml
index b408b681772..87d9de28ab1 100644
--- a/doc/manual/manual.xml
+++ b/doc/manual/manual.xml
@@ -32,6 +32,7 @@
 
   <xi:include href="introduction/introduction.xml" />
   <xi:include href="installation/installation.xml" />
+  <xi:include href="installation/upgrading.xml" />
   <xi:include href="packages/package-management.xml" />
   <xi:include href="expressions/writing-nix-expressions.xml" />
   <xi:include href="advanced-topics/advanced-topics.xml" />
diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index 20a8e4ea6ca..3cace13f10f 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -74,13 +74,6 @@ new features:</para>
 
 </itemizedlist>
 
-<para>
-  Multi-user Nix users on macOS can upgrade Nix by running
-  <command>sudo -i sh -c 'nix-channel --update &amp;&amp; nix-env -iA
-  nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo
-  launchctl start org.nixos.nix-daemon</command>.
-</para>
-
 <para>This release has contributions from
 
 Adrien Devresse,

From c29e5fbb1306a2da869260153cd38cf25986d662 Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Sat, 1 Sep 2018 17:11:56 -0300
Subject: [PATCH 1198/2196] improve the stats when profiling

---
 src/libexpr/eval.cc | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f41905787f9..77d1faced72 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1739,17 +1739,16 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
+    auto sum = bEnvs + bLists + bValues + bAttrsets;
+
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
     printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated count: %1%") % nrEnvs);
-    printMsg(v, format("  environments allocated bytes: %1%") % bEnvs);
-    printMsg(v, format("  list elements count: %1%") % nrListElems);
-    printMsg(v, format("  list elements bytes: %1%") % bLists);
+    printMsg(v, format("  environments allocated: %1% (%2% bytes, %3%%%)") % nrEnvs % bEnvs % ((bEnvs*100) / sum));
+    printMsg(v, format("  list elements: %1% (%2% bytes %3%%%)") % nrListElems % bLists % ((bLists*100)/sum));
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated count: %1%") % nrValues);
-    printMsg(v, format("  values allocated bytes: %1%") % bValues);
-    printMsg(v, format("  sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
+    printMsg(v, format("  values allocated: %1% (%2% bytes %3%%%)") % nrValues % bValues % ((bValues*100)/sum));
+    printMsg(v, format("  sets allocated: %1% (%2% bytes %3%%%)") % nrAttrsets % bAttrsets % ((bAttrsets*100)/sum));
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
     printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
@@ -1761,6 +1760,8 @@ void EvalState::printStats()
     printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
     printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
 
+    printMsg(v, format("  sets: %1% (%2% each, %3% mb total), attrs-in-sets: %4% (%5% each, %6% mb total)") % nrAttrsets % sizeof(Bindings) % ((nrAttrsets * sizeof(Bindings)) / 1024 / 1024) % nrAttrsInAttrsets % sizeof(Attr) % ((nrAttrsInAttrsets * sizeof(Attr)) / 1024 / 1024));
+
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);

From 2fd1008c708be68b56d39be2e1230f29b2ac5ab1 Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Sat, 1 Sep 2018 19:50:22 -0300
Subject: [PATCH 1199/2196] add JSON to NIX_SHOW_STATS

---
 src/libexpr/eval.cc | 114 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 90 insertions(+), 24 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 77d1faced72..9ad0b1a2043 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -6,12 +6,15 @@
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "download.hh"
+#include "json.hh"
 
 #include <algorithm>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <iostream>
+#include <fstream>
 
 #include <sys/time.h>
 #include <sys/resource.h>
@@ -23,6 +26,7 @@
 
 #endif
 
+using std::cout;
 
 namespace nix {
 
@@ -1723,10 +1727,10 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     }
 }
 
-
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
+    bool showJsonStats = getEnv("NIX_SHOW_STATS", "0") == "json";
     Verbosity v = showStats ? lvlInfo : lvlDebug;
     printMsg(v, "evaluation statistics:");
 
@@ -1741,33 +1745,95 @@ void EvalState::printStats()
 
     auto sum = bEnvs + bLists + bValues + bAttrsets;
 
-    printMsg(v, format("  time elapsed: %1%") % cpuTime);
-    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-    printMsg(v, format("  environments allocated: %1% (%2% bytes, %3%%%)") % nrEnvs % bEnvs % ((bEnvs*100) / sum));
-    printMsg(v, format("  list elements: %1% (%2% bytes %3%%%)") % nrListElems % bLists % ((bLists*100)/sum));
-    printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-    printMsg(v, format("  values allocated: %1% (%2% bytes %3%%%)") % nrValues % bValues % ((bValues*100)/sum));
-    printMsg(v, format("  sets allocated: %1% (%2% bytes %3%%%)") % nrAttrsets % bAttrsets % ((bAttrsets*100)/sum));
-    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
-    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
-    printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
-    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
-    printMsg(v, format("  number of thunks: %1%") % nrThunks);
-    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
-    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
-    printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
-    printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
-    printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-
-    printMsg(v, format("  sets: %1% (%2% each, %3% mb total), attrs-in-sets: %4% (%5% each, %6% mb total)") % nrAttrsets % sizeof(Bindings) % ((nrAttrsets * sizeof(Bindings)) / 1024 / 1024) % nrAttrsInAttrsets % sizeof(Attr) % ((nrAttrsInAttrsets * sizeof(Attr)) / 1024 / 1024));
-
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
-    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
-    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
 #endif
+    if (showJsonStats) {
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        std::fstream fs;
+        if (outPath != "-") {
+            fs.open(outPath, std::fstream::out);
+            printMsg(v, format("  written to: %1%") % outPath);
+        }
+        JSONObject topObj(outPath == "-" ? cout : fs, true);
+        topObj.attr("cpuTime",cpuTime);
+        {
+            auto envs = topObj.object("envs");
+            envs.attr("number", nrEnvs);
+            envs.attr("bytes", bEnvs);
+        }
+        {
+            auto lists = topObj.object("list");
+            lists.attr("elements", nrListElems);
+            lists.attr("bytes", bLists);
+            lists.attr("concats", nrListConcats);
+        }
+        {
+            auto values = topObj.object("values");
+            values.attr("number", nrValues);
+            values.attr("bytes", bValues);
+        }
+        {
+            auto syms = topObj.object("symbols");
+            syms.attr("number", symbols.size());
+            syms.attr("bytes", symbols.totalSize());
+        }
+        {
+            auto sets = topObj.object("sets");
+            sets.attr("number", nrAttrsets);
+            sets.attr("bytes", bAttrsets);
+        }
+        {
+            JSONObject sizes = topObj.object("sizes");
+            sizes.attr("Env", sizeof(Env));
+            sizes.attr("Value", sizeof(Value));
+            sizes.attr("Bindings", sizeof(Bindings));
+            sizes.attr("Attr", sizeof(Attr));
+        }
+        topObj.attr("nrOpUpdates", nrOpUpdates);
+        topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied);
+        topObj.attr("nrThunks", nrThunks);
+        topObj.attr("nrAvoided", nrAvoided);
+        topObj.attr("nrLookups", nrLookups);
+        topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
+        topObj.attr("nrFunctionCalls", nrFunctionCalls);
+#if HAVE_BOEHMGC
+        JSONObject gc = topObj.object("gc");
+        gc.attr("heapSize", heapSize);
+        gc.attr("totalBytes", totalBytes);
+#endif
+    } else {
+        if (getEnv("NIX_SHOW_STATS_PATH","-") != "-") {
+            printError("warning: $NIX_SHOW_STATS_PATH only works in combination with NIX_SHOW_STATS=json");
+        }
+        printMsg(v, format("  time elapsed: %1%") % cpuTime);
+        printMsg(v, format("  size of a value: %1%") % sizeof(Value));
+        printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
+        printMsg(v, format("  environments allocated: %1% (%2% bytes, %3%%%)") % nrEnvs % bEnvs % ((bEnvs*100) / sum));
+        printMsg(v, format("  list elements: %1% (%2% bytes %3%%%)") % nrListElems % bLists % ((bLists*100)/sum));
+        printMsg(v, format("  list concatenations: %1%") % nrListConcats);
+        printMsg(v, format("  values allocated: %1% (%2% bytes %3%%%)") % nrValues % bValues % ((bValues*100)/sum));
+        printMsg(v, format("  sets allocated: %1% (%2% bytes %3%%%)") % nrAttrsets % bAttrsets % ((bAttrsets*100)/sum));
+        printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
+        printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
+        printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
+        printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
+        printMsg(v, format("  number of thunks: %1%") % nrThunks);
+        printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
+        printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
+        printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
+        printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
+        printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
+
+        printMsg(v, format("  sets: %1% (%2% each, %3% mb total), attrs-in-sets: %4% (%5% each, %6% mb total)") % nrAttrsets % sizeof(Bindings) % ((nrAttrsets * sizeof(Bindings)) / 1024 / 1024) % nrAttrsInAttrsets % sizeof(Attr) % ((nrAttrsInAttrsets * sizeof(Attr)) / 1024 / 1024));
+
+#if HAVE_BOEHMGC
+        printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
+        printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
+#endif
+    }
+
 
     if (countCalls) {
         v = lvlInfo;

From c9a08540c3d64d1285928d1ce3d3d416a2547dd9 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 30 Aug 2018 00:59:29 +0200
Subject: [PATCH 1200/2196] nix doctor: add command

Inspired by the homebrew command, shows a combination of debugging
information and warnings with potential issues with a nix installation.
---
 src/nix/doctor.cc | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 src/nix/doctor.cc

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
new file mode 100644
index 00000000000..fb4fc2a6e7e
--- /dev/null
+++ b/src/nix/doctor.cc
@@ -0,0 +1,26 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdDoctor : StoreCommand
+{
+    std::string name() override
+    {
+        return "doctor";
+    }
+
+    std::string description() override
+    {
+        return "check your system for potential problems";
+    }
+
+    void run(ref<Store> store) override
+    {
+        std::cout << "Store uri: " << store->getUri() << std::endl;
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdDoctor>());
+

From 070823baa4c3c397c8a5eb0378944187e7f4903c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 30 Aug 2018 23:28:47 +0200
Subject: [PATCH 1201/2196] Store: expose the protocol version used by a store

---
 src/libstore/legacy-ssh-store.cc | 6 ++++++
 src/libstore/local-store.cc      | 6 ++++++
 src/libstore/local-store.hh      | 2 ++
 src/libstore/remote-store.cc     | 7 +++++++
 src/libstore/remote-store.hh     | 2 ++
 src/libstore/store-api.hh        | 6 ++++++
 6 files changed, 29 insertions(+)

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 88d2574e86e..26e1851981d 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -303,6 +303,12 @@ struct LegacySSHStore : public Store
     {
         auto conn(connections->get());
     }
+
+    unsigned int getProtocol() override
+    {
+        auto conn(connections->get());
+        return conn->remoteVersion;
+    }
 };
 
 static RegisterStoreImplementation regStore([](
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c91dbf241bc..c8117c0c650 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1332,6 +1332,12 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
 }
 
 
+unsigned int LocalStore::getProtocol()
+{
+    return PROTOCOL_VERSION;
+}
+
+
 #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
 
 static void makeMutable(const Path & path)
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 746bdbeed79..fce963433a5 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -209,6 +209,8 @@ public:
 
     void registerValidPaths(const ValidPathInfos & infos);
 
+    unsigned int getProtocol() override;
+
     void vacuumDB();
 
     /* Repair the contents of the given path by redownloading it using
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ea86ef052f5..eff5d252419 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -646,6 +646,13 @@ void RemoteStore::connect()
 }
 
 
+unsigned int RemoteStore::getProtocol()
+{
+    auto conn(connections->get());
+    return conn->daemonVersion;
+}
+
+
 void RemoteStore::flushBadConnections()
 {
     connections->flushBad();
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b488e34ce26..16daee8b673 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -97,6 +97,8 @@ public:
 
     void connect() override;
 
+    unsigned int getProtocol() override;
+
     void flushBadConnections();
 
 protected:
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7c5b495a448..c2f964e11f7 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -598,6 +598,12 @@ public:
        a notion of connection. Otherwise this is a no-op. */
     virtual void connect() { };
 
+    /* Get the protocol version of this store or it's connection. */
+    virtual unsigned int getProtocol()
+    {
+        return 0;
+    };
+
     /* Get the priority of the store, used to order substituters. In
        particular, binary caches can specify a priority field in their
        "nix-cache-info" file. Lower value means higher priority. */

From 7314dc7f07a90782dea5cc9d298c7f7148e3b7c3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 30 Aug 2018 23:42:28 +0200
Subject: [PATCH 1202/2196] nix doctor: add warning if client/daemon protocol
 mismatches

A protocol mismatch can sometimes cause problems when using specific
features with an older daemon. For example:

Nix 2.0 changed the way files are compied to the store.  The daemon is
backwards compatible and can still handle older clients, however a 1.11
nix-daemon isn't forwards compatible.
---
 src/nix/doctor.cc | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index fb4fc2a6e7e..6ef5eb9d30e 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -1,9 +1,20 @@
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "worker-protocol.hh"
 
 using namespace nix;
 
+std::string formatProtocol(unsigned int proto)
+{
+    if (proto) {
+        auto major = GET_PROTOCOL_MAJOR(proto) >> 8;
+        auto minor = GET_PROTOCOL_MINOR(proto);
+        return (format("%1%.%2%") % major % minor).str();
+    }
+    return "unknown";
+}
+
 struct CmdDoctor : StoreCommand
 {
     std::string name() override
@@ -19,8 +30,22 @@ struct CmdDoctor : StoreCommand
     void run(ref<Store> store) override
     {
         std::cout << "Store uri: " << store->getUri() << std::endl;
+        std::cout << std::endl;
+
+        checkStoreProtocol(store->getProtocol());
+    }
+
+    void checkStoreProtocol(unsigned int proto) {
+        if (PROTOCOL_VERSION != proto) {
+            std::cout << "Warning: protocol version of this client does not match the store." << std::endl;
+            std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl;
+            std::cout << "sync with the daemon." << std::endl;
+            std::cout << std::endl;
+            std::cout << "Client protocol: " << formatProtocol(PROTOCOL_VERSION) << std::endl;
+            std::cout << "Store protocol: " << formatProtocol(proto) << std::endl;
+            std::cout << std::endl;
+        }
     }
 };
 
 static RegisterCommand r1(make_ref<CmdDoctor>());
-

From 246acf93f2b61b2915e2140d761b19c2e836a96e Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 31 Aug 2018 01:01:59 +0200
Subject: [PATCH 1203/2196] nix doctor: handle serve protocol

The serve protocol used by LegacySSHStore has a different major and
shouldn't be compared to PROTOCOL_VERSION.
---
 src/nix/doctor.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 6ef5eb9d30e..a31df595da8 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "serve-protocol.hh"
 #include "shared.hh"
 #include "store-api.hh"
 #include "worker-protocol.hh"
@@ -35,14 +36,18 @@ struct CmdDoctor : StoreCommand
         checkStoreProtocol(store->getProtocol());
     }
 
-    void checkStoreProtocol(unsigned int proto) {
-        if (PROTOCOL_VERSION != proto) {
+    void checkStoreProtocol(unsigned int storeProto) {
+        auto clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
+            ? SERVE_PROTOCOL_VERSION
+            : PROTOCOL_VERSION;
+
+        if (clientProto != storeProto) {
             std::cout << "Warning: protocol version of this client does not match the store." << std::endl;
             std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl;
             std::cout << "sync with the daemon." << std::endl;
             std::cout << std::endl;
-            std::cout << "Client protocol: " << formatProtocol(PROTOCOL_VERSION) << std::endl;
-            std::cout << "Store protocol: " << formatProtocol(proto) << std::endl;
+            std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl;
+            std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl;
             std::cout << std::endl;
         }
     }

From 0f18dc54797a1850bc1b91673790ad73e8f4b82f Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 2 Sep 2018 01:01:23 +0200
Subject: [PATCH 1204/2196] nix doctor: add warning for multiple versions

It's pretty easy to unintentionally install a second version of nix
into the user profile when using a daemon install.  In this case it
looks like nix was upgraded while the nix-daemon is probably still
unning an older version.
---
 src/nix/doctor.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index a31df595da8..6265e9cfef2 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -33,9 +33,26 @@ struct CmdDoctor : StoreCommand
         std::cout << "Store uri: " << store->getUri() << std::endl;
         std::cout << std::endl;
 
+        checkNixInPath();
         checkStoreProtocol(store->getProtocol());
     }
 
+    void checkNixInPath() {
+        PathSet dirs;
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+            if (pathExists(dir + "/nix-env"))
+                dirs.insert(dirOf(canonPath(dir + "/nix-env", true)));
+
+        if (dirs.size() != 1) {
+            std::cout << "Warning: multiple versions of nix found in PATH." << std::endl;
+            std::cout << std::endl;
+            for (auto & dir : dirs)
+                std::cout << "  " << dir << std::endl;
+            std::cout << std::endl;
+        }
+    }
+
     void checkStoreProtocol(unsigned int storeProto) {
         auto clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
             ? SERVE_PROTOCOL_VERSION

From bfdca55868ac1cd336c5d73ff944098ce82d023d Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 2 Sep 2018 12:52:04 +0200
Subject: [PATCH 1205/2196] nix doctor: add check for profile roots

In most cases profiles that are in PATH should have a gcroot.
---
 src/nix/doctor.cc | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 6265e9cfef2..2a8af7780b9 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -34,6 +34,7 @@ struct CmdDoctor : StoreCommand
         std::cout << std::endl;
 
         checkNixInPath();
+        checkProfileRoots(store);
         checkStoreProtocol(store->getProtocol());
     }
 
@@ -53,6 +54,40 @@ struct CmdDoctor : StoreCommand
         }
     }
 
+    void checkProfileRoots(ref<Store> store) {
+        PathSet dirs;
+
+        Roots roots = store->findRoots();
+
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+            try {
+                auto profileDir = canonPath(dirOf(dir), true);
+                if (hasSuffix(profileDir, "user-environment") &&
+                    store->isValidPath(profileDir)) {
+                    PathSet referrers;
+                    store->computeFSClosure({profileDir}, referrers, true,
+                            settings.gcKeepOutputs, settings.gcKeepDerivations);
+                    bool found = false;
+                    for (auto & i : roots)
+                        if (referrers.find(i.second) != referrers.end())
+                            found = true;
+                    if (!found)
+                        dirs.insert(dir);
+
+                }
+            } catch (SysError &) {}
+
+        if (!dirs.empty()) {
+            std::cout << "Warning: found profiles without a gcroot." << std::endl;
+            std::cout << "The generation this profile points to will be deleted with the next gc, resulting" << std::endl;
+            std::cout << "in broken symlinks.  Make sure your profiles are in " << settings.nixStateDir << "/profiles." << std::endl;
+            std::cout << std::endl;
+            for (auto & dir : dirs)
+                std::cout << "  " << dir << std::endl;
+            std::cout << std::endl;
+        }
+    }
+
     void checkStoreProtocol(unsigned int storeProto) {
         auto clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
             ? SERVE_PROTOCOL_VERSION

From 80a4b44d3d1bd35ab13d6b534f1015d15fdb08fd Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 2 Sep 2018 13:14:39 +0200
Subject: [PATCH 1206/2196] nix doctor: only perform path/profile checks with a
 daemon/local store

Not all store types LegacySSHStore support these operations and it
doesn't really make sense to check those.
---
 src/nix/doctor.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 2a8af7780b9..6112b1f650a 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -33,8 +33,12 @@ struct CmdDoctor : StoreCommand
         std::cout << "Store uri: " << store->getUri() << std::endl;
         std::cout << std::endl;
 
-        checkNixInPath();
-        checkProfileRoots(store);
+        auto type = getStoreType();
+
+        if (type < tOther) {
+            checkNixInPath();
+            checkProfileRoots(store);
+        }
         checkStoreProtocol(store->getProtocol());
     }
 
@@ -56,7 +60,6 @@ struct CmdDoctor : StoreCommand
 
     void checkProfileRoots(ref<Store> store) {
         PathSet dirs;
-
         Roots roots = store->findRoots();
 
         for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))

From 4dd09210d7d32980ffbfc392dd7f8a42185dd9a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 2 Sep 2018 21:47:10 +0200
Subject: [PATCH 1207/2196] Release notes tweaks

---
 doc/manual/release-notes/rl-2.1.xml | 48 ++++++++++++++---------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.1.xml b/doc/manual/release-notes/rl-2.1.xml
index 3cace13f10f..16c243fc191 100644
--- a/doc/manual/release-notes/rl-2.1.xml
+++ b/doc/manual/release-notes/rl-2.1.xml
@@ -4,7 +4,7 @@
       version="5.0"
       xml:id="ssec-relnotes-2.1">
 
-<title>Release 2.1 (2018-08-31)</title>
+<title>Release 2.1 (2018-09-02)</title>
 
 <para>This is primarily a bug fix release. It also reduces memory
 consumption in certain situations. In addition, it has the following
@@ -12,6 +12,29 @@ new features:</para>
 
 <itemizedlist>
 
+  <listitem>
+    <para>The Nix installer will no longer default to the Multi-User
+    installation for macOS. You can still <link
+    linkend="sect-multi-user-installation">instruct the installer to
+    run in multi-user mode</link>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The Nix installer now supports performing a Multi-User
+    installation for Linux computers which are running systemd. You
+    can <link
+    linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the
+    <option>--daemon</option> flag to the installer: <command>sh &lt;(curl
+    https://nixos.org/nix/install) --daemon</command>.
+    </para>
+
+    <para>The multi-user installer cannot handle systems with SELinux.
+    If your system has SELinux enabled, you can <link
+    linkend="sect-single-user-installation">force the installer to run
+    in single-user mode</link>.</para>
+  </listitem>
+
   <listitem>
     <para>New builtin functions:
     <literal>builtins.bitAnd</literal>,
@@ -49,29 +72,6 @@ new features:</para>
     <varname>nix-support/propagated-user-env-packages</varname>.</para>
   </listitem>
 
-  <listitem>
-    <para>The Nix installer will no longer default to the Multi-User
-    installation for macOS. You can still <link
-    linkend="sect-multi-user-installation">instruct the installer to
-    run in multi-user mode</link>.
-    </para>
-  </listitem>
-
-  <listitem>
-    <para>The Nix installer now supports performing a Multi-User
-    installation for Linux computers which are running systemd. You
-    can <link
-    linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the
-    <option>--daemon</option> flag to the installer: <command>sh &lt;(curl
-    https://nixos.org/nix/install) --daemon</command>.
-    </para>
-
-    <para>The multi-user installer cannot handle systems with SELinux.
-    If your system has SELinux enabled, you can <link
-    linkend="sect-single-user-installation">force the installer to run
-    in single-user mode</link>.</para>
-  </listitem>
-
 </itemizedlist>
 
 <para>This release has contributions from

From 54996b51fba46e8f287b1ac6a58854e62b476985 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 2 Sep 2018 22:12:00 +0200
Subject: [PATCH 1208/2196] Bump version

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 42f7d2336ea..616187889b6 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.1
\ No newline at end of file
+2.2
\ No newline at end of file

From 4b034f390c207d997465112a814adb60797e1d4d Mon Sep 17 00:00:00 2001
From: Michael Bishop <cleverca22@gmail.com>
Date: Sun, 2 Sep 2018 18:20:18 -0300
Subject: [PATCH 1209/2196] remove the old text format output

---
 src/libexpr/eval.cc | 38 ++++----------------------------------
 1 file changed, 4 insertions(+), 34 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9ad0b1a2043..a67b7ae0dd6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1730,9 +1730,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
-    bool showJsonStats = getEnv("NIX_SHOW_STATS", "0") == "json";
     Verbosity v = showStats ? lvlInfo : lvlDebug;
-    printMsg(v, "evaluation statistics:");
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1743,13 +1741,12 @@ void EvalState::printStats()
     uint64_t bValues = nrValues * sizeof(Value);
     uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
 
-    auto sum = bEnvs + bLists + bValues + bAttrsets;
-
 #if HAVE_BOEHMGC
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
 #endif
-    if (showJsonStats) {
+    if (showStats) {
+        printMsg(v, "evaluation statistics:");
         auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
         std::fstream fs;
         if (outPath != "-") {
@@ -1761,6 +1758,7 @@ void EvalState::printStats()
         {
             auto envs = topObj.object("envs");
             envs.attr("number", nrEnvs);
+            envs.attr("elements", nrValuesInEnvs);
             envs.attr("bytes", bEnvs);
         }
         {
@@ -1783,6 +1781,7 @@ void EvalState::printStats()
             auto sets = topObj.object("sets");
             sets.attr("number", nrAttrsets);
             sets.attr("bytes", bAttrsets);
+            sets.attr("elements", nrAttrsInAttrsets);
         }
         {
             JSONObject sizes = topObj.object("sizes");
@@ -1802,35 +1801,6 @@ void EvalState::printStats()
         JSONObject gc = topObj.object("gc");
         gc.attr("heapSize", heapSize);
         gc.attr("totalBytes", totalBytes);
-#endif
-    } else {
-        if (getEnv("NIX_SHOW_STATS_PATH","-") != "-") {
-            printError("warning: $NIX_SHOW_STATS_PATH only works in combination with NIX_SHOW_STATS=json");
-        }
-        printMsg(v, format("  time elapsed: %1%") % cpuTime);
-        printMsg(v, format("  size of a value: %1%") % sizeof(Value));
-        printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
-        printMsg(v, format("  environments allocated: %1% (%2% bytes, %3%%%)") % nrEnvs % bEnvs % ((bEnvs*100) / sum));
-        printMsg(v, format("  list elements: %1% (%2% bytes %3%%%)") % nrListElems % bLists % ((bLists*100)/sum));
-        printMsg(v, format("  list concatenations: %1%") % nrListConcats);
-        printMsg(v, format("  values allocated: %1% (%2% bytes %3%%%)") % nrValues % bValues % ((bValues*100)/sum));
-        printMsg(v, format("  sets allocated: %1% (%2% bytes %3%%%)") % nrAttrsets % bAttrsets % ((bAttrsets*100)/sum));
-        printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
-        printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
-        printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
-        printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
-        printMsg(v, format("  number of thunks: %1%") % nrThunks);
-        printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
-        printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
-        printMsg(v, format("  number of primop calls: %1%") % nrPrimOpCalls);
-        printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
-        printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
-
-        printMsg(v, format("  sets: %1% (%2% each, %3% mb total), attrs-in-sets: %4% (%5% each, %6% mb total)") % nrAttrsets % sizeof(Bindings) % ((nrAttrsets * sizeof(Bindings)) / 1024 / 1024) % nrAttrsInAttrsets % sizeof(Attr) % ((nrAttrsInAttrsets * sizeof(Attr)) / 1024 / 1024));
-
-#if HAVE_BOEHMGC
-        printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
-        printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
 #endif
     }
 

From 5f3b72cfc2f729f6bd5cf1aea9720678b76b2bf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <vladimir.cunat@nic.cz>
Date: Mon, 3 Sep 2018 17:57:13 +0200
Subject: [PATCH 1210/2196] docs: change expired bzip2.org to archive.org

Fixes #2396.
---
 configure.ac                                     | 4 ++--
 doc/manual/installation/prerequisites-source.xml | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index cc354f6f302..9c85182efbf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -152,9 +152,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]
 
 # Look for libbz2, a required dependency.
 AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 
 
 # Look for SQLite, a required dependency.
diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index 01e9688d635..ef14a1d753d 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -29,7 +29,8 @@
   <literal>libbz2</literal> library.  Thus you must have bzip2
   installed, including development headers and libraries.  If your
   distribution does not provide these, you can obtain bzip2 from <link
-  xlink:href="http://www.bzip.org/"/>.</para></listitem>
+  xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/"
+  />.</para></listitem>
 
   <listitem><para><literal>liblzma</literal>, which is provided by
   XZ Utils. If your distribution does not provide this, you can

From 28418af920284e0ea830e2f0339efadb0660efa1 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 27 Aug 2018 08:41:17 -0500
Subject: [PATCH 1211/2196] download: fix size reported to progress bar

---
 src/libstore/download.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 973fca0b130..13913d031da 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -345,7 +345,7 @@ struct CurlDownloader : public Downloader
                 done = true;
 
                 try {
-                    act.progress(result.data->size(), result.data->size());
+                    act.progress(result.bodySize, result.bodySize);
                     callback(std::move(result));
                 } catch (...) {
                     done = true;

From 74f6d8767de13980b0e83a249633ee4a7e6fc547 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 4 Sep 2018 19:32:39 -0500
Subject: [PATCH 1212/2196] Get effective user in Nix commands
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

‘geteuid’ gives us the user that the command is being run as,
including in setuid modes. By using geteuid to determind id, we can
avoid the ‘sudo -i’ hack when upgrading Nix. So now, upgrading Nix on
macOS is as simple as:

$ sudo nix-channel --update
$ sudo nix-env -u

$ sudo launchctl stop org.nixos.nix-daemon
$ sudo launchctl start org.nixos.nix-daemon
or
$ sudo systemctl restart nix-daemon
---
 src/libutil/util.cc            | 2 +-
 src/nix-channel/nix-channel.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6bc64ae75a4..04c98dc80bf 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -468,7 +468,7 @@ static Lazy<Path> getHome2([]() {
         std::vector<char> buf(16384);
         struct passwd pwbuf;
         struct passwd * pw;
-        if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
+        if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
             || !pw || !pw->pw_dir || !pw->pw_dir[0])
             throw Error("cannot determine user's home directory");
         homeDir = pw->pw_dir;
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 55ebda43896..2083d3df5ca 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -169,7 +169,7 @@ int main(int argc, char ** argv)
 
         // Figure out the name of the channels profile.
         ;
-        auto pw = getpwuid(getuid());
+        auto pw = getpwuid(geteuid());
         std::string name = pw ? pw->pw_name : getEnv("USER", "");
         if (name.empty())
             throw Error("cannot figure out user name");

From 2d91012754dbe707fe60a0017496f78076d6694a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 5 Sep 2018 21:22:37 +0200
Subject: [PATCH 1213/2196] fetchurl: Respect unpack

Fixes #2393.
---
 src/libstore/builtins/fetchurl.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index b4dcb35f951..92aec63a037 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -24,6 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     Path storePath = getAttr("out");
     auto mainUrl = getAttr("url");
+    bool unpack = get(drv.env, "unpack", "") == "1";
 
     /* Note: have to use a fresh downloader here because we're in
        a forked process. */
@@ -40,12 +41,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
-                hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
+                unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
             downloader->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
-        if (get(drv.env, "unpack", "") == "1")
+        if (unpack)
             restorePath(storePath, *source);
         else
             writeFile(storePath, *source);

From 0a2545f95c155588a82f0e112aa1e8853e58477d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 5 Sep 2018 21:35:58 +0200
Subject: [PATCH 1214/2196] Log stats to stderr

We shouldn't pollute stdout.
---
 src/libexpr/eval.cc | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index a67b7ae0dd6..08a2c2bfc6b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -26,8 +26,6 @@
 
 #endif
 
-using std::cout;
-
 namespace nix {
 
 
@@ -1746,14 +1744,11 @@ void EvalState::printStats()
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
 #endif
     if (showStats) {
-        printMsg(v, "evaluation statistics:");
         auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
         std::fstream fs;
-        if (outPath != "-") {
+        if (outPath != "-")
             fs.open(outPath, std::fstream::out);
-            printMsg(v, format("  written to: %1%") % outPath);
-        }
-        JSONObject topObj(outPath == "-" ? cout : fs, true);
+        JSONObject topObj(outPath == "-" ? std::cerr : fs, true);
         topObj.attr("cpuTime",cpuTime);
         {
             auto envs = topObj.object("envs");

From 91405986f4d076d08051ebe980af6ac6ba124323 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 5 Sep 2018 21:57:54 +0200
Subject: [PATCH 1215/2196] Convert NIX_COUNT_CALLS to JSON too

---
 src/libexpr/eval.cc | 76 ++++++++++++++++++++++++---------------------
 1 file changed, 41 insertions(+), 35 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 08a2c2bfc6b..ab407e56907 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1728,7 +1728,6 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 void EvalState::printStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
-    Verbosity v = showStats ? lvlInfo : lvlDebug;
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1779,7 +1778,7 @@ void EvalState::printStats()
             sets.attr("elements", nrAttrsInAttrsets);
         }
         {
-            JSONObject sizes = topObj.object("sizes");
+            auto sizes = topObj.object("sizes");
             sizes.attr("Env", sizeof(Env));
             sizes.attr("Value", sizeof(Value));
             sizes.attr("Bindings", sizeof(Bindings));
@@ -1793,40 +1792,47 @@ void EvalState::printStats()
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
 #if HAVE_BOEHMGC
-        JSONObject gc = topObj.object("gc");
-        gc.attr("heapSize", heapSize);
-        gc.attr("totalBytes", totalBytes);
+        {
+            auto gc = topObj.object("gc");
+            gc.attr("heapSize", heapSize);
+            gc.attr("totalBytes", totalBytes);
+        }
 #endif
-    }
-
-
-    if (countCalls) {
-        v = lvlInfo;
-
-        printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
-        typedef std::multimap<size_t, Symbol> PrimOpCalls_;
-        PrimOpCalls_ primOpCalls_;
-        for (auto & i : primOpCalls)
-            primOpCalls_.insert(std::pair<size_t, Symbol>(i.second, i.first));
-        for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
-
-        printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<size_t, ExprLambda *> FunctionCalls_;
-        FunctionCalls_ functionCalls_;
-        for (auto & i : functionCalls)
-            functionCalls_.insert(std::pair<size_t, ExprLambda *>(i.second, i.first));
-        for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
-
-        printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
-        typedef std::multimap<size_t, Pos> AttrSelects_;
-        AttrSelects_ attrSelects_;
-        for (auto & i : attrSelects)
-            attrSelects_.insert(std::pair<size_t, Pos>(i.second, i.first));
-        for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
-
+        if (countCalls) {
+            {
+                auto obj = topObj.object("primops");
+                for (auto & i : primOpCalls)
+                    obj.attr(i.first, i.second);
+            }
+            {
+                auto list = topObj.list("functions");
+                for (auto & i : functionCalls) {
+                    auto obj = list.object();
+                    if (i.first->name.set())
+                        obj.attr("name", (const string &) i.first->name);
+                    else
+                        obj.attr("name", nullptr);
+                    if (i.first->pos) {
+                        obj.attr("file", (const string &) i.first->pos.file);
+                        obj.attr("line", i.first->pos.line);
+                        obj.attr("column", i.first->pos.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+            {
+                auto list = topObj.list("attributes");
+                for (auto & i : attrSelects) {
+                    auto obj = list.object();
+                    if (i.first) {
+                        obj.attr("file", (const string &) i.first.file);
+                        obj.attr("line", i.first.line);
+                        obj.attr("column", i.first.column);
+                    }
+                    obj.attr("count", i.second);
+                }
+            }
+        }
     }
 }
 

From 33c3f91885429b60cab551e761f297a640aff7d5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Sep 2018 16:35:48 +0200
Subject: [PATCH 1216/2196] Handle queryPathInfo() failure from substituters
 when fallback is enabled

Fixes #1990.
---
 src/libstore/build.cc       | 7 +++++++
 src/libstore/local-store.cc | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cd37f7a3fc0..96ca2874257 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3682,6 +3682,13 @@ void SubstitutionGoal::tryNext()
     } catch (InvalidPath &) {
         tryNext();
         return;
+    } catch (Error & e) {
+        if (settings.tryFallback) {
+            printError(e.what());
+            tryNext();
+            return;
+        }
+        throw;
     }
 
     /* Update the total expected download size. */
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c91dbf241bc..5e392c237e8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -880,6 +880,11 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
             } catch (InvalidPath) {
+            } catch (Error & e) {
+                if (settings.tryFallback)
+                    printError(e.what());
+                else
+                    throw;
             }
         }
     }

From bba3f0a308cceb56bad4aa1efe13927360ae463f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Sep 2018 17:08:43 +0200
Subject: [PATCH 1217/2196] If fallback is enabled, temporarily disable
 substituters after a failure

Otherwise, we just keep asking the substituter for other .narinfo
files, which can take a very long time due to retries/timeouts.
---
 src/libstore/build.cc                   |  6 ++++
 src/libstore/http-binary-cache-store.cc | 41 ++++++++++++++++++++++++-
 src/libstore/local-store.cc             |  1 +
 src/libstore/store-api.hh               |  1 +
 4 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 96ca2874257..1402bd097c3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3682,6 +3682,12 @@ void SubstitutionGoal::tryNext()
     } catch (InvalidPath &) {
         tryNext();
         return;
+    } catch (SubstituterDisabled &) {
+        if (settings.tryFallback) {
+            tryNext();
+            return;
+        }
+        throw;
     } catch (Error & e) {
         if (settings.tryFallback) {
             printError(e.what());
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index ab524d523cf..8da0e2f9d82 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -13,6 +13,14 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
     Path cacheUri;
 
+    struct State
+    {
+        bool enabled = true;
+        std::chrono::steady_clock::time_point disabledUntil;
+    };
+
+    Sync<State> _state;
+
 public:
 
     HttpBinaryCacheStore(
@@ -46,8 +54,33 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
 protected:
 
+    void maybeDisable()
+    {
+        auto state(_state.lock());
+        if (state->enabled && settings.tryFallback) {
+            int t = 60;
+            printError("disabling binary cache '%s' for %s seconds", getUri(), t);
+            state->enabled = false;
+            state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
+        }
+    }
+
+    void checkEnabled()
+    {
+        auto state(_state.lock());
+        if (state->enabled) return;
+        if (std::chrono::steady_clock::now() > state->disabledUntil) {
+            state->enabled = true;
+            debug("re-enabling binary cache '%s'", getUri());
+            return;
+        }
+        throw SubstituterDisabled("substituter '%s' is disabled", getUri());
+    }
+
     bool fileExists(const std::string & path) override
     {
+        checkEnabled();
+
         try {
             DownloadRequest request(cacheUri + "/" + path);
             request.head = true;
@@ -59,6 +92,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
                bucket is unlistable, so treat 403 as 404. */
             if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                 return false;
+            maybeDisable();
             throw;
         }
     }
@@ -86,12 +120,14 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
     void getFile(const std::string & path, Sink & sink) override
     {
+        checkEnabled();
         auto request(makeRequest(path));
         try {
             getDownloader()->download(std::move(request), sink);
         } catch (DownloadError & e) {
             if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
+            maybeDisable();
             throw;
         }
     }
@@ -99,15 +135,18 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     void getFile(const std::string & path,
         Callback<std::shared_ptr<std::string>> callback) override
     {
+        checkEnabled();
+
         auto request(makeRequest(path));
 
         getDownloader()->enqueueDownload(request,
-            {[callback](std::future<DownloadResult> result) {
+            {[callback, this](std::future<DownloadResult> result) {
                 try {
                     callback(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
                         return callback(std::shared_ptr<std::string>());
+                    maybeDisable();
                     callback.rethrow();
                 } catch (...) {
                     callback.rethrow();
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5e392c237e8..197b9d78995 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -880,6 +880,7 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
             } catch (InvalidPath) {
+            } catch (SubstituterDisabled) {
             } catch (Error & e) {
                 if (settings.tryFallback)
                     printError(e.what());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7c5b495a448..099818ed6f6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -23,6 +23,7 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
 MakeError(InvalidPath, Error)
 MakeError(Unsupported, Error)
 MakeError(SubstituteGone, Error)
+MakeError(SubstituterDisabled, Error)
 
 
 struct BasicDerivation;

From d6ac762bf7dcc844db28dcc2f2864d8cebbb5cf6 Mon Sep 17 00:00:00 2001
From: Ding Xiang Fei <dingxiangfei2009@gmail.com>
Date: Mon, 10 Sep 2018 15:07:50 +0800
Subject: [PATCH 1218/2196] auto args on repl

---
 src/nix/repl.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index b71e6f905f2..1bbe256b2d8 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -31,6 +31,7 @@ struct NixRepl
 {
     string curDir;
     EvalState state;
+    Bindings * autoArgs;
 
     Strings loadedFiles;
 
@@ -446,8 +447,7 @@ void NixRepl::loadFile(const Path & path)
     loadedFiles.push_back(path);
     Value v, v2;
     state.evalFile(lookupFileArg(state, path), v);
-    Bindings & bindings(*state.allocBindings(0));
-    state.autoCallFunction(bindings, v, v2);
+    state.autoCallFunction(*autoArgs, v, v2);
     addAttrsToScope(v2);
 }
 
@@ -699,6 +699,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
     void run(ref<Store> store) override
     {
         auto repl = std::make_unique<NixRepl>(searchPath, openStore());
+        repl->autoArgs = getAutoArgs(repl->state);
         repl->mainLoop(files);
     }
 };

From 0aca1ffb6ef655a5147e27e4307074f0b303000f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 7 Sep 2018 22:10:56 -0400
Subject: [PATCH 1219/2196] Prefer 'remote builder' over 'slave'

---
 doc/manual/command-ref/nix-store.xml |  2 +-
 tests/remote-builds.nix              | 36 ++++++++++++++--------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 5fff64a18f9..c827d85b381 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -275,7 +275,7 @@ as a means of providing Nix store access to a restricted ssh user.
 
     <listitem><para>Allow the connected client to request the realization
     of derivations. In effect, this can be used to make the host act
-    as a build slave.</para></listitem>
+    as a remote builder.</para></listitem>
 
   </varlistentry>
 
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index d7a4b21989e..b867f13b499 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -8,8 +8,8 @@ makeTest (
 
 let
 
-  # The configuration of the build slaves.
-  slave =
+  # The configuration of the remote builders.
+  builder =
     { config, pkgs, ... }:
     { services.openssh.enable = true;
       virtualisation.writableStore = true;
@@ -36,21 +36,21 @@ in
 {
 
   nodes =
-    { slave1 = slave;
-      slave2 = slave;
+    { builder1 = builder;
+      builder2 = builder;
 
       client =
         { config, pkgs, ... }:
         { nix.maxJobs = 0; # force remote building
           nix.distributedBuilds = true;
           nix.buildMachines =
-            [ { hostName = "slave1";
+            [ { hostName = "builder1";
                 sshUser = "root";
                 sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
                 maxJobs = 1;
               }
-              { hostName = "slave2";
+              { hostName = "builder2";
                 sshUser = "root";
                 sshKey = "/root/.ssh/id_ed25519";
                 system = "i686-linux";
@@ -75,33 +75,33 @@ in
       $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
       $client->succeed("chmod 600 /root/.ssh/id_ed25519");
 
-      # Install the SSH key on the slaves.
+      # Install the SSH key on the builders.
       $client->waitForUnit("network.target");
-      foreach my $slave ($slave1, $slave2) {
-          $slave->succeed("mkdir -p -m 700 /root/.ssh");
-          $slave->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
-          $slave->waitForUnit("sshd");
-          $client->succeed("ssh -o StrictHostKeyChecking=no " . $slave->name() . " 'echo hello world'");
+      foreach my $builder ($builder1, $builder2) {
+          $builder->succeed("mkdir -p -m 700 /root/.ssh");
+          $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
+          $builder->waitForUnit("sshd");
+          $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'");
       }
 
-      # Perform a build and check that it was performed on the slave.
+      # Perform a build and check that it was performed on the builder.
       my $out = $client->succeed(
         "nix-build ${expr nodes.client.config 1} 2> build-output",
         "grep -q Hello build-output"
       );
-      $slave1->succeed("test -e $out");
+      $builder1->succeed("test -e $out");
 
       # And a parallel build.
       my ($out1, $out2) = split /\s/,
           $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out');
-      $slave1->succeed("test -e $out1 -o -e $out2");
-      $slave2->succeed("test -e $out1 -o -e $out2");
+      $builder1->succeed("test -e $out1 -o -e $out2");
+      $builder2->succeed("test -e $out1 -o -e $out2");
 
       # And a failing build.
       $client->fail("nix-build ${expr nodes.client.config 5}");
 
-      # Test whether the build hook automatically skips unavailable slaves.
-      $slave1->block;
+      # Test whether the build hook automatically skips unavailable builders.
+      $builder1->block;
       $client->succeed("nix-build ${expr nodes.client.config 4}");
     '';
 

From 901dfc79788595f6382c8d8d9f584ec38a4cc010 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Sep 2018 16:36:11 +0200
Subject: [PATCH 1220/2196] Fix build failure if parallel xz is not available

---
 src/libutil/compression.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 204c63cd26f..0dd84e32034 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -250,7 +250,7 @@ struct XzCompressionSink : CompressionSink
             ret = lzma_stream_encoder_mt(&strm, &mt_options);
             done = true;
 #else
-            printMsg(lvlError, "warning: parallel compression requested but not supported for metho  d '%1%', falling back to single-threaded compression", method);
+            printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
 #endif
         }
 

From 52f6d541b9f872568f0043a20accc847ce6d5412 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Sep 2018 16:36:30 +0200
Subject: [PATCH 1221/2196] nix-env: Fix segfault if -f argument is not a
 directory or a Nix expression

Fixes #2425.
---
 src/nix-env/nix-env.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index a43b103f6ec..f9c8a8d313e 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -150,10 +150,8 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
     if (stat(path.c_str(), &st) == -1)
         throw SysError(format("getting information about '%1%'") % path);
 
-    if (isNixExpr(path, st)) {
+    if (isNixExpr(path, st))
         state.evalFile(path, v);
-        return;
-    }
 
     /* The path is a directory.  Put the Nix expressions in the
        directory in a set, with the file name of each expression as
@@ -161,13 +159,15 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
        set flat, not nested, to make it easier for a user to have a
        ~/.nix-defexpr directory that includes some system-wide
        directory). */
-    if (S_ISDIR(st.st_mode)) {
+    else if (S_ISDIR(st.st_mode)) {
         state.mkAttrs(v, 1024);
         state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0);
         StringSet attrs;
         getAllExprs(state, path, attrs, v);
         v.attrs->sort();
     }
+
+    else throw Error("path '%s' is not a directory or a Nix expression", path);
 }
 
 

From 0cd863197bd1f2ad5b5891e95bf75f7040d22351 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 17 Sep 2018 08:32:35 -0400
Subject: [PATCH 1222/2196] docs: document deprecated aliases

---
 doc/manual/command-ref/conf-file.xml | 191 ++++++++++++++++++++++++---
 1 file changed, 169 insertions(+), 22 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 26fe58d043b..fd09883be40 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -135,7 +135,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-builders">
     <term><literal>builders</literal></term>
     <listitem>
@@ -159,7 +158,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
 
     <listitem><para>This options specifies the Unix group containing
@@ -210,7 +208,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term>
 
     <listitem>
@@ -243,7 +240,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-extra-sandbox-paths">
     <term><literal>extra-sandbox-paths</literal></term>
 
@@ -283,7 +279,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
@@ -293,7 +288,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, changes to the
@@ -304,7 +298,6 @@ false</literal>.</para>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
 
     <listitem><para>A list of web servers used by
@@ -367,10 +360,8 @@ builtins.fetchurl {
     options a store path was built), so by default this option is on.
     Turn it off to save a bit of disk space (or a lot if
     <literal>keep-outputs</literal> is also turned on).</para></listitem>
-
   </varlistentry>
 
-
   <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
@@ -394,7 +385,6 @@ builtins.fetchurl {
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
@@ -408,10 +398,8 @@ builtins.fetchurl {
     only at build time (e.g., the C compiler, or source tarballs
     downloaded from the network).  To prevent it from doing so, set
     this option to <literal>true</literal>.</para></listitem>
-
   </varlistentry>
 
-
   <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term>
 
     <listitem>
@@ -444,10 +432,8 @@ builtins.fetchurl {
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>
-
   </varlistentry>
 
-
   <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>
 
     <listitem>
@@ -603,7 +589,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
@@ -615,7 +600,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),
@@ -679,7 +663,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term>
 
     <listitem><para>This option determines the maximum size of the
@@ -745,7 +728,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
@@ -754,7 +736,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-system"><term><literal>system</literal></term>
 
     <listitem><para>This option specifies the canonical Nix system
@@ -795,7 +776,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When
@@ -806,7 +786,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
@@ -819,7 +798,6 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
-
   <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term>
 
     <listitem>
@@ -845,8 +823,177 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 </variablelist>
+</para>
+
+<refsection>
+  <title>Deprecated Settings</title>
+
+<para>
+
+<variablelist>
+
+  <varlistentry xml:id="conf-binary-caches">
+    <term><literal>binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>binary-caches</literal> is now an alias to
+    <xref linkend="conf-substituters" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-binary-cache-public-keys">
+    <term><literal>binary-cache-public-keys</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>binary-cache-public-keys</literal> is now an alias to
+    <xref linkend="conf-trusted-public-keys" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-compress-log">
+    <term><literal>build-compress-log</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-compress-log</literal> is now an alias to
+    <xref linkend="conf-compress-build-log" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-cores">
+    <term><literal>build-cores</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-cores</literal> is now an alias to
+    <xref linkend="conf-cores" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-extra-chroot-dirs">
+    <term><literal>build-extra-chroot-dirs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-extra-chroot-dirs</literal> is now an alias to
+    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-extra-sandbox-paths">
+    <term><literal>build-extra-sandbox-paths</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-extra-sandbox-paths</literal> is now an alias to
+    <xref linkend="conf-extra-sandbox-paths" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-fallback">
+    <term><literal>build-fallback</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-fallback</literal> is now an alias to
+    <xref linkend="conf-fallback" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-jobs">
+    <term><literal>build-max-jobs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-jobs</literal> is now an alias to
+    <xref linkend="conf-max-jobs" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-log-size">
+    <term><literal>build-max-log-size</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-log-size</literal> is now an alias to
+    <xref linkend="conf-max-build-log-size" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-max-silent-time">
+    <term><literal>build-max-silent-time</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-max-silent-time</literal> is now an alias to
+    <xref linkend="conf-max-silent-time" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-repeat">
+    <term><literal>build-repeat</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-repeat</literal> is now an alias to
+    <xref linkend="conf-repeat" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-timeout">
+    <term><literal>build-timeout</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-timeout</literal> is now an alias to
+    <xref linkend="conf-timeout" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-use-chroot">
+    <term><literal>build-use-chroot</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-chroot</literal> is now an alias to
+    <xref linkend="conf-sandbox" />.</para></listitem>
+  </varlistentry>
 
+  <varlistentry xml:id="conf-build-use-sandbox">
+    <term><literal>build-use-sandbox</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-sandbox</literal> is now an alias to
+    <xref linkend="conf-sandbox" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-build-use-substitutes">
+    <term><literal>build-use-substitutes</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>build-use-substitutes</literal> is now an alias to
+    <xref linkend="conf-substitute" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-gc-keep-derivations">
+    <term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>gc-keep-derivations</literal> is now an alias to
+    <xref linkend="conf-keep-derivations" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-gc-keep-outputs">
+    <term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>gc-keep-outputs</literal> is now an alias to
+    <xref linkend="conf-keep-outputs" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-env-keep-derivations">
+    <term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>env-keep-derivations</literal> is now an alias to
+    <xref linkend="conf-keep-env-derivations" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-extra-binary-caches">
+    <term><literal>extra-binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>extra-binary-caches</literal> is now an alias to
+    <xref linkend="conf-extra-substituters" />.</para></listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-trusted-binary-caches">
+    <term><literal>trusted-binary-caches</literal></term>
+
+    <listitem><para><emphasis>Deprecated:</emphasis>
+    <literal>trusted-binary-caches</literal> is now an alias to
+    <xref linkend="conf-trusted-substituters" />.</para></listitem>
+  </varlistentry>
+</variablelist>
 </para>
+</refsection>
 
 </refsection>
 

From 1241a589756414222bbac731686dc3e2cc2538b3 Mon Sep 17 00:00:00 2001
From: Luke Clifton <Luke.Clifton@cba.com.au>
Date: Wed, 19 Sep 2018 15:10:06 +0800
Subject: [PATCH 1223/2196] Look inside the user profile

---
 scripts/nix-profile-daemon.sh.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 1be9a0755d8..432100d1622 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -61,8 +61,8 @@ elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
 elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
     export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
-elif [ -e "$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile
-    export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt
+elif [ -e "$NIX_USER_PROFILE_DIR/profile/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile
+    export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/profile/etc/ssl/certs/ca-bundle.crt
 elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the default Nix profile
     export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
 fi

From 10a7f199372c4bca5d81030e8a39ca7537ea47ca Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 19 Sep 2018 12:48:08 -0400
Subject: [PATCH 1224/2196] nix-shell: explain the merging property close to
 the example

---
 doc/manual/command-ref/nix-shell.xml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 5c44c4a8f44..3f12dda040a 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -317,13 +317,14 @@ while (my $token = $p->get_tag("a")) {
 
 </para>
 
-<para>Finally, the following Haskell script uses a specific branch of
-Nixpkgs/NixOS (the 14.12 stable branch):
+<para>Finally, using the merging of multiple nix-shell shebangs the
+following Haskell script uses a specific branch of Nixpkgs/NixOS (the
+18.03 stable branch):
 
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup

From be9e356e614a95e7ab839584c9d95838be1d0002 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 19 Sep 2018 13:01:27 -0400
Subject: [PATCH 1225/2196] nix-shell: document double quotes being necessary
 for a simple Nix expression in the shebang

---
 doc/manual/command-ref/nix-shell.xml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 3f12dda040a..cb443c888d3 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -317,6 +317,20 @@ while (my $token = $p->get_tag("a")) {
 
 </para>
 
+<para>Sometimes you need to pass a simple Nix expression to customize
+a package like Terraform:
+
+<programlisting><![CDATA[
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])"
+
+terraform apply
+]]></programlisting>
+
+<note><para>You must use double quotes (<literal>"</literal>) when
+passing a simple Nix expression in a nix-shell shebang.</para></note>
+</para>
+
 <para>Finally, using the merging of multiple nix-shell shebangs the
 following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 18.03 stable branch):

From 1915862767331dba33bc0f40bc09a3c32a22ebea Mon Sep 17 00:00:00 2001
From: Anders Riutta <ariutta@users.noreply.github.com>
Date: Mon, 17 Sep 2018 13:19:02 -0700
Subject: [PATCH 1226/2196] Upgrade docs: improve the upgrade command

and make it more copy-pastable.
---
 doc/manual/installation/upgrading.xml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml
index a3f86ade95c..30670d7fec9 100644
--- a/doc/manual/installation/upgrading.xml
+++ b/doc/manual/installation/upgrading.xml
@@ -7,15 +7,16 @@
   <title>Upgrading Nix</title>
 
   <para>
-    Multi-user Nix users on macOS can upgrade Nix by running
-    <command>sudo -i sh -c 'nix-channel --update &amp;&amp; nix-env
-    -iA nixpkgs.nix'; sudo launchctl stop org.nixos.nix-daemon; sudo
-    launchctl start org.nixos.nix-daemon</command>.
+    Multi-user Nix users on macOS can upgrade Nix by running:
+    <command>sudo -i sh -c 'nix-channel --update &amp;&amp;
+    nix-env -iA nixpkgs.nix &amp;&amp;
+    launchctl remove org.nixos.nix-daemon &amp;&amp;
+    launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command>
   </para>
 
+
   <para>
-    Single-user installations of Nix should run <command>nix-channel
-    --update; nix-env -iA nixpkgs.nix</command>.
+    Single-user installations of Nix should run this:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
   </para>
-
 </chapter>

From fb72104b80eb5747788ac32bcef9fc1db00d9825 Mon Sep 17 00:00:00 2001
From: Luke Clifton <Luke.Clifton@cba.com.au>
Date: Thu, 20 Sep 2018 07:33:35 +0800
Subject: [PATCH 1227/2196] Search NIX_PROFILE for SSL CA

---
 scripts/nix-profile-daemon.sh.in | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 432100d1622..567a543d95e 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -61,10 +61,13 @@ elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
 elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
     export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
-elif [ -e "$NIX_USER_PROFILE_DIR/profile/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile
-    export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/profile/etc/ssl/certs/ca-bundle.crt
-elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the default Nix profile
-    export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+else
+  # Fall back to what is in the nix profiles, favouring whatever is defined last.
+  for i in $NIX_PROFILES; do
+    if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
+      export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
+    fi
+  done
 fi
 
 export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"

From 8df367a92dc417363f06a10f3e6da87d986d04f2 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 24 Sep 2018 11:58:40 -0500
Subject: [PATCH 1228/2196] nix-daemon: allow setting builders to "" by any
 user (untrusted)

(cherry picked from commit a94a2eb1cb1c81e90a7529be5fecac27899a3442)
---
 src/nix-daemon/nix-daemon.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 644fa6681de..615c1f5dc06 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -557,7 +557,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     ;
                 else if (trusted
                     || name == settings.buildTimeout.name
-                    || name == "connect-timeout")
+                    || name == "connect-timeout"
+                    || (name == "builders" && value == ""))
                     settings.set(name, value);
                 else if (setSubstituters(settings.substituters))
                     ;

From 0ae8d4033d0b140b98e45c57e25b23d1970d356c Mon Sep 17 00:00:00 2001
From: Will Fancher <elvishjerricco@gmail.com>
Date: Tue, 25 Sep 2018 21:19:24 -0400
Subject: [PATCH 1229/2196] Fix overflow when verifying signatures of content
 addressable paths

---
 src/nix/verify.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 6540208a8a2..7ef571561a0 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand
                             for (auto sig : sigs) {
                                 if (sigsSeen.count(sig)) continue;
                                 sigsSeen.insert(sig);
-                                if (info->checkSignature(publicKeys, sig))
+                                if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig))
                                     validSigs++;
                             }
                         };

From b7091ce41e0073b147678f05b9a620b9bb2903c1 Mon Sep 17 00:00:00 2001
From: Will Fancher <elvishjerricco@gmail.com>
Date: Tue, 25 Sep 2018 22:18:52 -0400
Subject: [PATCH 1230/2196] Add a test for signed content-addressed paths

---
 tests/signing.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/signing.sh b/tests/signing.sh
index 46929639199..9e29e3fbf06 100644
--- a/tests/signing.sh
+++ b/tests/signing.sh
@@ -62,6 +62,10 @@ outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no
 nix verify $outPathCA
 nix verify $outPathCA --sigs-needed 1000
 
+# Check that signing a content-addressed path doesn't overflow validSigs
+nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA
+nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1
+
 # Copy to a binary cache.
 nix copy --to file://$cacheDir $outPath2
 

From 44e86304b611a955f4e934fc160f3f4a0a2b1c92 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Sep 2018 12:03:58 +0200
Subject: [PATCH 1231/2196] Make NAR header check more robust

Changes

  std::bad_alloc

into

  bad archive: input doesn't look like a Nix archive
---
 src/libutil/archive.cc   | 2 +-
 src/libutil/serialise.cc | 5 +++--
 src/libutil/serialise.hh | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 1be8934a2eb..bb68e82886d 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -283,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source)
 {
     string version;
     try {
-        version = readString(source);
+        version = readString(source, narVersionMagic1.size());
     } catch (SerialisationError & e) {
         /* This generally means the integer at the start couldn't be
            decoded.  Ignore and throw the exception below. */
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 17448f70efb..31df6fdfde8 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -268,16 +268,17 @@ void readPadding(size_t len, Source & source)
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
     auto len = readNum<size_t>(source);
-    if (len > max) throw Error("string is too long");
+    if (len > max) throw SerialisationError("string is too long");
     source(buf, len);
     readPadding(len, source);
     return len;
 }
 
 
-string readString(Source & source)
+string readString(Source & source, size_t max)
 {
     auto len = readNum<size_t>(source);
+    if (len > max) throw SerialisationError("string is too long");
     std::string res(len, 0);
     source((unsigned char*) res.data(), len);
     readPadding(len, source);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 4b6ad5da5b9..969e4dff383 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -284,7 +284,7 @@ inline uint64_t readLongLong(Source & source)
 
 void readPadding(size_t len, Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
-string readString(Source & source);
+string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);

From ed78582847d3932763ad9b7b239f843306fa6fe9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Sep 2018 21:19:34 +0200
Subject: [PATCH 1232/2196] sinkToSource(): Start the coroutine lazily

In particular this causes copyStorePath() from HttpBinaryCacheStore to
only start a download if needed. E.g. if the destination LocalStore
goes to sleep waiting for the path lock and another process creates
the path, then LocalStore::addToStore() will never read from the
source so we don't have to do the download.
---
 src/libutil/serialise.cc | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 31df6fdfde8..0e75eeec2bf 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -169,17 +169,13 @@ std::unique_ptr<Source> sinkToSource(
     {
         typedef boost::coroutines2::coroutine<std::string> coro_t;
 
+        std::function<void(Sink &)> fun;
         std::function<void()> eof;
-        coro_t::pull_type coro;
+        std::experimental::optional<coro_t::pull_type> coro;
+        bool started = false;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
-            : eof(eof)
-            , coro([&](coro_t::push_type & yield) {
-                LambdaSink sink([&](const unsigned char * data, size_t len) {
-                    if (len) yield(std::string((const char *) data, len));
-                });
-                fun(sink);
-            })
+            : fun(fun), eof(eof)
         {
         }
 
@@ -188,11 +184,19 @@ std::unique_ptr<Source> sinkToSource(
 
         size_t read(unsigned char * data, size_t len) override
         {
-            if (!coro) { eof(); abort(); }
+            if (!coro)
+                coro = coro_t::pull_type([&](coro_t::push_type & yield) {
+                    LambdaSink sink([&](const unsigned char * data, size_t len) {
+                            if (len) yield(std::string((const char *) data, len));
+                        });
+                    fun(sink);
+                });
+
+            if (!*coro) { eof(); abort(); }
 
             if (pos == cur.size()) {
-                if (!cur.empty()) coro();
-                cur = coro.get();
+                if (!cur.empty()) (*coro)();
+                cur = coro->get();
                 pos = 0;
             }
 

From 98b2cc2e6e63bfa49b8f75169a39b751b3e2c32c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Sep 2018 21:39:06 +0200
Subject: [PATCH 1233/2196] Untabify

---
 src/libstore/store-api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1f42097fccf..2c0f68651e4 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -610,7 +610,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
         });
         srcStore->narFromPath({storePath}, wrapperSink);
     }, [&]() {
-	throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
+        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
     });
 
     dstStore->addToStore(*info, *source, repair, checkSigs);

From 97504300032c7c57388d68bbe4a05b0a494e81aa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 26 Sep 2018 21:43:17 +0200
Subject: [PATCH 1234/2196] Ensure download thread liveness

* Don't wait forever for the client to remove data from the
  buffer. This does mean that the buffer can grow without bounds
  (e.g. when downloading is faster than writing to disk), but meh.

* Don't hold the state lock while calling the sink. The sink could
  take any amount of time to process the data (in particular when it's
  actually a coroutine), so we don't want to block the download
  thread.
---
 src/libstore/download.cc | 45 +++++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 13913d031da..f44f1836b31 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -710,11 +710,12 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
 
         /* If the buffer is full, then go to sleep until the calling
            thread wakes us up (i.e. when it has removed data from the
-           buffer). Note: this does stall the download thread. */
-        while (state->data.size() > 1024 * 1024) {
-            if (state->quit) return;
+           buffer). We don't wait forever to prevent stalling the
+           download thread. (Hopefully sleeping will throttle the
+           sender.) */
+        if (state->data.size() > 1024 * 1024) {
             debug("download buffer is full; going to sleep");
-            state.wait(state->request);
+            state.wait_for(state->request, std::chrono::seconds(10));
         }
 
         /* Append data to the buffer and wake up the calling
@@ -736,30 +737,36 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
             state->request.notify_one();
         }});
 
-    auto state(_state->lock());
-
     while (true) {
         checkInterrupt();
 
-        /* If no data is available, then wait for the download thread
-           to wake us up. */
-        if (state->data.empty()) {
+        std::string chunk;
+
+        /* Grab data if available, otherwise wait for the download
+           thread to wake us up. */
+        {
+            auto state(_state->lock());
+
+            while (state->data.empty()) {
 
-            if (state->quit) {
-                if (state->exc) std::rethrow_exception(state->exc);
-                break;
+                if (state->quit) {
+                    if (state->exc) std::rethrow_exception(state->exc);
+                    return;
+                }
+
+                state.wait(state->avail);
             }
 
-            state.wait(state->avail);
-        }
+            chunk = std::move(state->data);
 
-        /* If data is available, then flush it to the sink and wake up
-           the download thread if it's blocked on a full buffer. */
-        if (!state->data.empty()) {
-            sink((unsigned char *) state->data.data(), state->data.size());
-            state->data.clear();
             state->request.notify_one();
         }
+
+        /* Flush the data to the sink and wake up the download thread
+           if it's blocked on a full buffer. We don't hold the state
+           lock while doing this to prevent blocking the download
+           thread if sink() takes a long time. */
+        sink((unsigned char *) chunk.data(), chunk.size());
     }
 }
 

From d4f78a6b64a70c7135cd841eeaef8e698b0447f8 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 26 Sep 2018 15:37:13 -0500
Subject: [PATCH 1235/2196] bump base nixpkgs used by default 18.03 -> 18.09

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index e359ebcb2ce..415c87da7cb 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:

From ed25753501a7a3f77c097df01db299bdb60ede96 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 26 Sep 2018 22:59:41 +0200
Subject: [PATCH 1236/2196] nix doctor: reimplement profile warning without
 gcroot check

Calculating roots seems significantly slower on darwin compared to
linux.  Checking for /profile/ links could show some false positives but
should still catch most issues.
---
 src/nix/doctor.cc | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 6112b1f650a..b608b9d59ef 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -42,7 +42,8 @@ struct CmdDoctor : StoreCommand
         checkStoreProtocol(store->getProtocol());
     }
 
-    void checkNixInPath() {
+    void checkNixInPath()
+    {
         PathSet dirs;
 
         for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
@@ -58,32 +59,29 @@ struct CmdDoctor : StoreCommand
         }
     }
 
-    void checkProfileRoots(ref<Store> store) {
+    void checkProfileRoots(ref<Store> store)
+    {
         PathSet dirs;
-        Roots roots = store->findRoots();
 
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) {
+            Path profileDir = dirOf(dir);
             try {
-                auto profileDir = canonPath(dirOf(dir), true);
-                if (hasSuffix(profileDir, "user-environment") &&
-                    store->isValidPath(profileDir)) {
-                    PathSet referrers;
-                    store->computeFSClosure({profileDir}, referrers, true,
-                            settings.gcKeepOutputs, settings.gcKeepDerivations);
-                    bool found = false;
-                    for (auto & i : roots)
-                        if (referrers.find(i.second) != referrers.end())
-                            found = true;
-                    if (!found)
-                        dirs.insert(dir);
+                Path userEnv = canonPath(profileDir, true);
 
+                if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) {
+                    while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir))
+                        profileDir = absPath(readLink(profileDir), dirOf(profileDir));
+
+                    if (profileDir.find("/profiles/") == std::string::npos)
+                        dirs.insert(dir);
                 }
             } catch (SysError &) {}
+        }
 
         if (!dirs.empty()) {
-            std::cout << "Warning: found profiles without a gcroot." << std::endl;
-            std::cout << "The generation this profile points to will be deleted with the next gc, resulting" << std::endl;
-            std::cout << "in broken symlinks.  Make sure your profiles are in " << settings.nixStateDir << "/profiles." << std::endl;
+            std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl;
+            std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl;
+            std::cout << "garbage collected, resulting in broken symlinks." << std::endl;
             std::cout << std::endl;
             for (auto & dir : dirs)
                 std::cout << "  " << dir << std::endl;
@@ -91,7 +89,8 @@ struct CmdDoctor : StoreCommand
         }
     }
 
-    void checkStoreProtocol(unsigned int storeProto) {
+    void checkStoreProtocol(unsigned int storeProto)
+    {
         auto clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
             ? SERVE_PROTOCOL_VERSION
             : PROTOCOL_VERSION;

From 7cba4214a83fc613b60bb3abfe7ba18ff8201d37 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Sep 2018 21:01:19 +0200
Subject: [PATCH 1237/2196] Don't talk about a "current folder build output"

Presumably this refers to ./default.nix but the support for that in
'nix' is tenuous. Also folders are a Mac thing.
---
 src/nix/copy.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 91711c8b46d..96bd453d87b 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -69,12 +69,12 @@ struct CmdCopy : StorePathsCommand
             },
 #ifdef ENABLE_S3
             Example{
-                "To populate the current folder build output to a S3 binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1"
+                "To copy Hello to an S3 binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"
             },
             Example{
-                "To populate the current folder build output to an S3-compatible binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
+                "To copy Hello to an S3-compatible binary cache:",
+                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello"
             },
 #endif
         };

From f11acbaf175ebb2c844cb40900adb912ff58f613 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 27 Sep 2018 12:40:19 -0400
Subject: [PATCH 1238/2196] nix-shell: add bashInteractive to the start of the
 PATH, set SHELL

Tools which re-exec `$SHELL` or `$0` or `basename $SHELL` or even just
`bash` will otherwise  get the non-interactive bash, providing a
broken shell for the same reasons described in
https://github.com/NixOS/nixpkgs/issues/27493.

Extends c94f3d5575d7af5403274d1e9e2f3c9d72989751
---
 src/nix-build/nix-build.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 94d3a27560f..b78f3d9e424 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -417,16 +417,20 @@ void mainWrapped(int argc, char * * argv)
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
+                "PATH=\"%4%:$PATH\"; "
+                "SHELL=%5%; "
                 "set +e; "
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
                 "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; "
                 "unset NIX_ENFORCE_PURITY; "
                 "shopt -u nullglob; "
-                "unset TZ; %4%"
-                "%5%",
+                "unset TZ; %6%"
+                "%7%",
                 (Path) tmpDir,
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
+                dirOf(shell),
+                shell,
                 (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
                 envCommand));
 

From 63786cbd3bc0a2d0651c09eac6ad5ae609b82902 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 27 Sep 2018 21:32:25 +0200
Subject: [PATCH 1239/2196] S3BinaryCacheStore: Don't create buckets

This meant that making a typo in an s3:// URI would cause a bucket to
be created. Also it didn't handle eventual consistency very well. Now
it's up to the user to create the bucket.
---
 src/libstore/s3-binary-cache-store.cc | 28 ---------------------------
 1 file changed, 28 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 7711388f05a..ba11ce6bb6d 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -19,8 +19,6 @@
 #include <aws/core/utils/logging/LogMacros.h>
 #include <aws/core/utils/threading/Executor.h>
 #include <aws/s3/S3Client.h>
-#include <aws/s3/model/CreateBucketRequest.h>
-#include <aws/s3/model/GetBucketLocationRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
@@ -202,32 +200,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     {
         if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
 
-            /* Create the bucket if it doesn't already exists. */
-            // FIXME: HeadBucket would be more appropriate, but doesn't return
-            // an easily parsed 404 message.
-            auto res = s3Helper.client->GetBucketLocation(
-                Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
-
-            if (!res.IsSuccess()) {
-                if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
-                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());
-
-                printInfo("creating S3 bucket '%s'...", bucketName);
-
-                // Stupid S3 bucket locations.
-                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
-                if (s3Helper.config->region != "us-east-1")
-                    bucketConfig.SetLocationConstraint(
-                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
-                            s3Helper.config->region));
-
-                checkAws(format("AWS error creating bucket '%s'") % bucketName,
-                    s3Helper.client->CreateBucket(
-                        Aws::S3::Model::CreateBucketRequest()
-                        .WithBucket(bucketName)
-                        .WithCreateBucketConfiguration(bucketConfig)));
-            }
-
             BinaryCacheStore::init();
 
             diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);

From 51cbeec49ae78b0109dbed2e9efd641415853aa7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 27 Sep 2018 16:54:20 -0400
Subject: [PATCH 1240/2196] Update docs to describe how s3:// URLS does in fact
 support endpoint, region, and profile for upload

---
 doc/manual/packages/s3-substituter.xml | 202 +++++++++++++------------
 1 file changed, 107 insertions(+), 95 deletions(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index bcd91cfdbcc..ea654392c6b 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -12,8 +12,49 @@ from Amazon S3 and S3 compatible services. This uses the same
 <emphasis>binary</emphasis> cache mechanism that Nix usually uses to
 fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
 
+<para>The following options can be specified as URL parameters to
+the S3 URL:</para>
+
+<variablelist>
+  <varlistentry><term><literal>profile</literal></term>
+  <listitem>
+    <para>
+      The name of the AWS configuration profile to use. By default
+      Nix will use the <literal>default</literal> profile.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>region</literal></term>
+  <listitem>
+    <para>
+      The region of the S3 bucket. <literal>us–east-1</literal> by
+      default.
+    </para>
+
+    <para>
+      If your bucket is not in <literal>us–east-1</literal>, you
+      should always explicitly specify the region parameter.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>endpoint</literal></term>
+  <listitem>
+    <para>
+      The URL to your S3-compatible service, for when not using
+      Amazon S3. Do not specify this value if you're using Amazon
+      S3.
+    </para>
+    <note><para>This endpoint must support HTTPS and will use
+    path-based addressing instead of virtual host based
+    addressing.</para></note>
+  </listitem>
+  </varlistentry>
+</variablelist>
+
 <para>In this example we will use the bucket named
-<literal>example-bucket</literal>.</para>
+<literal>example-nix-cache</literal>.</para>
 
 <section xml:id="ssec-s3-substituter-anonymous-reads">
   <title>Anonymous Reads to your S3-compatible binary cache</title>
@@ -24,65 +65,56 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
   cache.</para>
 
   <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
-  compatible binary caches ago have to consult your software's
-  documentation.</para>
+  exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or
+  <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches,
+  consult that cache's documentation.</para>
 
   <para>Your bucket will need the following bucket policy:</para>
 
-  <programlisting>
-<![CDATA[
+  <programlisting><![CDATA[
 {
-  "Id": "DirectReads",
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Sid": "AlowDirectReads",
-      "Action": [
-        "s3:GetObject"
-      ],
-      "Effect": "Allow",
-      "Resource": "arn:aws:s3:::example-bucket/*",
-      "Principal": "*"
-    }
-  ]
+    "Id": "DirectReads",
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AlowDirectReads",
+            "Action": [
+                "s3:GetObject",
+                "s3:GetBucketLocation"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::example-nix-cache",
+                "arn:aws:s3:::example-nix-cache/*"
+            ],
+            "Principal": "*"
+        }
+    ]
 }
-]]>
-</programlisting>
+]]></programlisting>
 </section>
 
 <section xml:id="ssec-s3-substituter-authenticated-reads">
   <title>Authenticated Reads to your S3 binary cache</title>
 
   <para>For AWS S3 the binary cache URL for example bucket will be
-  exactly <uri>s3://example-bucket</uri>.</para>
+  exactly <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
   xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
-  <para>Nix supports authenticated writes to S3 compatible binary
-  caches but only supports Authenticated reads from Amazon S3.
-  Additionally, the following limitations are in place for
-  authenticated reads:</para>
-
-  <itemizedlist>
-    <listitem><para>The bucket must actually be hosted by Amazon S3 and
-    <emphasis>not</emphasis> an S3 compatible
-    service.</para></listitem>
-
-    <listitem><para>The bucket must be within the
-    <literal>us-east-1</literal> region.</para></listitem>
-
-    <listitem><para>The Amazon credentials, if stored in a credential
-    profile, must be stored in the <literal>default</literal>
-    profile.</para></listitem>
-  </itemizedlist>
+  <para>Nix supports authenticated reads from Amazon S3 and S3
+  compatible binary caches.</para>
 
   <para>Your bucket will need a bucket policy allowing the desired
-  users to perform the <literal>s3:GetObject</literal> action on all
-  objects in the bucket.</para>
+  users to perform the <literal>s3:GetObject</literal> and
+  <literal>s3:GetBucketLocation</literal> action on all objects in the
+  bucket. The anonymous policy in <xref
+  linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to
+  have a restricted <literal>Principal</literal> to support
+  this.</para>
 </section>
 
 
@@ -91,69 +123,49 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
 
   <para>Nix support fully supports writing to Amazon S3 and S3
   compatible buckets. The binary cache URL for our example bucket will
-  be <uri>s3://example-bucket</uri>.</para>
+  be <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
   xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
-  <para>The following options can be specified as URL parameters to
-  the S3 URL:</para>
-  <variablelist>
-    <varlistentry><term><literal>profile</literal></term>
-    <listitem>
-      <para>
-        The name of the AWS configuration profile to use. By default
-        Nix will use the <literal>default</literal> profile.
-      </para>
-    </listitem>
-    </varlistentry>
-
-    <varlistentry><term><literal>region</literal></term>
-    <listitem>
-      <para>
-        The region of the S3 bucket. <literal>us–east-1</literal> by
-        default.
-      </para>
-    </listitem>
-    </varlistentry>
-
-    <varlistentry><term><literal>endpoint</literal></term>
-    <listitem>
-      <para>
-        The URL to your S3-compatible service, for when not using
-        Amazon S3. Do not specify this value if you're using Amazon
-        S3.
-      </para>
-      <note><para>This endpoint must support HTTPS and will use
-      path-based addressing instead of virtual host based
-      addressing.</para></note>
-    </listitem>
-    </varlistentry>
-  </variablelist>
-
-  <example><title>Uploading with non-default credential profile for Amazon S3</title>
-    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
+  <para>Your account will need the following IAM policy to
+  upload to the cache:</para>
+
+  <programlisting><![CDATA[
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "UploadToCache",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:ListMultipartUploadParts",
+        "s3:ListObjects",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::example-nix-cache",
+        "arn:aws:s3:::example-nix-cache/*"
+      ]
+    }
+  ]
+}
+]]></programlisting>
+
+
+  <example><title>Uploading with a specific credential profile for Amazon S3</title>
+    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;region=eu-west-2' nixpkgs.hello</command></para>
   </example>
 
   <example><title>Uploading to an S3-Compatible Binary Cache</title>
-    <para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&amp;endpoint=minio.example.com</command></para>
+    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
   </example>
-
-  <para>The user writing to the bucket will need to perform the
-  following actions against the bucket:</para>
-
-  <itemizedlist>
-    <listitem><para><literal>s3:ListBucket</literal></para></listitem>
-    <listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
-    <listitem><para><literal>s3:ListObjects</literal></para></listitem>
-    <listitem><para><literal>s3:GetObject</literal></para></listitem>
-    <listitem><para><literal>s3:PutObject</literal></para></listitem>
-    <listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
-    <listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
-    <listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
-    <listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
-  </itemizedlist>
 </section>
 </section>

From c9ba33870e6da73420317e9ef80b8c9dee693c3d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Sep 2018 12:43:01 +0200
Subject: [PATCH 1241/2196] Support special attributes in structured attributes
 derivations

E.g. __noChroot and allowedReferences now work correctly. We also now
check that the attribute type is correct. For instance, instead of

  allowedReferences = "out";

you have to write

  allowedReferences = [ "out" ];

Fixes #2453.
---
 src/libstore/build.cc | 269 +++++++++++++++++++++++++++---------------
 1 file changed, 175 insertions(+), 94 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1402bd097c3..727b8b3444a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -740,6 +740,9 @@ class DerivationGoal : public Goal
     /* The derivation stored at drvPath. */
     std::unique_ptr<BasicDerivation> drv;
 
+    /* The contents of drv->env["__json"]. */
+    std::experimental::optional<nlohmann::json> structuredAttrs;
+
     /* The remainder is state held during the build. */
 
     /* Locks on the output paths. */
@@ -920,6 +923,13 @@ class DerivationGoal : public Goal
     /* Fill in the environment for the builder. */
     void initEnv();
 
+    /* Get an attribute from drv->env or from drv->env["__json"]. */
+    std::experimental::optional<std::string> getAttr(const std::string & name);
+
+    bool getBoolAttr(const std::string & name, bool def = false);
+
+    std::experimental::optional<Strings> getStringsAttr(const std::string & name);
+
     /* Write a JSON file containing the derivation attributes. */
     void writeStructuredAttrs();
 
@@ -1139,6 +1149,16 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
+    /* Parse the __json attribute, if any. */
+    auto jsonAttr = drv->env.find("__json");
+    if (jsonAttr != drv->env.end()) {
+        try {
+            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+        } catch (std::exception & e) {
+            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+        }
+    }
+
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1644,7 +1664,7 @@ HookReply DerivationGoal::tryBuildHook()
         /* Tell the hook about system features (beyond the system type)
            required from the build machine.  (The hook could parse the
            drv file itself, but this is easier.) */
-        Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures"));
+        auto features = getStringsAttr("requiredSystemFeatures").value_or(Strings());
         for (auto & i : features) checkStoreName(i); /* !!! abuse */
 
         /* Send the request to the hook. */
@@ -1803,13 +1823,14 @@ void DerivationGoal::startBuilder()
         preloadNSS();
 
 #if __APPLE__
-    additionalSandboxProfile = get(drv->env, "__sandboxProfile");
+    additionalSandboxProfile = getAttr("__sandboxProfile").value_or("");
 #endif
 
     /* Are we doing a chroot build? */
     {
+        auto noChroot = getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
-            if (get(drv->env, "__noChroot") == "1")
+            if (noChroot)
                 throw Error(format("derivation '%1%' has '__noChroot' set, "
                     "but that's not allowed when 'sandbox' is 'true'") % drvPath);
 #if __APPLE__
@@ -1822,7 +1843,7 @@ void DerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
+            useChroot = !fixedOutput && !noChroot;
     }
 
     if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -1873,7 +1894,7 @@ void DerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    if (!drv->env.count("__json")) {
+    if (!structuredAttrs) {
         /* The `exportReferencesGraph' feature allows the references graph
            to be passed to a builder.  This attribute should be a list of
            pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -1938,7 +1959,7 @@ void DerivationGoal::startBuilder()
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
-        Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps"));
+        auto impurePaths = getStringsAttr("__impureHostDeps").value_or(Strings());
 
         for (auto & i : impurePaths) {
             bool found = false;
@@ -2306,7 +2327,7 @@ void DerivationGoal::initEnv()
        passAsFile is ignored in structure mode because it's not
        needed (attributes are not passed through the environment, so
        there is no size constraint). */
-    if (!drv->env.count("__json")) {
+    if (!structuredAttrs) {
 
         StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
         int fileNr = 0;
@@ -2353,8 +2374,8 @@ void DerivationGoal::initEnv()
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
-        Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars"));
-        for (auto & i : varNames) env[i] = getEnv(i);
+        for (auto & i : getStringsAttr("impureEnvVars").value_or(Strings()))
+            env[i] = getEnv(i);
     }
 
     /* Currently structured log messages piggyback on stderr, but we
@@ -2364,116 +2385,176 @@ void DerivationGoal::initEnv()
 }
 
 
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
+std::experimental::optional<std::string> DerivationGoal::getAttr(const std::string & name)
 {
-    auto jsonAttr = drv->env.find("__json");
-    if (jsonAttr == drv->env.end()) return;
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_string())
+                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
+            return i->get<std::string>();
+        }
+    } else {
+        auto i = drv->env.find(name);
+        if (i == drv->env.end())
+            return {};
+        else
+            return i->second;
+    }
+}
 
-    try {
 
-        auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
+bool DerivationGoal::getBoolAttr(const std::string & name, bool def)
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return def;
+        else {
+            if (!i->is_boolean())
+                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
+            return i->get<bool>();
+        }
+    } else {
+        auto i = drv->env.find(name);
+        if (i == drv->env.end())
+            return def;
+        else
+            return i->second == "1";
+    }
+}
 
-        auto json = nlohmann::json::parse(jsonStr);
 
-        /* Add an "outputs" object containing the output paths. */
-        nlohmann::json outputs;
-        for (auto & i : drv->outputs)
-            outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
-        json["outputs"] = outputs;
-
-        /* Handle exportReferencesGraph. */
-        auto e = json.find("exportReferencesGraph");
-        if (e != json.end() && e->is_object()) {
-            for (auto i = e->begin(); i != e->end(); ++i) {
-                std::ostringstream str;
-                {
-                    JSONPlaceholder jsonRoot(str, true);
-                    PathSet storePaths;
-                    for (auto & p : *i)
-                        storePaths.insert(p.get<std::string>());
-                    worker.store.pathInfoToJSON(jsonRoot,
-                        exportReferences(storePaths), false, true);
-                }
-                json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+std::experimental::optional<Strings> DerivationGoal::getStringsAttr(const std::string & name)
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_array())
+                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+            Strings res;
+            for (auto j = i->begin(); j != i->end(); ++j) {
+                if (!j->is_string())
+                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                res.push_back(j->get<std::string>());
             }
+            return res;
         }
+    } else {
+        auto i = drv->env.find(name);
+        if (i == drv->env.end())
+            return {};
+        else
+            return tokenizeString<Strings>(i->second);
+    }
+}
+
+
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
-        writeFile(tmpDir + "/.attrs.json", json.dump());
 
-        /* As a convenience to bash scripts, write a shell file that
-           maps all attributes that are representable in bash -
-           namely, strings, integers, nulls, Booleans, and arrays and
-           objects consisting entirely of those values. (So nested
-           arrays or objects are not supported.) */
+void DerivationGoal::writeStructuredAttrs()
+{
+    if (!structuredAttrs) return;
 
-        auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
-            if (value.is_string())
-                return shellEscape(value);
+    auto json = *structuredAttrs;
 
-            if (value.is_number()) {
-                auto f = value.get<float>();
-                if (std::ceil(f) == f)
-                    return std::to_string(value.get<int>());
+    /* Add an "outputs" object containing the output paths. */
+    nlohmann::json outputs;
+    for (auto & i : drv->outputs)
+        outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+    json["outputs"] = outputs;
+
+    /* Handle exportReferencesGraph. */
+    auto e = json.find("exportReferencesGraph");
+    if (e != json.end() && e->is_object()) {
+        for (auto i = e->begin(); i != e->end(); ++i) {
+            std::ostringstream str;
+            {
+                JSONPlaceholder jsonRoot(str, true);
+                PathSet storePaths;
+                for (auto & p : *i)
+                    storePaths.insert(p.get<std::string>());
+                worker.store.pathInfoToJSON(jsonRoot,
+                    exportReferences(storePaths), false, true);
             }
+            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+        }
+    }
 
-            if (value.is_null())
-                return std::string("''");
+    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
 
-            if (value.is_boolean())
-                return value.get<bool>() ? std::string("1") : std::string("");
+    /* As a convenience to bash scripts, write a shell file that
+       maps all attributes that are representable in bash -
+       namely, strings, integers, nulls, Booleans, and arrays and
+       objects consisting entirely of those values. (So nested
+       arrays or objects are not supported.) */
 
-            return {};
-        };
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+        if (value.is_string())
+            return shellEscape(value);
 
-        std::string jsonSh;
+        if (value.is_number()) {
+            auto f = value.get<float>();
+            if (std::ceil(f) == f)
+                return std::to_string(value.get<int>());
+        }
 
-        for (auto i = json.begin(); i != json.end(); ++i) {
+        if (value.is_null())
+            return std::string("''");
 
-            if (!std::regex_match(i.key(), shVarName)) continue;
+        if (value.is_boolean())
+            return value.get<bool>() ? std::string("1") : std::string("");
 
-            auto & value = i.value();
+        return {};
+    };
 
-            auto s = handleSimpleType(value);
-            if (s)
-                jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+    std::string jsonSh;
 
-            else if (value.is_array()) {
-                std::string s2;
-                bool good = true;
+    for (auto i = json.begin(); i != json.end(); ++i) {
 
-                for (auto i = value.begin(); i != value.end(); ++i) {
-                    auto s3 = handleSimpleType(i.value());
-                    if (!s3) { good = false; break; }
-                    s2 += *s3; s2 += ' ';
-                }
+        if (!std::regex_match(i.key(), shVarName)) continue;
 
-                if (good)
-                    jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-            }
+        auto & value = i.value();
 
-            else if (value.is_object()) {
-                std::string s2;
-                bool good = true;
+        auto s = handleSimpleType(value);
+        if (s)
+            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
 
-                for (auto i = value.begin(); i != value.end(); ++i) {
-                    auto s3 = handleSimpleType(i.value());
-                    if (!s3) { good = false; break; }
-                    s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-                }
+        else if (value.is_array()) {
+            std::string s2;
+            bool good = true;
 
-                if (good)
-                    jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += *s3; s2 += ' ';
             }
+
+            if (good)
+                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
         }
 
-        writeFile(tmpDir + "/.attrs.sh", jsonSh);
+        else if (value.is_object()) {
+            std::string s2;
+            bool good = true;
 
-    } catch (std::exception & e) {
-        throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+            for (auto i = value.begin(); i != value.end(); ++i) {
+                auto s3 = handleSimpleType(i.value());
+                if (!s3) { good = false; break; }
+                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+            }
+
+            if (good)
+                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+        }
     }
+
+    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
 }
 
 
@@ -2917,7 +2998,7 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
-            bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1";
+            bool allowLocalNetworking = getBoolAttr("__darwinAllowLocalNetworking");
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
@@ -2989,10 +3070,9 @@ void DerivationGoal::runChild()
 /* Parse a list of reference specifiers.  Each element must either be
    a store path, or the symbolic name of the output of the derivation
    (such as `out'). */
-PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr)
+PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
 {
     PathSet result;
-    Paths paths = tokenizeString<Paths>(attr);
     for (auto & i : paths) {
         if (store.isStorePath(i))
             result.insert(i);
@@ -3121,7 +3201,7 @@ void DerivationGoal::registerOutputs()
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
 
-            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path));
 
             if (h != h2) {
 
@@ -3204,9 +3284,10 @@ void DerivationGoal::registerOutputs()
 
         /* Enforce `allowedReferences' and friends. */
         auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
-            if (drv->env.find(attrName) == drv->env.end()) return;
+            auto value = getStringsAttr(attrName);
+            if (!value) return;
 
-            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName));
+            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
 
             PathSet used;
             if (recursive) {

From 99d4bb2d4cd94e0234e8cfb12887db155d98ac50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Sep 2018 13:28:15 +0200
Subject: [PATCH 1242/2196] Remove obsolete check on system features

---
 src/libstore/build.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 727b8b3444a..69c3c2c1e04 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1665,7 +1665,6 @@ HookReply DerivationGoal::tryBuildHook()
            required from the build machine.  (The hook could parse the
            drv file itself, but this is easier.) */
         auto features = getStringsAttr("requiredSystemFeatures").value_or(Strings());
-        for (auto & i : features) checkStoreName(i); /* !!! abuse */
 
         /* Send the request to the hook. */
         worker.hook->sink

From 7ae7a38c9a7d0a5679e65c8213cd7b58dfdc1c52 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Sep 2018 14:31:16 +0200
Subject: [PATCH 1243/2196] Move structured attrs handling into a separate
 class

This is primarily because Derivation::{can,will}BuildLocally() depends
on attributes like preferLocalBuild and requiredSystemFeatures, but it
can't handle them properly because it doesn't have access to the
structured attributes.
---
 src/libstore/build.cc              | 113 ++++-------------------------
 src/libstore/derivations.cc        |  14 ----
 src/libstore/derivations.hh        |   4 -
 src/libstore/parsed-derivations.cc |  97 +++++++++++++++++++++++++
 src/libstore/parsed-derivations.hh |  33 +++++++++
 5 files changed, 145 insertions(+), 116 deletions(-)
 create mode 100644 src/libstore/parsed-derivations.cc
 create mode 100644 src/libstore/parsed-derivations.hh

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 69c3c2c1e04..eb7f106a20c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -11,6 +11,7 @@
 #include "compression.hh"
 #include "json.hh"
 #include "nar-info.hh"
+#include "parsed-derivations.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -740,8 +741,7 @@ class DerivationGoal : public Goal
     /* The derivation stored at drvPath. */
     std::unique_ptr<BasicDerivation> drv;
 
-    /* The contents of drv->env["__json"]. */
-    std::experimental::optional<nlohmann::json> structuredAttrs;
+    std::unique_ptr<ParsedDerivation> parsedDrv;
 
     /* The remainder is state held during the build. */
 
@@ -923,13 +923,6 @@ class DerivationGoal : public Goal
     /* Fill in the environment for the builder. */
     void initEnv();
 
-    /* Get an attribute from drv->env or from drv->env["__json"]. */
-    std::experimental::optional<std::string> getAttr(const std::string & name);
-
-    bool getBoolAttr(const std::string & name, bool def = false);
-
-    std::experimental::optional<Strings> getStringsAttr(const std::string & name);
-
     /* Write a JSON file containing the derivation attributes. */
     void writeStructuredAttrs();
 
@@ -1149,15 +1142,7 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    /* Parse the __json attribute, if any. */
-    auto jsonAttr = drv->env.find("__json");
-    if (jsonAttr != drv->env.end()) {
-        try {
-            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
-        } catch (std::exception & e) {
-            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
-        }
-    }
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
@@ -1415,7 +1400,7 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || drv->willBuildLocally();
+    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
 
     auto started = [&]() {
         auto msg = fmt(
@@ -1664,7 +1649,7 @@ HookReply DerivationGoal::tryBuildHook()
         /* Tell the hook about system features (beyond the system type)
            required from the build machine.  (The hook could parse the
            drv file itself, but this is easier.) */
-        auto features = getStringsAttr("requiredSystemFeatures").value_or(Strings());
+        auto features = parsedDrv->getStringsAttr("requiredSystemFeatures").value_or(Strings());
 
         /* Send the request to the hook. */
         worker.hook->sink
@@ -1812,7 +1797,7 @@ static void preloadNSS() {
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
-    if (!drv->canBuildLocally()) {
+    if (!parsedDrv->canBuildLocally()) {
         throw Error(
             format("a '%1%' is required to build '%3%', but I am a '%2%'")
             % drv->platform % settings.thisSystem % drvPath);
@@ -1822,12 +1807,12 @@ void DerivationGoal::startBuilder()
         preloadNSS();
 
 #if __APPLE__
-    additionalSandboxProfile = getAttr("__sandboxProfile").value_or("");
+    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
 
     /* Are we doing a chroot build? */
     {
-        auto noChroot = getBoolAttr("__noChroot");
+        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
             if (noChroot)
                 throw Error(format("derivation '%1%' has '__noChroot' set, "
@@ -1893,7 +1878,7 @@ void DerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    if (!structuredAttrs) {
+    if (!parsedDrv->getStructuredAttrs()) {
         /* The `exportReferencesGraph' feature allows the references graph
            to be passed to a builder.  This attribute should be a list of
            pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -1958,7 +1943,7 @@ void DerivationGoal::startBuilder()
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
-        auto impurePaths = getStringsAttr("__impureHostDeps").value_or(Strings());
+        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
 
         for (auto & i : impurePaths) {
             bool found = false;
@@ -2326,7 +2311,7 @@ void DerivationGoal::initEnv()
        passAsFile is ignored in structure mode because it's not
        needed (attributes are not passed through the environment, so
        there is no size constraint). */
-    if (!structuredAttrs) {
+    if (!parsedDrv->getStructuredAttrs()) {
 
         StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
         int fileNr = 0;
@@ -2373,7 +2358,7 @@ void DerivationGoal::initEnv()
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
-        for (auto & i : getStringsAttr("impureEnvVars").value_or(Strings()))
+        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i);
     }
 
@@ -2384,80 +2369,12 @@ void DerivationGoal::initEnv()
 }
 
 
-std::experimental::optional<std::string> DerivationGoal::getAttr(const std::string & name)
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return {};
-        else {
-            if (!i->is_string())
-                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
-            return i->get<std::string>();
-        }
-    } else {
-        auto i = drv->env.find(name);
-        if (i == drv->env.end())
-            return {};
-        else
-            return i->second;
-    }
-}
-
-
-bool DerivationGoal::getBoolAttr(const std::string & name, bool def)
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return def;
-        else {
-            if (!i->is_boolean())
-                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
-            return i->get<bool>();
-        }
-    } else {
-        auto i = drv->env.find(name);
-        if (i == drv->env.end())
-            return def;
-        else
-            return i->second == "1";
-    }
-}
-
-
-std::experimental::optional<Strings> DerivationGoal::getStringsAttr(const std::string & name)
-{
-    if (structuredAttrs) {
-        auto i = structuredAttrs->find(name);
-        if (i == structuredAttrs->end())
-            return {};
-        else {
-            if (!i->is_array())
-                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
-            Strings res;
-            for (auto j = i->begin(); j != i->end(); ++j) {
-                if (!j->is_string())
-                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
-                res.push_back(j->get<std::string>());
-            }
-            return res;
-        }
-    } else {
-        auto i = drv->env.find(name);
-        if (i == drv->env.end())
-            return {};
-        else
-            return tokenizeString<Strings>(i->second);
-    }
-}
-
-
 static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 
 void DerivationGoal::writeStructuredAttrs()
 {
+    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
 
     auto json = *structuredAttrs;
@@ -2997,7 +2914,7 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
-            bool allowLocalNetworking = getBoolAttr("__darwinAllowLocalNetworking");
+            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
@@ -3283,7 +3200,7 @@ void DerivationGoal::registerOutputs()
 
         /* Enforce `allowedReferences' and friends. */
         auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
-            auto value = getStringsAttr(attrName);
+            auto value = parsedDrv->getStringsAttr(attrName);
             if (!value) return;
 
             PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1e187ec5e95..3961126fff9 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const
 }
 
 
-bool BasicDerivation::willBuildLocally() const
-{
-    return get(env, "preferLocalBuild") == "1" && canBuildLocally();
-}
-
-
 bool BasicDerivation::substitutesAllowed() const
 {
     return get(env, "allowSubstitutes", "1") == "1";
@@ -54,14 +48,6 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-bool BasicDerivation::canBuildLocally() const
-{
-    return platform == settings.thisSystem
-        || settings.extraPlatforms.get().count(platform) > 0
-        || isBuiltin();
-}
-
-
 Path writeDerivation(ref<Store> store,
     const Derivation & drv, const string & name, RepairFlag repair)
 {
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 7b97730d3bf..9753e796db5 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -56,14 +56,10 @@ struct BasicDerivation
        the given derivation. */
     Path findOutput(const string & id) const;
 
-    bool willBuildLocally() const;
-
     bool substitutesAllowed() const;
 
     bool isBuiltin() const;
 
-    bool canBuildLocally() const;
-
     /* Return true iff this is a fixed-output derivation. */
     bool isFixedOutput() const;
 
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
new file mode 100644
index 00000000000..0d7acf046af
--- /dev/null
+++ b/src/libstore/parsed-derivations.cc
@@ -0,0 +1,97 @@
+#include "parsed-derivations.hh"
+
+namespace nix {
+
+ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
+    : drvPath(drvPath), drv(drv)
+{
+    /* Parse the __json attribute, if any. */
+    auto jsonAttr = drv.env.find("__json");
+    if (jsonAttr != drv.env.end()) {
+        try {
+            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+        } catch (std::exception & e) {
+            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+        }
+    }
+}
+
+std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_string())
+                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
+            return i->get<std::string>();
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return {};
+        else
+            return i->second;
+    }
+}
+
+bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return def;
+        else {
+            if (!i->is_boolean())
+                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
+            return i->get<bool>();
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return def;
+        else
+            return i->second == "1";
+    }
+}
+
+std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
+{
+    if (structuredAttrs) {
+        auto i = structuredAttrs->find(name);
+        if (i == structuredAttrs->end())
+            return {};
+        else {
+            if (!i->is_array())
+                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+            Strings res;
+            for (auto j = i->begin(); j != i->end(); ++j) {
+                if (!j->is_string())
+                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                res.push_back(j->get<std::string>());
+            }
+            return res;
+        }
+    } else {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end())
+            return {};
+        else
+            return tokenizeString<Strings>(i->second);
+    }
+}
+
+bool ParsedDerivation::canBuildLocally() const
+{
+    return drv.platform == settings.thisSystem
+        || settings.extraPlatforms.get().count(drv.platform) > 0
+        || drv.isBuiltin();
+}
+
+bool ParsedDerivation::willBuildLocally() const
+{
+    return getBoolAttr("preferLocalBuild") && canBuildLocally();
+}
+
+}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
new file mode 100644
index 00000000000..0c7dc32e1e0
--- /dev/null
+++ b/src/libstore/parsed-derivations.hh
@@ -0,0 +1,33 @@
+#include "derivations.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+class ParsedDerivation
+{
+    Path drvPath;
+    BasicDerivation & drv;
+    std::experimental::optional<nlohmann::json> structuredAttrs;
+
+public:
+
+    ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
+
+    const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const
+    {
+        return structuredAttrs;
+    }
+
+    std::experimental::optional<std::string> getStringAttr(const std::string & name) const;
+
+    bool getBoolAttr(const std::string & name, bool def = false) const;
+
+    std::experimental::optional<Strings> getStringsAttr(const std::string & name) const;
+
+    bool canBuildLocally() const;
+
+    bool willBuildLocally() const;
+};
+
+}

From 1e7b8deea7e052ed9ebf47d1411bcaf542054b41 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Sep 2018 15:57:27 +0200
Subject: [PATCH 1244/2196] Check requiredSystemFeatures for local builds

For example, this prevents a "kvm" build on machines that don't have
KVM.

Fixes #2012.
---
 doc/manual/command-ref/conf-file.xml       | 27 ++++++++++++++++++++++
 doc/manual/release-notes/release-notes.xml |  1 +
 src/libstore/build.cc                      | 21 ++++++++---------
 src/libstore/globals.cc                    | 15 ++++++++++++
 src/libstore/globals.hh                    |  6 +++++
 src/libstore/parsed-derivations.cc         | 20 +++++++++++++---
 src/libstore/parsed-derivations.hh         |  2 ++
 tests/build-remote.sh                      |  3 ++-
 8 files changed, 79 insertions(+), 16 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index fd09883be40..e9947ebc673 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -757,6 +757,33 @@ password <replaceable>my-password</replaceable>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term>
+
+    <listitem><para>A set of system “features” supported by this
+    machine, e.g. <literal>kvm</literal>. Derivations can express a
+    dependency on such features through the derivation attribute
+    <varname>requiredSystemFeatures</varname>. For example, the
+    attribute
+
+<programlisting>
+requiredSystemFeatures = [ "kvm" ];
+</programlisting>
+
+    ensures that the derivation can only be built on a machine with
+    the <literal>kvm</literal> feature.</para>
+
+    <para>This setting by default includes <literal>kvm</literal> if
+    <filename>/dev/kvm</filename> is accessible, and the
+    pseudo-features <literal>nixos-test</literal>,
+    <literal>benchmark</literal> and <literal>big-parallel</literal>
+    that are used in Nixpkgs to route builds to specific
+    machines.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
     <listitem>
diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index ff4085cb792..e8ff586fa43 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
 <xi:include href="rl-1.11.10.xml" />
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index eb7f106a20c..0073b9b727e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1646,18 +1646,13 @@ HookReply DerivationGoal::tryBuildHook()
 
     try {
 
-        /* Tell the hook about system features (beyond the system type)
-           required from the build machine.  (The hook could parse the
-           drv file itself, but this is easier.) */
-        auto features = parsedDrv->getStringsAttr("requiredSystemFeatures").value_or(Strings());
-
         /* Send the request to the hook. */
         worker.hook->sink
             << "try"
             << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
             << drv->platform
             << drvPath
-            << features;
+            << parsedDrv->getRequiredSystemFeatures();
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
@@ -1797,11 +1792,13 @@ static void preloadNSS() {
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
-    if (!parsedDrv->canBuildLocally()) {
-        throw Error(
-            format("a '%1%' is required to build '%3%', but I am a '%2%'")
-            % drv->platform % settings.thisSystem % drvPath);
-    }
+    if (!parsedDrv->canBuildLocally())
+        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
+            drv->platform,
+            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
+            drvPath,
+            settings.thisSystem,
+            concatStringsSep(", ", settings.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
@@ -2625,7 +2622,7 @@ void DerivationGoal::runChild()
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
                 ss.push_back("/dev/full");
-                if (pathExists("/dev/kvm"))
+                if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
                     ss.push_back("/dev/kvm");
                 ss.push_back("/dev/null");
                 ss.push_back("/dev/random");
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index d95db56726c..a9c07b23a6f 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -86,6 +86,21 @@ unsigned int Settings::getDefaultCores()
     return std::max(1U, std::thread::hardware_concurrency());
 }
 
+StringSet Settings::getDefaultSystemFeatures()
+{
+    /* For backwards compatibility, accept some "features" that are
+       used in Nixpkgs to route builds to certain machines but don't
+       actually require anything special on the machines. */
+    StringSet features{"nixos-test", "benchmark", "big-parallel"};
+
+    #if __linux__
+    if (access("/dev/kvm", R_OK | W_OK) == 0)
+        features.insert("kvm");
+    #endif
+
+    return features;
+}
+
 const string nixVersion = PACKAGE_VERSION;
 
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index f589078dbb9..cf4ae63cdc2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -32,6 +32,8 @@ class Settings : public Config {
 
     unsigned int getDefaultCores();
 
+    StringSet getDefaultSystemFeatures();
+
 public:
 
     Settings();
@@ -261,6 +263,10 @@ public:
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
         "or using hacks like qemu-user."};
 
+    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+        "system-features",
+        "Optional features that this system implements (like \"kvm\")."};
+
     Setting<Strings> substituters{this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 0d7acf046af..dc328648273 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -82,11 +82,25 @@ std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std:
     }
 }
 
+StringSet ParsedDerivation::getRequiredSystemFeatures() const
+{
+    StringSet res;
+    for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
+        res.insert(i);
+    return res;
+}
+
 bool ParsedDerivation::canBuildLocally() const
 {
-    return drv.platform == settings.thisSystem
-        || settings.extraPlatforms.get().count(drv.platform) > 0
-        || drv.isBuiltin();
+    if (drv.platform != settings.thisSystem.get()
+        && !settings.extraPlatforms.get().count(drv.platform)
+        && !drv.isBuiltin())
+        return false;
+
+    for (auto & feature : getRequiredSystemFeatures())
+        if (!settings.systemFeatures.get().count(feature)) return false;
+
+    return true;
 }
 
 bool ParsedDerivation::willBuildLocally() const
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 0c7dc32e1e0..0a82c146172 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -25,6 +25,8 @@ public:
 
     std::experimental::optional<Strings> getStringsAttr(const std::string & name) const;
 
+    StringSet getRequiredSystemFeatures() const;
+
     bool canBuildLocally() const;
 
     bool willBuildLocally() const;
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index 9bca0f4a385..ddd68f327a1 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -11,7 +11,8 @@ rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
 
 nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
   --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
-  --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo"
+  --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \
+  --system-features foo
 
 outPath=$TEST_ROOT/result
 

From 5b7cfa487e42d0ee198f628a2f2a17bcedc81dc1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 Oct 2018 11:58:54 +0200
Subject: [PATCH 1245/2196] Add missing file

---
 doc/manual/release-notes/rl-2.2.xml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/manual/release-notes/rl-2.2.xml

diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml
new file mode 100644
index 00000000000..bc28a56c940
--- /dev/null
+++ b/doc/manual/release-notes/rl-2.2.xml
@@ -0,0 +1,25 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.2">
+
+<title>Release 2.2 (201?-??-??)</title>
+
+<para>This release has the following changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The derivation attribute
+    <varname>requiredSystemFeatures</varname> is now enforced for
+    local builds, and not just to route builds to remote builders.
+    The supported features of a machine can be specified through the
+    configuration setting <varname>system-features</varname>.
+    </para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+

From 9cc876fb11dd28081d74c74834e75b05f08af8ef Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Mon, 1 Oct 2018 13:26:59 -0500
Subject: [PATCH 1246/2196] nix-profile-daemon: remove cruft

This removes part of the PATH that were being added automatically in multi-user installs:

- $HOME/.nix-profile/lib/kde4/libexec - shouldn't be needed anymore, we are now using kde5
- @localstatedir@/nix/profiles/default/lib/kde4/libexec - same as above
- @localstatedir@/nix/profiles/default - shouldn't ever contain binaries
---
 scripts/nix-profile-daemon.sh.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 1be9a0755d8..87d9fe5061a 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -68,4 +68,4 @@ elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then #
 fi
 
 export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
-export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH"
+export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

From 05cfc71cab4dd4c53f919d7451c36c8b5876919d Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 2 Oct 2018 15:38:16 +0200
Subject: [PATCH 1247/2196] manual: add XML IDs to all builtin list entries

---
 doc/manual/expressions/builtins.xml | 364 +++++++++++++++++-----------
 1 file changed, 217 insertions(+), 147 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 873f30b062e..8d12da9b135 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -21,7 +21,8 @@ available as <function>builtins.derivation</function>.</para>
 <variablelist>
 
 
-  <varlistentry><term><function>abort</function> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-abort'>
+    <term><function>abort</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Abort Nix expression evaluation, print error
     message <replaceable>s</replaceable>.</para></listitem>
@@ -29,8 +30,10 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.add</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-add'>
+    <term><function>builtins.add</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable>
+    </term>
 
     <listitem><para>Return the sum of the numbers
     <replaceable>e1</replaceable> and
@@ -39,8 +42,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.all</function>
-  <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-all'>
+    <term><function>builtins.all</function>
+    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the function
     <replaceable>pred</replaceable> returns <literal>true</literal>
@@ -50,8 +54,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.any</function>
-  <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-any'>
+    <term><function>builtins.any</function>
+    <replaceable>pred</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the function
     <replaceable>pred</replaceable> returns <literal>true</literal>
@@ -61,8 +66,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.attrNames</function>
-  <replaceable>set</replaceable></term>
+  <varlistentry xml:id='builtin-attrNames'>
+    <term><function>builtins.attrNames</function>
+    <replaceable>set</replaceable></term>
 
     <listitem><para>Return the names of the attributes in the set
     <replaceable>set</replaceable> in an alphabetically sorted list.  For instance,
@@ -72,8 +78,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.attrValues</function>
-  <replaceable>set</replaceable></term>
+  <varlistentry xml:id='builtin-attrValues'>
+    <term><function>builtins.attrValues</function>
+    <replaceable>set</replaceable></term>
 
     <listitem><para>Return the values of the attributes in the set
     <replaceable>set</replaceable> in the order corresponding to the
@@ -82,7 +89,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>baseNameOf</function> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-baseNameOf'>
+    <term><function>baseNameOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the <emphasis>base name</emphasis> of the
     string <replaceable>s</replaceable>, that is, everything following
@@ -92,8 +100,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.bitAnd</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-bitAnd'>
+    <term><function>builtins.bitAnd</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise AND of the integers
     <replaceable>e1</replaceable> and
@@ -102,8 +111,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.bitOr</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-bitOr'>
+    <term><function>builtins.bitOr</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise OR of the integers
     <replaceable>e1</replaceable> and
@@ -112,8 +122,9 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.bitXor</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-bitXor'>
+    <term><function>builtins.bitXor</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the bitwise XOR of the integers
     <replaceable>e1</replaceable> and
@@ -122,7 +133,8 @@ available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
 
-  <varlistentry><term><varname>builtins</varname></term>
+  <varlistentry xml:id='builtin-builtins'>
+    <term><varname>builtins</varname></term>
 
     <listitem><para>The set <varname>builtins</varname> contains all
     the built-in functions and values.  You can use
@@ -139,8 +151,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.compareVersions</function>
-  <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+  <varlistentry xml:id='builtin-compareVersions'>
+    <term><function>builtins.compareVersions</function>
+    <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
 
     <listitem><para>Compare two strings representing versions and
     return <literal>-1</literal> if version
@@ -156,8 +169,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.splitVersion</function>
-  <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-splitVersion'>
+    <term><function>builtins.splitVersion</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Split a string representing a version into its
     components, by the same version splitting logic underlying the
@@ -167,16 +181,18 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.concatLists</function>
-  <replaceable>lists</replaceable></term>
+  <varlistentry xml:id='builtin-concatLists'>
+    <term><function>builtins.concatLists</function>
+    <replaceable>lists</replaceable></term>
 
     <listitem><para>Concatenate a list of lists into a single
     list.</para></listitem>
 
   </varlistentry>
 
-  <varlistentry><term><function>builtins.concatStringsSep</function>
-  <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-concatStringsSep'>
+    <term><function>builtins.concatStringsSep</function>
+    <replaceable>separator</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Concatenate a list of strings with a separator
     between each element, e.g. <literal>concatStringsSep "/"
@@ -184,8 +200,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
-  <varlistentry
-  xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term>
+  <varlistentry xml:id='builtin-currentSystem'>
+    <term><varname>builtins.currentSystem</varname></term>
 
     <listitem><para>The built-in value <varname>currentSystem</varname>
     evaluates to the Nix platform identifier for the Nix installation
@@ -218,8 +234,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   -->
 
 
-  <varlistentry><term><function>builtins.deepSeq</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-deepSeq'>
+    <term><function>builtins.deepSeq</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>This is like <literal>seq
     <replaceable>e1</replaceable>
@@ -231,8 +248,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>derivation</function>
-  <replaceable>attrs</replaceable></term>
+  <varlistentry xml:id='builtin-derivation'>
+    <term><function>derivation</function>
+    <replaceable>attrs</replaceable></term>
 
     <listitem><para><function>derivation</function> is described in
     <xref linkend='ssec-derivation' />.</para></listitem>
@@ -240,7 +258,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>dirOf</function> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-dirOf'>
+    <term><function>dirOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the directory part of the string
     <replaceable>s</replaceable>, that is, everything before the final
@@ -250,8 +269,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.div</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-div'>
+    <term><function>builtins.div</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the quotient of the numbers
     <replaceable>e1</replaceable> and
@@ -259,8 +279,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   </varlistentry>
 
-  <varlistentry><term><function>builtins.elem</function>
-  <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
+  <varlistentry xml:id='builtin-elem'>
+    <term><function>builtins.elem</function>
+    <replaceable>x</replaceable> <replaceable>xs</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if a value equal to
     <replaceable>x</replaceable> occurs in the list
@@ -270,8 +291,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.elemAt</function>
-  <replaceable>xs</replaceable> <replaceable>n</replaceable></term>
+  <varlistentry xml:id='builtin-elemAt'>
+    <term><function>builtins.elemAt</function>
+    <replaceable>xs</replaceable> <replaceable>n</replaceable></term>
 
     <listitem><para>Return element <replaceable>n</replaceable> from
     the list <replaceable>xs</replaceable>.  Elements are counted
@@ -281,8 +303,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.fetchurl</function>
-  <replaceable>url</replaceable></term>
+  <varlistentry xml:id='builtin-fetchurl'>
+    <term><function>builtins.fetchurl</function>
+    <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL and return the path of
     the downloaded file. This function is not available if <link
@@ -292,8 +315,9 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>fetchTarball</function>
-  <replaceable>url</replaceable></term>
+  <varlistentry xml:id='builtin-fetchTarball'>
+    <term><function>fetchTarball</function>
+    <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL, unpack it and return
     the path of the unpacked tree. The file must be a tape archive
@@ -346,7 +370,7 @@ stdenv.mkDerivation { … }
 
   </varlistentry>
 
-  <varlistentry>
+  <varlistentry xml:id='builtin-fetchGit'>
     <term>
       <function>builtins.fetchGit</function>
       <replaceable>args</replaceable>
@@ -546,7 +570,8 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.foldl’</function>
+  <varlistentry xml:id='builtin-foldl-prime'>
+    <term><function>builtins.foldl’</function>
     <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Reduce a list by applying a binary operator, from
@@ -559,7 +584,8 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.functionArgs</function>
+  <varlistentry xml:id='builtin-functionArgs'>
+    <term><function>builtins.functionArgs</function>
     <replaceable>f</replaceable></term>
 
     <listitem><para>
@@ -577,7 +603,8 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-fromJSON'>
+    <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert a JSON string to a Nix
     value. For example,
@@ -592,8 +619,9 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.genList</function>
-  <replaceable>generator</replaceable> <replaceable>length</replaceable></term>
+  <varlistentry xml:id='builtin-genList'>
+    <term><function>builtins.genList</function>
+    <replaceable>generator</replaceable> <replaceable>length</replaceable></term>
 
     <listitem><para>Generate list of size
     <replaceable>length</replaceable>, with each element
@@ -610,8 +638,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.getAttr</function>
-  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+  <varlistentry xml:id='builtin-getAttr'>
+    <term><function>builtins.getAttr</function>
+    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>getAttr</function> returns the attribute
     named <replaceable>s</replaceable> from
@@ -623,8 +652,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.getEnv</function>
-  <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-getEnv'>
+    <term><function>builtins.getEnv</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para><function>getEnv</function> returns the value of
     the environment variable <replaceable>s</replaceable>, or an empty
@@ -641,8 +671,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.hasAttr</function>
-  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
+  <varlistentry xml:id='builtin-hasAttr'>
+    <term><function>builtins.hasAttr</function>
+    <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>hasAttr</function> returns
     <literal>true</literal> if <replaceable>set</replaceable> has an
@@ -655,8 +686,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.hashString</function>
-  <replaceable>type</replaceable> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-hashString'>
+    <term><function>builtins.hashString</function>
+    <replaceable>type</replaceable> <replaceable>s</replaceable></term>
 
     <listitem><para>Return a base-16 representation of the
     cryptographic hash of string <replaceable>s</replaceable>.  The
@@ -667,8 +699,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.head</function>
-  <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-head'>
+    <term><function>builtins.head</function>
+    <replaceable>list</replaceable></term>
 
     <listitem><para>Return the first element of a list; abort
     evaluation if the argument isn’t a list or is an empty list.  You
@@ -678,8 +711,9 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
-  <varlistentry><term><function>import</function>
-  <replaceable>path</replaceable></term>
+  <varlistentry xml:id='builtin-import'>
+    <term><function>import</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
@@ -733,8 +767,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.intersectAttrs</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-intersectAttrs'>
+    <term><function>builtins.intersectAttrs</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return a set consisting of the attributes in the
     set <replaceable>e2</replaceable> that also exist in the set
@@ -743,8 +778,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isAttrs</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isAttrs'>
+    <term><function>builtins.isAttrs</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a set, and
@@ -753,8 +789,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isList</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isList'>
+    <term><function>builtins.isList</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a list, and
@@ -763,7 +800,7 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isFunction</function>
+  <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function>
   <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
@@ -773,8 +810,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isString</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isString'>
+    <term><function>builtins.isString</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a string, and
@@ -783,8 +821,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isInt</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isInt'>
+    <term><function>builtins.isInt</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to an int, and
@@ -793,8 +832,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isFloat</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isFloat'>
+    <term><function>builtins.isFloat</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a float, and
@@ -803,8 +843,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.isBool</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isBool'>
+    <term><function>builtins.isBool</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to a bool, and
@@ -813,8 +854,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>isNull</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-isNull'>
+    <term><function>isNull</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to <literal>null</literal>,
@@ -828,8 +870,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.length</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-length'>
+    <term><function>builtins.length</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return the length of the list
     <replaceable>e</replaceable>.</para></listitem>
@@ -837,8 +880,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.lessThan</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-lessThan'>
+    <term><function>builtins.lessThan</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the number
     <replaceable>e1</replaceable> is less than the number
@@ -850,8 +894,9 @@ x: x + 456</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.listToAttrs</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-listToAttrs'>
+    <term><function>builtins.listToAttrs</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Construct a set from a list specifying the names
     and values of each attribute.  Each element of the list should be
@@ -877,8 +922,9 @@ builtins.listToAttrs
 
   </varlistentry>
 
-  <varlistentry><term><function>map</function>
-  <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-map'>
+    <term><function>map</function>
+    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Apply the function <replaceable>f</replaceable> to
     each element in the list <replaceable>list</replaceable>.  For
@@ -893,14 +939,15 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.match</function>
-  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+  <varlistentry xml:id='builtin-match'>
+    <term><function>builtins.match</function>
+    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
 
-  <listitem><para>Returns a list if the <link
-  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-  POSIX regular expression</link> <replaceable>regex</replaceable>
-  matches <replaceable>str</replaceable> precisely, otherwise returns
-  <literal>null</literal>.  Each item in the list is a regex group.
+    <listitem><para>Returns a list if the <link
+    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+    POSIX regular expression</link> <replaceable>regex</replaceable>
+    matches <replaceable>str</replaceable> precisely, otherwise returns
+    <literal>null</literal>.  Each item in the list is a regex group.
 
 <programlisting>
 builtins.match "ab" "abc"
@@ -926,11 +973,12 @@ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
 
 Evaluates to <literal>[ "foo" ]</literal>.
 
-  </para></listitem>
+    </para></listitem>
   </varlistentry>
 
-  <varlistentry><term><function>builtins.mul</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-mul'>
+    <term><function>builtins.mul</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the product of the numbers
     <replaceable>e1</replaceable> and
@@ -939,8 +987,9 @@ Evaluates to <literal>[ "foo" ]</literal>.
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.parseDrvName</function>
-  <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-parseDrvName'>
+    <term><function>builtins.parseDrvName</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Split the string <replaceable>s</replaceable> into
     a package name and version.  The package name is everything up to
@@ -953,7 +1002,7 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
-  <varlistentry>
+  <varlistentry xml:id='builtin-path'>
     <term>
       <function>builtins.path</function>
       <replaceable>args</replaceable>
@@ -1023,8 +1072,9 @@ Evaluates to <literal>[ "foo" ]</literal>.
     </listitem>
   </varlistentry>
 
-  <varlistentry><term><function>builtins.pathExists</function>
-  <replaceable>path</replaceable></term>
+  <varlistentry xml:id='builtin-pathExists'>
+    <term><function>builtins.pathExists</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the path
     <replaceable>path</replaceable> exists, and
@@ -1047,8 +1097,9 @@ in config.someSetting</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.readDir</function>
-  <replaceable>path</replaceable></term>
+  <varlistentry xml:id='builtin-readDir'>
+    <term><function>builtins.readDir</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Return the contents of the directory
     <replaceable>path</replaceable> as a set mapping directory entries
@@ -1069,8 +1120,9 @@ in config.someSetting</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.readFile</function>
-  <replaceable>path</replaceable></term>
+  <varlistentry xml:id='builtin-readFile'>
+    <term><function>builtins.readFile</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Return the contents of the file
     <replaceable>path</replaceable> as a string.</para></listitem>
@@ -1078,8 +1130,9 @@ in config.someSetting</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>removeAttrs</function>
-  <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-removeAttrs'>
+    <term><function>removeAttrs</function>
+    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Remove the attributes listed in
     <replaceable>list</replaceable> from
@@ -1094,8 +1147,9 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.replaceStrings</function>
-  <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-replaceStrings'>
+    <term><function>builtins.replaceStrings</function>
+    <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term>
 
     <listitem><para>Given string <replaceable>s</replaceable>, replace
     every occurrence of the strings in <replaceable>from</replaceable>
@@ -1111,8 +1165,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.seq</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-seq'>
+    <term><function>builtins.seq</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Evaluate <replaceable>e1</replaceable>, then
     evaluate and return <replaceable>e2</replaceable>. This ensures
@@ -1122,8 +1177,9 @@ builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar"
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.sort</function>
-  <replaceable>comparator</replaceable> <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-sort'>
+    <term><function>builtins.sort</function>
+    <replaceable>comparator</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Return <replaceable>list</replaceable> in sorted
     order. It repeatedly calls the function
@@ -1145,15 +1201,16 @@ builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.split</function>
-  <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
+  <varlistentry xml:id='builtin-split'>
+    <term><function>builtins.split</function>
+    <replaceable>regex</replaceable> <replaceable>str</replaceable></term>
 
-  <listitem><para>Returns a list composed of non matched strings interleaved
-  with the lists of the <link
-  xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
-  POSIX regular expression</link> <replaceable>regex</replaceable> matches
-  of <replaceable>str</replaceable>. Each item in the lists of matched
-  sequences is a regex group.
+    <listitem><para>Returns a list composed of non matched strings interleaved
+    with the lists of the <link
+    xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended
+    POSIX regular expression</link> <replaceable>regex</replaceable> matches
+    of <replaceable>str</replaceable>. Each item in the lists of matched
+    sequences is a regex group.
 
 <programlisting>
 builtins.split "(a)b" "abc"
@@ -1179,11 +1236,12 @@ builtins.split "([[:upper:]]+)" "  FOO   "
 
 Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
 
-  </para></listitem>
+    </para></listitem>
   </varlistentry>
 
-  <varlistentry><term><function>builtins.stringLength</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-stringLength'>
+    <term><function>builtins.stringLength</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return the length of the string
     <replaceable>e</replaceable>.  If <replaceable>e</replaceable> is
@@ -1192,8 +1250,9 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.sub</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-sub'>
+    <term><function>builtins.sub</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Return the difference between the numbers
     <replaceable>e1</replaceable> and
@@ -1202,9 +1261,10 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.substring</function>
-  <replaceable>start</replaceable> <replaceable>len</replaceable>
-  <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-substring'>
+    <term><function>builtins.substring</function>
+    <replaceable>start</replaceable> <replaceable>len</replaceable>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Return the substring of
     <replaceable>s</replaceable> from character position
@@ -1227,8 +1287,9 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.tail</function>
-  <replaceable>list</replaceable></term>
+  <varlistentry xml:id='builtin-tail'>
+    <term><function>builtins.tail</function>
+    <replaceable>list</replaceable></term>
 
     <listitem><para>Return the second to last elements of a list;
     abort evaluation if the argument isn’t a list or is an empty
@@ -1237,8 +1298,9 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry><term><function>throw</function>
-  <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-throw'>
+    <term><function>throw</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Throw an error message
     <replaceable>s</replaceable>.  This usually aborts Nix expression
@@ -1251,9 +1313,10 @@ builtins.substring 0 3 "nixos"
   </varlistentry>
 
 
-  <varlistentry
-  xml:id='builtin-toFile'><term><function>builtins.toFile</function>
-  <replaceable>name</replaceable> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-toFile'>
+    <term><function>builtins.toFile</function>
+    <replaceable>name</replaceable>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Store the string <replaceable>s</replaceable> in a
     file in the Nix store and return its path.  The file has suffix
@@ -1329,7 +1392,8 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.toJSON</function> <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-toJSON'>
+    <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string containing a JSON representation
     of <replaceable>e</replaceable>.  Strings, integers, floats, booleans,
@@ -1342,7 +1406,8 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
+  <varlistentry xml:id='builtin-toPath'>
+    <term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Convert the string value
     <replaceable>s</replaceable> into a path value.  The string
@@ -1355,7 +1420,8 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry><term><function>toString</function> <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-toString'>
+    <term><function>toString</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.
@@ -1374,7 +1440,8 @@ in foo</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-toXML'><term><function>builtins.toXML</function> <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-toXML'>
+    <term><function>builtins.toXML</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string containing an XML representation
     of <replaceable>e</replaceable>.  The main application for
@@ -1489,8 +1556,9 @@ stdenv.mkDerivation (rec {
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.trace</function>
-  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+  <varlistentry xml:id='builtin-trace'>
+    <term><function>builtins.trace</function>
+    <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem><para>Evaluate <replaceable>e1</replaceable> and print its
     abstract syntax representation on standard error.  Then return
@@ -1499,8 +1567,9 @@ stdenv.mkDerivation (rec {
 
   </varlistentry>
 
-  <varlistentry><term><function>builtins.tryEval</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-tryEval'>
+    <term><function>builtins.tryEval</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Try to evaluate <replaceable>e</replaceable>.
     Return a set containing the attributes <literal>success</literal>
@@ -1513,8 +1582,9 @@ stdenv.mkDerivation (rec {
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.typeOf</function>
-  <replaceable>e</replaceable></term>
+  <varlistentry xml:id='builtin-typeOf'>
+    <term><function>builtins.typeOf</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return a string representing the type of the value
     <replaceable>e</replaceable>, namely <literal>"int"</literal>,

From 65453e2d777dac8d90f98e9d24b6428fce9c2ab5 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Thu, 4 Oct 2018 02:37:15 +0000
Subject: [PATCH 1248/2196] Docs: update fixed output hashes

`fetchurl` will now throw if given an `md5`, and the hashes have generally
been upgraded to avoid it and use `sha256` as a default. This updates the
documentation examples in the manual accordingly.
---
 doc/manual/expressions/advanced-attributes.xml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 9422e82ff36..2af7a51acfb 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -216,7 +216,7 @@ fetchurl {
 <programlisting>
 { stdenv, curl }: # The <command>curl</command> program is used for downloading.
 
-{ url, md5 }:
+{ url, sha256 }:
 
 stdenv.mkDerivation {
   name = baseNameOf (toString url);
@@ -224,10 +224,10 @@ stdenv.mkDerivation {
   buildInputs = [ curl ];
 
   # This is a fixed-output derivation; the output must be a regular
-  # file with MD5 hash <varname>md5</varname>.
+  # file with SHA256 hash <varname>sha256</varname>.
   outputHashMode = "flat";
-  outputHashAlgo = "md5";
-  outputHash = md5;
+  outputHashAlgo = "sha256";
+  outputHash = sha256;
 
   inherit url;
 }
@@ -237,8 +237,8 @@ stdenv.mkDerivation {
 
     <para>The <varname>outputHashAlgo</varname> attribute specifies
     the hash algorithm used to compute the hash.  It can currently be
-    <literal>"md5"</literal>, <literal>"sha1"</literal> or
-    <literal>"sha256"</literal>.</para>
+    <literal>"sha1"</literal>, <literal>"sha256"</literal> or
+    <literal>"sha512"</literal>.</para>
 
     <para>The <varname>outputHashMode</varname> attribute determines
     how the hash is computed.  It must be one of the following two
@@ -251,7 +251,7 @@ stdenv.mkDerivation {
         <listitem><para>The output must be a non-executable regular
         file.  If it isn’t, the build fails.  The hash is simply
         computed over the contents of that file (so it’s equal to what
-        Unix commands like <command>md5sum</command> or
+        Unix commands like <command>sha256sum</command> or
         <command>sha1sum</command> produce).</para>
 
         <para>This is the default.</para></listitem>

From 0fda9b22c7cc0f0191a51d168c4b03d6c4e98d71 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Oct 2018 13:16:30 +0200
Subject: [PATCH 1249/2196] Remove unnecessary typecast

---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 615c1f5dc06..423ba15e2ad 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -709,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
 
         // FIXME: race if addToStore doesn't read source?
-        store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair,
+        store->addToStore(info, *source, (RepairFlag) repair,
             dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
 
         logger->stopWork();

From ea41838ae049817fd996ead14e040d922bb7c067 Mon Sep 17 00:00:00 2001
From: "Graham Christensen (Target)"
 <39596207+graham-at-target@users.noreply.github.com>
Date: Tue, 16 Oct 2018 10:22:36 -0400
Subject: [PATCH 1250/2196] install script: remove unportable command check,
 fixup errant escape

`which` isn't necessarily portable, but `command -v` is an equivalent form.

Additionally, the `\'` is not necessary, as it is already quoted by `"`.
---
 scripts/install.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install.in b/scripts/install.in
index 26ab85ba099..7bff7b216d9 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -11,14 +11,14 @@ oops() {
 }
 
 tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
-          oops "Can\'t create temporary directory for downloading the Nix binary tarball")"
+          oops "Can't create temporary directory for downloading the Nix binary tarball")"
 cleanup() {
     rm -rf "$tmpDir"
 }
 trap cleanup EXIT INT QUIT TERM
 
 require_util() {
-    type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 ||
+    type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 ||
         oops "you do not have '$1' installed, which I need to $2"
 }
 

From ba51100d64c18f627f97e606c4884ba2fb78dfa0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Oct 2018 22:58:01 +0200
Subject: [PATCH 1251/2196] Get rid of UDSRemoteStore::Connection

Since its superclass RemoteStore::Connection contains 'to' and 'from'
fields that refer to the file descriptor maintained in the subclass,
it was possible for the flush() call in Connection::~Connection() to
write to a closed file descriptor (or worse, a file descriptor now
referencing another file). So make sure that the file descriptor
survives 'to' and 'from'.
---
 src/libstore/remote-store.hh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b488e34ce26..9d768576bad 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -103,6 +103,7 @@ protected:
 
     struct Connection
     {
+        AutoCloseFD fd;
         FdSink to;
         FdSource from;
         unsigned int daemonVersion;
@@ -140,11 +141,6 @@ public:
 
 private:
 
-    struct Connection : RemoteStore::Connection
-    {
-        AutoCloseFD fd;
-    };
-
     ref<RemoteStore::Connection> openConnection() override;
     std::experimental::optional<std::string> path;
 };

From 79e358ce6d2a2c34f365c24d68ddbab7461380f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Oct 2018 23:36:15 +0200
Subject: [PATCH 1252/2196] RemoteStore: Close connection if an exception
 occurs

Fixes #2075.
---
 src/libstore/remote-store.cc | 171 ++++++++++++++++++++++-------------
 src/libstore/remote-store.hh |   7 +-
 src/libutil/pool.hh          |   6 +-
 3 files changed, 121 insertions(+), 63 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ea86ef052f5..ef8b0e53b80 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -161,7 +161,8 @@ void RemoteStore::initConnection(Connection & conn)
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
             conn.to << false;
 
-        conn.processStderr();
+        auto ex = conn.processStderr();
+        if (ex) std::rethrow_exception(ex);
     }
     catch (Error & e) {
         throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what());
@@ -195,22 +196,68 @@ void RemoteStore::setOptions(Connection & conn)
             conn.to << i.first << i.second.value;
     }
 
-    conn.processStderr();
+    auto ex = conn.processStderr();
+    if (ex) std::rethrow_exception(ex);
+}
+
+
+/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
+   the connection as bad (causing it to be closed) if a non-daemon
+   exception is thrown before the handle is closed. Such an exception
+   causes a deviation from the expected protocol and therefore a
+   desynchronization between the client and daemon. */
+struct ConnectionHandle
+{
+    Pool<RemoteStore::Connection>::Handle handle;
+    bool daemonException = false;
+
+    ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
+        : handle(std::move(handle))
+    { }
+
+    ConnectionHandle(ConnectionHandle && h)
+        : handle(std::move(h.handle))
+    { }
+
+    ~ConnectionHandle()
+    {
+        if (!daemonException && std::uncaught_exception()) {
+            handle.markBad();
+            debug("closing daemon connection because of an exception");
+        }
+    }
+
+    RemoteStore::Connection * operator -> () { return &*handle; }
+
+    void processStderr(Sink * sink = 0, Source * source = 0)
+    {
+        auto ex = handle->processStderr(sink, source);
+        if (ex) {
+            daemonException = true;
+            std::rethrow_exception(ex);
+        }
+    }
+};
+
+
+ConnectionHandle RemoteStore::getConnection()
+{
+    return ConnectionHandle(connections->get());
 }
 
 
 bool RemoteStore::isValidPathUncached(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopIsValidPath << path;
-    conn->processStderr();
+    conn.processStderr();
     return readInt(conn->from);
 }
 
 
 PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths)
@@ -218,7 +265,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
         return res;
     } else {
         conn->to << wopQueryValidPaths << paths;
-        conn->processStderr();
+        conn.processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -226,27 +273,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe
 
 PathSet RemoteStore::queryAllValidPaths()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
         PathSet res;
         for (auto & i : paths) {
             conn->to << wopHasSubstitutes << i;
-            conn->processStderr();
+            conn.processStderr();
             if (readInt(conn->from)) res.insert(i);
         }
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths << paths;
-        conn->processStderr();
+        conn.processStderr();
         return readStorePaths<PathSet>(*this, conn->from);
     }
 }
@@ -257,14 +304,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 {
     if (paths.empty()) return;
 
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
 
         for (auto & i : paths) {
             SubstitutablePathInfo info;
             conn->to << wopQuerySubstitutablePathInfo << i;
-            conn->processStderr();
+            conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
             info.deriver = readString(conn->from);
@@ -278,7 +325,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos << paths;
-        conn->processStderr();
+        conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
             Path path = readStorePath(*this, conn->from);
@@ -300,10 +347,10 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
     try {
         std::shared_ptr<ValidPathInfo> info;
         {
-            auto conn(connections->get());
+            auto conn(getConnection());
             conn->to << wopQueryPathInfo << path;
             try {
-                conn->processStderr();
+                conn.processStderr();
             } catch (Error & e) {
                 // Ugly backwards compatibility hack.
                 if (e.msg().find("is not valid") != std::string::npos)
@@ -335,9 +382,9 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
 void RemoteStore::queryReferrers(const Path & path,
     PathSet & referrers)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryReferrers << path;
-    conn->processStderr();
+    conn.processStderr();
     PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from);
     referrers.insert(referrers2.begin(), referrers2.end());
 }
@@ -345,36 +392,36 @@ void RemoteStore::queryReferrers(const Path & path,
 
 PathSet RemoteStore::queryValidDerivers(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryValidDerivers << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryDerivationOutputs << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStorePaths<PathSet>(*this, conn->from);
 }
 
 
 PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryDerivationOutputNames << path;
-    conn->processStderr();
+    conn.processStderr();
     return readStrings<PathSet>(conn->from);
 }
 
 
 Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopQueryPathFromHashPart << hashPart;
-    conn->processStderr();
+    conn.processStderr();
     Path path = readString(conn->from);
     if (!path.empty()) assertStorePath(path);
     return path;
@@ -384,7 +431,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
         conn->to << wopImportPaths;
@@ -403,7 +450,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                 ;
         });
 
-        conn->processStderr(0, source2.get());
+        conn.processStderr(0, source2.get());
 
         auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
         assert(importedPaths.size() <= 1);
@@ -417,7 +464,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << repair << !checkSigs;
         bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
         if (!tunnel) copyNAR(source, conn->to);
-        conn->processStderr(0, tunnel ? &source : nullptr);
+        conn.processStderr(0, tunnel ? &source : nullptr);
     }
 }
 
@@ -427,7 +474,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     Path srcPath(absPath(_srcPath));
 
@@ -445,13 +492,13 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
             dumpPath(srcPath, conn->to, filter);
         }
         conn->to.warn = false;
-        conn->processStderr();
+        conn.processStderr();
     } catch (SysError & e) {
         /* Daemon closed while we were sending the path. Probably OOM
            or I/O error. */
         if (e.errNo == EPIPE)
             try {
-                conn->processStderr();
+                conn.processStderr();
             } catch (EndOfFile & e) { }
         throw;
     }
@@ -465,17 +512,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddTextToStore << name << s << references;
 
-    conn->processStderr();
+    conn.processStderr();
     return readStorePath(*this, conn->from);
 }
 
 
 void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopBuildPaths;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
         conn->to << drvPaths;
@@ -494,7 +541,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
             drvPaths2.insert(string(i, 0, i.find('!')));
         conn->to << drvPaths2;
     }
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
@@ -502,9 +549,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopBuildDerivation << drvPath << drv << buildMode;
-    conn->processStderr();
+    conn.processStderr();
     BuildResult res;
     unsigned int status;
     conn->from >> status >> res.errorMsg;
@@ -515,45 +562,45 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
 
 void RemoteStore::ensurePath(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopEnsurePath << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addTempRoot(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddTempRoot << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::addIndirectRoot(const Path & path)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddIndirectRoot << path;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 void RemoteStore::syncWithGC()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopSyncWithGC;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 Roots RemoteStore::findRoots()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopFindRoots;
-    conn->processStderr();
+    conn.processStderr();
     size_t count = readNum<size_t>(conn->from);
     Roots result;
     while (count--) {
@@ -567,7 +614,7 @@ Roots RemoteStore::findRoots()
 
 void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
 
     conn->to
         << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
@@ -575,7 +622,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
         /* removed options */
         << 0 << 0 << 0;
 
-    conn->processStderr();
+    conn.processStderr();
 
     results.paths = readStrings<PathSet>(conn->from);
     results.bytesFreed = readLongLong(conn->from);
@@ -590,27 +637,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void RemoteStore::optimiseStore()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopOptimiseStore;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
 
 bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopVerifyStore << checkContents << repair;
-    conn->processStderr();
+    conn.processStderr();
     return readInt(conn->from);
 }
 
 
 void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
     conn->to << wopAddSignatures << storePath << sigs;
-    conn->processStderr();
+    conn.processStderr();
     readInt(conn->from);
 }
 
@@ -620,13 +667,13 @@ void RemoteStore::queryMissing(const PathSet & targets,
     unsigned long long & downloadSize, unsigned long long & narSize)
 {
     {
-        auto conn(connections->get());
+        auto conn(getConnection());
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19)
             // Don't hold the connection handle in the fallback case
             // to prevent a deadlock.
             goto fallback;
         conn->to << wopQueryMissing << targets;
-        conn->processStderr();
+        conn.processStderr();
         willBuild = readStorePaths<PathSet>(*this, conn->from);
         willSubstitute = readStorePaths<PathSet>(*this, conn->from);
         unknown = readStorePaths<PathSet>(*this, conn->from);
@@ -642,7 +689,7 @@ void RemoteStore::queryMissing(const PathSet & targets,
 
 void RemoteStore::connect()
 {
-    auto conn(connections->get());
+    auto conn(getConnection());
 }
 
 
@@ -679,7 +726,7 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
 {
     to.flush();
 
@@ -704,7 +751,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         else if (msg == STDERR_ERROR) {
             string error = readString(from);
             unsigned int status = readInt(from);
-            throw Error(status, error);
+            return std::make_exception_ptr(Error(status, error));
         }
 
         else if (msg == STDERR_NEXT)
@@ -738,6 +785,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         else
             throw Error("got unknown message type %x from Nix daemon", msg);
     }
+
+    return nullptr;
 }
 
 static std::string uriScheme = "unix://";
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 9d768576bad..7f9d7d1f56d 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -14,6 +14,7 @@ class Pid;
 struct FdSink;
 struct FdSource;
 template<typename T> class Pool;
+struct ConnectionHandle;
 
 
 /* FIXME: RemoteStore is a misnomer - should be something like
@@ -111,7 +112,7 @@ protected:
 
         virtual ~Connection();
 
-        void processStderr(Sink * sink = 0, Source * source = 0);
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
     };
 
     ref<Connection> openConnectionWrapper();
@@ -124,6 +125,10 @@ protected:
 
     virtual void setOptions(Connection & conn);
 
+    ConnectionHandle getConnection();
+
+    friend class ConnectionHandle;
+
 private:
 
     std::atomic_bool failed{false};
diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh
index 0b142b0597c..d49067bb95d 100644
--- a/src/libutil/pool.hh
+++ b/src/libutil/pool.hh
@@ -97,6 +97,7 @@ public:
     private:
         Pool & pool;
         std::shared_ptr<R> r;
+        bool bad = false;
 
         friend Pool;
 
@@ -112,7 +113,8 @@ public:
             if (!r) return;
             {
                 auto state_(pool.state.lock());
-                state_->idle.push_back(ref<R>(r));
+                if (!bad)
+                    state_->idle.push_back(ref<R>(r));
                 assert(state_->inUse);
                 state_->inUse--;
             }
@@ -121,6 +123,8 @@ public:
 
         R * operator -> () { return &*r; }
         R & operator * () { return *r; }
+
+        void markBad() { bad = true; }
     };
 
     Handle get()

From bd78544f66a001f00623d3d0e9d50b019ada6a9a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Oct 2018 23:39:36 +0200
Subject: [PATCH 1253/2196] Fix assertion failure in Store::queryPathInfo()

  $ nix-store -qR /nix/store/fnord
  nix-store: src/libstore/store-api.cc:80: std::__cxx11::string nix::storePathToHash(const Path&): Assertion `base.size() >= storePathHashLen' failed.
  Aborted
---
 src/libstore/store-api.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 2c0f68651e4..92e2685f7f6 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -320,6 +320,8 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 void Store::queryPathInfo(const Path & storePath,
     Callback<ref<ValidPathInfo>> callback)
 {
+    assertStorePath(storePath);
+
     auto hashPart = storePathToHash(storePath);
 
     try {

From 73c2ae43f08ca35cbb8f86ec7c2efc15ad8686b9 Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Sat, 29 Sep 2018 09:42:11 +0200
Subject: [PATCH 1254/2196] Add --graphml option to the nix-store --query
 command

This prints the references graph of the store paths in the graphML
format [1]. The graphML format is supported by several graph tools
such as the Python Networkx library or the Apache Thinkerpop project.

[1] http://graphml.graphdrawing.org
---
 doc/manual/command-ref/nix-store.xml | 12 ++++
 src/nix-store/graphml.cc             | 90 ++++++++++++++++++++++++++++
 src/nix-store/graphml.hh             | 11 ++++
 src/nix-store/nix-store.cc           | 14 ++++-
 4 files changed, 126 insertions(+), 1 deletion(-)
 create mode 100644 src/nix-store/graphml.cc
 create mode 100644 src/nix-store/graphml.hh

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index c827d85b381..41a04f265d7 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -679,6 +679,18 @@ query is applied to the target of the symlink.</para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--graphml</option></term>
+
+    <listitem><para>Prints the references graph of the store paths
+    <replaceable>paths</replaceable> in the <link
+    xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format.
+    This can be used to visualise dependency graphs. To obtain a
+    build-time dependency graph, apply this to a store derivation. To
+    obtain a runtime dependency graph, apply it to an output
+    path.</para></listitem>
+
+  </varlistentry>
+
   <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term>
     <term><option>-b</option> <replaceable>name</replaceable></term>
 
diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc
new file mode 100644
index 00000000000..670fbe227a4
--- /dev/null
+++ b/src/nix-store/graphml.cc
@@ -0,0 +1,90 @@
+#include "graphml.hh"
+#include "util.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+
+#include <iostream>
+
+
+using std::cout;
+
+namespace nix {
+
+
+static inline const string & xmlQuote(const string & s)
+{
+    // Luckily, store paths shouldn't contain any character that needs to be
+    // quoted.
+    return s;
+}
+
+
+static string symbolicName(const string & path)
+{
+    string p = baseNameOf(path);
+    return string(p, p.find('-') + 1);
+}
+
+
+static string makeEdge(const string & src, const string & dst)
+{
+    return fmt("  <edge source=\"%1%\" target=\"%2%\"/>\n",
+        xmlQuote(src), xmlQuote(dst));
+}
+
+
+static string makeNode(const ValidPathInfo & info)
+{
+    return fmt(
+        "  <node id=\"%1%\">\n"
+        "    <data key=\"narSize\">%2%</data>\n"
+        "    <data key=\"name\">%3%</data>\n"
+        "    <data key=\"type\">%4%</data>\n"
+        "  </node>\n",
+        info.path,
+        info.narSize,
+        symbolicName(info.path),
+        (isDerivation(info.path) ? "derivation" : "output-path"));
+}
+
+
+void printGraphML(ref<Store> store, const PathSet & roots)
+{
+    PathSet workList(roots);
+    PathSet doneSet;
+    std::pair<PathSet::iterator,bool> ret;
+
+    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
+         << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n"
+         << "    xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n"
+         << "    xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n"
+         << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>"
+         << "<key id='name' for='node' attr.name='name' attr.type='string'/>"
+         << "<key id='type' for='node' attr.name='type' attr.type='string'/>"
+         << "<graph id='G' edgedefault='directed'>\n";
+
+    while (!workList.empty()) {
+        Path path = *(workList.begin());
+        workList.erase(path);
+
+        ret = doneSet.insert(path);
+        if (ret.second == false) continue;
+
+        ValidPathInfo info = *(store->queryPathInfo(path));
+        cout << makeNode(info);
+
+        for (auto & p : store->queryPathInfo(path)->references) {
+            if (p != path) {
+                workList.insert(p);
+                cout << makeEdge(path, p);
+            }
+        }
+
+    }
+
+    cout << "</graph>\n";
+    cout << "</graphml>\n";
+}
+
+
+}
diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh
new file mode 100644
index 00000000000..b78df1e49a6
--- /dev/null
+++ b/src/nix-store/graphml.hh
@@ -0,0 +1,11 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+class Store;
+
+void printGraphML(ref<Store> store, const PathSet & roots);
+
+}
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index fe68f681ae2..4051fdbe166 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,6 +9,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "xmlgraph.hh"
+#include "graphml.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -273,7 +274,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
     enum QueryType
         { qDefault, qOutputs, qRequisites, qReferences, qReferrers
         , qReferrersClosure, qDeriver, qBinding, qHash, qSize
-        , qTree, qGraph, qXml, qResolve, qRoots };
+        , qTree, qGraph, qXml, qGraphML, qResolve, qRoots };
     QueryType query = qDefault;
     bool useOutput = false;
     bool includeOutputs = false;
@@ -300,6 +301,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--tree") query = qTree;
         else if (i == "--graph") query = qGraph;
         else if (i == "--xml") query = qXml;
+        else if (i == "--graphml") query = qGraphML;
         else if (i == "--resolve") query = qResolve;
         else if (i == "--roots") query = qRoots;
         else if (i == "--use-output" || i == "-u") useOutput = true;
@@ -413,6 +415,16 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
         }
 
+        case qGraphML: {
+            PathSet roots;
+            for (auto & i : opArgs) {
+                PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
+                roots.insert(paths.begin(), paths.end());
+            }
+            printGraphML(ref<Store>(store), roots);
+            break;
+        }
+
         case qResolve: {
             for (auto & i : opArgs)
                 cout << format("%1%\n") % store->followLinksToStorePath(i);

From d506342aa2b6945899988878b7c58de683cb573a Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Sat, 20 Oct 2018 09:48:53 +0200
Subject: [PATCH 1255/2196] Remove the `--xml` query command option

The `--graphml` option can be used instead.
---
 src/nix-store/nix-store.cc | 14 +-------
 src/nix-store/xmlgraph.cc  | 66 --------------------------------------
 src/nix-store/xmlgraph.hh  | 11 -------
 3 files changed, 1 insertion(+), 90 deletions(-)
 delete mode 100644 src/nix-store/xmlgraph.cc
 delete mode 100644 src/nix-store/xmlgraph.hh

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 4051fdbe166..e245bd643d4 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -8,7 +8,6 @@
 #include "shared.hh"
 #include "util.hh"
 #include "worker-protocol.hh"
-#include "xmlgraph.hh"
 #include "graphml.hh"
 
 #include <iostream>
@@ -274,7 +273,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
     enum QueryType
         { qDefault, qOutputs, qRequisites, qReferences, qReferrers
         , qReferrersClosure, qDeriver, qBinding, qHash, qSize
-        , qTree, qGraph, qXml, qGraphML, qResolve, qRoots };
+        , qTree, qGraph, qGraphML, qResolve, qRoots };
     QueryType query = qDefault;
     bool useOutput = false;
     bool includeOutputs = false;
@@ -300,7 +299,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--size") query = qSize;
         else if (i == "--tree") query = qTree;
         else if (i == "--graph") query = qGraph;
-        else if (i == "--xml") query = qXml;
         else if (i == "--graphml") query = qGraphML;
         else if (i == "--resolve") query = qResolve;
         else if (i == "--roots") query = qRoots;
@@ -405,16 +403,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
         }
 
-        case qXml: {
-            PathSet roots;
-            for (auto & i : opArgs) {
-                PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
-                roots.insert(paths.begin(), paths.end());
-            }
-            printXmlGraph(ref<Store>(store), roots);
-            break;
-        }
-
         case qGraphML: {
             PathSet roots;
             for (auto & i : opArgs) {
diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc
deleted file mode 100644
index 0f7be7f7a02..00000000000
--- a/src/nix-store/xmlgraph.cc
+++ /dev/null
@@ -1,66 +0,0 @@
-#include "xmlgraph.hh"
-#include "util.hh"
-#include "store-api.hh"
-
-#include <iostream>
-
-
-using std::cout;
-
-namespace nix {
-
-
-static inline const string & xmlQuote(const string & s)
-{
-    // Luckily, store paths shouldn't contain any character that needs to be
-    // quoted.
-    return s;
-}
-
-
-static string makeEdge(const string & src, const string & dst)
-{
-    format f = format("  <edge src=\"%1%\" dst=\"%2%\"/>\n")
-      % xmlQuote(src) % xmlQuote(dst);
-    return f.str();
-}
-
-
-static string makeNode(const string & id)
-{
-    format f = format("  <node name=\"%1%\"/>\n") % xmlQuote(id);
-    return f.str();
-}
-
-
-void printXmlGraph(ref<Store> store, const PathSet & roots)
-{
-    PathSet workList(roots);
-    PathSet doneSet;
-
-    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
-         << "<nix>\n";
-
-    while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
-
-        if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
-
-        cout << makeNode(path);
-
-        for (auto & p : store->queryPathInfo(path)->references) {
-            if (p != path) {
-                workList.insert(p);
-                cout << makeEdge(p, path);
-            }
-        }
-
-    }
-
-    cout << "</nix>\n";
-}
-
-
-}
diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh
deleted file mode 100644
index a6e7d4e2805..00000000000
--- a/src/nix-store/xmlgraph.hh
+++ /dev/null
@@ -1,11 +0,0 @@
-#pragma once
-
-#include "types.hh"
-
-namespace nix {
-
-class Store;
-
-void printXmlGraph(ref<Store> store, const PathSet & roots);
-
-}

From 3cd15c5b1f5a8e6de87d5b7e8cc2f1326b420c88 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 22 Oct 2018 21:49:56 +0200
Subject: [PATCH 1256/2196] Per-output reference and closure size checks

In structured-attributes derivations, you can now specify per-output
checks such as:

  outputChecks."out" = {
    # The closure of 'out' must not be larger than 256 MiB.
    maxClosureSize = 256 * 1024 * 1024;

    # It must not refer to C compiler or to the 'dev' output.
    disallowedRequisites = [ stdenv.cc "dev" ];
  };

  outputChecks."dev" = {
    # The 'dev' output must not be larger than 128 KiB.
    maxSize = 128 * 1024;
  };

Also fixed a bug in allowedRequisites that caused it to ignore
self-references.
---
 src/libstore/build.cc | 219 ++++++++++++++++++++++++++++++++----------
 tests/check-reqs.nix  |   2 +-
 2 files changed, 169 insertions(+), 52 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0073b9b727e..cf4218a261f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -21,6 +21,7 @@
 #include <future>
 #include <chrono>
 #include <regex>
+#include <queue>
 
 #include <limits.h>
 #include <sys/time.h>
@@ -857,7 +858,7 @@ class DerivationGoal : public Goal
        building multiple times. Since this contains the hash, it
        allows us to compare whether two rounds produced the same
        result. */
-    ValidPathInfos prevInfos;
+    std::map<Path, ValidPathInfo> prevInfos;
 
     const uid_t sandboxUid = 1000;
     const gid_t sandboxGid = 100;
@@ -938,6 +939,11 @@ class DerivationGoal : public Goal
        as valid. */
     void registerOutputs();
 
+    /* Check that an output meets the requirements specified by the
+       'outputChecks' attribute (or the legacy
+       '{allowed,disallowed}{References,Requisites}' attributes). */
+    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
@@ -3010,7 +3016,7 @@ void DerivationGoal::registerOutputs()
         if (allValid) return;
     }
 
-    ValidPathInfos infos;
+    std::map<std::string, ValidPathInfo> infos;
 
     /* Set of inodes seen during calls to canonicalisePathMetaData()
        for this build's outputs.  This needs to be shared between
@@ -3195,49 +3201,6 @@ void DerivationGoal::registerOutputs()
                 debug(format("referenced input: '%1%'") % i);
         }
 
-        /* Enforce `allowedReferences' and friends. */
-        auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
-            auto value = parsedDrv->getStringsAttr(attrName);
-            if (!value) return;
-
-            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
-
-            PathSet used;
-            if (recursive) {
-                /* Our requisites are the union of the closures of our references. */
-                for (auto & i : references)
-                    /* Don't call computeFSClosure on ourselves. */
-                    if (path != i)
-                        worker.store.computeFSClosure(i, used);
-            } else
-                used = references;
-
-            PathSet badPaths;
-
-            for (auto & i : used)
-                if (allowed) {
-                    if (spec.find(i) == spec.end())
-                        badPaths.insert(i);
-                } else {
-                    if (spec.find(i) != spec.end())
-                        badPaths.insert(i);
-                }
-
-            if (!badPaths.empty()) {
-                string badPathsStr;
-                for (auto & i : badPaths) {
-                    badPathsStr += "\n\t";
-                    badPathsStr += i;
-                }
-                throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr);
-            }
-        };
-
-        checkRefs("allowedReferences", true, false);
-        checkRefs("allowedRequisites", true, true);
-        checkRefs("disallowedReferences", false, false);
-        checkRefs("disallowedRequisites", false, true);
-
         if (curRound == nrRounds) {
             worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
             worker.markContentsGood(path);
@@ -3253,11 +3216,14 @@ void DerivationGoal::registerOutputs()
 
         if (!info.references.empty()) info.ca.clear();
 
-        infos.push_back(info);
+        infos[i.first] = info;
     }
 
     if (buildMode == bmCheck) return;
 
+    /* Apply output checks. */
+    checkOutputs(infos);
+
     /* Compare the result with the previous round, and report which
        path is different, if any.*/
     if (curRound > 1 && prevInfos != infos) {
@@ -3265,16 +3231,16 @@ void DerivationGoal::registerOutputs()
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
                 result.isNonDeterministic = true;
-                Path prev = i->path + checkSuffix;
+                Path prev = i->second.path + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 auto msg = prevExists
-                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev)
-                    : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath);
+                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
+                    : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
 
                 auto diffHook = settings.diffHook;
                 if (prevExists && diffHook != "" && runDiffHook) {
                     try {
-                        auto diff = runProgram(diffHook, true, {prev, i->path});
+                        auto diff = runProgram(diffHook, true, {prev, i->second.path});
                         if (diff != "")
                             printError(chomp(diff));
                     } catch (Error & error) {
@@ -3319,7 +3285,11 @@ void DerivationGoal::registerOutputs()
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
-    worker.store.registerValidPaths(infos);
+    {
+        ValidPathInfos infos2;
+        for (auto & i : infos) infos2.push_back(i.second);
+        worker.store.registerValidPaths(infos2);
+    }
 
     /* In case of a fixed-output derivation hash mismatch, throw an
        exception now that we have registered the output as valid. */
@@ -3328,6 +3298,153 @@ void DerivationGoal::registerOutputs()
 }
 
 
+void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+{
+    std::map<Path, const ValidPathInfo &> outputsByPath;
+    for (auto & output : outputs)
+        outputsByPath.emplace(output.second.path, output.second);
+
+    for (auto & output : outputs) {
+        auto & outputName = output.first;
+        auto & info = output.second;
+
+        struct Checks
+        {
+            std::experimental::optional<uint64_t> maxSize, maxClosureSize;
+            std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+        };
+
+        /* Compute the closure and closure size of some output. This
+           is slightly tricky because some of its references (namely
+           other outputs) may not be valid yet. */
+        auto getClosure = [&](const Path & path)
+        {
+            uint64_t closureSize = 0;
+            PathSet pathsDone;
+            std::queue<Path> pathsLeft;
+            pathsLeft.push(path);
+
+            while (!pathsLeft.empty()) {
+                auto path = pathsLeft.front();
+                pathsLeft.pop();
+                if (!pathsDone.insert(path).second) continue;
+
+                auto i = outputsByPath.find(path);
+                if (i != outputsByPath.end()) {
+                    closureSize += i->second.narSize;
+                    for (auto & ref : i->second.references)
+                        pathsLeft.push(ref);
+                } else {
+                    auto info = worker.store.queryPathInfo(path);
+                    closureSize += info->narSize;
+                    for (auto & ref : info->references)
+                        pathsLeft.push(ref);
+                }
+            }
+
+            return std::make_pair(pathsDone, closureSize);
+        };
+
+        auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
+        {
+            if (!value) return;
+
+            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
+
+            PathSet used = recursive ? getClosure(info.path).first : info.references;
+
+            PathSet badPaths;
+
+            for (auto & i : used)
+                if (allowed) {
+                    if (spec.find(i) == spec.end())
+                        badPaths.insert(i);
+                } else {
+                    if (spec.find(i) != spec.end())
+                        badPaths.insert(i);
+                }
+
+            if (!badPaths.empty()) {
+                string badPathsStr;
+                for (auto & i : badPaths) {
+                    badPathsStr += "\n  ";
+                    badPathsStr += i;
+                }
+                throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
+            }
+        };
+
+        auto applyChecks = [&](const Checks & checks)
+        {
+            if (checks.maxSize && info.narSize > *checks.maxSize)
+                throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
+                    info.path, info.narSize, *checks.maxSize);
+
+            if (checks.maxClosureSize) {
+                uint64_t closureSize = getClosure(info.path).second;
+                if (closureSize > *checks.maxClosureSize)
+                    throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
+                        info.path, closureSize, *checks.maxClosureSize);
+            }
+
+            checkRefs(checks.allowedReferences, true, false);
+            checkRefs(checks.allowedRequisites, true, true);
+            checkRefs(checks.disallowedReferences, false, false);
+            checkRefs(checks.disallowedRequisites, false, true);
+        };
+
+        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
+            auto outputChecks = structuredAttrs->find("outputChecks");
+            if (outputChecks != structuredAttrs->end()) {
+                auto output = outputChecks->find(outputName);
+
+                if (output != outputChecks->end()) {
+                    Checks checks;
+
+                    auto maxSize = output->find("maxSize");
+                    if (maxSize != output->end())
+                        checks.maxSize = maxSize->get<uint64_t>();
+
+                    auto maxClosureSize = output->find("maxClosureSize");
+                    if (maxClosureSize != output->end())
+                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
+
+                    auto get = [&](const std::string & name) -> std::experimental::optional<Strings> {
+                        auto i = output->find(name);
+                        if (i != output->end()) {
+                            Strings res;
+                            for (auto j = i->begin(); j != i->end(); ++j) {
+                                if (!j->is_string())
+                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                                res.push_back(j->get<std::string>());
+                            }
+                            checks.disallowedRequisites = res;
+                            return res;
+                        }
+                        return {};
+                    };
+
+                    checks.allowedReferences = get("allowedReferences");
+                    checks.allowedRequisites = get("allowedRequisites");
+                    checks.disallowedReferences = get("disallowedReferences");
+                    checks.disallowedRequisites = get("disallowedRequisites");
+
+                    applyChecks(checks);
+                }
+            }
+        } else {
+            // legacy non-structured-attributes case
+            Checks checks;
+            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
+            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
+            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
+            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
+            applyChecks(checks);
+        }
+    }
+}
+
+
 Path DerivationGoal::openLogFile()
 {
     logSize = 0;
diff --git a/tests/check-reqs.nix b/tests/check-reqs.nix
index 41436cb48e0..47b5b3d9c72 100644
--- a/tests/check-reqs.nix
+++ b/tests/check-reqs.nix
@@ -33,7 +33,7 @@ rec {
   };
 
   # When specifying all the requisites, the build succeeds.
-  test1 = makeTest 1 [ dep1 dep2 deps ];
+  test1 = makeTest 1 [ "out" dep1 dep2 deps ];
 
   # But missing anything it fails.
   test2 = makeTest 2 [ dep2 deps ];

From a25abe823fd2719eee09c28ccddf6d9b1c13a2e1 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Mon, 22 Oct 2018 12:14:02 +0200
Subject: [PATCH 1257/2196] Promote log-lines to a fully-qualified option

This allows commands like

nix build --log-lines 30 nixpkgs.hello

in order to obtain more information in case of a failure.
---
 src/libstore/globals.hh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index cf4ae63cdc2..6b3e204536f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -82,9 +82,9 @@ public:
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
-    /* If verboseBuild is false, the number of lines of the tail of
-       the log to show if a build fails. */
-    size_t logLines = 10;
+    Setting<size_t> logLines{this, 10, "log-lines",
+        "If verbose-build is false, the number of lines of the tail of "
+        "the log to show if a build fails."};
 
     MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
         "Maximum number of parallel build jobs. \"auto\" means use number of cores.",

From f6a3dfe4e06980b2d060fd1a646cb5ca20f29779 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 26 Oct 2018 11:35:46 +0200
Subject: [PATCH 1258/2196] Merge all nix-* binaries into nix

These are all symlinks to 'nix' now, reducing the installed size by
about ~1.7 MiB.
---
 Makefile                                      | 10 ---------
 local.mk                                      |  2 +-
 src/build-remote/build-remote.cc              | 19 +++++++++--------
 src/build-remote/local.mk                     |  9 --------
 src/nix-build/local.mk                        |  9 --------
 src/nix-build/nix-build.cc                    | 14 ++++---------
 src/nix-channel/local.mk                      |  7 -------
 src/nix-channel/nix-channel.cc                | 16 ++++++++------
 src/nix-collect-garbage/local.mk              |  7 -------
 .../nix-collect-garbage.cc                    | 15 +++++++------
 src/nix-copy-closure/local.mk                 |  7 -------
 src/nix-copy-closure/nix-copy-closure.cc      | 13 +++++++-----
 src/nix-daemon/local.mk                       | 13 ------------
 src/nix-daemon/nix-daemon.cc                  | 15 +++++++------
 src/nix-env/local.mk                          |  7 -------
 src/nix-env/nix-env.cc                        | 14 +++++++------
 src/nix-instantiate/local.mk                  |  7 -------
 src/nix-instantiate/nix-instantiate.cc        | 16 +++++++-------
 src/nix-prefetch-url/local.mk                 |  7 -------
 src/nix-prefetch-url/nix-prefetch-url.cc      | 14 +++++++------
 src/nix-store/local.mk                        |  9 --------
 src/nix-store/nix-store.cc                    | 13 +++++++-----
 src/nix/local.mk                              | 21 ++++++++++++++++---
 src/nix/main.cc                               |  6 +++---
 24 files changed, 105 insertions(+), 165 deletions(-)
 delete mode 100644 src/build-remote/local.mk
 delete mode 100644 src/nix-build/local.mk
 delete mode 100644 src/nix-channel/local.mk
 delete mode 100644 src/nix-collect-garbage/local.mk
 delete mode 100644 src/nix-copy-closure/local.mk
 delete mode 100644 src/nix-daemon/local.mk
 delete mode 100644 src/nix-env/local.mk
 delete mode 100644 src/nix-instantiate/local.mk
 delete mode 100644 src/nix-prefetch-url/local.mk
 delete mode 100644 src/nix-store/local.mk

diff --git a/Makefile b/Makefile
index 834f84b286b..45a3338ed21 100644
--- a/Makefile
+++ b/Makefile
@@ -5,17 +5,7 @@ makefiles = \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
-  src/nix-store/local.mk \
-  src/nix-instantiate/local.mk \
-  src/nix-env/local.mk \
-  src/nix-daemon/local.mk \
-  src/nix-collect-garbage/local.mk \
-  src/nix-copy-closure/local.mk \
-  src/nix-prefetch-url/local.mk \
   src/resolve-system-dependencies/local.mk \
-  src/nix-channel/local.mk \
-  src/nix-build/local.mk \
-  src/build-remote/local.mk \
   scripts/local.mk \
   corepkgs/local.mk \
   misc/systemd/local.mk \
diff --git a/local.mk b/local.mk
index 5d7e0fb2e42..4b380176f2e 100644
--- a/local.mk
+++ b/local.mk
@@ -6,7 +6,7 @@ dist-files += configure config.h.in nix.spec perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 38dbe3e58b2..abf3669b5b3 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,6 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -37,11 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-int main (int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         logger = makeJSONLogger(*logger);
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
@@ -80,7 +79,7 @@ int main (int argc, char * * argv)
 
         if (machines.empty()) {
             std::cerr << "# decline-permanently\n";
-            return;
+            return 0;
         }
 
         string drvPath;
@@ -90,8 +89,8 @@ int main (int argc, char * * argv)
 
             try {
                 auto s = readString(source);
-                if (s != "try") return;
-            } catch (EndOfFile &) { return; }
+                if (s != "try") return 0;
+            } catch (EndOfFile &) { return 0; }
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
@@ -253,6 +252,8 @@ int main (int argc, char * * argv)
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
-        return;
-    });
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("build-remote", _main);
diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
deleted file mode 100644
index 50b0409d188..00000000000
--- a/src/build-remote/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += build-remote
-
-build-remote_DIR := $(d)
-
-build-remote_INSTALL_DIR := $(libexecdir)/nix
-
-build-remote_LIBS = libmain libformat libstore libutil
-
-build-remote_SOURCES := $(d)/build-remote.cc
diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk
deleted file mode 100644
index a2d1c91dfd9..00000000000
--- a/src/nix-build/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += nix-build
-
-nix-build_DIR := $(d)
-
-nix-build_SOURCES := $(d)/nix-build.cc
-
-nix-build_LIBS = libmain libexpr libstore libutil libformat
-
-$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell))
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index b78f3d9e424..618895d387d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -16,6 +16,7 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
+#include "legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -66,11 +67,8 @@ std::vector<string> shellwords(const string & s)
     return res;
 }
 
-void mainWrapped(int argc, char * * argv)
+static void _main(int argc, char * * argv)
 {
-    initNix();
-    initGC();
-
     auto dryRun = false;
     auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$"));
     auto pure = false;
@@ -504,9 +502,5 @@ void mainWrapped(int argc, char * * argv)
     }
 }
 
-int main(int argc, char * * argv)
-{
-    return handleExceptions(argv[0], [&]() {
-        return mainWrapped(argc, argv);
-    });
-}
+static RegisterLegacyCommand s1("nix-build", _main);
+static RegisterLegacyCommand s2("nix-shell", _main);
diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk
deleted file mode 100644
index c14e8c359ca..00000000000
--- a/src/nix-channel/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-channel
-
-nix-channel_DIR := $(d)
-
-nix-channel_LIBS = libmain libformat libstore libutil
-
-nix-channel_SOURCES := $(d)/nix-channel.cc
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 2083d3df5ca..8b66cc7e314 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -1,9 +1,11 @@
 #include "shared.hh"
 #include "globals.hh"
 #include "download.hh"
+#include "store-api.hh"
+#include "legacy.hh"
+
 #include <fcntl.h>
 #include <regex>
-#include "store-api.hh"
 #include <pwd.h>
 
 using namespace nix;
@@ -157,11 +159,9 @@ static void update(const StringSet & channelNames)
     replaceSymlink(profile, channelLink);
 }
 
-int main(int argc, char ** argv)
+static int _main(int argc, char ** argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         // Figure out the name of the `.nix-channels' file to use
         auto home = getHome();
         channelsList = home + "/.nix-channels";
@@ -255,5 +255,9 @@ int main(int argc, char ** argv)
                 runProgram(settings.nixBinDir + "/nix-env", false, envArgs);
                 break;
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-channel", _main);
diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk
deleted file mode 100644
index 02d14cf6219..00000000000
--- a/src/nix-collect-garbage/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-collect-garbage
-
-nix-collect-garbage_DIR := $(d)
-
-nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc
-
-nix-collect-garbage_LIBS = libmain libstore libutil libformat
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 37fe22f4813..d4060ac937f 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -2,6 +2,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <cerrno>
@@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir)
     }
 }
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    bool removeOld = false;
-
-    return handleExceptions(argv[0], [&]() {
-        initNix();
+    {
+        bool removeOld = false;
 
         GCOptions options;
 
@@ -90,5 +89,9 @@ int main(int argc, char * * argv)
             PrintFreed freed(true, results);
             store->collectGarbage(options, results);
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-collect-garbage", _main);
diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk
deleted file mode 100644
index 5018ab975b4..00000000000
--- a/src/nix-copy-closure/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-copy-closure
-
-nix-copy-closure_DIR := $(d)
-
-nix-copy-closure_LIBS = libmain libformat libstore libutil
-
-nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index dfb1b8fc5dc..fdcde8b076b 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,13 +1,12 @@
 #include "shared.hh"
 #include "store-api.hh"
+#include "legacy.hh"
 
 using namespace nix;
 
-int main(int argc, char ** argv)
+static int _main(int argc, char ** argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         auto gzip = false;
         auto toMode = true;
         auto includeOutputs = false;
@@ -61,5 +60,9 @@ int main(int argc, char ** argv)
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-copy-closure", _main);
diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk
deleted file mode 100644
index 5a4474465b3..00000000000
--- a/src/nix-daemon/local.mk
+++ /dev/null
@@ -1,13 +0,0 @@
-programs += nix-daemon
-
-nix-daemon_DIR := $(d)
-
-nix-daemon_SOURCES := $(d)/nix-daemon.cc
-
-nix-daemon_LIBS = libmain libstore libutil libformat
-
-nix-daemon_LDFLAGS = -pthread
-
-ifeq ($(OS), SunOS)
-        nix-daemon_LDFLAGS += -lsocket
-endif
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 423ba15e2ad..8368c326614 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -9,6 +9,7 @@
 #include "monitor-fd.hh"
 #include "derivations.hh"
 #include "finally.hh"
+#include "legacy.hh"
 
 #include <algorithm>
 
@@ -1058,11 +1059,9 @@ static void daemonLoop(char * * argv)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         auto stdio = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
@@ -1122,7 +1121,7 @@ int main(int argc, char * * argv)
                         if (res == -1)
                             throw SysError("splicing data from stdin to daemon socket");
                         else if (res == 0)
-                            return;
+                            return 0;
                     }
                 }
             } else {
@@ -1131,5 +1130,9 @@ int main(int argc, char * * argv)
         } else {
             daemonLoop(argv);
         }
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-daemon", _main);
diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk
deleted file mode 100644
index e80719cd76f..00000000000
--- a/src/nix-env/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-env
-
-nix-env_DIR := $(d)
-
-nix-env_SOURCES := $(wildcard $(d)/*.cc)
-
-nix-env_LIBS = libexpr libmain libstore libutil libformat
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index f9c8a8d313e..56ed75daee4 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -13,6 +13,7 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
+#include "legacy.hh"
 
 #include <cerrno>
 #include <ctime>
@@ -1311,12 +1312,9 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         Strings opFlags, opArgs;
         Operation op = 0;
         RepairFlag repair = NoRepair;
@@ -1428,5 +1426,9 @@ int main(int argc, char * * argv)
         op(globals, opFlags, opArgs);
 
         globals.state->printStats();
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-env", _main);
diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk
deleted file mode 100644
index 7d1bc5ec9df..00000000000
--- a/src/nix-instantiate/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-instantiate
-
-nix-instantiate_DIR := $(d)
-
-nix-instantiate_SOURCES := $(d)/nix-instantiate.cc
-
-nix-instantiate_LIBS = libexpr libmain libstore libutil libformat
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index eb6d34dd821..a736caa8f05 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -9,6 +9,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
+#include "legacy.hh"
 
 #include <map>
 #include <iostream>
@@ -83,12 +84,9 @@ void processExpr(EvalState & state, const Strings & attrPaths,
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         Strings files;
         bool readStdin = false;
         bool fromArgs = false;
@@ -171,7 +169,7 @@ int main(int argc, char * * argv)
                 if (p == "") throw Error(format("unable to find '%1%'") % i);
                 std::cout << p << std::endl;
             }
-            return;
+            return 0;
         }
 
         if (readStdin) {
@@ -190,5 +188,9 @@ int main(int argc, char * * argv)
         }
 
         state->printStats();
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-instantiate", _main);
diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk
deleted file mode 100644
index 3e7735406af..00000000000
--- a/src/nix-prefetch-url/local.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-programs += nix-prefetch-url
-
-nix-prefetch-url_DIR := $(d)
-
-nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc
-
-nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index a3b025723cf..ddb72491321 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -6,6 +6,7 @@
 #include "eval-inline.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
+#include "legacy.hh"
 
 #include <iostream>
 
@@ -44,12 +45,9 @@ string resolveMirrorUri(EvalState & state, string uri)
 }
 
 
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-        initGC();
-
+    {
         HashType ht = htSHA256;
         std::vector<string> args;
         bool printPath = getEnv("PRINT_PATH") != "";
@@ -221,5 +219,9 @@ int main(int argc, char * * argv)
         std::cout << printHash16or32(hash) << std::endl;
         if (printPath)
             std::cout << storePath << std::endl;
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-prefetch-url", _main);
diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk
deleted file mode 100644
index ade0b233adf..00000000000
--- a/src/nix-store/local.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-programs += nix-store
-
-nix-store_DIR := $(d)
-
-nix-store_SOURCES := $(wildcard $(d)/*.cc)
-
-nix-store_LIBS = libmain libstore libutil libformat
-
-nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e245bd643d4..a9ad14762e6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,6 +9,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "graphml.hh"
+#include "legacy.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -993,11 +994,9 @@ static void opVersion(Strings opFlags, Strings opArgs)
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
-int main(int argc, char * * argv)
+static int _main(int argc, char * * argv)
 {
-    return handleExceptions(argv[0], [&]() {
-        initNix();
-
+    {
         Strings opFlags, opArgs;
         Operation op = 0;
 
@@ -1084,5 +1083,9 @@ int main(int argc, char * * argv)
             store = openStore();
 
         op(opFlags, opArgs);
-    });
+
+        return 0;
+    }
 }
+
+static RegisterLegacyCommand s1("nix-store", _main);
diff --git a/src/nix/local.mk b/src/nix/local.mk
index f76da194467..168936314dc 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -2,10 +2,25 @@ programs += nix
 
 nix_DIR := $(d)
 
-nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp)
+nix_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+  $(wildcard src/linenoise/*.cpp) \
+  $(wildcard src/build-remote/*.cc) \
+  $(wildcard src/nix-build/*.cc) \
+  $(wildcard src/nix-channel/*.cc) \
+  $(wildcard src/nix-collect-garbage/*.cc) \
+  $(wildcard src/nix-copy-closure/*.cc) \
+  $(wildcard src/nix-daemon/*.cc) \
+  $(wildcard src/nix-env/*.cc) \
+  $(wildcard src/nix-instantiate/*.cc) \
+  $(wildcard src/nix-prefetch-url/*.cc) \
+  $(wildcard src/nix-store/*.cc) \
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
-nix_LDFLAGS = -pthread
+nix_LDFLAGS = -pthread $(SODIUM_LIBS)
 
-$(eval $(call install-symlink, nix, $(bindir)/nix-hash))
+$(foreach name, \
+  nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-rul nix-shell nix-shore, \
+  $(eval $(call install-symlink, nix, $(bindir)/$(name))))
+$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 69791e223c2..64c1dc35787 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -67,9 +67,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
 void mainWrapped(int argc, char * * argv)
 {
-    verbosity = lvlError;
-    settings.verboseBuild = false;
-
     /* The chroot helper needs to be run before any threads have been
        started. */
     if (argc > 0 && argv[0] == chrootHelperName) {
@@ -88,6 +85,9 @@ void mainWrapped(int argc, char * * argv)
         if (legacy) return legacy(argc, argv);
     }
 
+    verbosity = lvlError;
+    settings.verboseBuild = false;
+
     NixArgs args;
 
     args.parseCmdline(argvToStrings(argc, argv));

From 27c2fcd4c067a176491317e47ca448772f8fcc92 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 26 Oct 2018 12:38:06 -0500
Subject: [PATCH 1259/2196] src/nix/local.mk: fix typos in names of symlinks

---
 src/nix/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index 168936314dc..bdcca33d2a6 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -21,6 +21,6 @@ nix_LIBS = libexpr libmain libstore libutil libformat
 nix_LDFLAGS = -pthread $(SODIUM_LIBS)
 
 $(foreach name, \
-  nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-rul nix-shell nix-shore, \
+  nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))

From 3283c0dc45e69f1a8e180cfcb226b47d3c826649 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 26 Oct 2018 12:34:29 -0500
Subject: [PATCH 1260/2196] remote-store.hh: ConnectionHandle is struct, minor
 fix warning

---
 src/libstore/remote-store.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 7f9d7d1f56d..3686dc360fe 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -127,7 +127,7 @@ protected:
 
     ConnectionHandle getConnection();
 
-    friend class ConnectionHandle;
+    friend struct ConnectionHandle;
 
 private:
 

From 9d24b5d56e024b51240fe1010810f1c343de01ff Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 20 Oct 2018 01:11:22 -0500
Subject: [PATCH 1261/2196] nix-prefetch-url: progressbar

---
 src/nix-prefetch-url/nix-prefetch-url.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index ddb72491321..54a402241e3 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -7,6 +7,8 @@
 #include "common-eval-args.hh"
 #include "attr-path.hh"
 #include "legacy.hh"
+#include "finally.hh"
+#include "progress-bar.hh"
 
 #include <iostream>
 
@@ -96,6 +98,11 @@ static int _main(int argc, char * * argv)
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
+        Finally f([]() { stopProgressBar(); });
+
+        if (isatty(STDERR_FILENO))
+          startProgressBar();
+
         auto store = openStore();
         auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
 

From 6a5bf9b1438ed0b721657568fbb3a7c0b829e89e Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Tue, 14 Jun 2016 17:42:46 +0200
Subject: [PATCH 1262/2196] simplify handling of extra '}'

---
 src/libexpr/lexer.l | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index a052447d3dc..c34e5c38392 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -6,9 +6,9 @@
 %option nounput noyy_top_state
 
 
+%s DEFAULT
 %x STRING
 %x IND_STRING
-%x INSIDE_DOLLAR_CURLY
 
 
 %{
@@ -99,8 +99,6 @@ URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~
 
 %%
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
-
 
 if          { return IF; }
 then        { return THEN; }
@@ -140,17 +138,19 @@ or          { return OR_KW; }
               return FLOAT;
             }
 
-\$\{        { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
-}
+\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 
-\}                           { return '}'; }
-<INSIDE_DOLLAR_CURLY>\}      { POP_STATE(); return '}'; }
-\{                           { return '{'; }
-<INSIDE_DOLLAR_CURLY>\{      { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; }
+\}          { /* State INITIAL only exists at the bottom of the stack and is
+                 used as a marker. DEFAULT replaces it everywhere else.
+                 Popping when in INITIAL state causes an empty stack exception,
+                 so don't */
+              if (YYSTATE != INITIAL)
+                POP_STATE();
+              return '}';
+            }
+\{          { PUSH_STATE(DEFAULT); return '{'; }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\" {
-                PUSH_STATE(STRING); return '"';
-              }
+\"          { PUSH_STATE(STRING); return '"'; }
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
 <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
@@ -159,7 +159,7 @@ or          { return OR_KW; }
                 yylval->e = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
-<STRING>\$\{  { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <STRING>\"    { POP_STATE(); return '"'; }
 <STRING>\$|\\|\$\\ {
                 /* This can only occur when we reach EOF, otherwise the above
@@ -169,7 +169,7 @@ or          { return OR_KW; }
                 return STR;
               }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
+\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
                    yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
@@ -187,14 +187,13 @@ or          { return OR_KW; }
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
-<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; }
+<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
                    yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 
-<INITIAL,INSIDE_DOLLAR_CURLY>{
 
 {PATH}      { if (yytext[yyleng-1] == '/')
                   throw ParseError("path '%s' has a trailing slash", yytext);
@@ -219,7 +218,5 @@ or          { return OR_KW; }
               return (unsigned char) yytext[0];
             }
 
-}
-
 %%
 

From 0c61515be1a247fd42c2290b9ac0f36b2f7b9b14 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 27 Oct 2018 13:07:26 +0200
Subject: [PATCH 1263/2196] Fix signedness warning

---
 src/nix/doctor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index b608b9d59ef..1ce3efcb041 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -91,7 +91,7 @@ struct CmdDoctor : StoreCommand
 
     void checkStoreProtocol(unsigned int storeProto)
     {
-        auto clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
+        unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
             ? SERVE_PROTOCOL_VERSION
             : PROTOCOL_VERSION;
 

From 18b4c53f71dfc626f5f5ffa0282afd1b9faad6a4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 27 Oct 2018 15:40:09 +0200
Subject: [PATCH 1264/2196] Restore old (dis)allowedRequisites behaviour for
 self-references

stdenv relies on this. So ignore self-references (but only in legacy non-structured attributes mode).
---
 src/libstore/build.cc | 63 +++++++++++++++++++++++--------------------
 tests/check-refs.sh   |  2 ++
 tests/check-reqs.nix  |  2 +-
 tests/check-reqs.sh   |  2 ++
 4 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf4218a261f..676ad5856b1 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3310,6 +3310,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
         struct Checks
         {
+            bool ignoreSelfRefs = false;
             std::experimental::optional<uint64_t> maxSize, maxClosureSize;
             std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
         };
@@ -3345,35 +3346,6 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
             return std::make_pair(pathsDone, closureSize);
         };
 
-        auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
-        {
-            if (!value) return;
-
-            PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
-
-            PathSet used = recursive ? getClosure(info.path).first : info.references;
-
-            PathSet badPaths;
-
-            for (auto & i : used)
-                if (allowed) {
-                    if (spec.find(i) == spec.end())
-                        badPaths.insert(i);
-                } else {
-                    if (spec.find(i) != spec.end())
-                        badPaths.insert(i);
-                }
-
-            if (!badPaths.empty()) {
-                string badPathsStr;
-                for (auto & i : badPaths) {
-                    badPathsStr += "\n  ";
-                    badPathsStr += i;
-                }
-                throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
-            }
-        };
-
         auto applyChecks = [&](const Checks & checks)
         {
             if (checks.maxSize && info.narSize > *checks.maxSize)
@@ -3387,6 +3359,38 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                         info.path, closureSize, *checks.maxClosureSize);
             }
 
+            auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
+            {
+                if (!value) return;
+
+                PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
+
+                PathSet used = recursive ? getClosure(info.path).first : info.references;
+
+                if (recursive && checks.ignoreSelfRefs)
+                    used.erase(info.path);
+
+                PathSet badPaths;
+
+                for (auto & i : used)
+                    if (allowed) {
+                        if (!spec.count(i))
+                            badPaths.insert(i);
+                    } else {
+                        if (spec.count(i))
+                            badPaths.insert(i);
+                    }
+
+                if (!badPaths.empty()) {
+                    string badPathsStr;
+                    for (auto & i : badPaths) {
+                        badPathsStr += "\n  ";
+                        badPathsStr += i;
+                    }
+                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
+                }
+            };
+
             checkRefs(checks.allowedReferences, true, false);
             checkRefs(checks.allowedRequisites, true, true);
             checkRefs(checks.disallowedReferences, false, false);
@@ -3435,6 +3439,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
         } else {
             // legacy non-structured-attributes case
             Checks checks;
+            checks.ignoreSelfRefs = true;
             checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
             checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
             checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
diff --git a/tests/check-refs.sh b/tests/check-refs.sh
index 34ee22cfc8f..16bbabc4098 100644
--- a/tests/check-refs.sh
+++ b/tests/check-refs.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+clearStore
+
 RESULT=$TEST_ROOT/result
 
 dep=$(nix-build -o $RESULT check-refs.nix -A dep)
diff --git a/tests/check-reqs.nix b/tests/check-reqs.nix
index 47b5b3d9c72..41436cb48e0 100644
--- a/tests/check-reqs.nix
+++ b/tests/check-reqs.nix
@@ -33,7 +33,7 @@ rec {
   };
 
   # When specifying all the requisites, the build succeeds.
-  test1 = makeTest 1 [ "out" dep1 dep2 deps ];
+  test1 = makeTest 1 [ dep1 dep2 deps ];
 
   # But missing anything it fails.
   test2 = makeTest 2 [ dep2 deps ];
diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh
index 77689215def..e9f65fc2a6d 100644
--- a/tests/check-reqs.sh
+++ b/tests/check-reqs.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+clearStore
+
 RESULT=$TEST_ROOT/result
 
 nix-build -o $RESULT check-reqs.nix -A test1

From 8e6bf492971347e18c1b5800e9e8fa5191a0839b Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Mon, 29 Oct 2018 12:09:22 +0000
Subject: [PATCH 1265/2196] nix doctor: return nonzero exitcode if a check
 fails

This makes it easier to use this when testing the installer or when
running the checks with other automated tooling.
---
 src/nix/doctor.cc | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 1ce3efcb041..7b544461947 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -18,6 +18,8 @@ std::string formatProtocol(unsigned int proto)
 
 struct CmdDoctor : StoreCommand
 {
+    bool success = true;
+
     std::string name() override
     {
         return "doctor";
@@ -36,13 +38,16 @@ struct CmdDoctor : StoreCommand
         auto type = getStoreType();
 
         if (type < tOther) {
-            checkNixInPath();
-            checkProfileRoots(store);
+            success &= checkNixInPath();
+            success &= checkProfileRoots(store);
         }
-        checkStoreProtocol(store->getProtocol());
+        success &= checkStoreProtocol(store->getProtocol());
+
+        if (!success)
+            throw Exit(2);
     }
 
-    void checkNixInPath()
+    bool checkNixInPath()
     {
         PathSet dirs;
 
@@ -56,10 +61,13 @@ struct CmdDoctor : StoreCommand
             for (auto & dir : dirs)
                 std::cout << "  " << dir << std::endl;
             std::cout << std::endl;
+            return false;
         }
+
+        return true;
     }
 
-    void checkProfileRoots(ref<Store> store)
+    bool checkProfileRoots(ref<Store> store)
     {
         PathSet dirs;
 
@@ -86,10 +94,13 @@ struct CmdDoctor : StoreCommand
             for (auto & dir : dirs)
                 std::cout << "  " << dir << std::endl;
             std::cout << std::endl;
+            return false;
         }
+
+        return true;
     }
 
-    void checkStoreProtocol(unsigned int storeProto)
+    bool checkStoreProtocol(unsigned int storeProto)
     {
         unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto)
             ? SERVE_PROTOCOL_VERSION
@@ -103,7 +114,10 @@ struct CmdDoctor : StoreCommand
             std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl;
             std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl;
             std::cout << std::endl;
+            return false;
         }
+
+        return true;
     }
 };
 

From 3d974d31facefe8eaf59af56b0187e6a63fdd0cc Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 29 Oct 2018 08:44:58 -0500
Subject: [PATCH 1266/2196] editline: wip

---
 Makefile.config.in                    |    1 +
 configure.ac                          |    2 +
 doc/manual/introduction/about-nix.xml |    6 -
 editline.nix                          |   24 +
 release-common.nix                    |    4 +-
 src/linenoise/ConvertUTF.cpp          |  542 ----
 src/linenoise/ConvertUTF.h            |  162 --
 src/linenoise/LICENSE                 |   66 -
 src/linenoise/linenoise.cpp           | 3450 -------------------------
 src/linenoise/linenoise.h             |   73 -
 src/linenoise/wcwidth.cpp             |  315 ---
 src/nix/local.mk                      |    3 +-
 src/nix/repl.cc                       |   89 +-
 13 files changed, 89 insertions(+), 4648 deletions(-)
 create mode 100644 editline.nix
 delete mode 100644 src/linenoise/ConvertUTF.cpp
 delete mode 100755 src/linenoise/ConvertUTF.h
 delete mode 100644 src/linenoise/LICENSE
 delete mode 100644 src/linenoise/linenoise.cpp
 delete mode 100644 src/linenoise/linenoise.h
 delete mode 100644 src/linenoise/wcwidth.cpp

diff --git a/Makefile.config.in b/Makefile.config.in
index 08edcb863ef..fad077d869b 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -15,6 +15,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
 brotli = @brotli@
diff --git a/configure.ac b/configure.ac
index 9c85182efbf..85fec77a29c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -164,6 +164,8 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
+# Look for editline, a required dependency.
+PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml
index e8c0a29753a..c21ed34ddc7 100644
--- a/doc/manual/introduction/about-nix.xml
+++ b/doc/manual/introduction/about-nix.xml
@@ -262,12 +262,6 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU
 LGPLv2.1 or (at your option) any later version</link>.</para>
 
-<para>Nix uses the <link
-xlink:href="https://github.com/arangodb/linenoise-ng">linenoise-ng
-library</link>, which has the following license:</para>
-
-<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting>
-
 </simplesect>
 
 
diff --git a/editline.nix b/editline.nix
new file mode 100644
index 00000000000..d9b5a2b64d6
--- /dev/null
+++ b/editline.nix
@@ -0,0 +1,24 @@
+{ stdenv, fetchFromGitHub, autoreconfHook }:
+
+stdenv.mkDerivation rec {
+  name = "editline-${version}";
+  version = "1.15.3";
+  src = fetchFromGitHub {
+    owner = "troglobit";
+    repo = "editline";
+    rev = version;
+    sha256 = "0dm5fgq0acpprr938idwml64nclg9l6c6avirsd8r6f40qicbgma";
+  };
+
+  nativeBuildInputs = [ autoreconfHook ];
+
+  dontDisableStatic = true;
+
+  meta = with stdenv.lib; {
+    homepage = http://troglobit.com/editline.html;
+    description = "A readline() replacement for UNIX without termcap (ncurses)";
+    license = licenses.bsdOriginal;
+    maintainers = with maintainers; [ dtzWill ];
+    platforms = platforms.all;
+  };
+}
diff --git a/release-common.nix b/release-common.nix
index ace2a4f9b91..690860abfdc 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -29,6 +29,8 @@ rec {
     '';
   });
 
+  editline = pkgs.editline or (pkgs.callPackage ./editline.nix {});
+
   configureFlags =
     [
       "--enable-gc"
@@ -49,7 +51,7 @@ rec {
 
   buildDeps =
     [ curl
-      bzip2 xz brotli
+      bzip2 xz brotli editline
       openssl pkgconfig sqlite boehmgc
       boost
 
diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp
deleted file mode 100644
index f7e5915d5e8..00000000000
--- a/src/linenoise/ConvertUTF.cpp
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-    Sept 2001: fixed const & error conditions per
-        mods suggested by S. Parent & A. Lillich.
-    June 2002: Tim Dodd added detection and handling of incomplete
-        source sequences, enhanced error detection, added casts
-        to eliminate compiler warnings.
-    July 2003: slight mods to back out aggressive FFFE detection.
-    Jan 2004: updated switches in from-UTF8 conversions.
-    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
-
-    See the header file "ConvertUTF.h" for complete documentation.
-
------------------------------------------------------------------------- */
-
-#include "ConvertUTF.h"
-#ifdef CVTUTF_DEBUG
-#include <stdio.h>
-#endif
-
-namespace linenoise_ng {
-
-static const int halfShift  = 10; /* used for shifting by 10 bits */
-
-static const UTF32 halfBase = 0x0010000UL;
-static const UTF32 halfMask = 0x3FFUL;
-
-#define UNI_SUR_HIGH_START  (UTF32)0xD800
-#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
-#define UNI_SUR_LOW_START   (UTF32)0xDC00
-#define UNI_SUR_LOW_END     (UTF32)0xDFFF
-#define false      0
-#define true        1
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF16 (
-        const UTF32** sourceStart, const UTF32* sourceEnd, 
-        char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    char16_t* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        if (target >= targetEnd) {
-            result = targetExhausted; break;
-        }
-        ch = *source++;
-        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = (UTF16)ch; /* normal case */
-            }
-        } else if (ch > UNI_MAX_LEGAL_UTF32) {
-            if (flags == strictConversion) {
-                result = sourceIllegal;
-            } else {
-                *target++ = UNI_REPLACEMENT_CHAR;
-            }
-        } else {
-            /* target is a character in range 0xFFFF - 0x10FFFF. */
-            if (target + 1 >= targetEnd) {
-                --source; /* Back up source pointer! */
-                result = targetExhausted; break;
-            }
-            ch -= halfBase;
-            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF32 (
-        const UTF16** sourceStart, const UTF16* sourceEnd, 
-        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF32* target = *targetStart;
-    UTF32 ch, ch2;
-    while (source < sourceEnd) {
-        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
-        ch = *source++;
-        /* If we have a surrogate pair, convert to UTF32 first. */
-        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-            /* If the 16 bits following the high surrogate are in the source buffer... */
-            if (source < sourceEnd) {
-                ch2 = *source;
-                /* If it's a low surrogate, convert to UTF32. */
-                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
-                    ++source;
-                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                }
-            } else { /* We don't have the 16 bits following the high surrogate. */
-                --source; /* return to the high surrogate */
-                result = sourceExhausted;
-                break;
-            }
-        } else if (flags == strictConversion) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        if (target >= targetEnd) {
-            source = oldSource; /* Back up source pointer! */
-            result = targetExhausted; break;
-        }
-        *target++ = ch;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-#ifdef CVTUTF_DEBUG
-if (result == sourceIllegal) {
-    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
-    fflush(stderr);
-}
-#endif
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Index into the table below with the first byte of a UTF-8 sequence to
- * get the number of trailing bytes that are supposed to follow it.
- * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
- * left as-is for anyone who may want to do such conversion, which was
- * allowed in earlier algorithms.
- */
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
-
-/*
- * Magic values subtracted from a buffer value during UTF8 conversion.
- * This table contains as many values as there might be trailing bytes
- * in a UTF-8 sequence.
- */
-static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
-                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
-
-/*
- * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
- * into the first byte, depending on how many bytes follow.  There are
- * as many entries in this table as there are UTF-8 sequence types.
- * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
- * for *legal* UTF-8 will be 4 or fewer bytes total.
- */
-static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
-
-/* --------------------------------------------------------------------- */
-
-/* The interface converts a whole buffer to avoid function-call overhead.
- * Constants have been gathered. Loops & conditionals have been removed as
- * much as possible for efficiency, in favor of drop-through switches.
- * (See "Note A" at the bottom of the file for equivalent code.)
- * If your compiler supports it, the "isLegalUTF8" call can be turned
- * into an inline function.
- */
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF8 (
-        const UTF16** sourceStart, const UTF16* sourceEnd, 
-        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        unsigned short bytesToWrite = 0;
-        const UTF32 byteMask = 0xBF;
-        const UTF32 byteMark = 0x80; 
-        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
-        ch = *source++;
-        /* If we have a surrogate pair, convert to UTF32 first. */
-        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-            /* If the 16 bits following the high surrogate are in the source buffer... */
-            if (source < sourceEnd) {
-                UTF32 ch2 = *source;
-                /* If it's a low surrogate, convert to UTF32. */
-                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
-                    ++source;
-                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-                    --source; /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                }
-            } else { /* We don't have the 16 bits following the high surrogate. */
-                --source; /* return to the high surrogate */
-                result = sourceExhausted;
-                break;
-            }
-        } else if (flags == strictConversion) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        /* Figure out how many bytes the result will require */
-        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
-        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-        } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
-        } else {                            bytesToWrite = 3;
-                                            ch = UNI_REPLACEMENT_CHAR;
-        }
-
-        target += bytesToWrite;
-        if (target > targetEnd) {
-            source = oldSource; /* Back up source pointer! */
-            target -= bytesToWrite; result = targetExhausted; break;
-        }
-        switch (bytesToWrite) { /* note: everything falls through. */
-            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
-        }
-        target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Utility routine to tell whether a sequence of bytes is legal UTF-8.
- * This must be called with the length pre-determined by the first byte.
- * If not calling this from ConvertUTF8to*, then the length can be set by:
- *  length = trailingBytesForUTF8[*source]+1;
- * and the sequence is illegal right away if there aren't that many bytes
- * available.
- * If presented with a length > 4, this returns false.  The Unicode
- * definition of UTF-8 goes up to 4-byte sequences.
- */
-
-static Boolean isLegalUTF8(const UTF8 *source, int length) {
-    UTF8 a;
-    const UTF8 *srcptr = source+length;
-    switch (length) {
-    default: return false;
-        /* Everything else falls through when "true"... */
-    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
-
-        switch (*source) {
-            /* no fall-through in this inner switch */
-            case 0xE0: if (a < 0xA0) return false; break;
-            case 0xED: if (a > 0x9F) return false; break;
-            case 0xF0: if (a < 0x90) return false; break;
-            case 0xF4: if (a > 0x8F) return false; break;
-            default:   if (a < 0x80) return false;
-        }
-
-    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
-    }
-    if (*source > 0xF4) return false;
-    return true;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Exported function to return whether a UTF-8 sequence is legal or not.
- * This is not used here; it's just exported.
- */
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
-    int length = trailingBytesForUTF8[*source]+1;
-    if (source+length > sourceEnd) {
-        return false;
-    }
-    return isLegalUTF8(source, length);
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF16 (
-        const UTF8** sourceStart, const UTF8* sourceEnd, 
-        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF16* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch = 0;
-        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-        if (source + extraBytesToRead >= sourceEnd) {
-            result = sourceExhausted; break;
-        }
-        /* Do this check whether lenient or strict */
-        if (! isLegalUTF8(source, extraBytesToRead+1)) {
-            result = sourceIllegal;
-            break;
-        }
-        /*
-         * The cases all fall through. See "Note A" below.
-         */
-        switch (extraBytesToRead) {
-            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 3: ch += *source++; ch <<= 6;
-            case 2: ch += *source++; ch <<= 6;
-            case 1: ch += *source++; ch <<= 6;
-            case 0: ch += *source++;
-        }
-        ch -= offsetsFromUTF8[extraBytesToRead];
-
-        if (target >= targetEnd) {
-            source -= (extraBytesToRead+1); /* Back up source pointer! */
-            result = targetExhausted; break;
-        }
-        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = (UTF16)ch; /* normal case */
-            }
-        } else if (ch > UNI_MAX_UTF16) {
-            if (flags == strictConversion) {
-                result = sourceIllegal;
-                source -= (extraBytesToRead+1); /* return to the start */
-                break; /* Bail out; shouldn't continue */
-            } else {
-                *target++ = UNI_REPLACEMENT_CHAR;
-            }
-        } else {
-            /* target is a character in range 0xFFFF - 0x10FFFF. */
-            if (target + 1 >= targetEnd) {
-                source -= (extraBytesToRead+1); /* Back up source pointer! */
-                result = targetExhausted; break;
-            }
-            ch -= halfBase;
-            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF8 (
-        const UTF32** sourceStart, const UTF32* sourceEnd, 
-        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch;
-        unsigned short bytesToWrite = 0;
-        const UTF32 byteMask = 0xBF;
-        const UTF32 byteMark = 0x80; 
-        ch = *source++;
-        if (flags == strictConversion ) {
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                --source; /* return to the illegal value itself */
-                result = sourceIllegal;
-                break;
-            }
-        }
-        /*
-         * Figure out how many bytes the result will require. Turn any
-         * illegally large UTF32 things (> Plane 17) into replacement chars.
-         */
-        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
-        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-        } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
-        } else {                            bytesToWrite = 3;
-                                            ch = UNI_REPLACEMENT_CHAR;
-                                            result = sourceIllegal;
-        }
-        
-        target += bytesToWrite;
-        if (target > targetEnd) {
-            --source; /* Back up source pointer! */
-            target -= bytesToWrite; result = targetExhausted; break;
-        }
-        switch (bytesToWrite) { /* note: everything falls through. */
-            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
-        }
-        target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF32 (
-        const UTF8** sourceStart, const UTF8* sourceEnd, 
-        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF32* target = *targetStart;
-    while (source < sourceEnd) {
-        UTF32 ch = 0;
-        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-        if (source + extraBytesToRead >= sourceEnd) {
-            result = sourceExhausted; break;
-        }
-        /* Do this check whether lenient or strict */
-        if (! isLegalUTF8(source, extraBytesToRead+1)) {
-            result = sourceIllegal;
-            break;
-        }
-        /*
-         * The cases all fall through. See "Note A" below.
-         */
-        switch (extraBytesToRead) {
-            case 5: ch += *source++; ch <<= 6;
-            case 4: ch += *source++; ch <<= 6;
-            case 3: ch += *source++; ch <<= 6;
-            case 2: ch += *source++; ch <<= 6;
-            case 1: ch += *source++; ch <<= 6;
-            case 0: ch += *source++;
-        }
-        ch -= offsetsFromUTF8[extraBytesToRead];
-
-        if (target >= targetEnd) {
-            source -= (extraBytesToRead+1); /* Back up the source pointer! */
-            result = targetExhausted; break;
-        }
-        if (ch <= UNI_MAX_LEGAL_UTF32) {
-            /*
-             * UTF-16 surrogate values are illegal in UTF-32, and anything
-             * over Plane 17 (> 0x10FFFF) is illegal.
-             */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-                if (flags == strictConversion) {
-                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-                    result = sourceIllegal;
-                    break;
-                } else {
-                    *target++ = UNI_REPLACEMENT_CHAR;
-                }
-            } else {
-                *target++ = ch;
-            }
-        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
-            result = sourceIllegal;
-            *target++ = UNI_REPLACEMENT_CHAR;
-        }
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-}
-
-/* ---------------------------------------------------------------------
-
-    Note A.
-    The fall-through switches in UTF-8 reading code save a
-    temp variable, some decrements & conditionals.  The switches
-    are equivalent to the following loop:
-        {
-            int tmpBytesToRead = extraBytesToRead+1;
-            do {
-                ch += *source++;
-                --tmpBytesToRead;
-                if (tmpBytesToRead) ch <<= 6;
-            } while (tmpBytesToRead > 0);
-        }
-    In UTF-8 writing code, the switches on "bytesToWrite" are
-    similarly unrolled loops.
-
-   --------------------------------------------------------------------- */
diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h
deleted file mode 100755
index 8a296235dcd..00000000000
--- a/src/linenoise/ConvertUTF.h
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
-
-    Several funtions are included here, forming a complete set of
-    conversions between the three formats.  UTF-7 is not included
-    here, but is handled in a separate source file.
-
-    Each of these routines takes pointers to input buffers and output
-    buffers.  The input buffers are const.
-
-    Each routine converts the text between *sourceStart and sourceEnd,
-    putting the result into the buffer between *targetStart and
-    targetEnd. Note: the end pointers are *after* the last item: e.g. 
-    *(sourceEnd - 1) is the last item.
-
-    The return result indicates whether the conversion was successful,
-    and if not, whether the problem was in the source or target buffers.
-    (Only the first encountered problem is indicated.)
-
-    After the conversion, *sourceStart and *targetStart are both
-    updated to point to the end of last text successfully converted in
-    the respective buffers.
-
-    Input parameters:
-        sourceStart - pointer to a pointer to the source buffer.
-                The contents of this are modified on return so that
-                it points at the next thing to be converted.
-        targetStart - similarly, pointer to pointer to the target buffer.
-        sourceEnd, targetEnd - respectively pointers to the ends of the
-                two buffers, for overflow checking only.
-
-    These conversion functions take a ConversionFlags argument. When this
-    flag is set to strict, both irregular sequences and isolated surrogates
-    will cause an error.  When the flag is set to lenient, both irregular
-    sequences and isolated surrogates are converted.
-
-    Whether the flag is strict or lenient, all illegal sequences will cause
-    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
-    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
-    must check for illegal sequences.
-
-    When the flag is set to lenient, characters over 0x10FFFF are converted
-    to the replacement character; otherwise (when the flag is set to strict)
-    they constitute an error.
-
-    Output parameters:
-        The value "sourceIllegal" is returned from some routines if the input
-        sequence is malformed.  When "sourceIllegal" is returned, the source
-        value will point to the illegal value that caused the problem. E.g.,
-        in UTF-8 when a sequence is malformed, it points to the start of the
-        malformed sequence.  
-
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-                 Fixes & updates, Sept 2001.
-
------------------------------------------------------------------------- */
-
-/* ---------------------------------------------------------------------
-    The following 4 definitions are compiler-specific.
-    The C standard does not guarantee that wchar_t has at least
-    16 bits, so wchar_t is no less portable than unsigned short!
-    All should be unsigned values to avoid sign extension during
-    bit mask & shift operations.
------------------------------------------------------------------------- */
-
-#if 0
-typedef unsigned long  UTF32; /* at least 32 bits */
-typedef unsigned short UTF16; /* at least 16 bits */
-typedef unsigned char  UTF8;  /* typically 8 bits */
-#endif
-
-#include <stdint.h>
-#include <string>
-
-namespace linenoise_ng {
-
-typedef uint32_t         UTF32;
-typedef uint16_t         UTF16;
-typedef uint8_t          UTF8;
-typedef unsigned char    Boolean; /* 0 or 1 */
-
-/* Some fundamental constants */
-#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
-#define UNI_MAX_BMP (UTF32)0x0000FFFF
-#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
-#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
-#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
-
-typedef enum {
-        conversionOK,           /* conversion successful */
-        sourceExhausted,        /* partial character in source, but hit end */
-        targetExhausted,        /* insuff. room in target for conversion */
-        sourceIllegal           /* source sequence is illegal/malformed */
-} ConversionResult;
-
-typedef enum {
-        strictConversion = 0,
-        lenientConversion
-} ConversionFlags;
-
-// /* This is for C++ and does no harm in C */
-// #ifdef __cplusplus
-// extern "C" {
-// #endif
-
-ConversionResult ConvertUTF8toUTF16 (
-                const UTF8** sourceStart, const UTF8* sourceEnd, 
-                UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF16toUTF8 (
-                const UTF16** sourceStart, const UTF16* sourceEnd, 
-                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-                
-ConversionResult ConvertUTF8toUTF32 (
-                const UTF8** sourceStart, const UTF8* sourceEnd, 
-                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF8 (
-                const UTF32** sourceStart, const UTF32* sourceEnd, 
-                UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-                
-ConversionResult ConvertUTF16toUTF32 (
-                const UTF16** sourceStart, const UTF16* sourceEnd, 
-                UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF16 (
-                const UTF32** sourceStart, const UTF32* sourceEnd, 
-                char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags);
-
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
-
-// #ifdef __cplusplus
-// }
-// #endif
-
-}
-
-/* --------------------------------------------------------------------- */
diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE
deleted file mode 100644
index b7c58c44586..00000000000
--- a/src/linenoise/LICENSE
+++ /dev/null
@@ -1,66 +0,0 @@
-linenoise.cpp
-=============
-
-Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
-Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-  * Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-  * Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-  * Neither the name of Redis nor the names of its contributors may be used
-    to endorse or promote products derived from this software without
-    specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-
-wcwidth.cpp
-===========
-
-Markus Kuhn -- 2007-05-26 (Unicode 5.0)
-
-Permission to use, copy, modify, and distribute this software
-for any purpose and without fee is hereby granted. The author
-disclaims all warranties with regard to this software.
-
-
-
-ConvertUTF.cpp
-==============
-
-Copyright 2001-2004 Unicode, Inc.
- 
-Disclaimer
- 
-This source code is provided as is by Unicode, Inc. No claims are
-made as to fitness for any particular purpose. No warranties of any
-kind are expressed or implied. The recipient agrees to determine
-applicability of information provided. If this file has been
-purchased on magnetic or optical media from Unicode, Inc., the
-sole remedy for any claim will be exchange of defective media
-within 90 days of receipt.
- 
-Limitations on Rights to Redistribute This Code
- 
-Unicode, Inc. hereby grants the right to freely use the information
-supplied in this file in the creation of products supporting the
-Unicode Standard, and to make copies of this file in any form
-for internal or external distribution as long as this notice
-remains attached.
diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp
deleted file mode 100644
index c57505d2fa9..00000000000
--- a/src/linenoise/linenoise.cpp
+++ /dev/null
@@ -1,3450 +0,0 @@
-/* linenoise.c -- guerrilla line editing library against the idea that a
- * line editing lib needs to be 20,000 lines of C code.
- *
- * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * line editing lib needs to be 20,000 lines of C code.
- *
- * You can find the latest source code at:
- *
- *   http://github.com/antirez/linenoise
- *
- * Does a number of crazy assumptions that happen to be true in 99.9999% of
- * the 2010 UNIX computers around.
- *
- * References:
- * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
- * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
- *
- * Todo list:
- * - Switch to gets() if $TERM is something we can't support.
- * - Filter bogus Ctrl+<char> combinations.
- * - Win32 support
- *
- * Bloat:
- * - Completion?
- * - History search like Ctrl+r in readline?
- *
- * List of escape sequences used by this program, we do everything just
- * with three sequences. In order to be so cheap we may have some
- * flickering effect with some slow terminal, but the lesser sequences
- * the more compatible.
- *
- * CHA (Cursor Horizontal Absolute)
- *    Sequence: ESC [ n G
- *    Effect: moves cursor to column n (1 based)
- *
- * EL (Erase Line)
- *    Sequence: ESC [ n K
- *    Effect: if n is 0 or missing, clear from cursor to end of line
- *    Effect: if n is 1, clear from beginning of line to cursor
- *    Effect: if n is 2, clear entire line
- *
- * CUF (Cursor Forward)
- *    Sequence: ESC [ n C
- *    Effect: moves cursor forward of n chars
- *
- * The following are used to clear the screen: ESC [ H ESC [ 2 J
- * This is actually composed of two sequences:
- *
- * cursorhome
- *    Sequence: ESC [ H
- *    Effect: moves the cursor to upper left corner
- *
- * ED2 (Clear entire screen)
- *    Sequence: ESC [ 2 J
- *    Effect: clear the whole screen
- *
- */
-
-#ifdef _WIN32
-
-#include <conio.h>
-#include <windows.h>
-#include <io.h>
-
-#if defined(_MSC_VER) && _MSC_VER < 1900
-#define snprintf _snprintf  // Microsoft headers use underscores in some names
-#endif
-
-#if !defined GNUC
-#define strcasecmp _stricmp
-#endif
-
-#define strdup _strdup
-#define isatty _isatty
-#define write _write
-#define STDIN_FILENO 0
-
-#else /* _WIN32 */
-
-#include <signal.h>
-#include <termios.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/ioctl.h>
-#include <cctype>
-#include <wctype.h>
-
-#endif /* _WIN32 */
-
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-
-#include "linenoise.h"
-#include "ConvertUTF.h"
-
-#include <string>
-#include <vector>
-#include <memory>
-
-using std::string;
-using std::vector;
-using std::unique_ptr;
-using namespace linenoise_ng;
-
-typedef unsigned char char8_t;
-
-static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
-                                        size_t& dstCount, const char* src) {
-  const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src);
-  const UTF8* sourceEnd = sourceStart + strlen(src);
-  UTF32* targetStart = reinterpret_cast<UTF32*>(dst);
-  UTF32* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF8toUTF32(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    dstCount = targetStart - reinterpret_cast<UTF32*>(dst);
-
-    if (dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-
-  return res;
-}
-
-static ConversionResult copyString8to32(char32_t* dst, size_t dstSize,
-                                        size_t& dstCount, const char8_t* src) {
-  return copyString8to32(dst, dstSize, dstCount,
-                         reinterpret_cast<const char*>(src));
-}
-
-static size_t strlen32(const char32_t* str) {
-  const char32_t* ptr = str;
-
-  while (*ptr) {
-    ++ptr;
-  }
-
-  return ptr - str;
-}
-
-static size_t strlen8(const char8_t* str) {
-  return strlen(reinterpret_cast<const char*>(str));
-}
-
-static char8_t* strdup8(const char* src) {
-  return reinterpret_cast<char8_t*>(strdup(src));
-}
-
-#ifdef _WIN32
-static const int FOREGROUND_WHITE =
-    FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
-static const int BACKGROUND_WHITE =
-    BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE;
-static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY;
-
-class WinAttributes {
- public:
-  WinAttributes() {
-    CONSOLE_SCREEN_BUFFER_INFO info;
-    GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
-    _defaultAttribute = info.wAttributes & INTENSITY;
-    _defaultColor = info.wAttributes & FOREGROUND_WHITE;
-    _defaultBackground = info.wAttributes & BACKGROUND_WHITE;
-
-    _consoleAttribute = _defaultAttribute;
-    _consoleColor = _defaultColor | _defaultBackground;
-  }
-
- public:
-  int _defaultAttribute;
-  int _defaultColor;
-  int _defaultBackground;
-
-  int _consoleAttribute;
-  int _consoleColor;
-};
-
-static WinAttributes WIN_ATTR;
-
-static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount,
-                             const char32_t* src, size_t srcSize) {
-  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
-  const UTF32* sourceEnd = sourceStart + srcSize;
-  char16_t* targetStart = reinterpret_cast<char16_t*>(dst);
-  char16_t* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF32toUTF16(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    *dstCount = targetStart - reinterpret_cast<char16_t*>(dst);
-
-    if (*dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-}
-#endif
-
-static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount,
-                            const char32_t* src, size_t srcSize) {
-  const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src);
-  const UTF32* sourceEnd = sourceStart + srcSize;
-  UTF8* targetStart = reinterpret_cast<UTF8*>(dst);
-  UTF8* targetEnd = targetStart + dstSize;
-
-  ConversionResult res = ConvertUTF32toUTF8(
-      &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion);
-
-  if (res == conversionOK) {
-    *dstCount = targetStart - reinterpret_cast<UTF8*>(dst);
-
-    if (*dstCount < dstSize) {
-      *targetStart = 0;
-    }
-  }
-}
-
-static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) {
-  size_t dstCount = 0;
-  copyString32to8(dst, dstLen, &dstCount, src, strlen32(src));
-}
-
-static void copyString32(char32_t* dst, const char32_t* src, size_t len) {
-  while (0 < len && *src) {
-    *dst++ = *src++;
-    --len;
-  }
-
-  *dst = 0;
-}
-
-static int strncmp32(const char32_t* left, const char32_t* right, size_t len) {
-  while (0 < len && *left) {
-    if (*left != *right) {
-      return *left - *right;
-    }
-
-    ++left;
-    ++right;
-    --len;
-  }
-
-  return 0;
-}
-
-#ifdef _WIN32
-#include <iostream>
-
-static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) {
-  size_t count16 = 0;
-
-  copyString32to16(text16, len32, &count16, text32, len32);
-  WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16,
-                static_cast<DWORD>(count16), nullptr, nullptr);
-
-  return count16;
-}
-
-static char32_t* HandleEsc(char32_t* p, char32_t* end) {
-  if (*p == '[') {
-    int code = 0;
-
-    for (++p; p < end; ++p) {
-      char32_t c = *p;
-
-      if ('0' <= c && c <= '9') {
-        code = code * 10 + (c - '0');
-      } else if (c == 'm' || c == ';') {
-        switch (code) {
-          case 0:
-            WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute;
-            WIN_ATTR._consoleColor =
-                WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground;
-            break;
-
-          case 1:  // BOLD
-          case 5:  // BLINK
-            WIN_ATTR._consoleAttribute =
-                (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY;
-            break;
-
-          case 30:
-            WIN_ATTR._consoleColor = BACKGROUND_WHITE;
-            break;
-
-          case 31:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_RED | WIN_ATTR._defaultBackground;
-            break;
-
-          case 32:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
-            break;
-
-          case 33:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground;
-            break;
-
-          case 34:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_BLUE | WIN_ATTR._defaultBackground;
-            break;
-
-          case 35:
-            WIN_ATTR._consoleColor =
-                FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground;
-            break;
-
-          case 36:
-            WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN |
-                                     WIN_ATTR._defaultBackground;
-            break;
-
-          case 37:
-            WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED |
-                                     FOREGROUND_BLUE |
-                                     WIN_ATTR._defaultBackground;
-            break;
-        }
-
-        code = 0;
-      }
-
-      if (*p == 'm') {
-        ++p;
-        break;
-      }
-    }
-  } else {
-    ++p;
-  }
-
-  auto handle = GetStdHandle(STD_OUTPUT_HANDLE);
-  SetConsoleTextAttribute(handle,
-                          WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor);
-
-  return p;
-}
-
-static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) {
-  char32_t* p = text32;
-  char32_t* q = p;
-  char32_t* e = text32 + len32;
-  size_t count16 = 0;
-
-  while (p < e) {
-    if (*p == 27) {
-      if (q < p) {
-        count16 += OutputWin(text16, q, p - q);
-      }
-
-      q = p = HandleEsc(p + 1, e);
-    } else {
-      ++p;
-    }
-  }
-
-  if (q < p) {
-    count16 += OutputWin(text16, q, p - q);
-  }
-
-  return count16;
-}
-#endif
-
-static int write32(int fd, char32_t* text32, int len32) {
-#ifdef _WIN32
-  if (isatty(fd)) {
-    size_t len16 = 2 * len32 + 1;
-    unique_ptr<char16_t[]> text16(new char16_t[len16]);
-    size_t count16 = WinWrite32(text16.get(), text32, len32);
-
-    return static_cast<int>(count16);
-  } else {
-    size_t len8 = 4 * len32 + 1;
-    unique_ptr<char[]> text8(new char[len8]);
-    size_t count8 = 0;
-
-    copyString32to8(text8.get(), len8, &count8, text32, len32);
-
-    return write(fd, text8.get(), static_cast<unsigned int>(count8));
-  }
-#else
-  size_t len8 = 4 * len32 + 1;
-  unique_ptr<char[]> text8(new char[len8]);
-  size_t count8 = 0;
-
-  copyString32to8(text8.get(), len8, &count8, text32, len32);
-
-  return write(fd, text8.get(), count8);
-#endif
-}
-
-class Utf32String {
- public:
-  Utf32String() : _length(0), _data(nullptr) { 
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[1](); 
-  }
-
-  explicit Utf32String(const char* src) : _length(0), _data(nullptr) {
-    size_t len = strlen(src);
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    copyString8to32(_data, len + 1, _length, src);
-  }
-
-  explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) {
-    size_t len = strlen(reinterpret_cast<const char*>(src));
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    copyString8to32(_data, len + 1, _length, src);
-  }
-
-  explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) {
-    for (_length = 0; src[_length] != 0; ++_length) {
-    }
-
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[_length + 1]();
-    memcpy(_data, src, _length * sizeof(char32_t));
-  }
-
-  explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) {
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len + 1]();
-    memcpy(_data, src, len * sizeof(char32_t));
-  }
-
-  explicit Utf32String(int len) : _length(0), _data(nullptr) { 
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[len](); 
-  }
-
-  explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) {
-    // note: parens intentional, _data must be properly initialized
-    _data = new char32_t[_length + 1]();
-    memcpy(_data, that._data, sizeof(char32_t) * _length);
-  }
-
-  Utf32String& operator=(const Utf32String& that) {
-    if (this != &that) {
-      delete[] _data;
-      _data = new char32_t[that._length]();
-      _length = that._length;
-      memcpy(_data, that._data, sizeof(char32_t) * _length);
-    }
-
-    return *this;
-  }
-
-  ~Utf32String() { delete[] _data; }
-
- public:
-  char32_t* get() const { return _data; }
-
-  size_t length() const { return _length; }
-
-  size_t chars() const { return _length; }
-
-  void initFromBuffer() {
-    for (_length = 0; _data[_length] != 0; ++_length) {
-    }
-  }
-
-  const char32_t& operator[](size_t pos) const { return _data[pos]; }
-
-  char32_t& operator[](size_t pos) { return _data[pos]; }
-
- private:
-  size_t _length;
-  char32_t* _data;
-};
-
-class Utf8String {
-  Utf8String(const Utf8String&) = delete;
-  Utf8String& operator=(const Utf8String&) = delete;
-
- public:
-  explicit Utf8String(const Utf32String& src) {
-    size_t len = src.length() * 4 + 1;
-    _data = new char[len];
-    copyString32to8(_data, len, src.get());
-  }
-
-  ~Utf8String() { delete[] _data; }
-
- public:
-  char* get() const { return _data; }
-
- private:
-  char* _data;
-};
-
-struct linenoiseCompletions {
-  vector<Utf32String> completionStrings;
-};
-
-#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
-#define LINENOISE_MAX_LINE 4096
-
-// make control-characters more readable
-#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40)
-
-/**
- * Recompute widths of all characters in a char32_t buffer
- * @param text          input buffer of Unicode characters
- * @param widths        output buffer of character widths
- * @param charCount     number of characters in buffer
- */
-namespace linenoise_ng {
-int mk_wcwidth(char32_t ucs);
-}
-
-static void recomputeCharacterWidths(const char32_t* text, char* widths,
-                                     int charCount) {
-  for (int i = 0; i < charCount; ++i) {
-    widths[i] = mk_wcwidth(text[i]);
-  }
-}
-
-/**
- * Calculate a new screen position given a starting position, screen width and
- * character count
- * @param x             initial x position (zero-based)
- * @param y             initial y position (zero-based)
- * @param screenColumns screen column count
- * @param charCount     character positions to advance
- * @param xOut          returned x position (zero-based)
- * @param yOut          returned y position (zero-based)
- */
-static void calculateScreenPosition(int x, int y, int screenColumns,
-                                    int charCount, int& xOut, int& yOut) {
-  xOut = x;
-  yOut = y;
-  int charsRemaining = charCount;
-  while (charsRemaining > 0) {
-    int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining
-                                                            : screenColumns - x;
-    xOut = x + charsThisRow;
-    yOut = y;
-    charsRemaining -= charsThisRow;
-    x = 0;
-    ++y;
-  }
-  if (xOut == screenColumns) {  // we have to special-case line wrap
-    xOut = 0;
-    ++yOut;
-  }
-}
-
-/**
- * Calculate a column width using mk_wcswidth()
- * @param buf32  text to calculate
- * @param len    length of text to calculate
- */
-namespace linenoise_ng {
-int mk_wcswidth(const char32_t* pwcs, size_t n);
-}
-
-static int calculateColumnPosition(char32_t* buf32, int len) {
-  int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len);
-  if (width == -1)
-    return len;
-  else
-    return width;
-}
-
-static bool isControlChar(char32_t testChar) {
-  return (testChar < ' ') ||                      // C0 controls
-         (testChar >= 0x7F && testChar <= 0x9F);  // DEL and C1 controls
-}
-
-struct PromptBase {            // a convenience struct for grouping prompt info
-  Utf32String promptText;      // our copy of the prompt text, edited
-  char* promptCharWidths;      // character widths from mk_wcwidth()
-  int promptChars;             // chars in promptText
-  int promptBytes;             // bytes in promptText
-  int promptExtraLines;        // extra lines (beyond 1) occupied by prompt
-  int promptIndentation;       // column offset to end of prompt
-  int promptLastLinePosition;  // index into promptText where last line begins
-  int promptPreviousInputLen;  // promptChars of previous input line, for
-                               // clearing
-  int promptCursorRowOffset;   // where the cursor is relative to the start of
-                               // the prompt
-  int promptScreenColumns;     // width of screen in columns
-  int promptPreviousLen;       // help erasing
-  int promptErrorCode;         // error code (invalid UTF-8) or zero
-
-  PromptBase() : promptPreviousInputLen(0) {}
-
-  bool write() {
-    if (write32(1, promptText.get(), promptBytes) == -1) return false;
-
-    return true;
-  }
-};
-
-struct PromptInfo : public PromptBase {
-  PromptInfo(const char* textPtr, int columns) {
-    promptExtraLines = 0;
-    promptLastLinePosition = 0;
-    promptPreviousLen = 0;
-    promptScreenColumns = columns;
-    Utf32String tempUnicode(textPtr);
-
-    // strip control characters from the prompt -- we do allow newline
-    char32_t* pIn = tempUnicode.get();
-    char32_t* pOut = pIn;
-
-    int len = 0;
-    int x = 0;
-
-    bool const strip = (isatty(1) == 0);
-
-    while (*pIn) {
-      char32_t c = *pIn;
-      if ('\n' == c || !isControlChar(c)) {
-        *pOut = c;
-        ++pOut;
-        ++pIn;
-        ++len;
-        if ('\n' == c || ++x >= promptScreenColumns) {
-          x = 0;
-          ++promptExtraLines;
-          promptLastLinePosition = len;
-        }
-      } else if (c == '\x1b') {
-        if (strip) {
-          // jump over control chars
-          ++pIn;
-          if (*pIn == '[') {
-            ++pIn;
-            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
-              ++pIn;
-            }
-            if (*pIn == 'm') {
-              ++pIn;
-            }
-          }
-        } else {
-          // copy control chars
-          *pOut = *pIn;
-          ++pOut;
-          ++pIn;
-          if (*pIn == '[') {
-            *pOut = *pIn;
-            ++pOut;
-            ++pIn;
-            while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) {
-              *pOut = *pIn;
-              ++pOut;
-              ++pIn;
-            }
-            if (*pIn == 'm') {
-              *pOut = *pIn;
-              ++pOut;
-              ++pIn;
-            }
-          }
-        }
-      } else {
-        ++pIn;
-      }
-    }
-    *pOut = 0;
-    promptChars = len;
-    promptBytes = static_cast<int>(pOut - tempUnicode.get());
-    promptText = tempUnicode;
-
-    promptIndentation = len - promptLastLinePosition;
-    promptCursorRowOffset = promptExtraLines;
-  }
-};
-
-// Used with DynamicPrompt (history search)
-//
-static const Utf32String forwardSearchBasePrompt("(i-search)`");
-static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`");
-static const Utf32String endSearchBasePrompt("': ");
-static Utf32String
-    previousSearchText;  // remembered across invocations of linenoise()
-
-// changing prompt for "(reverse-i-search)`text':" etc.
-//
-struct DynamicPrompt : public PromptBase {
-  Utf32String searchText;  // text we are searching for
-  char* searchCharWidths;  // character widths from mk_wcwidth()
-  int searchTextLen;       // chars in searchText
-  int direction;           // current search direction, 1=forward, -1=reverse
-
-  DynamicPrompt(PromptBase& pi, int initialDirection)
-      : searchTextLen(0), direction(initialDirection) {
-    promptScreenColumns = pi.promptScreenColumns;
-    promptCursorRowOffset = 0;
-    Utf32String emptyString(1);
-    searchText = emptyString;
-    const Utf32String* basePrompt =
-        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
-    size_t promptStartLength = basePrompt->length();
-    promptChars =
-        static_cast<int>(promptStartLength + endSearchBasePrompt.length());
-    promptBytes = promptChars;
-    promptLastLinePosition = promptChars;  // TODO fix this, we are asssuming
-                                           // that the history prompt won't wrap
-                                           // (!)
-    promptPreviousLen = promptChars;
-    Utf32String tempUnicode(promptChars + 1);
-    memcpy(tempUnicode.get(), basePrompt->get(),
-           sizeof(char32_t) * promptStartLength);
-    memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(),
-           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
-    tempUnicode.initFromBuffer();
-    promptText = tempUnicode;
-    calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars,
-                            promptIndentation, promptExtraLines);
-  }
-
-  void updateSearchPrompt(void) {
-    const Utf32String* basePrompt =
-        (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt;
-    size_t promptStartLength = basePrompt->length();
-    promptChars = static_cast<int>(promptStartLength + searchTextLen +
-                                   endSearchBasePrompt.length());
-    promptBytes = promptChars;
-    Utf32String tempUnicode(promptChars + 1);
-    memcpy(tempUnicode.get(), basePrompt->get(),
-           sizeof(char32_t) * promptStartLength);
-    memcpy(&tempUnicode[promptStartLength], searchText.get(),
-           sizeof(char32_t) * searchTextLen);
-    size_t endIndex = promptStartLength + searchTextLen;
-    memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(),
-           sizeof(char32_t) * (endSearchBasePrompt.length() + 1));
-    tempUnicode.initFromBuffer();
-    promptText = tempUnicode;
-  }
-
-  void updateSearchText(const char32_t* textPtr) {
-    Utf32String tempUnicode(textPtr);
-    searchTextLen = static_cast<int>(tempUnicode.chars());
-    searchText = tempUnicode;
-    updateSearchPrompt();
-  }
-};
-
-class KillRing {
-  static const int capacity = 10;
-  int size;
-  int index;
-  char indexToSlot[10];
-  vector<Utf32String> theRing;
-
- public:
-  enum action { actionOther, actionKill, actionYank };
-  action lastAction;
-  size_t lastYankSize;
-
-  KillRing() : size(0), index(0), lastAction(actionOther) {
-    theRing.reserve(capacity);
-  }
-
-  void kill(const char32_t* text, int textLen, bool forward) {
-    if (textLen == 0) {
-      return;
-    }
-    Utf32String killedText(text, textLen);
-    if (lastAction == actionKill && size > 0) {
-      int slot = indexToSlot[0];
-      int currentLen = static_cast<int>(theRing[slot].length());
-      int resultLen = currentLen + textLen;
-      Utf32String temp(resultLen + 1);
-      if (forward) {
-        memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t));
-        memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t));
-      } else {
-        memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t));
-        memcpy(&temp[textLen], theRing[slot].get(),
-               currentLen * sizeof(char32_t));
-      }
-      temp[resultLen] = 0;
-      temp.initFromBuffer();
-      theRing[slot] = temp;
-    } else {
-      if (size < capacity) {
-        if (size > 0) {
-          memmove(&indexToSlot[1], &indexToSlot[0], size);
-        }
-        indexToSlot[0] = size;
-        size++;
-        theRing.push_back(killedText);
-      } else {
-        int slot = indexToSlot[capacity - 1];
-        theRing[slot] = killedText;
-        memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1);
-        indexToSlot[0] = slot;
-      }
-      index = 0;
-    }
-  }
-
-  Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; }
-
-  Utf32String* yankPop() {
-    if (size == 0) {
-      return 0;
-    }
-    ++index;
-    if (index == size) {
-      index = 0;
-    }
-    return &theRing[indexToSlot[index]];
-  }
-};
-
-class InputBuffer {
-  char32_t* buf32;   // input buffer
-  char* charWidths;  // character widths from mk_wcwidth()
-  int buflen;        // buffer size in characters
-  int len;           // length of text in input buffer
-  int pos;           // character position in buffer ( 0 <= pos <= len )
-
-  void clearScreen(PromptBase& pi);
-  int incrementalHistorySearch(PromptBase& pi, int startChar);
-  int completeLine(PromptBase& pi);
-  void refreshLine(PromptBase& pi);
-
- public:
-  InputBuffer(char32_t* buffer, char* widthArray, int bufferLen)
-      : buf32(buffer),
-        charWidths(widthArray),
-        buflen(bufferLen - 1),
-        len(0),
-        pos(0) {
-    buf32[0] = 0;
-  }
-  void preloadBuffer(const char* preloadText) {
-    size_t ucharCount = 0;
-    copyString8to32(buf32, buflen + 1, ucharCount, preloadText);
-    recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount));
-    len = static_cast<int>(ucharCount);
-    pos = static_cast<int>(ucharCount);
-  }
-  int getInputLine(PromptBase& pi);
-  int length(void) const { return len; }
-};
-
-// Special codes for keyboard input:
-//
-// Between Windows and the various Linux "terminal" programs, there is some
-// pretty diverse behavior in the "scan codes" and escape sequences we are
-// presented with.  So ... we'll translate them all into our own pidgin
-// pseudocode, trying to stay out of the way of UTF-8 and international
-// characters.  Here's the general plan.
-//
-// "User input keystrokes" (key chords, whatever) will be encoded as a single
-// value.
-// The low 21 bits are reserved for Unicode characters.  Popular function-type
-// keys
-// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000,
-// currently
-// just arrow keys, Home, End and Delete.  Keypresses with Ctrl get ORed with
-// 0x20000000, with Alt get ORed with 0x40000000.  So, Ctrl+Alt+Home is encoded
-// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000.  To keep things
-// complicated,
-// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC
-// followed by
-// D is treated the same as Alt + D ... we'll just use Emacs terminology and
-// call
-// this "Meta".  So, we will encode both ESC followed by D and Alt held down
-// while D
-// is pressed the same, as Meta-D, encoded as 0x40000064.
-//
-// Here are the definitions of our component constants:
-//
-// Maximum unsigned 32-bit value    = 0xFFFFFFFF;   // For reference, max 32-bit
-// value
-// Highest allocated Unicode char   = 0x001FFFFF;   // For reference, max
-// Unicode value
-static const int META = 0x40000000;  // Meta key combination
-static const int CTRL = 0x20000000;  // Ctrl key combination
-// static const int SPECIAL_KEY = 0x10000000;   // Common bit for all special
-// keys
-static const int UP_ARROW_KEY = 0x10200000;  // Special keys
-static const int DOWN_ARROW_KEY = 0x10400000;
-static const int RIGHT_ARROW_KEY = 0x10600000;
-static const int LEFT_ARROW_KEY = 0x10800000;
-static const int HOME_KEY = 0x10A00000;
-static const int END_KEY = 0x10C00000;
-static const int DELETE_KEY = 0x10E00000;
-static const int PAGE_UP_KEY = 0x11000000;
-static const int PAGE_DOWN_KEY = 0x11200000;
-
-static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL};
-static linenoiseCompletionCallback* completionCallback = NULL;
-
-#ifdef _WIN32
-static HANDLE console_in, console_out;
-static DWORD oldMode;
-static WORD oldDisplayAttribute;
-#else
-static struct termios orig_termios; /* in order to restore at exit */
-#endif
-
-static KillRing killRing;
-
-static int rawmode = 0; /* for atexit() function to check if restore is needed*/
-static int atexit_registered = 0; /* register atexit just 1 time */
-static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
-static int historyLen = 0;
-static int historyIndex = 0;
-static char8_t** history = NULL;
-
-// used to emulate Windows command prompt on down-arrow after a recall
-// we use -2 as our "not set" value because we add 1 to the previous index on
-// down-arrow,
-// and zero is a valid index (so -1 is a valid "previous index")
-static int historyPreviousIndex = -2;
-static bool historyRecallMostRecent = false;
-
-static void linenoiseAtExit(void);
-
-static bool isUnsupportedTerm(void) {
-  char* term = getenv("TERM");
-  if (term == NULL) return false;
-  for (int j = 0; unsupported_term[j]; ++j)
-    if (!strcasecmp(term, unsupported_term[j])) {
-      return true;
-    }
-  return false;
-}
-
-static void beep() {
-  fprintf(stderr, "\x7");  // ctrl-G == bell/beep
-  fflush(stderr);
-}
-
-void linenoiseHistoryFree(void) {
-  if (history) {
-    for (int j = 0; j < historyLen; ++j) free(history[j]);
-    historyLen = 0;
-    free(history);
-    history = 0;
-  }
-}
-
-static int enableRawMode(void) {
-#ifdef _WIN32
-  if (!console_in) {
-    console_in = GetStdHandle(STD_INPUT_HANDLE);
-    console_out = GetStdHandle(STD_OUTPUT_HANDLE);
-
-    GetConsoleMode(console_in, &oldMode);
-    SetConsoleMode(console_in, oldMode &
-                                   ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT |
-                                     ENABLE_PROCESSED_INPUT));
-  }
-  return 0;
-#else
-  struct termios raw;
-
-  if (!isatty(STDIN_FILENO)) goto fatal;
-  if (!atexit_registered) {
-    atexit(linenoiseAtExit);
-    atexit_registered = 1;
-  }
-  if (tcgetattr(0, &orig_termios) == -1) goto fatal;
-
-  raw = orig_termios; /* modify the original mode */
-  /* input modes: no break, no CR to NL, no parity check, no strip char,
-   * no start/stop output control. */
-  raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
-  /* output modes - disable post processing */
-  // this is wrong, we don't want raw output, it turns newlines into straight
-  // linefeeds
-  // raw.c_oflag &= ~(OPOST);
-  /* control modes - set 8 bit chars */
-  raw.c_cflag |= (CS8);
-  /* local modes - echoing off, canonical off, no extended functions,
-   * no signal chars (^Z,^C) */
-  raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
-  /* control chars - set return condition: min number of bytes and timer.
-   * We want read to return every single byte, without timeout. */
-  raw.c_cc[VMIN] = 1;
-  raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
-
-  /* put terminal in raw mode after flushing */
-  if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal;
-  rawmode = 1;
-  return 0;
-
-fatal:
-  errno = ENOTTY;
-  return -1;
-#endif
-}
-
-static void disableRawMode(void) {
-#ifdef _WIN32
-  SetConsoleMode(console_in, oldMode);
-  console_in = 0;
-  console_out = 0;
-#else
-  if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0;
-#endif
-}
-
-// At exit we'll try to fix the terminal to the initial conditions
-static void linenoiseAtExit(void) { disableRawMode(); }
-
-static int getScreenColumns(void) {
-  int cols;
-#ifdef _WIN32
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
-  cols = inf.dwSize.X;
-#else
-  struct winsize ws;
-  cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col;
-#endif
-  // cols is 0 in certain circumstances like inside debugger, which creates
-  // further issues
-  return (cols > 0) ? cols : 80;
-}
-
-static int getScreenRows(void) {
-  int rows;
-#ifdef _WIN32
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf);
-  rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top;
-#else
-  struct winsize ws;
-  rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row;
-#endif
-  return (rows > 0) ? rows : 24;
-}
-
-static void setDisplayAttribute(bool enhancedDisplay, bool error) {
-#ifdef _WIN32
-  if (enhancedDisplay) {
-    CONSOLE_SCREEN_BUFFER_INFO inf;
-    GetConsoleScreenBufferInfo(console_out, &inf);
-    oldDisplayAttribute = inf.wAttributes;
-    BYTE oldLowByte = oldDisplayAttribute & 0xFF;
-    BYTE newLowByte;
-    switch (oldLowByte) {
-      case 0x07:
-        // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY;  // too dim
-        // newLowByte = FOREGROUND_BLUE;                         // even dimmer
-        newLowByte = FOREGROUND_BLUE |
-                     FOREGROUND_GREEN;  // most similar to xterm appearance
-        break;
-      case 0x70:
-        newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY;
-        break;
-      default:
-        newLowByte = oldLowByte ^ 0xFF;  // default to inverse video
-        break;
-    }
-    inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte;
-    SetConsoleTextAttribute(console_out, inf.wAttributes);
-  } else {
-    SetConsoleTextAttribute(console_out, oldDisplayAttribute);
-  }
-#else
-  if (enhancedDisplay) {
-    char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m");
-    if (write(1, p, 7) == -1)
-      return; /* bright blue (visible with both B&W bg) */
-  } else {
-    if (write(1, "\x1b[0m", 4) == -1) return; /* reset */
-  }
-#endif
-}
-
-/**
- * Display the dynamic incremental search prompt and the current user input
- * line.
- * @param pi   PromptBase struct holding information about the prompt and our
- * screen position
- * @param buf32  input buffer to be displayed
- * @param len  count of characters in the buffer
- * @param pos  current cursor position within the buffer (0 <= pos <= len)
- */
-static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) {
-  // calculate the position of the end of the prompt
-  int xEndOfPrompt, yEndOfPrompt;
-  calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars,
-                          xEndOfPrompt, yEndOfPrompt);
-  pi.promptIndentation = xEndOfPrompt;
-
-  // calculate the position of the end of the input line
-  int xEndOfInput, yEndOfInput;
-  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, len), xEndOfInput,
-                          yEndOfInput);
-
-  // calculate the desired position of the cursor
-  int xCursorPos, yCursorPos;
-  calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, pos), xCursorPos,
-                          yCursorPos);
-
-#ifdef _WIN32
-  // position at the start of the prompt, clear to end of previous input
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = 0;
-  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-  DWORD count;
-  FillConsoleOutputCharacterA(console_out, ' ',
-                              pi.promptPreviousLen + pi.promptPreviousInputLen,
-                              inf.dwCursorPosition, &count);
-  pi.promptPreviousLen = pi.promptIndentation;
-  pi.promptPreviousInputLen = len;
-
-  // display the prompt
-  if (!pi.write()) return;
-
-  // display the input line
-  if (write32(1, buf32, len) == -1) return;
-
-  // position the cursor
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-#else  // _WIN32
-  char seq[64];
-  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position at the start of the prompt, clear to end of screen
-  snprintf(seq, sizeof seq, "\x1b[1G\x1b[J");  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-
-  // display the prompt
-  if (!pi.write()) return;
-
-  // display the input line
-  if (write32(1, buf32, len) == -1) return;
-
-  // we have to generate our own newline on line wrap
-  if (xEndOfInput == 0 && yEndOfInput > 0)
-    if (write(1, "\n", 1) == -1) return;
-
-  // position the cursor
-  cursorRowMovement = yEndOfInput - yCursorPos;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position the cursor within the line
-  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-#endif
-
-  pi.promptCursorRowOffset =
-      pi.promptExtraLines + yCursorPos;  // remember row for next pass
-}
-
-/**
- * Refresh the user's input line: the prompt is already onscreen and is not
- * redrawn here
- * @param pi   PromptBase struct holding information about the prompt and our
- * screen position
- */
-void InputBuffer::refreshLine(PromptBase& pi) {
-  // check for a matching brace/bracket/paren, remember its position if found
-  int highlight = -1;
-  bool indicateError = false;
-  if (pos < len) {
-    /* this scans for a brace matching buf32[pos] to highlight */
-    unsigned char part1, part2;
-    int scanDirection = 0;
-    if (strchr("}])", buf32[pos])) {
-      scanDirection = -1; /* backwards */
-      if (buf32[pos] == '}') {
-        part1 = '}'; part2 = '{';
-      } else if (buf32[pos] == ']') {
-        part1 = ']'; part2 = '[';
-      } else {
-        part1 = ')'; part2 = '(';
-      }
-    }
-    else if (strchr("{[(", buf32[pos])) {
-      scanDirection = 1; /* forwards */
-      if (buf32[pos] == '{') {
-        //part1 = '{'; part2 = '}';
-        part1 = '}'; part2 = '{';
-      } else if (buf32[pos] == '[') {
-        //part1 = '['; part2 = ']';
-        part1 = ']'; part2 = '[';
-      } else {
-        //part1 = '('; part2 = ')';
-        part1 = ')'; part2 = '(';
-      }
-    }
-
-    if (scanDirection) {
-      int unmatched = scanDirection;
-      int unmatchedOther = 0;
-      for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) {
-        /* TODO: the right thing when inside a string */
-        if (strchr("}])", buf32[i])) {
-          if (buf32[i] == part1) {
-            --unmatched;
-          } else {
-            --unmatchedOther;
-          }
-        } else if (strchr("{[(", buf32[i])) {
-          if (buf32[i] == part2) {
-            ++unmatched;
-          } else {
-            ++unmatchedOther;
-          }
-        }
-/*
-        if (strchr("}])", buf32[i]))
-          --unmatched;
-        else if (strchr("{[(", buf32[i]))
-          ++unmatched;
-*/
-        if (unmatched == 0) {
-          highlight = i;
-          indicateError = (unmatchedOther != 0);
-          break;
-        }
-      }
-    }
-  }
-
-  // calculate the position of the end of the input line
-  int xEndOfInput, yEndOfInput;
-  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, len), xEndOfInput,
-                          yEndOfInput);
-
-  // calculate the desired position of the cursor
-  int xCursorPos, yCursorPos;
-  calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns,
-                          calculateColumnPosition(buf32, pos), xCursorPos,
-                          yCursorPos);
-
-#ifdef _WIN32
-  // position at the end of the prompt, clear to end of previous input
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = pi.promptIndentation;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-  DWORD count;
-  if (len < pi.promptPreviousInputLen)
-    FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen,
-                                inf.dwCursorPosition, &count);
-  pi.promptPreviousInputLen = len;
-
-  // display the input line
-  if (highlight == -1) {
-    if (write32(1, buf32, len) == -1) return;
-  } else {
-    if (write32(1, buf32, highlight) == -1) return;
-    setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */
-    if (write32(1, &buf32[highlight], 1) == -1) return;
-    setDisplayAttribute(false, indicateError);
-    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
-  }
-
-  // position the cursor
-  GetConsoleScreenBufferInfo(console_out, &inf);
-  inf.dwCursorPosition.X = xCursorPos;  // 0-based on Win32
-  inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos;
-  SetConsoleCursorPosition(console_out, inf.dwCursorPosition);
-#else  // _WIN32
-  char seq[64];
-  int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position at the end of the prompt, clear to end of screen
-  snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J",
-           pi.promptIndentation + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-
-  if (highlight == -1) {  // write unhighlighted text
-    if (write32(1, buf32, len) == -1) return;
-  } else {  // highlight the matching brace/bracket/parenthesis
-    if (write32(1, buf32, highlight) == -1) return;
-    setDisplayAttribute(true, indicateError);
-    if (write32(1, &buf32[highlight], 1) == -1) return;
-    setDisplayAttribute(false, indicateError);
-    if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return;
-  }
-
-  // we have to generate our own newline on line wrap
-  if (xEndOfInput == 0 && yEndOfInput > 0)
-    if (write(1, "\n", 1) == -1) return;
-
-  // position the cursor
-  cursorRowMovement = yEndOfInput - yCursorPos;
-  if (cursorRowMovement > 0) {  // move the cursor up as required
-    snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement);
-    if (write(1, seq, strlen(seq)) == -1) return;
-  }
-  // position the cursor within the line
-  snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1);  // 1-based on VT100
-  if (write(1, seq, strlen(seq)) == -1) return;
-#endif
-
-  pi.promptCursorRowOffset =
-      pi.promptExtraLines + yCursorPos;  // remember row for next pass
-}
-
-#ifndef _WIN32
-
-/**
- * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode
- * (char32_t) character it
- * encodes
- *
- * @return  char32_t Unicode character
- */
-static char32_t readUnicodeCharacter(void) {
-  static char8_t utf8String[5];
-  static size_t utf8Count = 0;
-  while (true) {
-    char8_t c;
-
-    /* Continue reading if interrupted by signal. */
-    ssize_t nread;
-    do {
-      nread = read(0, &c, 1);
-    } while ((nread == -1) && (errno == EINTR));
-
-    if (nread <= 0) return 0;
-    if (c <= 0x7F) {  // short circuit ASCII
-      utf8Count = 0;
-      return c;
-    } else if (utf8Count < sizeof(utf8String) - 1) {
-      utf8String[utf8Count++] = c;
-      utf8String[utf8Count] = 0;
-      char32_t unicodeChar[2];
-      size_t ucharCount;
-      ConversionResult res =
-          copyString8to32(unicodeChar, 2, ucharCount, utf8String);
-      if (res == conversionOK && ucharCount) {
-        utf8Count = 0;
-        return unicodeChar[0];
-      }
-    } else {
-      utf8Count =
-          0;  // this shouldn't happen: got four bytes but no UTF-8 character
-    }
-  }
-}
-
-namespace EscapeSequenceProcessing {  // move these out of global namespace
-
-// This chunk of code does parsing of the escape sequences sent by various Linux
-// terminals.
-//
-// It handles arrow keys, Home, End and Delete keys by interpreting the
-// sequences sent by
-// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and
-// Ctrl key
-// combinations that are understood by linenoise.
-//
-// The parsing uses tables, a bunch of intermediate dispatch routines and a
-// doDispatch
-// loop that reads the tables and sends control to "deeper" routines to continue
-// the
-// parsing.  The starting call to doDispatch( c, initialDispatch ) will
-// eventually return
-// either a character (with optional CTRL and META bits set), or -1 if parsing
-// fails, or
-// zero if an attempt to read from the keyboard fails.
-//
-// This is rather sloppy escape sequence processing, since we're not paying
-// attention to what the
-// actual TERM is set to and are processing all key sequences for all terminals,
-// but it works with
-// the most common keystrokes on the most common terminals.  It's intricate, but
-// the nested 'if'
-// statements required to do it directly would be worse.  This way has the
-// advantage of allowing
-// changes and extensions without having to touch a lot of code.
-
-// This is a typedef for the routine called by doDispatch().  It takes the
-// current character
-// as input, does any required processing including reading more characters and
-// calling other
-// dispatch routines, then eventually returns the final (possibly extended or
-// special) character.
-//
-typedef char32_t (*CharacterDispatchRoutine)(char32_t);
-
-// This structure is used by doDispatch() to hold a list of characters to test
-// for and
-// a list of routines to call if the character matches.  The dispatch routine
-// list is one
-// longer than the character list; the final entry is used if no character
-// matches.
-//
-struct CharacterDispatch {
-  unsigned int len;                    // length of the chars list
-  const char* chars;                   // chars to test
-  CharacterDispatchRoutine* dispatch;  // array of routines to call
-};
-
-// This dispatch routine is given a dispatch table and then farms work out to
-// routines
-// listed in the table based on the character it is called with.  The dispatch
-// routines can
-// read more input characters to decide what should eventually be returned.
-// Eventually,
-// a called routine returns either a character or -1 to indicate parsing
-// failure.
-//
-static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) {
-  for (unsigned int i = 0; i < dispatchTable.len; ++i) {
-    if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) {
-      return dispatchTable.dispatch[i](c);
-    }
-  }
-  return dispatchTable.dispatch[dispatchTable.len](c);
-}
-
-static char32_t thisKeyMetaCtrl =
-    0;  // holds pre-set Meta and/or Ctrl modifiers
-
-// Final dispatch routines -- return something
-//
-static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; }
-static char32_t upArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | UP_ARROW_KEY;
-}
-static char32_t downArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | DOWN_ARROW_KEY;
-}
-static char32_t rightArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | RIGHT_ARROW_KEY;
-}
-static char32_t leftArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | LEFT_ARROW_KEY;
-}
-static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; }
-static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; }
-static char32_t pageUpKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | PAGE_UP_KEY;
-}
-static char32_t pageDownKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | PAGE_DOWN_KEY;
-}
-static char32_t deleteCharRoutine(char32_t) {
-  return thisKeyMetaCtrl | ctrlChar('H');
-}  // key labeled Backspace
-static char32_t deleteKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | DELETE_KEY;
-}  // key labeled Delete
-static char32_t ctrlUpArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY;
-}
-static char32_t ctrlDownArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY;
-}
-static char32_t ctrlRightArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY;
-}
-static char32_t ctrlLeftArrowKeyRoutine(char32_t) {
-  return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY;
-}
-static char32_t escFailureRoutine(char32_t) {
-  beep();
-  return -1;
-}
-
-// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = {
-    upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine,
-    leftArrowKeyRoutine, escFailureRoutine};
-static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = {
-    4, "ABCD", escLeftBracket1Semicolon3or5Routines};
-
-// Handle ESC [ 1 ; <more stuff> escape sequences
-//
-static char32_t escLeftBracket1Semicolon3Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  thisKeyMetaCtrl |= META;
-  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
-}
-static char32_t escLeftBracket1Semicolon5Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  thisKeyMetaCtrl |= CTRL;
-  return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch);
-}
-static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = {
-    escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine,
-    escFailureRoutine};
-static CharacterDispatch escLeftBracket1SemicolonDispatch = {
-    2, "35", escLeftBracket1SemicolonRoutines};
-
-// Handle ESC [ 1 <more stuff> escape sequences
-//
-static char32_t escLeftBracket1SemicolonRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket1SemicolonDispatch);
-}
-static CharacterDispatchRoutine escLeftBracket1Routines[] = {
-    homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine};
-static CharacterDispatch escLeftBracket1Dispatch = {2, "~;",
-                                                    escLeftBracket1Routines};
-
-// Handle ESC [ 3 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket3Dispatch = {1, "~",
-                                                    escLeftBracket3Routines};
-
-// Handle ESC [ 4 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket4Dispatch = {1, "~",
-                                                    escLeftBracket4Routines};
-
-// Handle ESC [ 5 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket5Dispatch = {1, "~",
-                                                    escLeftBracket5Routines};
-
-// Handle ESC [ 6 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket6Dispatch = {1, "~",
-                                                    escLeftBracket6Routines};
-
-// Handle ESC [ 7 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket7Dispatch = {1, "~",
-                                                    escLeftBracket7Routines};
-
-// Handle ESC [ 8 <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine,
-                                                             escFailureRoutine};
-static CharacterDispatch escLeftBracket8Dispatch = {1, "~",
-                                                    escLeftBracket8Routines};
-
-// Handle ESC [ <digit> escape sequences
-//
-static char32_t escLeftBracket0Routine(char32_t c) {
-  return escFailureRoutine(c);
-}
-static char32_t escLeftBracket1Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket1Dispatch);
-}
-static char32_t escLeftBracket2Routine(char32_t c) {
-  return escFailureRoutine(c);  // Insert key, unused
-}
-static char32_t escLeftBracket3Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket3Dispatch);
-}
-static char32_t escLeftBracket4Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket4Dispatch);
-}
-static char32_t escLeftBracket5Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket5Dispatch);
-}
-static char32_t escLeftBracket6Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket6Dispatch);
-}
-static char32_t escLeftBracket7Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket7Dispatch);
-}
-static char32_t escLeftBracket8Routine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracket8Dispatch);
-}
-static char32_t escLeftBracket9Routine(char32_t c) {
-  return escFailureRoutine(c);
-}
-
-// Handle ESC [ <more stuff> escape sequences
-//
-static CharacterDispatchRoutine escLeftBracketRoutines[] = {
-    upArrowKeyRoutine,      downArrowKeyRoutine,    rightArrowKeyRoutine,
-    leftArrowKeyRoutine,    homeKeyRoutine,         endKeyRoutine,
-    escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine,
-    escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine,
-    escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine,
-    escLeftBracket9Routine, escFailureRoutine};
-static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789",
-                                                   escLeftBracketRoutines};
-
-// Handle ESC O <char> escape sequences
-//
-static CharacterDispatchRoutine escORoutines[] = {
-    upArrowKeyRoutine,       downArrowKeyRoutine,     rightArrowKeyRoutine,
-    leftArrowKeyRoutine,     homeKeyRoutine,          endKeyRoutine,
-    ctrlUpArrowKeyRoutine,   ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine,
-    ctrlLeftArrowKeyRoutine, escFailureRoutine};
-static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines};
-
-// Initial ESC dispatch -- could be a Meta prefix or the start of an escape
-// sequence
-//
-static char32_t escLeftBracketRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escLeftBracketDispatch);
-}
-static char32_t escORoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escODispatch);
-}
-static char32_t setMetaRoutine(char32_t c);  // need forward reference
-static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine,
-                                                 escORoutine, setMetaRoutine};
-static CharacterDispatch escDispatch = {2, "[O", escRoutines};
-
-// Initial dispatch -- we are not in the middle of anything yet
-//
-static char32_t escRoutine(char32_t c) {
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-  return doDispatch(c, escDispatch);
-}
-static CharacterDispatchRoutine initialRoutines[] = {
-    escRoutine, deleteCharRoutine, normalKeyRoutine};
-static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines};
-
-// Special handling for the ESC key because it does double duty
-//
-static char32_t setMetaRoutine(char32_t c) {
-  thisKeyMetaCtrl = META;
-  if (c == 0x1B) {  // another ESC, stay in ESC processing mode
-    c = readUnicodeCharacter();
-    if (c == 0) return 0;
-    return doDispatch(c, escDispatch);
-  }
-  return doDispatch(c, initialDispatch);
-}
-
-}  // namespace EscapeSequenceProcessing // move these out of global namespace
-
-#endif  // #ifndef _WIN32
-
-// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and
-// translate it
-// into an encoded "keystroke".  When convenient, extended keys are translated
-// into their
-// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B.
-//
-// A return value of zero means "no input available", and a return value of -1
-// means "invalid key".
-//
-static char32_t linenoiseReadChar(void) {
-#ifdef _WIN32
-
-  INPUT_RECORD rec;
-  DWORD count;
-  int modifierKeys = 0;
-  bool escSeen = false;
-  while (true) {
-    ReadConsoleInputW(console_in, &rec, 1, &count);
-#if 0  // helper for debugging keystrokes, display info in the debug "Output"
-       // window in the debugger
-        {
-            if ( rec.EventType == KEY_EVENT ) {
-                //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) {
-                    char buf[1024];
-                    sprintf(
-                            buf,
-                            "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, "
-                            "virtual scancode 0x%04X, key %s%s%s%s%s\n",
-                            rec.Event.KeyEvent.uChar.UnicodeChar,
-                            rec.Event.KeyEvent.wRepeatCount,
-                            rec.Event.KeyEvent.wVirtualKeyCode,
-                            rec.Event.KeyEvent.wVirtualScanCode,
-                            rec.Event.KeyEvent.bKeyDown ? "down" : "up",
-                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED)  ?
-                                    " L-Ctrl" : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ?
-                                    " R-Ctrl" : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED)   ?
-                                    " L-Alt"  : "",
-                                (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED)  ?
-                                    " R-Alt"  : ""
-                           );
-                    OutputDebugStringA( buf );
-                //}
-            }
-        }
-#endif
-    if (rec.EventType != KEY_EVENT) {
-      continue;
-    }
-    // Windows provides for entry of characters that are not on your keyboard by
-    // sending the
-    // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU ==
-    // Alt key) ...
-    // accept these characters, otherwise only process characters on "key down"
-    if (!rec.Event.KeyEvent.bKeyDown &&
-        rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) {
-      continue;
-    }
-    modifierKeys = 0;
-    // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't
-    // treat this
-    // combination as either CTRL or META we just turn off those two bits, so it
-    // is still
-    // possible to combine CTRL and/or META with an AltGr key by using
-    // right-Ctrl and/or
-    // left-Alt
-    if ((rec.Event.KeyEvent.dwControlKeyState &
-         (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) ==
-        (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) {
-      rec.Event.KeyEvent.dwControlKeyState &=
-          ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED);
-    }
-    if (rec.Event.KeyEvent.dwControlKeyState &
-        (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) {
-      modifierKeys |= CTRL;
-    }
-    if (rec.Event.KeyEvent.dwControlKeyState &
-        (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) {
-      modifierKeys |= META;
-    }
-    if (escSeen) {
-      modifierKeys |= META;
-    }
-    if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) {
-      switch (rec.Event.KeyEvent.wVirtualKeyCode) {
-        case VK_LEFT:
-          return modifierKeys | LEFT_ARROW_KEY;
-        case VK_RIGHT:
-          return modifierKeys | RIGHT_ARROW_KEY;
-        case VK_UP:
-          return modifierKeys | UP_ARROW_KEY;
-        case VK_DOWN:
-          return modifierKeys | DOWN_ARROW_KEY;
-        case VK_DELETE:
-          return modifierKeys | DELETE_KEY;
-        case VK_HOME:
-          return modifierKeys | HOME_KEY;
-        case VK_END:
-          return modifierKeys | END_KEY;
-        case VK_PRIOR:
-          return modifierKeys | PAGE_UP_KEY;
-        case VK_NEXT:
-          return modifierKeys | PAGE_DOWN_KEY;
-        default:
-          continue;  // in raw mode, ReadConsoleInput shows shift, ctrl ...
-      }              //  ... ignore them
-    } else if (rec.Event.KeyEvent.uChar.UnicodeChar ==
-               ctrlChar('[')) {  // ESC, set flag for later
-      escSeen = true;
-      continue;
-    } else {
-      // we got a real character, return it
-      return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar;
-    }
-  }
-
-#else
-  char32_t c;
-  c = readUnicodeCharacter();
-  if (c == 0) return 0;
-
-// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard
-// debugging mode
-// where we print out decimal and decoded values for whatever the "terminal"
-// program
-// gives us on different keystrokes.  Hit ctrl-C to exit this mode.
-//
-#define _DEBUG_LINUX_KEYBOARD
-#if defined(_DEBUG_LINUX_KEYBOARD)
-  if (c == ctrlChar('^')) {  // ctrl-^, special debug mode, prints all keys hit,
-                             // ctrl-C to get out
-    printf(
-        "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit "
-        "this mode\n");
-    while (true) {
-      unsigned char keys[10];
-      int ret = read(0, keys, 10);
-
-      if (ret <= 0) {
-        printf("\nret: %d\n", ret);
-      }
-      for (int i = 0; i < ret; ++i) {
-        char32_t key = static_cast<char32_t>(keys[i]);
-        char* friendlyTextPtr;
-        char friendlyTextBuf[10];
-        const char* prefixText = (key < 0x80) ? "" : "0x80+";
-        char32_t keyCopy = (key < 0x80) ? key : key - 0x80;
-        if (keyCopy >= '!' && keyCopy <= '~') {  // printable
-          friendlyTextBuf[0] = '\'';
-          friendlyTextBuf[1] = keyCopy;
-          friendlyTextBuf[2] = '\'';
-          friendlyTextBuf[3] = 0;
-          friendlyTextPtr = friendlyTextBuf;
-        } else if (keyCopy == ' ') {
-          friendlyTextPtr = const_cast<char*>("space");
-        } else if (keyCopy == 27) {
-          friendlyTextPtr = const_cast<char*>("ESC");
-        } else if (keyCopy == 0) {
-          friendlyTextPtr = const_cast<char*>("NUL");
-        } else if (keyCopy == 127) {
-          friendlyTextPtr = const_cast<char*>("DEL");
-        } else {
-          friendlyTextBuf[0] = '^';
-          friendlyTextBuf[1] = keyCopy + 0x40;
-          friendlyTextBuf[2] = 0;
-          friendlyTextPtr = friendlyTextBuf;
-        }
-        printf("%d x%02X (%s%s)  ", key, key, prefixText, friendlyTextPtr);
-      }
-      printf("\x1b[1G\n");  // go to first column of new line
-
-      // drop out of this loop on ctrl-C
-      if (keys[0] == ctrlChar('C')) {
-        printf("Leaving keyboard debugging mode (on ctrl-C)\n");
-        fflush(stdout);
-        return -2;
-      }
-    }
-  }
-#endif  // _DEBUG_LINUX_KEYBOARD
-
-  EscapeSequenceProcessing::thisKeyMetaCtrl =
-      0;  // no modifiers yet at initialDispatch
-  return EscapeSequenceProcessing::doDispatch(
-      c, EscapeSequenceProcessing::initialDispatch);
-#endif  // #_WIN32
-}
-
-/**
- * Free memory used in a recent command completion session
- *
- * @param lc pointer to a linenoiseCompletions struct
- */
-static void freeCompletions(linenoiseCompletions* lc) {
-  lc->completionStrings.clear();
-}
-
-/**
- * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into
- * ctrlChar( 'A' )
- * leave META alone
- *
- * @param c character to clean up
- * @return cleaned-up character
- */
-static int cleanupCtrl(int c) {
-  if (c & CTRL) {
-    int d = c & 0x1FF;
-    if (d >= 'a' && d <= 'z') {
-      c = (c + ('a' - ctrlChar('A'))) & ~CTRL;
-    }
-    if (d >= 'A' && d <= 'Z') {
-      c = (c + ('A' - ctrlChar('A'))) & ~CTRL;
-    }
-    if (d >= ctrlChar('A') && d <= ctrlChar('Z')) {
-      c = c & ~CTRL;
-    }
-  }
-  return c;
-}
-
-// break characters that may precede items to be completed
-static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}";
-
-// maximum number of completions to display without asking
-static const size_t completionCountCutoff = 100;
-
-/**
- * Handle command completion, using a completionCallback() routine to provide
- * possible substitutions
- * This routine handles the mechanics of updating the user's input buffer with
- * possible replacement
- * of text as the user selects a proposed completion string, or cancels the
- * completion attempt.
- * @param pi     PromptBase struct holding information about the prompt and our
- * screen position
- */
-int InputBuffer::completeLine(PromptBase& pi) {
-  linenoiseCompletions lc;
-  char32_t c = 0;
-
-  // completionCallback() expects a parsable entity, so find the previous break
-  // character and
-  // extract a copy to parse.  we also handle the case where tab is hit while
-  // not at end-of-line.
-  int startIndex = pos;
-  while (--startIndex >= 0) {
-    if (strchr(breakChars, buf32[startIndex])) {
-      break;
-    }
-  }
-  ++startIndex;
-  int itemLength = pos - startIndex;
-  Utf32String unicodeCopy(&buf32[startIndex], itemLength);
-  Utf8String parseItem(unicodeCopy);
-
-  // get a list of completions
-  completionCallback(parseItem.get(), &lc);
-
-  // if no completions, we are done
-  if (lc.completionStrings.size() == 0) {
-    beep();
-    freeCompletions(&lc);
-    return 0;
-  }
-
-  // at least one completion
-  int longestCommonPrefix = 0;
-  int displayLength = 0;
-  if (lc.completionStrings.size() == 1) {
-    longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length());
-  } else {
-    bool keepGoing = true;
-    while (keepGoing) {
-      for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) {
-        char32_t c1 = lc.completionStrings[j][longestCommonPrefix];
-        char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix];
-        if ((0 == c1) || (0 == c2) || (c1 != c2)) {
-          keepGoing = false;
-          break;
-        }
-      }
-      if (keepGoing) {
-        ++longestCommonPrefix;
-      }
-    }
-  }
-  if (lc.completionStrings.size() != 1) {  // beep if ambiguous
-    beep();
-  }
-
-  // if we can extend the item, extend it and return to main loop
-  if (longestCommonPrefix > itemLength) {
-    displayLength = len + longestCommonPrefix - itemLength;
-    if (displayLength > buflen) {
-      longestCommonPrefix -= displayLength - buflen;  // don't overflow buffer
-      displayLength = buflen;                         // truncate the insertion
-      beep();                                         // and make a noise
-    }
-    Utf32String displayText(displayLength + 1);
-    memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex);
-    memcpy(&displayText[startIndex], &lc.completionStrings[0][0],
-           sizeof(char32_t) * longestCommonPrefix);
-    int tailIndex = startIndex + longestCommonPrefix;
-    memcpy(&displayText[tailIndex], &buf32[pos],
-           sizeof(char32_t) * (displayLength - tailIndex + 1));
-    copyString32(buf32, displayText.get(), displayLength);
-    pos = startIndex + longestCommonPrefix;
-    len = displayLength;
-    refreshLine(pi);
-    return 0;
-  }
-
-  // we can't complete any further, wait for second tab
-  do {
-    c = linenoiseReadChar();
-    c = cleanupCtrl(c);
-  } while (c == static_cast<char32_t>(-1));
-
-  // if any character other than tab, pass it to the main loop
-  if (c != ctrlChar('I')) {
-    freeCompletions(&lc);
-    return c;
-  }
-
-  // we got a second tab, maybe show list of possible completions
-  bool showCompletions = true;
-  bool onNewLine = false;
-  if (lc.completionStrings.size() > completionCountCutoff) {
-    int savePos =
-        pos;  // move cursor to EOL to avoid overwriting the command line
-    pos = len;
-    refreshLine(pi);
-    pos = savePos;
-    printf("\nDisplay all %u possibilities? (y or n)",
-           static_cast<unsigned int>(lc.completionStrings.size()));
-    fflush(stdout);
-    onNewLine = true;
-    while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) {
-      do {
-        c = linenoiseReadChar();
-        c = cleanupCtrl(c);
-      } while (c == static_cast<char32_t>(-1));
-    }
-    switch (c) {
-      case 'n':
-      case 'N':
-        showCompletions = false;
-        freeCompletions(&lc);
-        break;
-      case ctrlChar('C'):
-        showCompletions = false;
-        freeCompletions(&lc);
-        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-        c = 0;
-        break;
-    }
-  }
-
-  // if showing the list, do it the way readline does it
-  bool stopList = false;
-  if (showCompletions) {
-    int longestCompletion = 0;
-    for (size_t j = 0; j < lc.completionStrings.size(); ++j) {
-      itemLength = static_cast<int>(lc.completionStrings[j].length());
-      if (itemLength > longestCompletion) {
-        longestCompletion = itemLength;
-      }
-    }
-    longestCompletion += 2;
-    int columnCount = pi.promptScreenColumns / longestCompletion;
-    if (columnCount < 1) {
-      columnCount = 1;
-    }
-    if (!onNewLine) {  // skip this if we showed "Display all %d possibilities?"
-      int savePos =
-          pos;  // move cursor to EOL to avoid overwriting the command line
-      pos = len;
-      refreshLine(pi);
-      pos = savePos;
-    }
-    size_t pauseRow = getScreenRows() - 1;
-    size_t rowCount =
-        (lc.completionStrings.size() + columnCount - 1) / columnCount;
-    for (size_t row = 0; row < rowCount; ++row) {
-      if (row == pauseRow) {
-        printf("\n--More--");
-        fflush(stdout);
-        c = 0;
-        bool doBeep = false;
-        while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' &&
-               c != 'n' && c != 'N' && c != 'q' && c != 'Q' &&
-               c != ctrlChar('C')) {
-          if (doBeep) {
-            beep();
-          }
-          doBeep = true;
-          do {
-            c = linenoiseReadChar();
-            c = cleanupCtrl(c);
-          } while (c == static_cast<char32_t>(-1));
-        }
-        switch (c) {
-          case ' ':
-          case 'y':
-          case 'Y':
-            printf("\r        \r");
-            pauseRow += getScreenRows() - 1;
-            break;
-          case '\r':
-          case '\n':
-            printf("\r        \r");
-            ++pauseRow;
-            break;
-          case 'n':
-          case 'N':
-          case 'q':
-          case 'Q':
-            printf("\r        \r");
-            stopList = true;
-            break;
-          case ctrlChar('C'):
-            if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-            stopList = true;
-            break;
-        }
-      } else {
-        printf("\n");
-      }
-      if (stopList) {
-        break;
-      }
-      for (int column = 0; column < columnCount; ++column) {
-        size_t index = (column * rowCount) + row;
-        if (index < lc.completionStrings.size()) {
-          itemLength = static_cast<int>(lc.completionStrings[index].length());
-          fflush(stdout);
-          if (write32(1, lc.completionStrings[index].get(), itemLength) == -1)
-            return -1;
-          if (((column + 1) * rowCount) + row < lc.completionStrings.size()) {
-            for (int k = itemLength; k < longestCompletion; ++k) {
-              printf(" ");
-            }
-          }
-        }
-      }
-    }
-    fflush(stdout);
-    freeCompletions(&lc);
-  }
-
-  // display the prompt on a new line, then redisplay the input buffer
-  if (!stopList || c == ctrlChar('C')) {
-    if (write(1, "\n", 1) == -1) return 0;
-  }
-  if (!pi.write()) return 0;
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return 0;
-#endif
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-  refreshLine(pi);
-  return 0;
-}
-
-/**
- * Clear the screen ONLY (no redisplay of anything)
- */
-void linenoiseClearScreen(void) {
-#ifdef _WIN32
-  COORD coord = {0, 0};
-  CONSOLE_SCREEN_BUFFER_INFO inf;
-  HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-  GetConsoleScreenBufferInfo(screenHandle, &inf);
-  SetConsoleCursorPosition(screenHandle, coord);
-  DWORD count;
-  FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y,
-                              coord, &count);
-#else
-  if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return;
-#endif
-}
-
-void InputBuffer::clearScreen(PromptBase& pi) {
-  linenoiseClearScreen();
-  if (!pi.write()) return;
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return;
-#endif
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-  refreshLine(pi);
-}
-
-/**
- * Incremental history search -- take over the prompt and keyboard as the user
- * types a search
- * string, deletes characters from it, changes direction, and either accepts the
- * found line (for
- * execution orediting) or cancels.
- * @param pi        PromptBase struct holding information about the (old,
- * static) prompt and our
- *                  screen position
- * @param startChar the character that began the search, used to set the initial
- * direction
- */
-int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) {
-  size_t bufferSize;
-  size_t ucharCount = 0;
-
-  // if not already recalling, add the current line to the history list so we
-  // don't have to
-  // special case it
-  if (historyIndex == historyLen - 1) {
-    free(history[historyLen - 1]);
-    bufferSize = sizeof(char32_t) * len + 1;
-    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-    copyString32to8(tempBuffer.get(), bufferSize, buf32);
-    history[historyLen - 1] = strdup8(tempBuffer.get());
-  }
-  int historyLineLength = len;
-  int historyLinePosition = pos;
-  char32_t emptyBuffer[1];
-  char emptyWidths[1];
-  InputBuffer empty(emptyBuffer, emptyWidths, 1);
-  empty.refreshLine(pi);  // erase the old input first
-  DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1);
-
-  dp.promptPreviousLen = pi.promptPreviousLen;
-  dp.promptPreviousInputLen = pi.promptPreviousInputLen;
-  dynamicRefresh(dp, buf32, historyLineLength,
-                 historyLinePosition);  // draw user's text with our prompt
-
-  // loop until we get an exit character
-  int c = 0;
-  bool keepLooping = true;
-  bool useSearchedLine = true;
-  bool searchAgain = false;
-  char32_t* activeHistoryLine = 0;
-  while (keepLooping) {
-    c = linenoiseReadChar();
-    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
-
-    switch (c) {
-      // these characters keep the selected text but do not execute it
-      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
-      case HOME_KEY:
-      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
-      case LEFT_ARROW_KEY:
-      case META + 'b':  // meta-B, move cursor left by one word
-      case META + 'B':
-      case CTRL + LEFT_ARROW_KEY:
-      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-      case ctrlChar('D'):
-      case META + 'd':  // meta-D, kill word to right of cursor
-      case META + 'D':
-      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
-      case END_KEY:
-      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
-      case RIGHT_ARROW_KEY:
-      case META + 'f':  // meta-F, move cursor right by one word
-      case META + 'F':
-      case CTRL + RIGHT_ARROW_KEY:
-      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-      case META + ctrlChar('H'):
-      case ctrlChar('J'):
-      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
-      case ctrlChar('M'):
-      case ctrlChar('N'):  // ctrl-N, recall next line in history
-      case ctrlChar('P'):  // ctrl-P, recall previous line in history
-      case DOWN_ARROW_KEY:
-      case UP_ARROW_KEY:
-      case ctrlChar('T'):  // ctrl-T, transpose characters
-      case ctrlChar(
-          'U'):  // ctrl-U, kill all characters to the left of the cursor
-      case ctrlChar('W'):
-      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
-      case META + 'Y':
-      case 127:
-      case DELETE_KEY:
-      case META + '<':  // start of history
-      case PAGE_UP_KEY:
-      case META + '>':  // end of history
-      case PAGE_DOWN_KEY:
-        keepLooping = false;
-        break;
-
-      // these characters revert the input line to its previous state
-      case ctrlChar('C'):  // ctrl-C, abort this line
-      case ctrlChar('G'):
-      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
-        keepLooping = false;
-        useSearchedLine = false;
-        if (c != ctrlChar('L')) {
-          c = -1;  // ctrl-C and ctrl-G just abort the search and do nothing
-                   // else
-        }
-        break;
-
-      // these characters stay in search mode and update the display
-      case ctrlChar('S'):
-      case ctrlChar('R'):
-        if (dp.searchTextLen ==
-            0) {  // if no current search text, recall previous text
-          if (previousSearchText.length()) {
-            dp.updateSearchText(previousSearchText.get());
-          }
-        }
-        if ((dp.direction == 1 && c == ctrlChar('R')) ||
-            (dp.direction == -1 && c == ctrlChar('S'))) {
-          dp.direction = 0 - dp.direction;  // reverse direction
-          dp.updateSearchPrompt();          // change the prompt
-        } else {
-          searchAgain = true;  // same direction, search again
-        }
-        break;
-
-// job control is its own thing
-#ifndef _WIN32
-      case ctrlChar('Z'):  // ctrl-Z, job control
-        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
-                           // mode
-        raise(SIGSTOP);    // Break out in mid-line
-        enableRawMode();   // Back from Linux shell, re-enter raw mode
-        {
-          bufferSize = historyLineLength + 1;
-          unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]);
-          copyString8to32(tempUnicode.get(), bufferSize, ucharCount,
-                          history[historyIndex]);
-          dynamicRefresh(dp, tempUnicode.get(), historyLineLength,
-                         historyLinePosition);
-        }
-        continue;
-        break;
-#endif
-
-      // these characters update the search string, and hence the selected input
-      // line
-      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
-        if (dp.searchTextLen > 0) {
-          unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]);
-          --dp.searchTextLen;
-          dp.searchText[dp.searchTextLen] = 0;
-          copyString32(tempUnicode.get(), dp.searchText.get(),
-                       dp.searchTextLen);
-          dp.updateSearchText(tempUnicode.get());
-        } else {
-          beep();
-        }
-        break;
-
-      case ctrlChar('Y'):  // ctrl-Y, yank killed text
-        break;
-
-      default:
-        if (!isControlChar(c) && c <= 0x0010FFFF) {  // not an action character
-          unique_ptr<char32_t[]> tempUnicode(
-              new char32_t[dp.searchTextLen + 2]);
-          copyString32(tempUnicode.get(), dp.searchText.get(),
-                       dp.searchTextLen);
-          tempUnicode[dp.searchTextLen] = c;
-          tempUnicode[dp.searchTextLen + 1] = 0;
-          dp.updateSearchText(tempUnicode.get());
-        } else {
-          beep();
-        }
-    }  // switch
-
-    // if we are staying in search mode, search now
-    if (keepLooping) {
-      bufferSize = historyLineLength + 1;
-      if (activeHistoryLine) {
-        delete[] activeHistoryLine;
-        activeHistoryLine = nullptr;
-      }
-      activeHistoryLine = new char32_t[bufferSize];
-      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                      history[historyIndex]);
-      if (dp.searchTextLen > 0) {
-        bool found = false;
-        int historySearchIndex = historyIndex;
-        int lineLength = static_cast<int>(ucharCount);
-        int lineSearchPos = historyLinePosition;
-        if (searchAgain) {
-          lineSearchPos += dp.direction;
-        }
-        searchAgain = false;
-        while (true) {
-          while ((dp.direction > 0) ? (lineSearchPos < lineLength)
-                                    : (lineSearchPos >= 0)) {
-            if (strncmp32(dp.searchText.get(),
-                          &activeHistoryLine[lineSearchPos],
-                          dp.searchTextLen) == 0) {
-              found = true;
-              break;
-            }
-            lineSearchPos += dp.direction;
-          }
-          if (found) {
-            historyIndex = historySearchIndex;
-            historyLineLength = lineLength;
-            historyLinePosition = lineSearchPos;
-            break;
-          } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1)
-                                        : (historySearchIndex > 0)) {
-            historySearchIndex += dp.direction;
-            bufferSize = strlen8(history[historySearchIndex]) + 1;
-            delete[] activeHistoryLine;
-            activeHistoryLine = nullptr;
-            activeHistoryLine = new char32_t[bufferSize];
-            copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                            history[historySearchIndex]);
-            lineLength = static_cast<int>(ucharCount);
-            lineSearchPos =
-                (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen);
-          } else {
-            beep();
-            break;
-          }
-        };  // while
-      }
-      if (activeHistoryLine) {
-        delete[] activeHistoryLine;
-        activeHistoryLine = nullptr;
-      }
-      bufferSize = historyLineLength + 1;
-      activeHistoryLine = new char32_t[bufferSize];
-      copyString8to32(activeHistoryLine, bufferSize, ucharCount,
-                      history[historyIndex]);
-      dynamicRefresh(dp, activeHistoryLine, historyLineLength,
-                     historyLinePosition);  // draw user's text with our prompt
-    }
-  }  // while
-
-  // leaving history search, restore previous prompt, maybe make searched line
-  // current
-  PromptBase pb;
-  pb.promptChars = pi.promptIndentation;
-  pb.promptBytes = pi.promptBytes;
-  Utf32String tempUnicode(pb.promptBytes + 1);
-
-  copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition],
-               pb.promptBytes - pi.promptLastLinePosition);
-  tempUnicode.initFromBuffer();
-  pb.promptText = tempUnicode;
-  pb.promptExtraLines = 0;
-  pb.promptIndentation = pi.promptIndentation;
-  pb.promptLastLinePosition = 0;
-  pb.promptPreviousInputLen = historyLineLength;
-  pb.promptCursorRowOffset = dp.promptCursorRowOffset;
-  pb.promptScreenColumns = pi.promptScreenColumns;
-  pb.promptPreviousLen = dp.promptChars;
-  if (useSearchedLine && activeHistoryLine) {
-    historyRecallMostRecent = true;
-    copyString32(buf32, activeHistoryLine, buflen + 1);
-    len = historyLineLength;
-    pos = historyLinePosition;
-  }
-  if (activeHistoryLine) {
-    delete[] activeHistoryLine;
-    activeHistoryLine = nullptr;
-  }
-  dynamicRefresh(pb, buf32, len,
-                 pos);  // redraw the original prompt with current input
-  pi.promptPreviousInputLen = len;
-  pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset;
-  previousSearchText =
-      dp.searchText;  // save search text for possible reuse on ctrl-R ctrl-R
-  return c;           // pass a character or -1 back to main loop
-}
-
-static bool isCharacterAlphanumeric(char32_t testChar) {
-#ifdef _WIN32
-  return (iswalnum((wint_t)testChar) != 0 ? true : false);
-#else
-  return (iswalnum(testChar) != 0 ? true : false);
-#endif
-}
-
-#ifndef _WIN32
-static bool gotResize = false;
-#endif
-static int keyType = 0;
-
-int InputBuffer::getInputLine(PromptBase& pi) {
-  keyType = 0;
-
-  // The latest history entry is always our current buffer
-  if (len > 0) {
-    size_t bufferSize = sizeof(char32_t) * len + 1;
-    unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-    copyString32to8(tempBuffer.get(), bufferSize, buf32);
-    linenoiseHistoryAdd(tempBuffer.get());
-  } else {
-    linenoiseHistoryAdd("");
-  }
-  historyIndex = historyLen - 1;
-  historyRecallMostRecent = false;
-
-  // display the prompt
-  if (!pi.write()) return -1;
-
-#ifndef _WIN32
-  // we have to generate our own newline on line wrap on Linux
-  if (pi.promptIndentation == 0 && pi.promptExtraLines > 0)
-    if (write(1, "\n", 1) == -1) return -1;
-#endif
-
-  // the cursor starts out at the end of the prompt
-  pi.promptCursorRowOffset = pi.promptExtraLines;
-
-  // kill and yank start in "other" mode
-  killRing.lastAction = KillRing::actionOther;
-
-  // when history search returns control to us, we execute its terminating
-  // keystroke
-  int terminatingKeystroke = -1;
-
-  // if there is already text in the buffer, display it first
-  if (len > 0) {
-    refreshLine(pi);
-  }
-
-  // loop collecting characters, respond to line editing characters
-  while (true) {
-    int c;
-    if (terminatingKeystroke == -1) {
-      c = linenoiseReadChar();  // get a new keystroke
-
-      keyType = 0; 
-      if (c != 0) {
-        // set flag that we got some input
-        if (c == ctrlChar('C')) {
-          keyType = 1;
-        } else if (c == ctrlChar('D')) {
-          keyType = 2;
-        }
-      }
-
-#ifndef _WIN32
-      if (c == 0 && gotResize) {
-        // caught a window resize event
-        // now redraw the prompt and line
-        gotResize = false;
-        pi.promptScreenColumns = getScreenColumns();
-        dynamicRefresh(pi, buf32, len,
-                       pos);  // redraw the original prompt with current input
-        continue;
-      }
-#endif
-    } else {
-      c = terminatingKeystroke;   // use the terminating keystroke from search
-      terminatingKeystroke = -1;  // clear it once we've used it
-    }
-
-    c = cleanupCtrl(c);  // convert CTRL + <char> into normal ctrl
-
-    if (c == 0) {
-      return len;
-    }
-
-    if (c == -1) {
-      refreshLine(pi);
-      continue;
-    }
-
-    if (c == -2) {
-      if (!pi.write()) return -1;
-      refreshLine(pi);
-      continue;
-    }
-
-    // ctrl-I/tab, command completion, needs to be before switch statement
-    if (c == ctrlChar('I') && completionCallback) {
-      killRing.lastAction = KillRing::actionOther;
-      historyRecallMostRecent = false;
-
-      // completeLine does the actual completion and replacement
-      c = completeLine(pi);
-
-      if (c < 0)  // return on error
-        return len;
-
-      if (c == 0)  // read next character when 0
-        continue;
-
-      // deliberate fall-through here, so we use the terminating character
-    }
-
-    switch (c) {
-      case ctrlChar('A'):  // ctrl-A, move cursor to start of line
-      case HOME_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        pos = 0;
-        refreshLine(pi);
-        break;
-
-      case ctrlChar('B'):  // ctrl-B, move cursor left by one character
-      case LEFT_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          --pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + 'b':  // meta-B, move cursor left by one word
-      case META + 'B':
-      case CTRL + LEFT_ARROW_KEY:
-      case META + LEFT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('C'):  // ctrl-C, abort this line
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        errno = EAGAIN;
-        --historyLen;
-        free(history[historyLen]);
-        // we need one last refresh with the cursor at the end of the line
-        // so we don't display the next prompt over the previous input line
-        pos = len;  // pass len as pos for EOL
-        refreshLine(pi);
-        if (write(1, "^C", 2) == -1) return -1;  // Display the ^C we got
-        return -1;
-
-      case META + 'c':  // meta-C, give word initial Cap
-      case META + 'C':
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        if (pos < len) {
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          if (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
-              buf32[pos] += 'A' - 'a';
-            }
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
-              buf32[pos] += 'a' - 'A';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      // ctrl-D, delete the character under the cursor
-      // on an empty line, exit the shell
-      case ctrlChar('D'):
-        killRing.lastAction = KillRing::actionOther;
-        if (len > 0 && pos < len) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
-          --len;
-          refreshLine(pi);
-        } else if (len == 0) {
-          --historyLen;
-          free(history[historyLen]);
-          return -1;
-        }
-        break;
-
-      case META + 'd':  // meta-D, kill word to right of cursor
-      case META + 'D':
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          int endingPos = pos;
-          while (endingPos < len &&
-                 !isCharacterAlphanumeric(buf32[endingPos])) {
-            ++endingPos;
-          }
-          while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) {
-            ++endingPos;
-          }
-          killRing.kill(&buf32[pos], endingPos - pos, true);
-          memmove(buf32 + pos, buf32 + endingPos,
-                  sizeof(char32_t) * (len - endingPos + 1));
-          len -= endingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('E'):  // ctrl-E, move cursor to end of line
-      case END_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        pos = len;
-        refreshLine(pi);
-        break;
-
-      case ctrlChar('F'):  // ctrl-F, move cursor right by one character
-      case RIGHT_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          ++pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + 'f':  // meta-F, move cursor right by one word
-      case META + 'F':
-      case CTRL + RIGHT_ARROW_KEY:
-      case META + RIGHT_ARROW_KEY:  // Emacs allows Meta, bash & readline don't
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('H'):  // backspace/ctrl-H, delete char to left of cursor
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos - 1, buf32 + pos,
-                  sizeof(char32_t) * (1 + len - pos));
-          --pos;
-          --len;
-          refreshLine(pi);
-        }
-        break;
-
-      // meta-Backspace, kill word to left of cursor
-      case META + ctrlChar('H'):
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          int startingPos = pos;
-          while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) {
-            --pos;
-          }
-          killRing.kill(&buf32[pos], startingPos - pos, false);
-          memmove(buf32 + pos, buf32 + startingPos,
-                  sizeof(char32_t) * (len - startingPos + 1));
-          len -= startingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('J'):  // ctrl-J/linefeed/newline, accept line
-      case ctrlChar('M'):  // ctrl-M/return/enter
-        killRing.lastAction = KillRing::actionOther;
-        // we need one last refresh with the cursor at the end of the line
-        // so we don't display the next prompt over the previous input line
-        pos = len;  // pass len as pos for EOL
-        refreshLine(pi);
-        historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2;
-        --historyLen;
-        free(history[historyLen]);
-        return len;
-
-      case ctrlChar('K'):  // ctrl-K, kill from cursor to end of line
-        killRing.kill(&buf32[pos], len - pos, true);
-        buf32[pos] = '\0';
-        len = pos;
-        refreshLine(pi);
-        killRing.lastAction = KillRing::actionKill;
-        historyRecallMostRecent = false;
-        break;
-
-      case ctrlChar('L'):  // ctrl-L, clear screen and redisplay line
-        clearScreen(pi);
-        break;
-
-      case META + 'l':  // meta-L, lowercase word
-      case META + 'L':
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') {
-              buf32[pos] += 'a' - 'A';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('N'):  // ctrl-N, recall next line in history
-      case ctrlChar('P'):  // ctrl-P, recall previous line in history
-      case DOWN_ARROW_KEY:
-      case UP_ARROW_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        // if not already recalling, add the current line to the history list so
-        // we don't
-        // have to special case it
-        if (historyIndex == historyLen - 1) {
-          free(history[historyLen - 1]);
-          size_t tempBufferSize = sizeof(char32_t) * len + 1;
-          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
-          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
-          history[historyLen - 1] = strdup8(tempBuffer.get());
-        }
-        if (historyLen > 1) {
-          if (c == UP_ARROW_KEY) {
-            c = ctrlChar('P');
-          }
-          if (historyPreviousIndex != -2 && c != ctrlChar('P')) {
-            historyIndex =
-                1 + historyPreviousIndex;  // emulate Windows down-arrow
-          } else {
-            historyIndex += (c == ctrlChar('P')) ? -1 : 1;
-          }
-          historyPreviousIndex = -2;
-          if (historyIndex < 0) {
-            historyIndex = 0;
-            break;
-          } else if (historyIndex >= historyLen) {
-            historyIndex = historyLen - 1;
-            break;
-          }
-          historyRecallMostRecent = true;
-          size_t ucharCount = 0;
-          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
-          len = pos = static_cast<int>(ucharCount);
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar('R'):  // ctrl-R, reverse history search
-      case ctrlChar('S'):  // ctrl-S, forward history search
-        terminatingKeystroke = incrementalHistorySearch(pi, c);
-        break;
-
-      case ctrlChar('T'):  // ctrl-T, transpose characters
-        killRing.lastAction = KillRing::actionOther;
-        if (pos > 0 && len > 1) {
-          historyRecallMostRecent = false;
-          size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1;
-          char32_t aux = buf32[leftCharPos];
-          buf32[leftCharPos] = buf32[leftCharPos + 1];
-          buf32[leftCharPos + 1] = aux;
-          if (pos != len) ++pos;
-          refreshLine(pi);
-        }
-        break;
-
-      case ctrlChar(
-          'U'):  // ctrl-U, kill all characters to the left of the cursor
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          killRing.kill(&buf32[0], pos, false);
-          len -= pos;
-          memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1));
-          pos = 0;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case META + 'u':  // meta-U, uppercase word
-      case META + 'U':
-        killRing.lastAction = KillRing::actionOther;
-        if (pos < len) {
-          historyRecallMostRecent = false;
-          while (pos < len && !isCharacterAlphanumeric(buf32[pos])) {
-            ++pos;
-          }
-          while (pos < len && isCharacterAlphanumeric(buf32[pos])) {
-            if (buf32[pos] >= 'a' && buf32[pos] <= 'z') {
-              buf32[pos] += 'A' - 'a';
-            }
-            ++pos;
-          }
-          refreshLine(pi);
-        }
-        break;
-
-      // ctrl-W, kill to whitespace (not word) to left of cursor
-      case ctrlChar('W'):
-        if (pos > 0) {
-          historyRecallMostRecent = false;
-          int startingPos = pos;
-          while (pos > 0 && buf32[pos - 1] == ' ') {
-            --pos;
-          }
-          while (pos > 0 && buf32[pos - 1] != ' ') {
-            --pos;
-          }
-          killRing.kill(&buf32[pos], startingPos - pos, false);
-          memmove(buf32 + pos, buf32 + startingPos,
-                  sizeof(char32_t) * (len - startingPos + 1));
-          len -= startingPos - pos;
-          refreshLine(pi);
-        }
-        killRing.lastAction = KillRing::actionKill;
-        break;
-
-      case ctrlChar('Y'):  // ctrl-Y, yank killed text
-        historyRecallMostRecent = false;
-        {
-          Utf32String* restoredText = killRing.yank();
-          if (restoredText) {
-            bool truncated = false;
-            size_t ucharCount = restoredText->length();
-            if (ucharCount > static_cast<size_t>(buflen - len)) {
-              ucharCount = buflen - len;
-              truncated = true;
-            }
-            memmove(buf32 + pos + ucharCount, buf32 + pos,
-                    sizeof(char32_t) * (len - pos + 1));
-            memmove(buf32 + pos, restoredText->get(),
-                    sizeof(char32_t) * ucharCount);
-            pos += static_cast<int>(ucharCount);
-            len += static_cast<int>(ucharCount);
-            refreshLine(pi);
-            killRing.lastAction = KillRing::actionYank;
-            killRing.lastYankSize = ucharCount;
-            if (truncated) {
-              beep();
-            }
-          } else {
-            beep();
-          }
-        }
-        break;
-
-      case META + 'y':  // meta-Y, "yank-pop", rotate popped text
-      case META + 'Y':
-        if (killRing.lastAction == KillRing::actionYank) {
-          historyRecallMostRecent = false;
-          Utf32String* restoredText = killRing.yankPop();
-          if (restoredText) {
-            bool truncated = false;
-            size_t ucharCount = restoredText->length();
-            if (ucharCount >
-                static_cast<size_t>(killRing.lastYankSize + buflen - len)) {
-              ucharCount = killRing.lastYankSize + buflen - len;
-              truncated = true;
-            }
-            if (ucharCount > killRing.lastYankSize) {
-              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
-                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
-              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
-                      sizeof(char32_t) * ucharCount);
-            } else {
-              memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(),
-                      sizeof(char32_t) * ucharCount);
-              memmove(buf32 + pos + ucharCount - killRing.lastYankSize,
-                      buf32 + pos, sizeof(char32_t) * (len - pos + 1));
-            }
-            pos += static_cast<int>(ucharCount - killRing.lastYankSize);
-            len += static_cast<int>(ucharCount - killRing.lastYankSize);
-            killRing.lastYankSize = ucharCount;
-            refreshLine(pi);
-            if (truncated) {
-              beep();
-            }
-            break;
-          }
-        }
-        beep();
-        break;
-
-#ifndef _WIN32
-      case ctrlChar('Z'):  // ctrl-Z, job control
-        disableRawMode();  // Returning to Linux (whatever) shell, leave raw
-                           // mode
-        raise(SIGSTOP);    // Break out in mid-line
-        enableRawMode();   // Back from Linux shell, re-enter raw mode
-        if (!pi.write()) break;  // Redraw prompt
-        refreshLine(pi);         // Refresh the line
-        break;
-#endif
-
-      // DEL, delete the character under the cursor
-      case 127:
-      case DELETE_KEY:
-        killRing.lastAction = KillRing::actionOther;
-        if (len > 0 && pos < len) {
-          historyRecallMostRecent = false;
-          memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos));
-          --len;
-          refreshLine(pi);
-        }
-        break;
-
-      case META + '<':     // meta-<, beginning of history
-      case PAGE_UP_KEY:    // Page Up, beginning of history
-      case META + '>':     // meta->, end of history
-      case PAGE_DOWN_KEY:  // Page Down, end of history
-        killRing.lastAction = KillRing::actionOther;
-        // if not already recalling, add the current line to the history list so
-        // we don't
-        // have to special case it
-        if (historyIndex == historyLen - 1) {
-          free(history[historyLen - 1]);
-          size_t tempBufferSize = sizeof(char32_t) * len + 1;
-          unique_ptr<char[]> tempBuffer(new char[tempBufferSize]);
-          copyString32to8(tempBuffer.get(), tempBufferSize, buf32);
-          history[historyLen - 1] = strdup8(tempBuffer.get());
-        }
-        if (historyLen > 1) {
-          historyIndex =
-              (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1;
-          historyPreviousIndex = -2;
-          historyRecallMostRecent = true;
-          size_t ucharCount = 0;
-          copyString8to32(buf32, buflen, ucharCount, history[historyIndex]);
-          len = pos = static_cast<int>(ucharCount);
-          refreshLine(pi);
-        }
-        break;
-
-      // not one of our special characters, maybe insert it in the buffer
-      default:
-        killRing.lastAction = KillRing::actionOther;
-        historyRecallMostRecent = false;
-        if (c & (META | CTRL)) {  // beep on unknown Ctrl and/or Meta keys
-          beep();
-          break;
-        }
-        if (len < buflen) {
-          if (isControlChar(c)) {  // don't insert control characters
-            beep();
-            break;
-          }
-          if (len == pos) {  // at end of buffer
-            buf32[pos] = c;
-            ++pos;
-            ++len;
-            buf32[len] = '\0';
-            int inputLen = calculateColumnPosition(buf32, len);
-            if (pi.promptIndentation + inputLen < pi.promptScreenColumns) {
-              if (inputLen > pi.promptPreviousInputLen)
-                pi.promptPreviousInputLen = inputLen;
-              /* Avoid a full update of the line in the
-               * trivial case. */
-              if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1)
-                return -1;
-            } else {
-              refreshLine(pi);
-            }
-          } else {  // not at end of buffer, have to move characters to our
-                    // right
-            memmove(buf32 + pos + 1, buf32 + pos,
-                    sizeof(char32_t) * (len - pos));
-            buf32[pos] = c;
-            ++len;
-            ++pos;
-            buf32[len] = '\0';
-            refreshLine(pi);
-          }
-        } else {
-          beep();  // buffer is full, beep on new characters
-        }
-        break;
-    }
-  }
-  return len;
-}
-
-static string preloadedBufferContents;  // used with linenoisePreloadBuffer
-static string preloadErrorMessage;
-
-/**
- * linenoisePreloadBuffer provides text to be inserted into the command buffer
- *
- * the provided text will be processed to be usable and will be used to preload
- * the input buffer on the next call to linenoise()
- *
- * @param preloadText text to begin with on the next call to linenoise()
- */
-void linenoisePreloadBuffer(const char* preloadText) {
-  if (!preloadText) {
-    return;
-  }
-  int bufferSize = static_cast<int>(strlen(preloadText) + 1);
-  unique_ptr<char[]> tempBuffer(new char[bufferSize]);
-  strncpy(&tempBuffer[0], preloadText, bufferSize);
-
-  // remove characters that won't display correctly
-  char* pIn = &tempBuffer[0];
-  char* pOut = pIn;
-  bool controlsStripped = false;
-  bool whitespaceSeen = false;
-  while (*pIn) {
-    unsigned char c =
-        *pIn++;       // we need unsigned so chars 0x80 and above are allowed
-    if ('\r' == c) {  // silently skip CR
-      continue;
-    }
-    if ('\n' == c || '\t' == c) {  // note newline or tab
-      whitespaceSeen = true;
-      continue;
-    }
-    if (isControlChar(
-            c)) {  // remove other control characters, flag for message
-      controlsStripped = true;
-      *pOut++ = ' ';
-      continue;
-    }
-    if (whitespaceSeen) {  // convert whitespace to a single space
-      *pOut++ = ' ';
-      whitespaceSeen = false;
-    }
-    *pOut++ = c;
-  }
-  *pOut = 0;
-  int processedLength = static_cast<int>(pOut - tempBuffer.get());
-  bool lineTruncated = false;
-  if (processedLength > (LINENOISE_MAX_LINE - 1)) {
-    lineTruncated = true;
-    tempBuffer[LINENOISE_MAX_LINE - 1] = 0;
-  }
-  preloadedBufferContents = tempBuffer.get();
-  if (controlsStripped) {
-    preloadErrorMessage +=
-        " [Edited line: control characters were converted to spaces]\n";
-  }
-  if (lineTruncated) {
-    preloadErrorMessage += " [Edited line: the line length was reduced from ";
-    char buf[128];
-    snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength,
-             (LINENOISE_MAX_LINE - 1));
-    preloadErrorMessage += buf;
-  }
-}
-
-/**
- * linenoise is a readline replacement.
- *
- * call it with a prompt to display and it will return a line of input from the
- * user
- *
- * @param prompt text of prompt to display to the user
- * @return       the returned string belongs to the caller on return and must be
- * freed to prevent
- *               memory leaks
- */
-char* linenoise(const char* prompt) {
-#ifndef _WIN32
-  gotResize = false;
-#endif
-  if (isatty(STDIN_FILENO)) {  // input is from a terminal
-    char32_t buf32[LINENOISE_MAX_LINE];
-    char charWidths[LINENOISE_MAX_LINE];
-    if (!preloadErrorMessage.empty()) {
-      printf("%s", preloadErrorMessage.c_str());
-      fflush(stdout);
-      preloadErrorMessage.clear();
-    }
-    PromptInfo pi(prompt, getScreenColumns());
-    if (isUnsupportedTerm()) {
-      if (!pi.write()) return 0;
-      fflush(stdout);
-      if (preloadedBufferContents.empty()) {
-        unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
-        if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
-          return NULL;
-        }
-        size_t len = strlen(buf8.get());
-        while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) {
-          --len;
-          buf8[len] = '\0';
-        }
-        return strdup(buf8.get());  // caller must free buffer
-      } else {
-        char* buf8 = strdup(preloadedBufferContents.c_str());
-        preloadedBufferContents.clear();
-        return buf8;  // caller must free buffer
-      }
-    } else {
-      if (enableRawMode() == -1) {
-        return NULL;
-      }
-      InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE);
-      if (!preloadedBufferContents.empty()) {
-        ib.preloadBuffer(preloadedBufferContents.c_str());
-        preloadedBufferContents.clear();
-      }
-      int count = ib.getInputLine(pi);
-      disableRawMode();
-      printf("\n");
-      if (count == -1) {
-        return NULL;
-      }
-      size_t bufferSize = sizeof(char32_t) * ib.length() + 1;
-      unique_ptr<char[]> buf8(new char[bufferSize]);
-      copyString32to8(buf8.get(), bufferSize, buf32);
-      return strdup(buf8.get());  // caller must free buffer
-    }
-  } else {  // input not from a terminal, we should work with piped input, i.e.
-            // redirected stdin
-    unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]);
-    if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) {
-      return NULL;
-    }
-
-    // if fgets() gave us the newline, remove it
-    int count = static_cast<int>(strlen(buf8.get()));
-    if (count > 0 && buf8[count - 1] == '\n') {
-      --count;
-      buf8[count] = '\0';
-    }
-    return strdup(buf8.get());  // caller must free buffer
-  }
-}
-
-/* Register a callback function to be called for tab-completion. */
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) {
-  completionCallback = fn;
-}
-
-void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) {
-  lc->completionStrings.push_back(Utf32String(str));
-}
-
-int linenoiseHistoryAdd(const char* line) {
-  if (historyMaxLen == 0) {
-    return 0;
-  }
-  if (history == NULL) {
-    history =
-        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen));
-    if (history == NULL) {
-      return 0;
-    }
-    memset(history, 0, (sizeof(char*) * historyMaxLen));
-  }
-  char8_t* linecopy = strdup8(line);
-  if (!linecopy) {
-    return 0;
-  }
-
-  // convert newlines in multi-line code to spaces before storing
-  char8_t* p = linecopy;
-  while (*p) {
-    if (*p == '\n') {
-      *p = ' ';
-    }
-    ++p;
-  }
-
-  // prevent duplicate history entries
-  if (historyLen > 0 && history[historyLen - 1] != nullptr &&
-      strcmp(reinterpret_cast<char const*>(history[historyLen - 1]),
-             reinterpret_cast<char const*>(linecopy)) == 0) {
-    free(linecopy);
-    return 0;
-  }
-
-  if (historyLen == historyMaxLen) {
-    free(history[0]);
-    memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1));
-    --historyLen;
-    if (--historyPreviousIndex < -1) {
-      historyPreviousIndex = -2;
-    }
-  }
-
-  history[historyLen] = linecopy;
-  ++historyLen;
-  return 1;
-}
-
-int linenoiseHistorySetMaxLen(int len) {
-  if (len < 1) {
-    return 0;
-  }
-  if (history) {
-    int tocopy = historyLen;
-    char8_t** newHistory =
-        reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len));
-    if (newHistory == NULL) {
-      return 0;
-    }
-    if (len < tocopy) {
-      tocopy = len;
-    }
-    memcpy(newHistory, history + historyMaxLen - tocopy,
-           sizeof(char8_t*) * tocopy);
-    free(history);
-    history = newHistory;
-  }
-  historyMaxLen = len;
-  if (historyLen > historyMaxLen) {
-    historyLen = historyMaxLen;
-  }
-  return 1;
-}
-
-/* Fetch a line of the history by (zero-based) index.  If the requested
- * line does not exist, NULL is returned.  The return value is a heap-allocated
- * copy of the line, and the caller is responsible for de-allocating it. */
-char* linenoiseHistoryLine(int index) {
-  if (index < 0 || index >= historyLen) return NULL;
-
-  return strdup(reinterpret_cast<char const*>(history[index]));
-}
-
-/* Save the history in the specified file. On success 0 is returned
- * otherwise -1 is returned. */
-int linenoiseHistorySave(const char* filename) {
-#if _WIN32
-  FILE* fp = fopen(filename, "wt");
-#else
-  int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR);
-
-  if (fd < 0) {
-    return -1;
-  }
-
-  FILE* fp = fdopen(fd, "wt");
-#endif
-
-  if (fp == NULL) {
-    return -1;
-  }
-
-  for (int j = 0; j < historyLen; ++j) {
-    if (history[j][0] != '\0') {
-      fprintf(fp, "%s\n", history[j]);
-    }
-  }
-
-  fclose(fp);
-
-  return 0;
-}
-
-/* Load the history from the specified file. If the file does not exist
- * zero is returned and no operation is performed.
- *
- * If the file exists and the operation succeeded 0 is returned, otherwise
- * on error -1 is returned. */
-int linenoiseHistoryLoad(const char* filename) {
-  FILE* fp = fopen(filename, "rt");
-  if (fp == NULL) {
-    return -1;
-  }
-
-  char buf[LINENOISE_MAX_LINE];
-  while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) {
-    char* p = strchr(buf, '\r');
-    if (!p) {
-      p = strchr(buf, '\n');
-    }
-    if (p) {
-      *p = '\0';
-    }
-    if (p != buf) {
-      linenoiseHistoryAdd(buf);
-    }
-  }
-  fclose(fp);
-  return 0;
-}
-
-/* Set if to use or not the multi line mode. */
-/* note that this is a stub only, as linenoise-ng always multi-line */
-void linenoiseSetMultiLine(int) {}
-
-/* This special mode is used by linenoise in order to print scan codes
- * on screen for debugging / development purposes. It is implemented
- * by the linenoise_example program using the --keycodes option. */
-void linenoisePrintKeyCodes(void) {
-  char quit[4];
-
-  printf(
-      "Linenoise key codes debugging mode.\n"
-      "Press keys to see scan codes. Type 'quit' at any time to exit.\n");
-  if (enableRawMode() == -1) return;
-  memset(quit, ' ', 4);
-  while (1) {
-    char c;
-    int nread;
-
-#if _WIN32
-    nread = _read(STDIN_FILENO, &c, 1);
-#else
-    nread = read(STDIN_FILENO, &c, 1);
-#endif
-    if (nread <= 0) continue;
-    memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */
-    quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */
-    if (memcmp(quit, "quit", sizeof(quit)) == 0) break;
-
-    printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c,
-           (int)c);
-    printf("\r"); /* Go left edge manually, we are in raw mode. */
-    fflush(stdout);
-  }
-  disableRawMode();
-}
-
-#ifndef _WIN32
-static void WindowSizeChanged(int) {
-  // do nothing here but setting this flag
-  gotResize = true;
-}
-#endif
-
-int linenoiseInstallWindowChangeHandler(void) {
-#ifndef _WIN32
-  struct sigaction sa;
-  sigemptyset(&sa.sa_mask);
-  sa.sa_flags = 0;
-  sa.sa_handler = &WindowSizeChanged;
-
-  if (sigaction(SIGWINCH, &sa, nullptr) == -1) {
-    return errno;
-  }
-#endif
-  return 0;
-}
-
-int linenoiseKeyType(void) {
-  return keyType;
-}
diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h
deleted file mode 100644
index 3a8eb9f7ee6..00000000000
--- a/src/linenoise/linenoise.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* linenoise.h -- guerrilla line editing library against the idea that a
- * line editing lib needs to be 20,000 lines of C code.
- *
- * See linenoise.c for more information.
- *
- * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __LINENOISE_H
-#define __LINENOISE_H
-
-#define LINENOISE_VERSION "1.0.0"
-#define LINENOISE_VERSION_MAJOR 1
-#define LINENOISE_VERSION_MINOR 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct linenoiseCompletions linenoiseCompletions;
-
-typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
-void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn);
-void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str);
-
-char* linenoise(const char* prompt);
-void linenoisePreloadBuffer(const char* preloadText);
-int linenoiseHistoryAdd(const char* line);
-int linenoiseHistorySetMaxLen(int len);
-char* linenoiseHistoryLine(int index);
-int linenoiseHistorySave(const char* filename);
-int linenoiseHistoryLoad(const char* filename);
-void linenoiseHistoryFree(void);
-void linenoiseClearScreen(void);
-void linenoiseSetMultiLine(int ml);
-void linenoisePrintKeyCodes(void);
-/* the following are extensions to the original linenoise API */
-int linenoiseInstallWindowChangeHandler(void);
-/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ 
-int linenoiseKeyType(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* __LINENOISE_H */
diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp
deleted file mode 100644
index deec0ba6b57..00000000000
--- a/src/linenoise/wcwidth.cpp
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * This is an implementation of wcwidth() and wcswidth() (defined in
- * IEEE Std 1002.1-2001) for Unicode.
- *
- * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
- * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
- *
- * In fixed-width output devices, Latin characters all occupy a single
- * "cell" position of equal width, whereas ideographic CJK characters
- * occupy two such cells. Interoperability between terminal-line
- * applications and (teletype-style) character terminals using the
- * UTF-8 encoding requires agreement on which character should advance
- * the cursor by how many cell positions. No established formal
- * standards exist at present on which Unicode character shall occupy
- * how many cell positions on character terminals. These routines are
- * a first attempt of defining such behavior based on simple rules
- * applied to data provided by the Unicode Consortium.
- *
- * For some graphical characters, the Unicode standard explicitly
- * defines a character-cell width via the definition of the East Asian
- * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
- * In all these cases, there is no ambiguity about which width a
- * terminal shall use. For characters in the East Asian Ambiguous (A)
- * class, the width choice depends purely on a preference of backward
- * compatibility with either historic CJK or Western practice.
- * Choosing single-width for these characters is easy to justify as
- * the appropriate long-term solution, as the CJK practice of
- * displaying these characters as double-width comes from historic
- * implementation simplicity (8-bit encoded characters were displayed
- * single-width and 16-bit ones double-width, even for Greek,
- * Cyrillic, etc.) and not any typographic considerations.
- *
- * Much less clear is the choice of width for the Not East Asian
- * (Neutral) class. Existing practice does not dictate a width for any
- * of these characters. It would nevertheless make sense
- * typographically to allocate two character cells to characters such
- * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
- * represented adequately with a single-width glyph. The following
- * routines at present merely assign a single-cell width to all
- * neutral characters, in the interest of simplicity. This is not
- * entirely satisfactory and should be reconsidered before
- * establishing a formal standard in this area. At the moment, the
- * decision which Not East Asian (Neutral) characters should be
- * represented by double-width glyphs cannot yet be answered by
- * applying a simple rule from the Unicode database content. Setting
- * up a proper standard for the behavior of UTF-8 character terminals
- * will require a careful analysis not only of each Unicode character,
- * but also of each presentation form, something the author of these
- * routines has avoided to do so far.
- *
- * http://www.unicode.org/unicode/reports/tr11/
- *
- * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
- *
- * Permission to use, copy, modify, and distribute this software
- * for any purpose and without fee is hereby granted. The author
- * disclaims all warranties with regard to this software.
- *
- * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
- */
-
-#include <wchar.h>
-#include <string>
-#include <memory>
-
-namespace linenoise_ng {
-
-struct interval {
-  char32_t first;
-  char32_t last;
-};
-
-/* auxiliary function for binary search in interval table */
-static int bisearch(char32_t ucs, const struct interval *table, int max) {
-  int min = 0;
-  int mid;
-
-  if (ucs < table[0].first || ucs > table[max].last)
-    return 0;
-  while (max >= min) {
-    mid = (min + max) / 2;
-    if (ucs > table[mid].last)
-      min = mid + 1;
-    else if (ucs < table[mid].first)
-      max = mid - 1;
-    else
-      return 1;
-  }
-
-  return 0;
-}
-
-
-/* The following two functions define the column width of an ISO 10646
- * character as follows:
- *
- *    - The null character (U+0000) has a column width of 0.
- *
- *    - Other C0/C1 control characters and DEL will lead to a return
- *      value of -1.
- *
- *    - Non-spacing and enclosing combining characters (general
- *      category code Mn or Me in the Unicode database) have a
- *      column width of 0.
- *
- *    - SOFT HYPHEN (U+00AD) has a column width of 1.
- *
- *    - Other format characters (general category code Cf in the Unicode
- *      database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
- *
- *    - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
- *      have a column width of 0.
- *
- *    - Spacing characters in the East Asian Wide (W) or East Asian
- *      Full-width (F) category as defined in Unicode Technical
- *      Report #11 have a column width of 2.
- *
- *    - All remaining characters (including all printable
- *      ISO 8859-1 and WGL4 characters, Unicode control characters,
- *      etc.) have a column width of 1.
- *
- * This implementation assumes that wchar_t characters are encoded
- * in ISO 10646.
- */
-
-int mk_wcwidth(char32_t ucs)
-{
-  /* sorted list of non-overlapping intervals of non-spacing characters */
-  /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
-  static const struct interval combining[] = {
-    { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
-    { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
-    { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
-    { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
-    { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
-    { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
-    { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
-    { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
-    { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
-    { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
-    { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
-    { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
-    { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
-    { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
-    { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
-    { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
-    { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
-    { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
-    { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
-    { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
-    { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
-    { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
-    { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
-    { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
-    { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
-    { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
-    { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
-    { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
-    { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
-    { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
-    { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
-    { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
-    { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
-    { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
-    { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
-    { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
-    { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
-    { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
-    { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
-    { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
-    { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
-    { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
-    { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
-    { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
-    { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
-    { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
-    { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
-    { 0xE0100, 0xE01EF }
-  };
-
-  /* test for 8-bit control characters */
-  if (ucs == 0)
-    return 0;
-  if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
-    return -1;
-
-  /* binary search in table of non-spacing characters */
-  if (bisearch(ucs, combining,
-               sizeof(combining) / sizeof(struct interval) - 1))
-    return 0;
-
-  /* if we arrive here, ucs is not a combining or C0/C1 control character */
-
-  return 1 + 
-    (ucs >= 0x1100 &&
-     (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
-      ucs == 0x2329 || ucs == 0x232a ||
-      (ucs >= 0x2e80 && ucs <= 0xa4cf &&
-       ucs != 0x303f) ||                  /* CJK ... Yi */
-      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
-      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
-      (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
-      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
-      (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
-      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
-      (ucs >= 0x20000 && ucs <= 0x2fffd) ||
-      (ucs >= 0x30000 && ucs <= 0x3fffd)));
-}
-
-
-int mk_wcswidth(const char32_t* pwcs, size_t n)
-{
-  int w, width = 0;
-
-  for (;*pwcs && n-- > 0; pwcs++)
-    if ((w = mk_wcwidth(*pwcs)) < 0)
-      return -1;
-    else
-      width += w;
-
-  return width;
-}
-
-
-/*
- * The following functions are the same as mk_wcwidth() and
- * mk_wcswidth(), except that spacing characters in the East Asian
- * Ambiguous (A) category as defined in Unicode Technical Report #11
- * have a column width of 2. This variant might be useful for users of
- * CJK legacy encodings who want to migrate to UCS without changing
- * the traditional terminal character-width behaviour. It is not
- * otherwise recommended for general use.
- */
-int mk_wcwidth_cjk(wchar_t ucs)
-{
-  /* sorted list of non-overlapping intervals of East Asian Ambiguous
-   * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
-  static const struct interval ambiguous[] = {
-    { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 },
-    { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 },
-    { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 },
-    { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 },
-    { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED },
-    { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA },
-    { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 },
-    { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B },
-    { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 },
-    { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 },
-    { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 },
-    { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE },
-    { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 },
-    { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA },
-    { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 },
-    { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB },
-    { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB },
-    { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 },
-    { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 },
-    { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 },
-    { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 },
-    { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 },
-    { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 },
-    { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 },
-    { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC },
-    { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 },
-    { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 },
-    { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 },
-    { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 },
-    { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 },
-    { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 },
-    { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B },
-    { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 },
-    { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 },
-    { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E },
-    { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 },
-    { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 },
-    { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F },
-    { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 },
-    { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF },
-    { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B },
-    { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 },
-    { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 },
-    { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 },
-    { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 },
-    { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 },
-    { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 },
-    { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 },
-    { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 },
-    { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F },
-    { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
-    { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
-  };
-
-  /* binary search in table of non-spacing characters */
-  if (bisearch(ucs, ambiguous,
-               sizeof(ambiguous) / sizeof(struct interval) - 1))
-    return 2;
-
-  return mk_wcwidth(ucs);
-}
-
-
-int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n)
-{
-  int w, width = 0;
-
-  for (;*pwcs && n-- > 0; pwcs++)
-    if ((w = mk_wcwidth_cjk(*pwcs)) < 0)
-      return -1;
-    else
-      width += w;
-
-  return width;
-}
-
-}
diff --git a/src/nix/local.mk b/src/nix/local.mk
index bdcca33d2a6..40a0e8d6bde 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -4,7 +4,6 @@ nix_DIR := $(d)
 
 nix_SOURCES := \
   $(wildcard $(d)/*.cc) \
-  $(wildcard src/linenoise/*.cpp) \
   $(wildcard src/build-remote/*.cc) \
   $(wildcard src/nix-build/*.cc) \
   $(wildcard src/nix-channel/*.cc) \
@@ -18,7 +17,7 @@ nix_SOURCES := \
 
 nix_LIBS = libexpr libmain libstore libutil libformat
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS)
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS)
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 1bbe256b2d8..7ce37077021 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -1,8 +1,12 @@
 #include <iostream>
 #include <cstdlib>
+#include <cstring>
+#include <climits>
 
 #include <setjmp.h>
 
+#include <editline.h>
+
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
@@ -15,8 +19,6 @@
 #include "command.hh"
 #include "finally.hh"
 
-#include "src/linenoise/linenoise.h"
-
 namespace nix {
 
 #define ESC_RED "\033[31m"
@@ -118,17 +120,53 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
 
 NixRepl::~NixRepl()
 {
-    linenoiseHistorySave(historyFile.c_str());
+    write_history(historyFile.c_str());
 }
 
-
 static NixRepl * curRepl; // ugly
 
-static void completionCallback(const char * s, linenoiseCompletions *lc)
-{
-    /* Otherwise, return all symbols that start with the prefix. */
-    for (auto & c : curRepl->completePrefix(s))
-        linenoiseAddCompletion(lc, c.c_str());
+static char * completionCallback(char * s, int *match) {
+  auto possible = curRepl->completePrefix(s);
+  if (possible.size() == 1) {
+    *match = 1;
+    auto *res = strdup(possible.begin()->c_str() + strlen(s));
+    if (!res) throw Error("allocation failure");
+    return res;
+  }
+
+  *match = 0;
+  return nullptr;
+}
+
+static int listPossibleCallback(char *s, char ***avp) {
+  auto possible = curRepl->completePrefix(s);
+
+  if (possible.size() > (INT_MAX / sizeof(char*)))
+    throw Error("too many completions");
+
+  int ac = 0;
+  char **vp = nullptr;
+
+  auto check = [&](auto *p) {
+    if (!p) {
+      if (vp) {
+        while (--ac >= 0)
+          free(vp[ac]);
+        free(vp);
+      }
+      throw Error("allocation failure");
+    }
+    return p;
+  };
+
+  vp = check((char **)malloc(possible.size() * sizeof(char*)));
+
+  for (auto & p : possible)
+    vp[ac++] = check(strdup(p.c_str()));
+
+  *avp = vp;
+
+  return ac;
 }
 
 
@@ -143,12 +181,16 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
     reloadFiles();
     if (!loadedFiles.empty()) std::cout << std::endl;
 
+    // Allow nix-repl specific settings in .inputrc
+    rl_readline_name = "nix-repl";
     createDirs(dirOf(historyFile));
-    linenoiseHistorySetMaxLen(1000);
-    linenoiseHistoryLoad(historyFile.c_str());
-
+    el_hist_size = 1000;
+    read_history(historyFile.c_str());
+    // rl_initialize();
+    // linenoiseSetCompletionCallback(completionCallback);
     curRepl = this;
-    linenoiseSetCompletionCallback(completionCallback);
+    rl_set_complete_func(completionCallback);
+    rl_set_list_possib_func(listPossibleCallback);
 
     std::string input;
 
@@ -174,12 +216,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
             printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
         }
 
-        if (input.size() > 0) {
-            // Remove trailing newline before adding to history
-            input.erase(input.size() - 1);
-            linenoiseHistoryAdd(input.c_str());
-        }
-
         // We handled the current input fully, so we should clear it
         // and read brand new input.
         input.clear();
@@ -190,19 +226,10 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 
 bool NixRepl::getLine(string & input, const std::string &prompt)
 {
-    char * s = linenoise(prompt.c_str());
+    char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
-    if (!s) {
-      switch (auto type = linenoiseKeyType()) {
-        case 1: // ctrl-C
-          input = "";
-          return true;
-        case 2: // ctrl-D
-          return false;
-        default:
-          throw Error(format("Unexpected linenoise keytype: %1%") % type);
-      }
-    }
+    if (!s)
+      return false;
     input += s;
     input += '\n';
     return true;

From 9f998096d23eab433f47d8dc1ad5eb1e2ff4b667 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 26 Jun 2018 13:06:46 -0500
Subject: [PATCH 1267/2196] repl: complete if all matches share prefix

---
 src/nix/repl.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 7ce37077021..838aa64ae29 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -132,6 +132,24 @@ static char * completionCallback(char * s, int *match) {
     auto *res = strdup(possible.begin()->c_str() + strlen(s));
     if (!res) throw Error("allocation failure");
     return res;
+  } else if (possible.size() > 1) {
+    auto checkAllHaveSameAt = [&](size_t pos) {
+      auto &first = *possible.begin();
+      for (auto &p : possible) {
+        if (p.size() <= pos || p[pos] != first[pos])
+          return false;
+      }
+      return true;
+    };
+    size_t start = strlen(s);
+    size_t len = 0;
+    while (checkAllHaveSameAt(start + len)) ++len;
+    if (len > 0) {
+      *match = 1;
+      auto *res = strdup(std::string(*possible.begin(), start, len).c_str());
+      if (!res) throw Error("allocation failure");
+      return res;
+    }
   }
 
   *match = 0;

From 2a8bdfd31a1711b19bbf19ee3b1bed15b724c007 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 29 Oct 2018 08:47:46 -0500
Subject: [PATCH 1268/2196] editline: 1.15.3 -> 1.16.0

Bump fallback editline expression to latest in nixpkgs.
---
 editline.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/editline.nix b/editline.nix
index d9b5a2b64d6..dc44c7d8705 100644
--- a/editline.nix
+++ b/editline.nix
@@ -2,12 +2,12 @@
 
 stdenv.mkDerivation rec {
   name = "editline-${version}";
-  version = "1.15.3";
+  version = "1.16.0";
   src = fetchFromGitHub {
     owner = "troglobit";
     repo = "editline";
     rev = version;
-    sha256 = "0dm5fgq0acpprr938idwml64nclg9l6c6avirsd8r6f40qicbgma";
+    sha256 = "0a751dp34mk9hwv59ss447csknpm5i5cgd607m3fqf24rszyhbf2";
   };
 
   nativeBuildInputs = [ autoreconfHook ];

From 49e272f6472360ba952aa4fa62cc8e61a1b4d6b7 Mon Sep 17 00:00:00 2001
From: Falco Peijnenburg <fpeijnenburg@gmail.com>
Date: Sat, 27 Oct 2018 18:12:02 +0200
Subject: [PATCH 1269/2196] copyStorePath: Fix hash errors when copying from
 older store

This commit partially reverts 48662d151bdf4a38670897beacea9d1bd750376a. When
copying from an older store (in my case a store running Nix 1.11.7), nix would
throw errors about there being no hash. This is fixed by recalculating the hash.
---
 src/libstore/store-api.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 92e2685f7f6..dc54c735fdb 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -588,15 +588,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
-    // FIXME
-#if 0
     if (!info->narHash) {
+        StringSink sink;
+        srcStore->narFromPath({storePath}, sink);
         auto info2 = make_ref<ValidPathInfo>(*info);
         info2->narHash = hashString(htSHA256, *sink.s);
         if (!info->narSize) info2->narSize = sink.s->size();
+        if (info->ultimate) info2->ultimate = false;
         info = info2;
+
+        StringSource source(*sink.s);
+        dstStore->addToStore(*info, source, repair, checkSigs);
+        return;
     }
-#endif
 
     if (info->ultimate) {
         auto info2 = make_ref<ValidPathInfo>(*info);

From 0163e8928c624251456adacb669a94a4adf230ff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 Oct 2018 11:45:31 +0100
Subject: [PATCH 1270/2196] Fix broken uploadProgressCallback closure

Since the callback is global we can't refer to 'path' in it. This
could cause a segfault or printing of arbitrary data.
---
 src/libstore/s3-binary-cache-store.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ba11ce6bb6d..13ee257ba25 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -269,14 +269,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             transferConfig.bufferSize = bufferSize;
 
             transferConfig.uploadProgressCallback =
-                [&](const TransferManager *transferManager,
-                    const std::shared_ptr<const TransferHandle>
-                    &transferHandle)
+                [](const TransferManager *transferManager,
+                   const std::shared_ptr<const TransferHandle>
+                   &transferHandle)
                 {
                     //FIXME: find a way to properly abort the multipart upload.
                     //checkInterrupt();
                     debug("upload progress ('%s'): '%d' of '%d' bytes",
-                        path,
+                        transferHandle->GetKey(),
                         transferHandle->GetBytesTransferred(),
                         transferHandle->GetBytesTotalSize());
                 };

From 9f99d62480cf7c58c0a110b180f2096b7d25adab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 Oct 2018 14:25:00 +0100
Subject: [PATCH 1271/2196] S3BinaryCacheStore: Allow disabling multipart
 uploads

The use of TransferManager has several issues, including that it
doesn't allow setting a Content-Encoding without a patch, and it
doesn't handle exceptions in worker threads (causing termination on
memory allocation failure).

Fixes #2493.
---
 src/libstore/s3-binary-cache-store.cc | 88 +++++++++++++++++----------
 1 file changed, 57 insertions(+), 31 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 13ee257ba25..c5c6b89b197 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -173,6 +173,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
     const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    const Setting<bool> multipartUpload{
+        this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
         this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
@@ -261,46 +263,70 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
             executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
 
-        std::call_once(transferManagerCreated, [&]() {
+        std::call_once(transferManagerCreated, [&]()
+        {
+            if (multipartUpload) {
+                TransferManagerConfiguration transferConfig(executor.get());
+
+                transferConfig.s3Client = s3Helper.client;
+                transferConfig.bufferSize = bufferSize;
+
+                transferConfig.uploadProgressCallback =
+                    [](const TransferManager *transferManager,
+                        const std::shared_ptr<const TransferHandle>
+                        &transferHandle)
+                    {
+                        //FIXME: find a way to properly abort the multipart upload.
+                        //checkInterrupt();
+                        debug("upload progress ('%s'): '%d' of '%d' bytes",
+                            transferHandle->GetKey(),
+                            transferHandle->GetBytesTransferred(),
+                            transferHandle->GetBytesTotalSize());
+                    };
+
+                transferManager = TransferManager::Create(transferConfig);
+            }
+        });
 
-            TransferManagerConfiguration transferConfig(executor.get());
+        auto now1 = std::chrono::steady_clock::now();
 
-            transferConfig.s3Client = s3Helper.client;
-            transferConfig.bufferSize = bufferSize;
+        if (transferManager) {
 
-            transferConfig.uploadProgressCallback =
-                [](const TransferManager *transferManager,
-                   const std::shared_ptr<const TransferHandle>
-                   &transferHandle)
-                {
-                    //FIXME: find a way to properly abort the multipart upload.
-                    //checkInterrupt();
-                    debug("upload progress ('%s'): '%d' of '%d' bytes",
-                        transferHandle->GetKey(),
-                        transferHandle->GetBytesTransferred(),
-                        transferHandle->GetBytesTotalSize());
-                };
+            std::shared_ptr<TransferHandle> transferHandle =
+                transferManager->UploadFile(
+                    stream, bucketName, path, mimeType,
+                    Aws::Map<Aws::String, Aws::String>(),
+                    nullptr, contentEncoding);
 
-            transferManager = TransferManager::Create(transferConfig);
-        });
+            transferHandle->WaitUntilFinished();
 
-        auto now1 = std::chrono::steady_clock::now();
+            if (transferHandle->GetStatus() == TransferStatus::FAILED)
+                throw Error("AWS error: failed to upload 's3://%s/%s': %s",
+                    bucketName, path, transferHandle->GetLastError().GetMessage());
 
-        std::shared_ptr<TransferHandle> transferHandle =
-            transferManager->UploadFile(
-                stream, bucketName, path, mimeType,
-                Aws::Map<Aws::String, Aws::String>(),
-                nullptr, contentEncoding);
+            if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
+                throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
+                    bucketName, path);
 
-        transferHandle->WaitUntilFinished();
+        } else {
 
-        if (transferHandle->GetStatus() == TransferStatus::FAILED)
-            throw Error("AWS error: failed to upload 's3://%s/%s': %s",
-                bucketName, path, transferHandle->GetLastError().GetMessage());
+            auto request =
+                Aws::S3::Model::PutObjectRequest()
+                .WithBucket(bucketName)
+                .WithKey(path);
 
-        if (transferHandle->GetStatus() != TransferStatus::COMPLETED)
-            throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
-                bucketName, path);
+            request.SetContentType(mimeType);
+
+            if (contentEncoding != "")
+                request.SetContentEncoding(contentEncoding);
+
+            auto stream = std::make_shared<istringstream_nocopy>(data);
+
+            request.SetBody(stream);
+
+            auto result = checkAws(fmt("AWS error uploading '%s'", path),
+                s3Helper.client->PutObject(request));
+        }
 
         printTalkative("upload of '%s' completed", path);
 

From 38d76d6d78e691c5ede8b929cf41ffe849349a8e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 30 Oct 2018 19:28:42 +0100
Subject: [PATCH 1272/2196] Remove redundant message

---
 src/libstore/s3-binary-cache-store.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index c5c6b89b197..1f755ba9eee 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -328,8 +328,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 s3Helper.client->PutObject(request));
         }
 
-        printTalkative("upload of '%s' completed", path);
-
         auto now2 = std::chrono::steady_clock::now();
 
         auto duration =

From f3b8173a93a3571046ef6765afe1a15efc1e0de6 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Thu, 25 Oct 2018 13:00:21 +0200
Subject: [PATCH 1273/2196] config: use all of XDG_CONFIG_DIRS

Previously, config would only be read from XDG_CONFIG_HOME. This change
allows reading config from additional directories, which enables e.g.
per-project binary caches or chroot stores with the help of direnv.
---
 src/libstore/globals.cc | 6 +++++-
 src/libutil/util.cc     | 9 +++++++++
 src/libutil/util.hh     | 3 +++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index a9c07b23a6f..1c2c08715a1 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -78,7 +78,11 @@ void loadConfFile()
        ~/.nix/nix.conf or the command line. */
     globalConfig.resetOverriden();
 
-    globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf");
+    auto dirs = getConfigDirs();
+    // Iterate over them in reverse so that the ones appearing first in the path take priority
+    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
+        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+    }
 }
 
 unsigned int Settings::getDefaultCores()
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 03f0be705c1..259eaf0a0dd 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -496,6 +496,15 @@ Path getConfigDir()
     return configDir;
 }
 
+std::vector<Path> getConfigDirs()
+{
+    Path configHome = getConfigDir();
+    string configDirs = getEnv("XDG_CONFIG_DIRS");
+    std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":");
+    result.insert(result.begin(), configHome);
+    return result;
+}
+
 
 Path getDataDir()
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fc25d27758c..bda87bee433 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -131,6 +131,9 @@ Path getCacheDir();
 /* Return $XDG_CONFIG_HOME or $HOME/.config. */
 Path getConfigDir();
 
+/* Return the directories to search for user configuration files */
+std::vector<Path> getConfigDirs();
+
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 

From d1b049c4ea795ba7c1d1f1640ccd087e538b9c79 Mon Sep 17 00:00:00 2001
From: Jan Path <jan@jpath.de>
Date: Wed, 31 Oct 2018 20:50:18 +0100
Subject: [PATCH 1274/2196] Fix typo in comments

---
 src/libexpr/get-drvs.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 4d9128e3f44..daaa635fe1b 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -44,7 +44,7 @@ public:
     string queryDrvPath() const;
     string queryOutPath() const;
     string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */
+    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
     Outputs queryOutputs(bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();

From 6323b0729a27f312ba8cce6279de68181644823e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 1 Nov 2018 15:17:35 +0100
Subject: [PATCH 1275/2196] Disable the S3 content-encoding patch

Since we're not using multi-part uploads at the moment, we can drop
this patch.
---
 release-common.nix                    | 2 ++
 shell.nix                             | 2 +-
 src/libstore/s3-binary-cache-store.cc | 5 ++++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index ace2a4f9b91..9f584993daa 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -64,10 +64,12 @@ rec {
         apis = ["s3" "transfer"];
         customMemoryManagement = false;
       }).overrideDerivation (args: {
+        /*
         patches = args.patches or [] ++ [ (fetchpatch {
           url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
           sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
         }) ];
+        */
       }));
 
   perlDeps =
diff --git a/shell.nix b/shell.nix
index c04bcd15130..817684b7646 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 1f755ba9eee..4f1e23198ff 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -292,11 +292,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
         if (transferManager) {
 
+            if (contentEncoding != "")
+                throw Error("setting a content encoding is not supported with S3 multi-part uploads");
+
             std::shared_ptr<TransferHandle> transferHandle =
                 transferManager->UploadFile(
                     stream, bucketName, path, mimeType,
                     Aws::Map<Aws::String, Aws::String>(),
-                    nullptr, contentEncoding);
+                    nullptr /*, contentEncoding */);
 
             transferHandle->WaitUntilFinished();
 

From 5a1a870849d47e198d3690e4befae9b7cd79f098 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 5 Nov 2018 23:28:32 +0100
Subject: [PATCH 1276/2196] Bump version

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 616187889b6..c0943d3e98d 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.2
\ No newline at end of file
+2.3
\ No newline at end of file

From 1109193ea3a25208d438c65b2d79e207ae1af039 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Nov 2018 11:37:43 +0100
Subject: [PATCH 1277/2196] Fix preferLocalBuild description

preferLocalBuild does not in fact prevent substitution.
---
 doc/manual/expressions/advanced-attributes.xml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 2af7a51acfb..db981b60713 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -312,9 +312,7 @@ big = "a very long string";
   <varlistentry><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
-    <literal>true</literal>, it has two effects.  First, the
-    derivation will always be built, not substituted, even if a
-    substitute is available.  Second, if <link
+    <literal>true</literal> and <link
     linkend="chap-distributed-builds">distributed building is
     enabled</link>, then, if possible, the derivaton will be built
     locally instead of forwarded to a remote machine.  This is

From 5a3f140856185ae0c6ee9270ad5d5fbc0505e3f2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Nov 2018 11:42:22 +0100
Subject: [PATCH 1278/2196] Document allowSubstitutes

---
 doc/manual/expressions/advanced-attributes.xml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index db981b60713..a9b97b91a0a 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -322,6 +322,20 @@ big = "a very long string";
 
   </varlistentry>
 
+
+  <varlistentry><term><varname>allowSubstitutes</varname></term>
+
+    <listitem><para>If this attribute is set to
+    <literal>false</literal>, then Nix will always build this
+    derivation; it will not try to substitute its outputs. This is
+    useful for very trivial derivations (such as
+    <function>writeText</function> in Nixpkgs) that are cheaper to
+    build locally than to substitute from a binary
+    cache.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </section>

From 812e39313c2bcf8909b83e1e8bc548a85dcd626c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 7 Nov 2018 17:08:28 +0100
Subject: [PATCH 1279/2196] Enable sandboxing by default

Closes #179.
---
 doc/manual/release-notes/release-notes.xml |  1 +
 doc/manual/release-notes/rl-2.3.xml        | 19 +++++++++++++++++++
 src/libstore/globals.hh                    |  8 +++++++-
 tests/common.sh.in                         | 15 ++++++---------
 tests/init.sh                              |  1 +
 5 files changed, 34 insertions(+), 10 deletions(-)
 create mode 100644 doc/manual/release-notes/rl-2.3.xml

diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index e8ff586fa43..2655d68e354 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.3.xml" />
 <xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
new file mode 100644
index 00000000000..6b68fbfd7e7
--- /dev/null
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -0,0 +1,19 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.3">
+
+<title>Release 2.3 (2019-??-??)</title>
+
+<para>This release has the following changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Sandbox builds are now enabled by default on Linux.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 6b3e204536f..53efc6a90fb 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -195,7 +195,13 @@ public:
     Setting<bool> showTrace{this, false, "show-trace",
         "Whether to show a stack trace on evaluation errors."};
 
-    Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox",
+    Setting<SandboxMode> sandboxMode{this,
+        #if __linux__
+          smEnabled
+        #else
+          smDisabled
+        #endif
+        , "sandbox",
         "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
         {"build-use-chroot", "build-use-sandbox"}};
 
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 2ee2f589dae..6a523ca9d83 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -85,16 +85,13 @@ killDaemon() {
     trap "" EXIT
 }
 
-canUseSandbox() {
-    if [[ $(uname) != Linux ]]; then return 1; fi
-
-    if [ ! -L /proc/self/ns/user ]; then
-        echo "Kernel doesn't support user namespaces, skipping this test..."
-        return 1
-    fi
+if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
+    _canUseSandbox=1
+fi
 
-    if ! unshare --user true ; then
-        echo "Unprivileged user namespaces disabled by sysctl, skipping this test..."
+canUseSandbox() {
+    if [[ ! $_canUseSandbox ]]; then
+        echo "Sandboxing not supported, skipping this test..."
         return 1
     fi
 
diff --git a/tests/init.sh b/tests/init.sh
index e5353598bcc..19a12c1e2d9 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -16,6 +16,7 @@ mkdir "$NIX_CONF_DIR"
 cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
+sandbox = false
 include nix.conf.extra
 EOF
 

From 0bea4a50e035c06805ac65360b1b50d3681e4f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 8 Nov 2018 12:03:50 +0000
Subject: [PATCH 1280/2196] Deprecate builtins.toPath

---
 doc/manual/expressions/builtins.xml | 29 ++++++-----------------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 873f30b062e..3f396ac1d99 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1027,22 +1027,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   <replaceable>path</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if the path
-    <replaceable>path</replaceable> exists, and
-    <literal>false</literal> otherwise.  One application of this
-    function is to conditionally include a Nix expression containing
-    user configuration:
-
-<programlisting>
-let
-  fileName = builtins.getEnv "CONFIG_FILE";
-  config =
-    if fileName != "" &amp;&amp; builtins.pathExists (builtins.toPath fileName)
-    then import (builtins.toPath fileName)
-    else { someSetting = false; }; <lineannotation># default configuration</lineannotation>
-in config.someSetting</programlisting>
-
-    (Note that <envar>CONFIG_FILE</envar> must be an absolute path for
-    this to work.)</para></listitem>
+    <replaceable>path</replaceable> exists at evaluation time, and
+    <literal>false</literal> otherwise.
 
   </varlistentry>
 
@@ -1344,13 +1330,10 @@ in foo</programlisting>
 
   <varlistentry><term><function>builtins.toPath</function> <replaceable>s</replaceable></term>
 
-    <listitem><para>Convert the string value
-    <replaceable>s</replaceable> into a path value.  The string
-    <replaceable>s</replaceable> must represent an absolute path
-    (i.e., must start with <literal>/</literal>).  The path need not
-    exist.  The resulting path is canonicalised, e.g.,
-    <literal>builtins.toPath "//foo/xyzzy/../bar/"</literal> returns
-    <literal>/foo/bar</literal>.</para></listitem>
+    <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal>
+    to convert a string into an absolute path. For relative paths,
+    use <literal>./. + "/path"</literal>.
+    </para></listitem>
 
   </varlistentry>
 

From 34d2948f219afdecb41ffd080f02f6ccc0a7bb7c Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Thu, 8 Nov 2018 14:07:19 +0100
Subject: [PATCH 1281/2196] Fix manual build

This was broken by some missing closing tags in 0bea4a50e03
---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 4a20ffd81be..03a60a3c4c7 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1078,7 +1078,7 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
     <listitem><para>Return <literal>true</literal> if the path
     <replaceable>path</replaceable> exists at evaluation time, and
-    <literal>false</literal> otherwise.
+    <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
 

From fdd19fa2d736bb8ff0e75b963071dfb9dffe1702 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Nov 2018 10:26:39 +0100
Subject: [PATCH 1282/2196] Revert "Bump version"

This reverts commit 5a1a870849d47e198d3690e4befae9b7cd79f098. Counting
is hard.
---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index c0943d3e98d..616187889b6 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.3
\ No newline at end of file
+2.2
\ No newline at end of file

From 4ea4d0b1a39deb0f12a66e10e4809f2d0ddb324e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Nov 2018 10:29:16 +0100
Subject: [PATCH 1283/2196] Urgh

---
 doc/manual/release-notes/release-notes.xml |  1 -
 doc/manual/release-notes/rl-2.2.xml        |  4 ++++
 doc/manual/release-notes/rl-2.3.xml        | 19 -------------------
 3 files changed, 4 insertions(+), 20 deletions(-)
 delete mode 100644 doc/manual/release-notes/rl-2.3.xml

diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index 2655d68e354..e8ff586fa43 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,7 +12,6 @@
 </partintro>
 -->
 
-<xi:include href="rl-2.3.xml" />
 <xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml
index bc28a56c940..abe9b49adad 100644
--- a/doc/manual/release-notes/rl-2.2.xml
+++ b/doc/manual/release-notes/rl-2.2.xml
@@ -19,6 +19,10 @@
     </para>
   </listitem>
 
+  <listitem>
+    <para>Sandbox builds are now enabled by default on Linux.</para>
+  </listitem>
+
 </itemizedlist>
 
 </section>
diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
deleted file mode 100644
index 6b68fbfd7e7..00000000000
--- a/doc/manual/release-notes/rl-2.3.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
-      xmlns:xlink="http://www.w3.org/1999/xlink"
-      xmlns:xi="http://www.w3.org/2001/XInclude"
-      version="5.0"
-      xml:id="ssec-relnotes-2.3">
-
-<title>Release 2.3 (2019-??-??)</title>
-
-<para>This release has the following changes:</para>
-
-<itemizedlist>
-
-  <listitem>
-    <para>Sandbox builds are now enabled by default on Linux.</para>
-  </listitem>
-
-</itemizedlist>
-
-</section>

From 77516166450008de8c64bb04c67f9b890ddfe38d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Nov 2018 10:34:12 +0100
Subject: [PATCH 1284/2196] nix-prefetch-url: Stop progress bar before printing
 results

---
 src/nix-prefetch-url/nix-prefetch-url.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 54a402241e3..f54706a8a01 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -220,6 +220,8 @@ static int _main(int argc, char * * argv)
             assert(storePath == store->makeFixedOutputPath(unpack, hash, name));
         }
 
+        stopProgressBar();
+
         if (!printPath)
             printInfo(format("path is '%1%'") % storePath);
 

From b05d6dac7ba662ccf7af8b5a112a20f460e0d50e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 9 Nov 2018 16:08:36 +0100
Subject: [PATCH 1285/2196] Manual: build locally -> build

---
 doc/manual/expressions/advanced-attributes.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index a9b97b91a0a..07b0d97d3f7 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -330,8 +330,7 @@ big = "a very long string";
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as
     <function>writeText</function> in Nixpkgs) that are cheaper to
-    build locally than to substitute from a binary
-    cache.</para></listitem>
+    build than to substitute from a binary cache.</para></listitem>
 
   </varlistentry>
 

From 18215be59d39741dae96a4a3be6d364b52c10e9b Mon Sep 17 00:00:00 2001
From: "Felix C. Stegerman" <flx@obfusk.net>
Date: Mon, 12 Nov 2018 17:50:39 +0100
Subject: [PATCH 1286/2196] fix typo (s/gift/git/)

---
 doc/manual/expressions/builtins.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 03a60a3c4c7..cdadfef707b 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -436,7 +436,7 @@ stdenv.mkDerivation { … }
         <title>Fetching a repository's specific commit on an arbitrary branch</title>
         <para>
           If the revision you're looking for is in the default branch
-          of the gift repository you don't strictly need to specify
+          of the git repository you don't strictly need to specify
           the branch name in the <varname>ref</varname> attribute.
         </para>
         <para>
@@ -465,7 +465,7 @@ stdenv.mkDerivation { … }
         <title>Fetching a repository's specific commit on the default branch</title>
         <para>
           If the revision you're looking for is in the default branch
-          of the gift repository you may omit the
+          of the git repository you may omit the
           <varname>ref</varname> attribute.
         </para>
         <programlisting>builtins.fetchGit {

From fb2c21f71c8a31229740cdbff5b2d08a1752133a Mon Sep 17 00:00:00 2001
From: "Felix C. Stegerman" <flx@obfusk.net>
Date: Tue, 13 Nov 2018 01:18:26 +0100
Subject: [PATCH 1287/2196] manual: quote $servlets

---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index cdadfef707b..412622714d4 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1480,7 +1480,7 @@ stdenv.mkDerivation (rec {
   builder = builtins.toFile "builder.sh" "
     source $stdenv/setup
     mkdir $out
-    echo $servlets | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[
+    echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[
   ";
 
   stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[

From a0ef21262f4d5652bfb65cfacaec01d89c475a93 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Nov 2018 16:15:30 +0100
Subject: [PATCH 1288/2196] Restore parent mount namespace before executing a
 child process

This ensures that they can't write to /nix/store. Fixes #2535.
---
 src/libstore/local-store.cc |  2 ++
 src/libstore/ssh.cc         |  3 +++
 src/libutil/util.cc         | 23 +++++++++++++++++++++++
 src/libutil/util.hh         |  9 +++++++++
 src/nix-build/nix-build.cc  |  4 ++--
 src/nix/edit.cc             |  5 +++++
 src/nix/repl.cc             |  2 ++
 src/nix/run.cc              |  4 ++--
 8 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 216f3417c4a..e1cb423d151 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -366,6 +366,8 @@ void LocalStore::makeStoreWritable()
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
+        saveMountNamespace();
+
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 5e0e44935cc..cf133b57cb2 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -1,4 +1,5 @@
 #include "ssh.hh"
+#include "affinity.hh"
 
 namespace nix {
 
@@ -34,7 +35,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
     auto conn = std::make_unique<Connection>();
     conn->sshPid = startProcess([&]() {
+        restoreAffinity();
         restoreSignals();
+        restoreMountNamespace();
 
         close(in.writeSide.get());
         close(out.readSide.get());
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 259eaf0a0dd..6e4536e6e4e 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -936,6 +936,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
                 throw SysError("setting death signal");
 #endif
             restoreAffinity();
+            restoreMountNamespace();
             fun();
         } catch (std::exception & e) {
             try {
@@ -1504,4 +1505,26 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
     return std::unique_ptr<InterruptCallback>(res.release());
 }
 
+static AutoCloseFD fdSavedMountNamespace;
+
+void saveMountNamespace()
+{
+#if __linux__
+    std::once_flag done;
+    std::call_once(done, []() {
+        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
+        if (!fdSavedMountNamespace)
+            throw SysError("saving parent mount namespace");
+    });
+#endif
+}
+
+void restoreMountNamespace()
+{
+#if __linux__
+    if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
+        throw SysError("restoring parent mount namespace");
+#endif
+}
+
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index bda87bee433..2689cbd8b41 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -514,4 +514,13 @@ typedef std::function<bool(const Path & path)> PathFilter;
 extern PathFilter defaultPathFilter;
 
 
+/* Save the current mount namespace. Ignored if called more than
+   once. */
+void saveMountNamespace();
+
+/* Restore the mount namespace saved by saveMountNamespace(). Ignored
+   if saveMountNamespace() was never called. */
+void restoreMountNamespace();
+
+
 }
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 618895d387d..11ea3b1f7ae 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -401,8 +401,6 @@ static void _main(int argc, char * * argv)
             } else
                 env[var.first] = var.second;
 
-        restoreAffinity();
-
         /* Run a shell using the derivation's environment.  For
            convenience, source $stdenv/setup to setup additional
            environment variables and shell functions.  Also don't
@@ -446,7 +444,9 @@ static void _main(int argc, char * * argv)
 
         auto argPtrs = stringsToCharPtrs(args);
 
+        restoreAffinity();
         restoreSignals();
+        restoreMountNamespace();
 
         execvp(shell.c_str(), argPtrs.data());
 
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index c9671f76d0f..d8d5895bd86 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -3,6 +3,7 @@
 #include "eval.hh"
 #include "attr-path.hh"
 #include "progress-bar.hh"
+#include "affinity.hh"
 
 #include <unistd.h>
 
@@ -72,6 +73,10 @@ struct CmdEdit : InstallableCommand
 
         stopProgressBar();
 
+        restoreAffinity();
+        restoreSignals();
+        restoreMountNamespace();
+
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("cannot run editor '%s'", editor);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 1bbe256b2d8..77898c63236 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -285,6 +285,8 @@ static int runProgram(const string & program, const Strings & args)
     if (pid == -1) throw SysError("forking");
     if (pid == 0) {
         restoreAffinity();
+        restoreSignals();
+        restoreMountNamespace();
         execvp(program.c_str(), stringsToCharPtrs(args2).data());
         _exit(1);
     }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 35b76334587..1297072989b 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -153,9 +153,9 @@ struct CmdRun : InstallablesCommand
 
         stopProgressBar();
 
-        restoreSignals();
-
         restoreAffinity();
+        restoreSignals();
+        restoreMountNamespace();
 
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location

From 9dc9b64aadadd9f3b8277fb942b00dad1a753fd3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 13 Nov 2018 20:55:08 +0100
Subject: [PATCH 1289/2196] Remove editline expression, not needed anymore

---
 editline.nix       | 24 ------------------------
 release-common.nix |  2 --
 2 files changed, 26 deletions(-)
 delete mode 100644 editline.nix

diff --git a/editline.nix b/editline.nix
deleted file mode 100644
index dc44c7d8705..00000000000
--- a/editline.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-{ stdenv, fetchFromGitHub, autoreconfHook }:
-
-stdenv.mkDerivation rec {
-  name = "editline-${version}";
-  version = "1.16.0";
-  src = fetchFromGitHub {
-    owner = "troglobit";
-    repo = "editline";
-    rev = version;
-    sha256 = "0a751dp34mk9hwv59ss447csknpm5i5cgd607m3fqf24rszyhbf2";
-  };
-
-  nativeBuildInputs = [ autoreconfHook ];
-
-  dontDisableStatic = true;
-
-  meta = with stdenv.lib; {
-    homepage = http://troglobit.com/editline.html;
-    description = "A readline() replacement for UNIX without termcap (ncurses)";
-    license = licenses.bsdOriginal;
-    maintainers = with maintainers; [ dtzWill ];
-    platforms = platforms.all;
-  };
-}
diff --git a/release-common.nix b/release-common.nix
index f495df1440d..4c556598526 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -29,8 +29,6 @@ rec {
     '';
   });
 
-  editline = pkgs.editline or (pkgs.callPackage ./editline.nix {});
-
   configureFlags =
     [
       "--enable-gc"

From a32ff2573ba4d0df4e3360c5a96398738da953f6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 15 Nov 2018 12:59:54 +0100
Subject: [PATCH 1290/2196] Fix 'Read-only file system' when building a
 derivation

---
 src/libstore/build.cc | 2 ++
 src/libutil/util.cc   | 3 ++-
 src/libutil/util.hh   | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 676ad5856b1..9c408e29c06 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2193,6 +2193,7 @@ void DerivationGoal::startBuilder()
         userNamespaceSync.create();
 
         options.allowVfork = false;
+        options.restoreMountNamespace = false;
 
         Pid helper = startProcess([&]() {
 
@@ -2259,6 +2260,7 @@ void DerivationGoal::startBuilder()
 #endif
     {
         options.allowVfork = !buildUser && !drv->isBuiltin();
+        options.restoreMountNamespace = false;
         pid = startProcess([&]() {
             runChild();
         }, options);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6e4536e6e4e..e12c4b258c2 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -936,7 +936,8 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
                 throw SysError("setting death signal");
 #endif
             restoreAffinity();
-            restoreMountNamespace();
+            if (options.restoreMountNamespace)
+                restoreMountNamespace();
             fun();
         } catch (std::exception & e) {
             try {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 2689cbd8b41..d67bddc138c 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -250,6 +250,7 @@ struct ProcessOptions
     bool dieWithParent = true;
     bool runExitHandlers = false;
     bool allowVfork = true;
+    bool restoreMountNamespace = true;
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());

From 40e0c9e925e266c1b0944d9810f3a7f3dd6ec544 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Fri, 16 Nov 2018 16:13:28 +0100
Subject: [PATCH 1291/2196] nix ls-nar: allow reading from FIFOs

fixes #2528
---
 src/nix/ls.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index e99622faf47..d089be42fb2 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -148,7 +148,7 @@ struct CmdLsNar : Command, MixLs
 
     void run() override
     {
-        list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
+        list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true))));
     }
 };
 

From b289d86cd1ef5a8df0ce177dfc04b4e5fec3fa14 Mon Sep 17 00:00:00 2001
From: Kai Harries <kai.harries@gmail.com>
Date: Sun, 18 Nov 2018 19:06:35 +0100
Subject: [PATCH 1292/2196] repl: Remove code that was commented out

---
 src/nix/repl.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index ff8477865b5..39c6520999c 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -204,8 +204,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
     createDirs(dirOf(historyFile));
     el_hist_size = 1000;
     read_history(historyFile.c_str());
-    // rl_initialize();
-    // linenoiseSetCompletionCallback(completionCallback);
     curRepl = this;
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);

From de5997332d0f9b1347978ea0d4e7ead2b59d2dd6 Mon Sep 17 00:00:00 2001
From: Kai Harries <kai.harries@gmail.com>
Date: Thu, 15 Nov 2018 21:15:11 +0100
Subject: [PATCH 1293/2196] repl: give user the choice between libeditline and
 libreadline

The goal is to support libeditline AND libreadline and let the user
decide at compile time which one to use.

Add a compile time option to use libreadline instead of
libeditline. If compiled against libreadline completion functionality
is lost because of a incompatibility between libeditlines and
libreadlines completion function. Completion with libreadline is
possible and can be added later.

To use libreadline instead of libeditline the environment
variables 'EDITLINE_LIBS' and 'EDITLINE_CFLAGS' have to been set
during the ./configure step.

Example:

  EDITLINE_LIBS="/usr/lib/x86_64-linux-gnu/libhistory.so /usr/lib/x86_64-linux-gnu/libreadline.so"
  EDITLINE_CFLAGS="-DREADLINE"

The reason for this change is that for example on Debian already three
different editline libraries exist but none of those is compatible the
flavor used by nix. My hope is that with this change it would be
easier to port nix to systems that have already libreadline available.
---
 src/nix/repl.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 39c6520999c..d93fd770e80 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -5,7 +5,12 @@
 
 #include <setjmp.h>
 
+#ifdef READLINE
+#include <readline/history.h>
+#include <readline/readline.h>
+#else
 #include <editline.h>
+#endif
 
 #include "shared.hh"
 #include "eval.hh"
@@ -202,11 +207,15 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
     // Allow nix-repl specific settings in .inputrc
     rl_readline_name = "nix-repl";
     createDirs(dirOf(historyFile));
+#ifndef READLINE
     el_hist_size = 1000;
+#endif
     read_history(historyFile.c_str());
     curRepl = this;
+#ifndef READLINE
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);
+#endif
 
     std::string input;
 

From 5e64470b192271ce60bea2f5c5fcc556227f86a1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Sep 2018 15:18:59 +0200
Subject: [PATCH 1294/2196] Fix typo

---
 src/libmain/stack.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index 13896aeecb6..e6224de7d28 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -63,7 +63,7 @@ void detectStackOverflow()
     act.sa_sigaction = sigsegvHandler;
     act.sa_flags = SA_SIGINFO | SA_ONSTACK;
     if (sigaction(SIGSEGV, &act, 0))
-        throw SysError("resetting SIGCHLD");
+        throw SysError("resetting SIGSEGV");
 #endif
 }
 

From 4aee93d5ce6cf77e314e93074b9da1dcff8979e9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 20 Nov 2018 20:59:44 +0100
Subject: [PATCH 1295/2196] fetchGit: Drop unnecessary localRef

---
 src/libexpr/primops/fetchGit.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 5d6249e565d..b46d2f25826 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -85,6 +85,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     if (rev != "" && !std::regex_match(rev, revRegex))
         throw Error("invalid Git revision '%s'", rev);
 
+    deletePath(getCacheDir() + "/nix/git");
+
     Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
 
     if (!pathExists(cacheDir)) {
@@ -92,9 +94,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false);
-
-    Path localRefFile = cacheDir + "/refs/heads/" + localRef;
+    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);
@@ -124,7 +124,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         // FIXME: git stderr messes up our progress indicator, so
         // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef });
+        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
 
         struct timeval times[2];
         times[0].tv_sec = now;

From fa5143c722dab749de9305dbaa85d618b5758440 Mon Sep 17 00:00:00 2001
From: CHEIKH Chawki <chawki.cheikh008@gmail.com>
Date: Tue, 4 Dec 2018 16:14:23 +0100
Subject: [PATCH 1296/2196] Solve hg "abandoned transaction" issue

---
 src/libexpr/primops/fetchMercurial.cc | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 97cda2458c9..66f49f37432 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -93,7 +93,22 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
 
             if (pathExists(cacheDir)) {
-                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                }
+                catch (ExecError & e){
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal))
+                    {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
+                    }
+                    else 
+                    {
+                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
+                    }
+                }
             } else {
                 createDirs(dirOf(cacheDir));
                 runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });

From 898823b67d4d9ceeaebf166957141706eb03ad72 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 7 Dec 2018 23:38:24 +0100
Subject: [PATCH 1297/2196] s3: make scheme configurable

This enables using for http for S3 request for debugging or
implementations that don't have https configured.  This is not a problem
for binary caches since they should not contain sensitive information.
Both package signatures and AWS auth already protect against tampering.
---
 src/libstore/download.cc              |  2 +-
 src/libstore/s3-binary-cache-store.cc | 12 ++++++++----
 src/libstore/s3.hh                    |  4 ++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 7773d903265..fef2cf7a388 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -622,7 +622,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
-                S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 4f1e23198ff..51de89e0d92 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -82,8 +82,8 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
-    : config(makeConfig(region, endpoint))
+S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
+    : config(makeConfig(region, scheme, endpoint))
     , client(make_ref<Aws::S3::S3Client>(
             profile == ""
             ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@@ -114,11 +114,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
     }
 };
 
-ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
 {
     initAWS();
     auto res = make_ref<Aws::Client::ClientConfiguration>();
     res->region = region;
+    if (!scheme.empty()) {
+        res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str());
+    }
     if (!endpoint.empty()) {
         res->endpointOverride = endpoint;
     }
@@ -169,6 +172,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
     const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
@@ -188,7 +192,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(profile, region, endpoint)
+        , s3Helper(profile, region, scheme, endpoint)
     {
         diskCache = getNarInfoDiskCache();
     }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 95d612b6633..ef5f23d0f25 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,9 +14,9 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
+    S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
+    ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
     struct DownloadResult
     {

From 77cc632186d6a0bb63c1e16238f628857bc6e7bd Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 7 Dec 2018 23:48:35 +0100
Subject: [PATCH 1298/2196] s3: document scheme query parameter

---
 doc/manual/packages/s3-substituter.xml | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index ea654392c6b..2ec9687a0c6 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -51,6 +51,18 @@ the S3 URL:</para>
     addressing.</para></note>
   </listitem>
   </varlistentry>
+
+  <varlistentry><term><literal>scheme</literal></term>
+  <listitem>
+    <para>
+      The scheme used for S3 requests, <literal>https</literal>
+      (default) or <literal>http</literal>.  This option allows you to
+      disable HTTPS for binary caches which don't support it.
+    </para>
+    <note><para>HTTPS should be used if the cache might contain
+    sensitive information.</para></note>
+  </listitem>
+  </varlistentry>
 </variablelist>
 
 <para>In this example we will use the bucket named
@@ -165,7 +177,7 @@ the S3 URL:</para>
   </example>
 
   <example><title>Uploading to an S3-Compatible Binary Cache</title>
-    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
+    <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
   </example>
 </section>
 </section>

From 419949bf617a73e92f6ebafbaf5288e3cf4c9f27 Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Sat, 8 Dec 2018 17:28:38 -0500
Subject: [PATCH 1299/2196] docs: add missing prerequisites: brotli, boost,
 libseccomp

---
 .../installation/prerequisites-source.xml     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index ef14a1d753d..f1705221663 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -25,6 +25,12 @@
   If your distribution does not provide it, you can get it from <link
   xlink:href="https://www.openssl.org"/>.</para></listitem>
 
+  <listitem><para>The <literal>libbrotlienc</literal> and
+  <literal>libbrotlidec</literal> libraries to provide implementation
+  of the Brotli compression algorithm. They are available for download
+  from the official repository <link
+  xlink:href="https://github.com/google/brotli" />.</para></listitem>
+
   <listitem><para>The bzip2 compressor program and the
   <literal>libbz2</literal> library.  Thus you must have bzip2
   installed, including development headers and libraries.  If your
@@ -52,6 +58,10 @@
   pass the flag <option>--enable-gc</option> to
   <command>configure</command>.</para></listitem>
 
+  <listitem><para>The <literal>boost</literal> library of version
+  1.61.0 or higher. It can be obtained from the official web site
+  <link xlink:href="https://www.boost.org/" />.</para></listitem>
+
   <listitem><para>The <command>xmllint</command> and
   <command>xsltproc</command> programs to build this manual and the
   man-pages.  These are part of the <literal>libxml2</literal> and
@@ -77,6 +87,15 @@
   modify the parser or when you are building from the Git
   repository.</para></listitem>
 
+  <listitem><para>The <literal>libseccomp</literal> is used to provide
+  syscall filtering on Linux. This is an optional dependency and can
+  be disabled passing a <option>--disable-seccomp-sandboxing</option>
+  option to the <command>configure</command> script (Not recommended
+  unless your system doesn't support
+  <literal>libseccomp</literal>). To get the library, visit <link
+  xlink:href="https://github.com/seccomp/libseccomp"
+  />.</para></listitem>
+
 </itemizedlist>
 
 </section>

From 6f890531084ab6596027ce0bf6ad302864affa5b Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 12 Dec 2018 01:04:34 +0100
Subject: [PATCH 1300/2196] nar-info-disk-cache: include ca in the cache
 entries

Without this information the content addressable state and hashes are
lost after the first request, this causes signatures to be required for
everything even tho the path could be verified without signing.
---
 src/libstore/nar-info-disk-cache.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 35403e5df56..5fdd7ce8916 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -31,6 +31,7 @@ create table if not exists NARs (
     refs             text,
     deriver          text,
     sigs             text,
+    ca               text,
     timestamp        integer not null,
     present          integer not null,
     primary key (cache, hashPart),
@@ -72,7 +73,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite";
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
@@ -94,7 +95,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
         state->insertNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
-            "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
+            "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)");
 
         state->insertMissingNAR.create(state->db,
             "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
@@ -210,6 +211,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                 narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
                 narInfo->sigs.insert(sig);
+            narInfo->ca = queryNAR.getStr(12);
 
             return {oValid, narInfo};
         });
@@ -243,6 +245,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
                     (concatStringsSep(" ", info->sigs))
+                    (info->ca)
                     (time(0)).exec();
 
             } else {

From f7425d55df5f5c39e778d437eba94540fee43f4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 12 Dec 2018 17:12:41 +0100
Subject: [PATCH 1301/2196] Fix assertion failure in NarInfoDiskCache

https://hydra.nixos.org/build/85827920
---
 src/libstore/nar-info-disk-cache.cc | 30 ++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 5fdd7ce8916..32ad7f2b27f 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -101,7 +101,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)");
 
         state->queryNAR.create(state->db,
-            "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
+            "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
 
         /* Periodically purge expired entries from the database. */
         retrySQLite<void>([&]() {
@@ -190,28 +190,28 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             if (!queryNAR.next())
                 return {oUnknown, 0};
 
-            if (!queryNAR.getInt(13))
+            if (!queryNAR.getInt(0))
                 return {oInvalid, 0};
 
             auto narInfo = make_ref<NarInfo>();
 
-            auto namePart = queryNAR.getStr(2);
+            auto namePart = queryNAR.getStr(1);
             narInfo->path = cache.storeDir + "/" +
                 hashPart + (namePart.empty() ? "" : "-" + namePart);
-            narInfo->url = queryNAR.getStr(3);
-            narInfo->compression = queryNAR.getStr(4);
-            if (!queryNAR.isNull(5))
-                narInfo->fileHash = Hash(queryNAR.getStr(5));
-            narInfo->fileSize = queryNAR.getInt(6);
-            narInfo->narHash = Hash(queryNAR.getStr(7));
-            narInfo->narSize = queryNAR.getInt(8);
-            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
+            narInfo->url = queryNAR.getStr(2);
+            narInfo->compression = queryNAR.getStr(3);
+            if (!queryNAR.isNull(4))
+                narInfo->fileHash = Hash(queryNAR.getStr(4));
+            narInfo->fileSize = queryNAR.getInt(5);
+            narInfo->narHash = Hash(queryNAR.getStr(6));
+            narInfo->narSize = queryNAR.getInt(7);
+            for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
                 narInfo->references.insert(cache.storeDir + "/" + r);
-            if (!queryNAR.isNull(10))
-                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10);
-            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
+            if (!queryNAR.isNull(9))
+                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
+            for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
-            narInfo->ca = queryNAR.getStr(12);
+            narInfo->ca = queryNAR.getStr(11);
 
             return {oValid, narInfo};
         });

From 5e6fa9092fb5be722f3568c687524416bc746423 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 12 Dec 2018 14:53:00 +0100
Subject: [PATCH 1302/2196] libstore: improve hash mismatch error messages

Use the same output ordering and format everywhere.

This is such a common issue that we trade the single-line error message for
more readability.

Old message:
```
fixed-output derivation produced path '/nix/store/d4nw9x2sy9q3r32f3g5l5h1k833c01vq-example.com' with sha256 hash '08y4734bm2zahw75b16bcmcg587vvyvh0n11gwiyir70divwp1rm' instead of the expected hash '1xzwnipjd54wl8g93vpw6hxnpmdabq0wqywriiwmh7x8k0lvpq5m'
```

New message:
```
hash mismatch in fixed-output derivation '/nix/store/d4nw9x2sy9q3r32f3g5l5h1k833c01vq-example.com':
  wanted: sha256:1xzwnipjd54wl8g93vpw6hxnpmdabq0wqywriiwmh7x8k0lvpq5m
  got:    sha256:08y4734bm2zahw75b16bcmcg587vvyvh0n11gwiyir70divwp1rm

```
---
 src/libstore/build.cc       | 4 ++--
 src/libstore/download.cc    | 4 ++--
 src/libstore/local-store.cc | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9c408e29c06..59abae9b90d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3129,8 +3129,8 @@ void DerivationGoal::registerOutputs()
                 /* Throw an error after registering the path as
                    valid. */
                 delayedException = std::make_exception_ptr(
-                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
-                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
+                    BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
+                        dest, h.to_string(), h2.to_string()));
 
                 Path actualDest = worker.store.toRealPath(dest);
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index fef2cf7a388..467f570bbf0 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -881,8 +881,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         Hash gotHash = unpack
             ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
             : hashFile(expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'",
-            url, gotHash.to_string(), expectedHash.to_string());
+        throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+            url, expectedHash.to_string(), gotHash.to_string());
     }
 
     return store->toRealPath(storePath);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e1cb423d151..5b4e7ca4ca9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1022,11 +1022,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             auto hashResult = hashSink.finish();
 
             if (hashResult.first != info.narHash)
-                throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'",
+                throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
                     info.path, info.narHash.to_string(), hashResult.first.to_string());
 
             if (hashResult.second != info.narSize)
-                throw Error("size mismatch importing path '%s'; expected %s, got %s",
+                throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
                     info.path, info.narSize, hashResult.second);
 
             autoGC();

From 0e6c84a771a2b124a200bfbedc85e51884ea63a6 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 12 Dec 2018 22:59:03 +0100
Subject: [PATCH 1303/2196] nix repl: don't create result symlinks

---
 src/nix/repl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index d93fd770e80..d4806d74adb 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -445,7 +445,7 @@ bool NixRepl::processLine(string line)
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", drvPath}) == 0) {
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
                 Derivation drv = readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)

From 21d494da8304aa496721c7695573b9ad796a30be Mon Sep 17 00:00:00 2001
From: volth <volth@webmaster.ms>
Date: Thu, 13 Dec 2018 02:45:50 +0000
Subject: [PATCH 1304/2196] probably typo

...at least MSVC unable to compile this
---
 src/libutil/util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e12c4b258c2..ce50334e1e6 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -202,7 +202,7 @@ bool isInDir(const Path & path, const Path & dir)
 
 bool isDirOrInDir(const Path & path, const Path & dir)
 {
-    return path == dir or isInDir(path, dir);
+    return path == dir || isInDir(path, dir);
 }
 
 

From 0cc4728f71b1a1dfd32897eeb79d6920cbf3f5aa Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Thu, 13 Dec 2018 01:50:39 -0500
Subject: [PATCH 1305/2196] docs: raise minimal boost version to 1.66

---
 doc/manual/installation/prerequisites-source.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index f1705221663..e87d0de21ef 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -59,7 +59,7 @@
   <command>configure</command>.</para></listitem>
 
   <listitem><para>The <literal>boost</literal> library of version
-  1.61.0 or higher. It can be obtained from the official web site
+  1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>
 
   <listitem><para>The <command>xmllint</command> and

From 6024dc1d97212130c19d3ff5ce6b1d102837eee6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Dec 2018 14:30:52 +0100
Subject: [PATCH 1306/2196] Support SRI hashes

SRI hashes (https://www.w3.org/TR/SRI/) combine the hash algorithm and
a base-64 hash. This allows more concise and standard hash
specifications. For example, instead of

  import <nix/fetchurl.nl> {
    url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
    sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
  };

you can write

  import <nix/fetchurl.nl> {
    url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
    hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
  };

In fixed-output derivations, the outputHashAlgo is no longer mandatory
if outputHash specifies the hash (either as an SRI or in the old
"<type>:<hash>" format).

'nix hash-{file,path}' now print hashes in SRI format by default. I
also reverted them to use SHA-256 by default because that's what we're
using most of the time in Nixpkgs.

Suggested by @zimbatm.
---
 corepkgs/fetchurl.nix      |  8 ++++++--
 src/libexpr/primops.cc     | 10 ++++------
 src/libutil/hash.cc        | 34 ++++++++++++++++++++--------------
 src/libutil/hash.hh        |  7 ++++---
 src/nix-store/nix-store.cc |  3 +++
 src/nix/hash.cc            | 23 ++++++++++++++---------
 tests/fetchurl.sh          | 11 +++++++++++
 tests/hash.sh              | 12 +++++++++++-
 8 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix
index 0ce1bab112f..a84777f5744 100644
--- a/corepkgs/fetchurl.nix
+++ b/corepkgs/fetchurl.nix
@@ -1,10 +1,14 @@
 { system ? "" # obsolete
 , url
+, hash ? "" # an SRI ash
+
+# Legacy hash specification
 , md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
 , outputHash ?
-    if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+    if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
 , outputHashAlgo ?
-    if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+    if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+
 , executable ? false
 , unpack ? false
 , name ? baseNameOf (toString url)
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7372134e2c9..60698f7402e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -724,16 +724,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
-        HashType ht = parseHashType(outputHashAlgo);
-        if (ht == htUnknown)
-            throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName);
+        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
-        outputHash = h.to_string(Base16, false);
-        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
 
         Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
+        drv.outputs["out"] = DerivationOutput(outPath,
+            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+            h.to_string(Base16, false));
     }
 
     else {
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 9d82f13a5e3..1c14ebb187c 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -105,9 +105,9 @@ string printHash16or32(const Hash & hash)
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
-    if (includeType) {
+    if (base == SRI || includeType) {
         s += printHashType(type);
-        s += ':';
+        s += base == SRI ? '-' : ':';
     }
     switch (base) {
     case Base16:
@@ -117,6 +117,7 @@ std::string Hash::to_string(Base base, bool includeType) const
         s += printHash32(*this);
         break;
     case Base64:
+    case SRI:
         s += base64Encode(std::string((const char *) hash, hashSize));
         break;
     }
@@ -127,28 +128,33 @@ std::string Hash::to_string(Base base, bool includeType) const
 Hash::Hash(const std::string & s, HashType type)
     : type(type)
 {
-    auto colon = s.find(':');
-
     size_t pos = 0;
-
-    if (colon == string::npos) {
-        if (type == htUnknown)
+    bool isSRI = false;
+
+    auto sep = s.find(':');
+    if (sep == string::npos) {
+        sep = s.find('-');
+        if (sep != string::npos) {
+            isSRI = true;
+        } else if (type == htUnknown)
             throw BadHash("hash '%s' does not include a type", s);
-    } else {
-        string hts = string(s, 0, colon);
+    }
+
+    if (sep != string::npos) {
+        string hts = string(s, 0, sep);
         this->type = parseHashType(hts);
         if (this->type == htUnknown)
             throw BadHash("unknown hash type '%s'", hts);
         if (type != htUnknown && type != this->type)
             throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
-        pos = colon + 1;
+        pos = sep + 1;
     }
 
     init();
 
     size_t size = s.size() - pos;
 
-    if (size == base16Len()) {
+    if (!isSRI && size == base16Len()) {
 
         auto parseHexDigit = [&](char c) {
             if (c >= '0' && c <= '9') return c - '0';
@@ -164,7 +170,7 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (size == base32Len()) {
+    else if (!isSRI && size == base32Len()) {
 
         for (unsigned int n = 0; n < size; ++n) {
             char c = s[pos + size - n - 1];
@@ -187,10 +193,10 @@ Hash::Hash(const std::string & s, HashType type)
         }
     }
 
-    else if (size == base64Len()) {
+    else if (isSRI || size == base64Len()) {
         auto d = base64Decode(std::string(s, pos));
         if (d.size() != hashSize)
-            throw BadHash("invalid base-64 hash '%s'", s);
+            throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
         assert(hashSize);
         memcpy(hash, d.data(), hashSize);
     }
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index fd7a61df8e4..2dbc3b63081 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -20,7 +20,7 @@ const int sha512HashSize = 64;
 
 extern const string base32Chars;
 
-enum Base : int { Base64, Base32, Base16 };
+enum Base : int { Base64, Base32, Base16, SRI };
 
 
 struct Hash
@@ -38,8 +38,9 @@ struct Hash
     Hash(HashType type) : type(type) { init(); };
 
     /* Initialize the hash from a string representation, in the format
-       "[<type>:]<base16|base32|base64>". If the 'type' argument is
-       htUnknown, then the hash type must be specified in the
+       "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
+       Subresource Integrity hash expression). If the 'type' argument
+       is htUnknown, then the hash type must be specified in the
        string. */
     Hash(const std::string & s, HashType type = htUnknown);
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a9ad14762e6..5b37237eb16 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -1000,6 +1000,9 @@ static int _main(int argc, char * * argv)
         Strings opFlags, opArgs;
         Operation op = 0;
 
+        Hash h("sha512-Q2bFTOhEALkN8hOms2FKTDLy7eugP2zFZ1T8LCvX42Fp3WoNr3bjZSAHeOsHrbV1Fu9/A0EzCinRE7Af1ofPrw==");
+        printError("GOT HASH %s", h.to_string(Base64));
+
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             Operation oldOp = op;
 
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 64062fb9795..af4105e2890 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -9,13 +9,14 @@ struct CmdHash : Command
 {
     enum Mode { mFile, mPath };
     Mode mode;
-    Base base = Base16;
+    Base base = SRI;
     bool truncate = false;
-    HashType ht = htSHA512;
+    HashType ht = htSHA256;
     std::vector<std::string> paths;
 
     CmdHash(Mode mode) : mode(mode)
     {
+        mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base16);
@@ -43,7 +44,7 @@ struct CmdHash : Command
             Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
-                h.to_string(base, false);
+                h.to_string(base, base == SRI);
         }
     }
 };
@@ -54,7 +55,7 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath));
 struct CmdToBase : Command
 {
     Base base;
-    HashType ht = htSHA512;
+    HashType ht = htUnknown;
     std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
@@ -70,26 +71,30 @@ struct CmdToBase : Command
         return
             base == Base16 ? "to-base16" :
             base == Base32 ? "to-base32" :
-            "to-base64";
+            base == Base64 ? "to-base64" :
+            "to-sri";
     }
 
     std::string description() override
     {
-        return fmt("convert a hash to base-%d representation",
-            base == Base16 ? 16 :
-            base == Base32 ? 32 : 64);
+        return fmt("convert a hash to %s representation",
+            base == Base16 ? "base-16" :
+            base == Base32 ? "base-32" :
+            base == Base64 ? "base-64" :
+            "SRI");
     }
 
     void run() override
     {
         for (auto s : args)
-            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false));
+            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI));
     }
 };
 
 static RegisterCommand r3(make_ref<CmdToBase>(Base16));
 static RegisterCommand r4(make_ref<CmdToBase>(Base32));
 static RegisterCommand r5(make_ref<CmdToBase>(Base64));
+static RegisterCommand r6(make_ref<CmdToBase>(SRI));
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 9bbf044f732..d51d081f5e3 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -18,6 +18,17 @@ outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh
 
 cmp $outPath fetchurl.sh
 
+# Now using an SRI hash.
+clearStore
+
+hash=$(nix hash-file ./fetchurl.sh)
+
+[[ $hash =~ ^sha512- ]]
+
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+
+cmp $outPath fetchurl.sh
+
 # Test the hashed mirror feature.
 clearStore
 
diff --git a/tests/hash.sh b/tests/hash.sh
index 9f234bc635b..4cfc9790101 100644
--- a/tests/hash.sh
+++ b/tests/hash.sh
@@ -2,7 +2,7 @@ source common.sh
 
 try () {
     printf "%s" "$2" > $TEST_ROOT/vector
-    hash=$(nix-hash $EXTRA --flat --type "$1" $TEST_ROOT/vector)
+    hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector)
     if test "$hash" != "$3"; then
         echo "hash $1, expected $3, got $hash"
         exit 1
@@ -33,6 +33,12 @@ EXTRA=--base32
 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
 EXTRA=
 
+EXTRA=--sri
+try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
+try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw=="
+try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="
+try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE="
+
 try2 () {
     hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path)
     if test "$hash" != "$2"; then
@@ -65,12 +71,16 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 try3() {
     h64=$(nix to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
+    sri=$(nix to-sri --type "$1" "$2")
+    [ "$sri" = "$1-$4" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
     h16=$(nix to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
+    h16=$(nix to-base16 "$sri")
+    [ "$h16" = "$2" ]
 }
 try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
 try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="

From 567941fb59960a05a5d99208fb229c8777977023 Mon Sep 17 00:00:00 2001
From: Patrick Hilhorst <git@hilhorst.be>
Date: Fri, 14 Dec 2018 09:36:19 +0100
Subject: [PATCH 1307/2196] Clarify nix-instantiate --read-write-mode

---
 doc/manual/command-ref/nix-instantiate.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml
index 39c1282fcc3..53f06aed124 100644
--- a/doc/manual/command-ref/nix-instantiate.xml
+++ b/doc/manual/command-ref/nix-instantiate.xml
@@ -154,7 +154,9 @@ input.</para>
     <listitem><para>When used with <option>--eval</option>, perform
     evaluation in read/write mode so nix language features that
     require it will still work (at the cost of needing to do
-    instantiation of every evaluated derivation).</para>
+    instantiation of every evaluated derivation). If this option is
+    not enabled, there may be uninstantiated store paths in the final
+    output.</para>
 
     </listitem>
 

From 7e35e914c1aa24957107c666c76f1d834ebae90a Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 14 Dec 2018 20:07:23 +0100
Subject: [PATCH 1308/2196] fetchGit: allow fetching explicit refs

Trying to fetch refs that are not in refs/heads currently fails because
it looks for refs/heads/refs/foo instead of refs/foo.

eg.

	builtins.fetchGit {
	  url = https://github.com/NixOS/nixpkgs.git;
	  ref = "refs/pull/1024/head;
	}
---
 src/libexpr/primops/fetchGit.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index b46d2f25826..588b0fa4d53 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -94,7 +94,11 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "init", "--bare", cacheDir });
     }
 
-    Path localRefFile = cacheDir + "/refs/heads/" + *ref;
+    Path localRefFile;
+    if (ref->compare(0, 5, "refs/") == 0)
+        localRefFile = cacheDir + "/" + *ref;
+    else
+        localRefFile = cacheDir + "/refs/heads/" + *ref;
 
     bool doFetch;
     time_t now = time(0);

From aa7e52abff64c4211b65e4092d17a35b9cccf36e Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 14 Dec 2018 22:37:20 -0600
Subject: [PATCH 1309/2196] tests/fetchurl: fix after changing default hash
 from 512 to 256

---
 tests/fetchurl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index d51d081f5e3..ec3399b08d0 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -23,7 +23,7 @@ clearStore
 
 hash=$(nix hash-file ./fetchurl.sh)
 
-[[ $hash =~ ^sha512- ]]
+[[ $hash =~ ^sha256- ]]
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
 

From c1112ae9a229c188f3415e69a5997d77383f6643 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Sat, 15 Dec 2018 09:59:47 -0600
Subject: [PATCH 1310/2196] nix-store: remove debugging print

---
 src/nix-store/nix-store.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 5b37237eb16..a9ad14762e6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -1000,9 +1000,6 @@ static int _main(int argc, char * * argv)
         Strings opFlags, opArgs;
         Operation op = 0;
 
-        Hash h("sha512-Q2bFTOhEALkN8hOms2FKTDLy7eugP2zFZ1T8LCvX42Fp3WoNr3bjZSAHeOsHrbV1Fu9/A0EzCinRE7Af1ofPrw==");
-        printError("GOT HASH %s", h.to_string(Base64));
-
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             Operation oldOp = op;
 

From 82f054d7d5fc8f9de45afa7107557644d1514c98 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 20 Dec 2018 20:05:14 +0100
Subject: [PATCH 1311/2196] installer: update macOS version check to 10.12.2

Nixpkgs will drop support for <10.12 soon and thus a nix release built
using the 19.03 channel will also require a newer version of macOS.
---
 scripts/install-nix-from-closure.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index ab20774bbf0..e635dcab391 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then
     exit 1
 fi
 
-# macOS support for 10.10 or higher
+# macOS support for 10.12.6 or higher
 if [ "$(uname -s)" = "Darwin" ]; then
-    if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then
-        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
+    macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
+    macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
+    if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then
+        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
         exit 1
     fi
 fi

From e78511743eaf7eca6f51480bad590ab6548519ab Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 17 Dec 2018 11:43:15 -0600
Subject: [PATCH 1312/2196] nlohmann: 3.0.1 -> 3.4.0

```
$ curl -L
"https://github.com/nlohmann/json/releases/download/v3.4.0/json.hpp" -o
src/nlohmann/json.hpp
```
---
 src/nlohmann/json.hpp | 14718 +++++++++++++++++++++++++++-------------
 1 file changed, 10059 insertions(+), 4659 deletions(-)

diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
index 5b0b0ea5b30..1e7cf51e0ac 100644
--- a/src/nlohmann/json.hpp
+++ b/src/nlohmann/json.hpp
@@ -1,11 +1,12 @@
 /*
     __ _____ _____ _____
  __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 3.0.1
+|  |  |__   |  |  | | | |  version 3.4.0
 |_____|_____|_____|_|___|  https://github.com/nlohmann/json
 
 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
-Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>.
+SPDX-License-Identifier: MIT
+Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>.
 
 Permission is hereby  granted, free of charge, to any  person obtaining a copy
 of this software and associated  documentation files (the "Software"), to deal
@@ -29,42 +30,104 @@ SOFTWARE.
 #ifndef NLOHMANN_JSON_HPP
 #define NLOHMANN_JSON_HPP
 
-#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform
-#include <array> // array
+#define NLOHMANN_JSON_VERSION_MAJOR 3
+#define NLOHMANN_JSON_VERSION_MINOR 4
+#define NLOHMANN_JSON_VERSION_PATCH 0
+
+#include <algorithm> // all_of, find, for_each
 #include <cassert> // assert
 #include <ciso646> // and, not, or
-#include <clocale> // lconv, localeconv
-#include <cmath> // isfinite, labs, ldexp, signbit
 #include <cstddef> // nullptr_t, ptrdiff_t, size_t
-#include <cstdint> // int64_t, uint64_t
-#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull
-#include <cstring> // memcpy, strlen
-#include <forward_list> // forward_list
-#include <functional> // function, hash, less
+#include <functional> // hash, less
 #include <initializer_list> // initializer_list
-#include <iomanip> // hex
-#include <iosfwd>   // istream, ostream
-#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
-#include <limits> // numeric_limits
-#include <locale> // locale
-#include <map> // map
-#include <memory> // addressof, allocator, allocator_traits, unique_ptr
+#include <iosfwd> // istream, ostream
+#include <iterator> // iterator_traits, random_access_iterator_tag
 #include <numeric> // accumulate
-#include <sstream> // stringstream
-#include <string> // getline, stoi, string, to_string
-#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
-#include <utility> // declval, forward, make_pair, move, pair, swap
-#include <valarray> // valarray
+#include <string> // string, stoi, to_string
+#include <utility> // declval, forward, move, pair, swap
+
+// #include <nlohmann/json_fwd.hpp>
+#ifndef NLOHMANN_JSON_FWD_HPP
+#define NLOHMANN_JSON_FWD_HPP
+
+#include <cstdint> // int64_t, uint64_t
+#include <map> // map
+#include <memory> // allocator
+#include <string> // string
 #include <vector> // vector
 
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer>
+class basic_json;
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+template<typename BasicJsonType>
+class json_pointer;
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+}  // namespace nlohmann
+
+#endif
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+
+// This file contains all internal macro definitions
+// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them
+
 // exclude unsupported compilers
-#if defined(__clang__)
-    #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
-        #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
-    #endif
-#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
-    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
-        #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)
+    #if defined(__clang__)
+        #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
+            #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
+    #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
+        #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800
+            #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
+        #endif
     #endif
 #endif
 
@@ -90,14 +153,36 @@ SOFTWARE.
 #endif
 
 // allow to disable exceptions
-#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION)
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
     #define JSON_THROW(exception) throw exception
     #define JSON_TRY try
     #define JSON_CATCH(exception) catch(exception)
+    #define JSON_INTERNAL_CATCH(exception) catch(exception)
 #else
     #define JSON_THROW(exception) std::abort()
     #define JSON_TRY if(true)
     #define JSON_CATCH(exception) if(false)
+    #define JSON_INTERNAL_CATCH(exception) if(false)
+#endif
+
+// override exception macros
+#if defined(JSON_THROW_USER)
+    #undef JSON_THROW
+    #define JSON_THROW JSON_THROW_USER
+#endif
+#if defined(JSON_TRY_USER)
+    #undef JSON_TRY
+    #define JSON_TRY JSON_TRY_USER
+#endif
+#if defined(JSON_CATCH_USER)
+    #undef JSON_CATCH
+    #define JSON_CATCH JSON_CATCH_USER
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_CATCH_USER
+#endif
+#if defined(JSON_INTERNAL_CATCH_USER)
+    #undef JSON_INTERNAL_CATCH
+    #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER
 #endif
 
 // manual branch prediction
@@ -118,25 +203,35 @@ SOFTWARE.
 #endif
 
 /*!
-@brief namespace for Niels Lohmann
-@see https://github.com/nlohmann
-@since version 1.0.0
+@brief macro to briefly define a mapping between an enum and JSON
+@def NLOHMANN_JSON_SERIALIZE_ENUM
+@since version 3.4.0
 */
-namespace nlohmann
-{
-template<typename = void, typename = void>
-struct adl_serializer;
-
-// forward declaration of basic_json (required to split the class)
-template<template<typename, typename, typename...> class ObjectType = std::map,
-         template<typename, typename...> class ArrayType = std::vector,
-         class StringType = std::string, class BooleanType = bool,
-         class NumberIntegerType = std::int64_t,
-         class NumberUnsignedType = std::uint64_t,
-         class NumberFloatType = double,
-         template<typename> class AllocatorType = std::allocator,
-         template<typename, typename = void> class JSONSerializer = adl_serializer>
-class basic_json;
+#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...)                                           \
+    template<typename BasicJsonType>                                                           \
+    inline void to_json(BasicJsonType& j, const ENUM_TYPE& e)                                  \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.first == e;                                                         \
+        });                                                                                    \
+        j = ((it != std::end(m)) ? it : std::begin(m))->second;                                \
+    }                                                                                          \
+    template<typename BasicJsonType>                                                           \
+    inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                \
+    {                                                                                          \
+        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
+        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
+        auto it = std::find_if(std::begin(m), std::end(m),                                     \
+                               [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
+        {                                                                                      \
+            return ej_pair.second == j;                                                        \
+        });                                                                                    \
+        e = ((it != std::end(m)) ? it : std::begin(m))->first;                                 \
+    }
 
 // Ugly macros to avoid uglier copy-paste when specializing basic_json. They
 // may be removed in the future once the class is split.
@@ -154,910 +249,991 @@ class basic_json;
     NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
     AllocatorType, JSONSerializer>
 
+// #include <nlohmann/detail/meta/cpp_future.hpp>
 
-/*!
-@brief unnamed namespace with internal helper functions
 
-This namespace collects some functions that could not be defined inside the
-@ref basic_json class.
+#include <ciso646> // not
+#include <cstddef> // size_t
+#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
 
-@since version 2.1.0
-*/
+namespace nlohmann
+{
 namespace detail
 {
-////////////////
-// exceptions //
-////////////////
-
-/*!
-@brief general exception of the @ref basic_json class
-
-This class is an extension of `std::exception` objects with a member @a id for
-exception ids. It is used as the base class for all exceptions thrown by the
-@ref basic_json class. This class can hence be used as "wildcard" to catch
-exceptions.
-
-Subclasses:
-- @ref parse_error for exceptions indicating a parse error
-- @ref invalid_iterator for exceptions indicating errors with iterators
-- @ref type_error for exceptions indicating executing a member function with
-                  a wrong type
-- @ref out_of_range for exceptions indicating access out of the defined range
-- @ref other_error for exceptions indicating other library errors
-
-@internal
-@note To have nothrow-copy-constructible exceptions, we internally use
-      `std::runtime_error` which can cope with arbitrary-length error messages.
-      Intermediate strings are built with static functions and then passed to
-      the actual constructor.
-@endinternal
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
 
-@liveexample{The following code shows how arbitrary library exceptions can be
-caught.,exception}
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
 
-@since version 3.0.0
-*/
-class exception : public std::exception
+// implementation of C++14 index_sequence and affiliates
+// source: https://stackoverflow.com/a/32223343
+template<std::size_t... Ints>
+struct index_sequence
 {
-  public:
-    /// returns the explanatory string
-    const char* what() const noexcept override
+    using type = index_sequence;
+    using value_type = std::size_t;
+    static constexpr std::size_t size() noexcept
     {
-        return m.what();
+        return sizeof...(Ints);
     }
+};
 
-    /// the id of the exception
-    const int id;
+template<class Sequence1, class Sequence2>
+struct merge_and_renumber;
 
-  protected:
-    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
+template<std::size_t... I1, std::size_t... I2>
+struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
+        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
 
-    static std::string name(const std::string& ename, int id_)
-    {
-        return "[json.exception." + ename + "." + std::to_string(id_) + "] ";
-    }
+template<std::size_t N>
+struct make_index_sequence
+    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
+      typename make_index_sequence < N - N / 2 >::type > {};
 
-  private:
-    /// an exception object as storage for error messages
-    std::runtime_error m;
+template<> struct make_index_sequence<0> : index_sequence<> {};
+template<> struct make_index_sequence<1> : index_sequence<0> {};
+
+template<typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+    static constexpr T value{};
 };
 
-/*!
-@brief exception indicating a parse error
+template<typename T>
+constexpr T static_const<T>::value;
+}  // namespace detail
+}  // namespace nlohmann
 
-This exception is thrown by the library when a parse error occurs. Parse errors
-can occur during the deserialization of JSON text, CBOR, MessagePack, as well
-as when using JSON Patch.
+// #include <nlohmann/detail/meta/type_traits.hpp>
 
-Member @a byte holds the byte index of the last read character in the input
-file.
 
-Exceptions have ids 1xx.
+#include <ciso646> // not
+#include <limits> // numeric_limits
+#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type
+#include <utility> // declval
 
-name / id                      | example message | description
------------------------------- | --------------- | -------------------------
-json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
-json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
-json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
-json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
-json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
-json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`.
-json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
-json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
-json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
-json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
-json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
-json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+// #include <nlohmann/json_fwd.hpp>
 
-@note For an input with n bytes, 1 is the index of the first character and n+1
-      is the index of the terminating null byte or the end of file. This also
-      holds true when reading a byte vector (CBOR or MessagePack).
+// #include <nlohmann/detail/meta/cpp_future.hpp>
 
-@liveexample{The following code shows how a `parse_error` exception can be
-caught.,parse_error}
+// #include <nlohmann/detail/meta/detected.hpp>
 
-@sa @ref exception for the base class of the library exceptions
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
 
-@since version 3.0.0
-*/
-class parse_error : public exception
+#include <type_traits>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
+
+namespace nlohmann
 {
-  public:
-    /*!
-    @brief create a parse error exception
-    @param[in] id_       the id of the exception
-    @param[in] byte_     the byte index where the error occurred (or 0 if the
-                         position cannot be determined)
-    @param[in] what_arg  the explanatory string
-    @return parse_error object
-    */
-    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
-    {
-        std::string w = exception::name("parse_error", id_) + "parse error" +
-                        (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") +
-                        ": " + what_arg;
-        return parse_error(id_, byte_, w.c_str());
-    }
+namespace detail
+{
+template <typename ...Ts> struct make_void
+{
+    using type = void;
+};
+template <typename ...Ts> using void_t = typename make_void<Ts...>::type;
+} // namespace detail
+}  // namespace nlohmann
 
-    /*!
-    @brief byte index of the parse error
 
-    The byte index of the last read character in the input file.
+// http://en.cppreference.com/w/cpp/experimental/is_detected
+namespace nlohmann
+{
+namespace detail
+{
+struct nonesuch
+{
+    nonesuch() = delete;
+    ~nonesuch() = delete;
+    nonesuch(nonesuch const&) = delete;
+    void operator=(nonesuch const&) = delete;
+};
 
-    @note For an input with n bytes, 1 is the index of the first character and
-          n+1 is the index of the terminating null byte or the end of file.
-          This also holds true when reading a byte vector (CBOR or MessagePack).
-    */
-    const std::size_t byte;
+template <class Default,
+          class AlwaysVoid,
+          template <class...> class Op,
+          class... Args>
+struct detector
+{
+    using value_t = std::false_type;
+    using type = Default;
+};
 
-  private:
-    parse_error(int id_, std::size_t byte_, const char* what_arg)
-        : exception(id_, what_arg), byte(byte_) {}
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, void_t<Op<Args...>>, Op, Args...>
+{
+    using value_t = std::true_type;
+    using type = Op<Args...>;
 };
 
-/*!
-@brief exception indicating errors with iterators
+template <template <class...> class Op, class... Args>
+using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;
 
-This exception is thrown if iterators passed to a library function do not match
-the expected semantics.
+template <template <class...> class Op, class... Args>
+using detected_t = typename detector<nonesuch, void, Op, Args...>::type;
 
-Exceptions have ids 2xx.
+template <class Default, template <class...> class Op, class... Args>
+using detected_or = detector<Default, void, Op, Args...>;
 
-name / id                           | example message | description
------------------------------------ | --------------- | -------------------------
-json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
-json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
-json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
-json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
-json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
-json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
-json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
-json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
-json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
-json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
-json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
-json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
-json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
-json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
+template <class Default, template <class...> class Op, class... Args>
+using detected_or_t = typename detected_or<Default, Op, Args...>::type;
 
-@liveexample{The following code shows how an `invalid_iterator` exception can be
-caught.,invalid_iterator}
+template <class Expected, template <class...> class Op, class... Args>
+using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;
 
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
+template <class To, template <class...> class Op, class... Args>
+using is_detected_convertible =
+    std::is_convertible<detected_t<Op, Args...>, To>;
+}  // namespace detail
+}  // namespace nlohmann
 
-@since version 3.0.0
-*/
-class invalid_iterator : public exception
-{
-  public:
-    static invalid_iterator create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("invalid_iterator", id_) + what_arg;
-        return invalid_iterator(id_, w.c_str());
-    }
+// #include <nlohmann/detail/macro_scope.hpp>
 
-  private:
-    invalid_iterator(int id_, const char* what_arg)
-        : exception(id_, what_arg) {}
-};
 
+namespace nlohmann
+{
 /*!
-@brief exception indicating executing a member function with a wrong type
+@brief detail namespace with internal helper functions
 
-This exception is thrown in case of a type error; that is, a library function is
-executed on a JSON value whose type does not match the expected semantics.
+This namespace collects functions that should not be exposed,
+implementations of some @ref basic_json methods, and meta-programming helpers.
 
-Exceptions have ids 3xx.
+@since version 2.1.0
+*/
+namespace detail
+{
+/////////////
+// helpers //
+/////////////
 
-name / id                     | example message | description
------------------------------ | --------------- | -------------------------
-json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
-json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
-json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
-json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
-json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
-json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
-json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
-json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
-json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
-json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
-json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
-json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
-json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
-json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
-json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
-json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
+// Note to maintainers:
+//
+// Every trait in this file expects a non CV-qualified type.
+// The only exceptions are in the 'aliases for detected' section
+// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))
+//
+// In this case, T has to be properly CV-qualified to constraint the function arguments
+// (e.g. to_json(BasicJsonType&, const T&))
 
-@liveexample{The following code shows how a `type_error` exception can be
-caught.,type_error}
+template<typename> struct is_basic_json : std::false_type {};
 
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
 
-@since version 3.0.0
-*/
-class type_error : public exception
-{
-  public:
-    static type_error create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("type_error", id_) + what_arg;
-        return type_error(id_, w.c_str());
-    }
+//////////////////////////
+// aliases for detected //
+//////////////////////////
 
-  private:
-    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
-};
+template <typename T>
+using mapped_type_t = typename T::mapped_type;
 
-/*!
-@brief exception indicating access out of the defined range
+template <typename T>
+using key_type_t = typename T::key_type;
 
-This exception is thrown in case a library function is called on an input
-parameter that exceeds the expected range, for instance in case of array
-indices or nonexisting object keys.
+template <typename T>
+using value_type_t = typename T::value_type;
 
-Exceptions have ids 4xx.
+template <typename T>
+using difference_type_t = typename T::difference_type;
 
-name / id                       | example message | description
-------------------------------- | --------------- | -------------------------
-json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
-json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
-json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
-json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
-json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
-json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+template <typename T>
+using pointer_t = typename T::pointer;
 
-@liveexample{The following code shows how an `out_of_range` exception can be
-caught.,out_of_range}
+template <typename T>
+using reference_t = typename T::reference;
 
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref other_error for exceptions indicating other library errors
+template <typename T>
+using iterator_category_t = typename T::iterator_category;
 
-@since version 3.0.0
-*/
-class out_of_range : public exception
-{
-  public:
-    static out_of_range create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("out_of_range", id_) + what_arg;
-        return out_of_range(id_, w.c_str());
-    }
+template <typename T>
+using iterator_t = typename T::iterator;
 
-  private:
-    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
-};
+template <typename T, typename... Args>
+using to_json_function = decltype(T::to_json(std::declval<Args>()...));
 
-/*!
-@brief exception indicating other library errors
+template <typename T, typename... Args>
+using from_json_function = decltype(T::from_json(std::declval<Args>()...));
 
-This exception is thrown in case of errors that cannot be classified with the
-other exception types.
+template <typename T, typename U>
+using get_template_function = decltype(std::declval<T>().template get<U>());
 
-Exceptions have ids 5xx.
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template <typename BasicJsonType, typename T, typename = void>
+struct has_from_json : std::false_type {};
 
-name / id                      | example message | description
------------------------------- | --------------- | -------------------------
-json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
+template <typename BasicJsonType, typename T>
+struct has_from_json<BasicJsonType, T,
+           enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
 
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
+    static constexpr bool value =
+        is_detected_exact<void, from_json_function, serializer,
+        const BasicJsonType&, T&>::value;
+};
 
-@liveexample{The following code shows how an `other_error` exception can be
-caught.,other_error}
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template <typename BasicJsonType, typename T, typename = void>
+struct has_non_default_from_json : std::false_type {};
 
-@since version 3.0.0
-*/
-class other_error : public exception
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
 {
-  public:
-    static other_error create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("other_error", id_) + what_arg;
-        return other_error(id_, w.c_str());
-    }
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
 
-  private:
-    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+    static constexpr bool value =
+        is_detected_exact<T, from_json_function, serializer,
+        const BasicJsonType&>::value;
 };
 
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.
+template <typename BasicJsonType, typename T, typename = void>
+struct has_to_json : std::false_type {};
 
+template <typename BasicJsonType, typename T>
+struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
+{
+    using serializer = typename BasicJsonType::template json_serializer<T, void>;
 
-///////////////////////////
-// JSON type enumeration //
-///////////////////////////
-
-/*!
-@brief the JSON type enumeration
+    static constexpr bool value =
+        is_detected_exact<void, to_json_function, serializer, BasicJsonType&,
+        T>::value;
+};
 
-This enumeration collects the different JSON types. It is internally used to
-distinguish the stored values, and the functions @ref basic_json::is_null(),
-@ref basic_json::is_object(), @ref basic_json::is_array(),
-@ref basic_json::is_string(), @ref basic_json::is_boolean(),
-@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
-@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
-@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
-@ref basic_json::is_structured() rely on it.
 
-@note There are three enumeration entries (number_integer, number_unsigned, and
-number_float), because the library distinguishes these three types for numbers:
-@ref basic_json::number_unsigned_t is used for unsigned integers,
-@ref basic_json::number_integer_t is used for signed integers, and
-@ref basic_json::number_float_t is used for floating-point numbers or to
-approximate integers which do not fit in the limits of their respective type.
+///////////////////
+// is_ functions //
+///////////////////
 
-@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
-value with the default value for a given type
+template <typename T, typename = void>
+struct is_iterator_traits : std::false_type {};
 
-@since version 1.0.0
-*/
-enum class value_t : uint8_t
+template <typename T>
+struct is_iterator_traits<std::iterator_traits<T>>
 {
-    null,             ///< null value
-    object,           ///< object (unordered set of name/value pairs)
-    array,            ///< array (ordered collection of values)
-    string,           ///< string value
-    boolean,          ///< boolean value
-    number_integer,   ///< number value (signed integer)
-    number_unsigned,  ///< number value (unsigned integer)
-    number_float,     ///< number value (floating-point)
-    discarded         ///< discarded by the the parser callback function
+  private:
+    using traits = std::iterator_traits<T>;
+
+  public:
+    static constexpr auto value =
+        is_detected<value_type_t, traits>::value &&
+        is_detected<difference_type_t, traits>::value &&
+        is_detected<pointer_t, traits>::value &&
+        is_detected<iterator_category_t, traits>::value &&
+        is_detected<reference_t, traits>::value;
 };
 
-/*!
-@brief comparison operator for JSON types
+// source: https://stackoverflow.com/a/37193089/4116453
 
-Returns an ordering that is similar to Python:
-- order: null < boolean < number < object < array < string
-- furthermore, each type is not smaller than itself
-- discarded values are not comparable
+template <typename T, typename = void>
+struct is_complete_type : std::false_type {};
 
-@since version 1.0.0
-*/
-inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+template <typename T>
+struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType,
+          typename = void>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type_impl <
+    BasicJsonType, CompatibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and
+    is_detected<key_type_t, CompatibleObjectType>::value >>
 {
-    static constexpr std::array<uint8_t, 8> order = {{
-            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
-            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
-        }
-    };
 
-    const auto l_index = static_cast<std::size_t>(lhs);
-    const auto r_index = static_cast<std::size_t>(rhs);
-    return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
-}
+    using object_t = typename BasicJsonType::object_t;
+
+    // macOS's is_constructible does not play well with nonesuch...
+    static constexpr bool value =
+        std::is_constructible<typename object_t::key_type,
+        typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename object_t::mapped_type,
+        typename CompatibleObjectType::mapped_type>::value;
+};
 
+template <typename BasicJsonType, typename CompatibleObjectType>
+struct is_compatible_object_type
+    : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};
 
-/////////////
-// helpers //
-/////////////
+template <typename BasicJsonType, typename ConstructibleObjectType,
+          typename = void>
+struct is_constructible_object_type_impl : std::false_type {};
 
-template<typename> struct is_basic_json : std::false_type {};
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type_impl <
+    BasicJsonType, ConstructibleObjectType,
+    enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and
+    is_detected<key_type_t, ConstructibleObjectType>::value >>
+{
+    using object_t = typename BasicJsonType::object_t;
 
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+    static constexpr bool value =
+        (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and
+         std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or
+        (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or
+         has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value);
+};
 
-// alias templates to reduce boilerplate
-template<bool B, typename T = void>
-using enable_if_t = typename std::enable_if<B, T>::type;
+template <typename BasicJsonType, typename ConstructibleObjectType>
+struct is_constructible_object_type
+    : is_constructible_object_type_impl<BasicJsonType,
+      ConstructibleObjectType> {};
 
-template<typename T>
-using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+template <typename BasicJsonType, typename CompatibleStringType,
+          typename = void>
+struct is_compatible_string_type_impl : std::false_type {};
 
-// implementation of C++14 index_sequence and affiliates
-// source: https://stackoverflow.com/a/32223343
-template<std::size_t... Ints>
-struct index_sequence
+template <typename BasicJsonType, typename CompatibleStringType>
+struct is_compatible_string_type_impl <
+    BasicJsonType, CompatibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, CompatibleStringType>::value >>
 {
-    using type = index_sequence;
-    using value_type = std::size_t;
-    static constexpr std::size_t size() noexcept
-    {
-        return sizeof...(Ints);
-    }
+    static constexpr auto value =
+        std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;
 };
 
-template<class Sequence1, class Sequence2>
-struct merge_and_renumber;
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_compatible_string_type
+    : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
 
-template<std::size_t... I1, std::size_t... I2>
-struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
-        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
+template <typename BasicJsonType, typename ConstructibleStringType,
+          typename = void>
+struct is_constructible_string_type_impl : std::false_type {};
 
-template<std::size_t N>
-struct make_index_sequence
-    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
-      typename make_index_sequence < N - N / 2 >::type > {};
-
-template<> struct make_index_sequence<0> : index_sequence<> {};
-template<> struct make_index_sequence<1> : index_sequence<0> {};
-
-template<typename... Ts>
-using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type_impl <
+    BasicJsonType, ConstructibleStringType,
+    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
+    value_type_t, ConstructibleStringType>::value >>
+{
+    static constexpr auto value =
+        std::is_constructible<ConstructibleStringType,
+        typename BasicJsonType::string_t>::value;
+};
 
-/*
-Implementation of two C++17 constructs: conjunction, negation. This is needed
-to avoid evaluating all the traits in a condition
+template <typename BasicJsonType, typename ConstructibleStringType>
+struct is_constructible_string_type
+    : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
+
+template <typename BasicJsonType, typename CompatibleArrayType, typename = void>
+struct is_compatible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type_impl <
+    BasicJsonType, CompatibleArrayType,
+    enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and
+    is_detected<iterator_t, CompatibleArrayType>::value and
+// This is needed because json_reverse_iterator has a ::iterator type...
+// Therefore it is detected as a CompatibleArrayType.
+// The real fix would be to have an Iterable concept.
+    not is_iterator_traits<
+    std::iterator_traits<CompatibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        std::is_constructible<BasicJsonType,
+        typename CompatibleArrayType::value_type>::value;
+};
 
-For example: not std::is_same<void, T>::value and has_value_type<T>::value
-will not compile when T = void (on MSVC at least). Whereas
-conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will
-stop evaluating if negation<...>::value == false
+template <typename BasicJsonType, typename CompatibleArrayType>
+struct is_compatible_array_type
+    : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType, typename = void>
+struct is_constructible_array_type_impl : std::false_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value >>
+            : std::true_type {};
+
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type_impl <
+    BasicJsonType, ConstructibleArrayType,
+    enable_if_t<not std::is_same<ConstructibleArrayType,
+    typename BasicJsonType::value_type>::value and
+    is_detected<value_type_t, ConstructibleArrayType>::value and
+    is_detected<iterator_t, ConstructibleArrayType>::value and
+    is_complete_type<
+    detected_t<value_type_t, ConstructibleArrayType>>::value >>
+{
+    static constexpr bool value =
+        // This is needed because json_reverse_iterator has a ::iterator type,
+        // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`...
+        // Therefore it is detected as a ConstructibleArrayType.
+        // The real fix would be to have an Iterable concept.
+        not is_iterator_traits <
+        std::iterator_traits<ConstructibleArrayType >>::value and
+
+        (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or
+         has_from_json<BasicJsonType,
+         typename ConstructibleArrayType::value_type>::value or
+         has_non_default_from_json <
+         BasicJsonType, typename ConstructibleArrayType::value_type >::value);
+};
 
-Please note that those constructs must be used with caution, since symbols can
-become very long quickly (which can slow down compilation and cause MSVC
-internal compiler errors). Only use it when you have to (see example ahead).
-*/
-template<class...> struct conjunction : std::true_type {};
-template<class B1> struct conjunction<B1> : B1 {};
-template<class B1, class... Bn>
-struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
+template <typename BasicJsonType, typename ConstructibleArrayType>
+struct is_constructible_array_type
+    : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};
 
-template<class B> struct negation : std::integral_constant<bool, not B::value> {};
+template <typename RealIntegerType, typename CompatibleNumberIntegerType,
+          typename = void>
+struct is_compatible_integer_type_impl : std::false_type {};
 
-// dispatch utility (taken from ranges-v3)
-template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
-template<> struct priority_tag<0> {};
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl <
+    RealIntegerType, CompatibleNumberIntegerType,
+    enable_if_t<std::is_integral<RealIntegerType>::value and
+    std::is_integral<CompatibleNumberIntegerType>::value and
+    not std::is_same<bool, CompatibleNumberIntegerType>::value >>
+{
+    // is there an assert somewhere on overflows?
+    using RealLimits = std::numeric_limits<RealIntegerType>;
+    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
 
+    static constexpr auto value =
+        std::is_constructible<RealIntegerType,
+        CompatibleNumberIntegerType>::value and
+        CompatibleLimits::is_integer and
+        RealLimits::is_signed == CompatibleLimits::is_signed;
+};
 
-//////////////////
-// constructors //
-//////////////////
+template <typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+    : is_compatible_integer_type_impl<RealIntegerType,
+      CompatibleNumberIntegerType> {};
 
-template<value_t> struct external_constructor;
+template <typename BasicJsonType, typename CompatibleType, typename = void>
+struct is_compatible_type_impl: std::false_type {};
 
-template<>
-struct external_constructor<value_t::boolean>
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type_impl <
+    BasicJsonType, CompatibleType,
+    enable_if_t<is_complete_type<CompatibleType>::value >>
 {
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
-    {
-        j.m_type = value_t::boolean;
-        j.m_value = b;
-        j.assert_invariant();
-    }
+    static constexpr bool value =
+        has_to_json<BasicJsonType, CompatibleType>::value;
 };
 
-template<>
-struct external_constructor<value_t::string>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
-    {
-        j.m_type = value_t::string;
-        j.m_value = s;
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
-    {
-        j.m_type = value_t::string;
-        j.m_value = std::move(s);
-        j.assert_invariant();
-    }
-};
+template <typename BasicJsonType, typename CompatibleType>
+struct is_compatible_type
+    : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
+}  // namespace detail
+}  // namespace nlohmann
 
-template<>
-struct external_constructor<value_t::number_float>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
-    {
-        j.m_type = value_t::number_float;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
+// #include <nlohmann/detail/exceptions.hpp>
 
-template<>
-struct external_constructor<value_t::number_unsigned>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
-    {
-        j.m_type = value_t::number_unsigned;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
 
-template<>
-struct external_constructor<value_t::number_integer>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
-    {
-        j.m_type = value_t::number_integer;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
+#include <exception> // exception
+#include <stdexcept> // runtime_error
+#include <string> // to_string
 
-template<>
-struct external_constructor<value_t::array>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = arr;
-        j.assert_invariant();
-    }
+// #include <nlohmann/detail/input/position_t.hpp>
 
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = std::move(arr);
-        j.assert_invariant();
-    }
 
-    template<typename BasicJsonType, typename CompatibleArrayType,
-             enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
-                         int> = 0>
-    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
-    {
-        using std::begin;
-        using std::end;
-        j.m_type = value_t::array;
-        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
-        j.assert_invariant();
-    }
+#include <cstddef> // size_t
 
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = value_t::array;
-        j.m_value.array->reserve(arr.size());
-        for (const bool x : arr)
-        {
-            j.m_value.array->push_back(x);
-        }
-        j.assert_invariant();
-    }
+namespace nlohmann
+{
+namespace detail
+{
+/// struct to capture the start position of the current token
+struct position_t
+{
+    /// the total number of characters read
+    std::size_t chars_read_total = 0;
+    /// the number of characters read in the current line
+    std::size_t chars_read_current_line = 0;
+    /// the number of lines read
+    std::size_t lines_read = 0;
 
-    template<typename BasicJsonType, typename T,
-             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
-    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
+    /// conversion to size_t to preserve SAX interface
+    constexpr operator size_t() const
     {
-        j.m_type = value_t::array;
-        j.m_value = value_t::array;
-        j.m_value.array->resize(arr.size());
-        std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());
-        j.assert_invariant();
+        return chars_read_total;
     }
 };
 
-template<>
-struct external_constructor<value_t::object>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
-    {
-        j.m_type = value_t::object;
-        j.m_value = obj;
-        j.assert_invariant();
-    }
+}
+}
 
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
-    {
-        j.m_type = value_t::object;
-        j.m_value = std::move(obj);
-        j.assert_invariant();
-    }
 
-    template<typename BasicJsonType, typename CompatibleObjectType,
-             enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0>
-    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
-    {
-        using std::begin;
-        using std::end;
+namespace nlohmann
+{
+namespace detail
+{
+////////////////
+// exceptions //
+////////////////
 
-        j.m_type = value_t::object;
-        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
-        j.assert_invariant();
-    }
-};
+/*!
+@brief general exception of the @ref basic_json class
 
+This class is an extension of `std::exception` objects with a member @a id for
+exception ids. It is used as the base class for all exceptions thrown by the
+@ref basic_json class. This class can hence be used as "wildcard" to catch
+exceptions.
 
-////////////////////////
-// has_/is_ functions //
-////////////////////////
+Subclasses:
+- @ref parse_error for exceptions indicating a parse error
+- @ref invalid_iterator for exceptions indicating errors with iterators
+- @ref type_error for exceptions indicating executing a member function with
+                  a wrong type
+- @ref out_of_range for exceptions indicating access out of the defined range
+- @ref other_error for exceptions indicating other library errors
 
-/*!
-@brief Helper to determine whether there's a key_type for T.
+@internal
+@note To have nothrow-copy-constructible exceptions, we internally use
+      `std::runtime_error` which can cope with arbitrary-length error messages.
+      Intermediate strings are built with static functions and then passed to
+      the actual constructor.
+@endinternal
 
-This helper is used to tell associative containers apart from other containers
-such as sequence containers. For instance, `std::map` passes the test as it
-contains a `mapped_type`, whereas `std::vector` fails the test.
+@liveexample{The following code shows how arbitrary library exceptions can be
+caught.,exception}
 
-@sa http://stackoverflow.com/a/7728728/266378
-@since version 1.0.0, overworked in version 2.0.6
+@since version 3.0.0
 */
-#define NLOHMANN_JSON_HAS_HELPER(type)                                        \
-    template<typename T> struct has_##type {                                  \
-    private:                                                                  \
-        template<typename U, typename = typename U::type>                     \
-        static int detect(U &&);                                              \
-        static void detect(...);                                              \
-    public:                                                                   \
-        static constexpr bool value =                                         \
-                std::is_integral<decltype(detect(std::declval<T>()))>::value; \
+class exception : public std::exception
+{
+  public:
+    /// returns the explanatory string
+    const char* what() const noexcept override
+    {
+        return m.what();
     }
 
-NLOHMANN_JSON_HAS_HELPER(mapped_type);
-NLOHMANN_JSON_HAS_HELPER(key_type);
-NLOHMANN_JSON_HAS_HELPER(value_type);
-NLOHMANN_JSON_HAS_HELPER(iterator);
-
-#undef NLOHMANN_JSON_HAS_HELPER
-
+    /// the id of the exception
+    const int id;
 
-template<bool B, class RealType, class CompatibleObjectType>
-struct is_compatible_object_type_impl : std::false_type {};
+  protected:
+    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
 
-template<class RealType, class CompatibleObjectType>
-struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType>
-{
-    static constexpr auto value =
-        std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and
-        std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value;
-};
+    static std::string name(const std::string& ename, int id_)
+    {
+        return "[json.exception." + ename + "." + std::to_string(id_) + "] ";
+    }
 
-template<class BasicJsonType, class CompatibleObjectType>
-struct is_compatible_object_type
-{
-    static auto constexpr value = is_compatible_object_type_impl <
-                                  conjunction<negation<std::is_same<void, CompatibleObjectType>>,
-                                  has_mapped_type<CompatibleObjectType>,
-                                  has_key_type<CompatibleObjectType>>::value,
-                                  typename BasicJsonType::object_t, CompatibleObjectType >::value;
+  private:
+    /// an exception object as storage for error messages
+    std::runtime_error m;
 };
 
-template<typename BasicJsonType, typename T>
-struct is_basic_json_nested_type
-{
-    static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::const_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::reverse_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value;
-};
+/*!
+@brief exception indicating a parse error
 
-template<class BasicJsonType, class CompatibleArrayType>
-struct is_compatible_array_type
-{
-    static auto constexpr value =
-        conjunction<negation<std::is_same<void, CompatibleArrayType>>,
-        negation<is_compatible_object_type<
-        BasicJsonType, CompatibleArrayType>>,
-        negation<std::is_constructible<typename BasicJsonType::string_t,
-        CompatibleArrayType>>,
-        negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>,
-        has_value_type<CompatibleArrayType>,
-        has_iterator<CompatibleArrayType>>::value;
-};
+This exception is thrown by the library when a parse error occurs. Parse errors
+can occur during the deserialization of JSON text, CBOR, MessagePack, as well
+as when using JSON Patch.
 
-template<bool, typename, typename>
-struct is_compatible_integer_type_impl : std::false_type {};
+Member @a byte holds the byte index of the last read character in the input
+file.
 
-template<typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType>
-{
-    // is there an assert somewhere on overflows?
-    using RealLimits = std::numeric_limits<RealIntegerType>;
-    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+Exceptions have ids 1xx.
 
-    static constexpr auto value =
-        std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and
-        CompatibleLimits::is_integer and
-        RealLimits::is_signed == CompatibleLimits::is_signed;
-};
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
+json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
+json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
+json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
+json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
+json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`.
+json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
+json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
+json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
+json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
+json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
+json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet).
 
-template<typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type
-{
-    static constexpr auto value =
-        is_compatible_integer_type_impl <
-        std::is_integral<CompatibleNumberIntegerType>::value and
-        not std::is_same<bool, CompatibleNumberIntegerType>::value,
-        RealIntegerType, CompatibleNumberIntegerType >::value;
-};
+@note For an input with n bytes, 1 is the index of the first character and n+1
+      is the index of the terminating null byte or the end of file. This also
+      holds true when reading a byte vector (CBOR or MessagePack).
 
+@liveexample{The following code shows how a `parse_error` exception can be
+caught.,parse_error}
 
-// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
-template<typename BasicJsonType, typename T>
-struct has_from_json
-{
-  private:
-    // also check the return type of from_json
-    template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json(
-                 std::declval<BasicJsonType>(), std::declval<T&>()))>::value>>
-    static int detect(U&&);
-    static void detect(...);
+@sa @ref exception for the base class of the library exceptions
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
 
+@since version 3.0.0
+*/
+class parse_error : public exception
+{
   public:
-    static constexpr bool value = std::is_integral<decltype(
-                                      detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
-};
+    /*!
+    @brief create a parse error exception
+    @param[in] id_       the id of the exception
+    @param[in] position  the position where the error occurred (or with
+                         chars_read_total=0 if the position cannot be
+                         determined)
+    @param[in] what_arg  the explanatory string
+    @return parse_error object
+    */
+    static parse_error create(int id_, const position_t& pos, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        position_string(pos) + ": " + what_arg;
+        return parse_error(id_, pos.chars_read_total, w.c_str());
+    }
 
-// This trait checks if JSONSerializer<T>::from_json(json const&) exists
-// this overload is used for non-default-constructible user-defined-types
-template<typename BasicJsonType, typename T>
-struct has_non_default_from_json
-{
-  private:
-    template<typename U, typename =
-             enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>>
-    static int detect(U&&);
-    static void detect(...);
+    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id_) + "parse error" +
+                        (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") +
+                        ": " + what_arg;
+        return parse_error(id_, byte_, w.c_str());
+    }
 
-  public:
-    static constexpr bool value = std::is_integral<decltype(detect(
-                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
-};
+    /*!
+    @brief byte index of the parse error
+
+    The byte index of the last read character in the input file.
+
+    @note For an input with n bytes, 1 is the index of the first character and
+          n+1 is the index of the terminating null byte or the end of file.
+          This also holds true when reading a byte vector (CBOR or MessagePack).
+    */
+    const std::size_t byte;
 
-// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
-template<typename BasicJsonType, typename T>
-struct has_to_json
-{
   private:
-    template<typename U, typename = decltype(uncvref_t<U>::to_json(
-                 std::declval<BasicJsonType&>(), std::declval<T>()))>
-    static int detect(U&&);
-    static void detect(...);
+    parse_error(int id_, std::size_t byte_, const char* what_arg)
+        : exception(id_, what_arg), byte(byte_) {}
 
-  public:
-    static constexpr bool value = std::is_integral<decltype(detect(
-                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+    static std::string position_string(const position_t& pos)
+    {
+        return " at line " + std::to_string(pos.lines_read + 1) +
+               ", column " + std::to_string(pos.chars_read_current_line);
+    }
 };
 
+/*!
+@brief exception indicating errors with iterators
 
-/////////////
-// to_json //
-/////////////
+This exception is thrown if iterators passed to a library function do not match
+the expected semantics.
 
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
-void to_json(BasicJsonType& j, T b) noexcept
-{
-    external_constructor<value_t::boolean>::construct(j, b);
-}
+Exceptions have ids 2xx.
 
-template<typename BasicJsonType, typename CompatibleString,
-         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
-void to_json(BasicJsonType& j, const CompatibleString& s)
-{
-    external_constructor<value_t::string>::construct(j, s);
-}
+name / id                           | example message | description
+----------------------------------- | --------------- | -------------------------
+json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
+json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
+json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
+json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
+json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
+json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
+json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
+json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
+json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
+json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
 
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
-{
-    external_constructor<value_t::string>::construct(j, std::move(s));
-}
+@liveexample{The following code shows how an `invalid_iterator` exception can be
+caught.,invalid_iterator}
 
-template<typename BasicJsonType, typename FloatType,
-         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
-void to_json(BasicJsonType& j, FloatType val) noexcept
-{
-    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
-}
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
 
-template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
-         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
-void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+@since version 3.0.0
+*/
+class invalid_iterator : public exception
 {
-    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
-}
+  public:
+    static invalid_iterator create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("invalid_iterator", id_) + what_arg;
+        return invalid_iterator(id_, w.c_str());
+    }
 
-template<typename BasicJsonType, typename CompatibleNumberIntegerType,
-         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
-void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
-{
-    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
-}
+  private:
+    invalid_iterator(int id_, const char* what_arg)
+        : exception(id_, what_arg) {}
+};
 
-template<typename BasicJsonType, typename EnumType,
-         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
-void to_json(BasicJsonType& j, EnumType e) noexcept
-{
-    using underlying_type = typename std::underlying_type<EnumType>::type;
-    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
-}
+/*!
+@brief exception indicating executing a member function with a wrong type
 
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, const std::vector<bool>& e)
-{
-    external_constructor<value_t::array>::construct(j, e);
-}
+This exception is thrown in case of a type error; that is, a library function is
+executed on a JSON value whose type does not match the expected semantics.
 
-template<typename BasicJsonType, typename CompatibleArrayType,
-         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
-                     std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
-                     int> = 0>
-void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
-{
-    external_constructor<value_t::array>::construct(j, arr);
-}
+Exceptions have ids 3xx.
 
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
-void to_json(BasicJsonType& j, std::valarray<T> arr)
-{
-    external_constructor<value_t::array>::construct(j, std::move(arr));
-}
+name / id                     | example message | description
+----------------------------- | --------------- | -------------------------
+json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
+json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
+json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
+json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
+json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
+json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
+json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
+json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
+json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
+json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
+json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
+json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
+json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
+json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
+json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
+json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
+json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) |
 
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
-{
-    external_constructor<value_t::array>::construct(j, std::move(arr));
-}
+@liveexample{The following code shows how a `type_error` exception can be
+caught.,type_error}
 
-template<typename BasicJsonType, typename CompatibleObjectType,
-         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
-void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
-{
-    external_constructor<value_t::object>::construct(j, obj);
-}
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
 
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+@since version 3.0.0
+*/
+class type_error : public exception
 {
-    external_constructor<value_t::object>::construct(j, std::move(obj));
-}
+  public:
+    static type_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("type_error", id_) + what_arg;
+        return type_error(id_, w.c_str());
+    }
 
-template<typename BasicJsonType, typename T, std::size_t N,
-         enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0>
-void to_json(BasicJsonType& j, T (&arr)[N])
-{
-    external_constructor<value_t::array>::construct(j, arr);
-}
+  private:
+    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
 
-template<typename BasicJsonType, typename... Args>
-void to_json(BasicJsonType& j, const std::pair<Args...>& p)
-{
-    j = {p.first, p.second};
-}
+/*!
+@brief exception indicating access out of the defined range
 
-template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>)
-{
-    j = {std::get<Idx>(t)...};
-}
+This exception is thrown in case a library function is called on an input
+parameter that exceeds the expected range, for instance in case of array
+indices or nonexisting object keys.
 
-template<typename BasicJsonType, typename... Args>
-void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
+Exceptions have ids 4xx.
+
+name / id                       | example message | description
+------------------------------- | --------------- | -------------------------
+json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
+json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
+json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
+json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
+json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
+json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. |
+json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. |
+json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string |
+
+@liveexample{The following code shows how an `out_of_range` exception can be
+caught.,out_of_range}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref other_error for exceptions indicating other library errors
+
+@since version 3.0.0
+*/
+class out_of_range : public exception
 {
-    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+  public:
+    static out_of_range create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("out_of_range", id_) + what_arg;
+        return out_of_range(id_, w.c_str());
+    }
+
+  private:
+    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+
+/*!
+@brief exception indicating other library errors
+
+This exception is thrown in case of errors that cannot be classified with the
+other exception types.
+
+Exceptions have ids 5xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+
+@liveexample{The following code shows how an `other_error` exception can be
+caught.,other_error}
+
+@since version 3.0.0
+*/
+class other_error : public exception
+{
+  public:
+    static other_error create(int id_, const std::string& what_arg)
+    {
+        std::string w = exception::name("other_error", id_) + what_arg;
+        return other_error(id_, w.c_str());
+    }
+
+  private:
+    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+#include <array> // array
+#include <ciso646> // and
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////////////
+// JSON type enumeration //
+///////////////////////////
+
+/*!
+@brief the JSON type enumeration
+
+This enumeration collects the different JSON types. It is internally used to
+distinguish the stored values, and the functions @ref basic_json::is_null(),
+@ref basic_json::is_object(), @ref basic_json::is_array(),
+@ref basic_json::is_string(), @ref basic_json::is_boolean(),
+@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
+@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
+@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
+@ref basic_json::is_structured() rely on it.
+
+@note There are three enumeration entries (number_integer, number_unsigned, and
+number_float), because the library distinguishes these three types for numbers:
+@ref basic_json::number_unsigned_t is used for unsigned integers,
+@ref basic_json::number_integer_t is used for signed integers, and
+@ref basic_json::number_float_t is used for floating-point numbers or to
+approximate integers which do not fit in the limits of their respective type.
+
+@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
+value with the default value for a given type
+
+@since version 1.0.0
+*/
+enum class value_t : std::uint8_t
+{
+    null,             ///< null value
+    object,           ///< object (unordered set of name/value pairs)
+    array,            ///< array (ordered collection of values)
+    string,           ///< string value
+    boolean,          ///< boolean value
+    number_integer,   ///< number value (signed integer)
+    number_unsigned,  ///< number value (unsigned integer)
+    number_float,     ///< number value (floating-point)
+    discarded         ///< discarded by the the parser callback function
+};
+
+/*!
+@brief comparison operator for JSON types
+
+Returns an ordering that is similar to Python:
+- order: null < boolean < number < object < array < string
+- furthermore, each type is not smaller than itself
+- discarded values are not comparable
+
+@since version 1.0.0
+*/
+inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+{
+    static constexpr std::array<std::uint8_t, 8> order = {{
+            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
+            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
+        }
+    };
+
+    const auto l_index = static_cast<std::size_t>(lhs);
+    const auto r_index = static_cast<std::size_t>(rhs);
+    return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
 }
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
 
-///////////////
-// from_json //
-///////////////
+
+#include <algorithm> // transform
+#include <array> // array
+#include <ciso646> // and, not
+#include <forward_list> // forward_list
+#include <iterator> // inserter, front_inserter, end
+#include <map> // map
+#include <string> // string
+#include <tuple> // tuple, make_tuple
+#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible
+#include <unordered_map> // unordered_map
+#include <utility> // pair, declval
+#include <valarray> // valarray
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename std::nullptr_t& n)
+{
+    if (JSON_UNLIKELY(not j.is_null()))
+    {
+        JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name())));
+    }
+    n = nullptr;
+}
 
 // overloads for basic_json template parameters
 template<typename BasicJsonType, typename ArithmeticType,
@@ -1109,6 +1285,23 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
     s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
 }
 
+template <
+    typename BasicJsonType, typename ConstructibleStringType,
+    enable_if_t <
+        is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and
+        not std::is_same<typename BasicJsonType::string_t,
+                         ConstructibleStringType>::value,
+        int > = 0 >
+void from_json(const BasicJsonType& j, ConstructibleStringType& s)
+{
+    if (JSON_UNLIKELY(not j.is_string()))
+    {
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
+    }
+
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
 template<typename BasicJsonType>
 void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
 {
@@ -1136,16 +1329,6 @@ void from_json(const BasicJsonType& j, EnumType& e)
     e = static_cast<EnumType>(val);
 }
 
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
-}
-
 // forward_list doesn't have an insert method
 template<typename BasicJsonType, typename T, typename Allocator,
          enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
@@ -1175,24 +1358,28 @@ void from_json(const BasicJsonType& j, std::valarray<T>& l)
     std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
 }
 
-template<typename BasicJsonType, typename CompatibleArrayType>
-void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/)
+template<typename BasicJsonType>
+void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)
 {
-    using std::end;
+    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
 
-    std::transform(j.begin(), j.end(),
-                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+template <typename BasicJsonType, typename T, std::size_t N>
+auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
+                          priority_tag<2> /*unused*/)
+-> decltype(j.template get<T>(), void())
+{
+    for (std::size_t i = 0; i < N; ++i)
     {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename CompatibleArrayType::value_type>();
-    });
+        arr[i] = j.at(i).template get<T>();
+    }
 }
 
-template<typename BasicJsonType, typename CompatibleArrayType>
-auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/)
+template<typename BasicJsonType, typename ConstructibleArrayType>
+auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
 -> decltype(
-    arr.reserve(std::declval<typename CompatibleArrayType::size_type>()),
+    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
+    j.template get<typename ConstructibleArrayType::value_type>(),
     void())
 {
     using std::end;
@@ -1203,36 +1390,51 @@ auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio
     {
         // get<BasicJsonType>() returns *this, this won't call a from_json
         // method when value_type is BasicJsonType
-        return i.template get<typename CompatibleArrayType::value_type>();
+        return i.template get<typename ConstructibleArrayType::value_type>();
     });
 }
 
-template<typename BasicJsonType, typename T, std::size_t N>
-void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/)
+template <typename BasicJsonType, typename ConstructibleArrayType>
+void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
+                          priority_tag<0> /*unused*/)
 {
-    for (std::size_t i = 0; i < N; ++i)
+    using std::end;
+
+    std::transform(
+        j.begin(), j.end(), std::inserter(arr, end(arr)),
+        [](const BasicJsonType & i)
     {
-        arr[i] = j.at(i).template get<T>();
-    }
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename ConstructibleArrayType::value_type>();
+    });
 }
 
-template<typename BasicJsonType, typename CompatibleArrayType,
-         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and
-                     std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and
-                     not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
-void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
+template <typename BasicJsonType, typename ConstructibleArrayType,
+          enable_if_t <
+              is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and
+              not is_basic_json<ConstructibleArrayType>::value,
+              int > = 0 >
+
+auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
+-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
+j.template get<typename ConstructibleArrayType::value_type>(),
+void())
 {
     if (JSON_UNLIKELY(not j.is_array()))
     {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+        JSON_THROW(type_error::create(302, "type must be array, but is " +
+                                      std::string(j.type_name())));
     }
 
-    from_json_array_impl(j, arr, priority_tag<2> {});
+    from_json_array_impl(j, arr, priority_tag<3> {});
 }
 
-template<typename BasicJsonType, typename CompatibleObjectType,
-         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
-void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
+template<typename BasicJsonType, typename ConstructibleObjectType,
+         enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
+void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
 {
     if (JSON_UNLIKELY(not j.is_object()))
     {
@@ -1240,13 +1442,13 @@ void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
     }
 
     auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
-    using value_type = typename CompatibleObjectType::value_type;
+    using value_type = typename ConstructibleObjectType::value_type;
     std::transform(
         inner_object->begin(), inner_object->end(),
         std::inserter(obj, obj.begin()),
         [](typename BasicJsonType::object_t::value_type const & p)
     {
-        return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>());
+        return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
     });
 }
 
@@ -1299,7 +1501,7 @@ void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
 }
 
 template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>)
+void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/)
 {
     t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
 }
@@ -1310,657 +1512,1299 @@ void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
     from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
 }
 
-struct to_json_fn
+template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
 {
-  private:
-    template<typename BasicJsonType, typename T>
-    auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
-    -> decltype(to_json(j, std::forward<T>(val)), void())
+    if (JSON_UNLIKELY(not j.is_array()))
     {
-        return to_json(j, std::forward<T>(val));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
     }
-
-    template<typename BasicJsonType, typename T>
-    void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept
+    for (const auto& p : j)
     {
-        static_assert(sizeof(BasicJsonType) == 0,
-                      "could not find to_json() method in T's namespace");
-
-#ifdef _MSC_VER
-        // MSVC does not show a stacktrace for the above assert
-        using decayed = uncvref_t<T>;
-        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
-                      "forcing MSVC stacktrace to show which T we're talking about.");
-#endif
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
     }
+}
 
-  public:
-    template<typename BasicJsonType, typename T>
-    void operator()(BasicJsonType& j, T&& val) const
-    noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {})))
+template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
+          typename = enable_if_t<not std::is_constructible<
+                                     typename BasicJsonType::string_t, Key>::value>>
+void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
     {
-        return call(j, std::forward<T>(val), priority_tag<1> {});
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
     }
-};
+    for (const auto& p : j)
+    {
+        if (JSON_UNLIKELY(not p.is_array()))
+        {
+            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
+        }
+        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+    }
+}
 
 struct from_json_fn
 {
-  private:
     template<typename BasicJsonType, typename T>
-    auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const
+    auto operator()(const BasicJsonType& j, T& val) const
     noexcept(noexcept(from_json(j, val)))
     -> decltype(from_json(j, val), void())
     {
         return from_json(j, val);
     }
-
-    template<typename BasicJsonType, typename T>
-    void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept
-    {
-        static_assert(sizeof(BasicJsonType) == 0,
-                      "could not find from_json() method in T's namespace");
-#ifdef _MSC_VER
-        // MSVC does not show a stacktrace for the above assert
-        using decayed = uncvref_t<T>;
-        static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0,
-                      "forcing MSVC stacktrace to show which T we're talking about.");
-#endif
-    }
-
-  public:
-    template<typename BasicJsonType, typename T>
-    void operator()(const BasicJsonType& j, T& val) const
-    noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {})))
-    {
-        return call(j, val, priority_tag<1> {});
-    }
 };
+}  // namespace detail
 
-// taken from ranges-v3
-template<typename T>
-struct static_const
+/// namespace to hold default `from_json` function
+/// to see why this is required:
+/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
+namespace
 {
-    static constexpr T value{};
-};
+constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
 
-template<typename T>
-constexpr T static_const<T>::value;
+// #include <nlohmann/detail/conversions/to_json.hpp>
 
-////////////////////
-// input adapters //
-////////////////////
 
-/*!
-@brief abstract input adapter interface
+#include <ciso646> // or, and, not
+#include <iterator> // begin, end
+#include <tuple> // tuple, get
+#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type
+#include <utility> // move, forward, declval, pair
+#include <valarray> // valarray
+#include <vector> // vector
 
-Produces a stream of std::char_traits<char>::int_type characters from a
-std::istream, a buffer, or some other input type.  Accepts the return of exactly
-one non-EOF character for future input.  The int_type characters returned
-consist of all valid char values as positive values (typically unsigned char),
-plus an EOF value outside that range, specified by the value of the function
-std::char_traits<char>::eof().  This value is typically -1, but could be any
-arbitrary value which is not a valid char value.
-*/
-struct input_adapter_protocol
-{
-    /// get a character [0,255] or std::char_traits<char>::eof().
-    virtual std::char_traits<char>::int_type get_character() = 0;
-    /// restore the last non-eof() character to input
-    virtual void unget_character() = 0;
-    virtual ~input_adapter_protocol() = default;
-};
+// #include <nlohmann/detail/meta/cpp_future.hpp>
 
-/// a type to simplify interfaces
-using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
+// #include <nlohmann/detail/meta/type_traits.hpp>
 
-/*!
-Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
-beginning of input. Does not support changing the underlying std::streambuf
-in mid-input. Maintains underlying std::istream and std::streambuf to support
-subsequent use of standard std::istream operations to process any input
-characters following those used in parsing the JSON input.  Clears the
-std::istream flags; any input errors (e.g., EOF) will be detected by the first
-subsequent call for input from the std::istream.
-*/
-class input_stream_adapter : public input_adapter_protocol
+// #include <nlohmann/detail/value_t.hpp>
+
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
+
+
+#include <cstddef> // size_t
+#include <string> // string, to_string
+#include <iterator> // input_iterator_tag
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
 {
-  public:
-    ~input_stream_adapter() override
+namespace detail
+{
+/// proxy class for the items() function
+template<typename IteratorType> class iteration_proxy
+{
+  private:
+    /// helper class for iteration
+    class iteration_proxy_internal
     {
-        // clear stream flags; we use underlying streambuf I/O, do not
-        // maintain ifstream flags
-        is.clear();
-    }
+      public:
+        using difference_type = std::ptrdiff_t;
+        using value_type = iteration_proxy_internal;
+        using pointer = iteration_proxy_internal*;
+        using reference = iteration_proxy_internal&;
+        using iterator_category = std::input_iterator_tag;
 
-    explicit input_stream_adapter(std::istream& i)
-        : is(i), sb(*i.rdbuf())
-    {
-        // skip byte order mark
-        std::char_traits<char>::int_type c;
-        if ((c = get_character()) == 0xEF)
+      private:
+        /// the iterator
+        IteratorType anchor;
+        /// an index for arrays (used to create key names)
+        std::size_t array_index = 0;
+        /// last stringified array index
+        mutable std::size_t array_index_last = 0;
+        /// a string representation of the array index
+        mutable std::string array_index_str = "0";
+        /// an empty string (to return a reference for primitive values)
+        const std::string empty_str = "";
+
+      public:
+        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
+
+        /// dereference operator (needed for range-based for)
+        iteration_proxy_internal& operator*()
+        {
+            return *this;
+        }
+
+        /// increment operator (needed for range-based for)
+        iteration_proxy_internal& operator++()
+        {
+            ++anchor;
+            ++array_index;
+
+            return *this;
+        }
+
+        /// equality operator (needed for InputIterator)
+        bool operator==(const iteration_proxy_internal& o) const noexcept
+        {
+            return anchor == o.anchor;
+        }
+
+        /// inequality operator (needed for range-based for)
+        bool operator!=(const iteration_proxy_internal& o) const noexcept
+        {
+            return anchor != o.anchor;
+        }
+
+        /// return key of the iterator
+        const std::string& key() const
         {
-            if ((c = get_character()) == 0xBB)
+            assert(anchor.m_object != nullptr);
+
+            switch (anchor.m_object->type())
             {
-                if ((c = get_character()) == 0xBF)
-                {
-                    return; // Ignore BOM
-                }
-                else if (c != std::char_traits<char>::eof())
+                // use integer array index as key
+                case value_t::array:
                 {
-                    is.unget();
+                    if (array_index != array_index_last)
+                    {
+                        array_index_str = std::to_string(array_index);
+                        array_index_last = array_index;
+                    }
+                    return array_index_str;
                 }
-                is.putback('\xBB');
-            }
-            else if (c != std::char_traits<char>::eof())
-            {
-                is.unget();
+
+                // use key from the object
+                case value_t::object:
+                    return anchor.key();
+
+                // use an empty key for all primitive types
+                default:
+                    return empty_str;
             }
-            is.putback('\xEF');
         }
-        else if (c != std::char_traits<char>::eof())
+
+        /// return value of the iterator
+        typename IteratorType::reference value() const
         {
-            is.unget(); // no byte order mark; process as usual
+            return anchor.value();
         }
-    }
+    };
 
-    // delete because of pointer members
-    input_stream_adapter(const input_stream_adapter&) = delete;
-    input_stream_adapter& operator=(input_stream_adapter&) = delete;
+    /// the container to iterate
+    typename IteratorType::reference container;
 
-    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
-    // ensure that std::char_traits<char>::eof() and the character 0xFF do not
-    // end up as the same value, eg. 0xFFFFFFFF.
-    std::char_traits<char>::int_type get_character() override
+  public:
+    /// construct iteration proxy from a container
+    explicit iteration_proxy(typename IteratorType::reference cont) noexcept
+        : container(cont) {}
+
+    /// return iterator begin (needed for range-based for)
+    iteration_proxy_internal begin() noexcept
     {
-        return sb.sbumpc();
+        return iteration_proxy_internal(container.begin());
     }
 
-    void unget_character() override
+    /// return iterator end (needed for range-based for)
+    iteration_proxy_internal end() noexcept
     {
-        sb.sungetc();  // is.unget() avoided for performance
+        return iteration_proxy_internal(container.end());
     }
+};
+}  // namespace detail
+}  // namespace nlohmann
 
-  private:
-    /// the associated input stream
-    std::istream& is;
-    std::streambuf& sb;
+
+namespace nlohmann
+{
+namespace detail
+{
+//////////////////
+// constructors //
+//////////////////
+
+template<value_t> struct external_constructor;
+
+template<>
+struct external_constructor<value_t::boolean>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
+    {
+        j.m_type = value_t::boolean;
+        j.m_value = b;
+        j.assert_invariant();
+    }
 };
 
-/// input adapter for buffer input
-class input_buffer_adapter : public input_adapter_protocol
+template<>
+struct external_constructor<value_t::string>
 {
-  public:
-    input_buffer_adapter(const char* b, const std::size_t l)
-        : cursor(b), limit(b + l), start(b)
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
     {
-        // skip byte order mark
-        if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
-        {
-            cursor += 3;
-        }
+        j.m_type = value_t::string;
+        j.m_value = s;
+        j.assert_invariant();
     }
 
-    // delete because of pointer members
-    input_buffer_adapter(const input_buffer_adapter&) = delete;
-    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = std::move(s);
+        j.assert_invariant();
+    }
 
-    std::char_traits<char>::int_type get_character() noexcept override
+    template<typename BasicJsonType, typename CompatibleStringType,
+             enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleStringType& str)
     {
-        if (JSON_LIKELY(cursor < limit))
-        {
-            return std::char_traits<char>::to_int_type(*(cursor++));
-        }
+        j.m_type = value_t::string;
+        j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
+        j.assert_invariant();
+    }
+};
 
-        return std::char_traits<char>::eof();
+template<>
+struct external_constructor<value_t::number_float>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
+    {
+        j.m_type = value_t::number_float;
+        j.m_value = val;
+        j.assert_invariant();
     }
+};
 
-    void unget_character() noexcept override
+template<>
+struct external_constructor<value_t::number_unsigned>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
     {
-        if (JSON_LIKELY(cursor > start))
-        {
-            --cursor;
-        }
+        j.m_type = value_t::number_unsigned;
+        j.m_value = val;
+        j.assert_invariant();
     }
+};
 
-  private:
-    /// pointer to the current character
-    const char* cursor;
-    /// pointer past the last character
-    const char* limit;
-    /// pointer to the first character
-    const char* start;
+template<>
+struct external_constructor<value_t::number_integer>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
+    {
+        j.m_type = value_t::number_integer;
+        j.m_value = val;
+        j.assert_invariant();
+    }
 };
 
-class input_adapter
+template<>
+struct external_constructor<value_t::array>
 {
-  public:
-    // native support
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = arr;
+        j.assert_invariant();
+    }
 
-    /// input adapter for input stream
-    input_adapter(std::istream& i)
-        : ia(std::make_shared<input_stream_adapter>(i)) {}
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = std::move(arr);
+        j.assert_invariant();
+    }
 
-    /// input adapter for input stream
-    input_adapter(std::istream&& i)
-        : ia(std::make_shared<input_stream_adapter>(i)) {}
+    template<typename BasicJsonType, typename CompatibleArrayType,
+             enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
+    {
+        using std::begin;
+        using std::end;
+        j.m_type = value_t::array;
+        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+        j.assert_invariant();
+    }
 
-    /// input adapter for buffer
-    template<typename CharT,
-             typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
-                 int>::type = 0>
-    input_adapter(CharT b, std::size_t l)
-        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
-
-    // derived support
-
-    /// input adapter for string literal
-    template<typename CharT,
-             typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
-                 int>::type = 0>
-    input_adapter(CharT b)
-        : input_adapter(reinterpret_cast<const char*>(b),
-                        std::strlen(reinterpret_cast<const char*>(b))) {}
-
-    /// input adapter for iterator range with contiguous storage
-    template<class IteratorType,
-             typename std::enable_if<
-                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
-                 int>::type = 0>
-    input_adapter(IteratorType first, IteratorType last)
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
     {
-        // assertion to check that the iterator range is indeed contiguous,
-        // see http://stackoverflow.com/a/35008842/266378 for more discussion
-        assert(std::accumulate(
-                   first, last, std::pair<bool, int>(true, 0),
-                   [&first](std::pair<bool, int> res, decltype(*first) val)
-        {
-            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
-            return res;
-        }).first);
-
-        // assertion to check that each element is 1 byte long
-        static_assert(
-            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
-            "each element in the iterator range must have the size of 1 byte");
-
-        const auto len = static_cast<size_t>(std::distance(first, last));
-        if (JSON_LIKELY(len > 0))
-        {
-            // there is at least one element: use the address of first
-            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
-        }
-        else
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->reserve(arr.size());
+        for (const bool x : arr)
         {
-            // the address of first cannot be used: use nullptr
-            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
+            j.m_value.array->push_back(x);
         }
+        j.assert_invariant();
     }
 
-    /// input adapter for array
-    template<class T, std::size_t N>
-    input_adapter(T (&array)[N])
-        : input_adapter(std::begin(array), std::end(array)) {}
-
-    /// input adapter for contiguous container
-    template<class ContiguousContainer, typename
-             std::enable_if<not std::is_pointer<ContiguousContainer>::value and
-                            std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
-                            int>::type = 0>
-    input_adapter(const ContiguousContainer& c)
-        : input_adapter(std::begin(c), std::end(c)) {}
-
-    operator input_adapter_t()
+    template<typename BasicJsonType, typename T,
+             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
     {
-        return ia;
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->resize(arr.size());
+        std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());
+        j.assert_invariant();
     }
-
-  private:
-    /// the actual adapter
-    input_adapter_t ia = nullptr;
 };
 
-//////////////////////
-// lexer and parser //
-//////////////////////
-
-/*!
-@brief lexical analysis
-
-This class organizes the lexical analysis during JSON deserialization.
-*/
-template<typename BasicJsonType>
-class lexer
+template<>
+struct external_constructor<value_t::object>
 {
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-
-  public:
-    /// token types for the parser
-    enum class token_type
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
     {
-        uninitialized,    ///< indicating the scanner is uninitialized
-        literal_true,     ///< the `true` literal
-        literal_false,    ///< the `false` literal
-        literal_null,     ///< the `null` literal
-        value_string,     ///< a string -- use get_string() for actual value
-        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
-        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
-        value_float,      ///< an floating point number -- use get_number_float() for actual value
-        begin_array,      ///< the character for array begin `[`
-        begin_object,     ///< the character for object begin `{`
-        end_array,        ///< the character for array end `]`
-        end_object,       ///< the character for object end `}`
-        name_separator,   ///< the name separator `:`
-        value_separator,  ///< the value separator `,`
-        parse_error,      ///< indicating a parse error
-        end_of_input,     ///< indicating the end of the input buffer
-        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
-    };
+        j.m_type = value_t::object;
+        j.m_value = obj;
+        j.assert_invariant();
+    }
 
-    /// return name of values of type token_type (only used for errors)
-    static const char* token_type_name(const token_type t) noexcept
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
     {
-        switch (t)
-        {
-            case token_type::uninitialized:
-                return "<uninitialized>";
-            case token_type::literal_true:
-                return "true literal";
-            case token_type::literal_false:
-                return "false literal";
-            case token_type::literal_null:
-                return "null literal";
-            case token_type::value_string:
-                return "string literal";
-            case lexer::token_type::value_unsigned:
-            case lexer::token_type::value_integer:
-            case lexer::token_type::value_float:
-                return "number literal";
-            case token_type::begin_array:
-                return "'['";
-            case token_type::begin_object:
-                return "'{'";
-            case token_type::end_array:
-                return "']'";
-            case token_type::end_object:
-                return "'}'";
-            case token_type::name_separator:
-                return "':'";
-            case token_type::value_separator:
-                return "','";
-            case token_type::parse_error:
-                return "<parse error>";
-            case token_type::end_of_input:
-                return "end of input";
-            case token_type::literal_or_value:
-                return "'[', '{', or a literal";
-            default: // catch non-enum values
-                return "unknown token"; // LCOV_EXCL_LINE
-        }
+        j.m_type = value_t::object;
+        j.m_value = std::move(obj);
+        j.assert_invariant();
     }
 
-    explicit lexer(detail::input_adapter_t adapter)
-        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
-
-    // delete because of pointer members
-    lexer(const lexer&) = delete;
-    lexer& operator=(lexer&) = delete;
-
-  private:
-    /////////////////////
-    // locales
-    /////////////////////
-
-    /// return the locale-dependent decimal point
-    static char get_decimal_point() noexcept
+    template<typename BasicJsonType, typename CompatibleObjectType,
+             enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
     {
-        const auto loc = localeconv();
-        assert(loc != nullptr);
-        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+        using std::begin;
+        using std::end;
+
+        j.m_type = value_t::object;
+        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
+        j.assert_invariant();
     }
+};
 
-    /////////////////////
-    // scan functions
-    /////////////////////
+/////////////
+// to_json //
+/////////////
 
-    /*!
-    @brief get codepoint from 4 hex characters following `\u`
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+void to_json(BasicJsonType& j, T b) noexcept
+{
+    external_constructor<value_t::boolean>::construct(j, b);
+}
 
-    For input "\u c1 c2 c3 c4" the codepoint is:
-      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
-    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+template<typename BasicJsonType, typename CompatibleString,
+         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleString& s)
+{
+    external_constructor<value_t::string>::construct(j, s);
+}
 
-    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
-    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
-    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
-    between the ASCII value of the character and the desired integer value.
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+{
+    external_constructor<value_t::string>::construct(j, std::move(s));
+}
 
-    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
-            non-hex character)
-    */
-    int get_codepoint()
-    {
-        // this function only makes sense after reading `\u`
-        assert(current == 'u');
-        int codepoint = 0;
+template<typename BasicJsonType, typename FloatType,
+         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
+void to_json(BasicJsonType& j, FloatType val) noexcept
+{
+    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
+}
 
-        const auto factors = { 12, 8, 4, 0 };
-        for (const auto factor : factors)
-        {
-            get();
+template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
+void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+{
+    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
+}
 
-            if (current >= '0' and current <= '9')
-            {
-                codepoint += ((current - 0x30) << factor);
-            }
-            else if (current >= 'A' and current <= 'F')
-            {
-                codepoint += ((current - 0x37) << factor);
-            }
-            else if (current >= 'a' and current <= 'f')
-            {
-                codepoint += ((current - 0x57) << factor);
-            }
-            else
-            {
-                return -1;
-            }
-        }
+template<typename BasicJsonType, typename CompatibleNumberIntegerType,
+         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
+void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
+{
+    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
+}
 
-        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
-        return codepoint;
-    }
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void to_json(BasicJsonType& j, EnumType e) noexcept
+{
+    using underlying_type = typename std::underlying_type<EnumType>::type;
+    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
+}
 
-    /*!
-    @brief check if the next byte(s) are inside a given range
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, const std::vector<bool>& e)
+{
+    external_constructor<value_t::array>::construct(j, e);
+}
 
-    Adds the current byte and, for each passed range, reads a new byte and
-    checks if it is inside the range. If a violation was detected, set up an
-    error message and return false. Otherwise, return true.
+template <typename BasicJsonType, typename CompatibleArrayType,
+          enable_if_t<is_compatible_array_type<BasicJsonType,
+                      CompatibleArrayType>::value and
+                      not is_compatible_object_type<
+                          BasicJsonType, CompatibleArrayType>::value and
+                      not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and
+                      not is_basic_json<CompatibleArrayType>::value,
+                      int> = 0>
+void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
 
-    @param[in] ranges  list of integers; interpreted as list of pairs of
-                       inclusive lower and upper bound, respectively
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+void to_json(BasicJsonType& j, const std::valarray<T>& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
 
-    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
-         1, 2, or 3 pairs. This precondition is enforced by an assertion.
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
 
-    @return true if and only if no range violation was detected
-    */
-    bool next_byte_in_range(std::initializer_list<int> ranges)
-    {
-        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
-        add(current);
+template<typename BasicJsonType, typename CompatibleObjectType,
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
+{
+    external_constructor<value_t::object>::construct(j, obj);
+}
 
-        for (auto range = ranges.begin(); range != ranges.end(); ++range)
-        {
-            get();
-            if (JSON_LIKELY(*range <= current and current <= *(++range)))
-            {
-                add(current);
-            }
-            else
-            {
-                error_message = "invalid string: ill-formed UTF-8 byte";
-                return false;
-            }
-        }
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+{
+    external_constructor<value_t::object>::construct(j, std::move(obj));
+}
 
-        return true;
-    }
+template <
+    typename BasicJsonType, typename T, std::size_t N,
+    enable_if_t<not std::is_constructible<typename BasicJsonType::string_t,
+                const T (&)[N]>::value,
+                int> = 0 >
+void to_json(BasicJsonType& j, const T (&arr)[N])
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
 
-    /*!
-    @brief scan a string literal
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::pair<Args...>& p)
+{
+    j = {p.first, p.second};
+}
 
-    This function scans a string according to Sect. 7 of RFC 7159. While
-    scanning, bytes are escaped and copied into buffer yytext. Then the function
-    returns successfully, yytext is *not* null-terminated (as it may contain \0
-    bytes), and yytext.size() is the number of bytes in the string.
+// for https://github.com/nlohmann/json/pull/1134
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_same<T, typename iteration_proxy<typename BasicJsonType::iterator>::iteration_proxy_internal>::value, int> = 0>
+void to_json(BasicJsonType& j, const T& b)
+{
+    j = {{b.key(), b.value()}};
+}
 
-    @return token_type::value_string if string could be successfully scanned,
-            token_type::parse_error otherwise
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
+{
+    j = {std::get<Idx>(t)...};
+}
 
-    @note In case of errors, variable error_message contains a textual
-          description.
-    */
-    token_type scan_string()
-    {
-        // reset yytext (ignore opening quote)
-        reset();
+template<typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
+{
+    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
 
-        // we entered the function by reading an open quote
-        assert(current == '\"');
+struct to_json_fn
+{
+    template<typename BasicJsonType, typename T>
+    auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    -> decltype(to_json(j, std::forward<T>(val)), void())
+    {
+        return to_json(j, std::forward<T>(val));
+    }
+};
+}  // namespace detail
 
-        while (true)
-        {
-            // get next character
-            switch (get())
-            {
-                // end of file while parsing string
-                case std::char_traits<char>::eof():
-                {
-                    error_message = "invalid string: missing closing quote";
-                    return token_type::parse_error;
-                }
+/// namespace to hold default `to_json` function
+namespace
+{
+constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
+} // namespace
+}  // namespace nlohmann
 
-                // closing quote
-                case '\"':
-                {
-                    return token_type::value_string;
-                }
+// #include <nlohmann/detail/input/input_adapters.hpp>
 
-                // escapes
-                case '\\':
-                {
-                    switch (get())
-                    {
-                        // quotation mark
-                        case '\"':
-                            add('\"');
-                            break;
-                        // reverse solidus
-                        case '\\':
-                            add('\\');
-                            break;
-                        // solidus
-                        case '/':
-                            add('/');
-                            break;
-                        // backspace
-                        case 'b':
-                            add('\b');
-                            break;
-                        // form feed
-                        case 'f':
-                            add('\f');
-                            break;
-                        // line feed
-                        case 'n':
-                            add('\n');
-                            break;
-                        // carriage return
-                        case 'r':
-                            add('\r');
-                            break;
-                        // tab
-                        case 't':
-                            add('\t');
-                            break;
 
-                        // unicode escapes
-                        case 'u':
-                        {
-                            const int codepoint1 = get_codepoint();
-                            int codepoint = codepoint1; // start with codepoint1
+#include <cassert> // assert
+#include <cstddef> // size_t
+#include <cstring> // strlen
+#include <istream> // istream
+#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
+#include <memory> // shared_ptr, make_shared, addressof
+#include <numeric> // accumulate
+#include <string> // string, char_traits
+#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
+#include <utility> // pair, declval
 
-                            if (JSON_UNLIKELY(codepoint1 == -1))
-                            {
-                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                return token_type::parse_error;
-                            }
+// #include <nlohmann/detail/macro_scope.hpp>
 
-                            // check if code point is a high surrogate
-                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
-                            {
-                                // expect next \uxxxx entry
-                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
-                                {
-                                    const int codepoint2 = get_codepoint();
 
-                                    if (JSON_UNLIKELY(codepoint2 == -1))
-                                    {
-                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                        return token_type::parse_error;
-                                    }
+namespace nlohmann
+{
+namespace detail
+{
+/// the supported input formats
+enum class input_format_t { json, cbor, msgpack, ubjson, bson };
 
-                                    // check if codepoint2 is a low surrogate
-                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
-                                    {
-                                        // overwrite codepoint
-                                        codepoint =
-                                            // high surrogate occupies the most significant 22 bits
-                                            (codepoint1 << 10)
-                                            // low surrogate occupies the least significant 15 bits
-                                            + codepoint2
-                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
-                                            // in the result so we have to subtract with:
-                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
-                                            - 0x35FDC00;
-                                    }
-                                    else
-                                    {
-                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                        return token_type::parse_error;
-                                    }
-                                }
-                                else
-                                {
-                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                    return token_type::parse_error;
-                                }
-                            }
-                            else
-                            {
-                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
-                                {
-                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
-                                    return token_type::parse_error;
-                                }
-                            }
+////////////////////
+// input adapters //
+////////////////////
 
-                            // result of the above calculation yields a proper codepoint
-                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+/*!
+@brief abstract input adapter interface
 
-                            // translate codepoint into bytes
-                            if (codepoint < 0x80)
-                            {
-                                // 1-byte characters: 0xxxxxxx (ASCII)
+Produces a stream of std::char_traits<char>::int_type characters from a
+std::istream, a buffer, or some other input type. Accepts the return of
+exactly one non-EOF character for future input. The int_type characters
+returned consist of all valid char values as positive values (typically
+unsigned char), plus an EOF value outside that range, specified by the value
+of the function std::char_traits<char>::eof(). This value is typically -1, but
+could be any arbitrary value which is not a valid char value.
+*/
+struct input_adapter_protocol
+{
+    /// get a character [0,255] or std::char_traits<char>::eof().
+    virtual std::char_traits<char>::int_type get_character() = 0;
+    virtual ~input_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
+
+/*!
+Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
+beginning of input. Does not support changing the underlying std::streambuf
+in mid-input. Maintains underlying std::istream and std::streambuf to support
+subsequent use of standard std::istream operations to process any input
+characters following those used in parsing the JSON input.  Clears the
+std::istream flags; any input errors (e.g., EOF) will be detected by the first
+subsequent call for input from the std::istream.
+*/
+class input_stream_adapter : public input_adapter_protocol
+{
+  public:
+    ~input_stream_adapter() override
+    {
+        // clear stream flags; we use underlying streambuf I/O, do not
+        // maintain ifstream flags
+        is.clear();
+    }
+
+    explicit input_stream_adapter(std::istream& i)
+        : is(i), sb(*i.rdbuf())
+    {}
+
+    // delete because of pointer members
+    input_stream_adapter(const input_stream_adapter&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&) = delete;
+    input_stream_adapter(input_stream_adapter&&) = delete;
+    input_stream_adapter& operator=(input_stream_adapter&&) = delete;
+
+    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
+    // ensure that std::char_traits<char>::eof() and the character 0xFF do not
+    // end up as the same value, eg. 0xFFFFFFFF.
+    std::char_traits<char>::int_type get_character() override
+    {
+        return sb.sbumpc();
+    }
+
+  private:
+    /// the associated input stream
+    std::istream& is;
+    std::streambuf& sb;
+};
+
+/// input adapter for buffer input
+class input_buffer_adapter : public input_adapter_protocol
+{
+  public:
+    input_buffer_adapter(const char* b, const std::size_t l) noexcept
+        : cursor(b), limit(b + l)
+    {}
+
+    // delete because of pointer members
+    input_buffer_adapter(const input_buffer_adapter&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+    input_buffer_adapter(input_buffer_adapter&&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&&) = delete;
+    ~input_buffer_adapter() override = default;
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        if (JSON_LIKELY(cursor < limit))
+        {
+            return std::char_traits<char>::to_int_type(*(cursor++));
+        }
+
+        return std::char_traits<char>::eof();
+    }
+
+  private:
+    /// pointer to the current character
+    const char* cursor;
+    /// pointer past the last character
+    const char* const limit;
+};
+
+template<typename WideStringType, size_t T>
+struct wide_string_input_helper
+{
+    // UTF-32
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
+    {
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
+        {
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-32 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F);
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (wc <= 0xFFFF)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F);
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else if (wc <= 0x10FFFF)
+            {
+                utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07);
+                utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F);
+                utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[3] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 4;
+            }
+            else
+            {
+                // unknown character
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+        }
+    }
+};
+
+template<typename WideStringType>
+struct wide_string_input_helper<WideStringType, 2>
+{
+    // UTF-16
+    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
+    {
+        utf8_bytes_index = 0;
+
+        if (current_wchar == str.size())
+        {
+            utf8_bytes[0] = std::char_traits<char>::eof();
+            utf8_bytes_filled = 1;
+        }
+        else
+        {
+            // get the current character
+            const auto wc = static_cast<int>(str[current_wchar++]);
+
+            // UTF-16 to UTF-8 encoding
+            if (wc < 0x80)
+            {
+                utf8_bytes[0] = wc;
+                utf8_bytes_filled = 1;
+            }
+            else if (wc <= 0x7FF)
+            {
+                utf8_bytes[0] = 0xC0 | ((wc >> 6));
+                utf8_bytes[1] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 2;
+            }
+            else if (0xD800 > wc or wc >= 0xE000)
+            {
+                utf8_bytes[0] = 0xE0 | ((wc >> 12));
+                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
+                utf8_bytes[2] = 0x80 | (wc & 0x3F);
+                utf8_bytes_filled = 3;
+            }
+            else
+            {
+                if (current_wchar < str.size())
+                {
+                    const auto wc2 = static_cast<int>(str[current_wchar++]);
+                    const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF));
+                    utf8_bytes[0] = 0xf0 | (charcode >> 18);
+                    utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F);
+                    utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F);
+                    utf8_bytes[3] = 0x80 | (charcode & 0x3F);
+                    utf8_bytes_filled = 4;
+                }
+                else
+                {
+                    // unknown character
+                    ++current_wchar;
+                    utf8_bytes[0] = wc;
+                    utf8_bytes_filled = 1;
+                }
+            }
+        }
+    }
+};
+
+template<typename WideStringType>
+class wide_string_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit wide_string_input_adapter(const WideStringType& w)  noexcept
+        : str(w)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        // check if buffer needs to be filled
+        if (utf8_bytes_index == utf8_bytes_filled)
+        {
+            fill_buffer<sizeof(typename WideStringType::value_type)>();
+
+            assert(utf8_bytes_filled > 0);
+            assert(utf8_bytes_index == 0);
+        }
+
+        // use buffer
+        assert(utf8_bytes_filled > 0);
+        assert(utf8_bytes_index < utf8_bytes_filled);
+        return utf8_bytes[utf8_bytes_index++];
+    }
+
+  private:
+    template<size_t T>
+    void fill_buffer()
+    {
+        wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);
+    }
+
+    /// the wstring to process
+    const WideStringType& str;
+
+    /// index of the current wchar in str
+    std::size_t current_wchar = 0;
+
+    /// a buffer for UTF-8 bytes
+    std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
+
+    /// index to the utf8_codes array for the next valid byte
+    std::size_t utf8_bytes_index = 0;
+    /// number of valid bytes in the utf8_codes array
+    std::size_t utf8_bytes_filled = 0;
+};
+
+class input_adapter
+{
+  public:
+    // native support
+
+    /// input adapter for input stream
+    input_adapter(std::istream& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    /// input adapter for input stream
+    input_adapter(std::istream&& i)
+        : ia(std::make_shared<input_stream_adapter>(i)) {}
+
+    input_adapter(const std::wstring& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {}
+
+    input_adapter(const std::u16string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {}
+
+    input_adapter(const std::u32string& ws)
+        : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {}
+
+    /// input adapter for buffer
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b, std::size_t l)
+        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
+
+    // derived support
+
+    /// input adapter for string literal
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b)
+        : input_adapter(reinterpret_cast<const char*>(b),
+                        std::strlen(reinterpret_cast<const char*>(b))) {}
+
+    /// input adapter for iterator range with contiguous storage
+    template<class IteratorType,
+             typename std::enable_if<
+                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
+                 int>::type = 0>
+    input_adapter(IteratorType first, IteratorType last)
+    {
+#ifndef NDEBUG
+        // assertion to check that the iterator range is indeed contiguous,
+        // see http://stackoverflow.com/a/35008842/266378 for more discussion
+        const auto is_contiguous = std::accumulate(
+                                       first, last, std::pair<bool, int>(true, 0),
+                                       [&first](std::pair<bool, int> res, decltype(*first) val)
+        {
+            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
+            return res;
+        }).first;
+        assert(is_contiguous);
+#endif
+
+        // assertion to check that each element is 1 byte long
+        static_assert(
+            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+            "each element in the iterator range must have the size of 1 byte");
+
+        const auto len = static_cast<size_t>(std::distance(first, last));
+        if (JSON_LIKELY(len > 0))
+        {
+            // there is at least one element: use the address of first
+            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
+        }
+        else
+        {
+            // the address of first cannot be used: use nullptr
+            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
+        }
+    }
+
+    /// input adapter for array
+    template<class T, std::size_t N>
+    input_adapter(T (&array)[N])
+        : input_adapter(std::begin(array), std::end(array)) {}
+
+    /// input adapter for contiguous container
+    template<class ContiguousContainer, typename
+             std::enable_if<not std::is_pointer<ContiguousContainer>::value and
+                            std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+                            int>::type = 0>
+    input_adapter(const ContiguousContainer& c)
+        : input_adapter(std::begin(c), std::end(c)) {}
+
+    operator input_adapter_t()
+    {
+        return ia;
+    }
+
+  private:
+    /// the actual adapter
+    input_adapter_t ia = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/lexer.hpp>
+
+
+#include <clocale> // localeconv
+#include <cstddef> // size_t
+#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
+#include <cstdio> // snprintf
+#include <initializer_list> // initializer_list
+#include <string> // char_traits, string
+#include <vector> // vector
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/position_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////
+// lexer //
+///////////
+
+/*!
+@brief lexical analysis
+
+This class organizes the lexical analysis during JSON deserialization.
+*/
+template<typename BasicJsonType>
+class lexer
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+  public:
+    /// token types for the parser
+    enum class token_type
+    {
+        uninitialized,    ///< indicating the scanner is uninitialized
+        literal_true,     ///< the `true` literal
+        literal_false,    ///< the `false` literal
+        literal_null,     ///< the `null` literal
+        value_string,     ///< a string -- use get_string() for actual value
+        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
+        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
+        value_float,      ///< an floating point number -- use get_number_float() for actual value
+        begin_array,      ///< the character for array begin `[`
+        begin_object,     ///< the character for object begin `{`
+        end_array,        ///< the character for array end `]`
+        end_object,       ///< the character for object end `}`
+        name_separator,   ///< the name separator `:`
+        value_separator,  ///< the value separator `,`
+        parse_error,      ///< indicating a parse error
+        end_of_input,     ///< indicating the end of the input buffer
+        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
+    };
+
+    /// return name of values of type token_type (only used for errors)
+    static const char* token_type_name(const token_type t) noexcept
+    {
+        switch (t)
+        {
+            case token_type::uninitialized:
+                return "<uninitialized>";
+            case token_type::literal_true:
+                return "true literal";
+            case token_type::literal_false:
+                return "false literal";
+            case token_type::literal_null:
+                return "null literal";
+            case token_type::value_string:
+                return "string literal";
+            case lexer::token_type::value_unsigned:
+            case lexer::token_type::value_integer:
+            case lexer::token_type::value_float:
+                return "number literal";
+            case token_type::begin_array:
+                return "'['";
+            case token_type::begin_object:
+                return "'{'";
+            case token_type::end_array:
+                return "']'";
+            case token_type::end_object:
+                return "'}'";
+            case token_type::name_separator:
+                return "':'";
+            case token_type::value_separator:
+                return "','";
+            case token_type::parse_error:
+                return "<parse error>";
+            case token_type::end_of_input:
+                return "end of input";
+            case token_type::literal_or_value:
+                return "'[', '{', or a literal";
+            // LCOV_EXCL_START
+            default: // catch non-enum values
+                return "unknown token";
+                // LCOV_EXCL_STOP
+        }
+    }
+
+    explicit lexer(detail::input_adapter_t&& adapter)
+        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
+
+    // delete because of pointer members
+    lexer(const lexer&) = delete;
+    lexer(lexer&&) = delete;
+    lexer& operator=(lexer&) = delete;
+    lexer& operator=(lexer&&) = delete;
+    ~lexer() = default;
+
+  private:
+    /////////////////////
+    // locales
+    /////////////////////
+
+    /// return the locale-dependent decimal point
+    static char get_decimal_point() noexcept
+    {
+        const auto loc = localeconv();
+        assert(loc != nullptr);
+        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+    }
+
+    /////////////////////
+    // scan functions
+    /////////////////////
+
+    /*!
+    @brief get codepoint from 4 hex characters following `\u`
+
+    For input "\u c1 c2 c3 c4" the codepoint is:
+      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
+    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+
+    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
+    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
+    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
+    between the ASCII value of the character and the desired integer value.
+
+    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
+            non-hex character)
+    */
+    int get_codepoint()
+    {
+        // this function only makes sense after reading `\u`
+        assert(current == 'u');
+        int codepoint = 0;
+
+        const auto factors = { 12, 8, 4, 0 };
+        for (const auto factor : factors)
+        {
+            get();
+
+            if (current >= '0' and current <= '9')
+            {
+                codepoint += ((current - 0x30) << factor);
+            }
+            else if (current >= 'A' and current <= 'F')
+            {
+                codepoint += ((current - 0x37) << factor);
+            }
+            else if (current >= 'a' and current <= 'f')
+            {
+                codepoint += ((current - 0x57) << factor);
+            }
+            else
+            {
+                return -1;
+            }
+        }
+
+        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
+        return codepoint;
+    }
+
+    /*!
+    @brief check if the next byte(s) are inside a given range
+
+    Adds the current byte and, for each passed range, reads a new byte and
+    checks if it is inside the range. If a violation was detected, set up an
+    error message and return false. Otherwise, return true.
+
+    @param[in] ranges  list of integers; interpreted as list of pairs of
+                       inclusive lower and upper bound, respectively
+
+    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
+         1, 2, or 3 pairs. This precondition is enforced by an assertion.
+
+    @return true if and only if no range violation was detected
+    */
+    bool next_byte_in_range(std::initializer_list<int> ranges)
+    {
+        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
+        add(current);
+
+        for (auto range = ranges.begin(); range != ranges.end(); ++range)
+        {
+            get();
+            if (JSON_LIKELY(*range <= current and current <= *(++range)))
+            {
+                add(current);
+            }
+            else
+            {
+                error_message = "invalid string: ill-formed UTF-8 byte";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief scan a string literal
+
+    This function scans a string according to Sect. 7 of RFC 7159. While
+    scanning, bytes are escaped and copied into buffer token_buffer. Then the
+    function returns successfully, token_buffer is *not* null-terminated (as it
+    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
+    string.
+
+    @return token_type::value_string if string could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note In case of errors, variable error_message contains a textual
+          description.
+    */
+    token_type scan_string()
+    {
+        // reset token_buffer (ignore opening quote)
+        reset();
+
+        // we entered the function by reading an open quote
+        assert(current == '\"');
+
+        while (true)
+        {
+            // get next character
+            switch (get())
+            {
+                // end of file while parsing string
+                case std::char_traits<char>::eof():
+                {
+                    error_message = "invalid string: missing closing quote";
+                    return token_type::parse_error;
+                }
+
+                // closing quote
+                case '\"':
+                {
+                    return token_type::value_string;
+                }
+
+                // escapes
+                case '\\':
+                {
+                    switch (get())
+                    {
+                        // quotation mark
+                        case '\"':
+                            add('\"');
+                            break;
+                        // reverse solidus
+                        case '\\':
+                            add('\\');
+                            break;
+                        // solidus
+                        case '/':
+                            add('/');
+                            break;
+                        // backspace
+                        case 'b':
+                            add('\b');
+                            break;
+                        // form feed
+                        case 'f':
+                            add('\f');
+                            break;
+                        // line feed
+                        case 'n':
+                            add('\n');
+                            break;
+                        // carriage return
+                        case 'r':
+                            add('\r');
+                            break;
+                        // tab
+                        case 't':
+                            add('\t');
+                            break;
+
+                        // unicode escapes
+                        case 'u':
+                        {
+                            const int codepoint1 = get_codepoint();
+                            int codepoint = codepoint1; // start with codepoint1
+
+                            if (JSON_UNLIKELY(codepoint1 == -1))
+                            {
+                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                return token_type::parse_error;
+                            }
+
+                            // check if code point is a high surrogate
+                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
+                            {
+                                // expect next \uxxxx entry
+                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
+                                {
+                                    const int codepoint2 = get_codepoint();
+
+                                    if (JSON_UNLIKELY(codepoint2 == -1))
+                                    {
+                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                        return token_type::parse_error;
+                                    }
+
+                                    // check if codepoint2 is a low surrogate
+                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
+                                    {
+                                        // overwrite codepoint
+                                        codepoint =
+                                            // high surrogate occupies the most significant 22 bits
+                                            (codepoint1 << 10)
+                                            // low surrogate occupies the least significant 15 bits
+                                            + codepoint2
+                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
+                                            // in the result so we have to subtract with:
+                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+                                            - 0x35FDC00;
+                                    }
+                                    else
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                        return token_type::parse_error;
+                                    }
+                                }
+                                else
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+                            else
+                            {
+                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+
+                            // result of the above calculation yields a proper codepoint
+                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+                            // translate codepoint into bytes
+                            if (codepoint < 0x80)
+                            {
+                                // 1-byte characters: 0xxxxxxx (ASCII)
                                 add(codepoint);
                             }
                             else if (codepoint <= 0x7FF)
@@ -1985,901 +2829,2006 @@ class lexer
                                 add(0x80 | (codepoint & 0x3F));
                             }
 
-                            break;
-                        }
+                            break;
+                        }
+
+                        // other characters after escape
+                        default:
+                            error_message = "invalid string: forbidden character after backslash";
+                            return token_type::parse_error;
+                    }
+
+                    break;
+                }
+
+                // invalid control characters
+                case 0x00:
+                {
+                    error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000";
+                    return token_type::parse_error;
+                }
+
+                case 0x01:
+                {
+                    error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001";
+                    return token_type::parse_error;
+                }
+
+                case 0x02:
+                {
+                    error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002";
+                    return token_type::parse_error;
+                }
+
+                case 0x03:
+                {
+                    error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003";
+                    return token_type::parse_error;
+                }
+
+                case 0x04:
+                {
+                    error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004";
+                    return token_type::parse_error;
+                }
+
+                case 0x05:
+                {
+                    error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005";
+                    return token_type::parse_error;
+                }
+
+                case 0x06:
+                {
+                    error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006";
+                    return token_type::parse_error;
+                }
+
+                case 0x07:
+                {
+                    error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007";
+                    return token_type::parse_error;
+                }
+
+                case 0x08:
+                {
+                    error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b";
+                    return token_type::parse_error;
+                }
+
+                case 0x09:
+                {
+                    error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t";
+                    return token_type::parse_error;
+                }
+
+                case 0x0A:
+                {
+                    error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n";
+                    return token_type::parse_error;
+                }
+
+                case 0x0B:
+                {
+                    error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B";
+                    return token_type::parse_error;
+                }
+
+                case 0x0C:
+                {
+                    error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f";
+                    return token_type::parse_error;
+                }
+
+                case 0x0D:
+                {
+                    error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r";
+                    return token_type::parse_error;
+                }
+
+                case 0x0E:
+                {
+                    error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E";
+                    return token_type::parse_error;
+                }
+
+                case 0x0F:
+                {
+                    error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F";
+                    return token_type::parse_error;
+                }
+
+                case 0x10:
+                {
+                    error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010";
+                    return token_type::parse_error;
+                }
+
+                case 0x11:
+                {
+                    error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011";
+                    return token_type::parse_error;
+                }
+
+                case 0x12:
+                {
+                    error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012";
+                    return token_type::parse_error;
+                }
+
+                case 0x13:
+                {
+                    error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013";
+                    return token_type::parse_error;
+                }
+
+                case 0x14:
+                {
+                    error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014";
+                    return token_type::parse_error;
+                }
+
+                case 0x15:
+                {
+                    error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015";
+                    return token_type::parse_error;
+                }
+
+                case 0x16:
+                {
+                    error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016";
+                    return token_type::parse_error;
+                }
+
+                case 0x17:
+                {
+                    error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017";
+                    return token_type::parse_error;
+                }
+
+                case 0x18:
+                {
+                    error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018";
+                    return token_type::parse_error;
+                }
+
+                case 0x19:
+                {
+                    error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019";
+                    return token_type::parse_error;
+                }
+
+                case 0x1A:
+                {
+                    error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A";
+                    return token_type::parse_error;
+                }
+
+                case 0x1B:
+                {
+                    error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B";
+                    return token_type::parse_error;
+                }
+
+                case 0x1C:
+                {
+                    error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C";
+                    return token_type::parse_error;
+                }
+
+                case 0x1D:
+                {
+                    error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D";
+                    return token_type::parse_error;
+                }
+
+                case 0x1E:
+                {
+                    error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E";
+                    return token_type::parse_error;
+                }
+
+                case 0x1F:
+                {
+                    error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F";
+                    return token_type::parse_error;
+                }
+
+                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+                case 0x20:
+                case 0x21:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2A:
+                case 0x2B:
+                case 0x2C:
+                case 0x2D:
+                case 0x2E:
+                case 0x2F:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                case 0x38:
+                case 0x39:
+                case 0x3A:
+                case 0x3B:
+                case 0x3C:
+                case 0x3D:
+                case 0x3E:
+                case 0x3F:
+                case 0x40:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x44:
+                case 0x45:
+                case 0x46:
+                case 0x47:
+                case 0x48:
+                case 0x49:
+                case 0x4A:
+                case 0x4B:
+                case 0x4C:
+                case 0x4D:
+                case 0x4E:
+                case 0x4F:
+                case 0x50:
+                case 0x51:
+                case 0x52:
+                case 0x53:
+                case 0x54:
+                case 0x55:
+                case 0x56:
+                case 0x57:
+                case 0x58:
+                case 0x59:
+                case 0x5A:
+                case 0x5B:
+                case 0x5D:
+                case 0x5E:
+                case 0x5F:
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6A:
+                case 0x6B:
+                case 0x6C:
+                case 0x6D:
+                case 0x6E:
+                case 0x6F:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78:
+                case 0x79:
+                case 0x7A:
+                case 0x7B:
+                case 0x7C:
+                case 0x7D:
+                case 0x7E:
+                case 0x7F:
+                {
+                    add(current);
+                    break;
+                }
+
+                // U+0080..U+07FF: bytes C2..DF 80..BF
+                case 0xC2:
+                case 0xC3:
+                case 0xC4:
+                case 0xC5:
+                case 0xC6:
+                case 0xC7:
+                case 0xC8:
+                case 0xC9:
+                case 0xCA:
+                case 0xCB:
+                case 0xCC:
+                case 0xCD:
+                case 0xCE:
+                case 0xCF:
+                case 0xD0:
+                case 0xD1:
+                case 0xD2:
+                case 0xD3:
+                case 0xD4:
+                case 0xD5:
+                case 0xD6:
+                case 0xD7:
+                case 0xD8:
+                case 0xD9:
+                case 0xDA:
+                case 0xDB:
+                case 0xDC:
+                case 0xDD:
+                case 0xDE:
+                case 0xDF:
+                {
+                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+                case 0xE0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+                case 0xE1:
+                case 0xE2:
+                case 0xE3:
+                case 0xE4:
+                case 0xE5:
+                case 0xE6:
+                case 0xE7:
+                case 0xE8:
+                case 0xE9:
+                case 0xEA:
+                case 0xEB:
+                case 0xEC:
+                case 0xEE:
+                case 0xEF:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+                case 0xED:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+                case 0xF0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+                case 0xF1:
+                case 0xF2:
+                case 0xF3:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+                case 0xF4:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // remaining bytes (80..C1 and F5..FF) are ill-formed
+                default:
+                {
+                    error_message = "invalid string: ill-formed UTF-8 byte";
+                    return token_type::parse_error;
+                }
+            }
+        }
+    }
+
+    static void strtof(float& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtof(str, endptr);
+    }
+
+    static void strtof(double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtod(str, endptr);
+    }
+
+    static void strtof(long double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtold(str, endptr);
+    }
+
+    /*!
+    @brief scan a number literal
+
+    This function scans a string according to Sect. 6 of RFC 7159.
+
+    The function is realized with a deterministic finite state machine derived
+    from the grammar described in RFC 7159. Starting in state "init", the
+    input is read and used to determined the next state. Only state "done"
+    accepts the number. State "error" is a trap state to model errors. In the
+    table below, "anything" means any character but the ones listed before.
+
+    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
+    ---------|----------|----------|----------|---------|---------|----------|-----------
+    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
+    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
+    zero     | done     | done     | exponent | done    | done    | decimal1 | done
+    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
+    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
+    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
+    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
+    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
+    any2     | any2     | any2     | done     | done    | done    | done     | done
+
+    The state machine is realized with one label per state (prefixed with
+    "scan_number_") and `goto` statements between them. The state machine
+    contains cycles, but any cycle can be left when EOF is read. Therefore,
+    the function is guaranteed to terminate.
+
+    During scanning, the read bytes are stored in token_buffer. This string is
+    then converted to a signed integer, an unsigned integer, or a
+    floating-point number.
+
+    @return token_type::value_unsigned, token_type::value_integer, or
+            token_type::value_float if number could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note The scanner is independent of the current locale. Internally, the
+          locale's decimal point is used instead of `.` to work with the
+          locale-dependent converters.
+    */
+    token_type scan_number()  // lgtm [cpp/use-of-goto]
+    {
+        // reset token_buffer to store the number's bytes
+        reset();
+
+        // the type of the parsed number; initially set to unsigned; will be
+        // changed if minus sign, decimal point or exponent is read
+        token_type number_type = token_type::value_unsigned;
+
+        // state (init): we just found out we need to scan a number
+        switch (current)
+        {
+            case '-':
+            {
+                add(current);
+                goto scan_number_minus;
+            }
+
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            // LCOV_EXCL_START
+            default:
+            {
+                // all other characters are rejected outside scan_number()
+                assert(false);
+            }
+                // LCOV_EXCL_STOP
+        }
+
+scan_number_minus:
+        // state: we just parsed a leading minus sign
+        number_type = token_type::value_integer;
+        switch (get())
+        {
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '-'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_zero:
+        // state: we just parse a zero (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_any1:
+        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_decimal1:
+        // state: we just parsed a decimal point
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '.'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_decimal2:
+        // we just parsed at least one number after a decimal point
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_exponent:
+        // we just parsed an exponent
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '+':
+            case '-':
+            {
+                add(current);
+                goto scan_number_sign;
+            }
+
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message =
+                    "invalid number; expected '+', '-', or digit after exponent";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_sign:
+        // we just parsed an exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after exponent sign";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_any2:
+        // we just parsed a number after the exponent or exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_done:
+        // unget the character after the number (we only read it to know that
+        // we are done scanning a number)
+        unget();
+
+        char* endptr = nullptr;
+        errno = 0;
+
+        // try to parse integers first and fall back to floats
+        if (number_type == token_type::value_unsigned)
+        {
+            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == token_buffer.data() + token_buffer.size());
+
+            if (errno == 0)
+            {
+                value_unsigned = static_cast<number_unsigned_t>(x);
+                if (value_unsigned == x)
+                {
+                    return token_type::value_unsigned;
+                }
+            }
+        }
+        else if (number_type == token_type::value_integer)
+        {
+            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == token_buffer.data() + token_buffer.size());
+
+            if (errno == 0)
+            {
+                value_integer = static_cast<number_integer_t>(x);
+                if (value_integer == x)
+                {
+                    return token_type::value_integer;
+                }
+            }
+        }
+
+        // this code is reached if we parse a floating-point number or if an
+        // integer conversion above failed
+        strtof(value_float, token_buffer.data(), &endptr);
+
+        // we checked the number format before
+        assert(endptr == token_buffer.data() + token_buffer.size());
+
+        return token_type::value_float;
+    }
+
+    /*!
+    @param[in] literal_text  the literal text to expect
+    @param[in] length        the length of the passed literal text
+    @param[in] return_type   the token type to return on success
+    */
+    token_type scan_literal(const char* literal_text, const std::size_t length,
+                            token_type return_type)
+    {
+        assert(current == literal_text[0]);
+        for (std::size_t i = 1; i < length; ++i)
+        {
+            if (JSON_UNLIKELY(get() != literal_text[i]))
+            {
+                error_message = "invalid literal";
+                return token_type::parse_error;
+            }
+        }
+        return return_type;
+    }
+
+    /////////////////////
+    // input management
+    /////////////////////
+
+    /// reset token_buffer; current character is beginning of token
+    void reset() noexcept
+    {
+        token_buffer.clear();
+        token_string.clear();
+        token_string.push_back(std::char_traits<char>::to_char_type(current));
+    }
+
+    /*
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns a
+    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters
+    for use in error messages.
+
+    @return character read from the input
+    */
+    std::char_traits<char>::int_type get()
+    {
+        ++position.chars_read_total;
+        ++position.chars_read_current_line;
+
+        if (next_unget)
+        {
+            // just reset the next_unget variable and work with current
+            next_unget = false;
+        }
+        else
+        {
+            current = ia->get_character();
+        }
+
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            token_string.push_back(std::char_traits<char>::to_char_type(current));
+        }
+
+        if (current == '\n')
+        {
+            ++position.lines_read;
+            ++position.chars_read_current_line = 0;
+        }
+
+        return current;
+    }
+
+    /*!
+    @brief unget current character (read it again on next get)
+
+    We implement unget by setting variable next_unget to true. The input is not
+    changed - we just simulate ungetting by modifying chars_read_total,
+    chars_read_current_line, and token_string. The next call to get() will
+    behave as if the unget character is read again.
+    */
+    void unget()
+    {
+        next_unget = true;
+
+        --position.chars_read_total;
+
+        // in case we "unget" a newline, we have to also decrement the lines_read
+        if (position.chars_read_current_line == 0)
+        {
+            if (position.lines_read > 0)
+            {
+                --position.lines_read;
+            }
+        }
+        else
+        {
+            --position.chars_read_current_line;
+        }
+
+        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        {
+            assert(token_string.size() != 0);
+            token_string.pop_back();
+        }
+    }
+
+    /// add a character to token_buffer
+    void add(int c)
+    {
+        token_buffer.push_back(std::char_traits<char>::to_char_type(c));
+    }
+
+  public:
+    /////////////////////
+    // value getters
+    /////////////////////
+
+    /// return integer value
+    constexpr number_integer_t get_number_integer() const noexcept
+    {
+        return value_integer;
+    }
+
+    /// return unsigned integer value
+    constexpr number_unsigned_t get_number_unsigned() const noexcept
+    {
+        return value_unsigned;
+    }
+
+    /// return floating-point value
+    constexpr number_float_t get_number_float() const noexcept
+    {
+        return value_float;
+    }
+
+    /// return current string value (implicitly resets the token; useful only once)
+    string_t& get_string()
+    {
+        return token_buffer;
+    }
+
+    /////////////////////
+    // diagnostics
+    /////////////////////
+
+    /// return position of last read token
+    constexpr position_t get_position() const noexcept
+    {
+        return position;
+    }
+
+    /// return the last read token (for errors only).  Will never contain EOF
+    /// (an arbitrary value that is not a valid char value, often -1), because
+    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
+    std::string get_token_string() const
+    {
+        // escape control characters
+        std::string result;
+        for (const auto c : token_string)
+        {
+            if ('\x00' <= c and c <= '\x1F')
+            {
+                // escape control characters
+                char cs[9];
+                snprintf(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
+                result += cs;
+            }
+            else
+            {
+                // add character as is
+                result.push_back(c);
+            }
+        }
+
+        return result;
+    }
+
+    /// return syntax error message
+    constexpr const char* get_error_message() const noexcept
+    {
+        return error_message;
+    }
+
+    /////////////////////
+    // actual scanner
+    /////////////////////
+
+    /*!
+    @brief skip the UTF-8 byte order mark
+    @return true iff there is no BOM or the correct BOM has been skipped
+    */
+    bool skip_bom()
+    {
+        if (get() == 0xEF)
+        {
+            // check if we completely parse the BOM
+            return get() == 0xBB and get() == 0xBF;
+        }
+
+        // the first character is not the beginning of the BOM; unget it to
+        // process is later
+        unget();
+        return true;
+    }
+
+    token_type scan()
+    {
+        // initially, skip the BOM
+        if (position.chars_read_total == 0 and not skip_bom())
+        {
+            error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
+            return token_type::parse_error;
+        }
+
+        // read next character and ignore whitespace
+        do
+        {
+            get();
+        }
+        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
+
+        switch (current)
+        {
+            // structural characters
+            case '[':
+                return token_type::begin_array;
+            case ']':
+                return token_type::end_array;
+            case '{':
+                return token_type::begin_object;
+            case '}':
+                return token_type::end_object;
+            case ':':
+                return token_type::name_separator;
+            case ',':
+                return token_type::value_separator;
+
+            // literals
+            case 't':
+                return scan_literal("true", 4, token_type::literal_true);
+            case 'f':
+                return scan_literal("false", 5, token_type::literal_false);
+            case 'n':
+                return scan_literal("null", 4, token_type::literal_null);
+
+            // string
+            case '\"':
+                return scan_string();
+
+            // number
+            case '-':
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+                return scan_number();
+
+            // end of input (the null byte is needed when parsing from
+            // string literals)
+            case '\0':
+            case std::char_traits<char>::eof():
+                return token_type::end_of_input;
+
+            // error
+            default:
+                error_message = "invalid literal";
+                return token_type::parse_error;
+        }
+    }
+
+  private:
+    /// input adapter
+    detail::input_adapter_t ia = nullptr;
+
+    /// the current character
+    std::char_traits<char>::int_type current = std::char_traits<char>::eof();
+
+    /// whether the next get() call should just return current
+    bool next_unget = false;
+
+    /// the start position of the current token
+    position_t position;
+
+    /// raw input token string (for error messages)
+    std::vector<char> token_string {};
+
+    /// buffer for variable-length tokens (numbers, strings)
+    string_t token_buffer {};
+
+    /// a description of occurred lexer errors
+    const char* error_message = "";
+
+    // number values
+    number_integer_t value_integer = 0;
+    number_unsigned_t value_unsigned = 0;
+    number_float_t value_float = 0;
+
+    /// the decimal point
+    const char decimal_point_char = '.';
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/parser.hpp>
+
+
+#include <cassert> // assert
+#include <cmath> // isfinite
+#include <cstdint> // uint8_t
+#include <functional> // function
+#include <string> // string
+#include <utility> // move
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+
+#include <cstdint> // size_t
+#include <utility> // declval
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename T>
+using null_function_t = decltype(std::declval<T&>().null());
+
+template <typename T>
+using boolean_function_t =
+    decltype(std::declval<T&>().boolean(std::declval<bool>()));
+
+template <typename T, typename Integer>
+using number_integer_function_t =
+    decltype(std::declval<T&>().number_integer(std::declval<Integer>()));
+
+template <typename T, typename Unsigned>
+using number_unsigned_function_t =
+    decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));
+
+template <typename T, typename Float, typename String>
+using number_float_function_t = decltype(std::declval<T&>().number_float(
+                                    std::declval<Float>(), std::declval<const String&>()));
+
+template <typename T, typename String>
+using string_function_t =
+    decltype(std::declval<T&>().string(std::declval<String&>()));
+
+template <typename T>
+using start_object_function_t =
+    decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));
+
+template <typename T, typename String>
+using key_function_t =
+    decltype(std::declval<T&>().key(std::declval<String&>()));
+
+template <typename T>
+using end_object_function_t = decltype(std::declval<T&>().end_object());
+
+template <typename T>
+using start_array_function_t =
+    decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));
+
+template <typename T>
+using end_array_function_t = decltype(std::declval<T&>().end_array());
+
+template <typename T, typename Exception>
+using parse_error_function_t = decltype(std::declval<T&>().parse_error(
+        std::declval<std::size_t>(), std::declval<const std::string&>(),
+        std::declval<const Exception&>()));
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static constexpr bool value =
+        is_detected_exact<bool, null_function_t, SAX>::value &&
+        is_detected_exact<bool, boolean_function_t, SAX>::value &&
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value &&
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value &&
+        is_detected_exact<bool, number_float_function_t, SAX, number_float_t,
+        string_t>::value &&
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, start_object_function_t, SAX>::value &&
+        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
+        is_detected_exact<bool, end_object_function_t, SAX>::value &&
+        is_detected_exact<bool, start_array_function_t, SAX>::value &&
+        is_detected_exact<bool, end_array_function_t, SAX>::value &&
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
+};
+
+template <typename SAX, typename BasicJsonType>
+struct is_sax_static_asserts
+{
+  private:
+    static_assert(is_basic_json<BasicJsonType>::value,
+                  "BasicJsonType must be of type basic_json<...>");
+
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using exception_t = typename BasicJsonType::exception;
+
+  public:
+    static_assert(is_detected_exact<bool, null_function_t, SAX>::value,
+                  "Missing/invalid function: bool null()");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
+                  "Missing/invalid function: bool boolean(bool)");
+    static_assert(
+        is_detected_exact<bool, number_integer_function_t, SAX,
+        number_integer_t>::value,
+        "Missing/invalid function: bool number_integer(number_integer_t)");
+    static_assert(
+        is_detected_exact<bool, number_unsigned_function_t, SAX,
+        number_unsigned_t>::value,
+        "Missing/invalid function: bool number_unsigned(number_unsigned_t)");
+    static_assert(is_detected_exact<bool, number_float_function_t, SAX,
+                  number_float_t, string_t>::value,
+                  "Missing/invalid function: bool number_float(number_float_t, const string_t&)");
+    static_assert(
+        is_detected_exact<bool, string_function_t, SAX, string_t>::value,
+        "Missing/invalid function: bool string(string_t&)");
+    static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_object(std::size_t)");
+    static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,
+                  "Missing/invalid function: bool key(string_t&)");
+    static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_object()");
+    static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool start_array(std::size_t)");
+    static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,
+                  "Missing/invalid function: bool end_array()");
+    static_assert(
+        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,
+        "Missing/invalid function: bool parse_error(std::size_t, const "
+        "std::string&, const exception&)");
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
 
-                        // other characters after escape
-                        default:
-                            error_message = "invalid string: forbidden character after backslash";
-                            return token_type::parse_error;
-                    }
+// #include <nlohmann/detail/input/json_sax.hpp>
 
-                    break;
-                }
 
-                // invalid control characters
-                case 0x00:
-                case 0x01:
-                case 0x02:
-                case 0x03:
-                case 0x04:
-                case 0x05:
-                case 0x06:
-                case 0x07:
-                case 0x08:
-                case 0x09:
-                case 0x0A:
-                case 0x0B:
-                case 0x0C:
-                case 0x0D:
-                case 0x0E:
-                case 0x0F:
-                case 0x10:
-                case 0x11:
-                case 0x12:
-                case 0x13:
-                case 0x14:
-                case 0x15:
-                case 0x16:
-                case 0x17:
-                case 0x18:
-                case 0x19:
-                case 0x1A:
-                case 0x1B:
-                case 0x1C:
-                case 0x1D:
-                case 0x1E:
-                case 0x1F:
-                {
-                    error_message = "invalid string: control character must be escaped";
-                    return token_type::parse_error;
-                }
+#include <cstddef>
+#include <string>
+#include <vector>
 
-                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
-                case 0x20:
-                case 0x21:
-                case 0x23:
-                case 0x24:
-                case 0x25:
-                case 0x26:
-                case 0x27:
-                case 0x28:
-                case 0x29:
-                case 0x2A:
-                case 0x2B:
-                case 0x2C:
-                case 0x2D:
-                case 0x2E:
-                case 0x2F:
-                case 0x30:
-                case 0x31:
-                case 0x32:
-                case 0x33:
-                case 0x34:
-                case 0x35:
-                case 0x36:
-                case 0x37:
-                case 0x38:
-                case 0x39:
-                case 0x3A:
-                case 0x3B:
-                case 0x3C:
-                case 0x3D:
-                case 0x3E:
-                case 0x3F:
-                case 0x40:
-                case 0x41:
-                case 0x42:
-                case 0x43:
-                case 0x44:
-                case 0x45:
-                case 0x46:
-                case 0x47:
-                case 0x48:
-                case 0x49:
-                case 0x4A:
-                case 0x4B:
-                case 0x4C:
-                case 0x4D:
-                case 0x4E:
-                case 0x4F:
-                case 0x50:
-                case 0x51:
-                case 0x52:
-                case 0x53:
-                case 0x54:
-                case 0x55:
-                case 0x56:
-                case 0x57:
-                case 0x58:
-                case 0x59:
-                case 0x5A:
-                case 0x5B:
-                case 0x5D:
-                case 0x5E:
-                case 0x5F:
-                case 0x60:
-                case 0x61:
-                case 0x62:
-                case 0x63:
-                case 0x64:
-                case 0x65:
-                case 0x66:
-                case 0x67:
-                case 0x68:
-                case 0x69:
-                case 0x6A:
-                case 0x6B:
-                case 0x6C:
-                case 0x6D:
-                case 0x6E:
-                case 0x6F:
-                case 0x70:
-                case 0x71:
-                case 0x72:
-                case 0x73:
-                case 0x74:
-                case 0x75:
-                case 0x76:
-                case 0x77:
-                case 0x78:
-                case 0x79:
-                case 0x7A:
-                case 0x7B:
-                case 0x7C:
-                case 0x7D:
-                case 0x7E:
-                case 0x7F:
-                {
-                    add(current);
-                    break;
-                }
+// #include <nlohmann/detail/input/parser.hpp>
 
-                // U+0080..U+07FF: bytes C2..DF 80..BF
-                case 0xC2:
-                case 0xC3:
-                case 0xC4:
-                case 0xC5:
-                case 0xC6:
-                case 0xC7:
-                case 0xC8:
-                case 0xC9:
-                case 0xCA:
-                case 0xCB:
-                case 0xCC:
-                case 0xCD:
-                case 0xCE:
-                case 0xCF:
-                case 0xD0:
-                case 0xD1:
-                case 0xD2:
-                case 0xD3:
-                case 0xD4:
-                case 0xD5:
-                case 0xD6:
-                case 0xD7:
-                case 0xD8:
-                case 0xD9:
-                case 0xDA:
-                case 0xDB:
-                case 0xDC:
-                case 0xDD:
-                case 0xDE:
-                case 0xDF:
-                {
-                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
+// #include <nlohmann/detail/exceptions.hpp>
 
-                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
-                case 0xE0:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
 
-                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
-                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
-                case 0xE1:
-                case 0xE2:
-                case 0xE3:
-                case 0xE4:
-                case 0xE5:
-                case 0xE6:
-                case 0xE7:
-                case 0xE8:
-                case 0xE9:
-                case 0xEA:
-                case 0xEB:
-                case 0xEC:
-                case 0xEE:
-                case 0xEF:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
+namespace nlohmann
+{
+
+/*!
+@brief SAX interface
+
+This class describes the SAX interface used by @ref nlohmann::json::sax_parse.
+Each function is called in different situations while the input is parsed. The
+boolean return value informs the parser whether to continue processing the
+input.
+*/
+template<typename BasicJsonType>
+struct json_sax
+{
+    /// type for (signed) integers
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    /// type for unsigned integers
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    /// type for floating-point numbers
+    using number_float_t = typename BasicJsonType::number_float_t;
+    /// type for strings
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @brief a null value was read
+    @return whether parsing should proceed
+    */
+    virtual bool null() = 0;
+
+    /*!
+    @brief a boolean value was read
+    @param[in] val  boolean value
+    @return whether parsing should proceed
+    */
+    virtual bool boolean(bool val) = 0;
+
+    /*!
+    @brief an integer number was read
+    @param[in] val  integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_integer(number_integer_t val) = 0;
+
+    /*!
+    @brief an unsigned integer number was read
+    @param[in] val  unsigned integer value
+    @return whether parsing should proceed
+    */
+    virtual bool number_unsigned(number_unsigned_t val) = 0;
+
+    /*!
+    @brief an floating-point number was read
+    @param[in] val  floating-point value
+    @param[in] s    raw token value
+    @return whether parsing should proceed
+    */
+    virtual bool number_float(number_float_t val, const string_t& s) = 0;
+
+    /*!
+    @brief a string was read
+    @param[in] val  string value
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool string(string_t& val) = 0;
+
+    /*!
+    @brief the beginning of an object was read
+    @param[in] elements  number of object elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_object(std::size_t elements) = 0;
+
+    /*!
+    @brief an object key was read
+    @param[in] val  object key
+    @return whether parsing should proceed
+    @note It is safe to move the passed string.
+    */
+    virtual bool key(string_t& val) = 0;
+
+    /*!
+    @brief the end of an object was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_object() = 0;
+
+    /*!
+    @brief the beginning of an array was read
+    @param[in] elements  number of array elements or -1 if unknown
+    @return whether parsing should proceed
+    @note binary formats may report the number of elements
+    */
+    virtual bool start_array(std::size_t elements) = 0;
 
-                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
-                case 0xED:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
+    /*!
+    @brief the end of an array was read
+    @return whether parsing should proceed
+    */
+    virtual bool end_array() = 0;
 
-                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
-                case 0xF0:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
+    /*!
+    @brief a parse error occurred
+    @param[in] position    the position in the input where the error occurs
+    @param[in] last_token  the last read token
+    @param[in] ex          an exception object describing the error
+    @return whether parsing should proceed (must return false)
+    */
+    virtual bool parse_error(std::size_t position,
+                             const std::string& last_token,
+                             const detail::exception& ex) = 0;
 
-                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
-                case 0xF1:
-                case 0xF2:
-                case 0xF3:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
+    virtual ~json_sax() = default;
+};
 
-                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
-                case 0xF4:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
 
-                // remaining bytes (80..C1 and F5..FF) are ill-formed
-                default:
-                {
-                    error_message = "invalid string: ill-formed UTF-8 byte";
-                    return token_type::parse_error;
-                }
-            }
-        }
-    }
+namespace detail
+{
+/*!
+@brief SAX implementation to create a JSON value from SAX events
 
-    static void strtof(float& f, const char* str, char** endptr) noexcept
+This class implements the @ref json_sax interface and processes the SAX events
+to create a JSON value which makes it basically a DOM parser. The structure or
+hierarchy of the JSON value is managed by the stack `ref_stack` which contains
+a pointer to the respective array or object for each recursion depth.
+
+After successful parsing, the value that is passed by reference to the
+constructor contains the parsed value.
+
+@tparam BasicJsonType  the JSON type
+*/
+template<typename BasicJsonType>
+class json_sax_dom_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+
+    /*!
+    @param[in, out] r  reference to a JSON value that is manipulated while
+                       parsing
+    @param[in] allow_exceptions_  whether parse errors yield exceptions
+    */
+    explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)
+        : root(r), allow_exceptions(allow_exceptions_)
+    {}
+
+    bool null()
     {
-        f = std::strtof(str, endptr);
+        handle_value(nullptr);
+        return true;
     }
 
-    static void strtof(double& f, const char* str, char** endptr) noexcept
+    bool boolean(bool val)
     {
-        f = std::strtod(str, endptr);
+        handle_value(val);
+        return true;
     }
 
-    static void strtof(long double& f, const char* str, char** endptr) noexcept
+    bool number_integer(number_integer_t val)
     {
-        f = std::strtold(str, endptr);
+        handle_value(val);
+        return true;
     }
 
-    /*!
-    @brief scan a number literal
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
 
-    This function scans a string according to Sect. 6 of RFC 7159.
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
 
-    The function is realized with a deterministic finite state machine derived
-    from the grammar described in RFC 7159. Starting in state "init", the
-    input is read and used to determined the next state. Only state "done"
-    accepts the number. State "error" is a trap state to model errors. In the
-    table below, "anything" means any character but the ones listed before.
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
 
-    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
-    ---------|----------|----------|----------|---------|---------|----------|-----------
-    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
-    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
-    zero     | done     | done     | exponent | done    | done    | decimal1 | done
-    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
-    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
-    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
-    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
-    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
-    any2     | any2     | any2     | done     | done    | done    | done     | done
+    bool start_object(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
 
-    The state machine is realized with one label per state (prefixed with
-    "scan_number_") and `goto` statements between them. The state machine
-    contains cycles, but any cycle can be left when EOF is read. Therefore,
-    the function is guaranteed to terminate.
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
+        {
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive object size: " + std::to_string(len)));
+        }
 
-    During scanning, the read bytes are stored in yytext. This string is
-    then converted to a signed integer, an unsigned integer, or a
-    floating-point number.
+        return true;
+    }
 
-    @return token_type::value_unsigned, token_type::value_integer, or
-            token_type::value_float if number could be successfully scanned,
-            token_type::parse_error otherwise
+    bool key(string_t& val)
+    {
+        // add null at given key and store the reference for later
+        object_element = &(ref_stack.back()->m_value.object->operator[](val));
+        return true;
+    }
 
-    @note The scanner is independent of the current locale. Internally, the
-          locale's decimal point is used instead of `.` to work with the
-          locale-dependent converters.
-    */
-    token_type scan_number()
+    bool end_object()
     {
-        // reset yytext to store the number's bytes
-        reset();
+        ref_stack.pop_back();
+        return true;
+    }
 
-        // the type of the parsed number; initially set to unsigned; will be
-        // changed if minus sign, decimal point or exponent is read
-        token_type number_type = token_type::value_unsigned;
+    bool start_array(std::size_t len)
+    {
+        ref_stack.push_back(handle_value(BasicJsonType::value_t::array));
 
-        // state (init): we just found out we need to scan a number
-        switch (current)
+        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
         {
-            case '-':
-            {
-                add(current);
-                goto scan_number_minus;
-            }
+            JSON_THROW(out_of_range::create(408,
+                                            "excessive array size: " + std::to_string(len)));
+        }
 
-            case '0':
-            {
-                add(current);
-                goto scan_number_zero;
-            }
+        return true;
+    }
 
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
+    bool end_array()
+    {
+        ref_stack.pop_back();
+        return true;
+    }
 
-            default:
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
+    {
+        errored = true;
+        if (allow_exceptions)
+        {
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
             {
-                // all other characters are rejected outside scan_number()
-                assert(false); // LCOV_EXCL_LINE
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
             }
         }
+        return false;
+    }
 
-scan_number_minus:
-        // state: we just parsed a leading minus sign
-        number_type = token_type::value_integer;
-        switch (get())
+    constexpr bool is_errored() const
+    {
+        return errored;
+    }
+
+  private:
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename Value>
+    BasicJsonType* handle_value(Value&& v)
+    {
+        if (ref_stack.empty())
         {
-            case '0':
-            {
-                add(current);
-                goto scan_number_zero;
-            }
+            root = BasicJsonType(std::forward<Value>(v));
+            return &root;
+        }
 
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
 
-            default:
-            {
-                error_message = "invalid number; expected digit after '-'";
-                return token_type::parse_error;
-            }
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));
+            return &(ref_stack.back()->m_value.array->back());
         }
-
-scan_number_zero:
-        // state: we just parse a zero (maybe with a leading minus sign)
-        switch (get())
+        else
         {
-            case '.':
-            {
-                add(decimal_point_char);
-                goto scan_number_decimal1;
-            }
+            assert(object_element);
+            *object_element = BasicJsonType(std::forward<Value>(v));
+            return object_element;
+        }
+    }
 
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+
+template<typename BasicJsonType>
+class json_sax_dom_callback_parser
+{
+  public:
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using parser_callback_t = typename BasicJsonType::parser_callback_t;
+    using parse_event_t = typename BasicJsonType::parse_event_t;
+
+    json_sax_dom_callback_parser(BasicJsonType& r,
+                                 const parser_callback_t cb,
+                                 const bool allow_exceptions_ = true)
+        : root(r), callback(cb), allow_exceptions(allow_exceptions_)
+    {
+        keep_stack.push_back(true);
+    }
+
+    bool null()
+    {
+        handle_value(nullptr);
+        return true;
+    }
+
+    bool boolean(bool val)
+    {
+        handle_value(val);
+        return true;
+    }
+
+    bool number_integer(number_integer_t val)
+    {
+        handle_value(val);
+        return true;
+    }
 
-            default:
-                goto scan_number_done;
-        }
+    bool number_unsigned(number_unsigned_t val)
+    {
+        handle_value(val);
+        return true;
+    }
 
-scan_number_any1:
-        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
+    bool number_float(number_float_t val, const string_t& /*unused*/)
+    {
+        handle_value(val);
+        return true;
+    }
 
-            case '.':
-            {
-                add(decimal_point_char);
-                goto scan_number_decimal1;
-            }
+    bool string(string_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
 
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
+    bool start_object(std::size_t len)
+    {
+        // check callback for object start
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);
+        keep_stack.push_back(keep);
 
-            default:
-                goto scan_number_done;
-        }
+        auto val = handle_value(BasicJsonType::value_t::object, true);
+        ref_stack.push_back(val.second);
 
-scan_number_decimal1:
-        // state: we just parsed a decimal point
-        number_type = token_type::value_float;
-        switch (get())
+        // check object limit
+        if (ref_stack.back())
         {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_decimal2;
-            }
-
-            default:
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
             {
-                error_message = "invalid number; expected digit after '.'";
-                return token_type::parse_error;
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive object size: " + std::to_string(len)));
             }
         }
 
-scan_number_decimal2:
-        // we just parsed at least one number after a decimal point
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_decimal2;
-            }
+        return true;
+    }
 
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
+    bool key(string_t& val)
+    {
+        BasicJsonType k = BasicJsonType(val);
 
-            default:
-                goto scan_number_done;
-        }
+        // check callback for key
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);
+        key_keep_stack.push_back(keep);
 
-scan_number_exponent:
-        // we just parsed an exponent
-        number_type = token_type::value_float;
-        switch (get())
+        // add discarded value at given key and store the reference for later
+        if (keep and ref_stack.back())
         {
-            case '+':
-            case '-':
-            {
-                add(current);
-                goto scan_number_sign;
-            }
+            object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);
+        }
 
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any2;
-            }
+        return true;
+    }
 
-            default:
+    bool end_object()
+    {
+        if (ref_stack.back())
+        {
+            if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))
             {
-                error_message =
-                    "invalid number; expected '+', '-', or digit after exponent";
-                return token_type::parse_error;
+                // discard object
+                *ref_stack.back() = discarded;
             }
         }
 
-scan_number_sign:
-        // we just parsed an exponent sign
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any2;
-            }
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
 
-            default:
+        if (not ref_stack.empty() and ref_stack.back())
+        {
+            // remove discarded value
+            if (ref_stack.back()->is_object())
             {
-                error_message = "invalid number; expected digit after exponent sign";
-                return token_type::parse_error;
+                for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+                {
+                    if (it->is_discarded())
+                    {
+                        ref_stack.back()->erase(it);
+                        break;
+                    }
+                }
             }
         }
 
-scan_number_any2:
-        // we just parsed a number after the exponent or exponent sign
-        switch (get())
+        return true;
+    }
+
+    bool start_array(std::size_t len)
+    {
+        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);
+        keep_stack.push_back(keep);
+
+        auto val = handle_value(BasicJsonType::value_t::array, true);
+        ref_stack.push_back(val.second);
+
+        // check array limit
+        if (ref_stack.back())
         {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
+            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
             {
-                add(current);
-                goto scan_number_any2;
+                JSON_THROW(out_of_range::create(408,
+                                                "excessive array size: " + std::to_string(len)));
             }
-
-            default:
-                goto scan_number_done;
         }
 
-scan_number_done:
-        // unget the character after the number (we only read it to know that
-        // we are done scanning a number)
-        unget();
+        return true;
+    }
 
-        char* endptr = nullptr;
-        errno = 0;
+    bool end_array()
+    {
+        bool keep = true;
 
-        // try to parse integers first and fall back to floats
-        if (number_type == token_type::value_unsigned)
+        if (ref_stack.back())
         {
-            const auto x = std::strtoull(yytext.data(), &endptr, 10);
-
-            // we checked the number format before
-            assert(endptr == yytext.data() + yytext.size());
-
-            if (errno == 0)
+            keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
+            if (not keep)
             {
-                value_unsigned = static_cast<number_unsigned_t>(x);
-                if (value_unsigned == x)
-                {
-                    return token_type::value_unsigned;
-                }
+                // discard array
+                *ref_stack.back() = discarded;
             }
         }
-        else if (number_type == token_type::value_integer)
-        {
-            const auto x = std::strtoll(yytext.data(), &endptr, 10);
 
-            // we checked the number format before
-            assert(endptr == yytext.data() + yytext.size());
+        assert(not ref_stack.empty());
+        assert(not keep_stack.empty());
+        ref_stack.pop_back();
+        keep_stack.pop_back();
 
-            if (errno == 0)
+        // remove discarded value
+        if (not keep and not ref_stack.empty())
+        {
+            if (ref_stack.back()->is_array())
             {
-                value_integer = static_cast<number_integer_t>(x);
-                if (value_integer == x)
-                {
-                    return token_type::value_integer;
-                }
+                ref_stack.back()->m_value.array->pop_back();
             }
         }
 
-        // this code is reached if we parse a floating-point number or if an
-        // integer conversion above failed
-        strtof(value_float, yytext.data(), &endptr);
-
-        // we checked the number format before
-        assert(endptr == yytext.data() + yytext.size());
-
-        return token_type::value_float;
+        return true;
     }
 
-    /*!
-    @param[in] literal_text  the literal text to expect
-    @param[in] length        the length of the passed literal text
-    @param[in] return_type   the token type to return on success
-    */
-    token_type scan_literal(const char* literal_text, const std::size_t length,
-                            token_type return_type)
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
+                     const detail::exception& ex)
     {
-        assert(current == literal_text[0]);
-        for (std::size_t i = 1; i < length; ++i)
+        errored = true;
+        if (allow_exceptions)
         {
-            if (JSON_UNLIKELY(get() != literal_text[i]))
+            // determine the proper exception type from the id
+            switch ((ex.id / 100) % 100)
             {
-                error_message = "invalid literal";
-                return token_type::parse_error;
+                case 1:
+                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
+                case 4:
+                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
+                // LCOV_EXCL_START
+                case 2:
+                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
+                case 3:
+                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
+                case 5:
+                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
+                default:
+                    assert(false);
+                    // LCOV_EXCL_STOP
             }
         }
-        return return_type;
+        return false;
     }
 
-    /////////////////////
-    // input management
-    /////////////////////
-
-    /// reset yytext; current character is beginning of token
-    void reset() noexcept
+    constexpr bool is_errored() const
     {
-        yytext.clear();
-        token_string.clear();
-        token_string.push_back(std::char_traits<char>::to_char_type(current));
+        return errored;
     }
 
-    /*
-    @brief get next character from the input
+  private:
+    /*!
+    @param[in] v  value to add to the JSON value we build during parsing
+    @param[in] skip_callback  whether we should skip calling the callback
+               function; this is required after start_array() and
+               start_object() SAX events, because otherwise we would call the
+               callback function with an empty array or object, respectively.
+
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+
+    @return pair of boolean (whether value should be kept) and pointer (to the
+            passed value in the ref_stack hierarchy; nullptr if not kept)
+    */
+    template<typename Value>
+    std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)
+    {
+        assert(not keep_stack.empty());
+
+        // do not handle this value if we know it would be added to a discarded
+        // container
+        if (not keep_stack.back())
+        {
+            return {false, nullptr};
+        }
+
+        // create value
+        auto value = BasicJsonType(std::forward<Value>(v));
+
+        // check callback
+        const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);
+
+        // do not handle this value if we just learnt it shall be discarded
+        if (not keep)
+        {
+            return {false, nullptr};
+        }
 
-    This function provides the interface to the used input adapter. It does
-    not throw in case the input reached EOF, but returns a
-    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters
-    for use in error messages.
+        if (ref_stack.empty())
+        {
+            root = std::move(value);
+            return {true, &root};
+        }
 
-    @return character read from the input
-    */
-    std::char_traits<char>::int_type get()
-    {
-        ++chars_read;
-        current = ia->get_character();
-        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        // skip this value if we already decided to skip the parent
+        // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)
+        if (not ref_stack.back())
         {
-            token_string.push_back(std::char_traits<char>::to_char_type(current));
+            return {false, nullptr};
         }
-        return current;
-    }
 
-    /// unget current character (return it again on next get)
-    void unget()
-    {
-        --chars_read;
-        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
+        // we now only expect arrays and objects
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
         {
-            ia->unget_character();
-            assert(token_string.size() != 0);
-            token_string.pop_back();
+            ref_stack.back()->m_value.array->push_back(std::move(value));
+            return {true, &(ref_stack.back()->m_value.array->back())};
         }
-    }
+        else
+        {
+            // check if we should store an element for the current key
+            assert(not key_keep_stack.empty());
+            const bool store_element = key_keep_stack.back();
+            key_keep_stack.pop_back();
 
-    /// add a character to yytext
-    void add(int c)
-    {
-        yytext.push_back(std::char_traits<char>::to_char_type(c));
+            if (not store_element)
+            {
+                return {false, nullptr};
+            }
+
+            assert(object_element);
+            *object_element = std::move(value);
+            return {true, object_element};
+        }
     }
 
+    /// the parsed JSON value
+    BasicJsonType& root;
+    /// stack to model hierarchy of values
+    std::vector<BasicJsonType*> ref_stack;
+    /// stack to manage which values to keep
+    std::vector<bool> keep_stack;
+    /// stack to manage which object keys to keep
+    std::vector<bool> key_keep_stack;
+    /// helper to hold the reference for the next object element
+    BasicJsonType* object_element = nullptr;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+    /// a discarded value for the callback
+    BasicJsonType discarded = BasicJsonType::value_t::discarded;
+};
+
+template<typename BasicJsonType>
+class json_sax_acceptor
+{
   public:
-    /////////////////////
-    // value getters
-    /////////////////////
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
 
-    /// return integer value
-    constexpr number_integer_t get_number_integer() const noexcept
+    bool null()
     {
-        return value_integer;
+        return true;
     }
 
-    /// return unsigned integer value
-    constexpr number_unsigned_t get_number_unsigned() const noexcept
+    bool boolean(bool /*unused*/)
     {
-        return value_unsigned;
+        return true;
     }
 
-    /// return floating-point value
-    constexpr number_float_t get_number_float() const noexcept
+    bool number_integer(number_integer_t /*unused*/)
     {
-        return value_float;
+        return true;
     }
 
-    /// return current string value (implicitly resets the token; useful only once)
-    std::string move_string()
+    bool number_unsigned(number_unsigned_t /*unused*/)
     {
-        return std::move(yytext);
+        return true;
     }
 
-    /////////////////////
-    // diagnostics
-    /////////////////////
-
-    /// return position of last read token
-    constexpr std::size_t get_position() const noexcept
+    bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)
     {
-        return chars_read;
+        return true;
     }
 
-    /// return the last read token (for errors only).  Will never contain EOF
-    /// (an arbitrary value that is not a valid char value, often -1), because
-    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
-    std::string get_token_string() const
+    bool string(string_t& /*unused*/)
     {
-        // escape control characters
-        std::string result;
-        for (const auto c : token_string)
-        {
-            if ('\x00' <= c and c <= '\x1F')
-            {
-                // escape control characters
-                std::stringstream ss;
-                ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0')
-                   << std::hex << static_cast<int>(c) << ">";
-                result += ss.str();
-            }
-            else
-            {
-                // add character as is
-                result.push_back(c);
-            }
-        }
-
-        return result;
+        return true;
     }
 
-    /// return syntax error message
-    constexpr const char* get_error_message() const noexcept
+    bool start_object(std::size_t  /*unused*/ = std::size_t(-1))
     {
-        return error_message;
+        return true;
     }
 
-    /////////////////////
-    // actual scanner
-    /////////////////////
-
-    token_type scan()
+    bool key(string_t& /*unused*/)
     {
-        // read next character and ignore whitespace
-        do
-        {
-            get();
-        }
-        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
-
-        switch (current)
-        {
-            // structural characters
-            case '[':
-                return token_type::begin_array;
-            case ']':
-                return token_type::end_array;
-            case '{':
-                return token_type::begin_object;
-            case '}':
-                return token_type::end_object;
-            case ':':
-                return token_type::name_separator;
-            case ',':
-                return token_type::value_separator;
-
-            // literals
-            case 't':
-                return scan_literal("true", 4, token_type::literal_true);
-            case 'f':
-                return scan_literal("false", 5, token_type::literal_false);
-            case 'n':
-                return scan_literal("null", 4, token_type::literal_null);
-
-            // string
-            case '\"':
-                return scan_string();
-
-            // number
-            case '-':
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-                return scan_number();
-
-            // end of input (the null byte is needed when parsing from
-            // string literals)
-            case '\0':
-            case std::char_traits<char>::eof():
-                return token_type::end_of_input;
+        return true;
+    }
 
-            // error
-            default:
-                error_message = "invalid literal";
-                return token_type::parse_error;
-        }
+    bool end_object()
+    {
+        return true;
     }
 
-  private:
-    /// input adapter
-    detail::input_adapter_t ia = nullptr;
+    bool start_array(std::size_t  /*unused*/ = std::size_t(-1))
+    {
+        return true;
+    }
 
-    /// the current character
-    std::char_traits<char>::int_type current = std::char_traits<char>::eof();
+    bool end_array()
+    {
+        return true;
+    }
 
-    /// the number of characters read
-    std::size_t chars_read = 0;
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)
+    {
+        return false;
+    }
+};
+}  // namespace detail
 
-    /// raw input token string (for error messages)
-    std::vector<char> token_string {};
+}  // namespace nlohmann
 
-    /// buffer for variable-length tokens (numbers, strings)
-    std::string yytext {};
+// #include <nlohmann/detail/input/lexer.hpp>
 
-    /// a description of occurred lexer errors
-    const char* error_message = "";
+// #include <nlohmann/detail/value_t.hpp>
 
-    // number values
-    number_integer_t value_integer = 0;
-    number_unsigned_t value_unsigned = 0;
-    number_float_t value_float = 0;
 
-    /// the decimal point
-    const char decimal_point_char = '.';
-};
+namespace nlohmann
+{
+namespace detail
+{
+////////////
+// parser //
+////////////
 
 /*!
 @brief syntax analysis
@@ -2892,6 +4841,7 @@ class parser
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
     using lexer_t = lexer<BasicJsonType>;
     using token_type = typename lexer_t::token_type;
 
@@ -2916,11 +4866,14 @@ class parser
         std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
 
     /// a parser reading from an input adapter
-    explicit parser(detail::input_adapter_t adapter,
+    explicit parser(detail::input_adapter_t&& adapter,
                     const parser_callback_t cb = nullptr,
                     const bool allow_exceptions_ = true)
-        : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_)
-    {}
+        : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_)
+    {
+        // read first token
+        get_token();
+    }
 
     /*!
     @brief public parser interface
@@ -2934,31 +4887,56 @@ class parser
     */
     void parse(const bool strict, BasicJsonType& result)
     {
-        // read first token
-        get_token();
+        if (callback)
+        {
+            json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
 
-        parse_internal(true, result);
-        result.assert_invariant();
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
 
-        // in strict mode, input must be completely read
-        if (strict)
-        {
-            get_token();
-            expect(token_type::end_of_input);
-        }
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
 
-        // in case of an error, return discarded value
-        if (errored)
-        {
-            result = value_t::discarded;
-            return;
+            // set top-level value to null if it was discarded by the callback
+            // function
+            if (result.is_discarded())
+            {
+                result = nullptr;
+            }
         }
-
-        // set top-level value to null if it was discarded by the callback
-        // function
-        if (result.is_discarded())
+        else
         {
-            result = nullptr;
+            json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);
+            sax_parse_internal(&sdp);
+            result.assert_invariant();
+
+            // in strict mode, input must be completely read
+            if (strict and (get_token() != token_type::end_of_input))
+            {
+                sdp.parse_error(m_lexer.get_position(),
+                                m_lexer.get_token_string(),
+                                parse_error::create(101, m_lexer.get_position(),
+                                                    exception_message(token_type::end_of_input, "value")));
+            }
+
+            // in case of an error, return discarded value
+            if (sdp.is_errored())
+            {
+                result = value_t::discarded;
+                return;
+            }
         }
     }
 
@@ -2970,413 +4948,317 @@ class parser
     */
     bool accept(const bool strict = true)
     {
-        // read first token
-        get_token();
-
-        if (not accept_internal())
-        {
-            return false;
-        }
-
-        // strict => last token must be EOF
-        return not strict or (get_token() == token_type::end_of_input);
+        json_sax_acceptor<BasicJsonType> sax_acceptor;
+        return sax_parse(&sax_acceptor, strict);
     }
 
-  private:
-    /*!
-    @brief the actual parser
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-    */
-    void parse_internal(bool keep, BasicJsonType& result)
+    template <typename SAX>
+    bool sax_parse(SAX* sax, const bool strict = true)
     {
-        // never parse after a parse error was detected
-        assert(not errored);
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+        const bool result = sax_parse_internal(sax);
 
-        // start with a discarded value
-        if (not result.is_discarded())
+        // strict mode: next byte must be EOF
+        if (result and strict and (get_token() != token_type::end_of_input))
         {
-            result.m_value.destroy(result.m_type);
-            result.m_type = value_t::discarded;
+            return sax->parse_error(m_lexer.get_position(),
+                                    m_lexer.get_token_string(),
+                                    parse_error::create(101, m_lexer.get_position(),
+                                            exception_message(token_type::end_of_input, "value")));
         }
 
-        switch (last_token)
-        {
-            case token_type::begin_object:
-            {
-                if (keep)
-                {
-                    if (callback)
-                    {
-                        keep = callback(depth++, parse_event_t::object_start, result);
-                    }
-
-                    if (not callback or keep)
-                    {
-                        // explicitly set result to object to cope with {}
-                        result.m_type = value_t::object;
-                        result.m_value = value_t::object;
-                    }
-                }
-
-                // read next token
-                get_token();
+        return result;
+    }
 
-                // closing } -> we are done
-                if (last_token == token_type::end_object)
-                {
-                    if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                    {
-                        result.m_value.destroy(result.m_type);
-                        result.m_type = value_t::discarded;
-                    }
-                    break;
-                }
+  private:
+    template <typename SAX>
+    bool sax_parse_internal(SAX* sax)
+    {
+        // stack to remember the hierarchy of structured values we are parsing
+        // true = array; false = object
+        std::vector<bool> states;
+        // value to avoid a goto (see comment where set to true)
+        bool skip_to_state_evaluation = false;
 
-                // parse values
-                std::string key;
-                BasicJsonType value;
-                while (true)
+        while (true)
+        {
+            if (not skip_to_state_evaluation)
+            {
+                // invariant: get_token() was called before each iteration
+                switch (last_token)
                 {
-                    // store key
-                    if (not expect(token_type::value_string))
+                    case token_type::begin_object:
                     {
-                        return;
-                    }
-                    key = m_lexer.move_string();
+                        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+                        {
+                            return false;
+                        }
 
-                    bool keep_tag = false;
-                    if (keep)
-                    {
-                        if (callback)
+                        // closing } -> we are done
+                        if (get_token() == token_type::end_object)
+                        {
+                            if (JSON_UNLIKELY(not sax->end_object()))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+
+                        // parse key
+                        if (JSON_UNLIKELY(last_token != token_type::value_string))
                         {
-                            BasicJsonType k(key);
-                            keep_tag = callback(depth, parse_event_t::key, k);
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
                         }
-                        else
+                        if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
                         {
-                            keep_tag = true;
+                            return false;
                         }
-                    }
 
-                    // parse separator (:)
-                    get_token();
-                    if (not expect(token_type::name_separator))
-                    {
-                        return;
-                    }
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
 
-                    // parse and add value
-                    get_token();
-                    value.m_value.destroy(value.m_type);
-                    value.m_type = value_t::discarded;
-                    parse_internal(keep, value);
+                        // remember we are now inside an object
+                        states.push_back(false);
 
-                    if (JSON_UNLIKELY(errored))
-                    {
-                        return;
+                        // parse values
+                        get_token();
+                        continue;
                     }
 
-                    if (keep and keep_tag and not value.is_discarded())
+                    case token_type::begin_array:
                     {
-                        result.m_value.object->emplace(std::move(key), std::move(value));
-                    }
+                        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
+                        {
+                            return false;
+                        }
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
-                    {
-                        get_token();
+                        // closing ] -> we are done
+                        if (get_token() == token_type::end_array)
+                        {
+                            if (JSON_UNLIKELY(not sax->end_array()))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+
+                        // remember we are now inside an array
+                        states.push_back(true);
+
+                        // parse values (no need to call get_token)
                         continue;
                     }
 
-                    // closing }
-                    if (not expect(token_type::end_object))
+                    case token_type::value_float:
                     {
-                        return;
-                    }
-                    break;
-                }
+                        const auto res = m_lexer.get_number_float();
 
-                if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                {
-                    result.m_value.destroy(result.m_type);
-                    result.m_type = value_t::discarded;
-                }
-                break;
-            }
+                        if (JSON_UNLIKELY(not std::isfinite(res)))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                            break;
+                        }
+                    }
 
-            case token_type::begin_array:
-            {
-                if (keep)
-                {
-                    if (callback)
+                    case token_type::literal_false:
                     {
-                        keep = callback(depth++, parse_event_t::array_start, result);
+                        if (JSON_UNLIKELY(not sax->boolean(false)))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    if (not callback or keep)
+                    case token_type::literal_null:
                     {
-                        // explicitly set result to array to cope with []
-                        result.m_type = value_t::array;
-                        result.m_value = value_t::array;
+                        if (JSON_UNLIKELY(not sax->null()))
+                        {
+                            return false;
+                        }
+                        break;
                     }
-                }
-
-                // read next token
-                get_token();
 
-                // closing ] -> we are done
-                if (last_token == token_type::end_array)
-                {
-                    if (callback and not callback(--depth, parse_event_t::array_end, result))
+                    case token_type::literal_true:
                     {
-                        result.m_value.destroy(result.m_type);
-                        result.m_type = value_t::discarded;
+                        if (JSON_UNLIKELY(not sax->boolean(true)))
+                        {
+                            return false;
+                        }
+                        break;
                     }
-                    break;
-                }
 
-                // parse values
-                BasicJsonType value;
-                while (true)
-                {
-                    // parse value
-                    value.m_value.destroy(value.m_type);
-                    value.m_type = value_t::discarded;
-                    parse_internal(keep, value);
-
-                    if (JSON_UNLIKELY(errored))
+                    case token_type::value_integer:
                     {
-                        return;
+                        if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    if (keep and not value.is_discarded())
+                    case token_type::value_string:
                     {
-                        result.m_value.array->push_back(std::move(value));
+                        if (JSON_UNLIKELY(not sax->string(m_lexer.get_string())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
+                    case token_type::value_unsigned:
                     {
-                        get_token();
-                        continue;
+                        if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned())))
+                        {
+                            return false;
+                        }
+                        break;
                     }
 
-                    // closing ]
-                    if (not expect(token_type::end_array))
+                    case token_type::parse_error:
                     {
-                        return;
+                        // using "uninitialized" to avoid "expected" message
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::uninitialized, "value")));
                     }
-                    break;
-                }
-
-                if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
-                {
-                    result.m_value.destroy(result.m_type);
-                    result.m_type = value_t::discarded;
-                }
-                break;
-            }
-
-            case token_type::literal_null:
-            {
-                result.m_type = value_t::null;
-                break;
-            }
-
-            case token_type::value_string:
-            {
-                result.m_type = value_t::string;
-                result.m_value = m_lexer.move_string();
-                break;
-            }
-
-            case token_type::literal_true:
-            {
-                result.m_type = value_t::boolean;
-                result.m_value = true;
-                break;
-            }
-
-            case token_type::literal_false:
-            {
-                result.m_type = value_t::boolean;
-                result.m_value = false;
-                break;
-            }
-
-            case token_type::value_unsigned:
-            {
-                result.m_type = value_t::number_unsigned;
-                result.m_value = m_lexer.get_number_unsigned();
-                break;
-            }
-
-            case token_type::value_integer:
-            {
-                result.m_type = value_t::number_integer;
-                result.m_value = m_lexer.get_number_integer();
-                break;
-            }
-
-            case token_type::value_float:
-            {
-                result.m_type = value_t::number_float;
-                result.m_value = m_lexer.get_number_float();
 
-                // throw in case of infinity or NAN
-                if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
-                {
-                    if (allow_exceptions)
+                    default: // the last token was unexpected
                     {
-                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" +
-                                                        m_lexer.get_token_string() + "'"));
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::literal_or_value, "value")));
                     }
-                    expect(token_type::uninitialized);
                 }
-                break;
             }
-
-            case token_type::parse_error:
+            else
             {
-                // using "uninitialized" to avoid "expected" message
-                if (not expect(token_type::uninitialized))
-                {
-                    return;
-                }
-                break; // LCOV_EXCL_LINE
+                skip_to_state_evaluation = false;
             }
 
-            default:
+            // we reached this line after we successfully parsed a value
+            if (states.empty())
             {
-                // the last token was unexpected; we expected a value
-                if (not expect(token_type::literal_or_value))
-                {
-                    return;
-                }
-                break; // LCOV_EXCL_LINE
+                // empty stack: we reached the end of the hierarchy: done
+                return true;
             }
-        }
-
-        if (keep and callback and not callback(depth, parse_event_t::value, result))
-        {
-            result.m_type = value_t::discarded;
-        }
-    }
-
-    /*!
-    @brief the actual acceptor
-
-    @invariant 1. The last token is not yet processed. Therefore, the caller
-                  of this function must make sure a token has been read.
-               2. When this function returns, the last token is processed.
-                  That is, the last read character was already considered.
-
-    This invariant makes sure that no token needs to be "unput".
-    */
-    bool accept_internal()
-    {
-        switch (last_token)
-        {
-            case token_type::begin_object:
+            else
             {
-                // read next token
-                get_token();
-
-                // closing } -> we are done
-                if (last_token == token_type::end_object)
-                {
-                    return true;
-                }
-
-                // parse values
-                while (true)
+                if (states.back())  // array
                 {
-                    // parse key
-                    if (last_token != token_type::value_string)
+                    // comma -> next value
+                    if (get_token() == token_type::value_separator)
                     {
-                        return false;
+                        // parse a new value
+                        get_token();
+                        continue;
                     }
 
-                    // parse separator (:)
-                    get_token();
-                    if (last_token != token_type::name_separator)
+                    // closing ]
+                    if (JSON_LIKELY(last_token == token_type::end_array))
                     {
-                        return false;
-                    }
+                        if (JSON_UNLIKELY(not sax->end_array()))
+                        {
+                            return false;
+                        }
 
-                    // parse value
-                    get_token();
-                    if (not accept_internal())
+                        // We are done with this array. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
+                        continue;
+                    }
+                    else
                     {
-                        return false;
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_array, "array")));
                     }
-
+                }
+                else  // object
+                {
                     // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
+                    if (get_token() == token_type::value_separator)
                     {
+                        // parse key
+                        if (JSON_UNLIKELY(get_token() != token_type::value_string))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::value_string, "object key")));
+                        }
+                        else
+                        {
+                            if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
+                            {
+                                return false;
+                            }
+                        }
+
+                        // parse separator (:)
+                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
+                        {
+                            return sax->parse_error(m_lexer.get_position(),
+                                                    m_lexer.get_token_string(),
+                                                    parse_error::create(101, m_lexer.get_position(),
+                                                            exception_message(token_type::name_separator, "object separator")));
+                        }
+
+                        // parse values
                         get_token();
                         continue;
                     }
 
                     // closing }
-                    return (last_token == token_type::end_object);
-                }
-            }
-
-            case token_type::begin_array:
-            {
-                // read next token
-                get_token();
-
-                // closing ] -> we are done
-                if (last_token == token_type::end_array)
-                {
-                    return true;
-                }
-
-                // parse values
-                while (true)
-                {
-                    // parse value
-                    if (not accept_internal())
+                    if (JSON_LIKELY(last_token == token_type::end_object))
                     {
-                        return false;
-                    }
+                        if (JSON_UNLIKELY(not sax->end_object()))
+                        {
+                            return false;
+                        }
 
-                    // comma -> next value
-                    get_token();
-                    if (last_token == token_type::value_separator)
-                    {
-                        get_token();
+                        // We are done with this object. Before we can parse a
+                        // new value, we need to evaluate the new state first.
+                        // By setting skip_to_state_evaluation to false, we
+                        // are effectively jumping to the beginning of this if.
+                        assert(not states.empty());
+                        states.pop_back();
+                        skip_to_state_evaluation = true;
                         continue;
                     }
-
-                    // closing ]
-                    return (last_token == token_type::end_array);
+                    else
+                    {
+                        return sax->parse_error(m_lexer.get_position(),
+                                                m_lexer.get_token_string(),
+                                                parse_error::create(101, m_lexer.get_position(),
+                                                        exception_message(token_type::end_object, "object")));
+                    }
                 }
             }
-
-            case token_type::value_float:
-            {
-                // reject infinity or NAN
-                return std::isfinite(m_lexer.get_number_float());
-            }
-
-            case token_type::literal_false:
-            case token_type::literal_null:
-            case token_type::literal_true:
-            case token_type::value_integer:
-            case token_type::value_string:
-            case token_type::value_unsigned:
-                return true;
-
-            default: // the last token was unexpected
-                return false;
         }
     }
 
@@ -3386,31 +5268,17 @@ class parser
         return (last_token = m_lexer.scan());
     }
 
-    /*!
-    @throw parse_error.101 if expected token did not occur
-    */
-    bool expect(token_type t)
+    std::string exception_message(const token_type expected, const std::string& context)
     {
-        if (JSON_UNLIKELY(t != last_token))
+        std::string error_msg = "syntax error ";
+
+        if (not context.empty())
         {
-            errored = true;
-            expected = t;
-            if (allow_exceptions)
-            {
-                throw_exception();
-            }
-            else
-            {
-                return false;
-            }
+            error_msg += "while parsing " + context + " ";
         }
 
-        return true;
-    }
+        error_msg += "- ";
 
-    [[noreturn]] void throw_exception() const
-    {
-        std::string error_msg = "syntax error - ";
         if (last_token == token_type::parse_error)
         {
             error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
@@ -3426,31 +5294,33 @@ class parser
             error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
         }
 
-        JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
+        return error_msg;
     }
 
   private:
-    /// current level of recursion
-    int depth = 0;
     /// callback function
     const parser_callback_t callback = nullptr;
     /// the type of the last read token
     token_type last_token = token_type::uninitialized;
     /// the lexer
     lexer_t m_lexer;
-    /// whether a syntax error occurred
-    bool errored = false;
-    /// possible reason for the syntax error
-    token_type expected = token_type::uninitialized;
     /// whether to throw exceptions in case of errors
     const bool allow_exceptions = true;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-///////////////
-// iterators //
-///////////////
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
 
-/*!
+
+#include <cstddef> // ptrdiff_t
+#include <limits>  // numeric_limits
+
+namespace nlohmann
+{
+namespace detail
+{
+/*
 @brief an iterator for primitive JSON types
 
 This class models an iterator for primitive JSON types (boolean, number,
@@ -3461,9 +5331,15 @@ end_value (`1`) models past the end.
 */
 class primitive_iterator_t
 {
-  public:
+  private:
     using difference_type = std::ptrdiff_t;
+    static constexpr difference_type begin_value = 0;
+    static constexpr difference_type end_value = begin_value + 1;
+
+    /// iterator as signed integer type
+    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
 
+  public:
     constexpr difference_type get_value() const noexcept
     {
         return m_it;
@@ -3503,10 +5379,10 @@ class primitive_iterator_t
         return lhs.m_it < rhs.m_it;
     }
 
-    primitive_iterator_t operator+(difference_type i)
+    primitive_iterator_t operator+(difference_type n) noexcept
     {
         auto result = *this;
-        result += i;
+        result += n;
         return result;
     }
 
@@ -3515,57 +5391,57 @@ class primitive_iterator_t
         return lhs.m_it - rhs.m_it;
     }
 
-    friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
-    {
-        return os << it.m_it;
-    }
-
-    primitive_iterator_t& operator++()
+    primitive_iterator_t& operator++() noexcept
     {
         ++m_it;
         return *this;
     }
 
-    primitive_iterator_t const operator++(int)
+    primitive_iterator_t const operator++(int) noexcept
     {
         auto result = *this;
-        m_it++;
+        ++m_it;
         return result;
     }
 
-    primitive_iterator_t& operator--()
+    primitive_iterator_t& operator--() noexcept
     {
         --m_it;
         return *this;
     }
 
-    primitive_iterator_t const operator--(int)
+    primitive_iterator_t const operator--(int) noexcept
     {
         auto result = *this;
-        m_it--;
+        --m_it;
         return result;
     }
 
-    primitive_iterator_t& operator+=(difference_type n)
+    primitive_iterator_t& operator+=(difference_type n) noexcept
     {
         m_it += n;
         return *this;
     }
 
-    primitive_iterator_t& operator-=(difference_type n)
+    primitive_iterator_t& operator-=(difference_type n) noexcept
     {
         m_it -= n;
         return *this;
     }
+};
+}  // namespace detail
+}  // namespace nlohmann
 
-  private:
-    static constexpr difference_type begin_value = 0;
-    static constexpr difference_type end_value = begin_value + 1;
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
 
-    /// iterator as signed integer type
-    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
-};
 
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
 /*!
 @brief an iterator value
 
@@ -3581,7 +5457,34 @@ template<typename BasicJsonType> struct internal_iterator
     /// generic iterator for all other types
     primitive_iterator_t primitive_iterator {};
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/iterators/iter_impl.hpp>
+
+
+#include <ciso646> // not
+#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next
+#include <type_traits> // conditional, is_const, remove_const
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/iterators/internal_iterator.hpp>
+
+// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
 
+namespace nlohmann
+{
+namespace detail
+{
+// forward declare, to be able to friend it later on
 template<typename IteratorType> class iteration_proxy;
 
 /*!
@@ -3597,7 +5500,7 @@ This class implements a both iterators (iterator and const_iterator) for the
 
 @requirement The class satisfies the following concept requirements:
 -
-[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
   The iterator that can be moved can be moved in both directions (i.e.
   incremented and decremented).
 
@@ -4149,7 +6052,7 @@ class iter_impl
     @brief  return the key of an object iterator
     @pre The iterator is initialized; i.e. `m_object != nullptr`.
     */
-    typename object_t::key_type key() const
+    const typename object_t::key_type& key() const
     {
         assert(m_object != nullptr);
 
@@ -4174,94 +6077,27 @@ class iter_impl
     /// associated JSON instance
     pointer m_object = nullptr;
     /// the actual iterator of the associated instance
-    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {};
+    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-/// proxy class for the iterator_wrapper functions
-template<typename IteratorType> class iteration_proxy
-{
-  private:
-    /// helper class for iteration
-    class iteration_proxy_internal
-    {
-      private:
-        /// the iterator
-        IteratorType anchor;
-        /// an index for arrays (used to create key names)
-        std::size_t array_index = 0;
-
-      public:
-        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
-
-        /// dereference operator (needed for range-based for)
-        iteration_proxy_internal& operator*()
-        {
-            return *this;
-        }
-
-        /// increment operator (needed for range-based for)
-        iteration_proxy_internal& operator++()
-        {
-            ++anchor;
-            ++array_index;
-
-            return *this;
-        }
-
-        /// inequality operator (needed for range-based for)
-        bool operator!=(const iteration_proxy_internal& o) const noexcept
-        {
-            return anchor != o.anchor;
-        }
-
-        /// return key of the iterator
-        std::string key() const
-        {
-            assert(anchor.m_object != nullptr);
-
-            switch (anchor.m_object->type())
-            {
-                // use integer array index as key
-                case value_t::array:
-                    return std::to_string(array_index);
-
-                // use key from the object
-                case value_t::object:
-                    return anchor.key();
-
-                // use an empty key for all primitive types
-                default:
-                    return "";
-            }
-        }
-
-        /// return value of the iterator
-        typename IteratorType::reference value() const
-        {
-            return anchor.value();
-        }
-    };
+// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
 
-    /// the container to iterate
-    typename IteratorType::reference container;
+// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
 
-  public:
-    /// construct iteration proxy from a container
-    explicit iteration_proxy(typename IteratorType::reference cont)
-        : container(cont) {}
 
-    /// return iterator begin (needed for range-based for)
-    iteration_proxy_internal begin() noexcept
-    {
-        return iteration_proxy_internal(container.begin());
-    }
+#include <cstddef> // ptrdiff_t
+#include <iterator> // reverse_iterator
+#include <utility> // declval
 
-    /// return iterator end (needed for range-based for)
-    iteration_proxy_internal end() noexcept
-    {
-        return iteration_proxy_internal(container.end());
-    }
-};
+namespace nlohmann
+{
+namespace detail
+{
+//////////////////////
+// reverse_iterator //
+//////////////////////
 
 /*!
 @brief a template for a reverse iterator class
@@ -4272,10 +6108,10 @@ create @ref const_reverse_iterator).
 
 @requirement The class satisfies the following concept requirements:
 -
-[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator):
+[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
   The iterator that can be moved can be moved in both directions (i.e.
   incremented and decremented).
-- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator):
+- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):
   It is possible to write to the pointed-to element (only if @a Base is
   @ref iterator).
 
@@ -4292,11 +6128,11 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
     using reference = typename Base::reference;
 
     /// create reverse iterator from iterator
-    json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+    explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
         : base_iterator(it) {}
 
     /// create reverse iterator from base class
-    json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+    explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
 
     /// post-increment (it++)
     json_reverse_iterator const operator++(int)
@@ -4366,11 +6202,25 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
         return it.operator * ();
     }
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
 
-/////////////////////
-// output adapters //
-/////////////////////
+#include <algorithm> // copy
+#include <cstddef> // size_t
+#include <ios> // streamsize
+#include <iterator> // back_inserter
+#include <memory> // shared_ptr, make_shared
+#include <ostream> // basic_ostream
+#include <string> // basic_string
+#include <vector> // vector
 
+namespace nlohmann
+{
+namespace detail
+{
 /// abstract output adapter interface
 template<typename CharType> struct output_adapter_protocol
 {
@@ -4388,7 +6238,9 @@ template<typename CharType>
 class output_vector_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {}
+    explicit output_vector_adapter(std::vector<CharType>& vec) noexcept
+        : v(vec)
+    {}
 
     void write_character(CharType c) override
     {
@@ -4409,7 +6261,9 @@ template<typename CharType>
 class output_stream_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {}
+    explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept
+        : stream(s)
+    {}
 
     void write_character(CharType c) override
     {
@@ -4426,139 +6280,416 @@ class output_stream_adapter : public output_adapter_protocol<CharType>
 };
 
 /// output adapter for basic_string
-template<typename CharType>
+template<typename CharType, typename StringType = std::basic_string<CharType>>
 class output_string_adapter : public output_adapter_protocol<CharType>
 {
   public:
-    explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {}
+    explicit output_string_adapter(StringType& s) noexcept
+        : str(s)
+    {}
 
     void write_character(CharType c) override
     {
         str.push_back(c);
     }
 
-    void write_characters(const CharType* s, std::size_t length) override
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        str.append(s, length);
+    }
+
+  private:
+    StringType& str;
+};
+
+template<typename CharType, typename StringType = std::basic_string<CharType>>
+class output_adapter
+{
+  public:
+    output_adapter(std::vector<CharType>& vec)
+        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
+
+    output_adapter(std::basic_ostream<CharType>& s)
+        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+
+    output_adapter(StringType& s)
+        : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}
+
+    operator output_adapter_t<CharType>()
+    {
+        return oa;
+    }
+
+  private:
+    output_adapter_t<CharType> oa = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+
+#include <algorithm> // generate_n
+#include <array> // array
+#include <cassert> // assert
+#include <cmath> // ldexp
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstdio> // snprintf
+#include <cstring> // memcpy
+#include <iterator> // back_inserter
+#include <limits> // numeric_limits
+#include <string> // char_traits, string
+#include <utility> // make_pair, move
+
+// #include <nlohmann/detail/input/input_adapters.hpp>
+
+// #include <nlohmann/detail/input/json_sax.hpp>
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/is_sax.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary reader //
+///////////////////
+
+/*!
+@brief deserialization of CBOR, MessagePack, and UBJSON values
+*/
+template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>>
+class binary_reader
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using string_t = typename BasicJsonType::string_t;
+    using json_sax_t = SAX;
+
+  public:
+    /*!
+    @brief create a binary reader
+
+    @param[in] adapter  input adapter to read from
+    */
+    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
+    {
+        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
+        assert(ia);
+    }
+
+    /*!
+    @param[in] format  the binary format to parse
+    @param[in] sax_    a SAX event processor
+    @param[in] strict  whether to expect the input to be consumed completed
+
+    @return
+    */
+    bool sax_parse(const input_format_t format,
+                   json_sax_t* sax_,
+                   const bool strict = true)
+    {
+        sax = sax_;
+        bool result = false;
+
+        switch (format)
+        {
+            case input_format_t::bson:
+                result = parse_bson_internal();
+                break;
+
+            case input_format_t::cbor:
+                result = parse_cbor_internal();
+                break;
+
+            case input_format_t::msgpack:
+                result = parse_msgpack_internal();
+                break;
+
+            case input_format_t::ubjson:
+                result = parse_ubjson_internal();
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
+        }
+
+        // strict mode: next byte must be EOF
+        if (result and strict)
+        {
+            if (format == input_format_t::ubjson)
+            {
+                get_ignore_noop();
+            }
+            else
+            {
+                get();
+            }
+
+            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            {
+                return sax->parse_error(chars_read, get_token_string(),
+                                        parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value")));
+            }
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief determine system byte order
+
+    @return true if and only if system's byte order is little endian
+
+    @note from http://stackoverflow.com/a/1001328/266378
+    */
+    static constexpr bool little_endianess(int num = 1) noexcept
+    {
+        return (*reinterpret_cast<char*>(&num) == 1);
+    }
+
+  private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @brief Reads in a BSON-object and passes it to the SAX-parser.
+    @return whether a valid BSON-value was passed to the SAX parser
+    */
+    bool parse_bson_internal()
+    {
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+        {
+            return false;
+        }
+
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false)))
+        {
+            return false;
+        }
+
+        return sax->end_object();
+    }
+
+    /*!
+    @brief Parses a C-style string from the BSON input.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @return `true` if the \x00-byte indicating the end of the string was
+             encountered before the EOF; false` indicates an unexpected EOF.
+    */
+    bool get_bson_cstr(string_t& result)
+    {
+        auto out = std::back_inserter(result);
+        while (true)
+        {
+            get();
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring")))
+            {
+                return false;
+            }
+            if (current == 0x00)
+            {
+                return true;
+            }
+            *out++ = static_cast<char>(current);
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief Parses a zero-terminated string of length @a len from the BSON
+           input.
+    @param[in] len  The length (including the zero-byte at the end) of the
+                    string to be read.
+    @param[in, out] result  A reference to the string variable where the read
+                            string is to be stored.
+    @tparam NumberType The type of the length @a len
+    @pre len >= 1
+    @return `true` if the string was successfully parsed
+    */
+    template<typename NumberType>
+    bool get_bson_string(const NumberType len, string_t& result)
     {
-        str.append(s, length);
-    }
+        if (JSON_UNLIKELY(len < 1))
+        {
+            auto last_token = get_token_string();
+            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string")));
+        }
 
-  private:
-    std::basic_string<CharType>& str;
-};
+        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof();
+    }
 
-template<typename CharType>
-class output_adapter
-{
-  public:
-    output_adapter(std::vector<CharType>& vec)
-        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
+    /*!
+    @brief Read a BSON document element of the given @a element_type.
+    @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html
+    @param[in] element_type_parse_position The position in the input stream,
+               where the `element_type` was read.
+    @warning Not all BSON element types are supported yet. An unsupported
+             @a element_type will give rise to a parse_error.114:
+             Unsupported BSON record type 0x...
+    @return whether a valid BSON-object/array was passed to the SAX parser
+    */
+    bool parse_bson_element_internal(const int element_type,
+                                     const std::size_t element_type_parse_position)
+    {
+        switch (element_type)
+        {
+            case 0x01: // double
+            {
+                double number;
+                return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
 
-    output_adapter(std::basic_ostream<CharType>& s)
-        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+            case 0x02: // string
+            {
+                std::int32_t len;
+                string_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value);
+            }
 
-    output_adapter(std::basic_string<CharType>& s)
-        : oa(std::make_shared<output_string_adapter<CharType>>(s)) {}
+            case 0x03: // object
+            {
+                return parse_bson_internal();
+            }
 
-    operator output_adapter_t<CharType>()
-    {
-        return oa;
-    }
+            case 0x04: // array
+            {
+                return parse_bson_array();
+            }
 
-  private:
-    output_adapter_t<CharType> oa = nullptr;
-};
+            case 0x08: // boolean
+            {
+                return sax->boolean(static_cast<bool>(get()));
+            }
 
-//////////////////////////////
-// binary reader and writer //
-//////////////////////////////
+            case 0x0A: // null
+            {
+                return sax->null();
+            }
 
-/*!
-@brief deserialization of CBOR and MessagePack values
-*/
-template<typename BasicJsonType>
-class binary_reader
-{
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+            case 0x10: // int32
+            {
+                std::int32_t value;
+                return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
 
-  public:
-    /*!
-    @brief create a binary reader
+            case 0x12: // int64
+            {
+                std::int64_t value;
+                return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value);
+            }
 
-    @param[in] adapter  input adapter to read from
-    */
-    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
-    {
-        assert(ia);
+            default: // anything else not supported (yet)
+            {
+                char cr[3];
+                snprintf(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
+                return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr)));
+            }
+        }
     }
 
     /*!
-    @brief create a JSON value from CBOR input
+    @brief Read a BSON element list (as specified in the BSON-spec)
 
-    @param[in] strict  whether to expect the input to be consumed completed
-    @return JSON value created from CBOR input
+    The same binary layout is used for objects and arrays, hence it must be
+    indicated with the argument @a is_array which one is expected
+    (true --> array, false --> object).
 
-    @throw parse_error.110 if input ended unexpectedly or the end of file was
-                           not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported byte was read
+    @param[in] is_array Determines if the element list being read is to be
+                        treated as an object (@a is_array == false), or as an
+                        array (@a is_array == true).
+    @return whether a valid BSON-object/array was passed to the SAX parser
     */
-    BasicJsonType parse_cbor(const bool strict)
+    bool parse_bson_element_list(const bool is_array)
     {
-        const auto res = parse_cbor_internal();
-        if (strict)
+        string_t key;
+        while (int element_type = get())
         {
-            get();
-            check_eof(true);
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list")))
+            {
+                return false;
+            }
+
+            const std::size_t element_type_parse_position = chars_read;
+            if (JSON_UNLIKELY(not get_bson_cstr(key)))
+            {
+                return false;
+            }
+
+            if (not is_array)
+            {
+                sax->key(key);
+            }
+
+            if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position)))
+            {
+                return false;
+            }
+
+            // get_bson_cstr only appends
+            key.clear();
         }
-        return res;
+
+        return true;
     }
 
     /*!
-    @brief create a JSON value from MessagePack input
-
-    @param[in] strict  whether to expect the input to be consumed completed
-    @return JSON value created from MessagePack input
-
-    @throw parse_error.110 if input ended unexpectedly or the end of file was
-                           not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported byte was read
+    @brief Reads an array from the BSON input and passes it to the SAX-parser.
+    @return whether a valid BSON-array was passed to the SAX parser
     */
-    BasicJsonType parse_msgpack(const bool strict)
+    bool parse_bson_array()
     {
-        const auto res = parse_msgpack_internal();
-        if (strict)
+        std::int32_t document_size;
+        get_number<std::int32_t, true>(input_format_t::bson, document_size);
+
+        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
         {
-            get();
-            check_eof(true);
+            return false;
         }
-        return res;
-    }
-
-    /*!
-    @brief determine system byte order
 
-    @return true if and only if system's byte order is little endian
+        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true)))
+        {
+            return false;
+        }
 
-    @note from http://stackoverflow.com/a/1001328/266378
-    */
-    static constexpr bool little_endianess(int num = 1) noexcept
-    {
-        return (*reinterpret_cast<char*>(&num) == 1);
+        return sax->end_array();
     }
 
-  private:
+    //////////
+    // CBOR //
+    //////////
+
     /*!
     @param[in] get_char  whether a new character should be retrieved from the
                          input (true, default) or whether the last read
                          character should be considered instead
+
+    @return whether a valid CBOR value was passed to the SAX parser
     */
-    BasicJsonType parse_cbor_internal(const bool get_char = true)
+    bool parse_cbor_internal(const bool get_char = true)
     {
         switch (get_char ? get() : current)
         {
             // EOF
             case std::char_traits<char>::eof():
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                return unexpect_eof(input_format_t::cbor, "value");
 
             // Integer 0x00..0x17 (0..23)
             case 0x00:
@@ -4585,19 +6716,31 @@ class binary_reader
             case 0x15:
             case 0x16:
             case 0x17:
-                return static_cast<number_unsigned_t>(current);
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             case 0x18: // Unsigned integer (one-byte uint8_t follows)
-                return get_number<uint8_t>();
+            {
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x19: // Unsigned integer (two-byte uint16_t follows)
-                return get_number<uint16_t>();
+            {
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x1A: // Unsigned integer (four-byte uint32_t follows)
-                return get_number<uint32_t>();
+            {
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
-                return get_number<uint64_t>();
+            {
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
+            }
 
             // Negative integer -1-0x00..-1-0x17 (-1..-24)
             case 0x20:
@@ -4624,28 +6767,31 @@ class binary_reader
             case 0x35:
             case 0x36:
             case 0x37:
-                return static_cast<int8_t>(0x20 - 1 - current);
+                return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current));
 
             case 0x38: // Negative integer (one-byte uint8_t follows)
             {
-                // must be uint8_t !
-                return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
+                uint8_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
             {
-                return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
+                uint16_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
             {
-                return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
+                uint32_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
             }
 
             case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
             {
-                return static_cast<number_integer_t>(-1) -
-                       static_cast<number_integer_t>(get_number<uint64_t>());
+                uint64_t number;
+                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1)
+                        - static_cast<number_integer_t>(number));
             }
 
             // UTF-8 string (0x00..0x17 bytes follow)
@@ -4679,7 +6825,8 @@ class binary_reader
             case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
             case 0x7F: // UTF-8 string (indefinite length)
             {
-                return get_cbor_string();
+                string_t s;
+                return get_cbor_string(s) and sax->string(s);
             }
 
             // array (0x00..0x17 data items follow)
@@ -4707,176 +6854,361 @@ class binary_reader
             case 0x95:
             case 0x96:
             case 0x97:
+                return get_cbor_array(static_cast<std::size_t>(current & 0x1F));
+
+            case 0x98: // array (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x99: // array (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9A: // array (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9B: // array (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
+            }
+
+            case 0x9F: // array (indefinite length)
+                return get_cbor_array(std::size_t(-1));
+
+            // map (0x00..0x17 pairs of data items follow)
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+                return get_cbor_object(static_cast<std::size_t>(current & 0x1F));
+
+            case 0xB8: // map (one-byte uint8_t for n follows)
+            {
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xB9: // map (two-byte uint16_t for n follow)
+            {
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBA: // map (four-byte uint32_t for n follow)
+            {
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBB: // map (eight-byte uint64_t for n follow)
+            {
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
+            }
+
+            case 0xBF: // map (indefinite length)
+                return get_cbor_object(std::size_t(-1));
+
+            case 0xF4: // false
+                return sax->boolean(false);
+
+            case 0xF5: // true
+                return sax->boolean(true);
+
+            case 0xF6: // null
+                return sax->null();
+
+            case 0xF9: // Half-Precision Float (two-byte IEEE 754)
+            {
+                const int byte1_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+                const int byte2_raw = get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
+                {
+                    return false;
+                }
+
+                const auto byte1 = static_cast<unsigned char>(byte1_raw);
+                const auto byte2 = static_cast<unsigned char>(byte2_raw);
+
+                // code from RFC 7049, Appendix D, Figure 3:
+                // As half-precision floating-point numbers were only added
+                // to IEEE 754 in 2008, today's programming platforms often
+                // still only have limited support for them. It is very
+                // easy to include at least decoding support for them even
+                // without such support. An example of a small decoder for
+                // half-precision floating-point numbers in the C language
+                // is shown in Fig. 3.
+                const int half = (byte1 << 8) + byte2;
+                const double val = [&half]
+                {
+                    const int exp = (half >> 10) & 0x1F;
+                    const int mant = half & 0x3FF;
+                    assert(0 <= exp and exp <= 32);
+                    assert(0 <= mant and mant <= 1024);
+                    switch (exp)
+                    {
+                        case 0:
+                            return std::ldexp(mant, -24);
+                        case 31:
+                            return (mant == 0)
+                            ? std::numeric_limits<double>::infinity()
+                            : std::numeric_limits<double>::quiet_NaN();
+                        default:
+                            return std::ldexp(mant + 1024, exp - 25);
+                    }
+                }();
+                return sax->number_float((half & 0x8000) != 0
+                                         ? static_cast<number_float_t>(-val)
+                                         : static_cast<number_float_t>(val), "");
+            }
+
+            case 0xFA: // Single-Precision Float (four-byte IEEE 754)
+            {
+                float number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
+            {
+                double number;
+                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            default: // anything else (0xFF is handled inside the other types)
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
+
+    /*!
+    @brief reads a CBOR string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+    Additionally, CBOR's strings with indefinite lengths are supported.
+
+    @param[out] result  created string
+
+    @return whether string creation completed
+    */
+    bool get_cbor_string(string_t& result)
+    {
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6A:
+            case 0x6B:
+            case 0x6C:
+            case 0x6D:
+            case 0x6E:
+            case 0x6F:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
             {
-                return get_cbor_array(current & 0x1F);
+                return get_string(input_format_t::cbor, current & 0x1F, result);
             }
 
-            case 0x98: // array (one-byte uint8_t for n follows)
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
             {
-                return get_cbor_array(get_number<uint8_t>());
+                uint8_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
             }
 
-            case 0x99: // array (two-byte uint16_t for n follow)
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
             {
-                return get_cbor_array(get_number<uint16_t>());
+                uint16_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
             }
 
-            case 0x9A: // array (four-byte uint32_t for n follow)
+            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
             {
-                return get_cbor_array(get_number<uint32_t>());
+                uint32_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
             }
 
-            case 0x9B: // array (eight-byte uint64_t for n follow)
+            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
             {
-                return get_cbor_array(get_number<uint64_t>());
+                uint64_t len;
+                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
             }
 
-            case 0x9F: // array (indefinite length)
+            case 0x7F: // UTF-8 string (indefinite length)
             {
-                BasicJsonType result = value_t::array;
                 while (get() != 0xFF)
                 {
-                    result.push_back(parse_cbor_internal(false));
+                    string_t chunk;
+                    if (not get_cbor_string(chunk))
+                    {
+                        return false;
+                    }
+                    result.append(chunk);
                 }
-                return result;
-            }
-
-            // map (0x00..0x17 pairs of data items follow)
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-            {
-                return get_cbor_object(current & 0x1F);
-            }
-
-            case 0xB8: // map (one-byte uint8_t for n follows)
-            {
-                return get_cbor_object(get_number<uint8_t>());
-            }
-
-            case 0xB9: // map (two-byte uint16_t for n follow)
-            {
-                return get_cbor_object(get_number<uint16_t>());
+                return true;
             }
 
-            case 0xBA: // map (four-byte uint32_t for n follow)
+            default:
             {
-                return get_cbor_object(get_number<uint32_t>());
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string")));
             }
+        }
+    }
 
-            case 0xBB: // map (eight-byte uint64_t for n follow)
-            {
-                return get_cbor_object(get_number<uint64_t>());
-            }
+    /*!
+    @param[in] len  the length of the array or std::size_t(-1) for an
+                    array of indefinite size
+    @return whether array creation completed
+    */
+    bool get_cbor_array(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_array(len)))
+        {
+            return false;
+        }
 
-            case 0xBF: // map (indefinite length)
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
             {
-                BasicJsonType result = value_t::object;
-                while (get() != 0xFF)
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
                 {
-                    auto key = get_cbor_string();
-                    result[key] = parse_cbor_internal();
+                    return false;
                 }
-                return result;
             }
-
-            case 0xF4: // false
+        }
+        else
+        {
+            while (get() != 0xFF)
             {
-                return false;
+                if (JSON_UNLIKELY(not parse_cbor_internal(false)))
+                {
+                    return false;
+                }
             }
+        }
 
-            case 0xF5: // true
-            {
-                return true;
-            }
+        return sax->end_array();
+    }
 
-            case 0xF6: // null
-            {
-                return value_t::null;
-            }
+    /*!
+    @param[in] len  the length of the object or std::size_t(-1) for an
+                    object of indefinite size
+    @return whether object creation completed
+    */
+    bool get_cbor_object(const std::size_t len)
+    {
+        if (not JSON_UNLIKELY(sax->start_object(len)))
+        {
+            return false;
+        }
 
-            case 0xF9: // Half-Precision Float (two-byte IEEE 754)
+        string_t key;
+        if (len != std::size_t(-1))
+        {
+            for (std::size_t i = 0; i < len; ++i)
             {
-                const int byte1 = get();
-                check_eof();
-                const int byte2 = get();
-                check_eof();
-
-                // code from RFC 7049, Appendix D, Figure 3:
-                // As half-precision floating-point numbers were only added
-                // to IEEE 754 in 2008, today's programming platforms often
-                // still only have limited support for them. It is very
-                // easy to include at least decoding support for them even
-                // without such support. An example of a small decoder for
-                // half-precision floating-point numbers in the C language
-                // is shown in Fig. 3.
-                const int half = (byte1 << 8) + byte2;
-                const int exp = (half >> 10) & 0x1F;
-                const int mant = half & 0x3FF;
-                double val;
-                if (exp == 0)
-                {
-                    val = std::ldexp(mant, -24);
-                }
-                else if (exp != 31)
+                get();
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
                 {
-                    val = std::ldexp(mant + 1024, exp - 25);
+                    return false;
                 }
-                else
+
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
                 {
-                    val = (mant == 0) ? std::numeric_limits<double>::infinity()
-                          : std::numeric_limits<double>::quiet_NaN();
+                    return false;
                 }
-                return (half & 0x8000) != 0 ? -val : val;
+                key.clear();
             }
-
-            case 0xFA: // Single-Precision Float (four-byte IEEE 754)
-            {
-                return get_number<float>();
-            }
-
-            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
+        }
+        else
+        {
+            while (get() != 0xFF)
             {
-                return get_number<double>();
-            }
+                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
+                {
+                    return false;
+                }
 
-            default: // anything else (0xFF is handled inside the other types)
-            {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
+                if (JSON_UNLIKELY(not parse_cbor_internal()))
+                {
+                    return false;
+                }
+                key.clear();
             }
         }
+
+        return sax->end_object();
     }
 
-    BasicJsonType parse_msgpack_internal()
+    /////////////
+    // MsgPack //
+    /////////////
+
+    /*!
+    @return whether a valid MessagePack value was passed to the SAX parser
+    */
+    bool parse_msgpack_internal()
     {
         switch (get())
         {
             // EOF
             case std::char_traits<char>::eof():
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                return unexpect_eof(input_format_t::msgpack, "value");
 
             // positive fixint
             case 0x00:
@@ -5007,7 +7339,7 @@ class binary_reader
             case 0x7D:
             case 0x7E:
             case 0x7F:
-                return static_cast<number_unsigned_t>(current);
+                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
 
             // fixmap
             case 0x80:
@@ -5026,31 +7358,228 @@ class binary_reader
             case 0x8D:
             case 0x8E:
             case 0x8F:
+                return get_msgpack_object(static_cast<std::size_t>(current & 0x0F));
+
+            // fixarray
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            case 0x98:
+            case 0x99:
+            case 0x9A:
+            case 0x9B:
+            case 0x9C:
+            case 0x9D:
+            case 0x9E:
+            case 0x9F:
+                return get_msgpack_array(static_cast<std::size_t>(current & 0x0F));
+
+            // fixstr
+            case 0xA0:
+            case 0xA1:
+            case 0xA2:
+            case 0xA3:
+            case 0xA4:
+            case 0xA5:
+            case 0xA6:
+            case 0xA7:
+            case 0xA8:
+            case 0xA9:
+            case 0xAA:
+            case 0xAB:
+            case 0xAC:
+            case 0xAD:
+            case 0xAE:
+            case 0xAF:
+            case 0xB0:
+            case 0xB1:
+            case 0xB2:
+            case 0xB3:
+            case 0xB4:
+            case 0xB5:
+            case 0xB6:
+            case 0xB7:
+            case 0xB8:
+            case 0xB9:
+            case 0xBA:
+            case 0xBB:
+            case 0xBC:
+            case 0xBD:
+            case 0xBE:
+            case 0xBF:
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
+
+            case 0xC0: // nil
+                return sax->null();
+
+            case 0xC2: // false
+                return sax->boolean(false);
+
+            case 0xC3: // true
+                return sax->boolean(true);
+
+            case 0xCA: // float 32
+            {
+                float number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xCB: // float 64
+            {
+                double number;
+                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
+
+            case 0xCC: // uint 8
+            {
+                uint8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCD: // uint 16
+            {
+                uint16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCE: // uint 32
+            {
+                uint32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xCF: // uint 64
+            {
+                uint64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
+            }
+
+            case 0xD0: // int 8
+            {
+                int8_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD1: // int 16
+            {
+                int16_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD2: // int 32
+            {
+                int32_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD3: // int 64
+            {
+                int64_t number;
+                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
+            }
+
+            case 0xD9: // str 8
+            case 0xDA: // str 16
+            case 0xDB: // str 32
+            {
+                string_t s;
+                return get_msgpack_string(s) and sax->string(s);
+            }
+
+            case 0xDC: // array 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
+            }
+
+            case 0xDD: // array 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
+            }
+
+            case 0xDE: // map 16
             {
-                return get_msgpack_object(current & 0x0F);
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
             }
 
-            // fixarray
-            case 0x90:
-            case 0x91:
-            case 0x92:
-            case 0x93:
-            case 0x94:
-            case 0x95:
-            case 0x96:
-            case 0x97:
-            case 0x98:
-            case 0x99:
-            case 0x9A:
-            case 0x9B:
-            case 0x9C:
-            case 0x9D:
-            case 0x9E:
-            case 0x9F:
+            case 0xDF: // map 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
+            }
+
+            // negative fixint
+            case 0xE0:
+            case 0xE1:
+            case 0xE2:
+            case 0xE3:
+            case 0xE4:
+            case 0xE5:
+            case 0xE6:
+            case 0xE7:
+            case 0xE8:
+            case 0xE9:
+            case 0xEA:
+            case 0xEB:
+            case 0xEC:
+            case 0xED:
+            case 0xEE:
+            case 0xEF:
+            case 0xF0:
+            case 0xF1:
+            case 0xF2:
+            case 0xF3:
+            case 0xF4:
+            case 0xF5:
+            case 0xF6:
+            case 0xF7:
+            case 0xF8:
+            case 0xF9:
+            case 0xFA:
+            case 0xFB:
+            case 0xFC:
+            case 0xFD:
+            case 0xFE:
+            case 0xFF:
+                return sax->number_integer(static_cast<int8_t>(current));
+
+            default: // anything else
             {
-                return get_msgpack_array(current & 0x0F);
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value")));
             }
+        }
+    }
+
+    /*!
+    @brief reads a MessagePack string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+
+    @param[out] result  created string
+
+    @return whether string creation completed
+    */
+    bool get_msgpack_string(string_t& result)
+    {
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string")))
+        {
+            return false;
+        }
 
+        switch (current)
+        {
             // fixstr
             case 0xA0:
             case 0xA1:
@@ -5084,117 +7613,522 @@ class binary_reader
             case 0xBD:
             case 0xBE:
             case 0xBF:
-                return get_msgpack_string();
+            {
+                return get_string(input_format_t::msgpack, current & 0x1F, result);
+            }
 
-            case 0xC0: // nil
-                return value_t::null;
+            case 0xD9: // str 8
+            {
+                uint8_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
 
-            case 0xC2: // false
+            case 0xDA: // str 16
+            {
+                uint16_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            case 0xDB: // str 32
+            {
+                uint32_t len;
+                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string")));
+            }
+        }
+    }
+
+    /*!
+    @param[in] len  the length of the array
+    @return whether array creation completed
+    */
+    bool get_msgpack_array(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_array(len)))
+        {
+            return false;
+        }
+
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
+            {
                 return false;
+            }
+        }
 
-            case 0xC3: // true
+        return sax->end_array();
+    }
+
+    /*!
+    @param[in] len  the length of the object
+    @return whether object creation completed
+    */
+    bool get_msgpack_object(const std::size_t len)
+    {
+        if (JSON_UNLIKELY(not sax->start_object(len)))
+        {
+            return false;
+        }
+
+        string_t key;
+        for (std::size_t i = 0; i < len; ++i)
+        {
+            get();
+            if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key)))
+            {
+                return false;
+            }
+
+            if (JSON_UNLIKELY(not parse_msgpack_internal()))
+            {
+                return false;
+            }
+            key.clear();
+        }
+
+        return sax->end_object();
+    }
+
+    ////////////
+    // UBJSON //
+    ////////////
+
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether a valid UBJSON value was passed to the SAX parser
+    */
+    bool parse_ubjson_internal(const bool get_char = true)
+    {
+        return get_ubjson_value(get_char ? get_ignore_noop() : current);
+    }
+
+    /*!
+    @brief reads a UBJSON string
+
+    This function is either called after reading the 'S' byte explicitly
+    indicating a string, or in case of an object key where the 'S' byte can be
+    left out.
+
+    @param[out] result   created string
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+
+    @return whether string creation completed
+    */
+    bool get_ubjson_string(string_t& result, const bool get_char = true)
+    {
+        if (get_char)
+        {
+            get();  // TODO: may we ignore N here?
+        }
+
+        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+        {
+            return false;
+        }
+
+        switch (current)
+        {
+            case 'U':
+            {
+                uint8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'i':
+            {
+                int8_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'I':
+            {
+                int16_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'l':
+            {
+                int32_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            case 'L':
+            {
+                int64_t len;
+                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
+            }
+
+            default:
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string")));
+        }
+    }
+
+    /*!
+    @param[out] result  determined size
+    @return whether size determination completed
+    */
+    bool get_ubjson_size_value(std::size_t& result)
+    {
+        switch (get_ignore_noop())
+        {
+            case 'U':
+            {
+                uint8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'i':
+            {
+                int8_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'I':
+            {
+                int16_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'l':
+            {
+                int32_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
+                return true;
+            }
+
+            case 'L':
+            {
+                int64_t number;
+                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
+                {
+                    return false;
+                }
+                result = static_cast<std::size_t>(number);
                 return true;
+            }
+
+            default:
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size")));
+            }
+        }
+    }
+
+    /*!
+    @brief determine the type and size for a container
+
+    In the optimized UBJSON format, a type and a size can be provided to allow
+    for a more compact representation.
+
+    @param[out] result  pair of the size and the type
+
+    @return whether pair creation completed
+    */
+    bool get_ubjson_size_type(std::pair<std::size_t, int>& result)
+    {
+        result.first = string_t::npos; // size
+        result.second = 0; // type
+
+        get_ignore_noop();
+
+        if (current == '$')
+        {
+            result.second = get();  // must not ignore 'N', because 'N' maybe the type
+            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type")))
+            {
+                return false;
+            }
+
+            get_ignore_noop();
+            if (JSON_UNLIKELY(current != '#'))
+            {
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
+                {
+                    return false;
+                }
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size")));
+            }
+
+            return get_ubjson_size_value(result.first);
+        }
+        else if (current == '#')
+        {
+            return get_ubjson_size_value(result.first);
+        }
+        return true;
+    }
+
+    /*!
+    @param prefix  the previously read or set type prefix
+    @return whether value creation completed
+    */
+    bool get_ubjson_value(const int prefix)
+    {
+        switch (prefix)
+        {
+            case std::char_traits<char>::eof():  // EOF
+                return unexpect_eof(input_format_t::ubjson, "value");
+
+            case 'T':  // true
+                return sax->boolean(true);
+            case 'F':  // false
+                return sax->boolean(false);
+
+            case 'Z':  // null
+                return sax->null();
+
+            case 'U':
+            {
+                uint8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number);
+            }
 
-            case 0xCA: // float 32
-                return get_number<float>();
+            case 'i':
+            {
+                int8_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
 
-            case 0xCB: // float 64
-                return get_number<double>();
+            case 'I':
+            {
+                int16_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
 
-            case 0xCC: // uint 8
-                return get_number<uint8_t>();
+            case 'l':
+            {
+                int32_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
 
-            case 0xCD: // uint 16
-                return get_number<uint16_t>();
+            case 'L':
+            {
+                int64_t number;
+                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
+            }
 
-            case 0xCE: // uint 32
-                return get_number<uint32_t>();
+            case 'd':
+            {
+                float number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
 
-            case 0xCF: // uint 64
-                return get_number<uint64_t>();
+            case 'D':
+            {
+                double number;
+                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
+            }
 
-            case 0xD0: // int 8
-                return get_number<int8_t>();
+            case 'C':  // char
+            {
+                get();
+                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char")))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(current > 127))
+                {
+                    auto last_token = get_token_string();
+                    return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char")));
+                }
+                string_t s(1, static_cast<char>(current));
+                return sax->string(s);
+            }
 
-            case 0xD1: // int 16
-                return get_number<int16_t>();
+            case 'S':  // string
+            {
+                string_t s;
+                return get_ubjson_string(s) and sax->string(s);
+            }
 
-            case 0xD2: // int 32
-                return get_number<int32_t>();
+            case '[':  // array
+                return get_ubjson_array();
 
-            case 0xD3: // int 64
-                return get_number<int64_t>();
+            case '{':  // object
+                return get_ubjson_object();
 
-            case 0xD9: // str 8
-            case 0xDA: // str 16
-            case 0xDB: // str 32
-                return get_msgpack_string();
+            default: // anything else
+            {
+                auto last_token = get_token_string();
+                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value")));
+            }
+        }
+    }
 
-            case 0xDC: // array 16
+    /*!
+    @return whether array creation completed
+    */
+    bool get_ubjson_array()
+    {
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
+
+        if (size_and_type.first != string_t::npos)
+        {
+            if (JSON_UNLIKELY(not sax->start_array(size_and_type.first)))
             {
-                return get_msgpack_array(get_number<uint16_t>());
+                return false;
             }
 
-            case 0xDD: // array 32
+            if (size_and_type.second != 0)
+            {
+                if (size_and_type.second != 'N')
+                {
+                    for (std::size_t i = 0; i < size_and_type.first; ++i)
+                    {
+                        if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                        {
+                            return false;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
             {
-                return get_msgpack_array(get_number<uint32_t>());
+                return false;
             }
 
-            case 0xDE: // map 16
+            while (current != ']')
             {
-                return get_msgpack_object(get_number<uint16_t>());
+                if (JSON_UNLIKELY(not parse_ubjson_internal(false)))
+                {
+                    return false;
+                }
+                get_ignore_noop();
             }
+        }
 
-            case 0xDF: // map 32
+        return sax->end_array();
+    }
+
+    /*!
+    @return whether object creation completed
+    */
+    bool get_ubjson_object()
+    {
+        std::pair<std::size_t, int> size_and_type;
+        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
+        {
+            return false;
+        }
+
+        string_t key;
+        if (size_and_type.first != string_t::npos)
+        {
+            if (JSON_UNLIKELY(not sax->start_object(size_and_type.first)))
             {
-                return get_msgpack_object(get_number<uint32_t>());
+                return false;
             }
 
-            // positive fixint
-            case 0xE0:
-            case 0xE1:
-            case 0xE2:
-            case 0xE3:
-            case 0xE4:
-            case 0xE5:
-            case 0xE6:
-            case 0xE7:
-            case 0xE8:
-            case 0xE9:
-            case 0xEA:
-            case 0xEB:
-            case 0xEC:
-            case 0xED:
-            case 0xEE:
-            case 0xEF:
-            case 0xF0:
-            case 0xF1:
-            case 0xF2:
-            case 0xF3:
-            case 0xF4:
-            case 0xF5:
-            case 0xF6:
-            case 0xF7:
-            case 0xF8:
-            case 0xF9:
-            case 0xFA:
-            case 0xFB:
-            case 0xFC:
-            case 0xFD:
-            case 0xFE:
-            case 0xFF:
-                return static_cast<int8_t>(current);
+            if (size_and_type.second != 0)
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+            else
+            {
+                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                {
+                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
+                    {
+                        return false;
+                    }
+                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                    {
+                        return false;
+                    }
+                    key.clear();
+                }
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
+            {
+                return false;
+            }
 
-            default: // anything else
+            while (current != '}')
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(112, chars_read,
-                                               "error reading MessagePack; last byte: 0x" + ss.str()));
+                if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key)))
+                {
+                    return false;
+                }
+                if (JSON_UNLIKELY(not parse_ubjson_internal()))
+                {
+                    return false;
+                }
+                get_ignore_noop();
+                key.clear();
             }
         }
+
+        return sax->end_object();
     }
 
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
     /*!
     @brief get next character from the input
 
@@ -5210,30 +8144,48 @@ class binary_reader
         return (current = ia->get_character());
     }
 
+    /*!
+    @return character read from the input after ignoring all 'N' entries
+    */
+    int get_ignore_noop()
+    {
+        do
+        {
+            get();
+        }
+        while (current == 'N');
+
+        return current;
+    }
+
     /*
     @brief read a number from the input
 
     @tparam NumberType the type of the number
+    @param[in] format   the current format (for diagnostics)
+    @param[out] result  number of type @a NumberType
 
-    @return number of type @a NumberType
+    @return whether conversion completed
 
     @note This function needs to respect the system's endianess, because
-          bytes in CBOR and MessagePack are stored in network order (big
-          endian) and therefore need reordering on little endian systems.
-
-    @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes
+          bytes in CBOR, MessagePack, and UBJSON are stored in network order
+          (big endian) and therefore need reordering on little endian systems.
     */
-    template<typename NumberType> NumberType get_number()
+    template<typename NumberType, bool InputIsLittleEndian = false>
+    bool get_number(const input_format_t format, NumberType& result)
     {
         // step 1: read input into array with system's byte order
         std::array<uint8_t, sizeof(NumberType)> vec;
         for (std::size_t i = 0; i < sizeof(NumberType); ++i)
         {
             get();
-            check_eof();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "number")))
+            {
+                return false;
+            }
 
             // reverse byte order prior to conversion if necessary
-            if (is_little_endian)
+            if (is_little_endian && !InputIsLittleEndian)
             {
                 vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
             }
@@ -5244,832 +8196,2609 @@ class binary_reader
         }
 
         // step 2: convert array into number of type T and return
-        NumberType result;
         std::memcpy(&result, vec.data(), sizeof(NumberType));
-        return result;
+        return true;
     }
 
     /*!
     @brief create a string by reading characters from the input
 
-    @param[in] len number of bytes to read
+    @tparam NumberType the type of the number
+    @param[in] format the current format (for diagnostics)
+    @param[in] len number of characters to read
+    @param[out] result string created by reading @a len bytes
+
+    @return whether string creation completed
 
     @note We can not reserve @a len bytes for the result, because @a len
-          may be too large. Usually, @ref check_eof() detects the end of
+          may be too large. Usually, @ref unexpect_eof() detects the end of
           the input before we run out of string memory.
-
-    @return string created by reading @a len bytes
-
-    @throw parse_error.110 if input has less than @a len bytes
     */
     template<typename NumberType>
-    std::string get_string(const NumberType len)
+    bool get_string(const input_format_t format,
+                    const NumberType len,
+                    string_t& result)
     {
-        std::string result;
-        std::generate_n(std::back_inserter(result), len, [this]()
+        bool success = true;
+        std::generate_n(std::back_inserter(result), len, [this, &success, &format]()
         {
             get();
-            check_eof();
+            if (JSON_UNLIKELY(not unexpect_eof(format, "string")))
+            {
+                success = false;
+            }
             return static_cast<char>(current);
         });
-        return result;
+        return success;
     }
 
     /*!
-    @brief reads a CBOR string
+    @param[in] format   the current format (for diagnostics)
+    @param[in] context  further context information (for diagnostics)
+    @return whether the last read character is not EOF
+    */
+    bool unexpect_eof(const input_format_t format, const char* context) const
+    {
+        if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+        {
+            return sax->parse_error(chars_read, "<end of file>",
+                                    parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context)));
+        }
+        return true;
+    }
 
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
-    Additionally, CBOR's strings with indefinite lengths are supported.
+    /*!
+    @return a string representation of the last read byte
+    */
+    std::string get_token_string() const
+    {
+        char cr[3];
+        snprintf(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
+        return std::string{cr};
+    }
+
+    /*!
+    @param[in] format   the current format
+    @param[in] detail   a detailed error message
+    @param[in] context  further contect information
+    @return a message string to use in the parse_error exceptions
+    */
+    std::string exception_message(const input_format_t format,
+                                  const std::string& detail,
+                                  const std::string& context) const
+    {
+        std::string error_msg = "syntax error while parsing ";
+
+        switch (format)
+        {
+            case input_format_t::cbor:
+                error_msg += "CBOR";
+                break;
+
+            case input_format_t::msgpack:
+                error_msg += "MessagePack";
+                break;
+
+            case input_format_t::ubjson:
+                error_msg += "UBJSON";
+                break;
+
+            case input_format_t::bson:
+                error_msg += "BSON";
+                break;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                // LCOV_EXCL_STOP
+        }
+
+        return error_msg + " " + context + ": " + detail;
+    }
+
+  private:
+    /// input adapter
+    input_adapter_t ia = nullptr;
+
+    /// the current character
+    int current = std::char_traits<char>::eof();
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+
+    /// whether we can assume little endianess
+    const bool is_little_endian = little_endianess();
+
+    /// the SAX parser
+    json_sax_t* sax = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
+
+
+#include <algorithm> // reverse
+#include <array> // array
+#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
+#include <cstring> // memcpy
+#include <limits> // numeric_limits
+
+// #include <nlohmann/detail/input/binary_reader.hpp>
+
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+///////////////////
+// binary writer //
+///////////////////
+
+/*!
+@brief serialization to CBOR and MessagePack values
+*/
+template<typename BasicJsonType, typename CharType>
+class binary_writer
+{
+    using string_t = typename BasicJsonType::string_t;
 
-    @return string
+  public:
+    /*!
+    @brief create a binary writer
 
-    @throw parse_error.110 if input ended
-    @throw parse_error.113 if an unexpected byte is read
+    @param[in] adapter  output adapter to write to
     */
-    std::string get_cbor_string()
+    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
     {
-        check_eof();
+        assert(oa);
+    }
 
-        switch (current)
+    /*!
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson(const BasicJsonType& j)
+    {
+        switch (j.type())
         {
-            // UTF-8 string (0x00..0x17 bytes follow)
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6A:
-            case 0x6B:
-            case 0x6C:
-            case 0x6D:
-            case 0x6E:
-            case 0x6F:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
+            case value_t::object:
             {
-                return get_string(current & 0x1F);
+                write_bson_object(*j.m_value.object);
+                break;
             }
 
-            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            default:
             {
-                return get_string(get_number<uint8_t>());
+                JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
             }
+        }
+    }
 
-            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+    /*!
+    @param[in] j  JSON value to serialize
+    */
+    void write_cbor(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
             {
-                return get_string(get_number<uint16_t>());
+                oa->write_character(to_char_type(0xF6));
+                break;
             }
 
-            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
+            case value_t::boolean:
             {
-                return get_string(get_number<uint32_t>());
+                oa->write_character(j.m_value.boolean
+                                    ? to_char_type(0xF5)
+                                    : to_char_type(0xF4));
+                break;
             }
 
-            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
+            case value_t::number_integer:
             {
-                return get_string(get_number<uint64_t>());
+                if (j.m_value.number_integer >= 0)
+                {
+                    // CBOR does not differentiate between positive signed
+                    // integers and unsigned integers. Therefore, we used the
+                    // code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_integer <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x18));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x19));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x1A));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else
+                    {
+                        oa->write_character(to_char_type(0x1B));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    // The conversions below encode the sign in the first
+                    // byte, and the value is converted to a positive number.
+                    const auto positive_number = -1 - j.m_value.number_integer;
+                    if (j.m_value.number_integer >= -24)
+                    {
+                        write_number(static_cast<uint8_t>(0x20 + positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x38));
+                        write_number(static_cast<uint8_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x39));
+                        write_number(static_cast<uint16_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(to_char_type(0x3A));
+                        write_number(static_cast<uint32_t>(positive_number));
+                    }
+                    else
+                    {
+                        oa->write_character(to_char_type(0x3B));
+                        write_number(static_cast<uint64_t>(positive_number));
+                    }
+                }
+                break;
             }
 
-            case 0x7F: // UTF-8 string (indefinite length)
+            case value_t::number_unsigned:
             {
-                std::string result;
-                while (get() != 0xFF)
+                if (j.m_value.number_unsigned <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x18));
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x19));
+                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x1A));
+                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                }
+                else
                 {
-                    check_eof();
-                    result.push_back(static_cast<char>(current));
+                    oa->write_character(to_char_type(0x1B));
+                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
                 }
-                return result;
+                break;
             }
 
-            default:
+            case value_t::number_float:
             {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
+                oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
+                write_number(j.m_value.number_float);
+                break;
             }
-        }
-    }
-
-    template<typename NumberType>
-    BasicJsonType get_cbor_array(const NumberType len)
-    {
-        BasicJsonType result = value_t::array;
-        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
-        {
-            return parse_cbor_internal();
-        });
-        return result;
-    }
-
-    template<typename NumberType>
-    BasicJsonType get_cbor_object(const NumberType len)
-    {
-        BasicJsonType result = value_t::object;
-        std::generate_n(std::inserter(*result.m_value.object,
-                                      result.m_value.object->end()),
-                        len, [this]()
-        {
-            get();
-            auto key = get_cbor_string();
-            auto val = parse_cbor_internal();
-            return std::make_pair(std::move(key), std::move(val));
-        });
-        return result;
-    }
-
-    /*!
-    @brief reads a MessagePack string
-
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
-
-    @return string
-
-    @throw parse_error.110 if input ended
-    @throw parse_error.113 if an unexpected byte is read
-    */
-    std::string get_msgpack_string()
-    {
-        check_eof();
 
-        switch (current)
-        {
-            // fixstr
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-            case 0xB8:
-            case 0xB9:
-            case 0xBA:
-            case 0xBB:
-            case 0xBC:
-            case 0xBD:
-            case 0xBE:
-            case 0xBF:
+            case value_t::string:
             {
-                return get_string(current & 0x1F);
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x60 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x78));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x79));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x7A));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x7B));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
             }
 
-            case 0xD9: // str 8
+            case value_t::array:
             {
-                return get_string(get_number<uint8_t>());
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x80 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x98));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x99));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x9A));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x9B));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_cbor(el);
+                }
+                break;
             }
 
-            case 0xDA: // str 16
+            case value_t::object:
             {
-                return get_string(get_number<uint16_t>());
-            }
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0xA0 + N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xB8));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xB9));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xBA));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0xBB));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
 
-            case 0xDB: // str 32
-            {
-                return get_string(get_number<uint32_t>());
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_cbor(el.first);
+                    write_cbor(el.second);
+                }
+                break;
             }
 
             default:
-            {
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current;
-                JSON_THROW(parse_error::create(113, chars_read,
-                                               "expected a MessagePack string; last byte: 0x" + ss.str()));
-            }
+                break;
         }
     }
 
-    template<typename NumberType>
-    BasicJsonType get_msgpack_array(const NumberType len)
-    {
-        BasicJsonType result = value_t::array;
-        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
-        {
-            return parse_msgpack_internal();
-        });
-        return result;
-    }
-
-    template<typename NumberType>
-    BasicJsonType get_msgpack_object(const NumberType len)
-    {
-        BasicJsonType result = value_t::object;
-        std::generate_n(std::inserter(*result.m_value.object,
-                                      result.m_value.object->end()),
-                        len, [this]()
-        {
-            get();
-            auto key = get_msgpack_string();
-            auto val = parse_msgpack_internal();
-            return std::make_pair(std::move(key), std::move(val));
-        });
-        return result;
-    }
-
     /*!
-    @brief check if input ended
-    @throw parse_error.110 if input ended
+    @param[in] j  JSON value to serialize
     */
-    void check_eof(const bool expect_eof = false) const
+    void write_msgpack(const BasicJsonType& j)
     {
-        if (expect_eof)
+        switch (j.type())
         {
-            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            case value_t::null: // nil
             {
-                JSON_THROW(parse_error::create(110, chars_read, "expected end of input"));
+                oa->write_character(to_char_type(0xC0));
+                break;
             }
-        }
-        else
-        {
-            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+
+            case value_t::boolean: // true and false
             {
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                oa->write_character(j.m_value.boolean
+                                    ? to_char_type(0xC3)
+                                    : to_char_type(0xC2));
+                break;
             }
-        }
-    }
 
-  private:
-    /// input adapter
-    input_adapter_t ia = nullptr;
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // MessagePack does not differentiate between positive
+                    // signed integers and unsigned integers. Therefore, we used
+                    // the code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_unsigned < 128)
+                    {
+                        // positive fixnum
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        // uint 8
+                        oa->write_character(to_char_type(0xCC));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        // uint 16
+                        oa->write_character(to_char_type(0xCD));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        // uint 32
+                        oa->write_character(to_char_type(0xCE));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                    {
+                        // uint 64
+                        oa->write_character(to_char_type(0xCF));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    if (j.m_value.number_integer >= -32)
+                    {
+                        // negative fixnum
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                    {
+                        // int 8
+                        oa->write_character(to_char_type(0xD0));
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                    {
+                        // int 16
+                        oa->write_character(to_char_type(0xD1));
+                        write_number(static_cast<int16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                    {
+                        // int 32
+                        oa->write_character(to_char_type(0xD2));
+                        write_number(static_cast<int32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
+                    {
+                        // int 64
+                        oa->write_character(to_char_type(0xD3));
+                        write_number(static_cast<int64_t>(j.m_value.number_integer));
+                    }
+                }
+                break;
+            }
 
-    /// the current character
-    int current = std::char_traits<char>::eof();
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned < 128)
+                {
+                    // positive fixnum
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // uint 8
+                    oa->write_character(to_char_type(0xCC));
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // uint 16
+                    oa->write_character(to_char_type(0xCD));
+                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // uint 32
+                    oa->write_character(to_char_type(0xCE));
+                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    // uint 64
+                    oa->write_character(to_char_type(0xCF));
+                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                }
+                break;
+            }
 
-    /// the number of characters read
-    std::size_t chars_read = 0;
+            case value_t::number_float:
+            {
+                oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
+                write_number(j.m_value.number_float);
+                break;
+            }
 
-    /// whether we can assume little endianess
-    const bool is_little_endian = little_endianess();
-};
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 31)
+                {
+                    // fixstr
+                    write_number(static_cast<uint8_t>(0xA0 | N));
+                }
+                else if (N <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // str 8
+                    oa->write_character(to_char_type(0xD9));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // str 16
+                    oa->write_character(to_char_type(0xDA));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // str 32
+                    oa->write_character(to_char_type(0xDB));
+                    write_number(static_cast<uint32_t>(N));
+                }
 
-/*!
-@brief serialization to CBOR and MessagePack values
-*/
-template<typename BasicJsonType, typename CharType>
-class binary_writer
-{
-  public:
-    /*!
-    @brief create a binary writer
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 15)
+                {
+                    // fixarray
+                    write_number(static_cast<uint8_t>(0x90 | N));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // array 16
+                    oa->write_character(to_char_type(0xDC));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // array 32
+                    oa->write_character(to_char_type(0xDD));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_msgpack(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 15)
+                {
+                    // fixmap
+                    write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
+                }
+                else if (N <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // map 16
+                    oa->write_character(to_char_type(0xDE));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // map 32
+                    oa->write_character(to_char_type(0xDF));
+                    write_number(static_cast<uint32_t>(N));
+                }
 
-    @param[in] adapter  output adapter to write to
-    */
-    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
-    {
-        assert(oa);
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_msgpack(el.first);
+                    write_msgpack(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
     }
 
     /*!
-    @brief[in] j  JSON value to serialize
+    @param[in] j  JSON value to serialize
+    @param[in] use_count   whether to use '#' prefixes (optimized format)
+    @param[in] use_type    whether to use '$' prefixes (optimized format)
+    @param[in] add_prefix  whether prefixes need to be used for this value
     */
-    void write_cbor(const BasicJsonType& j)
+    void write_ubjson(const BasicJsonType& j, const bool use_count,
+                      const bool use_type, const bool add_prefix = true)
     {
         switch (j.type())
         {
             case value_t::null:
             {
-                oa->write_character(static_cast<CharType>(0xF6));
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('Z'));
+                }
                 break;
             }
 
             case value_t::boolean:
             {
-                oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xF5)
-                                    : static_cast<CharType>(0xF4));
+                if (add_prefix)
+                {
+                    oa->write_character(j.m_value.boolean
+                                        ? to_char_type('T')
+                                        : to_char_type('F'));
+                }
                 break;
             }
 
             case value_t::number_integer:
             {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // CBOR does not differentiate between positive signed
-                    // integers and unsigned integers. Therefore, we used the
-                    // code from the value_t::number_unsigned case here.
-                    if (j.m_value.number_integer <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x18));
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x19));
-                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x1A));
-                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                    }
-                    else
-                    {
-                        oa->write_character(static_cast<CharType>(0x1B));
-                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                    }
-                }
-                else
+                write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix);
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix);
+                break;
+            }
+
+            case value_t::number_float:
+            {
+                write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix);
+                break;
+            }
+
+            case value_t::string:
+            {
+                if (add_prefix)
                 {
-                    // The conversions below encode the sign in the first
-                    // byte, and the value is converted to a positive number.
-                    const auto positive_number = -1 - j.m_value.number_integer;
-                    if (j.m_value.number_integer >= -24)
-                    {
-                        write_number(static_cast<uint8_t>(0x20 + positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x38));
-                        write_number(static_cast<uint8_t>(positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x39));
-                        write_number(static_cast<uint16_t>(positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        oa->write_character(static_cast<CharType>(0x3A));
-                        write_number(static_cast<uint32_t>(positive_number));
-                    }
-                    else
-                    {
-                        oa->write_character(static_cast<CharType>(0x3B));
-                        write_number(static_cast<uint64_t>(positive_number));
-                    }
+                    oa->write_character(to_char_type('S'));
                 }
+                write_number_with_ubjson_prefix(j.m_value.string->size(), true);
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
                 break;
             }
 
-            case value_t::number_unsigned:
+            case value_t::array:
             {
-                if (j.m_value.number_unsigned <= 0x17)
+                if (add_prefix)
                 {
-                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                    oa->write_character(to_char_type('['));
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.array->empty())
                 {
-                    oa->write_character(static_cast<CharType>(0x18));
-                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin() + 1, j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+
+                if (use_count)
                 {
-                    oa->write_character(static_cast<CharType>(0x19));
-                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.array->size(), true);
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+
+                for (const auto& el : *j.m_value.array)
                 {
-                    oa->write_character(static_cast<CharType>(0x1A));
-                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                    write_ubjson(el, use_count, use_type, prefix_required);
                 }
-                else
+
+                if (not use_count)
                 {
-                    oa->write_character(static_cast<CharType>(0x1B));
-                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
+                    oa->write_character(to_char_type(']'));
                 }
-                break;
-            }
 
-            case value_t::number_float: // Double-Precision Float
-            {
-                oa->write_character(static_cast<CharType>(0xFB));
-                write_number(j.m_value.number_float);
                 break;
             }
 
-            case value_t::string:
+            case value_t::object:
             {
-                // step 1: write control byte and the string length
-                const auto N = j.m_value.string->size();
-                if (N <= 0x17)
+                if (add_prefix)
                 {
-                    write_number(static_cast<uint8_t>(0x60 + N));
+                    oa->write_character(to_char_type('{'));
                 }
-                else if (N <= 0xFF)
+
+                bool prefix_required = true;
+                if (use_type and not j.m_value.object->empty())
                 {
-                    oa->write_character(static_cast<CharType>(0x78));
-                    write_number(static_cast<uint8_t>(N));
+                    assert(use_count);
+                    const CharType first_prefix = ubjson_prefix(j.front());
+                    const bool same_prefix = std::all_of(j.begin(), j.end(),
+                                                         [this, first_prefix](const BasicJsonType & v)
+                    {
+                        return ubjson_prefix(v) == first_prefix;
+                    });
+
+                    if (same_prefix)
+                    {
+                        prefix_required = false;
+                        oa->write_character(to_char_type('$'));
+                        oa->write_character(first_prefix);
+                    }
                 }
-                else if (N <= 0xFFFF)
+
+                if (use_count)
                 {
-                    oa->write_character(static_cast<CharType>(0x79));
-                    write_number(static_cast<uint16_t>(N));
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.object->size(), true);
                 }
-                else if (N <= 0xFFFFFFFF)
+
+                for (const auto& el : *j.m_value.object)
                 {
-                    oa->write_character(static_cast<CharType>(0x7A));
-                    write_number(static_cast<uint32_t>(N));
+                    write_number_with_ubjson_prefix(el.first.size(), true);
+                    oa->write_characters(
+                        reinterpret_cast<const CharType*>(el.first.c_str()),
+                        el.first.size());
+                    write_ubjson(el.second, use_count, use_type, prefix_required);
                 }
-                // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
+
+                if (not use_count)
                 {
-                    oa->write_character(static_cast<CharType>(0x7B));
-                    write_number(static_cast<uint64_t>(N));
+                    oa->write_character(to_char_type('}'));
                 }
+
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+  private:
+    //////////
+    // BSON //
+    //////////
+
+    /*!
+    @return The size of a BSON document entry header, including the id marker
+            and the entry name size (and its null-terminator).
+    */
+    static std::size_t calc_bson_entry_header_size(const string_t& name)
+    {
+        const auto it = name.find(static_cast<typename string_t::value_type>(0));
+        if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos))
+        {
+            JSON_THROW(out_of_range::create(409,
+                                            "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")"));
+        }
+
+        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
+    }
+
+    /*!
+    @brief Writes the given @a element_type and @a name to the output adapter
+    */
+    void write_bson_entry_header(const string_t& name,
+                                 const std::uint8_t element_type)
+    {
+        oa->write_character(to_char_type(element_type)); // boolean
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(name.c_str()),
+            name.size() + 1u);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and boolean value @a value
+    */
+    void write_bson_boolean(const string_t& name,
+                            const bool value)
+    {
+        write_bson_entry_header(name, 0x08);
+        oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and double value @a value
+    */
+    void write_bson_double(const string_t& name,
+                           const double value)
+    {
+        write_bson_entry_header(name, 0x01);
+        write_number<double, true>(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded string in @a value
+    */
+    static std::size_t calc_bson_string_size(const string_t& value)
+    {
+        return sizeof(std::int32_t) + value.size() + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and string value @a value
+    */
+    void write_bson_string(const string_t& name,
+                           const string_t& value)
+    {
+        write_bson_entry_header(name, 0x02);
+
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul));
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(value.c_str()),
+            value.size() + 1);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and null value
+    */
+    void write_bson_null(const string_t& name)
+    {
+        write_bson_entry_header(name, 0x0A);
+    }
+
+    /*!
+    @return The size of the BSON-encoded integer @a value
+    */
+    static std::size_t calc_bson_integer_size(const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            return sizeof(std::int32_t);
+        }
+        else
+        {
+            return sizeof(std::int64_t);
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and integer @a value
+    */
+    void write_bson_integer(const string_t& name,
+                            const std::int64_t value)
+    {
+        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
+        {
+            write_bson_entry_header(name, 0x10); // int32
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else
+        {
+            write_bson_entry_header(name, 0x12); // int64
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+    }
+
+    /*!
+    @return The size of the BSON-encoded unsigned integer in @a j
+    */
+    static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept
+    {
+        return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+               ? sizeof(std::int32_t)
+               : sizeof(std::int64_t);
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and unsigned @a value
+    */
+    void write_bson_unsigned(const string_t& name,
+                             const std::uint64_t value)
+    {
+        if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x10 /* int32 */);
+            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
+        }
+        else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+        {
+            write_bson_entry_header(name, 0x12 /* int64 */);
+            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64"));
+        }
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and object @a value
+    */
+    void write_bson_object_entry(const string_t& name,
+                                 const typename BasicJsonType::object_t& value)
+    {
+        write_bson_entry_header(name, 0x03); // object
+        write_bson_object(value);
+    }
+
+    /*!
+    @return The size of the BSON-encoded array @a value
+    */
+    static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)
+    {
+        std::size_t embedded_document_size = 0ul;
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el);
+        }
+
+        return sizeof(std::int32_t) + embedded_document_size + 1ul;
+    }
+
+    /*!
+    @brief Writes a BSON element with key @a name and array @a value
+    */
+    void write_bson_array(const string_t& name,
+                          const typename BasicJsonType::array_t& value)
+    {
+        write_bson_entry_header(name, 0x04); // array
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value)));
+
+        std::size_t array_index = 0ul;
+
+        for (const auto& el : value)
+        {
+            write_bson_element(std::to_string(array_index++), el);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    /*!
+    @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
+    @return The calculated size for the BSON document entry for @a j with the given @a name.
+    */
+    static std::size_t calc_bson_element_size(const string_t& name,
+            const BasicJsonType& j)
+    {
+        const auto header_size = calc_bson_entry_header_size(name);
+        switch (j.type())
+        {
+            case value_t::object:
+                return header_size + calc_bson_object_size(*j.m_value.object);
+
+            case value_t::array:
+                return header_size + calc_bson_array_size(*j.m_value.array);
+
+            case value_t::boolean:
+                return header_size + 1ul;
+
+            case value_t::number_float:
+                return header_size + 8ul;
+
+            case value_t::number_integer:
+                return header_size + calc_bson_integer_size(j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);
+
+            case value_t::string:
+                return header_size + calc_bson_string_size(*j.m_value.string);
+
+            case value_t::null:
+                return header_size + 0ul;
+
+            // LCOV_EXCL_START
+            default:
+                assert(false);
+                return 0ul;
                 // LCOV_EXCL_STOP
+        };
+    }
 
-                // step 2: write the string
-                oa->write_characters(
-                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
-                    j.m_value.string->size());
-                break;
-            }
+    /*!
+    @brief Serializes the JSON value @a j to BSON and associates it with the
+           key @a name.
+    @param name The name to associate with the JSON entity @a j within the
+                current BSON document
+    @return The size of the BSON entry
+    */
+    void write_bson_element(const string_t& name,
+                            const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::object:
+                return write_bson_object_entry(name, *j.m_value.object);
 
             case value_t::array:
-            {
-                // step 1: write control byte and the array size
-                const auto N = j.m_value.array->size();
-                if (N <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(0x80 + N));
-                }
-                else if (N <= 0xFF)
-                {
-                    oa->write_character(static_cast<CharType>(0x98));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= 0xFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0x99));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= 0xFFFFFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0x9A));
-                    write_number(static_cast<uint32_t>(N));
-                }
-                // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0x9B));
-                    write_number(static_cast<uint64_t>(N));
-                }
-                // LCOV_EXCL_STOP
+                return write_bson_array(name, *j.m_value.array);
 
-                // step 2: write each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    write_cbor(el);
-                }
-                break;
-            }
+            case value_t::boolean:
+                return write_bson_boolean(name, j.m_value.boolean);
 
-            case value_t::object:
-            {
-                // step 1: write control byte and the object size
-                const auto N = j.m_value.object->size();
-                if (N <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(0xA0 + N));
-                }
-                else if (N <= 0xFF)
-                {
-                    oa->write_character(static_cast<CharType>(0xB8));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= 0xFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0xB9));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= 0xFFFFFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0xBA));
-                    write_number(static_cast<uint32_t>(N));
-                }
-                // LCOV_EXCL_START
-                else if (N <= 0xFFFFFFFFFFFFFFFF)
-                {
-                    oa->write_character(static_cast<CharType>(0xBB));
-                    write_number(static_cast<uint64_t>(N));
-                }
-                // LCOV_EXCL_STOP
+            case value_t::number_float:
+                return write_bson_double(name, j.m_value.number_float);
 
-                // step 2: write each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    write_cbor(el.first);
-                    write_cbor(el.second);
-                }
-                break;
-            }
+            case value_t::number_integer:
+                return write_bson_integer(name, j.m_value.number_integer);
+
+            case value_t::number_unsigned:
+                return write_bson_unsigned(name, j.m_value.number_unsigned);
+
+            case value_t::string:
+                return write_bson_string(name, *j.m_value.string);
 
+            case value_t::null:
+                return write_bson_null(name);
+
+            // LCOV_EXCL_START
             default:
-                break;
-        }
+                assert(false);
+                return;
+                // LCOV_EXCL_STOP
+        };
     }
 
     /*!
-    @brief[in] j  JSON value to serialize
+    @brief Calculates the size of the BSON serialization of the given
+           JSON-object @a j.
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
     */
-    void write_msgpack(const BasicJsonType& j)
+    static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)
     {
-        switch (j.type())
+        std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul,
+                                    [](size_t result, const typename BasicJsonType::object_t::value_type & el)
         {
-            case value_t::null: // nil
+            return result += calc_bson_element_size(el.first, el.second);
+        });
+
+        return sizeof(std::int32_t) + document_size + 1ul;
+    }
+
+    /*!
+    @param[in] j  JSON value to serialize
+    @pre       j.type() == value_t::object
+    */
+    void write_bson_object(const typename BasicJsonType::object_t& value)
+    {
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value)));
+
+        for (const auto& el : value)
+        {
+            write_bson_element(el.first, el.second);
+        }
+
+        oa->write_character(to_char_type(0x00));
+    }
+
+    //////////
+    // CBOR //
+    //////////
+
+    static constexpr CharType get_cbor_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xFA);  // Single-Precision Float
+    }
+
+    static constexpr CharType get_cbor_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xFB);  // Double-Precision Float
+    }
+
+    /////////////
+    // MsgPack //
+    /////////////
+
+    static constexpr CharType get_msgpack_float_prefix(float /*unused*/)
+    {
+        return to_char_type(0xCA);  // float 32
+    }
+
+    static constexpr CharType get_msgpack_float_prefix(double /*unused*/)
+    {
+        return to_char_type(0xCB);  // float 64
+    }
+
+    ////////////
+    // UBJSON //
+    ////////////
+
+    // UBJSON: write number (floating point)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (add_prefix)
+        {
+            oa->write_character(get_ubjson_float_prefix(n));
+        }
+        write_number(n);
+    }
+
+    // UBJSON: write number (unsigned integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_unsigned<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)()))
+        {
+            if (add_prefix)
             {
-                oa->write_character(static_cast<CharType>(0xC0));
-                break;
+                oa->write_character(to_char_type('i'));  // int8
             }
-
-            case value_t::boolean: // true and false
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= (std::numeric_limits<uint8_t>::max)())
+        {
+            if (add_prefix)
             {
-                oa->write_character(j.m_value.boolean
-                                    ? static_cast<CharType>(0xC3)
-                                    : static_cast<CharType>(0xC2));
-                break;
+                oa->write_character(to_char_type('U'));  // uint8
+            }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
             }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+    }
 
-            case value_t::number_integer:
+    // UBJSON: write number (signed integer)
+    template<typename NumberType, typename std::enable_if<
+                 std::is_signed<NumberType>::value and
+                 not std::is_floating_point<NumberType>::value, int>::type = 0>
+    void write_number_with_ubjson_prefix(const NumberType n,
+                                         const bool add_prefix)
+    {
+        if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)())
+        {
+            if (add_prefix)
             {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // MessagePack does not differentiate between positive
-                    // signed integers and unsigned integers. Therefore, we used
-                    // the code from the value_t::number_unsigned case here.
-                    if (j.m_value.number_unsigned < 128)
-                    {
-                        // positive fixnum
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        // uint 8
-                        oa->write_character(static_cast<CharType>(0xCC));
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        // uint 16
-                        oa->write_character(static_cast<CharType>(0xCD));
-                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        // uint 32
-                        oa->write_character(static_cast<CharType>(0xCE));
-                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
-                    {
-                        // uint 64
-                        oa->write_character(static_cast<CharType>(0xCF));
-                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                    }
-                }
-                else
-                {
-                    if (j.m_value.number_integer >= -32)
-                    {
-                        // negative fixnum
-                        write_number(static_cast<int8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
-                    {
-                        // int 8
-                        oa->write_character(static_cast<CharType>(0xD0));
-                        write_number(static_cast<int8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
-                    {
-                        // int 16
-                        oa->write_character(static_cast<CharType>(0xD1));
-                        write_number(static_cast<int16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
-                    {
-                        // int 32
-                        oa->write_character(static_cast<CharType>(0xD2));
-                        write_number(static_cast<int32_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
-                    {
-                        // int 64
-                        oa->write_character(static_cast<CharType>(0xD3));
-                        write_number(static_cast<int64_t>(j.m_value.number_integer));
-                    }
-                }
-                break;
+                oa->write_character(to_char_type('i'));  // int8
+            }
+            write_number(static_cast<int8_t>(n));
+        }
+        else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)()))
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('U'));  // uint8
             }
+            write_number(static_cast<uint8_t>(n));
+        }
+        else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('I'));  // int16
+            }
+            write_number(static_cast<int16_t>(n));
+        }
+        else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('l'));  // int32
+            }
+            write_number(static_cast<int32_t>(n));
+        }
+        else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)())
+        {
+            if (add_prefix)
+            {
+                oa->write_character(to_char_type('L'));  // int64
+            }
+            write_number(static_cast<int64_t>(n));
+        }
+        // LCOV_EXCL_START
+        else
+        {
+            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
+        }
+        // LCOV_EXCL_STOP
+    }
 
-            case value_t::number_unsigned:
+    /*!
+    @brief determine the type prefix of container values
+
+    @note This function does not need to be 100% accurate when it comes to
+          integer limits. In case a number exceeds the limits of int64_t,
+          this will be detected by a later call to function
+          write_number_with_ubjson_prefix. Therefore, we return 'L' for any
+          value that does not fit the previous limits.
+    */
+    CharType ubjson_prefix(const BasicJsonType& j) const noexcept
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+                return 'Z';
+
+            case value_t::boolean:
+                return j.m_value.boolean ? 'T' : 'F';
+
+            case value_t::number_integer:
             {
-                if (j.m_value.number_unsigned < 128)
-                {
-                    // positive fixnum
-                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
                 {
-                    // uint 8
-                    oa->write_character(static_cast<CharType>(0xCC));
-                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    return 'i';
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    // uint 16
-                    oa->write_character(static_cast<CharType>(0xCD));
-                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    return 'U';
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
                 {
-                    // uint 32
-                    oa->write_character(static_cast<CharType>(0xCE));
-                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    return 'I';
                 }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
                 {
-                    // uint 64
-                    oa->write_character(static_cast<CharType>(0xCF));
-                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    return 'l';
                 }
-                break;
-            }
-
-            case value_t::number_float: // float 64
-            {
-                oa->write_character(static_cast<CharType>(0xCB));
-                write_number(j.m_value.number_float);
-                break;
+                // no check and assume int64_t (see note above)
+                return 'L';
             }
 
-            case value_t::string:
+            case value_t::number_unsigned:
             {
-                // step 1: write control byte and the string length
-                const auto N = j.m_value.string->size();
-                if (N <= 31)
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)())
                 {
-                    // fixstr
-                    write_number(static_cast<uint8_t>(0xA0 | N));
+                    return 'i';
                 }
-                else if (N <= 255)
+                if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
                 {
-                    // str 8
-                    oa->write_character(static_cast<CharType>(0xD9));
-                    write_number(static_cast<uint8_t>(N));
+                    return 'U';
                 }
-                else if (N <= 65535)
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)())
                 {
-                    // str 16
-                    oa->write_character(static_cast<CharType>(0xDA));
-                    write_number(static_cast<uint16_t>(N));
+                    return 'I';
                 }
-                else if (N <= 4294967295)
+                if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)())
                 {
-                    // str 32
-                    oa->write_character(static_cast<CharType>(0xDB));
-                    write_number(static_cast<uint32_t>(N));
+                    return 'l';
                 }
+                // no check and assume int64_t (see note above)
+                return 'L';
+            }
+
+            case value_t::number_float:
+                return get_ubjson_float_prefix(j.m_value.number_float);
+
+            case value_t::string:
+                return 'S';
+
+            case value_t::array:
+                return '[';
+
+            case value_t::object:
+                return '{';
+
+            default:  // discarded values
+                return 'N';
+        }
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(float /*unused*/)
+    {
+        return 'd';  // float 32
+    }
+
+    static constexpr CharType get_ubjson_float_prefix(double /*unused*/)
+    {
+        return 'D';  // float 64
+    }
+
+    ///////////////////////
+    // Utility functions //
+    ///////////////////////
+
+    /*
+    @brief write a number to output input
+    @param[in] n number of type @a NumberType
+    @tparam NumberType the type of the number
+    @tparam OutputIsLittleEndian Set to true if output data is
+                                 required to be little endian
+
+    @note This function needs to respect the system's endianess, because bytes
+          in CBOR, MessagePack, and UBJSON are stored in network order (big
+          endian) and therefore need reordering on little endian systems.
+    */
+    template<typename NumberType, bool OutputIsLittleEndian = false>
+    void write_number(const NumberType n)
+    {
+        // step 1: write number to array of length NumberType
+        std::array<CharType, sizeof(NumberType)> vec;
+        std::memcpy(vec.data(), &n, sizeof(NumberType));
+
+        // step 2: write array to output (with possible reordering)
+        if (is_little_endian and not OutputIsLittleEndian)
+        {
+            // reverse byte order prior to conversion if necessary
+            std::reverse(vec.begin(), vec.end());
+        }
+
+        oa->write_characters(vec.data(), sizeof(NumberType));
+    }
+
+  public:
+    // The following to_char_type functions are implement the conversion
+    // between uint8_t and CharType. In case CharType is not unsigned,
+    // such a conversion is required to allow values greater than 128.
+    // See <https://github.com/nlohmann/json/issues/1286> for a discussion.
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr >
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return *reinterpret_cast<char*>(&x);
+    }
+
+    template < typename C = CharType,
+               enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr >
+    static CharType to_char_type(std::uint8_t x) noexcept
+    {
+        static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t");
+        static_assert(std::is_pod<CharType>::value, "CharType must be POD");
+        CharType result;
+        std::memcpy(&result, &x, sizeof(x));
+        return result;
+    }
+
+    template<typename C = CharType,
+             enable_if_t<std::is_unsigned<C>::value>* = nullptr>
+    static constexpr CharType to_char_type(std::uint8_t x) noexcept
+    {
+        return x;
+    }
+
+    template < typename InputCharType, typename C = CharType,
+               enable_if_t <
+                   std::is_signed<C>::value and
+                   std::is_signed<char>::value and
+                   std::is_same<char, typename std::remove_cv<InputCharType>::type>::value
+                   > * = nullptr >
+    static constexpr CharType to_char_type(InputCharType x) noexcept
+    {
+        return x;
+    }
+
+  private:
+    /// whether we can assume little endianess
+    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
+
+    /// the output
+    output_adapter_t<CharType> oa = nullptr;
+};
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/output/serializer.hpp>
+
+
+#include <algorithm> // reverse, remove, fill, find, none_of
+#include <array> // array
+#include <cassert> // assert
+#include <ciso646> // and, or
+#include <clocale> // localeconv, lconv
+#include <cmath> // labs, isfinite, isnan, signbit
+#include <cstddef> // size_t, ptrdiff_t
+#include <cstdint> // uint8_t
+#include <cstdio> // snprintf
+#include <limits> // numeric_limits
+#include <string> // string
+#include <type_traits> // is_same
+
+// #include <nlohmann/detail/exceptions.hpp>
+
+// #include <nlohmann/detail/conversions/to_chars.hpp>
+
+
+#include <cassert> // assert
+#include <ciso646> // or, and, not
+#include <cmath>   // signbit, isfinite
+#include <cstdint> // intN_t, uintN_t
+#include <cstring> // memcpy, memmove
+
+namespace nlohmann
+{
+namespace detail
+{
+
+/*!
+@brief implements the Grisu2 algorithm for binary to decimal floating-point
+conversion.
+
+This implementation is a slightly modified version of the reference
+implementation which may be obtained from
+http://florian.loitsch.com/publications (bench.tar.gz).
+
+The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.
+
+For a detailed description of the algorithm see:
+
+[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with
+    Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming
+    Language Design and Implementation, PLDI 2010
+[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately",
+    Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language
+    Design and Implementation, PLDI 1996
+*/
+namespace dtoa_impl
+{
+
+template <typename Target, typename Source>
+Target reinterpret_bits(const Source source)
+{
+    static_assert(sizeof(Target) == sizeof(Source), "size mismatch");
+
+    Target target;
+    std::memcpy(&target, &source, sizeof(Source));
+    return target;
+}
+
+struct diyfp // f * 2^e
+{
+    static constexpr int kPrecision = 64; // = q
+
+    uint64_t f = 0;
+    int e = 0;
+
+    constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {}
+
+    /*!
+    @brief returns x - y
+    @pre x.e == y.e and x.f >= y.f
+    */
+    static diyfp sub(const diyfp& x, const diyfp& y) noexcept
+    {
+        assert(x.e == y.e);
+        assert(x.f >= y.f);
+
+        return {x.f - y.f, x.e};
+    }
+
+    /*!
+    @brief returns x * y
+    @note The result is rounded. (Only the upper q bits are returned.)
+    */
+    static diyfp mul(const diyfp& x, const diyfp& y) noexcept
+    {
+        static_assert(kPrecision == 64, "internal error");
+
+        // Computes:
+        //  f = round((x.f * y.f) / 2^q)
+        //  e = x.e + y.e + q
+
+        // Emulate the 64-bit * 64-bit multiplication:
+        //
+        // p = u * v
+        //   = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)
+        //   = (u_lo v_lo         ) + 2^32 ((u_lo v_hi         ) + (u_hi v_lo         )) + 2^64 (u_hi v_hi         )
+        //   = (p0                ) + 2^32 ((p1                ) + (p2                )) + 2^64 (p3                )
+        //   = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3                )
+        //   = (p0_lo             ) + 2^32 (p0_hi + p1_lo + p2_lo                      ) + 2^64 (p1_hi + p2_hi + p3)
+        //   = (p0_lo             ) + 2^32 (Q                                          ) + 2^64 (H                 )
+        //   = (p0_lo             ) + 2^32 (Q_lo + 2^32 Q_hi                           ) + 2^64 (H                 )
+        //
+        // (Since Q might be larger than 2^32 - 1)
+        //
+        //   = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)
+        //
+        // (Q_hi + H does not overflow a 64-bit int)
+        //
+        //   = p_lo + 2^64 p_hi
+
+        const uint64_t u_lo = x.f & 0xFFFFFFFF;
+        const uint64_t u_hi = x.f >> 32;
+        const uint64_t v_lo = y.f & 0xFFFFFFFF;
+        const uint64_t v_hi = y.f >> 32;
+
+        const uint64_t p0 = u_lo * v_lo;
+        const uint64_t p1 = u_lo * v_hi;
+        const uint64_t p2 = u_hi * v_lo;
+        const uint64_t p3 = u_hi * v_hi;
+
+        const uint64_t p0_hi = p0 >> 32;
+        const uint64_t p1_lo = p1 & 0xFFFFFFFF;
+        const uint64_t p1_hi = p1 >> 32;
+        const uint64_t p2_lo = p2 & 0xFFFFFFFF;
+        const uint64_t p2_hi = p2 >> 32;
+
+        uint64_t Q = p0_hi + p1_lo + p2_lo;
+
+        // The full product might now be computed as
+        //
+        // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)
+        // p_lo = p0_lo + (Q << 32)
+        //
+        // But in this particular case here, the full p_lo is not required.
+        // Effectively we only need to add the highest bit in p_lo to p_hi (and
+        // Q_hi + 1 does not overflow).
+
+        Q += uint64_t{1} << (64 - 32 - 1); // round, ties up
+
+        const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32);
+
+        return {h, x.e + y.e + 64};
+    }
+
+    /*!
+    @brief normalize x such that the significand is >= 2^(q-1)
+    @pre x.f != 0
+    */
+    static diyfp normalize(diyfp x) noexcept
+    {
+        assert(x.f != 0);
+
+        while ((x.f >> 63) == 0)
+        {
+            x.f <<= 1;
+            x.e--;
+        }
+
+        return x;
+    }
+
+    /*!
+    @brief normalize x such that the result has the exponent E
+    @pre e >= x.e and the upper e - x.e bits of x.f must be zero.
+    */
+    static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept
+    {
+        const int delta = x.e - target_exponent;
+
+        assert(delta >= 0);
+        assert(((x.f << delta) >> delta) == x.f);
+
+        return {x.f << delta, target_exponent};
+    }
+};
+
+struct boundaries
+{
+    diyfp w;
+    diyfp minus;
+    diyfp plus;
+};
+
+/*!
+Compute the (normalized) diyfp representing the input number 'value' and its
+boundaries.
+
+@pre value must be finite and positive
+*/
+template <typename FloatType>
+boundaries compute_boundaries(FloatType value)
+{
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // Convert the IEEE representation into a diyfp.
+    //
+    // If v is denormal:
+    //      value = 0.F * 2^(1 - bias) = (          F) * 2^(1 - bias - (p-1))
+    // If v is normalized:
+    //      value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))
+
+    static_assert(std::numeric_limits<FloatType>::is_iec559,
+                  "internal error: dtoa_short requires an IEEE-754 floating-point implementation");
+
+    constexpr int      kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)
+    constexpr int      kBias      = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);
+    constexpr int      kMinExp    = 1 - kBias;
+    constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1)
+
+    using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type;
+
+    const uint64_t bits = reinterpret_bits<bits_type>(value);
+    const uint64_t E = bits >> (kPrecision - 1);
+    const uint64_t F = bits & (kHiddenBit - 1);
+
+    const bool is_denormal = (E == 0);
+    const diyfp v = is_denormal
+                    ? diyfp(F, kMinExp)
+                    : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);
+
+    // Compute the boundaries m- and m+ of the floating-point value
+    // v = f * 2^e.
+    //
+    // Determine v- and v+, the floating-point predecessor and successor if v,
+    // respectively.
+    //
+    //      v- = v - 2^e        if f != 2^(p-1) or e == e_min                (A)
+    //         = v - 2^(e-1)    if f == 2^(p-1) and e > e_min                (B)
+    //
+    //      v+ = v + 2^e
+    //
+    // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_
+    // between m- and m+ round to v, regardless of how the input rounding
+    // algorithm breaks ties.
+    //
+    //      ---+-------------+-------------+-------------+-------------+---  (A)
+    //         v-            m-            v             m+            v+
+    //
+    //      -----------------+------+------+-------------+-------------+---  (B)
+    //                       v-     m-     v             m+            v+
+
+    const bool lower_boundary_is_closer = (F == 0 and E > 1);
+    const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);
+    const diyfp m_minus = lower_boundary_is_closer
+                          ? diyfp(4 * v.f - 1, v.e - 2)  // (B)
+                          : diyfp(2 * v.f - 1, v.e - 1); // (A)
+
+    // Determine the normalized w+ = m+.
+    const diyfp w_plus = diyfp::normalize(m_plus);
+
+    // Determine w- = m- such that e_(w-) = e_(w+).
+    const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);
+
+    return {diyfp::normalize(v), w_minus, w_plus};
+}
+
+// Given normalized diyfp w, Grisu needs to find a (normalized) cached
+// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies
+// within a certain range [alpha, gamma] (Definition 3.2 from [1])
+//
+//      alpha <= e = e_c + e_w + q <= gamma
+//
+// or
+//
+//      f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q
+//                          <= f_c * f_w * 2^gamma
+//
+// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies
+//
+//      2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma
+//
+// or
+//
+//      2^(q - 2 + alpha) <= c * w < 2^(q + gamma)
+//
+// The choice of (alpha,gamma) determines the size of the table and the form of
+// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well
+// in practice:
+//
+// The idea is to cut the number c * w = f * 2^e into two parts, which can be
+// processed independently: An integral part p1, and a fractional part p2:
+//
+//      f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e
+//              = (f div 2^-e) + (f mod 2^-e) * 2^e
+//              = p1 + p2 * 2^e
+//
+// The conversion of p1 into decimal form requires a series of divisions and
+// modulos by (a power of) 10. These operations are faster for 32-bit than for
+// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be
+// achieved by choosing
+//
+//      -e >= 32   or   e <= -32 := gamma
+//
+// In order to convert the fractional part
+//
+//      p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...
+//
+// into decimal form, the fraction is repeatedly multiplied by 10 and the digits
+// d[-i] are extracted in order:
+//
+//      (10 * p2) div 2^-e = d[-1]
+//      (10 * p2) mod 2^-e = d[-2] / 10^1 + ...
+//
+// The multiplication by 10 must not overflow. It is sufficient to choose
+//
+//      10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.
+//
+// Since p2 = f mod 2^-e < 2^-e,
+//
+//      -e <= 60   or   e >= -60 := alpha
+
+constexpr int kAlpha = -60;
+constexpr int kGamma = -32;
+
+struct cached_power // c = f * 2^e ~= 10^k
+{
+    uint64_t f;
+    int e;
+    int k;
+};
+
+/*!
+For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached
+power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c
+satisfies (Definition 3.2 from [1])
+
+     alpha <= e_c + e + q <= gamma.
+*/
+inline cached_power get_cached_power_for_binary_exponent(int e)
+{
+    // Now
+    //
+    //      alpha <= e_c + e + q <= gamma                                    (1)
+    //      ==> f_c * 2^alpha <= c * 2^e * 2^q
+    //
+    // and since the c's are normalized, 2^(q-1) <= f_c,
+    //
+    //      ==> 2^(q - 1 + alpha) <= c * 2^(e + q)
+    //      ==> 2^(alpha - e - 1) <= c
+    //
+    // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as
+    //
+    //      k = ceil( log_10( 2^(alpha - e - 1) ) )
+    //        = ceil( (alpha - e - 1) * log_10(2) )
+    //
+    // From the paper:
+    // "In theory the result of the procedure could be wrong since c is rounded,
+    //  and the computation itself is approximated [...]. In practice, however,
+    //  this simple function is sufficient."
+    //
+    // For IEEE double precision floating-point numbers converted into
+    // normalized diyfp's w = f * 2^e, with q = 64,
+    //
+    //      e >= -1022      (min IEEE exponent)
+    //           -52        (p - 1)
+    //           -52        (p - 1, possibly normalize denormal IEEE numbers)
+    //           -11        (normalize the diyfp)
+    //         = -1137
+    //
+    // and
+    //
+    //      e <= +1023      (max IEEE exponent)
+    //           -52        (p - 1)
+    //           -11        (normalize the diyfp)
+    //         = 960
+    //
+    // This binary exponent range [-1137,960] results in a decimal exponent
+    // range [-307,324]. One does not need to store a cached power for each
+    // k in this range. For each such k it suffices to find a cached power
+    // such that the exponent of the product lies in [alpha,gamma].
+    // This implies that the difference of the decimal exponents of adjacent
+    // table entries must be less than or equal to
+    //
+    //      floor( (gamma - alpha) * log_10(2) ) = 8.
+    //
+    // (A smaller distance gamma-alpha would require a larger table.)
+
+    // NB:
+    // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.
+
+    constexpr int kCachedPowersSize = 79;
+    constexpr int kCachedPowersMinDecExp = -300;
+    constexpr int kCachedPowersDecStep = 8;
+
+    static constexpr cached_power kCachedPowers[] =
+    {
+        { 0xAB70FE17C79AC6CA, -1060, -300 },
+        { 0xFF77B1FCBEBCDC4F, -1034, -292 },
+        { 0xBE5691EF416BD60C, -1007, -284 },
+        { 0x8DD01FAD907FFC3C,  -980, -276 },
+        { 0xD3515C2831559A83,  -954, -268 },
+        { 0x9D71AC8FADA6C9B5,  -927, -260 },
+        { 0xEA9C227723EE8BCB,  -901, -252 },
+        { 0xAECC49914078536D,  -874, -244 },
+        { 0x823C12795DB6CE57,  -847, -236 },
+        { 0xC21094364DFB5637,  -821, -228 },
+        { 0x9096EA6F3848984F,  -794, -220 },
+        { 0xD77485CB25823AC7,  -768, -212 },
+        { 0xA086CFCD97BF97F4,  -741, -204 },
+        { 0xEF340A98172AACE5,  -715, -196 },
+        { 0xB23867FB2A35B28E,  -688, -188 },
+        { 0x84C8D4DFD2C63F3B,  -661, -180 },
+        { 0xC5DD44271AD3CDBA,  -635, -172 },
+        { 0x936B9FCEBB25C996,  -608, -164 },
+        { 0xDBAC6C247D62A584,  -582, -156 },
+        { 0xA3AB66580D5FDAF6,  -555, -148 },
+        { 0xF3E2F893DEC3F126,  -529, -140 },
+        { 0xB5B5ADA8AAFF80B8,  -502, -132 },
+        { 0x87625F056C7C4A8B,  -475, -124 },
+        { 0xC9BCFF6034C13053,  -449, -116 },
+        { 0x964E858C91BA2655,  -422, -108 },
+        { 0xDFF9772470297EBD,  -396, -100 },
+        { 0xA6DFBD9FB8E5B88F,  -369,  -92 },
+        { 0xF8A95FCF88747D94,  -343,  -84 },
+        { 0xB94470938FA89BCF,  -316,  -76 },
+        { 0x8A08F0F8BF0F156B,  -289,  -68 },
+        { 0xCDB02555653131B6,  -263,  -60 },
+        { 0x993FE2C6D07B7FAC,  -236,  -52 },
+        { 0xE45C10C42A2B3B06,  -210,  -44 },
+        { 0xAA242499697392D3,  -183,  -36 },
+        { 0xFD87B5F28300CA0E,  -157,  -28 },
+        { 0xBCE5086492111AEB,  -130,  -20 },
+        { 0x8CBCCC096F5088CC,  -103,  -12 },
+        { 0xD1B71758E219652C,   -77,   -4 },
+        { 0x9C40000000000000,   -50,    4 },
+        { 0xE8D4A51000000000,   -24,   12 },
+        { 0xAD78EBC5AC620000,     3,   20 },
+        { 0x813F3978F8940984,    30,   28 },
+        { 0xC097CE7BC90715B3,    56,   36 },
+        { 0x8F7E32CE7BEA5C70,    83,   44 },
+        { 0xD5D238A4ABE98068,   109,   52 },
+        { 0x9F4F2726179A2245,   136,   60 },
+        { 0xED63A231D4C4FB27,   162,   68 },
+        { 0xB0DE65388CC8ADA8,   189,   76 },
+        { 0x83C7088E1AAB65DB,   216,   84 },
+        { 0xC45D1DF942711D9A,   242,   92 },
+        { 0x924D692CA61BE758,   269,  100 },
+        { 0xDA01EE641A708DEA,   295,  108 },
+        { 0xA26DA3999AEF774A,   322,  116 },
+        { 0xF209787BB47D6B85,   348,  124 },
+        { 0xB454E4A179DD1877,   375,  132 },
+        { 0x865B86925B9BC5C2,   402,  140 },
+        { 0xC83553C5C8965D3D,   428,  148 },
+        { 0x952AB45CFA97A0B3,   455,  156 },
+        { 0xDE469FBD99A05FE3,   481,  164 },
+        { 0xA59BC234DB398C25,   508,  172 },
+        { 0xF6C69A72A3989F5C,   534,  180 },
+        { 0xB7DCBF5354E9BECE,   561,  188 },
+        { 0x88FCF317F22241E2,   588,  196 },
+        { 0xCC20CE9BD35C78A5,   614,  204 },
+        { 0x98165AF37B2153DF,   641,  212 },
+        { 0xE2A0B5DC971F303A,   667,  220 },
+        { 0xA8D9D1535CE3B396,   694,  228 },
+        { 0xFB9B7CD9A4A7443C,   720,  236 },
+        { 0xBB764C4CA7A44410,   747,  244 },
+        { 0x8BAB8EEFB6409C1A,   774,  252 },
+        { 0xD01FEF10A657842C,   800,  260 },
+        { 0x9B10A4E5E9913129,   827,  268 },
+        { 0xE7109BFBA19C0C9D,   853,  276 },
+        { 0xAC2820D9623BF429,   880,  284 },
+        { 0x80444B5E7AA7CF85,   907,  292 },
+        { 0xBF21E44003ACDD2D,   933,  300 },
+        { 0x8E679C2F5E44FF8F,   960,  308 },
+        { 0xD433179D9C8CB841,   986,  316 },
+        { 0x9E19DB92B4E31BA9,  1013,  324 },
+    };
+
+    // This computation gives exactly the same results for k as
+    //      k = ceil((kAlpha - e - 1) * 0.30102999566398114)
+    // for |e| <= 1500, but doesn't require floating-point operations.
+    // NB: log_10(2) ~= 78913 / 2^18
+    assert(e >= -1500);
+    assert(e <=  1500);
+    const int f = kAlpha - e - 1;
+    const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);
+
+    const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;
+    assert(index >= 0);
+    assert(index < kCachedPowersSize);
+    static_cast<void>(kCachedPowersSize); // Fix warning.
+
+    const cached_power cached = kCachedPowers[index];
+    assert(kAlpha <= cached.e + e + 64);
+    assert(kGamma >= cached.e + e + 64);
+
+    return cached;
+}
+
+/*!
+For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.
+For n == 0, returns 1 and sets pow10 := 1.
+*/
+inline int find_largest_pow10(const uint32_t n, uint32_t& pow10)
+{
+    // LCOV_EXCL_START
+    if (n >= 1000000000)
+    {
+        pow10 = 1000000000;
+        return 10;
+    }
+    // LCOV_EXCL_STOP
+    else if (n >= 100000000)
+    {
+        pow10 = 100000000;
+        return  9;
+    }
+    else if (n >= 10000000)
+    {
+        pow10 = 10000000;
+        return  8;
+    }
+    else if (n >= 1000000)
+    {
+        pow10 = 1000000;
+        return  7;
+    }
+    else if (n >= 100000)
+    {
+        pow10 = 100000;
+        return  6;
+    }
+    else if (n >= 10000)
+    {
+        pow10 = 10000;
+        return  5;
+    }
+    else if (n >= 1000)
+    {
+        pow10 = 1000;
+        return  4;
+    }
+    else if (n >= 100)
+    {
+        pow10 = 100;
+        return  3;
+    }
+    else if (n >= 10)
+    {
+        pow10 = 10;
+        return  2;
+    }
+    else
+    {
+        pow10 = 1;
+        return 1;
+    }
+}
+
+inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta,
+                         uint64_t rest, uint64_t ten_k)
+{
+    assert(len >= 1);
+    assert(dist <= delta);
+    assert(rest <= delta);
+    assert(ten_k > 0);
+
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    //                                  ten_k
+    //                                <------>
+    //                                       <---- rest ---->
+    // --------------[------------------+----+--------------]--------------
+    //                                  w    V
+    //                                       = buf * 10^k
+    //
+    // ten_k represents a unit-in-the-last-place in the decimal representation
+    // stored in buf.
+    // Decrement buf by ten_k while this takes buf closer to w.
+
+    // The tests are written in this order to avoid overflow in unsigned
+    // integer arithmetic.
+
+    while (rest < dist
+            and delta - rest >= ten_k
+            and (rest + ten_k < dist or dist - rest > rest + ten_k - dist))
+    {
+        assert(buf[len - 1] != '0');
+        buf[len - 1]--;
+        rest += ten_k;
+    }
+}
+
+/*!
+Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.
+M- and M+ must be normalized and share the same exponent -60 <= e <= -32.
+*/
+inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,
+                             diyfp M_minus, diyfp w, diyfp M_plus)
+{
+    static_assert(kAlpha >= -60, "internal error");
+    static_assert(kGamma <= -32, "internal error");
+
+    // Generates the digits (and the exponent) of a decimal floating-point
+    // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's
+    // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.
+    //
+    //               <--------------------------- delta ---->
+    //                                  <---- dist --------->
+    // --------------[------------------+-------------------]--------------
+    //               M-                 w                   M+
+    //
+    // Grisu2 generates the digits of M+ from left to right and stops as soon as
+    // V is in [M-,M+].
+
+    assert(M_plus.e >= kAlpha);
+    assert(M_plus.e <= kGamma);
+
+    uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)
+    uint64_t dist  = diyfp::sub(M_plus, w      ).f; // (significand of (M+ - w ), implicit exponent is e)
+
+    // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):
+    //
+    //      M+ = f * 2^e
+    //         = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e
+    //         = ((p1        ) * 2^-e + (p2        )) * 2^e
+    //         = p1 + p2 * 2^e
+
+    const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e);
+
+    auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)
+    uint64_t p2 = M_plus.f & (one.f - 1);                    // p2 = f mod 2^-e
+
+    // 1)
+    //
+    // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]
+
+    assert(p1 > 0);
+
+    uint32_t pow10;
+    const int k = find_largest_pow10(p1, pow10);
+
+    //      10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)
+    //
+    //      p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))
+    //         = (d[k-1]         ) * 10^(k-1) + (p1 mod 10^(k-1))
+    //
+    //      M+ = p1                                             + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1))          + p2 * 2^e
+    //         = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e
+    //         = d[k-1] * 10^(k-1) + (                         rest) * 2^e
+    //
+    // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)
+    //
+    //      p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]
+    //
+    // but stop as soon as
+    //
+    //      rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e
+
+    int n = k;
+    while (n > 0)
+    {
+        // Invariants:
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)    (buffer = 0 for n = k)
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        //
+        const uint32_t d = p1 / pow10;  // d = p1 div 10^(n-1)
+        const uint32_t r = p1 % pow10;  // r = p1 mod 10^(n-1)
+        //
+        //      M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e
+        //         = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(n-1) + (r + p2 * 2^e)
+        //
+        p1 = r;
+        n--;
+        //
+        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)
+        //      pow10 = 10^n
+        //
+
+        // Now check if enough digits have been generated.
+        // Compute
+        //
+        //      p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e
+        //
+        // Note:
+        // Since rest and delta share the same exponent e, it suffices to
+        // compare the significands.
+        const uint64_t rest = (uint64_t{p1} << -one.e) + p2;
+        if (rest <= delta)
+        {
+            // V = buffer * 10^n, with M- <= V <= M+.
+
+            decimal_exponent += n;
+
+            // We may now just stop. But instead look if the buffer could be
+            // decremented to bring V closer to w.
+            //
+            // pow10 = 10^n is now 1 ulp in the decimal representation V.
+            // The rounding procedure works with diyfp's with an implicit
+            // exponent of e.
+            //
+            //      10^n = (10^n * 2^-e) * 2^e = ulp * 2^e
+            //
+            const uint64_t ten_n = uint64_t{pow10} << -one.e;
+            grisu2_round(buffer, length, dist, delta, rest, ten_n);
+
+            return;
+        }
+
+        pow10 /= 10;
+        //
+        //      pow10 = 10^(n-1) <= p1 < 10^n
+        // Invariants restored.
+    }
+
+    // 2)
+    //
+    // The digits of the integral part have been generated:
+    //
+    //      M+ = d[k-1]...d[1]d[0] + p2 * 2^e
+    //         = buffer            + p2 * 2^e
+    //
+    // Now generate the digits of the fractional part p2 * 2^e.
+    //
+    // Note:
+    // No decimal point is generated: the exponent is adjusted instead.
+    //
+    // p2 actually represents the fraction
+    //
+    //      p2 * 2^e
+    //          = p2 / 2^-e
+    //          = d[-1] / 10^1 + d[-2] / 10^2 + ...
+    //
+    // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)
+    //
+    //      p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m
+    //                      + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)
+    //
+    // using
+    //
+    //      10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)
+    //                = (                   d) * 2^-e + (                   r)
+    //
+    // or
+    //      10^m * p2 * 2^e = d + r * 2^e
+    //
+    // i.e.
+    //
+    //      M+ = buffer + p2 * 2^e
+    //         = buffer + 10^-m * (d + r * 2^e)
+    //         = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e
+    //
+    // and stop as soon as 10^-m * r * 2^e <= delta * 2^e
+
+    assert(p2 > delta);
+
+    int m = 0;
+    for (;;)
+    {
+        // Invariant:
+        //      M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e
+        //         = buffer * 10^-m + 10^-m * (p2                                 ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (10 * p2)                   ) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e
+        //
+        assert(p2 <= UINT64_MAX / 10);
+        p2 *= 10;
+        const uint64_t d = p2 >> -one.e;     // d = (10 * p2) div 2^-e
+        const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e
+        //
+        //      M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e
+        //         = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))
+        //         = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        assert(d <= 9);
+        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
+        //
+        //      M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e
+        //
+        p2 = r;
+        m++;
+        //
+        //      M+ = buffer * 10^-m + 10^-m * p2 * 2^e
+        // Invariant restored.
+
+        // Check if enough digits have been generated.
+        //
+        //      10^-m * p2 * 2^e <= delta * 2^e
+        //              p2 * 2^e <= 10^m * delta * 2^e
+        //                    p2 <= 10^m * delta
+        delta *= 10;
+        dist  *= 10;
+        if (p2 <= delta)
+        {
+            break;
+        }
+    }
+
+    // V = buffer * 10^-m, with M- <= V <= M+.
+
+    decimal_exponent -= m;
+
+    // 1 ulp in the decimal representation is now 10^-m.
+    // Since delta and dist are now scaled by 10^m, we need to do the
+    // same with ulp in order to keep the units in sync.
+    //
+    //      10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e
+    //
+    const uint64_t ten_m = one.f;
+    grisu2_round(buffer, length, dist, delta, p2, ten_m);
+
+    // By construction this algorithm generates the shortest possible decimal
+    // number (Loitsch, Theorem 6.2) which rounds back to w.
+    // For an input number of precision p, at least
+    //
+    //      N = 1 + ceil(p * log_10(2))
+    //
+    // decimal digits are sufficient to identify all binary floating-point
+    // numbers (Matula, "In-and-Out conversions").
+    // This implies that the algorithm does not produce more than N decimal
+    // digits.
+    //
+    //      N = 17 for p = 53 (IEEE double precision)
+    //      N = 9  for p = 24 (IEEE single precision)
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+inline void grisu2(char* buf, int& len, int& decimal_exponent,
+                   diyfp m_minus, diyfp v, diyfp m_plus)
+{
+    assert(m_plus.e == m_minus.e);
+    assert(m_plus.e == v.e);
+
+    //  --------(-----------------------+-----------------------)--------    (A)
+    //          m-                      v                       m+
+    //
+    //  --------------------(-----------+-----------------------)--------    (B)
+    //                      m-          v                       m+
+    //
+    // First scale v (and m- and m+) such that the exponent is in the range
+    // [alpha, gamma].
+
+    const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);
+
+    const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k
+
+    // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]
+    const diyfp w       = diyfp::mul(v,       c_minus_k);
+    const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);
+    const diyfp w_plus  = diyfp::mul(m_plus,  c_minus_k);
+
+    //  ----(---+---)---------------(---+---)---------------(---+---)----
+    //          w-                      w                       w+
+    //          = c*m-                  = c*v                   = c*m+
+    //
+    // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and
+    // w+ are now off by a small amount.
+    // In fact:
+    //
+    //      w - v * 10^k < 1 ulp
+    //
+    // To account for this inaccuracy, add resp. subtract 1 ulp.
+    //
+    //  --------+---[---------------(---+---)---------------]---+--------
+    //          w-  M-                  w                   M+  w+
+    //
+    // Now any number in [M-, M+] (bounds included) will round to w when input,
+    // regardless of how the input rounding algorithm breaks ties.
+    //
+    // And digit_gen generates the shortest possible such number in [M-, M+].
+    // Note that this does not mean that Grisu2 always generates the shortest
+    // possible number in the interval (m-, m+).
+    const diyfp M_minus(w_minus.f + 1, w_minus.e);
+    const diyfp M_plus (w_plus.f  - 1, w_plus.e );
+
+    decimal_exponent = -cached.k; // = -(-k) = k
+
+    grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);
+}
+
+/*!
+v = buf * 10^decimal_exponent
+len is the length of the buffer (number of decimal digits)
+The buffer must be large enough, i.e. >= max_digits10.
+*/
+template <typename FloatType>
+void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
+{
+    static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,
+                  "internal error: not enough precision");
+
+    assert(std::isfinite(value));
+    assert(value > 0);
+
+    // If the neighbors (and boundaries) of 'value' are always computed for double-precision
+    // numbers, all float's can be recovered using strtod (and strtof). However, the resulting
+    // decimal representations are not exactly "short".
+    //
+    // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)
+    // says "value is converted to a string as if by std::sprintf in the default ("C") locale"
+    // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars'
+    // does.
+    // On the other hand, the documentation for 'std::to_chars' requires that "parsing the
+    // representation using the corresponding std::from_chars function recovers value exactly". That
+    // indicates that single precision floating-point numbers should be recovered using
+    // 'std::strtof'.
+    //
+    // NB: If the neighbors are computed for single-precision numbers, there is a single float
+    //     (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision
+    //     value is off by 1 ulp.
+#if 0
+    const boundaries w = compute_boundaries(static_cast<double>(value));
+#else
+    const boundaries w = compute_boundaries(value);
+#endif
+
+    grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
+}
+
+/*!
+@brief appends a decimal representation of e to buf
+@return a pointer to the element following the exponent.
+@pre -1000 < e < 1000
+*/
+inline char* append_exponent(char* buf, int e)
+{
+    assert(e > -1000);
+    assert(e <  1000);
+
+    if (e < 0)
+    {
+        e = -e;
+        *buf++ = '-';
+    }
+    else
+    {
+        *buf++ = '+';
+    }
 
-                // step 2: write the string
-                oa->write_characters(
-                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
-                    j.m_value.string->size());
-                break;
-            }
+    auto k = static_cast<uint32_t>(e);
+    if (k < 10)
+    {
+        // Always print at least two digits in the exponent.
+        // This is for compatibility with printf("%g").
+        *buf++ = '0';
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else if (k < 100)
+    {
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
+    else
+    {
+        *buf++ = static_cast<char>('0' + k / 100);
+        k %= 100;
+        *buf++ = static_cast<char>('0' + k / 10);
+        k %= 10;
+        *buf++ = static_cast<char>('0' + k);
+    }
 
-            case value_t::array:
-            {
-                // step 1: write control byte and the array size
-                const auto N = j.m_value.array->size();
-                if (N <= 15)
-                {
-                    // fixarray
-                    write_number(static_cast<uint8_t>(0x90 | N));
-                }
-                else if (N <= 0xFFFF)
-                {
-                    // array 16
-                    oa->write_character(static_cast<CharType>(0xDC));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= 0xFFFFFFFF)
-                {
-                    // array 32
-                    oa->write_character(static_cast<CharType>(0xDD));
-                    write_number(static_cast<uint32_t>(N));
-                }
+    return buf;
+}
 
-                // step 2: write each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    write_msgpack(el);
-                }
-                break;
-            }
+/*!
+@brief prettify v = buf * 10^decimal_exponent
 
-            case value_t::object:
-            {
-                // step 1: write control byte and the object size
-                const auto N = j.m_value.object->size();
-                if (N <= 15)
-                {
-                    // fixmap
-                    write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
-                }
-                else if (N <= 65535)
-                {
-                    // map 16
-                    oa->write_character(static_cast<CharType>(0xDE));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= 4294967295)
-                {
-                    // map 32
-                    oa->write_character(static_cast<CharType>(0xDF));
-                    write_number(static_cast<uint32_t>(N));
-                }
+If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point
+notation. Otherwise it will be printed in exponential notation.
 
-                // step 2: write each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    write_msgpack(el.first);
-                    write_msgpack(el.second);
-                }
-                break;
-            }
+@pre min_exp < 0
+@pre max_exp > 0
+*/
+inline char* format_buffer(char* buf, int len, int decimal_exponent,
+                           int min_exp, int max_exp)
+{
+    assert(min_exp < 0);
+    assert(max_exp > 0);
 
-            default:
-                break;
-        }
+    const int k = len;
+    const int n = len + decimal_exponent;
+
+    // v = buf * 10^(n-k)
+    // k is the length of the buffer (number of decimal digits)
+    // n is the position of the decimal point relative to the start of the buffer.
+
+    if (k <= n and n <= max_exp)
+    {
+        // digits[000]
+        // len <= max_exp + 2
+
+        std::memset(buf + k, '0', static_cast<size_t>(n - k));
+        // Make it look like a floating-point number (#362, #378)
+        buf[n + 0] = '.';
+        buf[n + 1] = '0';
+        return buf + (n + 2);
     }
 
-  private:
-    /*
-    @brief write a number to output input
+    if (0 < n and n <= max_exp)
+    {
+        // dig.its
+        // len <= max_digits10 + 1
 
-    @param[in] n number of type @a NumberType
-    @tparam NumberType the type of the number
+        assert(k > n);
 
-    @note This function needs to respect the system's endianess, because bytes
-          in CBOR and MessagePack are stored in network order (big endian) and
-          therefore need reordering on little endian systems.
-    */
-    template<typename NumberType> void write_number(NumberType n)
+        std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n));
+        buf[n] = '.';
+        return buf + (k + 1);
+    }
+
+    if (min_exp < n and n <= 0)
     {
-        // step 1: write number to array of length NumberType
-        std::array<CharType, sizeof(NumberType)> vec;
-        std::memcpy(vec.data(), &n, sizeof(NumberType));
+        // 0.[000]digits
+        // len <= 2 + (-min_exp - 1) + max_digits10
 
-        // step 2: write array to output (with possible reordering)
-        if (is_little_endian)
-        {
-            // reverse byte order prior to conversion if necessary
-            std::reverse(vec.begin(), vec.end());
-        }
+        std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k));
+        buf[0] = '0';
+        buf[1] = '.';
+        std::memset(buf + 2, '0', static_cast<size_t>(-n));
+        return buf + (2 + (-n) + k);
+    }
 
-        oa->write_characters(vec.data(), sizeof(NumberType));
+    if (k == 1)
+    {
+        // dE+123
+        // len <= 1 + 5
+
+        buf += 1;
     }
+    else
+    {
+        // d.igitsE+123
+        // len <= max_digits10 + 1 + 5
 
-  private:
-    /// whether we can assume little endianess
-    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
+        std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1));
+        buf[1] = '.';
+        buf += 1 + k;
+    }
 
-    /// the output
-    output_adapter_t<CharType> oa = nullptr;
-};
+    *buf++ = 'e';
+    return append_exponent(buf, n - 1);
+}
+
+} // namespace dtoa_impl
+
+/*!
+@brief generates a decimal representation of the floating-point number value in [first, last).
+
+The format of the resulting decimal representation is similar to printf's %g
+format. Returns an iterator pointing past-the-end of the decimal representation.
+
+@note The input number must be finite, i.e. NaN's and Inf's are not supported.
+@note The buffer must be large enough.
+@note The result is NOT null-terminated.
+*/
+template <typename FloatType>
+char* to_chars(char* first, const char* last, FloatType value)
+{
+    static_cast<void>(last); // maybe unused - fix warning
+    assert(std::isfinite(value));
+
+    // Use signbit(value) instead of (value < 0) since signbit works for -0.
+    if (std::signbit(value))
+    {
+        value = -value;
+        *first++ = '-';
+    }
+
+    if (value == 0) // +-0
+    {
+        *first++ = '0';
+        // Make it look like a floating-point number (#362, #378)
+        *first++ = '.';
+        *first++ = '0';
+        return first;
+    }
+
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10);
+
+    // Compute v = buffer * 10^decimal_exponent.
+    // The decimal digits are stored in the buffer, which needs to be interpreted
+    // as an unsigned decimal integer.
+    // len is the length of the buffer, i.e. the number of decimal digits.
+    int len = 0;
+    int decimal_exponent = 0;
+    dtoa_impl::grisu2(first, len, decimal_exponent, value);
+
+    assert(len <= std::numeric_limits<FloatType>::max_digits10);
+
+    // Format the buffer like printf("%.*g", prec, value)
+    constexpr int kMinExp = -4;
+    // Use digits10 here to increase compatibility with version 2.
+    constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;
+
+    assert(last - first >= kMaxExp + 2);
+    assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);
+    assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);
+
+    return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
+}
+
+} // namespace detail
+} // namespace nlohmann
+
+// #include <nlohmann/detail/macro_scope.hpp>
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/output/binary_writer.hpp>
 
+// #include <nlohmann/detail/output/output_adapters.hpp>
+
+// #include <nlohmann/detail/value_t.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
 ///////////////////
 // serialization //
 ///////////////////
 
+/// how to treat decoding errors
+enum class error_handler_t
+{
+    strict,  ///< throw a type_error exception in case of invalid UTF-8
+    replace, ///< replace invalid UTF-8 sequences with U+FFFD
+    ignore   ///< ignore invalid UTF-8 sequences
+};
+
 template<typename BasicJsonType>
 class serializer
 {
@@ -6077,20 +10806,32 @@ class serializer
     using number_float_t = typename BasicJsonType::number_float_t;
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    static constexpr uint8_t UTF8_ACCEPT = 0;
+    static constexpr uint8_t UTF8_REJECT = 1;
+
   public:
     /*!
     @param[in] s  output stream to serialize to
     @param[in] ichar  indentation character to use
+    @param[in] error_handler_  how to react on decoding errors
     */
-    serializer(output_adapter_t<char> s, const char ichar)
-        : o(std::move(s)), loc(std::localeconv()),
-          thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)),
-          decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)),
-          indent_char(ichar), indent_string(512, indent_char) {}
+    serializer(output_adapter_t<char> s, const char ichar,
+               error_handler_t error_handler_ = error_handler_t::strict)
+        : o(std::move(s))
+        , loc(std::localeconv())
+        , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
+        , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
+        , indent_char(ichar)
+        , indent_string(512, indent_char)
+        , error_handler(error_handler_)
+    {}
 
     // delete because of pointer members
     serializer(const serializer&) = delete;
     serializer& operator=(const serializer&) = delete;
+    serializer(serializer&&) = delete;
+    serializer& operator=(serializer&&) = delete;
+    ~serializer() = default;
 
     /*!
     @brief internal implementation of the serialization function
@@ -6253,220 +10994,55 @@ class serializer
                 o->write_character('\"');
                 dump_escaped(*val.m_value.string, ensure_ascii);
                 o->write_character('\"');
-                return;
-            }
-
-            case value_t::boolean:
-            {
-                if (val.m_value.boolean)
-                {
-                    o->write_characters("true", 4);
-                }
-                else
-                {
-                    o->write_characters("false", 5);
-                }
-                return;
-            }
-
-            case value_t::number_integer:
-            {
-                dump_integer(val.m_value.number_integer);
-                return;
-            }
-
-            case value_t::number_unsigned:
-            {
-                dump_integer(val.m_value.number_unsigned);
-                return;
-            }
-
-            case value_t::number_float:
-            {
-                dump_float(val.m_value.number_float);
-                return;
-            }
-
-            case value_t::discarded:
-            {
-                o->write_characters("<discarded>", 11);
-                return;
-            }
-
-            case value_t::null:
-            {
-                o->write_characters("null", 4);
-                return;
-            }
-        }
-    }
-
-  private:
-    /*!
-    @brief returns the number of expected bytes following in UTF-8 string
-
-    @param[in]  u  the first byte of a UTF-8 string
-    @return  the number of expected bytes following
-    */
-    static constexpr std::size_t bytes_following(const uint8_t u)
-    {
-        return ((u <= 127) ? 0
-                : ((192 <= u and u <= 223) ? 1
-                   : ((224 <= u and u <= 239) ? 2
-                      : ((240 <= u and u <= 247) ? 3 : std::string::npos))));
-    }
-
-    /*!
-    @brief calculates the extra space to escape a JSON string
-
-    @param[in] s  the string to escape
-    @param[in] ensure_ascii  whether to escape non-ASCII characters with
-                             \uXXXX sequences
-    @return the number of characters required to escape string @a s
-
-    @complexity Linear in the length of string @a s.
-    */
-    static std::size_t extra_space(const string_t& s,
-                                   const bool ensure_ascii) noexcept
-    {
-        std::size_t res = 0;
-
-        for (std::size_t i = 0; i < s.size(); ++i)
-        {
-            switch (s[i])
-            {
-                // control characters that can be escaped with a backslash
-                case '"':
-                case '\\':
-                case '\b':
-                case '\f':
-                case '\n':
-                case '\r':
-                case '\t':
-                {
-                    // from c (1 byte) to \x (2 bytes)
-                    res += 1;
-                    break;
-                }
-
-                // control characters that need \uxxxx escaping
-                case 0x00:
-                case 0x01:
-                case 0x02:
-                case 0x03:
-                case 0x04:
-                case 0x05:
-                case 0x06:
-                case 0x07:
-                case 0x0B:
-                case 0x0E:
-                case 0x0F:
-                case 0x10:
-                case 0x11:
-                case 0x12:
-                case 0x13:
-                case 0x14:
-                case 0x15:
-                case 0x16:
-                case 0x17:
-                case 0x18:
-                case 0x19:
-                case 0x1A:
-                case 0x1B:
-                case 0x1C:
-                case 0x1D:
-                case 0x1E:
-                case 0x1F:
-                {
-                    // from c (1 byte) to \uxxxx (6 bytes)
-                    res += 5;
-                    break;
-                }
-
-                default:
-                {
-                    if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))
-                    {
-                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        // invalid characters will be detected by throw_if_invalid_utf8
-                        assert (bytes != std::string::npos);
-
-                        if (bytes == 3)
-                        {
-                            // codepoints that need 4 bytes (i.e., 3 additional
-                            // bytes) in UTF-8 need a surrogate pair when \u
-                            // escaping is used: from 4 bytes to \uxxxx\uxxxx
-                            // (12 bytes)
-                            res += (12 - bytes - 1);
-                        }
-                        else
-                        {
-                            // from x bytes to \uxxxx (6 bytes)
-                            res += (6 - bytes - 1);
-                        }
+                return;
+            }
 
-                        // skip the additional bytes
-                        i += bytes;
-                    }
-                    break;
+            case value_t::boolean:
+            {
+                if (val.m_value.boolean)
+                {
+                    o->write_characters("true", 4);
+                }
+                else
+                {
+                    o->write_characters("false", 5);
                 }
+                return;
             }
-        }
-
-        return res;
-    }
 
-    static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos)
-    {
-        // expecting a proper codepoint
-        assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+            case value_t::number_integer:
+            {
+                dump_integer(val.m_value.number_integer);
+                return;
+            }
 
-        // the last written character was the backslash before the 'u'
-        assert(result[pos] == '\\');
+            case value_t::number_unsigned:
+            {
+                dump_integer(val.m_value.number_unsigned);
+                return;
+            }
 
-        // write the 'u'
-        result[++pos] = 'u';
+            case value_t::number_float:
+            {
+                dump_float(val.m_value.number_float);
+                return;
+            }
 
-        // convert a number 0..15 to its hex representation (0..f)
-        static const std::array<char, 16> hexify =
-        {
+            case value_t::discarded:
             {
-                '0', '1', '2', '3', '4', '5', '6', '7',
-                '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+                o->write_characters("<discarded>", 11);
+                return;
             }
-        };
 
-        if (codepoint < 0x10000)
-        {
-            // codepoints U+0000..U+FFFF can be represented as \uxxxx.
-            result[++pos] = hexify[(codepoint >> 12) & 0x0F];
-            result[++pos] = hexify[(codepoint >> 8) & 0x0F];
-            result[++pos] = hexify[(codepoint >> 4) & 0x0F];
-            result[++pos] = hexify[codepoint & 0x0F];
-        }
-        else
-        {
-            // codepoints U+10000..U+10FFFF need a surrogate pair to be
-            // represented as \uxxxx\uxxxx.
-            // http://www.unicode.org/faq/utf_bom.html#utf16-4
-            codepoint -= 0x10000;
-            const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF);
-            const int low_surrogate = 0xDC00 | (codepoint & 0x3FF);
-            result[++pos] = hexify[(high_surrogate >> 12) & 0x0F];
-            result[++pos] = hexify[(high_surrogate >> 8) & 0x0F];
-            result[++pos] = hexify[(high_surrogate >> 4) & 0x0F];
-            result[++pos] = hexify[high_surrogate & 0x0F];
-            ++pos;  // backslash is already in output
-            result[++pos] = 'u';
-            result[++pos] = hexify[(low_surrogate >> 12) & 0x0F];
-            result[++pos] = hexify[(low_surrogate >> 8) & 0x0F];
-            result[++pos] = hexify[(low_surrogate >> 4) & 0x0F];
-            result[++pos] = hexify[low_surrogate & 0x0F];
+            case value_t::null:
+            {
+                o->write_characters("null", 4);
+                return;
+            }
         }
-
-        ++pos;
     }
 
+  private:
     /*!
     @brief dump escaped string
 
@@ -6481,145 +11057,236 @@ class serializer
 
     @complexity Linear in the length of string @a s.
     */
-    void dump_escaped(const string_t& s, const bool ensure_ascii) const
+    void dump_escaped(const string_t& s, const bool ensure_ascii)
     {
-        throw_if_invalid_utf8(s);
-
-        const auto space = extra_space(s, ensure_ascii);
-        if (space == 0)
-        {
-            o->write_characters(s.c_str(), s.size());
-            return;
-        }
+        uint32_t codepoint;
+        uint8_t state = UTF8_ACCEPT;
+        std::size_t bytes = 0;  // number of bytes written to string_buffer
 
-        // create a result string of necessary size
-        string_t result(s.size() + space, '\\');
-        std::size_t pos = 0;
+        // number of bytes written at the point of the last valid byte
+        std::size_t bytes_after_last_accept = 0;
+        std::size_t undumped_chars = 0;
 
         for (std::size_t i = 0; i < s.size(); ++i)
         {
-            switch (s[i])
+            const auto byte = static_cast<uint8_t>(s[i]);
+
+            switch (decode(state, codepoint, byte))
             {
-                case '"': // quotation mark (0x22)
+                case UTF8_ACCEPT:  // decode found a new code point
                 {
-                    result[pos + 1] = '"';
-                    pos += 2;
-                    break;
-                }
+                    switch (codepoint)
+                    {
+                        case 0x08: // backspace
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'b';
+                            break;
+                        }
 
-                case '\\': // reverse solidus (0x5C)
-                {
-                    // nothing to change
-                    pos += 2;
-                    break;
-                }
+                        case 0x09: // horizontal tab
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 't';
+                            break;
+                        }
 
-                case '\b': // backspace (0x08)
-                {
-                    result[pos + 1] = 'b';
-                    pos += 2;
-                    break;
-                }
+                        case 0x0A: // newline
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'n';
+                            break;
+                        }
 
-                case '\f': // formfeed (0x0C)
-                {
-                    result[pos + 1] = 'f';
-                    pos += 2;
-                    break;
-                }
+                        case 0x0C: // formfeed
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'f';
+                            break;
+                        }
 
-                case '\n': // newline (0x0A)
-                {
-                    result[pos + 1] = 'n';
-                    pos += 2;
-                    break;
-                }
+                        case 0x0D: // carriage return
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = 'r';
+                            break;
+                        }
 
-                case '\r': // carriage return (0x0D)
-                {
-                    result[pos + 1] = 'r';
-                    pos += 2;
-                    break;
-                }
+                        case 0x22: // quotation mark
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\"';
+                            break;
+                        }
 
-                case '\t': // horizontal tab (0x09)
-                {
-                    result[pos + 1] = 't';
-                    pos += 2;
+                        case 0x5C: // reverse solidus
+                        {
+                            string_buffer[bytes++] = '\\';
+                            string_buffer[bytes++] = '\\';
+                            break;
+                        }
+
+                        default:
+                        {
+                            // escape control characters (0x00..0x1F) or, if
+                            // ensure_ascii parameter is used, non-ASCII characters
+                            if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F)))
+                            {
+                                if (codepoint <= 0xFFFF)
+                                {
+                                    std::snprintf(string_buffer.data() + bytes, 7, "\\u%04x",
+                                                  static_cast<uint16_t>(codepoint));
+                                    bytes += 6;
+                                }
+                                else
+                                {
+                                    std::snprintf(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                                  static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
+                                                  static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
+                                    bytes += 12;
+                                }
+                            }
+                            else
+                            {
+                                // copy byte to buffer (all previous bytes
+                                // been copied have in default case above)
+                                string_buffer[bytes++] = s[i];
+                            }
+                            break;
+                        }
+                    }
+
+                    // write buffer and reset index; there must be 13 bytes
+                    // left, as this is the maximal number of bytes to be
+                    // written ("\uxxxx\uxxxx\0") for one code point
+                    if (string_buffer.size() - bytes < 13)
+                    {
+                        o->write_characters(string_buffer.data(), bytes);
+                        bytes = 0;
+                    }
+
+                    // remember the byte position of this accept
+                    bytes_after_last_accept = bytes;
+                    undumped_chars = 0;
                     break;
                 }
 
-                default:
+                case UTF8_REJECT:  // decode found invalid UTF-8 byte
                 {
-                    // escape control characters (0x00..0x1F) or, if
-                    // ensure_ascii parameter is used, non-ASCII characters
-                    if ((0x00 <= s[i] and s[i] <= 0x1F) or
-                            (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)))
+                    switch (error_handler)
                     {
-                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
-                        // invalid characters will be detected by throw_if_invalid_utf8
-                        assert (bytes != std::string::npos);
-
-                        // check that the additional bytes are present
-                        assert(i + bytes < s.size());
-
-                        // to use \uxxxx escaping, we first need to calculate
-                        // the codepoint from the UTF-8 bytes
-                        int codepoint = 0;
+                        case error_handler_t::strict:
+                        {
+                            std::string sn(3, '\0');
+                            snprintf(&sn[0], sn.size(), "%.2X", byte);
+                            JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
+                        }
 
-                        // bytes is unsigned type:
-                        assert(bytes <= 3);
-                        switch (bytes)
+                        case error_handler_t::ignore:
+                        case error_handler_t::replace:
                         {
-                            case 0:
+                            // in case we saw this character the first time, we
+                            // would like to read it again, because the byte
+                            // may be OK for itself, but just not OK for the
+                            // previous sequence
+                            if (undumped_chars > 0)
                             {
-                                codepoint = s[i] & 0xFF;
-                                break;
+                                --i;
                             }
 
-                            case 1:
-                            {
-                                codepoint = ((s[i] & 0x3F) << 6)
-                                            + (s[i + 1] & 0x7F);
-                                break;
-                            }
+                            // reset length buffer to the last accepted index;
+                            // thus removing/ignoring the invalid characters
+                            bytes = bytes_after_last_accept;
 
-                            case 2:
+                            if (error_handler == error_handler_t::replace)
                             {
-                                codepoint = ((s[i] & 0x1F) << 12)
-                                            + ((s[i + 1] & 0x7F) << 6)
-                                            + (s[i + 2] & 0x7F);
-                                break;
+                                // add a replacement character
+                                if (ensure_ascii)
+                                {
+                                    string_buffer[bytes++] = '\\';
+                                    string_buffer[bytes++] = 'u';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'f';
+                                    string_buffer[bytes++] = 'd';
+                                }
+                                else
+                                {
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
+                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
+                                }
+                                bytes_after_last_accept = bytes;
                             }
 
-                            case 3:
-                            {
-                                codepoint = ((s[i] & 0xF) << 18)
-                                            + ((s[i + 1] & 0x7F) << 12)
-                                            + ((s[i + 2] & 0x7F) << 6)
-                                            + (s[i + 3] & 0x7F);
-                                break;
-                            }
+                            undumped_chars = 0;
 
-                            default:
-                                break;  // LCOV_EXCL_LINE
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
                         }
+                    }
+                    break;
+                }
+
+                default:  // decode found yet incomplete multi-byte code point
+                {
+                    if (not ensure_ascii)
+                    {
+                        // code point will not be escaped - copy byte to buffer
+                        string_buffer[bytes++] = s[i];
+                    }
+                    ++undumped_chars;
+                    break;
+                }
+            }
+        }
+
+        // we finished processing the string
+        if (JSON_LIKELY(state == UTF8_ACCEPT))
+        {
+            // write buffer
+            if (bytes > 0)
+            {
+                o->write_characters(string_buffer.data(), bytes);
+            }
+        }
+        else
+        {
+            // we finish reading, but do not accept: string was incomplete
+            switch (error_handler)
+            {
+                case error_handler_t::strict:
+                {
+                    std::string sn(3, '\0');
+                    snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
+                    JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
+                }
 
-                        escape_codepoint(codepoint, result, pos);
-                        i += bytes;
+                case error_handler_t::ignore:
+                {
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
+                    break;
+                }
+
+                case error_handler_t::replace:
+                {
+                    // write all accepted bytes
+                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
+                    // add a replacement character
+                    if (ensure_ascii)
+                    {
+                        o->write_characters("\\ufffd", 6);
                     }
                     else
                     {
-                        // all other characters are added as-is
-                        result[pos++] = s[i];
+                        o->write_characters("\xEF\xBF\xBD", 3);
                     }
                     break;
                 }
             }
         }
-
-        assert(pos == result.size());
-        o->write_characters(result.c_str(), result.size());
     }
 
     /*!
@@ -6644,7 +11311,7 @@ class serializer
             return;
         }
 
-        const bool is_negative = (x <= 0) and (x != 0);  // see issue #755
+        const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0);  // see issue #755
         std::size_t i = 0;
 
         while (x != 0)
@@ -6679,14 +11346,36 @@ class serializer
     void dump_float(number_float_t x)
     {
         // NaN / inf
-        if (not std::isfinite(x) or std::isnan(x))
+        if (not std::isfinite(x))
         {
             o->write_characters("null", 4);
             return;
         }
 
-        // get number of digits for a text -> float -> text round-trip
-        static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
+        // If number_float_t is an IEEE-754 single or double precision number,
+        // use the Grisu2 algorithm to produce short numbers which are
+        // guaranteed to round-trip, using strtof and strtod, resp.
+        //
+        // NB: The test below works if <long double> == <double>.
+        static constexpr bool is_ieee_single_or_double
+            = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or
+              (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024);
+
+        dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());
+    }
+
+    void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)
+    {
+        char* begin = number_buffer.data();
+        char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);
+
+        o->write_characters(begin, static_cast<size_t>(end - begin));
+    }
+
+    void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)
+    {
+        // get number of digits for a float -> text -> float round-trip
+        static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
 
         // the actual conversion
         std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
@@ -6744,15 +11433,16 @@ class serializer
     followed.
 
     @param[in,out] state  the state of the decoding
+    @param[in,out] codep  codepoint (valid only if resulting state is UTF8_ACCEPT)
     @param[in] byte       next byte to decode
+    @return               new state
 
-    @note The function has been edited: a std::array is used and the code
-          point is not calculated.
+    @note The function has been edited: a std::array is used.
 
     @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
     @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
     */
-    static void decode(uint8_t& state, const uint8_t byte)
+    static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept
     {
         static const std::array<uint8_t, 400> utf8d =
         {
@@ -6775,42 +11465,13 @@ class serializer
         };
 
         const uint8_t type = utf8d[byte];
-        state = utf8d[256u + state * 16u + type];
-    }
-
-    /*!
-    @brief throw an exception if a string is not UTF-8 encoded
-
-    @param[in] str  UTF-8 string to check
-    @throw type_error.316 if passed string is not UTF-8 encoded
-
-    @since version 3.0.0
-    */
-    static void throw_if_invalid_utf8(const std::string& str)
-    {
-        // start with state 0 (= accept)
-        uint8_t state = 0;
 
-        for (size_t i = 0; i < str.size(); ++i)
-        {
-            const auto byte = static_cast<uint8_t>(str[i]);
-            decode(state, byte);
-            if (state == 1)
-            {
-                // state 1 means reject
-                std::stringstream ss;
-                ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte);
-                JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str()));
-            }
-        }
+        codep = (state != UTF8_ACCEPT)
+                ? (byte & 0x3fu) | (codep << 6)
+                : static_cast<uint32_t>(0xff >> type) & (byte);
 
-        if (state != 0)
-        {
-            // we finish reading, but do not accept: string was incomplete
-            std::stringstream ss;
-            ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back()));
-            JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str()));
-        }
+        state = utf8d[256u + state * 16u + type];
+        return state;
     }
 
   private:
@@ -6827,13 +11488,33 @@ class serializer
     /// the locale's decimal point character
     const char decimal_point = '\0';
 
+    /// string buffer
+    std::array<char, 512> string_buffer{{}};
+
     /// the indentation character
     const char indent_char;
-
     /// the indentation string
     string_t indent_string;
+
+    /// error_handler how to react on decoding errors
+    const error_handler_t error_handler;
 };
+}  // namespace detail
+}  // namespace nlohmann
+
+// #include <nlohmann/detail/json_ref.hpp>
+
+
+#include <initializer_list>
+#include <utility>
+
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
 
+namespace nlohmann
+{
+namespace detail
+{
 template<typename BasicJsonType>
 class json_ref
 {
@@ -6852,15 +11533,19 @@ class json_ref
         : owned_value(init), value_ref(&owned_value), is_rvalue(true)
     {}
 
-    template<class... Args>
-    json_ref(Args&& ... args)
-        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true)
-    {}
+    template <
+        class... Args,
+        enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >
+    json_ref(Args && ... args)
+        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value),
+          is_rvalue(true) {}
 
     // class should be movable only
     json_ref(json_ref&&) = default;
     json_ref(const json_ref&) = delete;
     json_ref& operator=(const json_ref&) = delete;
+    json_ref& operator=(json_ref&&) = delete;
+    ~json_ref() = default;
 
     value_type moved_or_copied() const
     {
@@ -6886,74 +11571,30 @@ class json_ref
     value_type* value_ref = nullptr;
     const bool is_rvalue;
 };
+}  // namespace detail
+}  // namespace nlohmann
 
-} // namespace detail
-
-/// namespace to hold default `to_json` / `from_json` functions
-namespace
-{
-constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
-constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
-}
-
-
-/*!
-@brief default JSONSerializer template argument
-
-This serializer ignores the template arguments and uses ADL
-([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl))
-for serialization.
-*/
-template<typename, typename>
-struct adl_serializer
-{
-    /*!
-    @brief convert a JSON value to any value type
-
-    This function is usually called by the `get()` function of the
-    @ref basic_json class (either explicit or via conversion operators).
-
-    @param[in] j         JSON value to read from
-    @param[in,out] val  value to write to
-    */
-    template<typename BasicJsonType, typename ValueType>
-    static void from_json(BasicJsonType&& j, ValueType& val) noexcept(
-        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
-    {
-        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
-    }
+// #include <nlohmann/detail/json_pointer.hpp>
 
-    /*!
-    @brief convert any value type to a JSON value
 
-    This function is usually called by the constructors of the @ref basic_json
-    class.
+#include <cassert> // assert
+#include <numeric> // accumulate
+#include <string> // string
+#include <vector> // vector
 
-    @param[in,out] j  JSON value to write to
-    @param[in] val     value to read from
-    */
-    template<typename BasicJsonType, typename ValueType>
-    static void to_json(BasicJsonType& j, ValueType&& val) noexcept(
-        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
-    {
-        ::nlohmann::to_json(j, std::forward<ValueType>(val));
-    }
-};
+// #include <nlohmann/detail/macro_scope.hpp>
 
-/*!
-@brief JSON Pointer
+// #include <nlohmann/detail/exceptions.hpp>
 
-A JSON pointer defines a string syntax for identifying a specific value
-within a JSON document. It can be used with functions `at` and
-`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+// #include <nlohmann/detail/value_t.hpp>
 
-@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
 
-@since version 2.0.0
-*/
+namespace nlohmann
+{
+template<typename BasicJsonType>
 class json_pointer
 {
-    /// allow basic_json to access private members
+    // allow basic_json to access private members
     NLOHMANN_BASIC_JSON_TPL_DECLARATION
     friend class basic_json;
 
@@ -6967,19 +11608,21 @@ class json_pointer
     @param[in] s  string representing the JSON pointer; if omitted, the empty
                   string is assumed which references the whole JSON value
 
-    @throw parse_error.107 if the given JSON pointer @a s is nonempty and
-    does not begin with a slash (`/`); see example below
+    @throw parse_error.107 if the given JSON pointer @a s is nonempty and does
+                           not begin with a slash (`/`); see example below
 
-    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
-    is not followed by `0` (representing `~`) or `1` (representing `/`);
-    see example below
+    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is
+    not followed by `0` (representing `~`) or `1` (representing `/`); see
+    example below
 
-    @liveexample{The example shows the construction several valid JSON
-    pointers as well as the exceptional behavior.,json_pointer}
+    @liveexample{The example shows the construction several valid JSON pointers
+    as well as the exceptional behavior.,json_pointer}
 
     @since version 2.0.0
     */
-    explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {}
+    explicit json_pointer(const std::string& s = "")
+        : reference_tokens(split(s))
+    {}
 
     /*!
     @brief return a string representation of the JSON pointer
@@ -6996,7 +11639,7 @@ class json_pointer
 
     @since version 2.0.0
     */
-    std::string to_string() const noexcept
+    std::string to_string() const
     {
         return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
                                std::string{},
@@ -7021,7 +11664,7 @@ class json_pointer
     */
     static int array_index(const std::string& s)
     {
-        size_t processed_chars = 0;
+        std::size_t processed_chars = 0;
         const int res = std::stoi(s, &processed_chars);
 
         // check if the string was completely read
@@ -7051,7 +11694,7 @@ class json_pointer
     }
 
     /// return whether pointer points to the root document
-    bool is_root() const
+    bool is_root() const noexcept
     {
         return reference_tokens.empty();
     }
@@ -7076,8 +11719,66 @@ class json_pointer
     @throw parse_error.109 if array index is not a number
     @throw type_error.313 if value cannot be unflattened
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const;
+    BasicJsonType& get_and_create(BasicJsonType& j) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        auto result = &j;
+
+        // in case no reference tokens exist, return a reference to the JSON value
+        // j which will be overwritten by a primitive value
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (result->m_type)
+            {
+                case detail::value_t::null:
+                {
+                    if (reference_token == "0")
+                    {
+                        // start a new array if reference token is 0
+                        result = &result->operator[](0);
+                    }
+                    else
+                    {
+                        // start a new object otherwise
+                        result = &result->operator[](reference_token);
+                    }
+                    break;
+                }
+
+                case detail::value_t::object:
+                {
+                    // create an entry in the object
+                    result = &result->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // create an entry in the array
+                    JSON_TRY
+                    {
+                        result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                /*
+                The following code is only reached if there exists a reference
+                token _and_ the current value is primitive. In this case, we have
+                an error situation, because primitive values may only occur as
+                single value; that is, with an empty list of reference tokens.
+                */
+                default:
+                    JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
+            }
+        }
+
+        return *result;
+    }
 
     /*!
     @brief return a reference to the pointed to value
@@ -7098,8 +11799,75 @@ class json_pointer
     @throw parse_error.109   if an array index was not a number
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    BasicJsonType& get_unchecked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            // convert null values to arrays or objects before continuing
+            if (ptr->m_type == detail::value_t::null)
+            {
+                // check if reference token is a number
+                const bool nums =
+                    std::all_of(reference_token.begin(), reference_token.end(),
+                                [](const char x)
+                {
+                    return (x >= '0' and x <= '9');
+                });
+
+                // change value to array for numbers or "-" or to object otherwise
+                *ptr = (nums or reference_token == "-")
+                       ? detail::value_t::array
+                       : detail::value_t::object;
+            }
+
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    if (reference_token == "-")
+                    {
+                        // explicitly treat "-" as index beyond the end
+                        ptr = &ptr->operator[](ptr->m_value.array->size());
+                    }
+                    else
+                    {
+                        // convert array index to number; unchecked access
+                        JSON_TRY
+                        {
+                            ptr = &ptr->operator[](
+                                static_cast<size_type>(array_index(reference_token)));
+                        }
+                        JSON_CATCH(std::invalid_argument&)
+                        {
+                            JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @throw parse_error.106   if an array index begins with '0'
@@ -7107,8 +11875,57 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    BasicJsonType& get_checked(BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @brief return a const reference to the pointed to value
@@ -7123,8 +11940,58 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // use unchecked object access
+                    ptr = &ptr->operator[](reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" cannot be used for const access
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // use unchecked array access
+                    JSON_TRY
+                    {
+                        ptr = &ptr->operator[](
+                            static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @throw parse_error.106   if an array index begins with '0'
@@ -7132,8 +11999,57 @@ class json_pointer
     @throw out_of_range.402  if the array index '-' is used
     @throw out_of_range.404  if the JSON pointer can not be resolved
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+    const BasicJsonType& get_checked(const BasicJsonType* ptr) const
+    {
+        using size_type = typename BasicJsonType::size_type;
+        for (const auto& reference_token : reference_tokens)
+        {
+            switch (ptr->m_type)
+            {
+                case detail::value_t::object:
+                {
+                    // note: at performs range check
+                    ptr = &ptr->at(reference_token);
+                    break;
+                }
+
+                case detail::value_t::array:
+                {
+                    if (JSON_UNLIKELY(reference_token == "-"))
+                    {
+                        // "-" always fails the range check
+                        JSON_THROW(detail::out_of_range::create(402,
+                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                                ") is out of range"));
+                    }
+
+                    // error condition (cf. RFC 6901, Sect. 4)
+                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                    {
+                        JSON_THROW(detail::parse_error::create(106, 0,
+                                                               "array index '" + reference_token +
+                                                               "' must not begin with '0'"));
+                    }
+
+                    // note: at performs range check
+                    JSON_TRY
+                    {
+                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                    break;
+                }
+
+                default:
+                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+            }
+        }
+
+        return *ptr;
+    }
 
     /*!
     @brief split the string input to reference tokens
@@ -7170,11 +12086,11 @@ class json_pointer
             std::size_t slash = reference_string.find_first_of('/', 1),
             // set the beginning of the first reference token
             start = 1;
-            // we can stop if start == string::npos+1 = 0
+            // we can stop if start == 0 (if slash == std::string::npos)
             start != 0;
             // set the beginning of the next reference token
             // (will eventually be 0 if slash == std::string::npos)
-            start = slash + 1,
+            start = (slash == std::string::npos) ? 0 : slash + 1,
             // find next slash
             slash = reference_string.find_first_of('/', start))
         {
@@ -7230,7 +12146,7 @@ class json_pointer
         {}
     }
 
-    /// escape "~"" to "~0" and "/" to "~1"
+    /// escape "~" to "~0" and "/" to "~1"
     static std::string escape(std::string s)
     {
         replace_substring(s, "~", "~0");
@@ -7252,34 +12168,174 @@ class json_pointer
 
     @note Empty objects or arrays are flattened to `null`.
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
     static void flatten(const std::string& reference_string,
-                        const NLOHMANN_BASIC_JSON_TPL& value,
-                        NLOHMANN_BASIC_JSON_TPL& result);
+                        const BasicJsonType& value,
+                        BasicJsonType& result)
+    {
+        switch (value.m_type)
+        {
+            case detail::value_t::array:
+            {
+                if (value.m_value.array->empty())
+                {
+                    // flatten empty array as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate array and use index as reference string
+                    for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
+                    {
+                        flatten(reference_string + "/" + std::to_string(i),
+                                value.m_value.array->operator[](i), result);
+                    }
+                }
+                break;
+            }
+
+            case detail::value_t::object:
+            {
+                if (value.m_value.object->empty())
+                {
+                    // flatten empty object as null
+                    result[reference_string] = nullptr;
+                }
+                else
+                {
+                    // iterate object and use keys as reference string
+                    for (const auto& element : *value.m_value.object)
+                    {
+                        flatten(reference_string + "/" + escape(element.first), element.second, result);
+                    }
+                }
+                break;
+            }
+
+            default:
+            {
+                // add primitive value with its reference string
+                result[reference_string] = value;
+                break;
+            }
+        }
+    }
+
+    /*!
+    @param[in] value  flattened JSON
+
+    @return unflattened JSON
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+    @throw type_error.313  if value cannot be unflattened
+    */
+    static BasicJsonType
+    unflatten(const BasicJsonType& value)
+    {
+        if (JSON_UNLIKELY(not value.is_object()))
+        {
+            JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
+        }
+
+        BasicJsonType result;
+
+        // iterate the JSON object values
+        for (const auto& element : *value.m_value.object)
+        {
+            if (JSON_UNLIKELY(not element.second.is_primitive()))
+            {
+                JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+            }
+
+            // assign value to reference pointed to by JSON pointer; Note that if
+            // the JSON pointer is "" (i.e., points to the whole value), function
+            // get_and_create returns a reference to result itself. An assignment
+            // will then create a primitive value.
+            json_pointer(element.first).get_and_create(result) = element.second;
+        }
+
+        return result;
+    }
+
+    friend bool operator==(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept
+    {
+        return (lhs.reference_tokens == rhs.reference_tokens);
+    }
+
+    friend bool operator!=(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept
+    {
+        return not (lhs == rhs);
+    }
+
+    /// the reference tokens
+    std::vector<std::string> reference_tokens;
+};
+}  // namespace nlohmann
+
+// #include <nlohmann/adl_serializer.hpp>
+
+
+#include <utility>
+
+// #include <nlohmann/detail/conversions/from_json.hpp>
+
+// #include <nlohmann/detail/conversions/to_json.hpp>
+
+
+namespace nlohmann
+{
+
+template<typename, typename>
+struct adl_serializer
+{
+    /*!
+    @brief convert a JSON value to any value type
+
+    This function is usually called by the `get()` function of the
+    @ref basic_json class (either explicit or via conversion operators).
+
+    @param[in] j        JSON value to read from
+    @param[in,out] val  value to write to
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static auto from_json(BasicJsonType&& j, ValueType& val) noexcept(
+        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())
+    {
+        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+    }
 
     /*!
-    @param[in] value  flattened JSON
+    @brief convert any value type to a JSON value
 
-    @return unflattened JSON
+    This function is usually called by the constructors of the @ref basic_json
+    class.
 
-    @throw parse_error.109 if array index is not a number
-    @throw type_error.314  if value is not an object
-    @throw type_error.315  if object values are not primitive
-    @throw type_error.313  if value cannot be unflattened
+    @param[in,out] j  JSON value to write to
+    @param[in] val    value to read from
     */
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    static NLOHMANN_BASIC_JSON_TPL
-    unflatten(const NLOHMANN_BASIC_JSON_TPL& value);
+    template <typename BasicJsonType, typename ValueType>
+    static auto to_json(BasicJsonType& j, ValueType&& val) noexcept(
+        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
+    -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void())
+    {
+        ::nlohmann::to_json(j, std::forward<ValueType>(val));
+    }
+};
 
-    friend bool operator==(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept;
+}  // namespace nlohmann
 
-    friend bool operator!=(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept;
 
-    /// the reference tokens
-    std::vector<std::string> reference_tokens;
-};
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann
+{
 
 /*!
 @brief a class to store JSON values
@@ -7305,42 +12361,42 @@ and `from_json()` (@ref adl_serializer by default)
 
 @requirement The class satisfies the following concept requirements:
 - Basic
- - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible):
+ - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible):
    JSON values can be default constructed. The result will be a JSON null
    value.
- - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible):
+ - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible):
    A JSON value can be constructed from an rvalue argument.
- - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible):
+ - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible):
    A JSON value can be copy-constructed from an lvalue expression.
- - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable):
+ - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable):
    A JSON value van be assigned from an rvalue argument.
- - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable):
+ - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable):
    A JSON value can be copy-assigned from an lvalue expression.
- - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible):
+ - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible):
    JSON values can be destructed.
 - Layout
- - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType):
+ - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType):
    JSON values have
-   [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
+   [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
    All non-static data members are private and standard layout types, the
    class has no virtual functions or (virtual) base classes.
 - Library-wide
- - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable):
+ - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable):
    JSON values can be compared with `==`, see @ref
    operator==(const_reference,const_reference).
- - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable):
+ - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable):
    JSON values can be compared with `<`, see @ref
    operator<(const_reference,const_reference).
- - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable):
+ - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable):
    Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of
    other compatible types, using unqualified function call @ref swap().
- - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer):
+ - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer):
    JSON values can be compared against `std::nullptr_t` objects which are used
    to model the `null` value.
 - Container
- - [Container](http://en.cppreference.com/w/cpp/concept/Container):
+ - [Container](https://en.cppreference.com/w/cpp/named_req/Container):
    JSON values can be used like STL containers and provide iterator access.
- - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer);
+ - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer);
    JSON values can be used like STL containers and provide reverse iterator
    access.
 
@@ -7367,15 +12423,19 @@ class basic_json
 {
   private:
     template<detail::value_t> friend struct detail::external_constructor;
-    friend ::nlohmann::json_pointer;
+    friend ::nlohmann::json_pointer<basic_json>;
     friend ::nlohmann::detail::parser<basic_json>;
     friend ::nlohmann::detail::serializer<basic_json>;
     template<typename BasicJsonType>
     friend class ::nlohmann::detail::iter_impl;
     template<typename BasicJsonType, typename CharType>
     friend class ::nlohmann::detail::binary_writer;
-    template<typename BasicJsonType>
+    template<typename BasicJsonType, typename SAX>
     friend class ::nlohmann::detail::binary_reader;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_parser;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::json_sax_dom_callback_parser;
 
     /// workaround type for MSVC
     using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
@@ -7403,13 +12463,19 @@ class basic_json
 
   public:
     using value_t = detail::value_t;
-    /// @copydoc nlohmann::json_pointer
-    using json_pointer = ::nlohmann::json_pointer;
+    /// JSON Pointer, see @ref nlohmann::json_pointer
+    using json_pointer = ::nlohmann::json_pointer<basic_json>;
     template<typename T, typename SFINAE>
     using json_serializer = JSONSerializer<T, SFINAE>;
+    /// how to treat decoding errors
+    using error_handler_t = detail::error_handler_t;
     /// helper type for initializer lists of basic_json values
     using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
 
+    using input_format_t = detail::input_format_t;
+    /// SAX interface type, see @ref nlohmann::json_sax
+    using json_sax_t = json_sax<basic_json>;
+
     ////////////////
     // exceptions //
     ////////////////
@@ -7517,10 +12583,13 @@ class basic_json
         result["copyright"] = "(C) 2013-2017 Niels Lohmann";
         result["name"] = "JSON for Modern C++";
         result["url"] = "https://github.com/nlohmann/json";
-        result["version"] =
-        {
-            {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1}
-        };
+        result["version"]["string"] =
+            std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." +
+            std::to_string(NLOHMANN_JSON_VERSION_PATCH);
+        result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR;
+        result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR;
+        result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH;
 
 #ifdef _WIN32
         result["platform"] = "win32";
@@ -7622,10 +12691,10 @@ class basic_json
     - When all names are unique, objects will be interoperable in the sense
       that all software implementations receiving that object will agree on
       the name-value mappings.
-    - When the names within an object are not unique, later stored name/value
-      pairs overwrite previously stored name/value pairs, leaving the used
-      names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will
-      be treated as equal and both stored as `{"key": 1}`.
+    - When the names within an object are not unique, it is unspecified which
+      one of the values for a given key will be chosen. For instance,
+      `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or
+      `{"key": 2}`.
     - Internally, name/value pairs are stored in lexicographical order of the
       names. Objects will also be serialized (see @ref dump) in this order.
       For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored
@@ -8139,7 +13208,7 @@ class basic_json
                     object = nullptr;  // silence warning, see #821
                     if (JSON_UNLIKELY(t == value_t::null))
                     {
-                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.4.0")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -8182,7 +13251,7 @@ class basic_json
             array = create<array_t>(std::move(value));
         }
 
-        void destroy(value_t t)
+        void destroy(value_t t) noexcept
         {
             switch (t)
             {
@@ -8227,7 +13296,7 @@ class basic_json
     value is changed, because the invariant expresses a relationship between
     @a m_type and @a m_value.
     */
-    void assert_invariant() const
+    void assert_invariant() const noexcept
     {
         assert(m_type != value_t::object or m_value.object != nullptr);
         assert(m_type != value_t::array or m_value.array != nullptr);
@@ -8307,7 +13376,6 @@ class basic_json
     */
     using parser_callback_t = typename parser::parser_callback_t;
 
-
     //////////////////
     // constructors //
     //////////////////
@@ -8409,6 +13477,7 @@ class basic_json
     - @a CompatibleType is not derived from `std::istream`,
     - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
          constructors),
+    - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments)
     - @a CompatibleType is not a @ref basic_json nested type (e.g.,
          @ref json_pointer, @ref iterator, etc ...)
     - @ref @ref json_serializer<U> has a
@@ -8432,20 +13501,90 @@ class basic_json
 
     @since version 2.1.0
     */
-    template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>,
-             detail::enable_if_t<not std::is_base_of<std::istream, U>::value and
-                                 not std::is_same<U, basic_json_t>::value and
-                                 not detail::is_basic_json_nested_type<
-                                     basic_json_t, U>::value and
-                                 detail::has_to_json<basic_json, U>::value,
-                                 int> = 0>
-    basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json(
-                std::declval<basic_json_t&>(), std::forward<CompatibleType>(val))))
+    template <typename CompatibleType,
+              typename U = detail::uncvref_t<CompatibleType>,
+              detail::enable_if_t<
+                  not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0>
+    basic_json(CompatibleType && val) noexcept(noexcept(
+                JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
+                                           std::forward<CompatibleType>(val))))
     {
         JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
         assert_invariant();
     }
 
+    /*!
+    @brief create a JSON value from an existing one
+
+    This is a constructor for existing @ref basic_json types.
+    It does not hijack copy/move constructors, since the parameter has different
+    template arguments than the current ones.
+
+    The constructor tries to convert the internal @ref m_value of the parameter.
+
+    @tparam BasicJsonType a type such that:
+    - @a BasicJsonType is a @ref basic_json type.
+    - @a BasicJsonType has different template arguments than @ref basic_json_t.
+
+    @param[in] val the @ref basic_json value to be converted.
+
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
+
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
+
+    @since version 3.2.0
+    */
+    template <typename BasicJsonType,
+              detail::enable_if_t<
+                  detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0>
+    basic_json(const BasicJsonType& val)
+    {
+        using other_boolean_t = typename BasicJsonType::boolean_t;
+        using other_number_float_t = typename BasicJsonType::number_float_t;
+        using other_number_integer_t = typename BasicJsonType::number_integer_t;
+        using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+        using other_string_t = typename BasicJsonType::string_t;
+        using other_object_t = typename BasicJsonType::object_t;
+        using other_array_t = typename BasicJsonType::array_t;
+
+        switch (val.type())
+        {
+            case value_t::boolean:
+                JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());
+                break;
+            case value_t::number_float:
+                JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());
+                break;
+            case value_t::number_integer:
+                JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());
+                break;
+            case value_t::number_unsigned:
+                JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());
+                break;
+            case value_t::string:
+                JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());
+                break;
+            case value_t::object:
+                JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());
+                break;
+            case value_t::array:
+                JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());
+                break;
+            case value_t::null:
+                *this = nullptr;
+                break;
+            case value_t::discarded:
+                m_type = value_t::discarded;
+                break;
+        }
+        assert_invariant();
+    }
+
     /*!
     @brief create a container (array or object) from an initializer list
 
@@ -8718,7 +13857,7 @@ class basic_json
     @warning A precondition is enforced with a runtime assertion that will
              result in calling `std::abort` if this precondition is not met.
              Assertions can be disabled by defining `NDEBUG` at compile time.
-             See http://en.cppreference.com/w/cpp/error/assert for more
+             See https://en.cppreference.com/w/cpp/error/assert for more
              information.
 
     @throw invalid_iterator.201 if iterators @a first and @a last are not
@@ -8858,7 +13997,7 @@ class basic_json
     changes to any JSON value.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
     - As postcondition, it holds: `other == basic_json(other)`.
@@ -8943,7 +14082,7 @@ class basic_json
     exceptions.
 
     @requirement This function helps `basic_json` satisfying the
-    [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible)
+    [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible)
     requirements.
 
     @liveexample{The code below shows the move constructor explicitly called
@@ -8977,7 +14116,7 @@ class basic_json
     @complexity Linear.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
 
@@ -8988,7 +14127,7 @@ class basic_json
 
     @since version 1.0.0
     */
-    reference& operator=(basic_json other) noexcept (
+    basic_json& operator=(basic_json other) noexcept (
         std::is_nothrow_move_constructible<value_t>::value and
         std::is_nothrow_move_assignable<value_t>::value and
         std::is_nothrow_move_constructible<json_value>::value and
@@ -9014,14 +14153,14 @@ class basic_json
     @complexity Linear.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is linear.
     - All stored elements are destroyed and all memory is freed.
 
     @since version 1.0.0
     */
-    ~basic_json()
+    ~basic_json() noexcept
     {
         assert_invariant();
         m_value.destroy(m_type);
@@ -9054,6 +14193,10 @@ class basic_json
     @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
     in the output are escaped with `\uXXXX` sequences, and the result consists
     of ASCII characters only.
+    @param[in] error_handler  how to react on decoding errors; there are three
+    possible values: `strict` (throws and exception in case a decoding error
+    occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
+    and `ignore` (ignore invalid UTF-8 sequences during serialization).
 
     @return string containing the serialization of the JSON value
 
@@ -9072,13 +14215,16 @@ class basic_json
     @see https://docs.python.org/2/library/json.html#json.dump
 
     @since version 1.0.0; indentation character @a indent_char, option
-           @a ensure_ascii and exceptions added in version 3.0.0
+           @a ensure_ascii and exceptions added in version 3.0.0; error
+           handlers added in version 3.4.0.
     */
-    string_t dump(const int indent = -1, const char indent_char = ' ',
-                  const bool ensure_ascii = false) const
+    string_t dump(const int indent = -1,
+                  const char indent_char = ' ',
+                  const bool ensure_ascii = false,
+                  const error_handler_t error_handler = error_handler_t::strict) const
     {
         string_t result;
-        serializer s(detail::output_adapter<char>(result), indent_char);
+        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
 
         if (indent >= 0)
         {
@@ -9618,12 +14764,35 @@ class basic_json
         return *this;
     }
 
+    /*!
+    @brief get special-case overload
+
+    This overloads converts the current @ref basic_json in a different
+    @ref basic_json type
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this, converted into @tparam BasicJsonType
+
+    @complexity Depending on the implementation of the called `from_json()`
+                method.
+
+    @since version 3.2.0
+    */
+    template<typename BasicJsonType, detail::enable_if_t<
+                 not std::is_same<BasicJsonType, basic_json>::value and
+                 detail::is_basic_json<BasicJsonType>::value, int> = 0>
+    BasicJsonType get() const
+    {
+        return *this;
+    }
+
     /*!
     @brief get a value (explicit)
 
     Explicit type conversion between the JSON value and a compatible value
-    which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
-    and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
     The value is converted by calling the @ref json_serializer<ValueType>
     `from_json()` method.
 
@@ -9659,7 +14828,7 @@ class basic_json
     */
     template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
              detail::enable_if_t <
-                 not std::is_same<basic_json_t, ValueType>::value and
+                 not detail::is_basic_json<ValueType>::value and
                  detail::has_from_json<basic_json_t, ValueType>::value and
                  not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
                  int> = 0>
@@ -9683,8 +14852,8 @@ class basic_json
     @brief get a value (explicit); special case
 
     Explicit type conversion between the JSON value and a compatible value
-    which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
-    and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
+    and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
     The value is converted by calling the @ref json_serializer<ValueType>
     `from_json()` method.
 
@@ -9723,17 +14892,64 @@ class basic_json
     }
 
     /*!
-    @brief get a pointer value (explicit)
+    @brief get a value (explicit)
 
-    Explicit pointer access to the internally stored JSON value. No copies are
+    Explicit type conversion between the JSON value and a compatible value.
+    The value is filled into the input parameter by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType v;
+    JSONSerializer<ValueType>::from_json(*this, v);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const basic_json&, ValueType&)`, and
+
+    @tparam ValueType the input parameter type.
+
+    @return the input parameter, allowing chaining calls.
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @liveexample{The example below shows several conversions from JSON values
+    to other types. There a few things to note: (1) Floating-point numbers can
+    be converted to integers\, (2) A JSON array can be converted to a standard
+    `std::vector<short>`\, (3) A JSON object can be converted to C++
+    associative containers such as `std::unordered_map<std::string\,
+    json>`.,get_to}
+
+    @since version 3.3.0
+    */
+    template<typename ValueType,
+             detail::enable_if_t <
+                 not detail::is_basic_json<ValueType>::value and
+                 detail::has_from_json<basic_json_t, ValueType>::value,
+                 int> = 0>
+    ValueType & get_to(ValueType& v) const noexcept(noexcept(
+                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))
+    {
+        JSONSerializer<ValueType>::from_json(*this, v);
+        return v;
+    }
+
+
+    /*!
+    @brief get a pointer value (implicit)
+
+    Implicit pointer access to the internally stored JSON value. No copies are
     made.
 
-    @warning The pointer becomes invalid if the underlying JSON object
-    changes.
+    @warning Writing data to the pointee of the result yields an undefined
+    state.
 
     @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
     object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t.
+    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
+    assertion.
 
     @return pointer to the internally stored JSON value if the requested
     pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
@@ -9743,45 +14959,43 @@ class basic_json
     @liveexample{The example below shows how pointers to internal values of a
     JSON value can be requested. Note that no type conversions are made and a
     `nullptr` is returned if the value and the requested pointer type does not
-    match.,get__PointerType}
-
-    @sa @ref get_ptr() for explicit pointer-member access
+    match.,get_ptr}
 
     @since version 1.0.0
     */
     template<typename PointerType, typename std::enable_if<
                  std::is_pointer<PointerType>::value, int>::type = 0>
-    PointerType get() noexcept
+    auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
     {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
+        // delegate the call to get_impl_ptr<>()
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
     }
 
     /*!
-    @brief get a pointer value (explicit)
-    @copydoc get()
+    @brief get a pointer value (implicit)
+    @copydoc get_ptr()
     */
     template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value, int>::type = 0>
-    constexpr const PointerType get() const noexcept
+                 std::is_pointer<PointerType>::value and
+                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
+    constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
     {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
+        // delegate the call to get_impl_ptr<>() const
+        return get_impl_ptr(static_cast<PointerType>(nullptr));
     }
 
     /*!
-    @brief get a pointer value (implicit)
+    @brief get a pointer value (explicit)
 
-    Implicit pointer access to the internally stored JSON value. No copies are
+    Explicit pointer access to the internally stored JSON value. No copies are
     made.
 
-    @warning Writing data to the pointee of the result yields an undefined
-    state.
+    @warning The pointer becomes invalid if the underlying JSON object
+    changes.
 
     @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
     object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
-    assertion.
+    @ref number_unsigned_t, or @ref number_float_t.
 
     @return pointer to the internally stored JSON value if the requested
     pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
@@ -9791,59 +15005,30 @@ class basic_json
     @liveexample{The example below shows how pointers to internal values of a
     JSON value can be requested. Note that no type conversions are made and a
     `nullptr` is returned if the value and the requested pointer type does not
-    match.,get_ptr}
+    match.,get__PointerType}
+
+    @sa @ref get_ptr() for explicit pointer-member access
 
     @since version 1.0.0
     */
     template<typename PointerType, typename std::enable_if<
                  std::is_pointer<PointerType>::value, int>::type = 0>
-    PointerType get_ptr() noexcept
+    auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())
     {
-        // get the type of the PointerType (remove pointer and const)
-        using pointee_t = typename std::remove_const<typename
-                          std::remove_pointer<typename
-                          std::remove_const<PointerType>::type>::type>::type;
-        // make sure the type matches the allowed types
-        static_assert(
-            std::is_same<object_t, pointee_t>::value
-            or std::is_same<array_t, pointee_t>::value
-            or std::is_same<string_t, pointee_t>::value
-            or std::is_same<boolean_t, pointee_t>::value
-            or std::is_same<number_integer_t, pointee_t>::value
-            or std::is_same<number_unsigned_t, pointee_t>::value
-            or std::is_same<number_float_t, pointee_t>::value
-            , "incompatible pointer type");
-
-        // delegate the call to get_impl_ptr<>()
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
     }
 
     /*!
-    @brief get a pointer value (implicit)
-    @copydoc get_ptr()
+    @brief get a pointer value (explicit)
+    @copydoc get()
     */
     template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value and
-                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
-    constexpr const PointerType get_ptr() const noexcept
+                 std::is_pointer<PointerType>::value, int>::type = 0>
+    constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())
     {
-        // get the type of the PointerType (remove pointer and const)
-        using pointee_t = typename std::remove_const<typename
-                          std::remove_pointer<typename
-                          std::remove_const<PointerType>::type>::type>::type;
-        // make sure the type matches the allowed types
-        static_assert(
-            std::is_same<object_t, pointee_t>::value
-            or std::is_same<array_t, pointee_t>::value
-            or std::is_same<string_t, pointee_t>::value
-            or std::is_same<boolean_t, pointee_t>::value
-            or std::is_same<number_integer_t, pointee_t>::value
-            or std::is_same<number_unsigned_t, pointee_t>::value
-            or std::is_same<number_float_t, pointee_t>::value
-            , "incompatible pointer type");
-
-        // delegate the call to get_impl_ptr<>() const
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
+        // delegate the call to get_ptr
+        return get_ptr<PointerType>();
     }
 
     /*!
@@ -9925,13 +15110,16 @@ class basic_json
     template < typename ValueType, typename std::enable_if <
                    not std::is_pointer<ValueType>::value and
                    not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
-                   not std::is_same<ValueType, typename string_t::value_type>::value
+                   not std::is_same<ValueType, typename string_t::value_type>::value and
+                   not detail::is_basic_json<ValueType>::value
+
 #ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
                    and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
-#endif
-#if defined(JSON_HAS_CPP_17)
+#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914
                    and not std::is_same<ValueType, typename std::string_view>::value
 #endif
+#endif
+                   and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value
                    , int >::type = 0 >
     operator ValueType() const
     {
@@ -10195,7 +15383,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10225,7 +15413,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10271,7 +15459,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10313,7 +15501,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10360,7 +15548,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10403,7 +15591,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
     }
 
     /*!
@@ -10476,7 +15664,7 @@ class basic_json
 
     /*!
     @brief overload for a default value of type const char*
-    @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const
+    @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const
     */
     string_t value(const typename object_t::key_type& key, const char* default_value) const
     {
@@ -10512,7 +15700,7 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw type_error.306 if the JSON value is not an objec; in that case,
+    @throw type_error.306 if the JSON value is not an object; in that case,
     using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
@@ -10536,7 +15724,7 @@ class basic_json
             {
                 return ptr.get_checked(this);
             }
-            JSON_CATCH (out_of_range&)
+            JSON_INTERNAL_CATCH (out_of_range&)
             {
                 return default_value;
             }
@@ -11047,7 +16235,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
 
@@ -11086,7 +16274,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).begin()`.
@@ -11118,7 +16306,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
 
@@ -11157,7 +16345,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).end()`.
@@ -11187,7 +16375,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `reverse_iterator(end())`.
@@ -11224,7 +16412,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `reverse_iterator(begin())`.
@@ -11261,7 +16449,7 @@ class basic_json
     @complexity Constant.
 
     @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
     requirements:
     - The complexity is constant.
     - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`.
@@ -11279,45 +16467,117 @@ class basic_json
         return const_reverse_iterator(cend());
     }
 
-    /*!
-    @brief returns a const reverse iterator to one before the first
+    /*!
+    @brief returns a const reverse iterator to one before the first
+
+    Returns a const reverse iterator to the reverse-end; that is, one before
+    the first element.
+
+    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+
+    @complexity Constant.
+
+    @requirement This function helps `basic_json` satisfying the
+    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
+    requirements:
+    - The complexity is constant.
+    - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
+
+    @liveexample{The following code shows an example for `crend()`.,crend}
+
+    @sa @ref rend() -- returns a reverse iterator to the end
+    @sa @ref rbegin() -- returns a reverse iterator to the beginning
+    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+
+    @since version 1.0.0
+    */
+    const_reverse_iterator crend() const noexcept
+    {
+        return const_reverse_iterator(cbegin());
+    }
+
+  public:
+    /*!
+    @brief wrapper to access iterator member functions in range-based for
+
+    This function allows to access @ref iterator::key() and @ref
+    iterator::value() during range-based for loops. In these loops, a
+    reference to the JSON values is returned, so there is no access to the
+    underlying iterator.
+
+    For loop without iterator_wrapper:
+
+    @code{cpp}
+    for (auto it = j_object.begin(); it != j_object.end(); ++it)
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    Range-based for loop without iterator proxy:
+
+    @code{cpp}
+    for (auto it : j_object)
+    {
+        // "it" is of type json::reference and has no key() member
+        std::cout << "value: " << it << '\n';
+    }
+    @endcode
+
+    Range-based for loop with iterator proxy:
+
+    @code{cpp}
+    for (auto it : json::iterator_wrapper(j_object))
+    {
+        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+    }
+    @endcode
+
+    @note When iterating over an array, `key()` will return the index of the
+          element as string (see example).
 
-    Returns a const reverse iterator to the reverse-end; that is, one before
-    the first element.
+    @param[in] ref  reference to a JSON value
+    @return iteration proxy object wrapping @a ref with an interface to use in
+            range-based for loops
 
-    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
+    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
 
-    @complexity Constant.
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
 
-    @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
+    @complexity Constant.
 
-    @liveexample{The following code shows an example for `crend()`.,crend}
+    @note The name of this function is not yet final and may change in the
+    future.
 
-    @sa @ref rend() -- returns a reverse iterator to the end
-    @sa @ref rbegin() -- returns a reverse iterator to the beginning
-    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use @ref items() instead;
+                that is, replace `json::iterator_wrapper(j)` with `j.items()`.
+    */
+    JSON_DEPRECATED
+    static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept
+    {
+        return ref.items();
+    }
 
-    @since version 1.0.0
+    /*!
+    @copydoc iterator_wrapper(reference)
     */
-    const_reverse_iterator crend() const noexcept
+    JSON_DEPRECATED
+    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept
     {
-        return const_reverse_iterator(cbegin());
+        return ref.items();
     }
 
-  public:
     /*!
-    @brief wrapper to access iterator member functions in range-based for
+    @brief helper to access iterator member functions in range-based for
 
     This function allows to access @ref iterator::key() and @ref
     iterator::value() during range-based for loops. In these loops, a
     reference to the JSON values is returned, so there is no access to the
     underlying iterator.
 
-    For loop without iterator_wrapper:
+    For loop without `items()` function:
 
     @code{cpp}
     for (auto it = j_object.begin(); it != j_object.end(); ++it)
@@ -11326,7 +16586,7 @@ class basic_json
     }
     @endcode
 
-    Range-based for loop without iterator proxy:
+    Range-based for loop without `items()` function:
 
     @code{cpp}
     for (auto it : j_object)
@@ -11336,43 +16596,42 @@ class basic_json
     }
     @endcode
 
-    Range-based for loop with iterator proxy:
+    Range-based for loop with `items()` function:
 
     @code{cpp}
-    for (auto it : json::iterator_wrapper(j_object))
+    for (auto it : j_object.items())
     {
         std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
     }
     @endcode
 
     @note When iterating over an array, `key()` will return the index of the
-          element as string (see example).
+          element as string (see example). For primitive types (e.g., numbers),
+          `key()` returns an empty string.
 
-    @param[in] ref  reference to a JSON value
     @return iteration proxy object wrapping @a ref with an interface to use in
             range-based for loops
 
-    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
+    @liveexample{The following code shows how the function is used.,items}
 
     @exceptionsafety Strong guarantee: if an exception is thrown, there are no
     changes in the JSON value.
 
     @complexity Constant.
 
-    @note The name of this function is not yet final and may change in the
-    future.
+    @since version 3.1.0.
     */
-    static iteration_proxy<iterator> iterator_wrapper(reference ref)
+    iteration_proxy<iterator> items() noexcept
     {
-        return iteration_proxy<iterator>(ref);
+        return iteration_proxy<iterator>(*this);
     }
 
     /*!
-    @copydoc iterator_wrapper(reference)
+    @copydoc items()
     */
-    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref)
+    iteration_proxy<const_iterator> items() const noexcept
     {
-        return iteration_proxy<const_iterator>(ref);
+        return iteration_proxy<const_iterator>(*this);
     }
 
     /// @}
@@ -11417,7 +16676,7 @@ class basic_json
     false in the case of a string.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `begin() == end()`.
@@ -11488,7 +16747,7 @@ class basic_json
     the case of a string.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `std::distance(begin(), end())`.
@@ -11558,7 +16817,7 @@ class basic_json
     @exceptionsafety No-throw guarantee: this function never throws exceptions.
 
     @requirement This function helps `basic_json` satisfying the
-    [Container](http://en.cppreference.com/w/cpp/concept/Container)
+    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of returning `b.size()` where `b` is the largest
@@ -11970,6 +17229,26 @@ class basic_json
         return {it, res.second};
     }
 
+    /// Helper for insertion of an iterator
+    /// @note: This uses std::distance to support GCC 4.8,
+    ///        see https://github.com/nlohmann/json/pull/1257
+    template<typename... Args>
+    iterator insert_iterator(const_iterator pos, Args&& ... args)
+    {
+        iterator result(this);
+        assert(m_value.array != nullptr);
+
+        auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);
+        m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);
+        result.m_it.array_iterator = m_value.array->begin() + insert_pos;
+
+        // This could have been written as:
+        // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
+        // but the return value of insert is missing in GCC 4.8, so it is written this way instead.
+
+        return result;
+    }
+
     /*!
     @brief inserts element
 
@@ -12004,9 +17283,7 @@ class basic_json
             }
 
             // insert to array and return iterator
-            iterator result(this);
-            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val);
-            return result;
+            return insert_iterator(pos, val);
         }
 
         JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
@@ -12057,9 +17334,7 @@ class basic_json
             }
 
             // insert to array and return iterator
-            iterator result(this);
-            result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
-            return result;
+            return insert_iterator(pos, cnt, val);
         }
 
         JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
@@ -12121,12 +17396,7 @@ class basic_json
         }
 
         // insert to array and return iterator
-        iterator result(this);
-        result.m_it.array_iterator = m_value.array->insert(
-                                         pos.m_it.array_iterator,
-                                         first.m_it.array_iterator,
-                                         last.m_it.array_iterator);
-        return result;
+        return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
     }
 
     /*!
@@ -12168,9 +17438,7 @@ class basic_json
         }
 
         // insert to array and return iterator
-        iterator result(this);
-        result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end());
-        return result;
+        return insert_iterator(pos, ilist.begin(), ilist.end());
     }
 
     /*!
@@ -12312,7 +17580,7 @@ class basic_json
 
         // passed iterators must belong to objects
         if (JSON_UNLIKELY(not first.m_object->is_object()
-                          or not first.m_object->is_object()))
+                          or not last.m_object->is_object()))
         {
             JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
         }
@@ -12943,8 +18211,8 @@ class basic_json
 
     /*!
     @brief serialize to stream
-    @deprecated This stream operator is deprecated and will be removed in a
-                future version of the library. Please use
+    @deprecated This stream operator is deprecated and will be removed in
+                future 4.0.0 of the library. Please use
                 @ref operator<<(std::ostream&, const basic_json&)
                 instead; that is, replace calls like `j >> o;` with `o << j;`.
     @since version 1.0.0; deprecated since version 3.0.0
@@ -12999,6 +18267,8 @@ class basic_json
     @param[in] cb  a parser callback function of type @ref parser_callback_t
     which is used to control the deserialization by filtering unwanted values
     (optional)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
 
     @return result of the deserialization
 
@@ -13027,19 +18297,7 @@ class basic_json
 
     @since version 2.0.3 (contiguous containers)
     */
-    static basic_json parse(detail::input_adapter i,
-                            const parser_callback_t cb = nullptr,
-                            const bool allow_exceptions = true)
-    {
-        basic_json result;
-        parser(i, cb, allow_exceptions).parse(true, result);
-        return result;
-    }
-
-    /*!
-    @copydoc basic_json parse(detail::input_adapter, const parser_callback_t)
-    */
-    static basic_json parse(detail::input_adapter& i,
+    static basic_json parse(detail::input_adapter&& i,
                             const parser_callback_t cb = nullptr,
                             const bool allow_exceptions = true)
     {
@@ -13048,14 +18306,80 @@ class basic_json
         return result;
     }
 
-    static bool accept(detail::input_adapter i)
+    static bool accept(detail::input_adapter&& i)
     {
         return parser(i).accept(true);
     }
 
-    static bool accept(detail::input_adapter& i)
+    /*!
+    @brief generate SAX events
+
+    The SAX event lister must follow the interface of @ref json_sax.
+
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
+
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
+    @param[in,out] sax  SAX event listener
+    @param[in] format  the format to parse (JSON, CBOR, MessagePack, or UBJSON)
+    @param[in] strict  whether the input has to be consumed completely
+
+    @return return value of the last processed SAX event
+
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
+    @complexity Linear in the length of the input. The parser is a predictive
+    LL(1) parser. The complexity can be higher if the SAX consumer @a sax has
+    a super-linear complexity.
+
+    @note A UTF-8 byte order mark is silently ignored.
+
+    @liveexample{The example below demonstrates the `sax_parse()` function
+    reading from string and processing the events with a user-defined SAX
+    event consumer.,sax_parse}
+
+    @since version 3.2.0
+    */
+    template <typename SAX>
+    static bool sax_parse(detail::input_adapter&& i, SAX* sax,
+                          input_format_t format = input_format_t::json,
+                          const bool strict = true)
     {
-        return parser(i).accept(true);
+        assert(sax);
+        switch (format)
+        {
+            case input_format_t::json:
+                return parser(std::move(i)).sax_parse(sax, strict);
+            default:
+                return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict);
+        }
     }
 
     /*!
@@ -13127,10 +18451,19 @@ class basic_json
         return parser(detail::input_adapter(first, last)).accept(true);
     }
 
+    template<class IteratorType, class SAX, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool sax_parse(IteratorType first, IteratorType last, SAX* sax)
+    {
+        return parser(detail::input_adapter(first, last)).sax_parse(sax);
+    }
+
     /*!
     @brief deserialize from stream
-    @deprecated This stream operator is deprecated and will be removed in a
-                future version of the library. Please use
+    @deprecated This stream operator is deprecated and will be removed in
+                version 4.0.0 of the library. Please use
                 @ref operator>>(std::istream&, basic_json&)
                 instead; that is, replace calls like `j << i;` with `i >> j;`.
     @since version 1.0.0; deprecated since version 3.0.0
@@ -13331,9 +18664,11 @@ class basic_json
     vector in CBOR format.,to_cbor}
 
     @sa http://cbor.io
-    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
         analogous deserialization
     @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
 
     @since version 2.0.9
     */
@@ -13426,9 +18761,10 @@ class basic_json
     vector in MessagePack format.,to_msgpack}
 
     @sa http://msgpack.org
-    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
-        analogous deserialization
+    @sa @ref from_msgpack for the analogous deserialization
     @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
 
     @since version 2.0.9
     */
@@ -13449,6 +18785,192 @@ class basic_json
         binary_writer<char>(o).write_msgpack(j);
     }
 
+    /*!
+    @brief create a UBJSON serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the UBJSON
+    (Universal Binary JSON) serialization format. UBJSON aims to be more compact
+    than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    UBJSON types according to the UBJSON specification:
+
+    JSON value type | value/range                       | UBJSON type | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | `Z`
+    boolean         | `true`                            | true        | `T`
+    boolean         | `false`                           | false       | `F`
+    number_integer  | -9223372036854775808..-2147483649 | int64       | `L`
+    number_integer  | -2147483648..-32769               | int32       | `l`
+    number_integer  | -32768..-129                      | int16       | `I`
+    number_integer  | -128..127                         | int8        | `i`
+    number_integer  | 128..255                          | uint8       | `U`
+    number_integer  | 256..32767                        | int16       | `I`
+    number_integer  | 32768..2147483647                 | int32       | `l`
+    number_integer  | 2147483648..9223372036854775807   | int64       | `L`
+    number_unsigned | 0..127                            | int8        | `i`
+    number_unsigned | 128..255                          | uint8       | `U`
+    number_unsigned | 256..32767                        | int16       | `I`
+    number_unsigned | 32768..2147483647                 | int32       | `l`
+    number_unsigned | 2147483648..9223372036854775807   | int64       | `L`
+    number_float    | *any value*                       | float64     | `D`
+    string          | *with shortest length indicator*  | string      | `S`
+    array           | *see notes on optimized format*   | array       | `[`
+    object          | *see notes on optimized format*   | map         | `{`
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a UBJSON value.
+
+    @note The following values can **not** be converted to a UBJSON value:
+          - strings with more than 9223372036854775807 bytes (theoretical)
+          - unsigned integer numbers above 9223372036854775807
+
+    @note The following markers are not used in the conversion:
+          - `Z`: no-op values are not created.
+          - `C`: single-byte strings are serialized with `S` markers.
+
+    @note Any UBJSON output created @ref to_ubjson can be successfully parsed
+          by @ref from_ubjson.
+
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
+    @note The optimized formats for containers are supported: Parameter
+          @a use_size adds size information to the beginning of a container and
+          removes the closing marker. Parameter @a use_type further checks
+          whether all elements of a container have the same type and adds the
+          type marker to the beginning of the container. The @a use_type
+          parameter must only be used together with @a use_size = true. Note
+          that @a use_size = true alone may result in larger representations -
+          the benefit of this parameter is that the receiving side is
+          immediately informed on the number of elements of the container.
+
+    @param[in] j  JSON value to serialize
+    @param[in] use_size  whether to add size annotations to container types
+    @param[in] use_type  whether to add type annotations to container types
+                         (must be combined with @a use_size = true)
+    @return UBJSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in UBJSON format.,to_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+
+    @since version 3.1.0
+    */
+    static std::vector<uint8_t> to_ubjson(const basic_json& j,
+                                          const bool use_size = false,
+                                          const bool use_type = false)
+    {
+        std::vector<uint8_t> result;
+        to_ubjson(j, result, use_size, use_type);
+        return result;
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type);
+    }
+
+    static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
+                          const bool use_size = false, const bool use_type = false)
+    {
+        binary_writer<char>(o).write_ubjson(j, use_size, use_type);
+    }
+
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and returns a vector
+           containing the corresponding BSON-representation.
+
+    BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are
+    stored as a single entity (a so-called document).
+
+    The library uses the following mapping from JSON values types to BSON types:
+
+    JSON value type | value/range                       | BSON type   | marker
+    --------------- | --------------------------------- | ----------- | ------
+    null            | `null`                            | null        | 0x0A
+    boolean         | `true`, `false`                   | boolean     | 0x08
+    number_integer  | -9223372036854775808..-2147483649 | int64       | 0x12
+    number_integer  | -2147483648..2147483647           | int32       | 0x10
+    number_integer  | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 0..2147483647                     | int32       | 0x10
+    number_unsigned | 2147483648..9223372036854775807   | int64       | 0x12
+    number_unsigned | 9223372036854775808..18446744073709551615| --   | --
+    number_float    | *any value*                       | double      | 0x01
+    string          | *any value*                       | string      | 0x02
+    array           | *any value*                       | document    | 0x04
+    object          | *any value*                       | document    | 0x03
+
+    @warning The mapping is **incomplete**, since only JSON-objects (and things
+    contained therein) can be serialized to BSON.
+    Also, integers larger than 9223372036854775807 cannot be serialized to BSON,
+    and the keys may not contain U+0000, since they are serialized a
+    zero-terminated c-strings.
+
+    @throw out_of_range.407  if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807`
+    @throw out_of_range.409  if a key in `j` contains a NULL (U+0000)
+    @throw type_error.317    if `!j.is_object()`
+
+    @pre The input `j` is required to be an object: `j.is_object() == true`.
+
+    @note Any BSON output created via @ref to_bson can be successfully parsed
+          by @ref from_bson.
+
+    @param[in] j  JSON value to serialize
+    @return BSON serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in BSON format.,to_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the
+        analogous deserialization
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             related UBJSON format
+    @sa @ref to_cbor(const basic_json&) for the related CBOR format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
+    */
+    static std::vector<uint8_t> to_bson(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        to_bson(j, result);
+        return result;
+    }
+
+    /*!
+    @brief Serializes the given JSON object `j` to BSON and forwards the
+           corresponding BSON-representation to the given output_adapter `o`.
+    @param j The JSON object to convert to BSON.
+    @param o The output adapter that receives the binary BSON representation.
+    @pre The input `j` shall be an object: `j.is_object() == true`
+    @sa @ref to_bson(const basic_json&)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_bson(j);
+    }
+
+    /*!
+    @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>)
+    */
+    static void to_bson(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_bson(j);
+    }
+
+
     /*!
     @brief create a JSON value from an input in CBOR format
 
@@ -13490,7 +19012,7 @@ class basic_json
     map                    | object          | 0xBF
     False                  | `false`         | 0xF4
     True                   | `true`          | 0xF5
-    Nill                   | `null`          | 0xF6
+    Null                   | `null`          | 0xF6
     Half-Precision Float   | number_float    | 0xF9
     Single-Precision Float | number_float    | 0xFA
     Double-Precision Float | number_float    | 0xFB
@@ -13518,6 +19040,9 @@ class basic_json
     @param[in] i  an input in CBOR format convertible to an input adapter
     @param[in] strict  whether to expect the input to be consumed until EOF
                        (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
     @return deserialized JSON value
 
     @throw parse_error.110 if the given input ends prematurely or the end of
@@ -13533,27 +19058,39 @@ class basic_json
 
     @sa http://cbor.io
     @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(detail::input_adapter, const bool) for the
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the
         related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
 
     @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
            consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
     */
-    static basic_json from_cbor(detail::input_adapter i,
-                                const bool strict = true)
+    static basic_json from_cbor(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(i).parse_cbor(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
-    @copydoc from_cbor(detail::input_adapter, const bool)
+    @copydoc from_cbor(detail::input_adapter&&, const bool, const bool)
     */
     template<typename A1, typename A2,
              detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true)
+    static basic_json from_cbor(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
@@ -13606,6 +19143,10 @@ class basic_json
                   adapter
     @param[in] strict  whether to expect the input to be consumed until EOF
                        (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
 
     @throw parse_error.110 if the given input ends prematurely or the end of
     file was not reached when @a strict was set to true
@@ -13620,29 +19161,212 @@ class basic_json
 
     @sa http://msgpack.org
     @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR
-        format
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for
+        the related UBJSON format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
 
     @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
            consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0
+           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
+           since 3.2.0
+    */
+    static basic_json from_msgpack(detail::input_adapter&& i,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_msgpack(A1 && a1, A2 && a2,
+                                   const bool strict = true,
+                                   const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief create a JSON value from an input in UBJSON format
+
+    Deserializes a given input @a i to a JSON value using the UBJSON (Universal
+    Binary JSON) serialization format.
+
+    The library maps UBJSON types to JSON value types as follows:
+
+    UBJSON type | JSON value type                         | marker
+    ----------- | --------------------------------------- | ------
+    no-op       | *no value, next value is read*          | `N`
+    null        | `null`                                  | `Z`
+    false       | `false`                                 | `F`
+    true        | `true`                                  | `T`
+    float32     | number_float                            | `d`
+    float64     | number_float                            | `D`
+    uint8       | number_unsigned                         | `U`
+    int8        | number_integer                          | `i`
+    int16       | number_integer                          | `I`
+    int32       | number_integer                          | `l`
+    int64       | number_integer                          | `L`
+    string      | string                                  | `S`
+    char        | string                                  | `C`
+    array       | array (optimized values are supported)  | `[`
+    object      | object (optimized values are supported) | `{`
+
+    @note The mapping is **complete** in the sense that any UBJSON value can
+          be converted to a JSON value.
+
+    @param[in] i  an input in UBJSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
+    @throw parse_error.112 if a parse error occurs
+    @throw parse_error.113 if a string could not be parsed successfully
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    UBJSON format to a JSON value.,from_ubjson}
+
+    @sa http://ubjson.org
+    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
+             analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
+        the related BSON format
+
+    @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0
+    */
+    static basic_json from_ubjson(detail::input_adapter&& i,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_ubjson(A1 && a1, A2 && a2,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
+    }
+
+    /*!
+    @brief Create a JSON value from an input in BSON format
+
+    Deserializes a given input @a i to a JSON value using the BSON (Binary JSON)
+    serialization format.
+
+    The library maps BSON record types to JSON value types as follows:
+
+    BSON type       | BSON marker byte | JSON value type
+    --------------- | ---------------- | ---------------------------
+    double          | 0x01             | number_float
+    string          | 0x02             | string
+    document        | 0x03             | object
+    array           | 0x04             | array
+    binary          | 0x05             | still unsupported
+    undefined       | 0x06             | still unsupported
+    ObjectId        | 0x07             | still unsupported
+    boolean         | 0x08             | boolean
+    UTC Date-Time   | 0x09             | still unsupported
+    null            | 0x0A             | null
+    Regular Expr.   | 0x0B             | still unsupported
+    DB Pointer      | 0x0C             | still unsupported
+    JavaScript Code | 0x0D             | still unsupported
+    Symbol          | 0x0E             | still unsupported
+    JavaScript Code | 0x0F             | still unsupported
+    int32           | 0x10             | number_integer
+    Timestamp       | 0x11             | still unsupported
+    128-bit decimal float | 0x13       | still unsupported
+    Max Key         | 0x7F             | still unsupported
+    Min Key         | 0xFF             | still unsupported
+
+    @warning The mapping is **incomplete**. The unsupported mappings
+             are indicated in the table above.
+
+    @param[in] i  an input in BSON format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
+    @param[in] allow_exceptions  whether to throw exceptions in case of a
+    parse error (optional, true by default)
+
+    @return deserialized JSON value
+
+    @throw parse_error.114 if an unsupported BSON record type is encountered
+
+    @complexity Linear in the size of the input @a i.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    BSON format to a JSON value.,from_bson}
+
+    @sa http://bsonspec.org/spec.html
+    @sa @ref to_bson(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
+        related CBOR format
+    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
+        the related MessagePack format
+    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
+        related UBJSON format
     */
-    static basic_json from_msgpack(detail::input_adapter i,
-                                   const bool strict = true)
+    static basic_json from_bson(detail::input_adapter&& i,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(i).parse_msgpack(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
     /*!
-    @copydoc from_msgpack(detail::input_adapter, const bool)
+    @copydoc from_bson(detail::input_adapter&&, const bool, const bool)
     */
     template<typename A1, typename A2,
              detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true)
+    static basic_json from_bson(A1 && a1, A2 && a2,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
     {
-        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict);
+        basic_json result;
+        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
+        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict);
+        return res ? result : basic_json(value_t::discarded);
     }
 
+
+
     /// @}
 
     //////////////////////////
@@ -14008,20 +19732,20 @@ class basic_json
                                 // avoid undefined behavior
                                 JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
                             }
-                            else
-                            {
-                                // default case: insert add offset
-                                parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
-                            }
+
+                            // default case: insert add offset
+                            parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
                         }
                         break;
                     }
 
+                    // LCOV_EXCL_START
                     default:
                     {
                         // if there exists a parent it cannot be primitive
-                        assert(false);  // LCOV_EXCL_LINE
+                        assert(false);
                     }
+                        // LCOV_EXCL_STOP
                 }
             }
         };
@@ -14066,7 +19790,7 @@ class basic_json
             // wrapper to get a value for an operation
             const auto get_value = [&val](const std::string & op,
                                           const std::string & member,
-                                          bool string_type) -> basic_json&
+                                          bool string_type) -> basic_json &
             {
                 // find value
                 auto it = val.m_value.object->find(member);
@@ -14163,7 +19887,7 @@ class basic_json
                         // the "path" location must exist - use at()
                         success = (result.at(ptr) == get_value("test", "value", false));
                     }
-                    JSON_CATCH (out_of_range&)
+                    JSON_INTERNAL_CATCH (out_of_range&)
                     {
                         // ignore out of range errors: success remains false
                     }
@@ -14216,6 +19940,7 @@ class basic_json
     diff for two JSON values.,diff}
 
     @sa @ref patch -- apply a JSON patch
+    @sa @ref merge_patch -- apply a JSON Merge Patch
 
     @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
 
@@ -14347,418 +20072,86 @@ class basic_json
     }
 
     /// @}
-};
-
-/////////////
-// presets //
-/////////////
-
-/*!
-@brief default JSON class
-
-This type is the default specialization of the @ref basic_json class which
-uses the standard template types.
-
-@since version 1.0.0
-*/
-using json = basic_json<>;
-
-//////////////////
-// json_pointer //
-//////////////////
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    auto result = &j;
-
-    // in case no reference tokens exist, return a reference to the JSON value
-    // j which will be overwritten by a primitive value
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (result->m_type)
-        {
-            case detail::value_t::null:
-            {
-                if (reference_token == "0")
-                {
-                    // start a new array if reference token is 0
-                    result = &result->operator[](0);
-                }
-                else
-                {
-                    // start a new object otherwise
-                    result = &result->operator[](reference_token);
-                }
-                break;
-            }
 
-            case detail::value_t::object:
-            {
-                // create an entry in the object
-                result = &result->operator[](reference_token);
-                break;
-            }
+    ////////////////////////////////
+    // JSON Merge Patch functions //
+    ////////////////////////////////
 
-            case detail::value_t::array:
-            {
-                // create an entry in the array
-                JSON_TRY
-                {
-                    result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
+    /// @name JSON Merge Patch functions
+    /// @{
 
-            /*
-            The following code is only reached if there exists a reference
-            token _and_ the current value is primitive. In this case, we have
-            an error situation, because primitive values may only occur as
-            single value; that is, with an empty list of reference tokens.
-            */
-            default:
-                JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
-        }
-    }
+    /*!
+    @brief applies a JSON Merge Patch
+
+    The merge patch format is primarily intended for use with the HTTP PATCH
+    method as a means of describing a set of modifications to a target
+    resource's content. This function applies a merge patch to the current
+    JSON value.
+
+    The function implements the following algorithm from Section 2 of
+    [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396):
+
+    ```
+    define MergePatch(Target, Patch):
+      if Patch is an Object:
+        if Target is not an Object:
+          Target = {} // Ignore the contents and set it to an empty Object
+        for each Name/Value pair in Patch:
+          if Value is null:
+            if Name exists in Target:
+              remove the Name/Value pair from Target
+          else:
+            Target[Name] = MergePatch(Target[Name], Value)
+        return Target
+      else:
+        return Patch
+    ```
+
+    Thereby, `Target` is the current object; that is, the patch is applied to
+    the current value.
+
+    @param[in] patch  the patch to apply
+
+    @complexity Linear in the lengths of @a patch.
+
+    @liveexample{The following code shows how a JSON Merge Patch is applied to
+    a JSON document.,merge_patch}
 
-    return *result;
-}
+    @sa @ref patch -- apply a JSON patch
+    @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
 
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
+    @since version 3.0.0
+    */
+    void merge_patch(const basic_json& patch)
     {
-        // convert null values to arrays or objects before continuing
-        if (ptr->m_type == detail::value_t::null)
+        if (patch.is_object())
         {
-            // check if reference token is a number
-            const bool nums =
-                std::all_of(reference_token.begin(), reference_token.end(),
-                            [](const char x)
+            if (not is_object())
             {
-                return (x >= '0' and x <= '9');
-            });
-
-            // change value to array for numbers or "-" or to object otherwise
-            *ptr = (nums or reference_token == "-")
-                   ? detail::value_t::array
-                   : detail::value_t::object;
-        }
-
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // use unchecked object access
-                ptr = &ptr->operator[](reference_token);
-                break;
+                *this = object();
             }
-
-            case detail::value_t::array:
+            for (auto it = patch.begin(); it != patch.end(); ++it)
             {
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                if (reference_token == "-")
+                if (it.value().is_null())
                 {
-                    // explicitly treat "-" as index beyond the end
-                    ptr = &ptr->operator[](ptr->m_value.array->size());
+                    erase(it.key());
                 }
                 else
                 {
-                    // convert array index to number; unchecked access
-                    JSON_TRY
-                    {
-                        ptr = &ptr->operator[](
-                            static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // note: at performs range check
-                ptr = &ptr->at(reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" always fails the range check
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // note: at performs range check
-                JSON_TRY
-                {
-                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-const NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // use unchecked object access
-                ptr = &ptr->operator[](reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" cannot be used for const access
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // use unchecked array access
-                JSON_TRY
-                {
-                    ptr = &ptr->operator[](
-                        static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-const NLOHMANN_BASIC_JSON_TPL&
-json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
-{
-    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
-    for (const auto& reference_token : reference_tokens)
-    {
-        switch (ptr->m_type)
-        {
-            case detail::value_t::object:
-            {
-                // note: at performs range check
-                ptr = &ptr->at(reference_token);
-                break;
-            }
-
-            case detail::value_t::array:
-            {
-                if (JSON_UNLIKELY(reference_token == "-"))
-                {
-                    // "-" always fails the range check
-                    JSON_THROW(detail::out_of_range::create(402,
-                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                }
-
-                // error condition (cf. RFC 6901, Sect. 4)
-                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                {
-                    JSON_THROW(detail::parse_error::create(106, 0,
-                                                           "array index '" + reference_token +
-                                                           "' must not begin with '0'"));
-                }
-
-                // note: at performs range check
-                JSON_TRY
-                {
-                    ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
-                }
-                JSON_CATCH(std::invalid_argument&)
-                {
-                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                }
-                break;
-            }
-
-            default:
-                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-        }
-    }
-
-    return *ptr;
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-void json_pointer::flatten(const std::string& reference_string,
-                           const NLOHMANN_BASIC_JSON_TPL& value,
-                           NLOHMANN_BASIC_JSON_TPL& result)
-{
-    switch (value.m_type)
-    {
-        case detail::value_t::array:
-        {
-            if (value.m_value.array->empty())
-            {
-                // flatten empty array as null
-                result[reference_string] = nullptr;
-            }
-            else
-            {
-                // iterate array and use index as reference string
-                for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
-                {
-                    flatten(reference_string + "/" + std::to_string(i),
-                            value.m_value.array->operator[](i), result);
-                }
-            }
-            break;
-        }
-
-        case detail::value_t::object:
-        {
-            if (value.m_value.object->empty())
-            {
-                // flatten empty object as null
-                result[reference_string] = nullptr;
-            }
-            else
-            {
-                // iterate object and use keys as reference string
-                for (const auto& element : *value.m_value.object)
-                {
-                    flatten(reference_string + "/" + escape(element.first), element.second, result);
+                    operator[](it.key()).merge_patch(it.value());
                 }
             }
-            break;
-        }
-
-        default:
-        {
-            // add primitive value with its reference string
-            result[reference_string] = value;
-            break;
         }
-    }
-}
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-NLOHMANN_BASIC_JSON_TPL
-json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value)
-{
-    if (JSON_UNLIKELY(not value.is_object()))
-    {
-        JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
-    }
-
-    NLOHMANN_BASIC_JSON_TPL result;
-
-    // iterate the JSON object values
-    for (const auto& element : *value.m_value.object)
-    {
-        if (JSON_UNLIKELY(not element.second.is_primitive()))
+        else
         {
-            JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+            *this = patch;
         }
-
-        // assign value to reference pointed to by JSON pointer; Note that if
-        // the JSON pointer is "" (i.e., points to the whole value), function
-        // get_and_create returns a reference to result itself. An assignment
-        // will then create a primitive value.
-        json_pointer(element.first).get_and_create(result) = element.second;
     }
 
-    return result;
-}
-
-inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept
-{
-    return (lhs.reference_tokens == rhs.reference_tokens);
-}
-
-inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept
-{
-    return not (lhs == rhs);
-}
+    /// @}
+};
 } // namespace nlohmann
 
-
 ///////////////////////
 // nonmember support //
 ///////////////////////
@@ -14766,20 +20159,6 @@ inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcep
 // specialization of std::swap, and std::hash
 namespace std
 {
-/*!
-@brief exchanges the values of two JSON objects
-
-@since version 1.0.0
-*/
-template<>
-inline void swap(nlohmann::json& j1,
-                 nlohmann::json& j2) noexcept(
-                     is_nothrow_move_constructible<nlohmann::json>::value and
-                     is_nothrow_move_assignable<nlohmann::json>::value
-                 )
-{
-    j1.swap(j2);
-}
 
 /// hash value for JSON objects
 template<>
@@ -14815,6 +20194,20 @@ struct less< ::nlohmann::detail::value_t>
     }
 };
 
+/*!
+@brief exchanges the values of two JSON objects
+
+@since version 1.0.0
+*/
+template<>
+inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept(
+    is_nothrow_move_constructible<nlohmann::json>::value and
+    is_nothrow_move_assignable<nlohmann::json>::value
+)
+{
+    j1.swap(j2);
+}
+
 } // namespace std
 
 /*!
@@ -14853,6 +20246,9 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
     return nlohmann::json::json_pointer(std::string(s, n));
 }
 
+// #include <nlohmann/detail/macro_unscope.hpp>
+
+
 // restore GCC/clang diagnostic settings
 #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
     #pragma GCC diagnostic pop
@@ -14862,13 +20258,17 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
 #endif
 
 // clean up
+#undef JSON_INTERNAL_CATCH
 #undef JSON_CATCH
 #undef JSON_THROW
 #undef JSON_TRY
 #undef JSON_LIKELY
 #undef JSON_UNLIKELY
 #undef JSON_DEPRECATED
+#undef JSON_HAS_CPP_14
+#undef JSON_HAS_CPP_17
 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
 #undef NLOHMANN_BASIC_JSON_TPL
 
+
 #endif

From 3f8b78a84d4e53208cfcda7dea33bc1a6a5415cf Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Fri, 21 Dec 2018 22:38:06 -0600
Subject: [PATCH 1313/2196] nlohmann-json: 3.4.0 -> 3.5.0

https://github.com/nlohmann/json/releases/tag/v3.5.0
---
 src/nlohmann/json.hpp | 390 ++++++++++++++++++++++++++++--------------
 1 file changed, 261 insertions(+), 129 deletions(-)

diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
index 1e7cf51e0ac..c9af0bed36d 100644
--- a/src/nlohmann/json.hpp
+++ b/src/nlohmann/json.hpp
@@ -1,7 +1,7 @@
 /*
     __ _____ _____ _____
  __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 3.4.0
+|  |  |__   |  |  | | | |  version 3.5.0
 |_____|_____|_____|_|___|  https://github.com/nlohmann/json
 
 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
@@ -31,7 +31,7 @@ SOFTWARE.
 #define NLOHMANN_JSON_HPP
 
 #define NLOHMANN_JSON_VERSION_MAJOR 3
-#define NLOHMANN_JSON_VERSION_MINOR 4
+#define NLOHMANN_JSON_VERSION_MINOR 5
 #define NLOHMANN_JSON_VERSION_PATCH 0
 
 #include <algorithm> // all_of, find, for_each
@@ -41,7 +41,7 @@ SOFTWARE.
 #include <functional> // hash, less
 #include <initializer_list> // initializer_list
 #include <iosfwd> // istream, ostream
-#include <iterator> // iterator_traits, random_access_iterator_tag
+#include <iterator> // random_access_iterator_tag
 #include <numeric> // accumulate
 #include <string> // string, stoi, to_string
 #include <utility> // declval, forward, move, pair, swap
@@ -324,12 +324,10 @@ constexpr T static_const<T>::value;
 
 // #include <nlohmann/json_fwd.hpp>
 
-// #include <nlohmann/detail/meta/cpp_future.hpp>
+// #include <nlohmann/detail/iterators/iterator_traits.hpp>
 
-// #include <nlohmann/detail/meta/detected.hpp>
 
-
-#include <type_traits>
+#include <iterator> // random_access_iterator_tag
 
 // #include <nlohmann/detail/meta/void_t.hpp>
 
@@ -346,6 +344,63 @@ template <typename ...Ts> using void_t = typename make_void<Ts...>::type;
 } // namespace detail
 }  // namespace nlohmann
 
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+
+namespace nlohmann
+{
+namespace detail
+{
+template <typename It, typename = void>
+struct iterator_types {};
+
+template <typename It>
+struct iterator_types <
+    It,
+    void_t<typename It::difference_type, typename It::value_type, typename It::pointer,
+    typename It::reference, typename It::iterator_category >>
+{
+    using difference_type = typename It::difference_type;
+    using value_type = typename It::value_type;
+    using pointer = typename It::pointer;
+    using reference = typename It::reference;
+    using iterator_category = typename It::iterator_category;
+};
+
+// This is required as some compilers implement std::iterator_traits in a way that
+// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.
+template <typename T, typename = void>
+struct iterator_traits
+{
+};
+
+template <typename T>
+struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>
+            : iterator_types<T>
+{
+};
+
+template <typename T>
+struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>
+{
+    using iterator_category = std::random_access_iterator_tag;
+    using value_type = T;
+    using difference_type = ptrdiff_t;
+    using pointer = T*;
+    using reference = T&;
+};
+}
+}
+
+// #include <nlohmann/detail/meta/cpp_future.hpp>
+
+// #include <nlohmann/detail/meta/detected.hpp>
+
+
+#include <type_traits>
+
+// #include <nlohmann/detail/meta/void_t.hpp>
+
 
 // http://en.cppreference.com/w/cpp/experimental/is_detected
 namespace nlohmann
@@ -522,10 +577,10 @@ template <typename T, typename = void>
 struct is_iterator_traits : std::false_type {};
 
 template <typename T>
-struct is_iterator_traits<std::iterator_traits<T>>
+struct is_iterator_traits<iterator_traits<T>>
 {
   private:
-    using traits = std::iterator_traits<T>;
+    using traits = iterator_traits<T>;
 
   public:
     static constexpr auto value =
@@ -642,7 +697,7 @@ struct is_compatible_array_type_impl <
 // Therefore it is detected as a CompatibleArrayType.
 // The real fix would be to have an Iterable concept.
     not is_iterator_traits<
-    std::iterator_traits<CompatibleArrayType>>::value >>
+    iterator_traits<CompatibleArrayType>>::value >>
 {
     static constexpr bool value =
         std::is_constructible<BasicJsonType,
@@ -679,7 +734,7 @@ struct is_constructible_array_type_impl <
         // Therefore it is detected as a ConstructibleArrayType.
         // The real fix would be to have an Iterable concept.
         not is_iterator_traits <
-        std::iterator_traits<ConstructibleArrayType >>::value and
+        iterator_traits<ConstructibleArrayType >>::value and
 
         (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or
          has_from_json<BasicJsonType,
@@ -1594,105 +1649,107 @@ constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::va
 #include <cstddef> // size_t
 #include <string> // string, to_string
 #include <iterator> // input_iterator_tag
+#include <tuple> // tuple_size, get, tuple_element
 
 // #include <nlohmann/detail/value_t.hpp>
 
+// #include <nlohmann/detail/meta/type_traits.hpp>
+
 
 namespace nlohmann
 {
 namespace detail
 {
-/// proxy class for the items() function
-template<typename IteratorType> class iteration_proxy
+template <typename IteratorType> class iteration_proxy_value
 {
+  public:
+    using difference_type = std::ptrdiff_t;
+    using value_type = iteration_proxy_value;
+    using pointer = value_type * ;
+    using reference = value_type & ;
+    using iterator_category = std::input_iterator_tag;
+
   private:
-    /// helper class for iteration
-    class iteration_proxy_internal
-    {
-      public:
-        using difference_type = std::ptrdiff_t;
-        using value_type = iteration_proxy_internal;
-        using pointer = iteration_proxy_internal*;
-        using reference = iteration_proxy_internal&;
-        using iterator_category = std::input_iterator_tag;
+    /// the iterator
+    IteratorType anchor;
+    /// an index for arrays (used to create key names)
+    std::size_t array_index = 0;
+    /// last stringified array index
+    mutable std::size_t array_index_last = 0;
+    /// a string representation of the array index
+    mutable std::string array_index_str = "0";
+    /// an empty string (to return a reference for primitive values)
+    const std::string empty_str = "";
 
-      private:
-        /// the iterator
-        IteratorType anchor;
-        /// an index for arrays (used to create key names)
-        std::size_t array_index = 0;
-        /// last stringified array index
-        mutable std::size_t array_index_last = 0;
-        /// a string representation of the array index
-        mutable std::string array_index_str = "0";
-        /// an empty string (to return a reference for primitive values)
-        const std::string empty_str = "";
+  public:
+    explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {}
 
-      public:
-        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
+    /// dereference operator (needed for range-based for)
+    iteration_proxy_value& operator*()
+    {
+        return *this;
+    }
 
-        /// dereference operator (needed for range-based for)
-        iteration_proxy_internal& operator*()
-        {
-            return *this;
-        }
+    /// increment operator (needed for range-based for)
+    iteration_proxy_value& operator++()
+    {
+        ++anchor;
+        ++array_index;
 
-        /// increment operator (needed for range-based for)
-        iteration_proxy_internal& operator++()
-        {
-            ++anchor;
-            ++array_index;
+        return *this;
+    }
 
-            return *this;
-        }
+    /// equality operator (needed for InputIterator)
+    bool operator==(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor == o.anchor;
+    }
 
-        /// equality operator (needed for InputIterator)
-        bool operator==(const iteration_proxy_internal& o) const noexcept
-        {
-            return anchor == o.anchor;
-        }
+    /// inequality operator (needed for range-based for)
+    bool operator!=(const iteration_proxy_value& o) const noexcept
+    {
+        return anchor != o.anchor;
+    }
 
-        /// inequality operator (needed for range-based for)
-        bool operator!=(const iteration_proxy_internal& o) const noexcept
-        {
-            return anchor != o.anchor;
-        }
+    /// return key of the iterator
+    const std::string& key() const
+    {
+        assert(anchor.m_object != nullptr);
 
-        /// return key of the iterator
-        const std::string& key() const
+        switch (anchor.m_object->type())
         {
-            assert(anchor.m_object != nullptr);
-
-            switch (anchor.m_object->type())
+            // use integer array index as key
+            case value_t::array:
             {
-                // use integer array index as key
-                case value_t::array:
+                if (array_index != array_index_last)
                 {
-                    if (array_index != array_index_last)
-                    {
-                        array_index_str = std::to_string(array_index);
-                        array_index_last = array_index;
-                    }
-                    return array_index_str;
+                    array_index_str = std::to_string(array_index);
+                    array_index_last = array_index;
                 }
+                return array_index_str;
+            }
 
-                // use key from the object
-                case value_t::object:
-                    return anchor.key();
+            // use key from the object
+            case value_t::object:
+                return anchor.key();
 
-                // use an empty key for all primitive types
-                default:
-                    return empty_str;
-            }
+            // use an empty key for all primitive types
+            default:
+                return empty_str;
         }
+    }
 
-        /// return value of the iterator
-        typename IteratorType::reference value() const
-        {
-            return anchor.value();
-        }
-    };
+    /// return value of the iterator
+    typename IteratorType::reference value() const
+    {
+        return anchor.value();
+    }
+};
 
+/// proxy class for the items() function
+template<typename IteratorType> class iteration_proxy
+{
+  private:
     /// the container to iterate
     typename IteratorType::reference container;
 
@@ -1702,20 +1759,55 @@ template<typename IteratorType> class iteration_proxy
         : container(cont) {}
 
     /// return iterator begin (needed for range-based for)
-    iteration_proxy_internal begin() noexcept
+    iteration_proxy_value<IteratorType> begin() noexcept
     {
-        return iteration_proxy_internal(container.begin());
+        return iteration_proxy_value<IteratorType>(container.begin());
     }
 
     /// return iterator end (needed for range-based for)
-    iteration_proxy_internal end() noexcept
+    iteration_proxy_value<IteratorType> end() noexcept
     {
-        return iteration_proxy_internal(container.end());
+        return iteration_proxy_value<IteratorType>(container.end());
     }
 };
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())
+{
+    return i.key();
+}
+// Structured Bindings Support
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>
+auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())
+{
+    return i.value();
+}
 }  // namespace detail
 }  // namespace nlohmann
 
+// The Addition to the STD Namespace is required to add
+// Structured Bindings Support to the iteration_proxy_value class
+// For further reference see https://blog.tartanllama.xyz/structured-bindings/
+// And see https://github.com/nlohmann/json/pull/1391
+namespace std
+{
+template <typename IteratorType>
+class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>
+            : public std::integral_constant<std::size_t, 2> {};
+
+template <std::size_t N, typename IteratorType>
+class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>
+{
+  public:
+    using type = decltype(
+                     get<N>(std::declval <
+                            ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));
+};
+}
 
 namespace nlohmann
 {
@@ -1994,9 +2086,9 @@ void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
 template <
     typename BasicJsonType, typename T, std::size_t N,
     enable_if_t<not std::is_constructible<typename BasicJsonType::string_t,
-                const T (&)[N]>::value,
+                const T(&)[N]>::value,
                 int> = 0 >
-void to_json(BasicJsonType& j, const T (&arr)[N])
+void to_json(BasicJsonType& j, const T(&arr)[N])
 {
     external_constructor<value_t::array>::construct(j, arr);
 }
@@ -2004,21 +2096,21 @@ void to_json(BasicJsonType& j, const T (&arr)[N])
 template<typename BasicJsonType, typename... Args>
 void to_json(BasicJsonType& j, const std::pair<Args...>& p)
 {
-    j = {p.first, p.second};
+    j = { p.first, p.second };
 }
 
 // for https://github.com/nlohmann/json/pull/1134
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_same<T, typename iteration_proxy<typename BasicJsonType::iterator>::iteration_proxy_internal>::value, int> = 0>
+template < typename BasicJsonType, typename T,
+           enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>
 void to_json(BasicJsonType& j, const T& b)
 {
-    j = {{b.key(), b.value()}};
+    j = { {b.key(), b.value()} };
 }
 
 template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
 void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
 {
-    j = {std::get<Idx>(t)...};
+    j = { std::get<Idx>(t)... };
 }
 
 template<typename BasicJsonType, typename... Args>
@@ -2058,6 +2150,7 @@ constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
 #include <string> // string, char_traits
 #include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
 #include <utility> // pair, declval
+#include <cstdio> //FILE *
 
 // #include <nlohmann/detail/macro_scope.hpp>
 
@@ -2094,6 +2187,27 @@ struct input_adapter_protocol
 /// a type to simplify interfaces
 using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
 
+/*!
+Input adapter for stdio file access. This adapter read only 1 byte and do not use any
+ buffer. This adapter is a very low level adapter.
+*/
+class file_input_adapter : public input_adapter_protocol
+{
+  public:
+    explicit file_input_adapter(std::FILE* f)  noexcept
+        : m_file(f)
+    {}
+
+    std::char_traits<char>::int_type get_character() noexcept override
+    {
+        return std::fgetc(m_file);
+    }
+  private:
+    /// the file pointer to read from
+    std::FILE* m_file;
+};
+
+
 /*!
 Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
 beginning of input. Does not support changing the underlying std::streambuf
@@ -2109,8 +2223,8 @@ class input_stream_adapter : public input_adapter_protocol
     ~input_stream_adapter() override
     {
         // clear stream flags; we use underlying streambuf I/O, do not
-        // maintain ifstream flags
-        is.clear();
+        // maintain ifstream flags, except eof
+        is.clear(is.rdstate() & std::ios::eofbit);
     }
 
     explicit input_stream_adapter(std::istream& i)
@@ -2128,7 +2242,13 @@ class input_stream_adapter : public input_adapter_protocol
     // end up as the same value, eg. 0xFFFFFFFF.
     std::char_traits<char>::int_type get_character() override
     {
-        return sb.sbumpc();
+        auto res = sb.sbumpc();
+        // set eof manually, as we don't use the istream interface.
+        if (res == EOF)
+        {
+            is.clear(is.rdstate() | std::ios::eofbit);
+        }
+        return res;
     }
 
   private:
@@ -2336,7 +2456,8 @@ class input_adapter
 {
   public:
     // native support
-
+    input_adapter(std::FILE* file)
+        : ia(std::make_shared<file_input_adapter>(file)) {}
     /// input adapter for input stream
     input_adapter(std::istream& i)
         : ia(std::make_shared<input_stream_adapter>(i)) {}
@@ -2380,7 +2501,7 @@ class input_adapter
     /// input adapter for iterator range with contiguous storage
     template<class IteratorType,
              typename std::enable_if<
-                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
+                 std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
                  int>::type = 0>
     input_adapter(IteratorType first, IteratorType last)
     {
@@ -2399,7 +2520,7 @@ class input_adapter
 
         // assertion to check that each element is 1 byte long
         static_assert(
-            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+            sizeof(typename iterator_traits<IteratorType>::value_type) == 1,
             "each element in the iterator range must have the size of 1 byte");
 
         const auto len = static_cast<size_t>(std::distance(first, last));
@@ -2423,7 +2544,7 @@ class input_adapter
     /// input adapter for contiguous container
     template<class ContiguousContainer, typename
              std::enable_if<not std::is_pointer<ContiguousContainer>::value and
-                            std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+                            std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
                             int>::type = 0>
     input_adapter(const ContiguousContainer& c)
         : input_adapter(std::begin(c), std::end(c)) {}
@@ -3806,7 +3927,7 @@ class lexer
             {
                 // escape control characters
                 char cs[9];
-                snprintf(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
+                (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
                 result += cs;
             }
             else
@@ -5486,24 +5607,21 @@ namespace detail
 {
 // forward declare, to be able to friend it later on
 template<typename IteratorType> class iteration_proxy;
+template<typename IteratorType> class iteration_proxy_value;
 
 /*!
 @brief a template for a bidirectional iterator for the @ref basic_json class
-
 This class implements a both iterators (iterator and const_iterator) for the
 @ref basic_json class.
-
 @note An iterator is called *initialized* when a pointer to a JSON value has
       been set (e.g., by a constructor or a copy assignment). If the iterator is
       default-constructed, it is *uninitialized* and most methods are undefined.
       **The library uses assertions to detect calls on uninitialized iterators.**
-
 @requirement The class satisfies the following concept requirements:
 -
 [BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
   The iterator that can be moved can be moved in both directions (i.e.
   incremented and decremented).
-
 @since version 1.0.0, simplified in version 2.0.9, change to bidirectional
        iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
 */
@@ -5514,6 +5632,7 @@ class iter_impl
     friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
     friend BasicJsonType;
     friend iteration_proxy<iter_impl>;
+    friend iteration_proxy_value<iter_impl>;
 
     using object_t = typename BasicJsonType::object_t;
     using array_t = typename BasicJsonType::array_t;
@@ -6080,8 +6199,7 @@ class iter_impl
     internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it;
 };
 }  // namespace detail
-}  // namespace nlohmann
-
+} // namespace nlohmann
 // #include <nlohmann/detail/iterators/iteration_proxy.hpp>
 
 // #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
@@ -6576,7 +6694,7 @@ class binary_reader
 
             case 0x08: // boolean
             {
-                return sax->boolean(static_cast<bool>(get()));
+                return sax->boolean(get() != 0);
             }
 
             case 0x0A: // null
@@ -6599,7 +6717,7 @@ class binary_reader
             default: // anything else not supported (yet)
             {
                 char cr[3];
-                snprintf(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
+                (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
                 return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr)));
             }
         }
@@ -6635,7 +6753,10 @@ class binary_reader
 
             if (not is_array)
             {
-                sax->key(key);
+                if (not sax->key(key))
+                {
+                    return false;
+                }
             }
 
             if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position)))
@@ -8253,7 +8374,7 @@ class binary_reader
     std::string get_token_string() const
     {
         char cr[3];
-        snprintf(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
+        (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
         return std::string{cr};
     }
 
@@ -11134,15 +11255,15 @@ class serializer
                             {
                                 if (codepoint <= 0xFFFF)
                                 {
-                                    std::snprintf(string_buffer.data() + bytes, 7, "\\u%04x",
-                                                  static_cast<uint16_t>(codepoint));
+                                    (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
+                                                    static_cast<uint16_t>(codepoint));
                                     bytes += 6;
                                 }
                                 else
                                 {
-                                    std::snprintf(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
-                                                  static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
-                                                  static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
+                                    (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                                    static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
+                                                    static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
                                     bytes += 12;
                                 }
                             }
@@ -11178,7 +11299,7 @@ class serializer
                         case error_handler_t::strict:
                         {
                             std::string sn(3, '\0');
-                            snprintf(&sn[0], sn.size(), "%.2X", byte);
+                            (std::snprintf)(&sn[0], sn.size(), "%.2X", byte);
                             JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
                         }
 
@@ -11259,7 +11380,7 @@ class serializer
                 case error_handler_t::strict:
                 {
                     std::string sn(3, '\0');
-                    snprintf(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
+                    (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
                     JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
                 }
 
@@ -11378,7 +11499,7 @@ class serializer
         static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
 
         // the actual conversion
-        std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+        std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
 
         // negative value indicates an error
         assert(len > 0);
@@ -13208,7 +13329,7 @@ class basic_json
                     object = nullptr;  // silence warning, see #821
                     if (JSON_UNLIKELY(t == value_t::null))
                     {
-                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.4.0")); // LCOV_EXCL_LINE
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -16599,9 +16720,20 @@ class basic_json
     Range-based for loop with `items()` function:
 
     @code{cpp}
-    for (auto it : j_object.items())
+    for (auto& el : j_object.items())
     {
-        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
+        std::cout << "key: " << el.key() << ", value:" << el.value() << '\n';
+    }
+    @endcode
+
+    The `items()` function also allows to use
+    [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding)
+    (C++17):
+
+    @code{cpp}
+    for (auto& [key, val] : j_object.items())
+    {
+        std::cout << "key: " << key << ", value:" << val << '\n';
     }
     @endcode
 
@@ -16619,7 +16751,7 @@ class basic_json
 
     @complexity Constant.
 
-    @since version 3.1.0.
+    @since version 3.1.0, structured bindings support since 3.5.0.
     */
     iteration_proxy<iterator> items() noexcept
     {
@@ -20110,7 +20242,7 @@ class basic_json
     Thereby, `Target` is the current object; that is, the patch is applied to
     the current value.
 
-    @param[in] patch  the patch to apply
+    @param[in] apply_patch  the patch to apply
 
     @complexity Linear in the lengths of @a patch.
 
@@ -20122,15 +20254,15 @@ class basic_json
 
     @since version 3.0.0
     */
-    void merge_patch(const basic_json& patch)
+    void merge_patch(const basic_json& apply_patch)
     {
-        if (patch.is_object())
+        if (apply_patch.is_object())
         {
             if (not is_object())
             {
                 *this = object();
             }
-            for (auto it = patch.begin(); it != patch.end(); ++it)
+            for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)
             {
                 if (it.value().is_null())
                 {
@@ -20144,7 +20276,7 @@ class basic_json
         }
         else
         {
-            *this = patch;
+            *this = apply_patch;
         }
     }
 

From 93c9ba3e783a0250786fe7f305551cee6f0ea3ef Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Tue, 25 Dec 2018 22:35:46 +0300
Subject: [PATCH 1314/2196] remove some dead code in nix/repl.cc

Fixes a static analyzer warning:

Expression 'isDrv' is always false.

Fixes: 3beb6f6e7 ('Show derivations more concisely')
---
 src/nix/repl.cc | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index d4806d74adb..dd3d9ed9749 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -643,30 +643,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
             for (auto & i : *v.attrs)
                 sorted[i.name] = i.value;
 
-            /* If this is a derivation, then don't show the
-               self-references ("all", "out", etc.). */
-            StringSet hidden;
-            if (isDrv) {
-                hidden.insert("all");
-                Bindings::iterator i = v.attrs->find(state.sOutputs);
-                if (i == v.attrs->end())
-                    hidden.insert("out");
-                else {
-                    state.forceList(*i->value);
-                    for (unsigned int j = 0; j < i->value->listSize(); ++j)
-                        hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j]));
-                }
-            }
-
             for (auto & i : sorted) {
                 if (isVarName(i.first))
                     str << i.first;
                 else
                     printStringValue(str, i.first.c_str());
                 str << " = ";
-                if (hidden.find(i.first) != hidden.end())
-                    str << "«...»";
-                else if (seen.find(i.second) != seen.end())
+                if (seen.find(i.second) != seen.end())
                     str << "«repeated»";
                 else
                     try {

From e8b0efdcc9d6329b299937790360cac8b9b256c9 Mon Sep 17 00:00:00 2001
From: Mateusz Piotrowski <0mp@FreeBSD.org>
Date: Sun, 30 Dec 2018 02:32:29 +0100
Subject: [PATCH 1315/2196] Remove some bashisms from configure.ac

"configure.ac" is used to generate "configure", which is supposed to be
run with /bin/sh (as suggested by the shebang line of "configure"). As a
result it is a good idea to remove any /bin/sh-incompatible syntax from
configure.ac. Otherwise, systems that do not use Bash as their /bin/sh
are unable to run "configure" due to syntax errors.
---
 configure.ac | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/configure.ac b/configure.ac
index 85fec77a29c..ecacb8ab45b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -214,9 +214,11 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
+  aws_version_tokens="$(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')"
+  aws_major_version="$(awk -v ver="${aws_version_tokens}" '{print $1}')"
+  aws_minor_version="$(awk -v ver="${aws_version_tokens}" '{print $2}')"
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_major_version}, [Major version of aws-sdk-cpp.])
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_minor_version}, [Minor version of aws-sdk-cpp.])
 fi
 
 
@@ -279,9 +281,12 @@ AC_SUBST(sandbox_shell)
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
 for name in $ac_subst_vars; do
-    declare $name="$(eval echo "${!name}")"
-    declare $name="$(eval echo "${!name}")"
-    declare $name="$(eval echo "${!name}")"
+    for _ in 1 2 3; do
+    if printf '%s' "${name}" | egrep '^[_[:alpha:]][_[:alnum:]]*$'
+    then
+        eval "${name}"="$(eval echo "\${${name}}")"
+    fi
+    done
 done
 
 rm -f Makefile.config

From 21ea00d3ecc5428bcb02e8a7174cfd82ed421b2f Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Mon, 31 Dec 2018 10:18:28 -0600
Subject: [PATCH 1316/2196] EvalState::resetFileCache: clear parse cache as
 well as eval cache

Fixes #2546.

(at least the basic reproduction I've been testing)
---
 src/libexpr/eval.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index ab407e56907..2a194d0e08c 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -757,6 +757,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
 void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
+    fileParseCache.clear();
 }
 
 

From 318153f4c27973b6c16e83ba284f552b3f1fb0ce Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sat, 5 Jan 2019 13:30:42 -0500
Subject: [PATCH 1317/2196] Remove dead code computing `dynlib_suffix` from
 comfigure.ac

---
 configure.ac | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index 85fec77a29c..25891954715 100644
--- a/configure.ac
+++ b/configure.ac
@@ -253,11 +253,6 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-# Figure out the extension of dynamic libraries.
-eval dynlib_suffix=$shrext_cmds
-AC_SUBST(dynlib_suffix)
-
-
 # Do we have GNU tar?
 AC_MSG_CHECKING([if you have a recent GNU tar])
 if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then

From e10d6ed2a7463d8a36b94bfc8769e6aea32c0efd Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sat, 5 Jan 2019 14:25:54 -0500
Subject: [PATCH 1318/2196] brotli is only used as a library now

---
 Makefile.config.in   | 1 -
 configure.ac         | 1 -
 src/libutil/local.mk | 2 --
 3 files changed, 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index fad077d869b..b01a4afbfba 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -18,7 +18,6 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-brotli = @brotli@
 lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
diff --git a/configure.ac b/configure.ac
index 25891954715..69aa9a946d2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -128,7 +128,6 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
-AC_PATH_PROGS(brotli, brotli bro, bro)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 824f48fbfc9..21c1e78ccde 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -9,5 +9,3 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
 
 libutil_LIBS = libformat
-
-libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\"

From fef9f5653ba656d0028b5bea8f4e1c6571e5573a Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sat, 5 Jan 2019 14:31:29 -0500
Subject: [PATCH 1319/2196] Remove mentions of `libformat`, it no longer exists

---
 src/libexpr/local.mk                     | 2 +-
 src/libmain/local.mk                     | 2 +-
 src/libstore/local.mk                    | 2 +-
 src/libutil/local.mk                     | 2 --
 src/nix/local.mk                         | 2 +-
 src/resolve-system-dependencies/local.mk | 2 +-
 6 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index daa3258f0d3..ccd5293e4e5 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore libformat
+libexpr_LIBS = libutil libstore
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
diff --git a/src/libmain/local.mk b/src/libmain/local.mk
index f1fd3eb7242..0c80f5a0a03 100644
--- a/src/libmain/local.mk
+++ b/src/libmain/local.mk
@@ -8,7 +8,7 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
 
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
-libmain_LIBS = libstore libutil libformat
+libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 3799257f83f..89fc918c30f 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil libformat
+libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 21c1e78ccde..3ccc23fd5c1 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -7,5 +7,3 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
-
-libutil_LIBS = libformat
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 40a0e8d6bde..ca4604d566c 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,7 +15,7 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil libformat
+nix_LIBS = libexpr libmain libstore libutil
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS)
 
diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk
index 8792a4a252f..f9db16268be 100644
--- a/src/resolve-system-dependencies/local.mk
+++ b/src/resolve-system-dependencies/local.mk
@@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d)
 
 resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix
 
-resolve-system-dependencies_LIBS := libstore libmain libutil libformat
+resolve-system-dependencies_LIBS := libstore libmain libutil
 
 resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc

From 27332870466c971d9c9bbc4a0955c7882468e132 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 7 Jan 2019 16:38:57 -0500
Subject: [PATCH 1320/2196] Don't look for a "pv" program, or mention it in the
 manual source

The manual reference was commented out, and the sole reference to this
program other than the configure script. Removed both.
---
 configure.ac                                | 1 -
 doc/manual/command-ref/nix-copy-closure.xml | 9 ---------
 2 files changed, 10 deletions(-)

diff --git a/configure.ac b/configure.ac
index 69aa9a946d2..5a2526672fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -127,7 +127,6 @@ NEED_PROG(bzip2, bzip2)
 NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
-AC_PATH_PROG(pv, pv, pv)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml
index 800e1db6ab0..e6dcf180ad6 100644
--- a/doc/manual/command-ref/nix-copy-closure.xml
+++ b/doc/manual/command-ref/nix-copy-closure.xml
@@ -95,15 +95,6 @@ those paths.  If this bothers you, use
 
   </varlistentry>
 
-  <!--
-  <varlistentry><term><option>- -show-progress</option></term>
-
-    <listitem><para>Show the progress of each path's transfer as it's made.
-    This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem>
-
-  </varlistentry>
-  -->
-
   <varlistentry><term><option>--include-outputs</option></term>
 
     <listitem><para>Also copy the outputs of store derivations

From 80f464d9d72839841422f75e78ce725365cc3d71 Mon Sep 17 00:00:00 2001
From: Mateusz Piotrowski <0mp@FreeBSD.org>
Date: Wed, 9 Jan 2019 14:45:00 +0100
Subject: [PATCH 1321/2196] Escape square brackets in configure.ac

This patch attempts to solve the regression introduced in e8b0efdc
(#2607).
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 4f00ff0143b..55600d90bbf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -275,7 +275,7 @@ test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
 for name in $ac_subst_vars; do
     for _ in 1 2 3; do
-    if printf '%s' "${name}" | egrep '^[_[:alpha:]][_[:alnum:]]*$'
+    if printf '%s' "${name}" | egrep '^[[_[[:alpha:]]]][[_[[:alnum:]]]]*$'
     then
         eval "${name}"="$(eval echo "\${${name}}")"
     fi

From 7af5f70139d85cae773a4cd58b0708363bf44f9a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Jan 2019 10:10:20 +0100
Subject: [PATCH 1322/2196] Revert "Escape square brackets in configure.ac"

This reverts commit 80f464d9d72839841422f75e78ce725365cc3d71.
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 55600d90bbf..4f00ff0143b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -275,7 +275,7 @@ test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
 for name in $ac_subst_vars; do
     for _ in 1 2 3; do
-    if printf '%s' "${name}" | egrep '^[[_[[:alpha:]]]][[_[[:alnum:]]]]*$'
+    if printf '%s' "${name}" | egrep '^[_[:alpha:]][_[:alnum:]]*$'
     then
         eval "${name}"="$(eval echo "\${${name}}")"
     fi

From 44a8b17556464306d6de0ff9cc6907fd7971fba8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Jan 2019 10:10:44 +0100
Subject: [PATCH 1323/2196] Revert "Remove some bashisms from configure.ac"

This reverts commit e8b0efdcc9d6329b299937790360cac8b9b256c9.
---
 configure.ac | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/configure.ac b/configure.ac
index 4f00ff0143b..5a2526672fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -212,11 +212,9 @@ AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
 
 if test -n "$enable_s3"; then
-  aws_version_tokens="$(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')"
-  aws_major_version="$(awk -v ver="${aws_version_tokens}" '{print $1}')"
-  aws_minor_version="$(awk -v ver="${aws_version_tokens}" '{print $2}')"
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_major_version}, [Major version of aws-sdk-cpp.])
-  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_minor_version}, [Minor version of aws-sdk-cpp.])
+  declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
+  AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
 fi
 
 
@@ -274,12 +272,9 @@ AC_SUBST(sandbox_shell)
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'
 for name in $ac_subst_vars; do
-    for _ in 1 2 3; do
-    if printf '%s' "${name}" | egrep '^[_[:alpha:]][_[:alnum:]]*$'
-    then
-        eval "${name}"="$(eval echo "\${${name}}")"
-    fi
-    done
+    declare $name="$(eval echo "${!name}")"
+    declare $name="$(eval echo "${!name}")"
+    declare $name="$(eval echo "${!name}")"
 done
 
 rm -f Makefile.config

From bbaf865655cff51f30be254f94c22407e368cb96 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Jan 2019 21:01:47 +0100
Subject: [PATCH 1324/2196] Add 2.2 release notes

---
 doc/manual/release-notes/rl-2.2.xml | 122 +++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 4 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.2.xml b/doc/manual/release-notes/rl-2.2.xml
index abe9b49adad..d29eb87e82c 100644
--- a/doc/manual/release-notes/rl-2.2.xml
+++ b/doc/manual/release-notes/rl-2.2.xml
@@ -4,25 +4,139 @@
       version="5.0"
       xml:id="ssec-relnotes-2.2">
 
-<title>Release 2.2 (201?-??-??)</title>
+<title>Release 2.2 (2019-01-11)</title>
 
-<para>This release has the following changes:</para>
+<para>This is primarily a bug fix release. It also has the following
+changes:</para>
 
 <itemizedlist>
 
+  <listitem>
+    <para>In derivations that use structured attributes (i.e. that
+    specify set the <varname>__structuredAttrs</varname> attribute to
+    <literal>true</literal> to cause all attributes to be passed to
+    the builder in JSON format), you can now specify closure checks
+    per output, e.g.:
+
+<programlisting>
+outputChecks."out" = {
+  # The closure of 'out' must not be larger than 256 MiB.
+  maxClosureSize = 256 * 1024 * 1024;
+
+  # It must not refer to C compiler or to the 'dev' output.
+  disallowedRequisites = [ stdenv.cc "dev" ];
+};
+
+outputChecks."dev" = {
+  # The 'dev' output must not be larger than 128 KiB.
+  maxSize = 128 * 1024;
+};
+</programlisting>
+
+    </para>
+  </listitem>
+
+
   <listitem>
     <para>The derivation attribute
     <varname>requiredSystemFeatures</varname> is now enforced for
     local builds, and not just to route builds to remote builders.
     The supported features of a machine can be specified through the
-    configuration setting <varname>system-features</varname>.
-    </para>
+    configuration setting <varname>system-features</varname>.</para>
+
+    <para>By default, <varname>system-features</varname> includes
+    <literal>kvm</literal> if <filename>/dev/kvm</filename>
+    exists. For compatibility, it also includes the pseudo-features
+    <literal>nixos-test</literal>, <literal>benchmark</literal> and
+    <literal>big-parallel</literal> which are used by Nixpkgs to route
+    builds to particular Hydra build machines.</para>
+
   </listitem>
 
   <listitem>
     <para>Sandbox builds are now enabled by default on Linux.</para>
   </listitem>
 
+  <listitem>
+    <para>The new command <command>nix doctor</command> shows
+    potential issues with your Nix installation.</para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>fetchGit</literal> builtin function now uses a
+    caching scheme that puts different remote repositories in distinct
+    local repositories, rather than a single shared repository. This
+    may require more disk space but is faster.</para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>dirOf</literal> builtin function now works on
+    relative paths.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now supports <link
+    xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>,
+    allowing the hash algorithm and hash to be specified in a single
+    string. For example, you can write:
+
+<programlisting>
+import &lt;nix/fetchurl.nix> {
+  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+  hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ=";
+};
+</programlisting>
+
+    instead of
+
+<programlisting>
+import &lt;nix/fetchurl.nix> {
+  url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz;
+  sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4";
+};
+</programlisting>
+
+    </para>
+
+    <para>In fixed-output derivations, the
+    <varname>outputHashAlgo</varname> attribute is no longer mandatory
+    if <varname>outputHash</varname> specifies the hash.</para>
+
+    <para><command>nix hash-file</command> and <command>nix
+    hash-path</command> now print hashes in SRI format by
+    default. They also use SHA-256 by default instead of SHA-512
+    because that's what we use most of the time in Nixpkgs.</para>
+  </listitem>
+
+  <listitem>
+    <para>Integers are now 64 bits on all platforms.</para>
+  </listitem>
+
+  <listitem>
+    <para>The evaluator now prints profiling statistics (enabled via
+    the <envar>NIX_SHOW_STATS</envar> and
+    <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON
+    format.</para>
+  </listitem>
+
+  <listitem>
+    <para>The option <option>--xml</option> in <command>nix-store
+    --query</command> has been removed. Instead, there now is an
+    option <option>--graphml</option> to output the dependency graph
+    in GraphML format.</para>
+  </listitem>
+
+  <listitem>
+    <para>All <filename>nix-*</filename> commands are now symlinks to
+    <filename>nix</filename>. This saves a bit of disk space.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix repl</command> now uses
+    <literal>libeditline</literal> or
+    <literal>libreadline</literal>.</para>
+  </listitem>
+
 </itemizedlist>
 
 </section>

From 8ce1986611ccf536c939de3feecfefe64f47fc8f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 Jan 2019 16:30:36 +0100
Subject: [PATCH 1325/2196] Bump version

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 616187889b6..c0943d3e98d 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.2
\ No newline at end of file
+2.3
\ No newline at end of file

From 4ae6e849015fb269414c1544e815aba75085cf7b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 Jan 2019 16:30:43 +0100
Subject: [PATCH 1326/2196] install: Use base-16 hashes

Issue #2623.
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 415c87da7cb..27164506752 100644
--- a/release.nix
+++ b/release.nix
@@ -296,7 +296,7 @@ let
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
+              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
               [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
             } \
             --replace '@nixVersion@' ${build.x86_64-linux.src.version}

From 087be7281a02a6d0e548ae56b910771837023d21 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 13 Jan 2019 10:33:15 -0500
Subject: [PATCH 1327/2196] Treat plain derivation paths in context as normal
 paths.

Previously, plain derivation paths in the string context (e.g. those
that arose from builtins.storePath on a drv file, not those that arose
from accessing .drvPath of a derivation) were treated somewhat like
derivaiton paths derived from .drvPath, except their dependencies
weren't recursively added to the input set. With this change, such
plain derivation paths are simply treated as paths and added to the
source inputs set accordingly, simplifying context handling code and
removing the inconsistency. If drvPath-like behavior is desired, the
.drv file can be imported and then .drvPath can be accessed.

This is a backwards-incompatibility, but storePath is never used on
drv files within nixpkgs and almost never used elsewhere.
---
 .gitignore             |  3 +++
 src/libexpr/primops.cc | 20 +++-----------------
 2 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0f2f3ddeec1..b75c5d48905 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,6 +81,9 @@ perl/Makefile.config
 /tests/common.sh
 /tests/dummy
 /tests/result*
+/tests/restricted-innocent
+/tests/shell
+/tests/shell.drv
 
 # /tests/lang/
 /tests/lang/*.out
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 60698f7402e..113ceaf2940 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -687,21 +687,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
         }
 
-        /* See prim_unsafeDiscardOutputDependency. */
-        else if (path.at(0) == '~')
-            drv.inputSrcs.insert(string(path, 1));
-
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
             drv.inputDrvs[ctx.first].insert(ctx.second);
         }
 
-        /* Handle derivation contexts returned by
-           ‘builtins.storePath’. */
-        else if (isDerivation(path))
-            drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path);
-
         /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
@@ -1004,13 +995,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     PathSet refs;
 
     for (auto path : context) {
-        if (path.at(0) == '=') path = string(path, 1);
-        if (isDerivation(path)) {
-            /* See prim_unsafeDiscardOutputDependency. */
-            if (path.at(0) != '~')
-                throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-            path = string(path, 1);
-        }
+        if (path.at(0) != '/')
+            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
         refs.insert(path);
     }
 
@@ -1823,7 +1809,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
 
     PathSet context2;
     for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p);
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
 
     mkString(v, s, context2);
 }

From 1d757292d0cb78beec32fcdfe15c2944a4bc4a95 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 13 Jan 2019 12:43:46 -0500
Subject: [PATCH 1328/2196] Add builtins.getContext.

This can be very helpful when debugging, as well as enabling complex
black magic like surgically removing a single dependency from a
string's context.
---
 src/libexpr/eval.hh                           |   3 +
 src/libexpr/primops.cc                        |  38 -----
 src/libexpr/primops/context.cc                | 132 ++++++++++++++++++
 .../lang/eval-okay-context-introspection.exp  |   1 +
 .../lang/eval-okay-context-introspection.nix  |  22 +++
 5 files changed, 158 insertions(+), 38 deletions(-)
 create mode 100644 src/libexpr/primops/context.cc
 create mode 100644 tests/lang/eval-okay-context-introspection.exp
 create mode 100644 tests/lang/eval-okay-context-introspection.nix

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index d0f298e168e..9fe3878916d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -316,6 +316,9 @@ private:
 /* Return a string representing the type of the value `v'. */
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(const string & s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 113ceaf2940..0da9f702f4b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1780,41 +1780,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
-}
-
-
-static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
-}
-
-
-/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
-   builder without causing the derivation to be built (for instance,
-   in the derivation that builds NARs in nix-push, when doing
-   source-only deployment).  This primop marks the string context so
-   that builtins.derivation adds the path to drv.inputSrcs rather than
-   drv.inputDrvs. */
-static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-
-    PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
-
-    mkString(v, s, context2);
-}
-
-
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2285,9 +2250,6 @@ void EvalState::createBaseEnv()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
-    addPrimOp("__hasContext", 1, prim_hasContext);
-    addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
     addPrimOp("__hashString", 2, prim_hashString);
     addPrimOp("__match", 2, prim_match);
     addPrimOp("__split", 2, prim_split);
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
new file mode 100644
index 00000000000..6849aa26119
--- /dev/null
+++ b/src/libexpr/primops/context.cc
@@ -0,0 +1,132 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+
+namespace nix {
+
+static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+    mkString(v, s, PathSet());
+}
+
+static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+
+
+static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    mkBool(v, !context.empty());
+}
+
+static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+
+
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    string s = state.coerceToString(pos, *args[0], context);
+
+    PathSet context2;
+    for (auto & p : context)
+        context2.insert(p.at(0) == '=' ? string(p, 1) : p);
+
+    mkString(v, s, context2);
+}
+
+static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+
+
+/* Extract the context of a string as a structured Nix value.
+
+   The context is represented as an attribute set whose keys are the
+   paths in the context set and whose values are attribute sets with
+   the following keys:
+     path: True if the relevant path is in the context as a plain store
+           path (i.e. the kind of context you get when interpolating
+           a Nix path (e.g. ./.) into a string). False if missing.
+     allOutputs: True if the relevant path is a derivation and it is
+                  in the context as a drv file with all of its outputs
+                  (i.e. the kind of context you get when referencing
+                  .drvPath of some derivation). False if missing.
+     outputs: If a non-empty list, the relevant path is a derivation
+              and the provided outputs are referenced in the context
+              (i.e. the kind of context you get when referencing
+              .outPath of some derivation). Empty list if missing.
+   Note that for a given path any combination of the above attributes
+   may be present, but at least one must be set to something other
+   than the default.
+*/
+static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    struct ContextInfo {
+        bool path = false;
+        bool allOutputs = false;
+        Strings outputs;
+    };
+    PathSet context;
+    state.forceString(*args[0], context, pos);
+    auto contextInfos = std::map<Path, ContextInfo>();
+    for (const auto & p : context) {
+        Path drv;
+        string output;
+        const Path * path = &p;
+        if (p.at(0) == '=') {
+            drv = string(p, 1);
+            path = &drv;
+        } else if (p.at(0) == '!') {
+            std::pair<string, string> ctx = decodeContext(p);
+            drv = ctx.first;
+            output = ctx.second;
+            path = &drv;
+        }
+        auto isPath = drv.empty();
+        auto isAllOutputs = (!drv.empty()) && output.empty();
+
+        auto iter = contextInfos.find(*path);
+        if (iter == contextInfos.end()) {
+            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
+        } else {
+            if (isPath)
+                iter->second.path = true;
+            else if (isAllOutputs)
+                iter->second.allOutputs = true;
+            else
+                iter->second.outputs.emplace_back(std::move(output));
+        }
+    }
+
+    state.mkAttrs(v, contextInfos.size());
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (const auto & info : contextInfos) {
+        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
+        state.mkAttrs(infoVal, 3);
+        if (info.second.path)
+            mkBool(*state.allocAttr(infoVal, sPath), true);
+        if (info.second.allOutputs)
+            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+        if (!info.second.outputs.empty()) {
+            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            state.mkList(outputsVal, info.second.outputs.size());
+            size_t i = 0;
+            for (const auto & output : info.second.outputs) {
+                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            }
+        }
+        infoVal.attrs->sort();
+    }
+    v.attrs->sort();
+}
+
+static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+
+}
diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp
new file mode 100644
index 00000000000..27ba77ddaf6
--- /dev/null
+++ b/tests/lang/eval-okay-context-introspection.exp
@@ -0,0 +1 @@
+true
diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix
new file mode 100644
index 00000000000..d9b2ea35499
--- /dev/null
+++ b/tests/lang/eval-okay-context-introspection.nix
@@ -0,0 +1,22 @@
+let
+  drv = derivation {
+    name = "fail";
+    builder = "/bin/false";
+    system = "x86_64-linux";
+    outputs = [ "out" "foo" ];
+  };
+
+  path = "${./eval-okay-context-introspection.nix}";
+
+  desired-context = {
+    "${builtins.unsafeDiscardStringContext path}" = {
+      path = true;
+    };
+    "${builtins.unsafeDiscardStringContext drv.drvPath}" = {
+      outputs = [ "foo" "out" ];
+      allOutputs = true;
+    };
+  };
+
+  legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}";
+in builtins.getContext legit-context == desired-context

From 2ae5624b2f32e3c84c57d164f2c7fef205b458cc Mon Sep 17 00:00:00 2001
From: Samuel Evans-Powell <samuel.evans-powell@tiny.cloud>
Date: Fri, 18 Jan 2019 10:50:54 +1000
Subject: [PATCH 1329/2196] Update instructions for Haskell script using
 nix-shell interpreter

- The instructions for using nix-shell as an interpreter has a Haskell script
  example that doesn't work on more recent versions of Nix. Update the
  instructions with a working command
---
 doc/manual/command-ref/nix-shell.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index cb443c888d3..bb4a4e42012 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -337,7 +337,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
 #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
 
 import Network.HTTP

From 7cc1a2593ebece4df28abc86c8b83f5d14dc23a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 18 Jan 2019 13:34:23 +0100
Subject: [PATCH 1330/2196] unsupported(): Show the name of the unsupported
 operation

---
 src/libstore/binary-cache-store.hh | 31 +++----------------------
 src/libstore/legacy-ssh-store.cc   | 37 ++++--------------------------
 src/libstore/store-api.cc          |  4 ++--
 src/libstore/store-api.hh          | 35 +++++++++++++++++-----------
 4 files changed, 31 insertions(+), 76 deletions(-)

diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 6bc83fc50ca..953f3b90af4 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -72,24 +72,11 @@ public:
 
     bool isValidPathUncached(const Path & path) override;
 
-    PathSet queryAllValidPaths() override
-    { unsupported(); }
-
     void queryPathInfoUncached(const Path & path,
         Callback<std::shared_ptr<ValidPathInfo>> callback) override;
 
-    void queryReferrers(const Path & path,
-        PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
-
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     bool wantMassQuery() override { return wantMassQuery_; }
 
@@ -108,22 +95,10 @@ public:
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
-    { unsupported(); }
+    { unsupported("buildDerivation"); }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     ref<FSAccessor> getFSAccessor() override;
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 26e1851981d..7c9bc2b68ba 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -187,28 +187,17 @@ struct LegacySSHStore : public Store
         copyNAR(conn->from, sink);
     }
 
-    PathSet queryAllValidPaths() override { unsupported(); }
-
-    void queryReferrers(const Path & path, PathSet & referrers) override
-    { unsupported(); }
-
-    PathSet queryDerivationOutputs(const Path & path) override
-    { unsupported(); }
-
-    StringSet queryDerivationOutputNames(const Path & path) override
-    { unsupported(); }
-
     Path queryPathFromHashPart(const string & hashPart) override
-    { unsupported(); }
+    { unsupported("queryPathFromHashPart"); }
 
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addToStore"); }
 
     Path addTextToStore(const string & name, const string & s,
         const PathSet & references, RepairFlag repair) override
-    { unsupported(); }
+    { unsupported("addTextToStore"); }
 
     BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
@@ -242,25 +231,7 @@ struct LegacySSHStore : public Store
     }
 
     void ensurePath(const Path & path) override
-    { unsupported(); }
-
-    void addTempRoot(const Path & path) override
-    { unsupported(); }
-
-    void addIndirectRoot(const Path & path) override
-    { unsupported(); }
-
-    Roots findRoots() override
-    { unsupported(); }
-
-    void collectGarbage(const GCOptions & options, GCResults & results) override
-    { unsupported(); }
-
-    ref<FSAccessor> getFSAccessor() override
-    { unsupported(); }
-
-    void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { unsupported(); }
+    { unsupported("ensurePath"); }
 
     void computeFSClosure(const PathSet & paths,
         PathSet & out, bool flipDirection = false,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index dc54c735fdb..913a1112141 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -562,10 +562,10 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 {
     for (auto & path : paths)
         if (isDerivation(path))
-            unsupported();
+            unsupported("buildPaths");
 
     if (queryValidPaths(paths).size() != paths.size())
-        unsupported();
+        unsupported("buildPaths");
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 106b2be5e6b..f504735e0f8 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -349,7 +349,8 @@ public:
        (i.e. you'll get /nix/store/<hash> rather than
        /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
        full store path. */
-    virtual PathSet queryAllValidPaths() = 0;
+    virtual PathSet queryAllValidPaths()
+    { unsupported("queryAllValidPaths"); }
 
     /* Query information about a valid path. It is permitted to omit
        the name part of the store path. */
@@ -368,8 +369,8 @@ public:
 
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
-    virtual void queryReferrers(const Path & path,
-        PathSet & referrers) = 0;
+    virtual void queryReferrers(const Path & path, PathSet & referrers)
+    { unsupported("queryReferrers"); }
 
     /* Return all currently valid derivations that have `path' as an
        output.  (Note that the result of `queryDeriver()' is the
@@ -378,10 +379,12 @@ public:
     virtual PathSet queryValidDerivers(const Path & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual PathSet queryDerivationOutputs(const Path & path) = 0;
+    virtual PathSet queryDerivationOutputs(const Path & path)
+    { unsupported("queryDerivationOutputs"); }
 
     /* Query the output names of the derivation denoted by `path'. */
-    virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
+    virtual StringSet queryDerivationOutputNames(const Path & path)
+    { unsupported("queryDerivationOutputNames"); }
 
     /* Query the full store path given the hash part of a valid store
        path, or "" if the path doesn't exist. */
@@ -447,14 +450,16 @@ public:
 
     /* Add a store path as a temporary root of the garbage collector.
        The root disappears as soon as we exit. */
-    virtual void addTempRoot(const Path & path) = 0;
+    virtual void addTempRoot(const Path & path)
+    { unsupported("addTempRoot"); }
 
     /* Add an indirect root, which is merely a symlink to `path' from
        /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
        to be a symlink to a store path.  The garbage collector will
        automatically remove the indirect root when it finds that
        `path' has disappeared. */
-    virtual void addIndirectRoot(const Path & path) = 0;
+    virtual void addIndirectRoot(const Path & path)
+    { unsupported("addIndirectRoot"); }
 
     /* Acquire the global GC lock, then immediately release it.  This
        function must be called after registering a new permanent root,
@@ -479,10 +484,12 @@ public:
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
        outside of the Nix store that point to `storePath'.  */
-    virtual Roots findRoots() = 0;
+    virtual Roots findRoots()
+    { unsupported("findRoots"); }
 
     /* Perform a garbage collection. */
-    virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
+    virtual void collectGarbage(const GCOptions & options, GCResults & results)
+    { unsupported("collectGarbage"); }
 
     /* Return a string representing information about the path that
        can be loaded into the database using `nix-store --load-db' or
@@ -513,11 +520,13 @@ public:
     virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; };
 
     /* Return an object to access files in the Nix store. */
-    virtual ref<FSAccessor> getFSAccessor() = 0;
+    virtual ref<FSAccessor> getFSAccessor()
+    { unsupported("getFSAccessor"); }
 
     /* Add signatures to the specified store path. The signatures are
        not verified. */
-    virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0;
+    virtual void addSignatures(const Path & storePath, const StringSet & sigs)
+    { unsupported("addSignatures"); }
 
     /* Utility functions. */
 
@@ -620,9 +629,9 @@ protected:
     Stats stats;
 
     /* Unsupported methods. */
-    [[noreturn]] void unsupported()
+    [[noreturn]] void unsupported(const std::string & op)
     {
-        throw Unsupported("requested operation is not supported by store '%s'", getUri());
+        throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri());
     }
 
 };

From 15efd5437377aa7054bca753abff5451396bfdca Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Wed, 16 Jan 2019 14:53:57 -0500
Subject: [PATCH 1331/2196] manual: "sandbox" option default is "true" on Linux

Fixes: 812e39313c ('Enable sandboxing by default')
---
 doc/manual/command-ref/conf-file.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index e9947ebc673..f0da1f612fe 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -657,7 +657,8 @@ password <replaceable>my-password</replaceable>
     <varname>__noChroot</varname> attribute set to
     <literal>true</literal> do not run in sandboxes.</para>
 
-    <para>The default is <literal>false</literal>.</para>
+    <para>The default is <literal>true</literal> on Linux and
+    <literal>false</literal> on all other platforms.</para>
 
     </listitem>
 

From 5f1891b795c4c4daac5336d1ec60c94f521ede1d Mon Sep 17 00:00:00 2001
From: Spencer Baugh <sbaugh@catern.com>
Date: Wed, 12 Dec 2018 14:19:54 -0500
Subject: [PATCH 1332/2196] nix-store: make --dump-db take a list of paths to
 dump

Inside a derivation, exportReferencesGraph already provides a way to
dump the Nix database for a specific closure. On the command line,
--dump-db gave us the same information, but only for the entire Nix
database at once.

With this change, one can now pass a list of paths to --dump-db to get
the Nix database dumped for just those paths. (The user is responsible
for ensuring this is a closure, like for --export).

Among other things, this is useful for deploying a closure to a new
host without using --import/--export; one can use tar to transfer the
store paths, and --dump-db/--load-db to transfer the validity
information. This is useful if the new host doesn't actually have Nix
yet, and the closure that is being deployed itself contains Nix.
---
 doc/manual/command-ref/nix-store.xml |  8 ++++++++
 src/nix-store/nix-store.cc           | 15 ++++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 41a04f265d7..d73cb92ee22 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -1282,6 +1282,7 @@ ktorrent-2.2.1/NEWS
   <cmdsynopsis>
     <command>nix-store</command>
     <arg choice='plain'><option>--dump-db</option></arg>
+    <arg rep='repeat'><replaceable>paths</replaceable></arg>
   </cmdsynopsis>
 </refsection>
 
@@ -1292,6 +1293,13 @@ Nix database to standard output.  It can be loaded into an empty Nix
 store using <option>--load-db</option>.  This is useful for making
 backups and when migrating to different database schemas.</para>
 
+<para>By default, <option>--dump-db</option> will dump the entire Nix
+database.  When one or more store paths is passed, only the subset of
+the Nix database for those store paths is dumped.  As with
+<option>--export</option>, the user is responsible for passing all the
+store paths for a closure.  See <option>--export</option> for an
+example.</para>
+
 </refsection>
 
 </refsection>
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index a9ad14762e6..33138baff38 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -485,11 +485,16 @@ static void opReadLog(Strings opFlags, Strings opArgs)
 static void opDumpDB(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty())
-        throw UsageError("no arguments expected");
-    PathSet validPaths = store->queryAllValidPaths();
-    for (auto & i : validPaths)
-        cout << store->makeValidityRegistration({i}, true, true);
+    if (!opArgs.empty()) {
+        for (auto & i : opArgs)
+            i = store->followLinksToStorePath(i);
+        for (auto & i : opArgs)
+            cout << store->makeValidityRegistration({i}, true, true);
+    } else {
+        PathSet validPaths = store->queryAllValidPaths();
+        for (auto & i : validPaths)
+            cout << store->makeValidityRegistration({i}, true, true);
+    }
 }
 
 

From b30be6b450f872f8be6dc8afa28f4b030fa8d1d1 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Mon, 14 Jan 2019 11:34:54 -0500
Subject: [PATCH 1333/2196] Add builtins.appendContext.

A partner of builtins.getContext, useful for the same reasons.
---
 src/libexpr/primops/context.cc                | 59 ++++++++++++++++++-
 .../lang/eval-okay-context-introspection.nix  |  6 +-
 2 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 6849aa26119..2d79739ea04 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "derivations.hh"
 
 namespace nix {
 
@@ -61,8 +62,7 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar
               (i.e. the kind of context you get when referencing
               .outPath of some derivation). Empty list if missing.
    Note that for a given path any combination of the above attributes
-   may be present, but at least one must be set to something other
-   than the default.
+   may be present.
 */
 static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -129,4 +129,59 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
 
 static RegisterPrimOp r4("__getContext", 1, prim_getContext);
 
+
+/* Append the given context to a given string.
+
+   See the commentary above unsafeGetContext for details of the
+   context representation.
+*/
+static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    auto orig = state.forceString(*args[0], context, pos);
+
+    state.forceAttrs(*args[1], pos);
+
+    auto sPath = state.symbols.create("path");
+    auto sAllOutputs = state.symbols.create("allOutputs");
+    for (auto & i : *args[1]->attrs) {
+        if (!state.store->isStorePath(i.name))
+            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+        if (!settings.readOnlyMode)
+            state.store->ensurePath(i.name);
+        state.forceAttrs(*i.value, *i.pos);
+        auto iter = i.value->attrs->find(sPath);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos))
+                context.insert(i.name);
+        }
+
+        iter = i.value->attrs->find(sAllOutputs);
+        if (iter != i.value->attrs->end()) {
+            if (state.forceBool(*iter->value, *iter->pos)) {
+                if (!isDerivation(i.name)) {
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                }
+                context.insert("=" + string(i.name));
+            }
+        }
+
+        iter = i.value->attrs->find(state.sOutputs);
+        if (iter != i.value->attrs->end()) {
+            state.forceList(*iter->value, *iter->pos);
+            if (iter->value->listSize() && !isDerivation(i.name)) {
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+            }
+            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
+                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+                context.insert("!" + name + "!" + string(i.name));
+            }
+        }
+    }
+
+    mkString(v, orig, context);
+}
+
+static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+
 }
diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix
index d9b2ea35499..43178bd2eef 100644
--- a/tests/lang/eval-okay-context-introspection.nix
+++ b/tests/lang/eval-okay-context-introspection.nix
@@ -18,5 +18,7 @@ let
     };
   };
 
-  legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}";
-in builtins.getContext legit-context == desired-context
+  legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}";
+
+  constructed-context = builtins.getContext (builtins.appendContext "" desired-context);
+in legit-context == constructed-context

From 01d07b1e92c298f729a73705907b2987da9a4d0c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Feb 2019 10:49:19 +0100
Subject: [PATCH 1334/2196] Revert "Restore parent mount namespace before
 executing a child process"

This reverts commit a0ef21262f4d5652bfb65cfacaec01d89c475a93. This
doesn't work in 'nix run' and nix-shell because setns() fails in
multithreaded programs, and Boehm GC mark threads are uncancellable.

Fixes #2646.
---
 src/libstore/build.cc       |  2 --
 src/libstore/local-store.cc |  2 --
 src/libstore/ssh.cc         |  3 ---
 src/libutil/util.cc         | 24 ------------------------
 src/libutil/util.hh         | 10 ----------
 src/nix-build/nix-build.cc  |  4 ++--
 src/nix/edit.cc             |  5 -----
 src/nix/repl.cc             |  2 --
 src/nix/run.cc              |  4 ++--
 9 files changed, 4 insertions(+), 52 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 59abae9b90d..47ee8b48f4b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2193,7 +2193,6 @@ void DerivationGoal::startBuilder()
         userNamespaceSync.create();
 
         options.allowVfork = false;
-        options.restoreMountNamespace = false;
 
         Pid helper = startProcess([&]() {
 
@@ -2260,7 +2259,6 @@ void DerivationGoal::startBuilder()
 #endif
     {
         options.allowVfork = !buildUser && !drv->isBuiltin();
-        options.restoreMountNamespace = false;
         pid = startProcess([&]() {
             runChild();
         }, options);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5b4e7ca4ca9..485fdd69193 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -366,8 +366,6 @@ void LocalStore::makeStoreWritable()
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
-        saveMountNamespace();
-
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index cf133b57cb2..5e0e44935cc 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -1,5 +1,4 @@
 #include "ssh.hh"
-#include "affinity.hh"
 
 namespace nix {
 
@@ -35,9 +34,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
     auto conn = std::make_unique<Connection>();
     conn->sshPid = startProcess([&]() {
-        restoreAffinity();
         restoreSignals();
-        restoreMountNamespace();
 
         close(in.writeSide.get());
         close(out.readSide.get());
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index ce50334e1e6..7eca35577b0 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -936,8 +936,6 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
                 throw SysError("setting death signal");
 #endif
             restoreAffinity();
-            if (options.restoreMountNamespace)
-                restoreMountNamespace();
             fun();
         } catch (std::exception & e) {
             try {
@@ -1506,26 +1504,4 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
     return std::unique_ptr<InterruptCallback>(res.release());
 }
 
-static AutoCloseFD fdSavedMountNamespace;
-
-void saveMountNamespace()
-{
-#if __linux__
-    std::once_flag done;
-    std::call_once(done, []() {
-        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
-        if (!fdSavedMountNamespace)
-            throw SysError("saving parent mount namespace");
-    });
-#endif
-}
-
-void restoreMountNamespace()
-{
-#if __linux__
-    if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
-        throw SysError("restoring parent mount namespace");
-#endif
-}
-
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index d67bddc138c..bda87bee433 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -250,7 +250,6 @@ struct ProcessOptions
     bool dieWithParent = true;
     bool runExitHandlers = false;
     bool allowVfork = true;
-    bool restoreMountNamespace = true;
 };
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@@ -515,13 +514,4 @@ typedef std::function<bool(const Path & path)> PathFilter;
 extern PathFilter defaultPathFilter;
 
 
-/* Save the current mount namespace. Ignored if called more than
-   once. */
-void saveMountNamespace();
-
-/* Restore the mount namespace saved by saveMountNamespace(). Ignored
-   if saveMountNamespace() was never called. */
-void restoreMountNamespace();
-
-
 }
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 11ea3b1f7ae..618895d387d 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -401,6 +401,8 @@ static void _main(int argc, char * * argv)
             } else
                 env[var.first] = var.second;
 
+        restoreAffinity();
+
         /* Run a shell using the derivation's environment.  For
            convenience, source $stdenv/setup to setup additional
            environment variables and shell functions.  Also don't
@@ -444,9 +446,7 @@ static void _main(int argc, char * * argv)
 
         auto argPtrs = stringsToCharPtrs(args);
 
-        restoreAffinity();
         restoreSignals();
-        restoreMountNamespace();
 
         execvp(shell.c_str(), argPtrs.data());
 
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index d8d5895bd86..c9671f76d0f 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -3,7 +3,6 @@
 #include "eval.hh"
 #include "attr-path.hh"
 #include "progress-bar.hh"
-#include "affinity.hh"
 
 #include <unistd.h>
 
@@ -73,10 +72,6 @@ struct CmdEdit : InstallableCommand
 
         stopProgressBar();
 
-        restoreAffinity();
-        restoreSignals();
-        restoreMountNamespace();
-
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("cannot run editor '%s'", editor);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index dd3d9ed9749..227affc60e2 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -337,8 +337,6 @@ static int runProgram(const string & program, const Strings & args)
     if (pid == -1) throw SysError("forking");
     if (pid == 0) {
         restoreAffinity();
-        restoreSignals();
-        restoreMountNamespace();
         execvp(program.c_str(), stringsToCharPtrs(args2).data());
         _exit(1);
     }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 1297072989b..35b76334587 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -153,9 +153,9 @@ struct CmdRun : InstallablesCommand
 
         stopProgressBar();
 
-        restoreAffinity();
         restoreSignals();
-        restoreMountNamespace();
+
+        restoreAffinity();
 
         /* If this is a diverted store (i.e. its "logical" location
            (typically /nix/store) differs from its "physical" location

From 8ac1130cc2f0d717fa49adf18465ecf5613d295c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 9 Feb 2019 20:32:12 +0100
Subject: [PATCH 1335/2196] nix-daemon: add variable to disable fork safety

Since macOS 10.14 this has become an error, causing problems if the
nix-daemon loads nix during substitution (this is a forked process).

Workaround for #2523.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 549619a57d5..92ed12fa509 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -2,6 +2,11 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
   <dict>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
+      <string>YES</string>
+    </dict>
     <key>Label</key>
     <string>org.nixos.nix-daemon</string>
     <key>KeepAlive</key>

From 7ce1fae59f6cb4341f61a48a5216cb252e4eca54 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 13 Feb 2019 00:03:10 -0500
Subject: [PATCH 1336/2196] Support --disable-shared flag.

This tells Nix to not build the shared libraries.
---
 Makefile.config.in |  2 ++
 configure.ac       | 10 ++++++++++
 mk/libraries.mk    |  2 +-
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index b01a4afbfba..59730b64638 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -1,4 +1,6 @@
+AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
diff --git a/configure.ac b/configure.ac
index 5a2526672fd..410b20972f2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -64,6 +64,7 @@ AC_PROG_CXX
 AC_PROG_CPP
 AX_CXX_COMPILE_STDCXX_14
 
+AC_CHECK_TOOL([AR], [ar])
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -267,6 +268,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+fi
+
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
diff --git a/mk/libraries.mk b/mk/libraries.mk
index 14c95fa91cf..3953446cba3 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -125,7 +125,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) ar crs $$@ $$?
+	$(trace-ar) $(AR) crs $$@ $$?
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

From e58a71442ad4a538b48fc7a9938c3690628c4741 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Thu, 14 Feb 2019 13:24:16 -0500
Subject: [PATCH 1337/2196] nix.sh: Be set -u compliant.

---
 scripts/nix-profile.sh.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index a5f52274fc7..db03e16ba89 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
-    if [ -n "${MANPATH}" ]; then
+    if [ -n "${MANPATH-}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 

From b9567aa8b6f935522d1037377e0e5205a7529fd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Wed, 13 Feb 2019 11:28:28 +0100
Subject: [PATCH 1338/2196] install script: don't abort when "nix-channel
 --update" fails

Instead, print a message about what happened and tell the user what can
be done (run "[sudo -i] nix-channel --update nixpkgs" again at a later
time). This change allows installing Nix when you're offline.

Since the multi-user installer is so verbose, the message isn't printed
until the end.

Fixes issue #2650 ("installation without internet connection").
---
 scripts/install-multi-user.sh       | 10 +++++++++-
 scripts/install-nix-from-closure.sh |  5 ++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 6ee8dd48582..3d2acaac82f 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -240,10 +240,16 @@ EOF
 }
 trap finish_fail EXIT
 
+channel_update_failed=0
 function finish_success {
     finish_cleanup
 
     ok "Alright! We're done!"
+    if [ "x$channel_update_failed" = x1 ]; then
+        echo ""
+        echo "But fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
+    fi
     cat <<EOF
 
 Before Nix will work in your existing shells, you'll need to close
@@ -737,7 +743,9 @@ setup_default_profile() {
     # otherwise it will be lost in environments where sudo doesn't pass
     # all the environment variables by default.
     _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+          || channel_update_failed=1
+
 }
 
 
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index ab20774bbf0..62f7cb673cc 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -138,7 +138,10 @@ if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
     $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
 fi
 if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    $nix/bin/nix-channel --update nixpkgs
+    if ! $nix/bin/nix-channel --update nixpkgs; then
+        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+    fi
 fi
 
 added=

From b402148d8f584144b80fe159e76e553a1792ecb7 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Fri, 22 Feb 2019 21:07:48 +0100
Subject: [PATCH 1339/2196] remove noop uses of nix-store --init

the nix-store --init command is a noop apparently
---
 release.nix                         | 1 -
 scripts/install-multi-user.sh       | 3 ---
 scripts/install-nix-from-closure.sh | 6 ------
 tests/common.sh.in                  | 1 -
 tests/init.sh                       | 3 ---
 5 files changed, 14 deletions(-)

diff --git a/release.nix b/release.nix
index 27164506752..5c0027301b3 100644
--- a/release.nix
+++ b/release.nix
@@ -278,7 +278,6 @@ let
       pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
         ''
           export NIX_STATE_DIR=$TMPDIR
-          nix-store --init
 
           nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
             --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 6ee8dd48582..f93be9d3a45 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -674,9 +674,6 @@ $NIX_INSTALLED_NIX.
 EOF
         fi
 
-        _sudo "to initialize the Nix Database" \
-              $NIX_INSTALLED_NIX/bin/nix-store --init
-
         cat ./.reginfo \
             | _sudo "to load data for the first time in to the Nix Database" \
                    "$NIX_INSTALLED_NIX/bin/nix-store" --load-db
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index ab20774bbf0..fc633fa2337 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -109,12 +109,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
 done
 echo "" >&2
 
-echo "initialising Nix database..." >&2
-if ! $nix/bin/nix-store --init; then
-    echo "$0: failed to initialize the Nix database" >&2
-    exit 1
-fi
-
 if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then
     echo "$0: unable to register valid paths" >&2
     exit 1
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 6a523ca9d83..28d0e8bfd5f 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -53,7 +53,6 @@ clearStore() {
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
     mkdir "$NIX_STATE_DIR"
-    nix-store --init
     clearProfiles
 }
 
diff --git a/tests/init.sh b/tests/init.sh
index 19a12c1e2d9..6fa4f815bfe 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -25,9 +25,6 @@ fsync-metadata = false
 !include nix.conf.extra.not-there
 EOF
 
-# Initialise the database.
-nix-store --init
-
 # Did anything happen?
 test -e "$NIX_STATE_DIR"/db/db.sqlite
 

From d7a7a029ff9c1fc6c5a22b91718e28c4c1062d9f Mon Sep 17 00:00:00 2001
From: Austin Seipp <aseipp@pobox.com>
Date: Fri, 22 Feb 2019 13:07:35 -0600
Subject: [PATCH 1340/2196] scripts: remove default 'sandbox = false' from
 multi-user installer

Sandboxing is now enabled by default on Linux, but is still disabled on
macOS. However, the installer always turned it off to ensure consistent
behavior.

Remove this default configuration, so we fall back to the default
platform-specific value.

Signed-off-by: Austin Seipp <aseipp@pobox.com>
---
 scripts/install-multi-user.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 6ee8dd48582..4c793247e64 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -747,7 +747,6 @@ build-users-group = $NIX_BUILD_GROUP_NAME
 
 max-jobs = $NIX_USER_COUNT
 cores = 1
-sandbox = false
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \
           install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf

From 07f992a74b64f4376d5b415d0042babc924772f3 Mon Sep 17 00:00:00 2001
From: Dzmitry Zaitsau <pradd.me@gmail.com>
Date: Thu, 21 Feb 2019 11:44:25 +0100
Subject: [PATCH 1341/2196] Extract and expose splitUriAndParams function

which splits a URL into localtor and parameter parts
---
 src/libstore/store-api.cc | 16 ++++++++++++----
 src/libstore/store-api.hh |  4 ++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 913a1112141..c13ff11564e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -842,12 +842,11 @@ namespace nix {
 
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
-
-ref<Store> openStore(const std::string & uri_,
-    const Store::Params & extraParams)
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
 {
     auto uri(uri_);
-    Store::Params params(extraParams);
+    Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
         for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
         }
         uri = uri_.substr(0, q);
     }
+    return {uri, params};
+}
+
+ref<Store> openStore(const std::string & uri_,
+    const Store::Params & extraParams)
+{
+    auto [uri, uriParams] = splitUriAndParams(uri_);
+    auto params = extraParams;
+    params.insert(uriParams.begin(), uriParams.end());
 
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f504735e0f8..ad0f8df11b8 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
    for paths created by makeFixedOutputPath() / addToStore(). */
 std::string makeFixedOutputCA(bool recursive, const Hash & hash);
 
+
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
+
 }

From 56c18c67d98078dbed1d05ac68663cc52d2cb543 Mon Sep 17 00:00:00 2001
From: Dzmitry Zaitsau <pradd.me@gmail.com>
Date: Mon, 25 Feb 2019 17:56:52 +0100
Subject: [PATCH 1342/2196] Extend S3 URL parsing with parameters extraction

---
 src/libstore/download.cc | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 467f570bbf0..8bc496515f7 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
 
+#ifdef ENABLE_S3
+    std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
+    {
+        auto [path, params] = splitUriAndParams(uri);
+
+        auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
+           if (slash == std::string::npos)
+               throw nix::Error("bad S3 URI '%s'", path);
+
+        std::string bucketName(path, 5, slash - 5);
+        std::string key(path, slash + 1);
+
+        return {bucketName, key, params};
+    }
+#endif
+
     void enqueueDownload(const DownloadRequest & request,
         Callback<DownloadResult> callback) override
     {
@@ -622,12 +638,8 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             try {
 #ifdef ENABLE_S3
+                auto [bucketName, key, params] = parseS3Uri(request.uri);
                 S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
-                auto slash = request.uri.find('/', 5);
-                if (slash == std::string::npos)
-                    throw nix::Error("bad S3 URI '%s'", request.uri);
-                std::string bucketName(request.uri, 5, slash - 5);
-                std::string key(request.uri, slash + 1);
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;

From ac200c3678c0f569cd962d8bbd22bb74b223d748 Mon Sep 17 00:00:00 2001
From: Dzmitry Zaitsau <pradd.me@gmail.com>
Date: Thu, 21 Feb 2019 14:26:50 +0100
Subject: [PATCH 1343/2196] Apply param values on S3Helper initialization

---
 src/libstore/download.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8bc496515f7..f1666d293ea 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -639,7 +639,14 @@ struct CurlDownloader : public Downloader
             try {
 #ifdef ENABLE_S3
                 auto [bucketName, key, params] = parseS3Uri(request.uri);
-                S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
+
+                std::string profile = get(params, "profile", "");
+                std::string region = get(params, "region", Aws::Region::US_EAST_1);
+                std::string scheme = get(params, "scheme", "");
+                std::string endpoint = get(params, "endpoint", "");
+
+                S3Helper s3Helper(profile, region, scheme, endpoint);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DownloadResult res;

From 06d633598727763c54b4b049dbc213106474d10c Mon Sep 17 00:00:00 2001
From: Dzmitry Zaitsau <pradd.me@gmail.com>
Date: Tue, 26 Feb 2019 11:04:18 +0100
Subject: [PATCH 1344/2196] fix indentation

---
 src/libstore/download.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index f1666d293ea..22382ab1d6e 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -620,8 +620,8 @@ struct CurlDownloader : public Downloader
         auto [path, params] = splitUriAndParams(uri);
 
         auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
-           if (slash == std::string::npos)
-               throw nix::Error("bad S3 URI '%s'", path);
+            if (slash == std::string::npos)
+                throw nix::Error("bad S3 URI '%s'", path);
 
         std::string bucketName(path, 5, slash - 5);
         std::string key(path, slash + 1);

From 0963479741d476393260b831a242c74c918ed6e6 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 4 Apr 2018 13:48:50 -0500
Subject: [PATCH 1345/2196] archive.cc: ignore more posix_fallocate "not
 supported" error codes

Fixes w/musl.
---
 src/libutil/archive.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index bb68e82886d..3aa12027097 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -331,7 +331,7 @@ struct RestoreSink : ParseSink
                filesystem doesn't support preallocation (e.g. on
                OpenSolaris).  Since preallocation is just an
                optimisation, ignore it. */
-            if (errno && errno != EINVAL)
+            if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
                 throw SysError(format("preallocating file of %1% bytes") % len);
         }
 #endif

From 4cfc131ec43ccbfa2f87efc3e96bdbe127ab3b1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <vladimir.cunat@nic.cz>
Date: Fri, 1 Mar 2019 14:30:30 +0100
Subject: [PATCH 1346/2196] manual nitpick: document sha512 support in
 hashString

---
 doc/manual/expressions/builtins.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 412622714d4..22d998bd0ed 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -693,8 +693,8 @@ builtins.genList (x: x * x) 5
     <listitem><para>Return a base-16 representation of the
     cryptographic hash of string <replaceable>s</replaceable>.  The
     hash algorithm specified by <replaceable>type</replaceable> must
-    be one of <literal>"md5"</literal>, <literal>"sha1"</literal> or
-    <literal>"sha256"</literal>.</para></listitem>
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
+    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
 
   </varlistentry>
 

From b71e1fb342b2344b738733698b20c5ec03dabe5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Mar 2019 18:27:45 +0800
Subject: [PATCH 1347/2196] Restore --init calls in tests

---
 tests/common.sh.in | 1 +
 tests/init.sh      | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/tests/common.sh.in b/tests/common.sh.in
index 28d0e8bfd5f..6a523ca9d83 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -53,6 +53,7 @@ clearStore() {
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
     mkdir "$NIX_STATE_DIR"
+    nix-store --init
     clearProfiles
 }
 
diff --git a/tests/init.sh b/tests/init.sh
index 6fa4f815bfe..19a12c1e2d9 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -25,6 +25,9 @@ fsync-metadata = false
 !include nix.conf.extra.not-there
 EOF
 
+# Initialise the database.
+nix-store --init
+
 # Did anything happen?
 test -e "$NIX_STATE_DIR"/db/db.sqlite
 

From fff8db205c393c53e42326f274b322eb70259250 Mon Sep 17 00:00:00 2001
From: volth <volth@volth.com>
Date: Wed, 6 Mar 2019 05:03:25 +0000
Subject: [PATCH 1348/2196] canBuildLocally: check for features

It could happen that the local builder match the system but lacks some features.
Now it results a failure.
The fix gracefully excludes the local builder from the set of available builders for derivation which requires the feature, so the derivation is built on remote builders only (as though it has incompatible system, like ```aarch64-linux``` when local is x86)
---
 src/build-remote/build-remote.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index abf3669b5b3..279ae62f69c 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
+static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+    for (auto & feature : requiredFeatures)
+        if (!settings.systemFeatures.get().count(feature)) return false;
+    return true;
+}
+
 static int _main(int argc, char * * argv)
 {
     {
@@ -97,9 +103,10 @@ static int _main(int argc, char * * argv)
             source >> drvPath;
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-            auto canBuildLocally = amWilling
-                &&  (  neededSystem == settings.thisSystem
-                    || settings.extraPlatforms.get().count(neededSystem) > 0);
+             auto canBuildLocally = amWilling
+                 &&  (  neededSystem == settings.thisSystem
+                     || settings.extraPlatforms.get().count(neededSystem) > 0)
+                 &&  allSupportedLocally(requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);

From 43331d634498154f6a1835c8c417dd07eaefd45d Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Wed, 27 Feb 2019 23:32:12 +0100
Subject: [PATCH 1349/2196] detail memory roots

---
 src/libstore/gc.cc          | 61 +++++++++++++++++++------------------
 src/libstore/local-store.hh |  2 +-
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index b415d542147..ecfa5e1ed44 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -333,9 +333,7 @@ Roots LocalStore::findRootsNoTemp()
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    size_t n = 0;
-    for (auto & root : findRuntimeRoots())
-        roots[fmt("{memory:%d}", n++)] = root;
+    findRuntimeRoots(roots);
 
     return roots;
 }
@@ -357,8 +355,7 @@ Roots LocalStore::findRoots()
     return roots;
 }
 
-
-static void readProcLink(const string & file, StringSet & paths)
+static void readProcLink(const string & file, Roots & roots)
 {
     /* 64 is the starting buffer size gnu readlink uses... */
     auto bufsiz = ssize_t{64};
@@ -377,7 +374,7 @@ static void readProcLink(const string & file, StringSet & paths)
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        paths.emplace(static_cast<char *>(buf), res);
+        roots.emplace(file, std::string(static_cast<char *>(buf), res));
     return;
 }
 
@@ -387,20 +384,20 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
-static void readFileRoots(const char * path, StringSet & paths)
+static void readFileRoots(const char * path, Roots & roots)
 {
     try {
-        paths.emplace(readFile(path));
+        roots.emplace(path, readFile(path));
     } catch (SysError & e) {
         if (e.errNo != ENOENT && e.errNo != EACCES)
             throw;
     }
 }
 
-PathSet LocalStore::findRuntimeRoots()
+void LocalStore::findRuntimeRoots(Roots & roots)
 {
-    PathSet roots;
-    StringSet paths;
+    Roots unchecked;
+
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
         struct dirent * ent;
@@ -410,8 +407,8 @@ PathSet LocalStore::findRuntimeRoots()
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
-                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
+                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked);
+                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked);
 
                 auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
@@ -423,7 +420,7 @@ PathSet LocalStore::findRuntimeRoots()
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
                     if (fd_ent->d_name[0] != '.') {
-                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
+                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked);
                     }
                 }
                 if (errno) {
@@ -434,18 +431,22 @@ PathSet LocalStore::findRuntimeRoots()
                 fdDir.reset();
 
                 try {
-                    auto mapLines =
-                        tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
+                    auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str();
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    int n = 0;
                     for (const auto& line : mapLines) {
+                        n++;
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            paths.emplace(match[1]);
+                            unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]);
                     }
 
-                    auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
+                    auto envFile = (format("/proc/%1%/environ") % ent->d_name).str();
+                    auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
+                    n = 0;
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        paths.emplace(i->str());
+                        unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str());
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -462,10 +463,11 @@ PathSet LocalStore::findRuntimeRoots()
         std::regex lsofRegex(R"(^n(/.*)$)");
         auto lsofLines =
             tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
+        int n = 0;
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                paths.emplace(match[1]);
+                unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]);
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
@@ -473,21 +475,20 @@ PathSet LocalStore::findRuntimeRoots()
 #endif
 
 #if defined(__linux__)
-    readFileRoots("/proc/sys/kernel/modprobe", paths);
-    readFileRoots("/proc/sys/kernel/fbsplash", paths);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
+    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
+    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
+    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 #endif
 
-    for (auto & i : paths)
-        if (isInStore(i)) {
-            Path path = toStorePath(i);
-            if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
+    for (auto & root : unchecked) {
+        if (isInStore(root.second)) {
+            Path path = toStorePath(root.second);
+            if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots.insert(path);
+                roots.emplace(root.first, path);
             }
         }
-
-    return roots;
+    }
 }
 
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fce963433a5..730d4c917d3 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -269,7 +269,7 @@ private:
 
     Roots findRootsNoTemp();
 
-    PathSet findRuntimeRoots();
+    void findRuntimeRoots(Roots & roots);
 
     void removeUnusedLinks(const GCState & state);
 

From a17f86ce3a67dd2dab2329d7262bc4ad4e7c37ff Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@uclouvain.be>
Date: Thu, 28 Feb 2019 23:26:07 +0100
Subject: [PATCH 1350/2196] Obfuscate memory roots for non-root users

---
 src/libstore/gc.cc           | 14 +++++++-------
 src/nix-daemon/nix-daemon.cc | 11 +++++++++--
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ecfa5e1ed44..73630f36dcf 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -374,7 +374,8 @@ static void readProcLink(const string & file, Roots & roots)
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        roots.emplace(file, std::string(static_cast<char *>(buf), res));
+        roots.emplace((format("{memory:%1%") % file).str(),
+                std::string(static_cast<char *>(buf), res));
     return;
 }
 
@@ -407,8 +408,8 @@ void LocalStore::findRuntimeRoots(Roots & roots)
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked);
-                readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked);
+                readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked);
+                readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked);
 
                 auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
@@ -435,10 +436,9 @@ void LocalStore::findRuntimeRoots(Roots & roots)
                     auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
                     int n = 0;
                     for (const auto& line : mapLines) {
-                        n++;
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]);
+                            unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]);
                     }
 
                     auto envFile = (format("/proc/%1%/environ") % ent->d_name).str();
@@ -446,7 +446,7 @@ void LocalStore::findRuntimeRoots(Roots & roots)
                     auto env_end = std::sregex_iterator{};
                     n = 0;
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str());
+                        unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str());
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -467,7 +467,7 @@ void LocalStore::findRuntimeRoots(Roots & roots)
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]);
+                unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]);
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 8368c326614..faa23b26862 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -478,8 +478,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         Roots roots = store->findRoots();
         logger->stopWork();
         to << roots.size();
-        for (auto & i : roots)
-            to << i.first << i.second;
+        int n = 0;
+        for (auto & i : roots) {
+            // Obfuscate 'memory' roots as they exposes information about other users,
+            if (i.first.rfind("{memory:", 0) == 0) {
+               to << fmt("{memory:%d}", n++) << i.second;
+            } else {
+               to << i.first << i.second;
+            }
+        }
         break;
     }
 

From ebc86550f92ec76cda0961ecc625944ec402d2cd Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@uclouvain.be>
Date: Fri, 1 Mar 2019 00:54:52 +0100
Subject: [PATCH 1351/2196] Make roots a map of store paths to pinning links

This new structure makes more sense as there may be many sources rooting
the same store path. Many profiles can reference the same path but this
is even more true with /proc/<pid>/maps where distinct pids can and
often do map the same store path.
This implementation is also more efficient as the `Roots` map contains
only one entry per rooted store path.
---
 src/libstore/gc.cc           | 39 +++++++++++++++++-------------------
 src/libstore/remote-store.cc |  2 +-
 src/libstore/store-api.hh    |  2 +-
 src/nix-daemon/nix-daemon.cc | 19 +++++++++++-------
 src/nix-store/nix-store.cc   | 12 ++++++-----
 5 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 73630f36dcf..da6799f6fc6 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -130,7 +130,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        gcroots directory. */
     if (settings.checkRootReachability) {
         Roots roots = findRoots();
-        if (roots.find(gcRoot) == roots.end())
+        if (roots[storePath].count(gcRoot) == 0)
             printError(
                 format(
                     "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@@ -266,7 +266,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     auto foundRoot = [&](const Path & path, const Path & target) {
         Path storePath = toStorePath(target);
         if (isStorePath(storePath) && isValidPath(storePath))
-            roots[path] = storePath;
+            roots[storePath].emplace(path);
         else
             printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
     };
@@ -306,7 +306,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         else if (type == DT_REG) {
             Path storePath = storeDir + "/" + baseNameOf(path);
             if (isStorePath(storePath) && isValidPath(storePath))
-                roots[path] = storePath;
+                roots[storePath].emplace(path);
         }
 
     }
@@ -346,10 +346,10 @@ Roots LocalStore::findRoots()
     FDs fds;
     pid_t prev = -1;
     size_t n = 0;
-    for (auto & root : readTempRoots(fds)) {
-        if (prev != root.first) n = 0;
-        prev = root.first;
-        roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
+    for (auto & [pid, root] : readTempRoots(fds)) {
+        if (prev != pid) n = 0;
+        prev = pid;
+        roots[root].emplace(fmt("{temp:%d:%d}", pid, n++));
     }
 
     return roots;
@@ -374,8 +374,8 @@ static void readProcLink(const string & file, Roots & roots)
         goto try_again;
     }
     if (res > 0 && buf[0] == '/')
-        roots.emplace((format("{memory:%1%") % file).str(),
-                std::string(static_cast<char *>(buf), res));
+        roots[std::string(static_cast<char *>(buf), res)]
+            .emplace((format("{memory:%1%") % file).str());
     return;
 }
 
@@ -388,7 +388,7 @@ static string quoteRegexChars(const string & raw)
 static void readFileRoots(const char * path, Roots & roots)
 {
     try {
-        roots.emplace(path, readFile(path));
+        roots[readFile(path)].emplace(path);
     } catch (SysError & e) {
         if (e.errNo != ENOENT && e.errNo != EACCES)
             throw;
@@ -434,19 +434,17 @@ void LocalStore::findRuntimeRoots(Roots & roots)
                 try {
                     auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str();
                     auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
-                    int n = 0;
                     for (const auto& line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]);
+                            unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str());
                     }
 
                     auto envFile = (format("/proc/%1%/environ") % ent->d_name).str();
                     auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
-                    n = 0;
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str());
+                        unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str());
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -463,11 +461,10 @@ void LocalStore::findRuntimeRoots(Roots & roots)
         std::regex lsofRegex(R"(^n(/.*)$)");
         auto lsofLines =
             tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
-        int n = 0;
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]);
+                unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str());
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
@@ -480,12 +477,12 @@ void LocalStore::findRuntimeRoots(Roots & roots)
     readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
 #endif
 
-    for (auto & root : unchecked) {
-        if (isInStore(root.second)) {
-            Path path = toStorePath(root.second);
+    for (auto & [target, links] : unchecked) {
+        if (isInStore(target)) {
+            Path path = toStorePath(target);
             if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots.emplace(root.first, path);
+                roots[path].insert(links.begin(), links.end());
             }
         }
     }
@@ -757,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printError(format("finding garbage collector roots..."));
     Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
 
-    for (auto & i : rootMap) state.roots.insert(i.second);
+    for (auto & i : rootMap) state.roots.insert(i.first);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index def140cfbe1..a4dd28af0ea 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -606,7 +606,7 @@ Roots RemoteStore::findRoots()
     while (count--) {
         Path link = readString(conn->from);
         Path target = readStorePath(*this, conn->from);
-        result[link] = target;
+        result[target].emplace(link);
     }
     return result;
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ad0f8df11b8..b4e5f551159 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -47,7 +47,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::map<Path, Path> Roots;
+typedef std::map<Path, std::set<std::string>> Roots;
 
 
 struct GCOptions
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index faa23b26862..014378d27e7 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -477,14 +477,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         Roots roots = store->findRoots();
         logger->stopWork();
-        to << roots.size();
+        size_t total_length = 0;
+        for (auto & root : roots)
+            total_length += root.second.size();
+        to << total_length;
         int n = 0;
-        for (auto & i : roots) {
-            // Obfuscate 'memory' roots as they exposes information about other users,
-            if (i.first.rfind("{memory:", 0) == 0) {
-               to << fmt("{memory:%d}", n++) << i.second;
-            } else {
-               to << i.first << i.second;
+        for (auto & [target, links] : roots) {
+            for (auto & link : links) {
+                // Obfuscate 'memory' roots as they expose information about other users,
+                if (link.rfind("{memory:", 0) == 0) {
+                    to << fmt("{memory:%d}", n++) << target;
+                } else {
+                    to << link << target;
+                }
             }
         }
         break;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 33138baff38..b281ea2dd32 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -428,9 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
             Roots roots = store->findRoots();
-            for (auto & i : roots)
-                if (referrers.find(i.second) != referrers.end())
-                    cout << format("%1%\n") % i.first;
+            for (auto & [path, roots] : roots)
+                if (referrers.find(path) != referrers.end())
+                    for (auto & root : roots)
+                        cout << format("%1% -> %2%\n") % root % path;
             break;
         }
 
@@ -591,8 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs)
 
     if (printRoots) {
         Roots roots = store->findRoots();
-        for (auto & i : roots)
-            cout << i.first << " -> " << i.second << std::endl;
+        for (auto & [path, roots] : roots)
+            for (auto & root : roots)
+                cout << root << " -> " << path << std::endl;
     }
 
     else {

From 8574b703424b73f6f75fc549b1738a6534d6a01e Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@uclouvain.be>
Date: Fri, 1 Mar 2019 01:15:10 +0100
Subject: [PATCH 1352/2196] fixup! Make roots a map of store paths to pinning
 links

---
 src/nix-store/nix-store.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index b281ea2dd32..89831eb3d30 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
             Roots roots = store->findRoots();
-            for (auto & [path, roots] : roots)
+            for (auto & [path, links] : roots)
                 if (referrers.find(path) != referrers.end())
-                    for (auto & root : roots)
-                        cout << format("%1% -> %2%\n") % root % path;
+                    for (auto & link : links)
+                        cout << format("%1%\n") % link;
             break;
         }
 

From fc02b1b3eee7632049040a96b056d769a088b2ea Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@uclouvain.be>
Date: Fri, 1 Mar 2019 01:16:25 +0100
Subject: [PATCH 1353/2196] Also print rooted path in `nix-store -q --roots`

---
 src/nix-store/nix-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 89831eb3d30..11d0d573f07 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -431,7 +431,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
             for (auto & [path, links] : roots)
                 if (referrers.find(path) != referrers.end())
                     for (auto & link : links)
-                        cout << format("%1%\n") % link;
+                        cout << format("%1% -> %2%\n") % link % path;
             break;
         }
 

From 5c56570726bc3c546b5cd24da96ceed23b9fdc90 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@uclouvain.be>
Date: Fri, 1 Mar 2019 01:29:37 +0100
Subject: [PATCH 1354/2196] Also obfuscate the number of memory roots

---
 src/nix-daemon/nix-daemon.cc | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 014378d27e7..63b5eb87d9d 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -478,15 +478,29 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         Roots roots = store->findRoots();
         logger->stopWork();
         size_t total_length = 0;
-        for (auto & root : roots)
-            total_length += root.second.size();
+        bool hasMemoryLink;
+        for (auto & [target, links] : roots) {
+            hasMemoryLink = false;
+            for (auto & link : links) {
+                if (link.rfind("{memory:", 0) == 0) {
+                    if (hasMemoryLink) continue;
+                    ++total_length;
+                    hasMemoryLink = true;
+                } else {
+                    ++total_length;
+                }
+            }
+        }
         to << total_length;
         int n = 0;
         for (auto & [target, links] : roots) {
+            bool hasMemoryLink = false;
             for (auto & link : links) {
                 // Obfuscate 'memory' roots as they expose information about other users,
                 if (link.rfind("{memory:", 0) == 0) {
+                    if (hasMemoryLink) continue;
                     to << fmt("{memory:%d}", n++) << target;
+                    hasMemoryLink = true;
                 } else {
                     to << link << target;
                 }

From 9d87e3fbd2c690f03eadbcada7ff8b880033155c Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Sat, 9 Mar 2019 23:55:15 +0100
Subject: [PATCH 1355/2196] require c++ 17

This fixes warnings about "structured binding declaration" requiring
"-std=c++1z".
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 410b20972f2..4eea008b02a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -62,7 +62,7 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_14
+AX_CXX_COMPILE_STDCXX_17
 
 AC_CHECK_TOOL([AR], [ar])
 

From 9d7221183a6825d798267288ebe7d3e2adcd8cde Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Sun, 10 Mar 2019 00:04:37 +0100
Subject: [PATCH 1356/2196] unify naming of roots as links & targets

---
 src/nix-store/nix-store.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 11d0d573f07..b5530b39e1c 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
             Roots roots = store->findRoots();
-            for (auto & [path, links] : roots)
-                if (referrers.find(path) != referrers.end())
+            for (auto & [target, links] : roots)
+                if (referrers.find(target) != referrers.end())
                     for (auto & link : links)
-                        cout << format("%1% -> %2%\n") % link % path;
+                        cout << format("%1% -> %2%\n") % link % target;
             break;
         }
 
@@ -592,9 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs)
 
     if (printRoots) {
         Roots roots = store->findRoots();
-        for (auto & [path, roots] : roots)
-            for (auto & root : roots)
-                cout << root << " -> " << path << std::endl;
+        for (auto & [target, links] : roots)
+            for (auto & link : links)
+                cout << link << " -> " << target << std::endl;
     }
 
     else {

From 38ee16ae9c277b0512559282c950c48c71e27697 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Sun, 10 Mar 2019 00:37:52 +0100
Subject: [PATCH 1357/2196] Unify internal findRootsXxx() api

---
 src/libstore/gc.cc          | 35 +++++++++++++----------------------
 src/libstore/local-store.hh |  4 ++--
 2 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index da6799f6fc6..2a314035436 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -197,10 +197,8 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
+void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots)
 {
-    std::set<std::pair<pid_t, Path>> tempRoots;
-
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
@@ -250,14 +248,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots.emplace(pid, root);
+            tempRoots[root].emplace(fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
         fds.push_back(fd); /* keep open */
     }
-
-    return tempRoots;
 }
 
 
@@ -321,10 +317,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-Roots LocalStore::findRootsNoTemp()
+void LocalStore::findRootsNoTemp(Roots & roots)
 {
-    Roots roots;
-
     /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@@ -334,23 +328,16 @@ Roots LocalStore::findRootsNoTemp()
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
     findRuntimeRoots(roots);
-
-    return roots;
 }
 
 
 Roots LocalStore::findRoots()
 {
-    Roots roots = findRootsNoTemp();
+    Roots roots;
+    findRootsNoTemp(roots);
 
     FDs fds;
-    pid_t prev = -1;
-    size_t n = 0;
-    for (auto & [pid, root] : readTempRoots(fds)) {
-        if (prev != pid) n = 0;
-        prev = pid;
-        roots[root].emplace(fmt("{temp:%d:%d}", pid, n++));
-    }
+    findTempRoots(fds, roots);
 
     return roots;
 }
@@ -752,7 +739,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printError(format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
+    Roots rootMap;
+    if (!options.ignoreLiveness)
+       findRootsNoTemp(rootMap);
 
     for (auto & i : rootMap) state.roots.insert(i.first);
 
@@ -760,8 +749,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
     FDs fds;
-    for (auto & root : readTempRoots(fds))
-        state.tempRoots.insert(root.second);
+    Roots tempRoots;
+    findTempRoots(fds, tempRoots);
+    for (auto & root : tempRoots)
+        state.tempRoots.insert(root.first);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 730d4c917d3..39a34597e2b 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -180,7 +180,7 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
+    void findTempRoots(FDs & fds, Roots & roots);
 
 public:
 
@@ -267,7 +267,7 @@ private:
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    Roots findRootsNoTemp();
+    void findRootsNoTemp(Roots & roots);
 
     void findRuntimeRoots(Roots & roots);
 

From 4f4391193cca2f7a28d351947e12f954d9432693 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Sun, 10 Mar 2019 00:45:55 +0100
Subject: [PATCH 1358/2196] Fix warning about unused variable

---
 src/nix-daemon/nix-daemon.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 63b5eb87d9d..0feafb01385 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -477,11 +477,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         Roots roots = store->findRoots();
         logger->stopWork();
+
+        // Pre-compute roots length using same algo as below.
         size_t total_length = 0;
         bool hasMemoryLink;
-        for (auto & [target, links] : roots) {
+        for (auto & root : roots) {
             hasMemoryLink = false;
-            for (auto & link : links) {
+            for (auto & link : root.second) {
                 if (link.rfind("{memory:", 0) == 0) {
                     if (hasMemoryLink) continue;
                     ++total_length;
@@ -491,6 +493,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 }
             }
         }
+
         to << total_length;
         int n = 0;
         for (auto & [target, links] : roots) {

From 115e2c8c67a949d3fa78fda450186803cd713958 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <layus.on@gmail.com>
Date: Sun, 10 Mar 2019 01:50:33 +0100
Subject: [PATCH 1359/2196] Update tests to the new --roots format

---
 tests/gc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/gc.sh b/tests/gc.sh
index 0adb05bf173..8b4f8d28218 100644
--- a/tests/gc.sh
+++ b/tests/gc.sh
@@ -7,7 +7,7 @@ outPath=$(nix-store -rvv "$drvPath")
 rm -f "$NIX_STATE_DIR"/gcroots/foo
 ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
 
-[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR"/gcroots/foo ]
+[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ]
 
 nix-store --gc --print-roots | grep $outPath
 nix-store --gc --print-live | grep $outPath

From a3f37d87eabcfb5dc581abcfa46e5e7d387dfa8c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Mar 2019 13:27:16 +0100
Subject: [PATCH 1360/2196] findRuntimeRoots: Simplify/fix handling of /proc
 files

Scanning of /proc/<pid>/{exe,cwd} was broken because '{memory:' was
prepended twice. Also, get rid of the whole '{memory:...}' thing
because it's unnecessary, we can just list the file in /proc directly.
---
 src/libstore/gc.cc        | 26 ++++++++++++--------------
 src/libstore/store-api.hh |  3 ++-
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 2a314035436..8c3607f6614 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -362,8 +362,7 @@ static void readProcLink(const string & file, Roots & roots)
     }
     if (res > 0 && buf[0] == '/')
         roots[std::string(static_cast<char *>(buf), res)]
-            .emplace((format("{memory:%1%") % file).str());
-    return;
+            .emplace(file);
 }
 
 static string quoteRegexChars(const string & raw)
@@ -395,10 +394,10 @@ void LocalStore::findRuntimeRoots(Roots & roots)
         while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
-                readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked);
-                readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked);
+                readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
+                readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
 
-                auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
+                auto fdStr = fmt("/proc/%s/fd", ent->d_name);
                 auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
@@ -407,9 +406,8 @@ void LocalStore::findRuntimeRoots(Roots & roots)
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                    if (fd_ent->d_name[0] != '.') {
-                        readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked);
-                    }
+                    if (fd_ent->d_name[0] != '.')
+                        readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
                 }
                 if (errno) {
                     if (errno == ESRCH)
@@ -419,19 +417,19 @@ void LocalStore::findRuntimeRoots(Roots & roots)
                 fdDir.reset();
 
                 try {
-                    auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str();
+                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
                     auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
-                    for (const auto& line : mapLines) {
+                    for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
-                            unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str());
+                            unchecked[match[1]].emplace(mapFile);
                     }
 
-                    auto envFile = (format("/proc/%1%/environ") % ent->d_name).str();
+                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
                     auto envString = readFile(envFile, true);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str());
+                        unchecked[i->str()].emplace(envFile);
                 } catch (SysError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
@@ -451,7 +449,7 @@ void LocalStore::findRuntimeRoots(Roots & roots)
         for (const auto & line : lsofLines) {
             std::smatch match;
             if (std::regex_match(line, match, lsofRegex))
-                unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str());
+                unchecked[match[1]].emplace("{lsof}");
         }
     } catch (ExecError & e) {
         /* lsof not installed, lsof failed */
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b4e5f551159..bb16659765f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -11,6 +11,7 @@
 #include <atomic>
 #include <limits>
 #include <map>
+#include <unordered_set>
 #include <memory>
 #include <string>
 
@@ -47,7 +48,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::map<Path, std::set<std::string>> Roots;
+typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
 
 
 struct GCOptions

From 53522cb6ac19bd1da35a657988231cce9387be9c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Mar 2019 13:50:07 +0100
Subject: [PATCH 1361/2196] findRoots(): Add 'censor' parameter

This is less brittle than filtering paths after the fact in
nix-daemon.
---
 src/libstore/gc.cc           | 30 +++++++++++++++-----------
 src/libstore/local-store.hh  |  8 +++----
 src/libstore/remote-store.cc |  2 +-
 src/libstore/remote-store.hh |  2 +-
 src/libstore/store-api.hh    |  6 ++++--
 src/nix-daemon/nix-daemon.cc | 41 +++++++++---------------------------
 src/nix-store/nix-store.cc   |  4 ++--
 7 files changed, 40 insertions(+), 53 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 8c3607f6614..d8a5da0d49e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -129,7 +129,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        check if the root is in a directory in or linked from the
        gcroots directory. */
     if (settings.checkRootReachability) {
-        Roots roots = findRoots();
+        Roots roots = findRoots(false);
         if (roots[storePath].count(gcRoot) == 0)
             printError(
                 format(
@@ -197,7 +197,10 @@ void LocalStore::addTempRoot(const Path & path)
 }
 
 
-void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots)
+static std::string censored = "{censored}";
+
+
+void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
@@ -248,7 +251,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots)
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
             assertStorePath(root);
-            tempRoots[root].emplace(fmt("{temp:%d}", pid));
+            tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
@@ -317,7 +320,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 }
 
 
-void LocalStore::findRootsNoTemp(Roots & roots)
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
 {
     /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
@@ -327,17 +330,17 @@ void LocalStore::findRootsNoTemp(Roots & roots)
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    findRuntimeRoots(roots);
+    findRuntimeRoots(roots, censor);
 }
 
 
-Roots LocalStore::findRoots()
+Roots LocalStore::findRoots(bool censor)
 {
     Roots roots;
-    findRootsNoTemp(roots);
+    findRootsNoTemp(roots, censor);
 
     FDs fds;
-    findTempRoots(fds, roots);
+    findTempRoots(fds, roots, censor);
 
     return roots;
 }
@@ -381,7 +384,7 @@ static void readFileRoots(const char * path, Roots & roots)
     }
 }
 
-void LocalStore::findRuntimeRoots(Roots & roots)
+void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
     Roots unchecked;
 
@@ -467,7 +470,10 @@ void LocalStore::findRuntimeRoots(Roots & roots)
             Path path = toStorePath(target);
             if (isStorePath(path) && isValidPath(path)) {
                 debug(format("got additional root '%1%'") % path);
-                roots[path].insert(links.begin(), links.end());
+                if (censor)
+                    roots[path].insert(censored);
+                else
+                    roots[path].insert(links.begin(), links.end());
             }
         }
     }
@@ -739,7 +745,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printError(format("finding garbage collector roots..."));
     Roots rootMap;
     if (!options.ignoreLiveness)
-       findRootsNoTemp(rootMap);
+        findRootsNoTemp(rootMap, true);
 
     for (auto & i : rootMap) state.roots.insert(i.first);
 
@@ -748,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        can be added to the set of temporary roots. */
     FDs fds;
     Roots tempRoots;
-    findTempRoots(fds, tempRoots);
+    findTempRoots(fds, tempRoots, true);
     for (auto & root : tempRoots)
         state.tempRoots.insert(root.first);
     state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 39a34597e2b..6b655647b03 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -180,11 +180,11 @@ private:
     typedef std::shared_ptr<AutoCloseFD> FDPtr;
     typedef list<FDPtr> FDs;
 
-    void findTempRoots(FDs & fds, Roots & roots);
+    void findTempRoots(FDs & fds, Roots & roots, bool censor);
 
 public:
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
@@ -267,9 +267,9 @@ private:
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
-    void findRootsNoTemp(Roots & roots);
+    void findRootsNoTemp(Roots & roots, bool censor);
 
-    void findRuntimeRoots(Roots & roots);
+    void findRuntimeRoots(Roots & roots, bool censor);
 
     void removeUnusedLinks(const GCState & state);
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a4dd28af0ea..15faf78a526 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -596,7 +596,7 @@ void RemoteStore::syncWithGC()
 }
 
 
-Roots RemoteStore::findRoots()
+Roots RemoteStore::findRoots(bool censor)
 {
     auto conn(getConnection());
     conn->to << wopFindRoots;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4f554b5980e..e5ab79e59d3 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -82,7 +82,7 @@ public:
 
     void syncWithGC() override;
 
-    Roots findRoots() override;
+    Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bb16659765f..40887b6aa3a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -484,8 +484,10 @@ public:
 
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
-       outside of the Nix store that point to `storePath'.  */
-    virtual Roots findRoots()
+       outside of the Nix store that point to `storePath'. If
+       'censor' is true, privacy-sensitive information about roots
+       found in /proc is censored. */
+    virtual Roots findRoots(bool censor)
     { unsupported("findRoots"); }
 
     /* Perform a garbage collection. */
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 0feafb01385..dd8b9bd1e89 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -475,40 +475,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopFindRoots: {
         logger->startWork();
-        Roots roots = store->findRoots();
+        Roots roots = store->findRoots(true);
         logger->stopWork();
 
-        // Pre-compute roots length using same algo as below.
-        size_t total_length = 0;
-        bool hasMemoryLink;
-        for (auto & root : roots) {
-            hasMemoryLink = false;
-            for (auto & link : root.second) {
-                if (link.rfind("{memory:", 0) == 0) {
-                    if (hasMemoryLink) continue;
-                    ++total_length;
-                    hasMemoryLink = true;
-                } else {
-                    ++total_length;
-                }
-            }
-        }
+        size_t size = 0;
+        for (auto & i : roots)
+            size += i.second.size();
+
+        to << size;
+
+        for (auto & [target, links] : roots)
+            for (auto & link : links)
+                to << link << target;
 
-        to << total_length;
-        int n = 0;
-        for (auto & [target, links] : roots) {
-            bool hasMemoryLink = false;
-            for (auto & link : links) {
-                // Obfuscate 'memory' roots as they expose information about other users,
-                if (link.rfind("{memory:", 0) == 0) {
-                    if (hasMemoryLink) continue;
-                    to << fmt("{memory:%d}", n++) << target;
-                    hasMemoryLink = true;
-                } else {
-                    to << link << target;
-                }
-            }
-        }
         break;
     }
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index b5530b39e1c..6f315e5093d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -427,7 +427,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise),
                     referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
-            Roots roots = store->findRoots();
+            Roots roots = store->findRoots(false);
             for (auto & [target, links] : roots)
                 if (referrers.find(target) != referrers.end())
                     for (auto & link : links)
@@ -591,7 +591,7 @@ static void opGC(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     if (printRoots) {
-        Roots roots = store->findRoots();
+        Roots roots = store->findRoots(false);
         for (auto & [target, links] : roots)
             for (auto & link : links)
                 cout << link << " -> " << target << std::endl;

From 578ed7a259ddf7a85c85fe4e0200179ae908171e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Mar 2019 13:52:13 +0100
Subject: [PATCH 1362/2196] findRoots(): Don't censor for trusted users

They're pretty much root anyway.
---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index dd8b9bd1e89..8d63b8f362e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -475,7 +475,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopFindRoots: {
         logger->startWork();
-        Roots roots = store->findRoots(true);
+        Roots roots = store->findRoots(!trusted);
         logger->stopWork();
 
         size_t size = 0;

From 86f3b94c8c28defd7402245d60af15ce429986ff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 14 Mar 2019 13:59:10 +0100
Subject: [PATCH 1363/2196] nix-store --gc --print-roots: Sort output

---
 src/nix-store/nix-store.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 6f315e5093d..f324056bb3a 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -592,9 +592,13 @@ static void opGC(Strings opFlags, Strings opArgs)
 
     if (printRoots) {
         Roots roots = store->findRoots(false);
+        std::set<std::pair<Path, Path>> roots2;
+        // Transpose and sort the roots.
         for (auto & [target, links] : roots)
             for (auto & link : links)
-                cout << link << " -> " << target << std::endl;
+                roots2.emplace(link, target);
+        for (auto & [link, target] : roots2)
+            std::cout << link << " -> " << target << "\n";
     }
 
     else {

From ef52ccf035fe0bcd78226891022171a28dd6fcbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Feb 2019 13:43:32 +0100
Subject: [PATCH 1364/2196] experimental/optional -> optional

---
 src/libexpr/eval.hh                |  2 +-
 src/libexpr/get-drvs.cc            |  2 +-
 src/libexpr/get-drvs.hh            |  2 +-
 src/libexpr/primops.cc             |  2 +-
 src/libexpr/primops/fetchGit.cc    |  4 ++--
 src/libstore/build.cc              | 10 +++++-----
 src/libstore/parsed-derivations.cc |  4 ++--
 src/libstore/parsed-derivations.hh |  8 ++++----
 src/libstore/remote-store.hh       |  2 +-
 src/libutil/lru-cache.hh           |  4 ++--
 src/libutil/serialise.cc           |  2 +-
 src/libutil/util.cc                |  2 +-
 src/libutil/util.hh                |  6 +++---
 src/nix/add-to-store.cc            |  2 +-
 14 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9fe3878916d..a314e01e0a7 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -81,7 +81,7 @@ public:
 
     /* The allowed filesystem paths in restricted or pure evaluation
        mode. */
-    std::experimental::optional<PathSet> allowedPaths;
+    std::optional<PathSet> allowedPaths;
 
     Value vEmptySet;
 
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index d38ed2df3b1..21a4d7917fc 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index daaa635fe1b..d7860fc6a4b 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos;
 
 /* If value `v' denotes a derivation, return a DrvInfo object
    describing it. Otherwise return nothing. */
-std::experimental::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0da9f702f4b..6b0c55e72cd 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -555,7 +555,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
-    std::experimental::optional<std::string> outputHash;
+    std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     bool outputHashRecursive = false;
 
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index b46d2f25826..aaf02c856d4 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -26,7 +26,7 @@ struct GitInfo
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::experimental::optional<std::string> ref, std::string rev,
+    std::optional<std::string> ref, std::string rev,
     const std::string & name)
 {
     if (evalSettings.pureEval && rev == "")
@@ -190,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::experimental::optional<std::string> ref;
+    std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
     PathSet context;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 47ee8b48f4b..6b88b130730 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2413,7 +2413,7 @@ void DerivationGoal::writeStructuredAttrs()
        objects consisting entirely of those values. (So nested
        arrays or objects are not supported.) */
 
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
         if (value.is_string())
             return shellEscape(value);
 
@@ -3311,8 +3311,8 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
         struct Checks
         {
             bool ignoreSelfRefs = false;
-            std::experimental::optional<uint64_t> maxSize, maxClosureSize;
-            std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+            std::optional<uint64_t> maxSize, maxClosureSize;
+            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
         };
 
         /* Compute the closure and closure size of some output. This
@@ -3359,7 +3359,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                         info.path, closureSize, *checks.maxClosureSize);
             }
 
-            auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive)
+            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
             {
                 if (!value) return;
 
@@ -3413,7 +3413,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                     if (maxClosureSize != output->end())
                         checks.maxClosureSize = maxClosureSize->get<uint64_t>();
 
-                    auto get = [&](const std::string & name) -> std::experimental::optional<Strings> {
+                    auto get = [&](const std::string & name) -> std::optional<Strings> {
                         auto i = output->find(name);
                         if (i != output->end()) {
                             Strings res;
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index dc328648273..17fde00a016 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
     }
 }
 
-std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
+std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
 {
     if (structuredAttrs) {
         auto i = structuredAttrs->find(name);
@@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
     }
 }
 
-std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
+std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
 {
     if (structuredAttrs) {
         auto i = structuredAttrs->find(name);
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 0a82c146172..ed07dc652e8 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -8,22 +8,22 @@ class ParsedDerivation
 {
     Path drvPath;
     BasicDerivation & drv;
-    std::experimental::optional<nlohmann::json> structuredAttrs;
+    std::optional<nlohmann::json> structuredAttrs;
 
 public:
 
     ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
 
-    const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const
+    const std::optional<nlohmann::json> & getStructuredAttrs() const
     {
         return structuredAttrs;
     }
 
-    std::experimental::optional<std::string> getStringAttr(const std::string & name) const;
+    std::optional<std::string> getStringAttr(const std::string & name) const;
 
     bool getBoolAttr(const std::string & name, bool def = false) const;
 
-    std::experimental::optional<Strings> getStringsAttr(const std::string & name) const;
+    std::optional<Strings> getStringsAttr(const std::string & name) const;
 
     StringSet getRequiredSystemFeatures() const;
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index e5ab79e59d3..80f18ab715d 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -149,7 +149,7 @@ public:
 private:
 
     ref<RemoteStore::Connection> openConnection() override;
-    std::experimental::optional<std::string> path;
+    std::optional<std::string> path;
 };
 
 
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 9b8290e634c..8b83f842c32 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -2,7 +2,7 @@
 
 #include <map>
 #include <list>
-#include <experimental/optional>
+#include <optional>
 
 namespace nix {
 
@@ -64,7 +64,7 @@ public:
 
     /* Look up an item in the cache. If it exists, it becomes the most
        recently used item. */
-    std::experimental::optional<Value> get(const Key & key)
+    std::optional<Value> get(const Key & key)
     {
         auto i = data.find(key);
         if (i == data.end()) return {};
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 0e75eeec2bf..8201549fd7d 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -171,7 +171,7 @@ std::unique_ptr<Source> sinkToSource(
 
         std::function<void(Sink &)> fun;
         std::function<void()> eof;
-        std::experimental::optional<coro_t::pull_type> coro;
+        std::optional<coro_t::pull_type> coro;
         bool started = false;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 7eca35577b0..e3dcd246c68 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -965,7 +965,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
 
 
 string runProgram(Path program, bool searchPath, const Strings & args,
-    const std::experimental::optional<std::string> & input)
+    const std::optional<std::string> & input)
 {
     RunOptions opts(program, args);
     opts.searchPath = searchPath;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index bda87bee433..9f239bff371 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -14,7 +14,7 @@
 #include <cstdio>
 #include <map>
 #include <sstream>
-#include <experimental/optional>
+#include <optional>
 #include <future>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
@@ -259,14 +259,14 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
-    const std::experimental::optional<std::string> & input = {});
+    const std::optional<std::string> & input = {});
 
 struct RunOptions
 {
     Path program;
     bool searchPath = true;
     Strings args;
-    std::experimental::optional<std::string> input;
+    std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
     bool _killStderr = false;
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index d0003790d3b..e86b96e3f3f 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -8,7 +8,7 @@ using namespace nix;
 struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
-    std::experimental::optional<std::string> namePart;
+    std::optional<std::string> namePart;
 
     CmdAddToStore()
     {

From 1f64f4c7c83d8efd67051a399c738c71815e0594 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 Feb 2019 13:44:20 +0100
Subject: [PATCH 1365/2196] pkg-config files: Use c++17

---
 src/libexpr/nix-expr.pc.in   | 2 +-
 src/libmain/nix-main.pc.in   | 2 +-
 src/libstore/nix-store.pc.in | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in
index 79f3e2f4506..80f7a492b1a 100644
--- a/src/libexpr/nix-expr.pc.in
+++ b/src/libexpr/nix-expr.pc.in
@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in
index 38bc85c484e..37b03dcd42c 100644
--- a/src/libmain/nix-main.pc.in
+++ b/src/libmain/nix-main.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in
index 5cf22faadcb..6d67b1e0380 100644
--- a/src/libstore/nix-store.pc.in
+++ b/src/libstore/nix-store.pc.in
@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil
-Cflags: -I${includedir}/nix -std=c++14
+Cflags: -I${includedir}/nix -std=c++17

From 34fade478a84d822646a8daa334014134c0ec59e Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Thu, 14 Mar 2019 23:51:18 +0100
Subject: [PATCH 1366/2196] docs: fix typo in AllowDirectReads

It was just missing an `l` but should be fixed anyway.
---
 doc/manual/packages/s3-substituter.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index 2ec9687a0c6..e7589ffdb03 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -89,7 +89,7 @@ the S3 URL:</para>
     "Version": "2012-10-17",
     "Statement": [
         {
-            "Sid": "AlowDirectReads",
+            "Sid": "AllowDirectReads",
             "Action": [
                 "s3:GetObject",
                 "s3:GetBucketLocation"

From 81a23fa7e2eb4d58d5edf2a145af5c8bbe4c331e Mon Sep 17 00:00:00 2001
From: Chaker Benhamed <chaker.benhamed@gmail.com>
Date: Fri, 15 Mar 2019 13:16:20 +0100
Subject: [PATCH 1367/2196] Add a 5 seconds of timeouts to connect to S3.

The default is 1000ms, but we can hit it a lot of we don't have direct
link to AWS (e.g. using VPN).
---
 src/libstore/s3-binary-cache-store.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 51de89e0d92..cd547a96485 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -126,6 +126,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
         res->endpointOverride = endpoint;
     }
     res->requestTimeoutMs = 600 * 1000;
+    res->connectTimeoutMs = 5 * 1000;
     res->retryStrategy = std::make_shared<RetryStrategy>();
     res->caFile = settings.caFile;
     return res;

From 684c7fff80b575da8055015c6b87693fd1d858ca Mon Sep 17 00:00:00 2001
From: Joachim Breitner <mail@joachim-breitner.de>
Date: Sat, 16 Mar 2019 19:03:03 +0100
Subject: [PATCH 1368/2196] Link to `builders-use-substitutes` in chapter on
 distributed builds

fixes #2730.
---
 doc/manual/advanced-topics/distributed-builds.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index bbb573e3540..ce2e077ed7b 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -180,4 +180,8 @@ builders = @/etc/nix/machines
 causes the list of machines in <filename>/etc/nix/machines</filename>
 to be included. (This is the default.)</para>
 
+<para>If you want the builders to use caches, you likely want to set
+the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
+in your local <filename>nix.conf</filename>.</para>
+
 </chapter>

From 6f093073b6ff38fb2c4e22331e73930baa1b30e7 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Thu, 21 Mar 2019 09:35:43 +0100
Subject: [PATCH 1369/2196] manual: include builtins.* for globally available
 builtins

This improves searchability.
---
 doc/manual/expressions/builtins.xml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 22d998bd0ed..cefc8b78eeb 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -23,6 +23,7 @@ available as <function>builtins.derivation</function>.</para>
 
   <varlistentry xml:id='builtin-abort'>
     <term><function>abort</function> <replaceable>s</replaceable></term>
+    <term><function>builtins.abort</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Abort Nix expression evaluation, print error
     message <replaceable>s</replaceable>.</para></listitem>
@@ -251,6 +252,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   <varlistentry xml:id='builtin-derivation'>
     <term><function>derivation</function>
     <replaceable>attrs</replaceable></term>
+    <term><function>builtins.derivation</function>
+    <replaceable>attrs</replaceable></term>
 
     <listitem><para><function>derivation</function> is described in
     <xref linkend='ssec-derivation' />.</para></listitem>
@@ -260,6 +263,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
   <varlistentry xml:id='builtin-dirOf'>
     <term><function>dirOf</function> <replaceable>s</replaceable></term>
+    <term><function>builtins.dirOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the directory part of the string
     <replaceable>s</replaceable>, that is, everything before the final
@@ -318,6 +322,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   <varlistentry xml:id='builtin-fetchTarball'>
     <term><function>fetchTarball</function>
     <replaceable>url</replaceable></term>
+    <term><function>builtins.fetchTarball</function>
+    <replaceable>url</replaceable></term>
 
     <listitem><para>Download the specified URL, unpack it and return
     the path of the unpacked tree. The file must be a tape archive
@@ -714,6 +720,8 @@ builtins.genList (x: x * x) 5
   <varlistentry xml:id='builtin-import'>
     <term><function>import</function>
     <replaceable>path</replaceable></term>
+    <term><function>builtins.import</function>
+    <replaceable>path</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
@@ -857,6 +865,8 @@ x: x + 456</programlisting>
   <varlistentry xml:id='builtin-isNull'>
     <term><function>isNull</function>
     <replaceable>e</replaceable></term>
+    <term><function>builtins.isNull</function>
+    <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
     <replaceable>e</replaceable> evaluates to <literal>null</literal>,
@@ -925,6 +935,8 @@ builtins.listToAttrs
   <varlistentry xml:id='builtin-map'>
     <term><function>map</function>
     <replaceable>f</replaceable> <replaceable>list</replaceable></term>
+    <term><function>builtins.map</function>
+    <replaceable>f</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Apply the function <replaceable>f</replaceable> to
     each element in the list <replaceable>list</replaceable>.  For
@@ -1119,6 +1131,8 @@ Evaluates to <literal>[ "foo" ]</literal>.
   <varlistentry xml:id='builtin-removeAttrs'>
     <term><function>removeAttrs</function>
     <replaceable>set</replaceable> <replaceable>list</replaceable></term>
+    <term><function>builtins.removeAttrs</function>
+    <replaceable>set</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Remove the attributes listed in
     <replaceable>list</replaceable> from
@@ -1287,6 +1301,8 @@ builtins.substring 0 3 "nixos"
   <varlistentry xml:id='builtin-throw'>
     <term><function>throw</function>
     <replaceable>s</replaceable></term>
+    <term><function>builtins.throw</function>
+    <replaceable>s</replaceable></term>
 
     <listitem><para>Throw an error message
     <replaceable>s</replaceable>.  This usually aborts Nix expression
@@ -1405,6 +1421,7 @@ in foo</programlisting>
 
   <varlistentry xml:id='builtin-toString'>
     <term><function>toString</function> <replaceable>e</replaceable></term>
+    <term><function>builtins.toString</function> <replaceable>e</replaceable></term>
 
     <listitem><para>Convert the expression
     <replaceable>e</replaceable> to a string.

From 2aa89daab3a8351a03f6d57d3e1f737cccd59850 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Thu, 21 Mar 2019 14:32:20 +0100
Subject: [PATCH 1370/2196] eval: improve type description for primops and
 applied primops

This can make type errors a little easier to understand.
---
 src/libexpr/eval.cc | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2a194d0e08c..211f7a55f73 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -130,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 }
 
 
+const Value *getPrimOp(const Value &v) {
+    const Value * primOp = &v;
+    while (primOp->type == tPrimOpApp) {
+        primOp = primOp->primOpApp.left;
+    }
+    assert(primOp->type == tPrimOp);
+    return primOp;
+}
+
+
 string showType(const Value & v)
 {
     switch (v.type) {
@@ -144,8 +154,10 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
+        case tPrimOp:
+            return fmt("the built-in function '%s'", string(v.primOp->name));
+        case tPrimOpApp:
+            return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tFloat: return "a float";
     }

From fcd766097636943ff11b84747d11a4e52d3d8e38 Mon Sep 17 00:00:00 2001
From: xbreak <rosenquist.calle@gmail.com>
Date: Sun, 24 Mar 2019 09:39:48 +0000
Subject: [PATCH 1371/2196] repl: Restore CTRL-C behaviour

Install signal handler during `readline` to handle SIGINT to abort
partially typed expressions.
---
 src/nix/repl.cc | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 227affc60e2..d8f81214906 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -192,6 +192,14 @@ static int listPossibleCallback(char *s, char ***avp) {
   return ac;
 }
 
+namespace {
+    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+    volatile sig_atomic_t g_signal_received = 0;
+
+    void sigintHandler(int signo) {
+        g_signal_received = signo;
+    }
+}
 
 void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
@@ -251,8 +259,40 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 
 bool NixRepl::getLine(string & input, const std::string &prompt)
 {
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
+    restoreSignals();
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
     if (!s)
       return false;
     input += s;

From 514b3c7f8345cfcbbe166981214497ed9d93ae18 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 29 Jan 2018 12:36:59 +0000
Subject: [PATCH 1372/2196] Add isPath primop

this is added for completeness' sake since all the other possible
`builtins.typeOf` results have a corresponding `builtins.is<Type>`
---
 doc/manual/expressions/builtins.xml | 8 ++++++++
 src/libexpr/primops.cc              | 7 +++++++
 tests/lang/eval-okay-types.nix      | 2 ++
 3 files changed, 17 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index cefc8b78eeb..0fb5261b384 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -861,6 +861,14 @@ x: x + 456</programlisting>
 
   </varlistentry>
 
+  <varlistentry><term><function>builtins.isPath</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a path, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
 
   <varlistentry xml:id='builtin-isNull'>
     <term><function>isNull</function>
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 6b0c55e72cd..39073725e9c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu
     mkBool(v, args[0]->type == tBool);
 }
 
+/* Determine whether the argument is a path. */
+static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    mkBool(v, args[0]->type == tPath);
+}
 
 struct CompareValues
 {
@@ -2169,6 +2175,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__isInt", 1, prim_isInt);
     addPrimOp("__isFloat", 1, prim_isFloat);
     addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__isPath", 1, prim_isPath);
     addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix
index a34775f5e60..9b58be5d1dd 100644
--- a/tests/lang/eval-okay-types.nix
+++ b/tests/lang/eval-okay-types.nix
@@ -20,6 +20,8 @@ with builtins;
   (isFloat (1 - 2.0))
   (isBool (true && false))
   (isBool null)
+  (isPath /nix/store)
+  (isPath ./.)
   (isAttrs { x = 123; })
   (isAttrs null)
   (typeOf (3 * 4))

From 0bebca402afa5f334c6d268d937097efbe1762d5 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 14 Mar 2019 09:55:30 -0500
Subject: [PATCH 1373/2196] version -> .version, avoid conflict with C++20
 <version>

---
 version => .version | 0
 release.nix         | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename version => .version (100%)

diff --git a/version b/.version
similarity index 100%
rename from version
rename to .version
diff --git a/release.nix b/release.nix
index 5c0027301b3..55808407eb8 100644
--- a/release.nix
+++ b/release.nix
@@ -18,7 +18,7 @@ let
 
       releaseTools.sourceTarball {
         name = "nix-tarball";
-        version = builtins.readFile ./version;
+        version = builtins.readFile ./.version;
         versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
         src = nix;
         inherit officialRelease;

From 42e2d5e7b7214772a0afa57e87de08715994dc29 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 14 Mar 2019 10:00:00 -0500
Subject: [PATCH 1374/2196] store-api.hh: add missing include for unordered_map

---
 src/libstore/store-api.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 40887b6aa3a..7a1b31d0ff5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -11,6 +11,7 @@
 #include <atomic>
 #include <limits>
 #include <map>
+#include <unordered_map>
 #include <unordered_set>
 #include <memory>
 #include <string>

From 1f9c8cd68b9cd06a7c604ee7a86a83eef676273c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 14 Mar 2019 10:21:22 -0500
Subject: [PATCH 1375/2196] configure.ac: update for new version file location

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 4eea008b02a..f5b1614f19f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"]))
+AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"]))
 AC_CONFIG_SRCDIR(README.md)
 AC_CONFIG_AUX_DIR(config)
 

From 63e7fc509628874bf60d9e436af387bfb1bc00a5 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Thu, 14 Mar 2019 10:28:10 -0500
Subject: [PATCH 1376/2196] perl/configure.ac: fix for new version location too

---
 perl/configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/configure.ac b/perl/configure.ac
index 9f49db4d281..966700695ff 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"]))
+AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"]))
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 

From 75ec68f93ac98956552c06a7fdf4dc8da6841a65 Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Mon, 25 Mar 2019 20:55:28 -0400
Subject: [PATCH 1377/2196] manual: mention the "channel:" shorthand for
 NIX_PATH

Bumped to 15.09 because older channels, when downloaded from
nixos.org, require firefox to be accessed via `pkgs.firefox`
---
 doc/manual/command-ref/env-common.xml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 361d3e2b033..c532ffddea2 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -52,10 +52,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
-    14.12 channel.</para>
+    15.09 channel.</para>
+
+    <para>A following shorthand can be used to refer to the official channels:
+    
+    <screen>nixpkgs=channel:nixos-15.09</screen>
+    </para>
 
     <para>The search path can be extended using the <option
     linkend="opt-I">-I</option> option, which takes precedence over

From d854e7dfd6d512c8ed687b61d7aae3358eda71cd Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Tue, 26 Mar 2019 21:08:22 -0400
Subject: [PATCH 1378/2196] install-multi-user: Detect and fail lack of systemd
 separately

Otherwise, the user is shown:

```
Sorry, I don't know what to do on Linux
```

Which is... not exactly right.
---
 scripts/install-multi-user.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 4b65783a20f..80d1c73fbb4 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -753,9 +753,13 @@ main() {
     if [ "$(uname -s)" = "Darwin" ]; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
-    elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
-        # shellcheck source=./install-systemd-multi-user.sh
-        . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+    elif [ "$(uname -s)" = "Linux" ]; then
+        if [ -e /run/systemd/system ]; then
+            # shellcheck source=./install-systemd-multi-user.sh
+            . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
+        else
+            failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)"
+        fi
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi

From dbe4c043d75f90f30932d326d2c80fd030b1f06d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Wed, 27 Mar 2019 16:18:53 +0100
Subject: [PATCH 1379/2196] install-multi-user: reduce max-jobs from 32 to 1

Having max-jobs = 32 ($NIX_USER_COUNT is hardcoded to that value) may
severely overload the machine. The nix.conf(5) manual page says max-jobs
defaults to 1, so let's use that value.

NOTE: Both max-jobs and cores are now being set to their default value,
so they can be removed alltogether.
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 4b65783a20f..6a76761cc4d 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -742,7 +742,7 @@ place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
 build-users-group = $NIX_BUILD_GROUP_NAME
 
-max-jobs = $NIX_USER_COUNT
+max-jobs = 1
 cores = 1
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \

From 07d9981f34c8423e74a8dc1b4f978fe58b421aee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Wed, 27 Mar 2019 16:24:58 +0100
Subject: [PATCH 1380/2196] install-multi-user: remove unneeded settings from
 nix.conf

Hardcoding the "max-jobs" and "cores" settings in nix.conf at install
time, to the same value as Nix' built-in default, makes little sense to
me.
---
 scripts/install-multi-user.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 6a76761cc4d..1e1db21c3ec 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -741,9 +741,6 @@ setup_default_profile() {
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
 build-users-group = $NIX_BUILD_GROUP_NAME
-
-max-jobs = 1
-cores = 1
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \
           install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf

From e489f5cabf95aaaaf0e67bb52d27dcd13a6cb1c0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 21:07:04 +0100
Subject: [PATCH 1381/2196] Update eval-okay-types.exp to match #1828

---
 tests/lang/eval-okay-types.exp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/lang/eval-okay-types.exp b/tests/lang/eval-okay-types.exp
index 9a8ea0bcbd8..92a15329935 100644
--- a/tests/lang/eval-okay-types.exp
+++ b/tests/lang/eval-okay-types.exp
@@ -1 +1 @@
-[ true false true false true false true false true true true true true true true true true true true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ]
+[ true false true false true false true false true true true true true true true true true true true false true true true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ]

From f32fbf952d8eb15963da65a162da169de72f36fa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 21:09:31 +0100
Subject: [PATCH 1382/2196] Fix Bison 3.3 warning

---
 src/libexpr/parser.y | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index cbd576d7d12..7870393076b 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -1,7 +1,7 @@
 %glr-parser
 %pure-parser
 %locations
-%error-verbose
+%define parse.error verbose
 %defines
 /* %no-lines */
 %parse-param { void * scanner }

From caa76c369a6d77eb3ccaa14bd8144581c8cd50a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sun, 31 Mar 2019 03:47:43 +0200
Subject: [PATCH 1383/2196] docs: Mention `--max-jobs 0` to build remotely only

---
 doc/manual/advanced-topics/distributed-builds.xml |  3 +++
 doc/manual/command-ref/opt-common.xml             | 12 ++++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml
index ce2e077ed7b..9ac4a92cd5b 100644
--- a/doc/manual/advanced-topics/distributed-builds.xml
+++ b/doc/manual/advanced-topics/distributed-builds.xml
@@ -184,4 +184,7 @@ to be included. (This is the default.)</para>
 the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
 in your local <filename>nix.conf</filename>.</para>
 
+<para>To build only on remote builders and disable building on the local machine,
+you can use the option <option>--max-jobs 0</option>.</para>
+
 </chapter>
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 4c572e12944..b8a2f260e8f 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -107,14 +107,22 @@
 <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
 <replaceable>number</replaceable></term>
 
-  <listitem><para>Sets the maximum number of build jobs that Nix will
+  <listitem>
+
+  <para>Sets the maximum number of build jobs that Nix will
   perform in parallel to the specified number.  Specify
   <literal>auto</literal> to use the number of CPUs in the system.
   The default is specified by the <link
   linkend='conf-max-jobs'><literal>max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.</para></listitem>
+  exploit I/O latency.</para>
+
+  <para> Setting it to <literal>0</literal> disallows building on the local
+  machine, which is useful when you want builds to happen only on remote
+  builders.</para>
+
+  </listitem>
 
 </varlistentry>
 

From 2f59b302517dff622743eb081884deacc906b3d1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 Apr 2019 21:09:25 +0200
Subject: [PATCH 1384/2196] Use Nixpkgs 19.03

---
 release.nix | 2 +-
 shell.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 55808407eb8..ab13451ff3d 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }
+, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
diff --git a/shell.nix b/shell.nix
index 817684b7646..73e75fb29c4 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {};
+with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From 2bc6304793e14e44d6d24c1fb35666352c7106f2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 Apr 2019 21:09:35 +0200
Subject: [PATCH 1385/2196] getMachines(): Cache result

---
 src/libstore/machines.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index edd03d14783..f848582dafd 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines)
 
 Machines getMachines()
 {
-    Machines machines;
-
-    parseMachines(settings.builders, machines);
-
+    static auto machines = [&]() {
+        Machines machines;
+        parseMachines(settings.builders, machines);
+        return machines;
+    }();
     return machines;
 }
 

From 41ba5135e0a2d9940c4e8cbedff44eb1a47af782 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 Apr 2019 21:09:49 +0200
Subject: [PATCH 1386/2196] primeCache(): Barf if builds are needed but not
 allowed

Fixes #2753.
---
 src/libstore/build.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 6b88b130730..53a0c743b03 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -12,6 +12,7 @@
 #include "json.hh"
 #include "nar-info.hh"
 #include "parsed-derivations.hh"
+#include "machines.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -4411,6 +4412,11 @@ static void primeCache(Store & store, const PathSet & paths)
     PathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+    if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
+        throw Error(
+            "%d derivations need to be built, but neither local builds ('--max-jobs') "
+            "nor remote builds ('--builders') are enabled", willBuild.size());
 }
 
 

From bb6e6923f25841874b6a915d234d884ddd4c92dd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 Apr 2019 23:04:13 +0200
Subject: [PATCH 1387/2196] Add environment variable NIX_SHOW_SYMBOLS for
 dumping the symbol table

---
 src/libexpr/eval.cc         | 6 ++++++
 src/libexpr/symbol-table.hh | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 211f7a55f73..d8e10d9f20e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1811,6 +1811,7 @@ void EvalState::printStats()
             gc.attr("totalBytes", totalBytes);
         }
 #endif
+
         if (countCalls) {
             {
                 auto obj = topObj.object("primops");
@@ -1846,6 +1847,11 @@ void EvalState::printStats()
                 }
             }
         }
+
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index 44929f7eea0..91faea122ce 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -75,6 +75,13 @@ public:
     }
 
     size_t totalSize() const;
+
+    template<typename T>
+    void dump(T callback)
+    {
+        for (auto & s : symbols)
+            callback(s);
+    }
 };
 
 }

From 288f93cec0efeba5a767c09e4d58203c67704a6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sun, 14 Apr 2019 03:18:33 +0200
Subject: [PATCH 1388/2196] manual: "Nix Package collection" -> "Nixpkgs
 package collection".

Makes difference between Nix and Nixpkgs clearer to avoid
some common confusion this sentence on IRC.

Also disambiguate an "it" reference.
---
 doc/manual/packages/basic-package-mgmt.xml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml
index e8d1419da09..0f21297f31b 100644
--- a/doc/manual/packages/basic-package-mgmt.xml
+++ b/doc/manual/packages/basic-package-mgmt.xml
@@ -24,11 +24,11 @@ symlinks to the files of the active applications.  </para>
 <para>Components are installed from a set of <emphasis>Nix
 expressions</emphasis> that tell Nix how to build those packages,
 including, if necessary, their dependencies.  There is a collection of
-Nix expressions called the Nix Package collection that contains
+Nix expressions called the Nixpkgs package collection that contains
 packages ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
-tied to the Nix Package collection; you could write your own Nix
-expressions based on it, or completely new ones.)</para>
+tied to the Nixpkgs package collection; you could write your own Nix
+expressions based on Nixpkgs, or completely new ones.)</para>
 
 <para>You can manually download the latest version of Nixpkgs from
 <link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However,

From 5112a33fb17f792ceb6d641738277cbbe6a58bfc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Apr 2019 19:17:17 +0200
Subject: [PATCH 1389/2196] Fix release script

---
 maintainers/upload-release.pl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 8432c95960c..1cdf5ed16dc 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -67,10 +67,10 @@ sub downloadFile {
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt\n";
+        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 

From b614e0e53da1f825e198996f88935a1d1a89725f Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 17 Apr 2019 21:42:41 +0200
Subject: [PATCH 1390/2196] build: only skip hash rewriting for sandboxing on
 linux

The sandbox on darwin, and possibly other future platforms, doesn't have
enough isolation to redirect outputs without hash rewriting.
---
 src/libstore/build.cc | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 53a0c743b03..dbadfacd51e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -803,6 +803,9 @@ class DerivationGoal : public Goal
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
 
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite;
+
     Path chrootRootDir;
 
     /* RAII object to delete the chroot directory. */
@@ -994,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
+#if __linux__
+    needsHashRewrite = !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    needsHashRewrite = true;
+#endif
+
     state = &DerivationGoal::getDerivation;
     name = (format("building of '%1%'") % drvPath).str();
     trace("created");
@@ -2073,7 +2083,7 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    else {
+    if (needsHashRewrite) {
 
         if (pathExists(homeDir))
             throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@@ -2873,6 +2883,10 @@ void DerivationGoal::runChild()
                 for (auto & i : missingPaths) {
                     sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
                 }
+                /* Also add redirected outputs to the chroot */
+                for (auto & i : redirectedOutputs) {
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
+                }
                 sandboxProfile += ")\n";
 
                 /* Our inputs (transitive dependencies and any impurities computed above)
@@ -3051,7 +3065,9 @@ void DerivationGoal::registerOutputs()
                         throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
-        } else {
+        }
+
+        if (needsHashRewrite) {
             Path redirected = redirectedOutputs[path];
             if (buildMode == bmRepair
                 && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()

From cbc7d9a4124343c7ba68b695b5e9b3c8188fb267 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Tue, 30 Apr 2019 22:43:24 -0400
Subject: [PATCH 1391/2196] findRootsNoTemp: fixes comment about
 findRuntimeRoots

The NIX_ROOT_FINDER environment variable was removed in
3c46fe62b833a4e66845665edc99555022d3d98c when porting from perl to C.
---
 src/libstore/gc.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index d8a5da0d49e..26e2b0dca7c 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -326,10 +326,9 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
-    /* Add additional roots returned by the program specified by the
-       NIX_ROOT_FINDER environment variable.  This is typically used
-       to add running programs to the set of roots (to prevent them
-       from being garbage collected). */
+    /* Add additional roots returned by different platforms-specific
+       heuristics.  This is typically used to add running programs to
+       the set of roots (to prevent them from being garbage collected). */
     findRuntimeRoots(roots, censor);
 }
 

From f9a2ea44867cd1dbb408bca4df0ced806137b7f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 May 2019 10:44:32 +0200
Subject: [PATCH 1392/2196] Fix "Bad system call" running i686-linux binaries
 on x86_64-linux

To determine which seccomp filters to install, we were incorrectly
using settings.thisSystem, which doesn't denote the actual system when
--system is used.

Fixes #2791.
---
 src/libstore/build.cc | 8 ++++----
 src/libutil/util.cc   | 3 +++
 src/libutil/util.hh   | 4 ++++
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dbadfacd51e..91eb97dfb87 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2510,17 +2510,17 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
         throw SysError("unable to add X32 seccomp architecture");
 
-    if (settings.thisSystem == "aarch64-linux" &&
+    if (nativeSystem == "aarch64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes.");
+        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e3dcd246c68..a7170566533 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -38,6 +38,9 @@ extern char * * environ;
 namespace nix {
 
 
+const std::string nativeSystem = SYSTEM;
+
+
 BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
     prefix_ = fs.s + prefix_;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 9f239bff371..54936a5cb10 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -30,6 +30,10 @@ struct Sink;
 struct Source;
 
 
+/* The system for which Nix is compiled. */
+extern const std::string nativeSystem;
+
+
 /* Return an environment variable. */
 string getEnv(const string & key, const string & def = "");
 

From 3f192ac80ca421888c668896b63485486e1397ae Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Fri, 3 May 2019 14:30:29 +0200
Subject: [PATCH 1393/2196] Add builtins.hashFile

For text files it is possible to do it like so:
`builtins.hashString "sha256" (builtins.readFile /tmp/a)`
but that doesn't work for binary files.

With builtins.hashFile any kind of file can be conveniently hashed.
---
 doc/manual/expressions/builtins.xml              |  13 +++++++++++++
 src/libexpr/primops.cc                           |  15 +++++++++++++++
 tests/lang/binary-data                           | Bin 0 -> 1024 bytes
 tests/lang/eval-fail-hashfile-missing.nix        |   5 +++++
 tests/lang/eval-okay-hash.exp                    |   1 -
 tests/lang/eval-okay-hashfile.exp                |   1 +
 tests/lang/eval-okay-hashfile.nix                |   4 ++++
 tests/lang/eval-okay-hashstring.exp              |   1 +
 ...al-okay-hash.nix => eval-okay-hashstring.nix} |   0
 9 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 tests/lang/binary-data
 create mode 100644 tests/lang/eval-fail-hashfile-missing.nix
 create mode 100644 tests/lang/eval-okay-hashfile.exp
 create mode 100644 tests/lang/eval-okay-hashfile.nix
 create mode 100644 tests/lang/eval-okay-hashstring.exp
 rename tests/lang/{eval-okay-hash.nix => eval-okay-hashstring.nix} (100%)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 0fb5261b384..a87639a075a 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -705,6 +705,19 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
+  <varlistentry xml:id='builtin-hashFile'>
+    <term><function>builtins.hashFile</function>
+    <replaceable>type</replaceable> <replaceable>p</replaceable></term>
+
+    <listitem><para>Return a base-16 representation of the
+    cryptographic hash of the file at path <replaceable>p</replaceable>.  The
+    hash algorithm specified by <replaceable>type</replaceable> must
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
+    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id='builtin-head'>
     <term><function>builtins.head</function>
     <replaceable>list</replaceable></term>
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 39073725e9c..06f577f36fc 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -923,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+/* Return the cryptographic hash of a file in base-16. */
+static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0], pos);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+
+    PathSet context; // discarded
+    Path p = state.coerceToPath(pos, *args[1], context);
+
+    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+}
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2202,6 +2216,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__readFile", 1, prim_readFile);
     addPrimOp("__readDir", 1, prim_readDir);
     addPrimOp("__findFile", 2, prim_findFile);
+    addPrimOp("__hashFile", 2, prim_hashFile);
 
     // Creating files
     addPrimOp("__toXML", 1, prim_toXML);
diff --git a/tests/lang/binary-data b/tests/lang/binary-data
new file mode 100644
index 0000000000000000000000000000000000000000..06d7405020018ddf3cacee90fd4af10487da3d20
GIT binary patch
literal 1024
ScmZQz7zLvtFd70QH3R?z00031

literal 0
HcmV?d00001

diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix
new file mode 100644
index 00000000000..42fb1ec7ed8
--- /dev/null
+++ b/tests/lang/eval-fail-hashfile-missing.nix
@@ -0,0 +1,5 @@
+let
+  paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ];
+in
+  builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])
+
diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp
index d720a082ddb..e69de29bb2d 100644
--- a/tests/lang/eval-okay-hash.exp
+++ b/tests/lang/eval-okay-hash.exp
@@ -1 +0,0 @@
-[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]
diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/lang/eval-okay-hashfile.exp
new file mode 100644
index 00000000000..ff1e8293ef2
--- /dev/null
+++ b/tests/lang/eval-okay-hashfile.exp
@@ -0,0 +1 @@
+[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ]
diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/lang/eval-okay-hashfile.nix
new file mode 100644
index 00000000000..aff5a185681
--- /dev/null
+++ b/tests/lang/eval-okay-hashfile.nix
@@ -0,0 +1,4 @@
+let
+  paths = [ ./data ./binary-data ];
+in
+  builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])
diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/lang/eval-okay-hashstring.exp
new file mode 100644
index 00000000000..d720a082ddb
--- /dev/null
+++ b/tests/lang/eval-okay-hashstring.exp
@@ -0,0 +1 @@
+[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]
diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hashstring.nix
similarity index 100%
rename from tests/lang/eval-okay-hash.nix
rename to tests/lang/eval-okay-hashstring.nix

From a83486187616fd2de877ce72f5fd419385a2c541 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Tue, 7 May 2019 17:32:01 -0500
Subject: [PATCH 1394/2196] fix hashfile test that wasn't failing due to eval
 laziness

See:
https://github.com/NixOS/nix/commit/7becb1bf1c2ec1544a5374580a97b36273506baf#r33450554
---
 tests/lang/eval-fail-hashfile-missing.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix
index 42fb1ec7ed8..ce098b82380 100644
--- a/tests/lang/eval-fail-hashfile-missing.nix
+++ b/tests/lang/eval-fail-hashfile-missing.nix
@@ -1,5 +1,5 @@
 let
   paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ];
 in
-  builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])
+  toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]))
 

From 7c6391ddc730519a632cc0ee526c94a04812d871 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 8 May 2019 14:18:57 +0200
Subject: [PATCH 1395/2196] nix-shell: Don't absolutize '-p' expressions

This prevents spurious syscalls like

  25011 lstat("/home/eelco/with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ (hello) ]; } \"\"", 0x7ffe9c67f580) = -1 ENOENT (No such file or directory)
---
 src/nix-build/nix-build.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 618895d387d..c6a4d416648 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -274,19 +274,21 @@ static void _main(int argc, char * * argv)
         exprs = {state->parseStdin()};
     else
         for (auto i : left) {
-            auto absolute = i;
-            try {
-                absolute = canonPath(absPath(i), true);
-            } catch (Error e) {};
             if (fromArgs)
                 exprs.push_back(state->parseExprFromString(i, absPath(".")));
-            else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+            else {
+                auto absolute = i;
+                try {
+                    absolute = canonPath(absPath(i), true);
+                } catch (Error e) {};
+                if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
                 drvs.push_back(DrvInfo(*state, store, absolute));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
                 exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
                     inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+            }
         }
 
     /* Evaluate them into derivations. */

From 6ade7ec022c836b7d1f9bd06be45e2c07835ec8c Mon Sep 17 00:00:00 2001
From: Florian Klink <flokli@flokli.de>
Date: Mon, 6 May 2019 22:23:15 +0200
Subject: [PATCH 1396/2196] progress-bar: hide expected if expected is 0
 (unknown)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sometimes, "expected" can be "0", but in fact means "unknown".

This is for example the case when downloading a file while the http
server doesn't send the `Content-Length` header, like when running `nix
build` pointing to a nixpkgs checkout streamed from GitHub:

⇒  nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello
[1.8/0.0 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz'

In that case, don't show that weird progress bar, but only the (slowly
increasing) downloaded size ("done").

⇒  nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello
[1.8 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz'

This commit also updates fmt calls with three numbers (when something is
currently 'running' too) - I'm not sure if this can be provoked, but
showing "0" as expected doesn't make any sense, as we're obviously doing
more than nothing.
---
 src/nix/progress-bar.cc | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 40b905ba324..8da72bc9481 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -333,11 +333,18 @@ class ProgressBar : public Logger
 
             if (running || done || expected || failed) {
                 if (running)
-                    s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        running / unit, done / unit, expected / unit);
+                    if (expected != 0)
+                        s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                            running / unit, done / unit, expected / unit);
+                    else
+                        s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL,
+                            running / unit, done / unit);
                 else if (expected != done)
-                    s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
-                        done / unit, expected / unit);
+                    if (expected != 0)
+                        s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt,
+                            done / unit, expected / unit);
+                    else
+                        s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit);
                 else
                     s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit);
                 s = fmt(itemFmt, s);

From ff6867ab94cbe9ddcb4ba18d68a4a2dcb79b865d Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 12 May 2019 15:53:40 +0200
Subject: [PATCH 1397/2196] build: move needsHashRewrite initialization to
 startBuilder

The value of useChroot is not set yet in the constructor, resulting in
hash rewriting being enabled in certain cases where it should not be.

Fixes #2801
---
 src/libstore/build.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 91eb97dfb87..30825add43b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -997,13 +997,6 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-#if __linux__
-    needsHashRewrite = !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    needsHashRewrite = true;
-#endif
-
     state = &DerivationGoal::getDerivation;
     name = (format("building of '%1%'") % drvPath).str();
     trace("created");
@@ -1852,6 +1845,13 @@ void DerivationGoal::startBuilder()
         #endif
     }
 
+#if __linux__
+    needsHashRewrite = !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    needsHashRewrite = true;
+#endif
+
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
     if (settings.buildUsersGroup != "" && getuid() == 0) {

From d75bdb5793e5ebf9e480f5a0012d141347725801 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 12 May 2019 16:46:21 +0200
Subject: [PATCH 1398/2196] build: add test for sandboxed --check

---
 tests/linux-sandbox.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index acfd46c5417..52967d07dda 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -25,3 +25,6 @@ nix path-info -r $outPath | grep input-2
 nix ls-store -R -l $outPath | grep foobar
 
 nix cat-store $outPath/foobar | grep FOOBAR
+
+# Test --check without hash rewriting.
+nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store

From c78686e411e0a14cff51836fe6c35d7584171df3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 10 May 2019 16:39:31 -0400
Subject: [PATCH 1399/2196] build: run diff-hook under --check and document
 diff-hook

---
 .../advanced-topics/advanced-topics.xml       |   1 +
 doc/manual/advanced-topics/diff-hook.xml      | 207 ++++++++++++++++++
 doc/manual/command-ref/conf-file.xml          |  81 ++++++-
 src/libstore/build.cc                         |  30 ++-
 4 files changed, 303 insertions(+), 16 deletions(-)
 create mode 100644 doc/manual/advanced-topics/diff-hook.xml

diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml
index b710f9f2b51..c304367aaf8 100644
--- a/doc/manual/advanced-topics/advanced-topics.xml
+++ b/doc/manual/advanced-topics/advanced-topics.xml
@@ -7,5 +7,6 @@
 <title>Advanced Topics</title>
 
 <xi:include href="distributed-builds.xml" />
+<xi:include href="diff-hook.xml" />
 
 </part>
diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml
new file mode 100644
index 00000000000..d2613f6df22
--- /dev/null
+++ b/doc/manual/advanced-topics/diff-hook.xml
@@ -0,0 +1,207 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="chap-diff-hook"
+      version="5.0"
+      >
+
+<title>Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option></title>
+
+<subtitle>Check build reproducibility by running builds multiple times
+and comparing their results.</subtitle>
+
+<para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to
+compare build results when two builds produce different results. Note:
+this hook is only executed if the results are not the same, this hook
+is not used for determining if the results are the same.</para>
+
+<para>For purposes of demonstration, we'll use the following Nix file,
+<filename>deterministic.nix</filename> for testing:</para>
+
+<programlisting>
+let
+  inherit (import &lt;nixpkgs&gt; {}) runCommand;
+in {
+  stable = runCommand "stable" {} ''
+    touch $out
+  '';
+
+  unstable = runCommand "unstable" {} ''
+    echo $RANDOM > $out
+  '';
+}
+</programlisting>
+
+<para>Additionally, <filename>nix.conf</filename> contains:
+
+<programlisting>
+diff-hook = /etc/nix/my-diff-hook
+run-diff-hook = true
+</programlisting>
+
+where <filename>/etc/nix/my-diff-hook</filename> is an executable
+file containing:
+
+<programlisting>
+#!/bin/sh
+exec &gt;&amp;2
+echo "For derivation $3:"
+/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2"
+</programlisting>
+
+<warning>
+  <para>The diff hook can be run as root. Take care to run as little
+  as possible as root, for this example we use <command>runuser</command>
+  to drop privileges.
+  </para>
+</warning>
+</para>
+
+<section>
+  <title>
+    Spot-Checking Build Determinism
+  </title>
+
+  <para>
+    Verify a path which already exists in the Nix store by passing
+    <option>--check</option> to the build command.
+  </para>
+
+  <para>If the build passes and is deterministic, Nix will exit with a
+  status code of 0:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A stable
+these derivations will be built:
+  /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
+building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+
+$ nix-build ./deterministic.nix -A stable --check
+checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
+/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
+</screen>
+
+  <para>If the build is not deterministic, Nix will exit with a status
+  code of 1:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A unstable
+these derivations will be built:
+  /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
+building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
+
+$ nix-build ./deterministic.nix -A unstable --check
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs
+</screen>
+
+<para>In the Nix daemon's log, we will now see:
+<screen>
+For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv:
+1c1
+&lt; 8108
+---
+&gt; 30204
+</screen>
+</para>
+
+  <para>Using <option>--check</option> with <option>--keep-failed</option>
+  will cause Nix to keep the second build's output in a special,
+  <literal>.check</literal> path:</para>
+
+  <screen>
+$ nix-build ./deterministic.nix -A unstable --check --keep-failed
+checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
+note: keeping build directory '/tmp/nix-build-unstable.drv-0'
+error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check'
+</screen>
+
+  <para>In particular, notice the
+  <literal>/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check</literal>
+  output. Nix has copied the build results to that directory where you
+  can examine it.</para>
+
+  <note xml:id="check-dirs-are-unregistered">
+    <title><literal>.check</literal> paths are not registered store paths</title>
+
+    <para>Check paths are not protected against garbage collection,
+    and this path will be deleted on the next garbage collection.</para>
+
+    <para>The path is guaranteed to be alive for the duration of
+    <xref linkend="conf-diff-hook" />'s execution, but may be deleted
+    any time after.</para>
+
+    <para>If the comparison is performed as part of automated tooling,
+    please use the diff-hook or author your tooling to handle the case
+    where the build was not deterministic and also a check path does
+    not exist.</para>
+  </note>
+
+  <para>
+    <option>--check</option> is only usable if the derivation has
+    been built on the system already. If the derivation has not been
+    built Nix will fail with the error:
+    <screen>
+error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible
+</screen>
+
+    Run the build without <option>--check</option>, and then try with
+    <option>--check</option> again.
+  </para>
+</section>
+
+<section>
+  <title>
+    Automatic and Optionally Enforced Determinism Verification
+  </title>
+
+  <para>
+    Automatically verify every build at build time by executing the
+    build multiple times.
+  </para>
+
+  <para>
+    Setting <xref linkend="conf-repeat" /> and
+    <xref linkend="conf-enforce-determinism" /> in your
+    <filename>nix.conf</filename> permits the automated verification
+    of every build Nix performs.
+  </para>
+
+  <para>
+    The following configuration will run each build three times, and
+    will require the build to be deterministic:
+
+    <programlisting>
+enforce-determinism = true
+repeat = 2
+</programlisting>
+  </para>
+
+  <para>
+    Setting <xref linkend="conf-enforce-determinism" /> to false as in
+    the following configuration will run the build multiple times,
+    execute the build hook, but will allow the build to succeed even
+    if it does not build reproducibly:
+
+    <programlisting>
+enforce-determinism = false
+repeat = 1
+</programlisting>
+  </para>
+
+  <para>
+    An example output of this configuration:
+    <screen>
+$ nix-build ./test.nix -A unstable
+these derivations will be built:
+  /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
+building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...
+output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round
+/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable
+</screen>
+  </para>
+</section>
+</chapter>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index f0da1f612fe..a1a5d6e1297 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -1,7 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
           xmlns:xi="http://www.w3.org/2001/XInclude"
-          xml:id="sec-conf-file">
+          xml:id="sec-conf-file"
+          version="5">
 
 <refmeta>
   <refentrytitle>nix.conf</refentrytitle>
@@ -240,6 +242,64 @@ false</literal>.</para>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term>
+  <listitem>
+    <para>
+      Absolute path to an executable capable of diffing build results.
+      The hook executes if <xref linkend="conf-run-diff-hook" /> is
+      true, and the output of a build is known to not be the same.
+      This program is not executed to determine if two results are the
+      same.
+    </para>
+
+    <warning>
+      <para>
+        The root user executes the diff hook in a daemonised
+        installation. See <xref linkend="chap-diff-hook" /> for
+        information on using the diff hook safely.
+      </para>
+    </warning>
+
+    <para>The diff hook program receives three parameters:</para>
+
+    <orderedlist>
+      <listitem>
+        <para>
+          A path to the previous build's results
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          A path to the current build's results
+        </para>
+      </listitem>
+
+      <listitem>
+        <para>
+          The path to the build's derivation
+        </para>
+      </listitem>
+    </orderedlist>
+
+    <para>The diff hook should not print data to stderr or stdout, as
+    output is not displayed to the user. However, if information is
+    printed, it will be printed in the <command>nix-daemon</command>
+    log.</para>
+
+    <para>When using the Nix daemon, <literal>diff-hook</literal> must
+    be set in the <filename>nix.conf</filename> configuration file, and
+    cannot be passed at the command line.
+    </para>
+  </listitem>
+  </varlistentry>
+
+  <varlistentry xml:id="conf-enforce-determinism">
+    <term><literal>enforce-determinism</literal></term>
+
+    <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-extra-sandbox-paths">
     <term><literal>extra-sandbox-paths</literal></term>
 
@@ -595,9 +655,9 @@ password <replaceable>my-password</replaceable>
     they are deterministic. The default value is 0. If the value is
     non-zero, every build is repeated the specified number of
     times. If the contents of any of the runs differs from the
-    previous ones, the build is rejected and the resulting store paths
-    are not registered as “valid” in Nix’s database.</para></listitem>
-
+    previous ones and <xref linkend="conf-enforce-determinism" /> is
+    true, the build is rejected and the resulting store paths are not
+    registered as “valid” in Nix’s database.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term>
@@ -628,6 +688,19 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term>
+  <listitem>
+    <para>
+      If true, enable the execution of <xref linkend="conf-diff-hook" />.
+    </para>
+
+    <para>
+      When using the Nix daemon, <literal>run-diff-hook</literal> must
+      be set in the <filename>nix.conf</filename> configuration file,
+      and cannot be passed at the command line.
+    </para>
+  </listitem>
+  </varlistentry>
 
   <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term>
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 91eb97dfb87..026828c535c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -461,6 +461,19 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
+void handleDiffHook(Path tryA, Path tryB, Path drvPath)
+{
+    auto diffHook = settings.diffHook;
+    if (diffHook != "" && settings.runDiffHook) {
+        try {
+            auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath});
+            if (diff != "")
+                printError(chomp(diff));
+        } catch (Error & error) {
+            printError("diff hook execution failed: %s", error.what());
+        }
+    }
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -3039,8 +3052,7 @@ void DerivationGoal::registerOutputs()
     InodesSeen inodesSeen;
 
     Path checkSuffix = ".check";
-    bool runDiffHook = settings.runDiffHook;
-    bool keepPreviousRound = settings.keepFailed || runDiffHook;
+    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
 
     std::exception_ptr delayedException;
 
@@ -3185,11 +3197,14 @@ void DerivationGoal::registerOutputs()
             if (!worker.store.isValidPath(path)) continue;
             auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
+                handleDiffHook(path, actualPath, drvPath);
+
                 if (settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
                         throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+
                     throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
@@ -3254,16 +3269,7 @@ void DerivationGoal::registerOutputs()
                     ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
                     : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
 
-                auto diffHook = settings.diffHook;
-                if (prevExists && diffHook != "" && runDiffHook) {
-                    try {
-                        auto diff = runProgram(diffHook, true, {prev, i->second.path});
-                        if (diff != "")
-                            printError(chomp(diff));
-                    } catch (Error & error) {
-                        printError("diff hook execution failed: %s", error.what());
-                    }
-                }
+                handleDiffHook(prev, i->second.path, drvPath);
 
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);

From 6df61db0600ca73ccd51e3e5bec5312a04e99da1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 10 May 2019 20:59:39 -0400
Subject: [PATCH 1400/2196] diff hook: execute as the build user, and pass the
 temp dir

---
 doc/manual/advanced-topics/diff-hook.xml | 12 +++----
 doc/manual/command-ref/conf-file.xml     | 20 ++++++++----
 src/libstore/build.cc                    | 41 +++++++++++++++++-------
 src/libutil/util.cc                      |  4 +--
 src/libutil/util.hh                      |  2 ++
 5 files changed, 51 insertions(+), 28 deletions(-)

diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml
index d2613f6df22..fb4bf819f94 100644
--- a/doc/manual/advanced-topics/diff-hook.xml
+++ b/doc/manual/advanced-topics/diff-hook.xml
@@ -46,17 +46,15 @@ file containing:
 #!/bin/sh
 exec &gt;&amp;2
 echo "For derivation $3:"
-/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2"
+/run/current-system/sw/bin/diff -r "$1" "$2"
 </programlisting>
 
-<warning>
-  <para>The diff hook can be run as root. Take care to run as little
-  as possible as root, for this example we use <command>runuser</command>
-  to drop privileges.
-  </para>
-</warning>
 </para>
 
+<para>The diff hook is executed by the same user and group who ran the
+build. However, the diff hook does not have write access to the store
+path just built.</para>
+
 <section>
   <title>
     Spot-Checking Build Determinism
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index a1a5d6e1297..c5f90481b13 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -252,13 +252,11 @@ false</literal>.</para>
       same.
     </para>
 
-    <warning>
-      <para>
-        The root user executes the diff hook in a daemonised
-        installation. See <xref linkend="chap-diff-hook" /> for
-        information on using the diff hook safely.
-      </para>
-    </warning>
+    <para>
+      The diff hook is executed by the same user and group who ran the
+      build. However, the diff hook does not have write access to the
+      store path just built.
+    </para>
 
     <para>The diff hook program receives three parameters:</para>
 
@@ -280,6 +278,14 @@ false</literal>.</para>
           The path to the build's derivation
         </para>
       </listitem>
+
+      <listitem>
+        <para>
+          The path to the build's scratch directory. This directory
+          will exist only if the build was run with
+          <option>--keep-failed</option>.
+        </para>
+      </listitem>
     </orderedlist>
 
     <para>The diff hook should not print data to stderr or stdout, as
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 026828c535c..f38d2eaa0cd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -461,17 +461,26 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
-void handleDiffHook(Path tryA, Path tryB, Path drvPath)
+void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
 {
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
-        try {
-            auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath});
-            if (diff != "")
-                printError(chomp(diff));
-        } catch (Error & error) {
-            printError("diff hook execution failed: %s", error.what());
-        }
+        auto wrapper = [&]() {
+            if (setgid(gid) == -1)
+                throw SysError("setgid failed");
+            if (setuid(uid) == -1)
+                throw SysError("setuid failed");
+
+            try {
+                auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir});
+                if (diff != "")
+                    printError(chomp(diff));
+            } catch (Error & error) {
+                printError("diff hook execution failed: %s", error.what());
+            }
+        };
+
+        doFork(allowVfork, wrapper);
     }
 }
 
@@ -3197,14 +3206,18 @@ void DerivationGoal::registerOutputs()
             if (!worker.store.isValidPath(path)) continue;
             auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
-                handleDiffHook(path, actualPath, drvPath);
-
-                if (settings.keepFailed) {
+                if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
                         throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
 
+                    handleDiffHook(
+                        !buildUser && !drv->isBuiltin(),
+                        buildUser ? buildUser->getUID() : getuid(),
+                        buildUser ? buildUser->getGID() : getgid(),
+                        path, dst, drvPath, tmpDir);
+
                     throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
@@ -3269,7 +3282,11 @@ void DerivationGoal::registerOutputs()
                     ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
                     : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
 
-                handleDiffHook(prev, i->second.path, drvPath);
+                handleDiffHook(
+                    !buildUser && !drv->isBuiltin(),
+                    buildUser ? buildUser->getUID() : getuid(),
+                    buildUser ? buildUser->getGID() : getgid(),
+                    prev, i->second.path, drvPath, tmpDir);
 
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a7170566533..0f4d3d92b86 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -914,8 +914,8 @@ void killUser(uid_t uid)
 
 /* Wrapper around vfork to prevent the child process from clobbering
    the caller's stack frame in the parent. */
-static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
-static pid_t doFork(bool allowVfork, std::function<void()> fun)
+pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
+pid_t doFork(bool allowVfork, std::function<void()> fun)
 {
 #ifdef __linux__
     pid_t pid = allowVfork ? vfork() : fork();
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 54936a5cb10..824a35b987e 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -265,6 +265,8 @@ string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
     const std::optional<std::string> & input = {});
 
+pid_t doFork(bool allowVfork, std::function<void()> fun);
+
 struct RunOptions
 {
     Path program;

From dde8eeb39ae9fb73011462c74e5fa6405e432147 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 11 May 2019 15:57:38 -0400
Subject: [PATCH 1401/2196] chdir, setgroups

---
 src/libstore/build.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f38d2eaa0cd..8397cd0d1dd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -466,8 +466,12 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB,
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
         auto wrapper = [&]() {
+            if (chdir("/") == -1)
+                throw SysError("chdir / failed");
             if (setgid(gid) == -1)
                 throw SysError("setgid failed");
+            if (setgroups(0, 0) == -1)
+                throw SysError("setgroups failed");
             if (setuid(uid) == -1)
                 throw SysError("setuid failed");
 

From b4a05edbfe49f87555fd284dfb0d6c56ed43217d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sat, 11 May 2019 16:35:53 -0400
Subject: [PATCH 1402/2196] runProgram: support gid, uid, chdir

---
 src/libstore/build.cc | 36 ++++++++++++++++--------------------
 src/libutil/util.cc   | 15 +++++++++++++--
 src/libutil/util.hh   |  5 +++--
 3 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8397cd0d1dd..8902e22bd8a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -465,26 +465,22 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB,
 {
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
-        auto wrapper = [&]() {
-            if (chdir("/") == -1)
-                throw SysError("chdir / failed");
-            if (setgid(gid) == -1)
-                throw SysError("setgid failed");
-            if (setgroups(0, 0) == -1)
-                throw SysError("setgroups failed");
-            if (setuid(uid) == -1)
-                throw SysError("setuid failed");
-
-            try {
-                auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir});
-                if (diff != "")
-                    printError(chomp(diff));
-            } catch (Error & error) {
-                printError("diff hook execution failed: %s", error.what());
-            }
-        };
-
-        doFork(allowVfork, wrapper);
+        try {
+            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
+            diffHookOptions.searchPath = true;
+            diffHookOptions.uid = uid;
+            diffHookOptions.gid = gid;
+            diffHookOptions.chdir = "/";
+
+            auto diffRes = runProgram(diffHookOptions);
+            if (!statusOk(diffRes.first))
+                throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)));
+
+            if (diffRes.second != "")
+                printError(chomp(diffRes.second));
+        } catch (Error & error) {
+            printError("diff hook execution failed: %s", error.what());
+        }
     }
 }
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 0f4d3d92b86..55b9144f314 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -16,6 +16,7 @@
 #include <future>
 
 #include <fcntl.h>
+#include <grp.h>
 #include <limits.h>
 #include <pwd.h>
 #include <sys/ioctl.h>
@@ -914,8 +915,8 @@ void killUser(uid_t uid)
 
 /* Wrapper around vfork to prevent the child process from clobbering
    the caller's stack frame in the parent. */
-pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
-pid_t doFork(bool allowVfork, std::function<void()> fun)
+static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
+static pid_t doFork(bool allowVfork, std::function<void()> fun)
 {
 #ifdef __linux__
     pid_t pid = allowVfork ? vfork() : fork();
@@ -1025,6 +1026,16 @@ void runProgram2(const RunOptions & options)
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
+        //if (options.chdir && chdir((*options.chdir).c_str()) == -1)
+        //    throw SysError("chdir failed");
+        if (options.gid && setgid(*options.gid) == -1)
+            throw SysError("setgid failed");
+        /* Drop all other groups if we're setgid. */
+        if (options.gid && setgroups(0, 0) == -1)
+            throw SysError("setgroups failed");
+        if (options.uid && setuid(*options.uid) == -1)
+            throw SysError("setuid failed");
+
         Strings args_(options.args);
         args_.push_front(options.program);
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 824a35b987e..7c57d0afad9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -265,10 +265,11 @@ string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings(),
     const std::optional<std::string> & input = {});
 
-pid_t doFork(bool allowVfork, std::function<void()> fun);
-
 struct RunOptions
 {
+    std::optional<uid_t> uid;
+    std::optional<uid_t> gid;
+    std::optional<Path> chdir;
     Path program;
     bool searchPath = true;
     Strings args;

From a5efe617862484ab7dd234a495d315e7b08aa519 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 12 May 2019 13:23:30 -0400
Subject: [PATCH 1403/2196] Clarify where output from the diff hook goes.

---
 doc/manual/command-ref/conf-file.xml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c5f90481b13..24fbf28cff2 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -288,10 +288,11 @@ false</literal>.</para>
       </listitem>
     </orderedlist>
 
-    <para>The diff hook should not print data to stderr or stdout, as
-    output is not displayed to the user. However, if information is
-    printed, it will be printed in the <command>nix-daemon</command>
-    log.</para>
+    <para>
+      The stderr and stdout output from the diff hook will not be
+      displayed to the user. Instead, it will print to the nix-daemon's
+      log.
+    </para>
 
     <para>When using the Nix daemon, <literal>diff-hook</literal> must
     be set in the <filename>nix.conf</filename> configuration file, and

From 73b797c207e1c7a0fd9059d2cf1e3479502f8f1b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 12 May 2019 13:44:22 -0400
Subject: [PATCH 1404/2196] handleDiffHook: stop passing allowVfork

---
 src/libstore/build.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 8902e22bd8a..b07461013cc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -461,7 +461,7 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
-void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
+void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
 {
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
@@ -3213,7 +3213,6 @@ void DerivationGoal::registerOutputs()
                         throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
 
                     handleDiffHook(
-                        !buildUser && !drv->isBuiltin(),
                         buildUser ? buildUser->getUID() : getuid(),
                         buildUser ? buildUser->getGID() : getgid(),
                         path, dst, drvPath, tmpDir);
@@ -3283,7 +3282,6 @@ void DerivationGoal::registerOutputs()
                     : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
 
                 handleDiffHook(
-                    !buildUser && !drv->isBuiltin(),
                     buildUser ? buildUser->getUID() : getuid(),
                     buildUser ? buildUser->getGID() : getgid(),
                     prev, i->second.path, drvPath, tmpDir);

From ce02fc74b2db35e45906865c8a3ce2e98871eeb8 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 12 May 2019 22:47:41 +0200
Subject: [PATCH 1405/2196] build: make needsHashRewrite a method

---
 src/libstore/build.cc | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 30825add43b..79dcdddbed7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -803,9 +803,6 @@ class DerivationGoal : public Goal
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
 
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
-    bool needsHashRewrite;
-
     Path chrootRootDir;
 
     /* RAII object to delete the chroot directory. */
@@ -885,6 +882,9 @@ class DerivationGoal : public Goal
         Worker & worker, BuildMode buildMode = bmNormal);
     ~DerivationGoal();
 
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite();
+
     void timedOut() override;
 
     string key() override
@@ -1037,6 +1037,17 @@ DerivationGoal::~DerivationGoal()
 }
 
 
+inline bool DerivationGoal::needsHashRewrite()
+{
+#if __linux__
+    return !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    return true;
+#endif
+}
+
+
 void DerivationGoal::killChild()
 {
     if (pid != -1) {
@@ -1845,13 +1856,6 @@ void DerivationGoal::startBuilder()
         #endif
     }
 
-#if __linux__
-    needsHashRewrite = !useChroot;
-#else
-    /* Darwin requires hash rewriting even when sandboxing is enabled. */
-    needsHashRewrite = true;
-#endif
-
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
     if (settings.buildUsersGroup != "" && getuid() == 0) {
@@ -2083,7 +2087,7 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    if (needsHashRewrite) {
+    if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
             throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@@ -3067,7 +3071,7 @@ void DerivationGoal::registerOutputs()
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
         }
 
-        if (needsHashRewrite) {
+        if (needsHashRewrite()) {
             Path redirected = redirectedOutputs[path];
             if (buildMode == bmRepair
                 && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()

From f1b8e9efe77014655f059b44afa05c38990dc4aa Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 12 May 2019 17:03:01 -0400
Subject: [PATCH 1406/2196] runProgram: Uncomment chdir support

---
 src/libutil/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 55b9144f314..17aee2d5c3d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1026,8 +1026,8 @@ void runProgram2(const RunOptions & options)
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
-        //if (options.chdir && chdir((*options.chdir).c_str()) == -1)
-        //    throw SysError("chdir failed");
+        if (options.chdir && chdir((*options.chdir).c_str()) == -1)
+            throw SysError("chdir failed");
         if (options.gid && setgid(*options.gid) == -1)
             throw SysError("setgid failed");
         /* Drop all other groups if we're setgid. */

From 3fd5425f948afa5122ff6b0aad60a6b961b57161 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 May 2019 13:13:14 +0200
Subject: [PATCH 1407/2196] Fix shellcheck error

https://hydra.nixos.org/build/93359951
---
 scripts/install-nix-from-closure.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 4dd249923b5..fc999d336d1 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -26,7 +26,7 @@ fi
 if [ "$(uname -s)" = "Darwin" ]; then
     macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
     macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
-    if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then
+    if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then
         echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
         exit 1
     fi

From b6eb8a2d7e2ea8b083fdac15f537679ffe633183 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 May 2019 14:30:09 +0200
Subject: [PATCH 1408/2196] nix-profile: Add all channels to $NIX_PATH

Fixes #2709.
---
 scripts/nix-profile.sh.in | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index db03e16ba89..f3cfa157c76 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -51,10 +51,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         unset __nix_defexpr
     fi
 
-    # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that
-    # <nixpkgs> paths work when the user has fetched the Nixpkgs
-    # channel.
-    export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs"
+    # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
+    # paths work when the user has fetched the Nixpkgs channel.
+    export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
 
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix

From 66b8a62101cb1dfe2e368346cf99efd32e9328ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 May 2019 17:33:56 +0200
Subject: [PATCH 1409/2196] nix: Add --print-build-logs flag

This causes 'nix' to print build log output to stderr rather than
showing the last log line in the progress bar. Log lines are prefixed
by the name of the derivation (minus the version string), e.g.

  binutils> make[1]: Leaving directory '/build/binutils-2.31.1'
  binutils-wrapper> unpacking sources
  binutils-wrapper> patching sources
  ...
  binutils-wrapper> Using dynamic linker: '/nix/store/kr51dlsj9v5cr4n8700jliyz8v5b2q7q-bootstrap-stage0-glibc/lib/ld-linux-x86-64.so.2'
  bootstrap-stage2-gcc-wrapper> unpacking sources
  ...
  linux-headers> unpacking sources
  linux-headers> unpacking source archive /nix/store/8javli69jhj3bkql2c35gsj5vl91p382-linux-4.19.16.tar.xz
---
 src/libutil/util.hh     |  1 +
 src/nix/main.cc         | 10 ++++++++--
 src/nix/progress-bar.cc | 39 +++++++++++++++++++++++++++++----------
 src/nix/progress-bar.hh |  2 +-
 4 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 9f239bff371..23360de2e8d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -401,6 +401,7 @@ void ignoreException();
 /* Some ANSI escape sequences. */
 #define ANSI_NORMAL "\e[0m"
 #define ANSI_BOLD "\e[1m"
+#define ANSI_FAINT "\e[2m"
 #define ANSI_RED "\e[31;1m"
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_BLUE "\e[34;1m"
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 64c1dc35787..4f87ad72b65 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -20,6 +20,8 @@ std::string programPath;
 
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
+    bool printBuildLogs = false;
+
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
         mkFlag()
@@ -41,6 +43,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
                 throw Exit();
             });
 
+        mkFlag()
+            .longName("print-build-logs")
+            .description("print full build logs on stderr")
+            .set(&printBuildLogs, true);
+
         mkFlag()
             .longName("version")
             .description("show version information")
@@ -98,8 +105,7 @@ void mainWrapped(int argc, char * * argv)
 
     Finally f([]() { stopProgressBar(); });
 
-    if (isatty(STDERR_FILENO))
-        startProgressBar();
+    startProgressBar(args.printBuildLogs);
 
     args.command->prepare();
     args.command->run();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 40b905ba324..304f918cc5f 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "sync.hh"
 #include "store-api.hh"
+#include "names.hh"
 
 #include <atomic>
 #include <map>
@@ -38,6 +39,7 @@ class ProgressBar : public Logger
         std::map<ActivityType, uint64_t> expectedByType;
         bool visible = true;
         ActivityId parent;
+        std::optional<std::string> name;
     };
 
     struct ActivitiesByType
@@ -68,10 +70,16 @@ class ProgressBar : public Logger
 
     std::condition_variable quitCV, updateCV;
 
+    bool printBuildLogs;
+    bool isTTY;
+
 public:
 
-    ProgressBar()
+    ProgressBar(bool printBuildLogs, bool isTTY)
+        : printBuildLogs(printBuildLogs)
+        , isTTY(isTTY)
     {
+        state_.lock()->active = isTTY;
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
@@ -109,8 +117,14 @@ class ProgressBar : public Logger
 
     void log(State & state, Verbosity lvl, const std::string & s)
     {
-        writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
-        draw(state);
+        if (state.active) {
+            writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
+            draw(state);
+        } else {
+            auto s2 = s + ANSI_NORMAL "\n";
+            if (!isTTY) s2 = filterANSIEscapes(s2, true);
+            writeToStderr(s2);
+        }
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
@@ -141,6 +155,7 @@ class ProgressBar : public Logger
             auto nrRounds = getI(fields, 3);
             if (nrRounds != 1)
                 i->s += fmt(" (round %d/%d)", curRound, nrRounds);
+            i->name = DrvName(name).name;
         }
 
         if (type == actSubstitute) {
@@ -217,11 +232,15 @@ class ProgressBar : public Logger
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
-                state->activities.erase(i->second);
-                info.lastLine = lastLine;
-                state->activities.emplace_back(info);
-                i->second = std::prev(state->activities.end());
-                update();
+                if (printBuildLogs) {
+                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
+                } else {
+                    state->activities.erase(i->second);
+                    info.lastLine = lastLine;
+                    state->activities.emplace_back(info);
+                    i->second = std::prev(state->activities.end());
+                    update();
+                }
             }
         }
 
@@ -395,9 +414,9 @@ class ProgressBar : public Logger
     }
 };
 
-void startProgressBar()
+void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar();
+    logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO));
 }
 
 void stopProgressBar()
diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh
index af8eda5a84f..4d61175c24e 100644
--- a/src/nix/progress-bar.hh
+++ b/src/nix/progress-bar.hh
@@ -4,7 +4,7 @@
 
 namespace nix {
 
-void startProgressBar();
+void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();
 

From 7c20ee448fa924d898bcebf84bd0a7caf368a656 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 15 May 2019 22:04:39 -0400
Subject: [PATCH 1410/2196] Sync NIX_PROFILES between single-user and
 multi-user modes

When we are in single user mode, we still want to have access to
profiles. This way things in Nixpkgs that rely on them getting set
accurately are done in both cases. The point where I hit this is with
using aspell which looks in NIX_PROFILES:

https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/libraries/aspell/default.nix

Before this patch, NIX_PROFILES was never set in single user mode!
This corrects that.
---
 scripts/nix-profile.sh.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index f3cfa157c76..85f1d6e5dae 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -57,7 +57,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
-    NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR"
+    export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
     if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
@@ -79,5 +79,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     fi
 
     export PATH="$NIX_LINK/bin:$__savedpath"
-    unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES
+    unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR
 fi

From 92f461e4f46b595e3712b41c30b8403905dd8673 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 15 May 2019 22:24:24 -0400
Subject: [PATCH 1411/2196] =?UTF-8?q?Don=E2=80=99t=20set=20NIX=5FREMOTE=3D?=
 =?UTF-8?q?daemon=20in=20daemon=20profile?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is now autodetected. There is no need to put it in the profile.
---
 scripts/nix-profile-daemon.sh.in | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 6940969cca7..23da5e8559e 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -2,12 +2,6 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-# Set up secure multi-user builds: non-root users build through the
-# Nix daemon.
-if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then
-    export NIX_REMOTE=daemon
-fi
-
 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 

From c0559a1d6092b94c7ce46a065d72fc74e4db51dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <v@cunat.cz>
Date: Fri, 17 May 2019 09:50:42 +0200
Subject: [PATCH 1412/2196] docs: describe $IN_NIX_SHELL values (#2796)

See commit 1bffd83e1a9
---
 doc/manual/command-ref/env-common.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index c532ffddea2..6a3aaae717e 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -14,7 +14,8 @@
 <varlistentry><term><envar>IN_NIX_SHELL</envar></term>
 
   <listitem><para>Indicator that tells if the current environment was set up by
-  <command>nix-shell</command>.</para></listitem>
+  <command>nix-shell</command>.  Since Nix 2.0 the values are
+  <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem>
 
 </varlistentry>
 

From 14c877b4ab097667734b050a79b45658dcf2695d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Apr 2019 14:41:59 +0200
Subject: [PATCH 1413/2196] fetchGit -> fetchTarball

(cherry picked from commit cbfdea685764bf66443a999e672656c54289b8c9)
---
 release.nix | 2 +-
 shell.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index ab13451ff3d..78b39108f85 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
diff --git a/shell.nix b/shell.nix
index 73e75fb29c4..8167f87a292 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From b502b6682b9c73ae5760967eaf7161b3e8523e9e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 20 May 2019 19:17:18 +0200
Subject: [PATCH 1414/2196] doc: clarify that optional attrs in a function
 argument will be ignored unless specified

In `args@{ a ? 1 }: /* ... */` the value `a` won't be a part of `args`
unless it's specified when calling the function, the default value will
be ignored in this case.

My personal point of view is that this behavior is a matter of taste, at
least I was pretty sure that unmatched arguments will be a part of
`args@` while debugging some Nix code last week.

I decided to add a warning to the docs which hopefully reduces the
confusion of further Nix developers who thought the same about `args@`.
---
 .../expressions/language-constructs.xml       | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index f961ed921bc..923b5d8c401 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -217,7 +217,25 @@ but can also be written as:
   ellipsis(<literal>...</literal>) as you can access attribute names as 
   <literal>a</literal>, using <literal>args.a</literal>, which was given as an
   additional attribute to the function.
-  </para></listitem>
+  </para>
+
+  <warning>
+   <para>
+    The <literal>args@</literal> expression is bound to the argument passed to the function which
+    means that attributes with defaults that aren't explicitly specified in the function call
+    won't cause an evaluation error, but won't exist in <literal>args</literal>.
+   </para>
+   <para>
+    For instance
+<programlisting>
+let
+  function = args@{ a ? 23, ... }: args;
+in
+ function {}
+</programlisting>
+    will evaluate to an empty attribute set.
+   </para>
+  </warning></listitem>
 
 </itemizedlist>
 

From 22f2744afdfa048de7bd078ba9d24dafbcc7d7c0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 May 2019 23:05:08 +0200
Subject: [PATCH 1415/2196] Iterate over references

---
 src/libexpr/parser.y | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 97e1a65849e..78a50390718 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -83,7 +83,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
     assert(!attrPath.empty());
     // Checking attrPath validity.
     // ===========================
-    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { 
+    for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
         if (i->symbol.set()) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
@@ -106,18 +106,18 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
     }
     // Expr insertion.
     // ==========================
-    if (i->symbol.set()) { 
+    if (i->symbol.set()) {
         ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
         if (j != attrs->attrs.end()) {
             // This attr path is already defined. However, if both
             // e and the expr pointed by the attr path are two attribute sets,
             // we want to merge them.
             // Otherwise, throw an error.
-            ExprAttrs* ae = dynamic_cast<ExprAttrs *>(e);
-            ExprAttrs* jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            auto ae = dynamic_cast<ExprAttrs *>(e);
+            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
             if (jAttrs && ae) {
-                for (auto ad: ae->attrs) {
-                    ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first);
+                for (auto & ad : ae->attrs) {
+                    auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
                     jAttrs->attrs[ad.first] = ad.second;

From abdedcdb387628aaa4d9dc362b7e588bbc62baae Mon Sep 17 00:00:00 2001
From: Ding Xiang Fei <dingxiangfei2009@gmail.com>
Date: Wed, 29 May 2019 17:01:39 +0800
Subject: [PATCH 1416/2196] bump cpptoml to v0.1.1

---
 src/cpptoml/cpptoml.h | 569 +++++++++++++++++++++++++++++-------------
 1 file changed, 390 insertions(+), 179 deletions(-)

diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h
index 07de010b152..5a00da3b4cd 100644
--- a/src/cpptoml/cpptoml.h
+++ b/src/cpptoml/cpptoml.h
@@ -4,8 +4,8 @@
  * @date May 2013
  */
 
-#ifndef _CPPTOML_H_
-#define _CPPTOML_H_
+#ifndef CPPTOML_H
+#define CPPTOML_H
 
 #include <algorithm>
 #include <cassert>
@@ -84,11 +84,12 @@ class option
         return &value_;
     }
 
-    const T& value_or(const T& alternative) const
+    template <class U>
+    T value_or(U&& alternative) const
     {
         if (!empty_)
             return value_;
-        return alternative;
+        return static_cast<T>(std::forward<U>(alternative));
     }
 
   private:
@@ -295,13 +296,12 @@ struct valid_value_or_string_convertible
 };
 
 template <class T>
-struct value_traits<T, typename std::
-                           enable_if<valid_value_or_string_convertible<T>::
-                                         value>::type>
+struct value_traits<T, typename std::enable_if<
+                           valid_value_or_string_convertible<T>::value>::type>
 {
-    using value_type = typename std::
-        conditional<valid_value<typename std::decay<T>::type>::value,
-                    typename std::decay<T>::type, std::string>::type;
+    using value_type = typename std::conditional<
+        valid_value<typename std::decay<T>::type>::value,
+        typename std::decay<T>::type, std::string>::type;
 
     using type = value<value_type>;
 
@@ -312,12 +312,11 @@ struct value_traits<T, typename std::
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_floating_point<
-                                         typename std::decay<T>::type>::value>::
-                            type>
+struct value_traits<
+    T,
+    typename std::enable_if<
+        !valid_value_or_string_convertible<T>::value
+        && std::is_floating_point<typename std::decay<T>::type>::value>::type>
 {
     using value_type = typename std::decay<T>::type;
 
@@ -330,11 +329,11 @@ struct value_traits<T,
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_signed<typename std::decay<T>::
-                                                        type>::value>::type>
+struct value_traits<
+    T, typename std::enable_if<
+           !valid_value_or_string_convertible<T>::value
+           && !std::is_floating_point<typename std::decay<T>::type>::value
+           && std::is_signed<typename std::decay<T>::type>::value>::type>
 {
     using value_type = int64_t;
 
@@ -356,11 +355,10 @@ struct value_traits<T,
 };
 
 template <class T>
-struct value_traits<T,
-                    typename std::
-                        enable_if<!valid_value_or_string_convertible<T>::value
-                                  && std::is_unsigned<typename std::decay<T>::
-                                                          type>::value>::type>
+struct value_traits<
+    T, typename std::enable_if<
+           !valid_value_or_string_convertible<T>::value
+           && std::is_unsigned<typename std::decay<T>::type>::value>::type>
 {
     using value_type = int64_t;
 
@@ -395,10 +393,15 @@ struct array_of_trait<array>
 template <class T>
 inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val);
 inline std::shared_ptr<array> make_array();
+
+namespace detail
+{
 template <class T>
 inline std::shared_ptr<T> make_element();
+}
+
 inline std::shared_ptr<table> make_table();
-inline std::shared_ptr<table_array> make_table_array();
+inline std::shared_ptr<table_array> make_table_array(bool is_inline = false);
 
 #if defined(CPPTOML_NO_RTTI)
 /// Base type used to store underlying data type explicitly if RTTI is disabled
@@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base>
 #if defined(CPPTOML_NO_RTTI)
     base_type type() const
     {
-      return type_;
+        return type_;
     }
 
   protected:
@@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as()
     if (type() == base_type::INT)
     {
         auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this());
-        return make_value<double>(static_cast<double>(v->get()));;
+        return make_value<double>(static_cast<double>(v->get()));
     }
 #else
     if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this()))
@@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const
 {
 #if defined(CPPTOML_NO_RTTI)
     if (type() == base_type::FLOAT)
-        return std::static_pointer_cast<const value<double>>(shared_from_this());
+        return std::static_pointer_cast<const value<double>>(
+            shared_from_this());
 
     if (type() == base_type::INT)
     {
@@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array()
     return std::make_shared<make_shared_enabler>();
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<array> make_element<array>()
 {
     return make_array();
 }
+} // namespace detail
 
 /**
  * Obtains a option<vector<T>>. The option will be empty if the array
@@ -1060,7 +1067,7 @@ class table;
 class table_array : public base
 {
     friend class table;
-    friend std::shared_ptr<table_array> make_table_array();
+    friend std::shared_ptr<table_array> make_table_array(bool);
 
   public:
     std::shared_ptr<base> clone() const override;
@@ -1152,14 +1159,25 @@ class table_array : public base
         array_.reserve(n);
     }
 
+    /**
+     * Whether or not the table array is declared inline. This mostly
+     * matters for parsing, where statically defined arrays cannot be
+     * appended to using the array-of-table syntax.
+     */
+    bool is_inline() const
+    {
+        return is_inline_;
+    }
+
   private:
 #if defined(CPPTOML_NO_RTTI)
-    table_array() : base(base_type::TABLE_ARRAY)
+    table_array(bool is_inline = false)
+        : base(base_type::TABLE_ARRAY), is_inline_(is_inline)
     {
         // nothing
     }
 #else
-    table_array()
+    table_array(bool is_inline = false) : is_inline_(is_inline)
     {
         // nothing
     }
@@ -1169,26 +1187,30 @@ class table_array : public base
     table_array& operator=(const table_array& rhs) = delete;
 
     std::vector<std::shared_ptr<table>> array_;
+    const bool is_inline_ = false;
 };
 
-inline std::shared_ptr<table_array> make_table_array()
+inline std::shared_ptr<table_array> make_table_array(bool is_inline)
 {
     struct make_shared_enabler : public table_array
     {
-        make_shared_enabler()
+        make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline)
         {
             // nothing
         }
     };
 
-    return std::make_shared<make_shared_enabler>();
+    return std::make_shared<make_shared_enabler>(is_inline);
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<table_array> make_element<table_array>()
 {
-    return make_table_array();
+    return make_table_array(true);
 }
+} // namespace detail
 
 // The below are overloads for fetching specific value types out of a value
 // where special casting behavior (like bounds checking) is desired
@@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table()
     return std::make_shared<make_shared_enabler>();
 }
 
+namespace detail
+{
 template <>
 inline std::shared_ptr<table> make_element<table>()
 {
     return make_table();
 }
+} // namespace detail
 
 template <class T>
 std::shared_ptr<base> value<T>::clone() const
@@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const
 
 inline std::shared_ptr<base> table_array::clone() const
 {
-    auto result = make_table_array();
+    auto result = make_table_array(is_inline());
     result->reserve(array_.size());
     for (const auto& ptr : array_)
         result->array_.push_back(ptr->clone()->as_table());
@@ -1738,6 +1763,11 @@ inline bool is_number(char c)
     return c >= '0' && c <= '9';
 }
 
+inline bool is_hex(char c)
+{
+    return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
+
 /**
  * Helper object for consuming expected characters.
  */
@@ -1766,6 +1796,13 @@ class consumer
                       [&](char c) { (*this)(c); });
     }
 
+    void eat_or(char a, char b)
+    {
+        if (it_ == end_ || (*it_ != a && *it_ != b))
+            on_error_();
+        ++it_;
+    }
+
     int eat_digits(int len)
     {
         int val = 0;
@@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line)
         line.push_back(static_cast<char>(c));
     }
 }
-}
+} // namespace detail
 
 /**
  * The parser class.
@@ -1914,21 +1951,25 @@ class parser
 
         std::string full_table_name;
         bool inserted = false;
-        while (it != end && *it != ']')
-        {
-            auto part = parse_key(it, end,
-                                  [](char c) { return c == '.' || c == ']'; });
 
+        auto key_end = [](char c) { return c == ']'; };
+
+        auto key_part_handler = [&](const std::string& part) {
             if (part.empty())
                 throw_parse_exception("Empty component of table name");
 
             if (!full_table_name.empty())
-                full_table_name += ".";
+                full_table_name += '.';
             full_table_name += part;
 
             if (curr_table->contains(part))
             {
+#if !defined(__PGI)
                 auto b = curr_table->get(part);
+#else
+                // Workaround for PGI compiler
+                std::shared_ptr<base> b = curr_table->get(part);
+#endif
                 if (b->is_table())
                     curr_table = static_cast<table*>(b.get());
                 else if (b->is_table_array())
@@ -1946,16 +1987,23 @@ class parser
                 curr_table->insert(part, make_table());
                 curr_table = static_cast<table*>(curr_table->get(part).get());
             }
-            consume_whitespace(it, end);
-            if (it != end && *it == '.')
-                ++it;
-            consume_whitespace(it, end);
-        }
+        };
+
+        key_part_handler(parse_key(it, end, key_end, key_part_handler));
 
         if (it == end)
             throw_parse_exception(
                 "Unterminated table declaration; did you forget a ']'?");
 
+        if (*it != ']')
+        {
+            std::string errmsg{"Unexpected character in table definition: "};
+            errmsg += '"';
+            errmsg += *it;
+            errmsg += '"';
+            throw_parse_exception(errmsg);
+        }
+
         // table already existed
         if (!inserted)
         {
@@ -1969,8 +2017,9 @@ class parser
             // since it has already been defined. If there aren't any
             // values, then it was implicitly created by something like
             // [a.b]
-            if (curr_table->empty() || std::any_of(curr_table->begin(),
-                                                   curr_table->end(), is_value))
+            if (curr_table->empty()
+                || std::any_of(curr_table->begin(), curr_table->end(),
+                               is_value))
             {
                 throw_parse_exception("Redefinition of table "
                                       + full_table_name);
@@ -1989,36 +2038,45 @@ class parser
         if (it == end || *it == ']')
             throw_parse_exception("Table array name cannot be empty");
 
-        std::string full_ta_name;
-        while (it != end && *it != ']')
-        {
-            auto part = parse_key(it, end,
-                                  [](char c) { return c == '.' || c == ']'; });
+        auto key_end = [](char c) { return c == ']'; };
 
+        std::string full_ta_name;
+        auto key_part_handler = [&](const std::string& part) {
             if (part.empty())
                 throw_parse_exception("Empty component of table array name");
 
             if (!full_ta_name.empty())
-                full_ta_name += ".";
+                full_ta_name += '.';
             full_ta_name += part;
 
-            consume_whitespace(it, end);
-            if (it != end && *it == '.')
-                ++it;
-            consume_whitespace(it, end);
-
             if (curr_table->contains(part))
             {
+#if !defined(__PGI)
                 auto b = curr_table->get(part);
+#else
+                // Workaround for PGI compiler
+                std::shared_ptr<base> b = curr_table->get(part);
+#endif
 
                 // if this is the end of the table array name, add an
-                // element to the table array that we just looked up
+                // element to the table array that we just looked up,
+                // provided it was not declared inline
                 if (it != end && *it == ']')
                 {
                     if (!b->is_table_array())
+                    {
                         throw_parse_exception("Key " + full_ta_name
                                               + " is not a table array");
+                    }
+
                     auto v = b->as_table_array();
+
+                    if (v->is_inline())
+                    {
+                        throw_parse_exception("Static array " + full_ta_name
+                                              + " cannot be appended to");
+                    }
+
                     v->get().push_back(make_table());
                     curr_table = v->get().back().get();
                 }
@@ -2059,15 +2117,16 @@ class parser
                         = static_cast<table*>(curr_table->get(part).get());
                 }
             }
-        }
+        };
+
+        key_part_handler(parse_key(it, end, key_end, key_part_handler));
 
         // consume the last "]]"
-        if (it == end)
-            throw_parse_exception("Unterminated table array name");
-        ++it;
-        if (it == end)
+        auto eat = make_consumer(it, end, [this]() {
             throw_parse_exception("Unterminated table array name");
-        ++it;
+        });
+        eat(']');
+        eat(']');
 
         consume_whitespace(it, end);
         eol_or_comment(it, end);
@@ -2076,7 +2135,35 @@ class parser
     void parse_key_value(std::string::iterator& it, std::string::iterator& end,
                          table* curr_table)
     {
-        auto key = parse_key(it, end, [](char c) { return c == '='; });
+        auto key_end = [](char c) { return c == '='; };
+
+        auto key_part_handler = [&](const std::string& part) {
+            // two cases: this key part exists already, in which case it must
+            // be a table, or it doesn't exist in which case we must create
+            // an implicitly defined table
+            if (curr_table->contains(part))
+            {
+                auto val = curr_table->get(part);
+                if (val->is_table())
+                {
+                    curr_table = static_cast<table*>(val.get());
+                }
+                else
+                {
+                    throw_parse_exception("Key " + part
+                                          + " already exists as a value");
+                }
+            }
+            else
+            {
+                auto newtable = make_table();
+                curr_table->insert(part, newtable);
+                curr_table = newtable.get();
+            }
+        };
+
+        auto key = parse_key(it, end, key_end, key_part_handler);
+
         if (curr_table->contains(key))
             throw_parse_exception("Key " + key + " already present");
         if (it == end || *it != '=')
@@ -2087,18 +2174,57 @@ class parser
         consume_whitespace(it, end);
     }
 
-    template <class Function>
-    std::string parse_key(std::string::iterator& it,
-                          const std::string::iterator& end, Function&& fun)
+    template <class KeyEndFinder, class KeyPartHandler>
+    std::string
+    parse_key(std::string::iterator& it, const std::string::iterator& end,
+              KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler)
+    {
+        // parse the key as a series of one or more simple-keys joined with '.'
+        while (it != end && !key_end(*it))
+        {
+            auto part = parse_simple_key(it, end);
+            consume_whitespace(it, end);
+
+            if (it == end || key_end(*it))
+            {
+                return part;
+            }
+
+            if (*it != '.')
+            {
+                std::string errmsg{"Unexpected character in key: "};
+                errmsg += '"';
+                errmsg += *it;
+                errmsg += '"';
+                throw_parse_exception(errmsg);
+            }
+
+            key_part_handler(part);
+
+            // consume the dot
+            ++it;
+        }
+
+        throw_parse_exception("Unexpected end of key");
+    }
+
+    std::string parse_simple_key(std::string::iterator& it,
+                                 const std::string::iterator& end)
     {
         consume_whitespace(it, end);
-        if (*it == '"')
+
+        if (it == end)
+            throw_parse_exception("Unexpected end of key (blank key?)");
+
+        if (*it == '"' || *it == '\'')
         {
-            return parse_quoted_key(it, end);
+            return string_literal(it, end, *it);
         }
         else
         {
-            auto bke = std::find_if(it, end, std::forward<Function>(fun));
+            auto bke = std::find_if(it, end, [](char c) {
+                return c == '.' || c == '=' || c == ']';
+            });
             return parse_bare_key(it, bke);
         }
     }
@@ -2142,12 +2268,6 @@ class parser
         return key;
     }
 
-    std::string parse_quoted_key(std::string::iterator& it,
-                                 const std::string::iterator& end)
-    {
-        return string_literal(it, end, '"');
-    }
-
     enum class parse_type
     {
         STRING = 1,
@@ -2193,7 +2313,7 @@ class parser
     parse_type determine_value_type(const std::string::iterator& it,
                                     const std::string::iterator& end)
     {
-        if(it == end)
+        if (it == end)
         {
             throw_parse_exception("Failed to parse value type");
         }
@@ -2209,7 +2329,11 @@ class parser
         {
             return *dtype;
         }
-        else if (is_number(*it) || *it == '-' || *it == '+')
+        else if (is_number(*it) || *it == '-' || *it == '+'
+                 || (*it == 'i' && it + 1 != end && it[1] == 'n'
+                     && it + 2 != end && it[2] == 'f')
+                 || (*it == 'n' && it + 1 != end && it[1] == 'a'
+                     && it + 2 != end && it[2] == 'n'))
         {
             return determine_number_type(it, end);
         }
@@ -2235,6 +2359,13 @@ class parser
         auto check_it = it;
         if (*check_it == '-' || *check_it == '+')
             ++check_it;
+
+        if (check_it == end)
+            throw_parse_exception("Malformed number");
+
+        if (*check_it == 'i' || *check_it == 'n')
+            return parse_type::FLOAT;
+
         while (check_it != end && is_number(*check_it))
             ++check_it;
         if (check_it != end && *check_it == '.')
@@ -2283,57 +2414,56 @@ class parser
         bool consuming = false;
         std::shared_ptr<value<std::string>> ret;
 
-        auto handle_line
-            = [&](std::string::iterator& local_it,
-                  std::string::iterator& local_end) {
-                  if (consuming)
-                  {
-                      local_it = std::find_if_not(local_it, local_end, is_ws);
-
-                      // whole line is whitespace
-                      if (local_it == local_end)
-                          return;
-                  }
-
-                  consuming = false;
-
-                  while (local_it != local_end)
-                  {
-                      // handle escaped characters
-                      if (delim == '"' && *local_it == '\\')
-                      {
-                          auto check = local_it;
-                          // check if this is an actual escape sequence or a
-                          // whitespace escaping backslash
-                          ++check;
-                          consume_whitespace(check, local_end);
-                          if (check == local_end)
-                          {
-                              consuming = true;
-                              break;
-                          }
-
-                          ss << parse_escape_code(local_it, local_end);
-                          continue;
-                      }
-
-                      // if we can end the string
-                      if (std::distance(local_it, local_end) >= 3)
-                      {
-                          auto check = local_it;
-                          // check for """
-                          if (*check++ == delim && *check++ == delim
-                              && *check++ == delim)
-                          {
-                              local_it = check;
-                              ret = make_value<std::string>(ss.str());
-                              break;
-                          }
-                      }
-
-                      ss << *local_it++;
-                  }
-              };
+        auto handle_line = [&](std::string::iterator& local_it,
+                               std::string::iterator& local_end) {
+            if (consuming)
+            {
+                local_it = std::find_if_not(local_it, local_end, is_ws);
+
+                // whole line is whitespace
+                if (local_it == local_end)
+                    return;
+            }
+
+            consuming = false;
+
+            while (local_it != local_end)
+            {
+                // handle escaped characters
+                if (delim == '"' && *local_it == '\\')
+                {
+                    auto check = local_it;
+                    // check if this is an actual escape sequence or a
+                    // whitespace escaping backslash
+                    ++check;
+                    consume_whitespace(check, local_end);
+                    if (check == local_end)
+                    {
+                        consuming = true;
+                        break;
+                    }
+
+                    ss << parse_escape_code(local_it, local_end);
+                    continue;
+                }
+
+                // if we can end the string
+                if (std::distance(local_it, local_end) >= 3)
+                {
+                    auto check = local_it;
+                    // check for """
+                    if (*check++ == delim && *check++ == delim
+                        && *check++ == delim)
+                    {
+                        local_it = check;
+                        ret = make_value<std::string>(ss.str());
+                        break;
+                    }
+                }
+
+                ss << *local_it++;
+            }
+        };
 
         // handle the remainder of the current line
         handle_line(it, end);
@@ -2514,17 +2644,13 @@ class parser
         return value;
     }
 
-    bool is_hex(char c)
-    {
-        return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
-    }
-
     uint32_t hex_to_digit(char c)
     {
         if (is_number(c))
             return static_cast<uint32_t>(c - '0');
-        return 10 + static_cast<uint32_t>(
-                        c - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
+        return 10
+               + static_cast<uint32_t>(c
+                                       - ((c >= 'a' && c <= 'f') ? 'a' : 'A'));
     }
 
     std::shared_ptr<base> parse_number(std::string::iterator& it,
@@ -2538,17 +2664,23 @@ class parser
                 ++check_it;
         };
 
-        eat_sign();
+        auto check_no_leading_zero = [&]() {
+            if (check_it != end && *check_it == '0' && check_it + 1 != check_end
+                && check_it[1] != '.')
+            {
+                throw_parse_exception("Numbers may not have leading zeros");
+            }
+        };
 
-        auto eat_numbers = [&]() {
+        auto eat_digits = [&](bool (*check_char)(char)) {
             auto beg = check_it;
-            while (check_it != end && is_number(*check_it))
+            while (check_it != end && check_char(*check_it))
             {
                 ++check_it;
                 if (check_it != end && *check_it == '_')
                 {
                     ++check_it;
-                    if (check_it == end || !is_number(*check_it))
+                    if (check_it == end || !check_char(*check_it))
                         throw_parse_exception("Malformed number");
                 }
             }
@@ -2557,15 +2689,63 @@ class parser
                 throw_parse_exception("Malformed number");
         };
 
-        auto check_no_leading_zero = [&]() {
-            if (check_it != end && *check_it == '0' && check_it + 1 != check_end
-                && check_it[1] != '.')
+        auto eat_hex = [&]() { eat_digits(&is_hex); };
+
+        auto eat_numbers = [&]() { eat_digits(&is_number); };
+
+        if (check_it != end && *check_it == '0' && check_it + 1 != check_end
+            && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b'))
+        {
+            ++check_it;
+            char base = *check_it;
+            ++check_it;
+            if (base == 'x')
             {
-                throw_parse_exception("Numbers may not have leading zeros");
+                eat_hex();
+                return parse_int(it, check_it, 16);
             }
-        };
+            else if (base == 'o')
+            {
+                auto start = check_it;
+                eat_numbers();
+                auto val = parse_int(start, check_it, 8, "0");
+                it = start;
+                return val;
+            }
+            else // if (base == 'b')
+            {
+                auto start = check_it;
+                eat_numbers();
+                auto val = parse_int(start, check_it, 2);
+                it = start;
+                return val;
+            }
+        }
 
+        eat_sign();
         check_no_leading_zero();
+
+        if (check_it != end && check_it + 1 != end && check_it + 2 != end)
+        {
+            if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f')
+            {
+                auto val = std::numeric_limits<double>::infinity();
+                if (*it == '-')
+                    val = -val;
+                it = check_it + 3;
+                return make_value(val);
+            }
+            else if (check_it[0] == 'n' && check_it[1] == 'a'
+                     && check_it[2] == 'n')
+            {
+                auto val = std::numeric_limits<double>::quiet_NaN();
+                if (*it == '-')
+                    val = -val;
+                it = check_it + 3;
+                return make_value(val);
+            }
+        }
+
         eat_numbers();
 
         if (check_it != end
@@ -2604,14 +2784,17 @@ class parser
     }
 
     std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it,
-                                              const std::string::iterator& end)
+                                              const std::string::iterator& end,
+                                              int base = 10,
+                                              const char* prefix = "")
     {
         std::string v{it, end};
+        v = prefix + v;
         v.erase(std::remove(v.begin(), v.end(), '_'), v.end());
         it = end;
         try
         {
-            return make_value<int64_t>(std::stoll(v));
+            return make_value<int64_t>(std::stoll(v, nullptr, base));
         }
         catch (const std::invalid_argument& ex)
         {
@@ -2674,18 +2857,33 @@ class parser
     std::string::iterator find_end_of_number(std::string::iterator it,
                                              std::string::iterator end)
     {
-        return std::find_if(it, end, [](char c) {
+        auto ret = std::find_if(it, end, [](char c) {
             return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E'
-                   && c != '-' && c != '+';
+                   && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b';
         });
+        if (ret != end && ret + 1 != end && ret + 2 != end)
+        {
+            if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f')
+                || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n'))
+            {
+                ret = ret + 3;
+            }
+        }
+        return ret;
     }
 
     std::string::iterator find_end_of_date(std::string::iterator it,
                                            std::string::iterator end)
     {
-        return std::find_if(it, end, [](char c) {
-            return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-'
-                   && c != '+' && c != '.';
+        auto end_of_date = std::find_if(it, end, [](char c) {
+            return !is_number(c) && c != '-';
+        });
+        if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end
+            && is_number(end_of_date[1]))
+            end_of_date++;
+        return std::find_if(end_of_date, end, [](char c) {
+            return !is_number(c) && c != 'T' && c != 'Z' && c != ':'
+                   && c != '-' && c != '+' && c != '.';
         });
     }
 
@@ -2754,7 +2952,7 @@ class parser
         if (it == date_end)
             return make_value(ldate);
 
-        eat('T');
+        eat.eat_or('T', ' ');
 
         local_datetime ldt;
         static_cast<local_date&>(ldt) = ldate;
@@ -2850,9 +3048,9 @@ class parser
         auto arr = make_array();
         while (it != end && *it != ']')
         {
-            auto value = parse_value(it, end);
-            if (auto v = value->as<Value>())
-                arr->get().push_back(value);
+            auto val = parse_value(it, end);
+            if (auto v = val->as<Value>())
+                arr->get().push_back(val);
             else
                 throw_parse_exception("Arrays must be homogeneous");
             skip_whitespace_and_comments(it, end);
@@ -2871,7 +3069,7 @@ class parser
                                                std::string::iterator& it,
                                                std::string::iterator& end)
     {
-        auto arr = make_element<Object>();
+        auto arr = detail::make_element<Object>();
 
         while (it != end && *it != ']')
         {
@@ -2881,7 +3079,7 @@ class parser
             arr->get().push_back(((*this).*fun)(it, end));
             skip_whitespace_and_comments(it, end);
 
-            if (*it != ',')
+            if (it == end || *it != ',')
                 break;
 
             ++it;
@@ -2906,8 +3104,11 @@ class parser
                 throw_parse_exception("Unterminated inline table");
 
             consume_whitespace(it, end);
-            parse_key_value(it, end, tbl.get());
-            consume_whitespace(it, end);
+            if (it != end && *it != '}')
+            {
+                parse_key_value(it, end, tbl.get());
+                consume_whitespace(it, end);
+            }
         } while (*it == ',');
 
         if (it == end || *it != '}')
@@ -2987,7 +3188,8 @@ class parser
         if (it[4] != '-' || it[7] != '-')
             return {};
 
-        if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end))
+        if (len >= 19 && (it[10] == 'T' || it[10] == ' ')
+            && is_time(it + 11, date_end))
         {
             // datetime type
             auto time_end = find_end_of_time(it + 11, date_end);
@@ -3243,7 +3445,7 @@ class toml_writer
             {
                 res += "\\\\";
             }
-            else if ((const uint32_t)*it <= 0x001f)
+            else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f))
             {
                 res += "\\u";
                 std::stringstream ss;
@@ -3274,12 +3476,21 @@ class toml_writer
      */
     void write(const value<double>& v)
     {
-        std::ios::fmtflags flags{stream_.flags()};
-
-        stream_ << std::showpoint;
-        write(v.get());
-
-        stream_.flags(flags);
+        std::stringstream ss;
+        ss << std::showpoint
+           << std::setprecision(std::numeric_limits<double>::max_digits10)
+           << v.get();
+
+        auto double_str = ss.str();
+        auto pos = double_str.find("e0");
+        if (pos != std::string::npos)
+            double_str.replace(pos, 2, "e");
+        pos = double_str.find("e-0");
+        if (pos != std::string::npos)
+            double_str.replace(pos, 3, "e-");
+
+        stream_ << double_str;
+        has_naked_endline_ = false;
     }
 
     /**
@@ -3287,9 +3498,9 @@ class toml_writer
      * offset_datetime.
      */
     template <class T>
-    typename std::enable_if<is_one_of<T, int64_t, local_date, local_time,
-                                      local_datetime,
-                                      offset_datetime>::value>::type
+    typename std::enable_if<
+        is_one_of<T, int64_t, local_date, local_time, local_datetime,
+                  offset_datetime>::value>::type
     write(const value<T>& v)
     {
         write(v.get());
@@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a)
     a.accept(writer);
     return stream;
 }
-}
-#endif
+} // namespace cpptoml
+#endif // CPPTOML_H

From cfd74aef1ef98e8830b18a300057dc8b80df2a9f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 29 May 2019 12:12:02 +0200
Subject: [PATCH 1417/2196] Fix eval-okay-fromTOML test

Turns out we were mis-parsing single-quoted attributes, e.g. 'key2'.
---
 tests/lang/eval-okay-fromTOML.exp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp
index 5b9d47122cc..10ad2fe99ba 100644
--- a/tests/lang/eval-okay-fromTOML.exp
+++ b/tests/lang/eval-okay-fromTOML.exp
@@ -1 +1 @@
-[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
+[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]

From 17ef3e6f411b2cf1fa87593ad84bc2a94a1edd93 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 29 May 2019 12:22:52 +0200
Subject: [PATCH 1418/2196] Enable more fromTOML tests

cpptoml now parses almost all examples from the spec.
---
 tests/lang/eval-okay-fromTOML.exp |  2 +-
 tests/lang/eval-okay-fromTOML.nix | 30 ++++++++++++++----------------
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp
index 10ad2fe99ba..392ff7a728a 100644
--- a/tests/lang/eval-okay-fromTOML.exp
+++ b/tests/lang/eval-okay-fromTOML.exp
@@ -1 +1 @@
-[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
+[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix
index 8e7cbd1c61e..5626ef3382f 100644
--- a/tests/lang/eval-okay-fromTOML.nix
+++ b/tests/lang/eval-okay-fromTOML.nix
@@ -46,15 +46,15 @@
     "character encoding" = "value"
     "ʎǝʞ" = "value"
     'key2' = "value"
-    #'quoted "value"' = "value"
+    'quoted "value"' = "value"
 
     name = "Orange"
 
-    # FIXME: cpptoml doesn't handle dotted keys properly yet.
-    #physical.color = "orange"
-    #physical.shape = "round"
-    #site."google.com" = true
+    physical.color = "orange"
+    physical.shape = "round"
+    site."google.com" = true
 
+    # This is legal according to the spec, but cpptoml doesn't handle it.
     #a.b.c = 1
     #a.d = 2
 
@@ -68,16 +68,14 @@
     int6 = 5_349_221
     int7 = 1_2_3_4_5
 
-    # FIXME: cpptoml doesn't support these yet:
+    hex1 = 0xDEADBEEF
+    hex2 = 0xdeadbeef
+    hex3 = 0xdead_beef
 
-    #hex1 = 0xDEADBEEF
-    #hex2 = 0xdeadbeef
-    #hex3 = 0xdead_beef
+    oct1 = 0o01234567
+    oct2 = 0o755
 
-    #oct1 = 0o01234567
-    #oct2 = 0o755
-
-    #bin1 = 0b11010110
+    bin1 = 0b11010110
 
     flt1 = +1.0
     flt2 = 3.1415
@@ -126,8 +124,8 @@
     key1 = "another string"
     key2 = 456
 
-    #[dog."tater.man"]
-    #type.name = "pug"
+    [dog."tater.man"]
+    type.name = "pug"
 
     [a.b.c]
     [ d.e.f ]
@@ -137,7 +135,7 @@
 
     name = { first = "Tom", last = "Preston-Werner" }
     point = { x = 1, y = 2 }
-    #animal = { type.name = "pug" }
+    animal = { type.name = "pug" }
 
     [[products]]
     name = "Hammer"

From a8251ba2ed9e833ef3e8b20d233064aba74446db Mon Sep 17 00:00:00 2001
From: Johannes Climacus <johannesclimacus@tuta.io>
Date: Sat, 25 May 2019 05:06:43 -0400
Subject: [PATCH 1419/2196] Replace `type` with `command -v` in install script

In POSIX sh, `type` is undefined.

cf. https://pubs.opengroup.org/onlinepubs/9699919799/utilities/command.html#tag_20_22_04
---
 scripts/install.in | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/install.in b/scripts/install.in
index 7bff7b216d9..4857638c026 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -18,7 +18,7 @@ cleanup() {
 trap cleanup EXIT INT QUIT TERM
 
 require_util() {
-    type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 ||
+    command -v "$1" > /dev/null 2>&1 ||
         oops "you do not have '$1' installed, which I need to $2"
 }
 
@@ -41,11 +41,11 @@ require_util tar "unpack the binary tarball"
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
 
-if type sha256sum > /dev/null 2>&1; then
+if command -v sha256sum > /dev/null 2>&1; then
     hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
-elif type shasum > /dev/null 2>&1; then
+elif command -v shasum > /dev/null 2>&1; then
     hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
-elif type openssl > /dev/null 2>&1; then
+elif command -v openssl > /dev/null 2>&1; then
     hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
 else
     oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"

From 3b1cc8b0cbc0a2530e4671231257a497db8eede2 Mon Sep 17 00:00:00 2001
From: ln-nl <44435420+ln-nl@users.noreply.github.com>
Date: Thu, 30 May 2019 18:08:33 +0300
Subject: [PATCH 1420/2196] release-common: fix build with `config.allowAliases
 = false`

```sh
> nix build -f release.nix build.x86_64-linux
error: undefined variable 'docbook5_xsl' at /vcs/nix/release-common.nix:45:7
```
---
 release-common.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release-common.nix b/release-common.nix
index 4c556598526..2e8a951b9cd 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -42,7 +42,7 @@ rec {
       libxml2
       libxslt
       docbook5
-      docbook5_xsl
+      docbook_xsl_ns
       autoconf-archive
       autoreconfHook
     ];

From 2d34028b1e4e46d600926d4ad0c687c4d1d58c75 Mon Sep 17 00:00:00 2001
From: worldofpeace <worldofpeace@protonmail.ch>
Date: Fri, 31 May 2019 22:19:46 -0400
Subject: [PATCH 1421/2196] Add .github/FUNDING.yml

---
 .github/FUNDING.yml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .github/FUNDING.yml

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 00000000000..bbaabf93c7a
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+custom: https://nixos.org/nixos/foundation.html

From aec545c20b15eace71c6733107dd6fde7736afbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 1 Jun 2019 15:27:43 +0200
Subject: [PATCH 1422/2196] Fix segfault in builtin fetchurl with hashed
 mirrors + SRI hashes

---
 src/libstore/builtins/fetchurl.cc | 3 ++-
 tests/fetchurl.sh                 | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 92aec63a037..b1af3b4fc31 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
                 auto ht = parseHashType(getAttr("outputHashAlgo"));
-                fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
+                auto h = Hash(getAttr("outputHash"), ht);
+                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index ec3399b08d0..7319ced2b59 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -42,6 +42,10 @@ ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
 
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
 
+# Test hashed mirrors with an SRI hash.
+nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
+          --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+
 # Test unpacking a NAR.
 rm -rf $TEST_ROOT/archive
 mkdir -p $TEST_ROOT/archive

From 4b0d6133836113572418f383ddd14c92bfebc768 Mon Sep 17 00:00:00 2001
From: JorisE <JorisE@users.noreply.github.com>
Date: Tue, 4 Jun 2019 14:12:03 +0200
Subject: [PATCH 1423/2196] Minor typo

---
 doc/manual/expressions/language-constructs.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml
index 923b5d8c401..0d0cbbe1553 100644
--- a/doc/manual/expressions/language-constructs.xml
+++ b/doc/manual/expressions/language-constructs.xml
@@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para>
 
 <simplesect xml:id="sect-let-expressions"><title>Let-expressions</title>
 
-<para>A let-expression allows you define local variables for an
+<para>A let-expression allows you to define local variables for an
 expression.  For instance,
 
 <programlisting>

From 4a3e96281d6b9ae3dbc20e638f389677df9a649e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 5 Jun 2019 00:40:45 -0400
Subject: [PATCH 1424/2196] Handle SIGWINCH in main thread

For the SIGWINCH signal to be caught, it needs to be set in sigaction
on the main thread. Previously, this was broken, and updateWindowSize
was never being called. Tested on macOS 10.14.
---
 src/libmain/shared.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 4ed34e54dc5..f2aeec9e1db 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -125,6 +125,9 @@ void initNix()
     act.sa_handler = sigHandler;
     if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
 
+    /* Make sure SIGWINCH is handled as well. */
+    if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH");
+
     /* Register a SIGSEGV handler to detect stack overflows. */
     detectStackOverflow();
 

From 5011a52cf3f291c22d784aff6f5bbf8b99ae186a Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 5 Jun 2019 20:18:47 -0400
Subject: [PATCH 1425/2196] Just enable hack on macOS

This is not needed on linux at all! Tried to explain as much as I
understand with the problem.
---
 src/libmain/shared.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index f2aeec9e1db..a6101342ac7 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -125,8 +125,14 @@ void initNix()
     act.sa_handler = sigHandler;
     if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1");
 
-    /* Make sure SIGWINCH is handled as well. */
-    if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH");
+#if __APPLE__
+    /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH.
+     * Instead, add a dummy sigaction handler, and signalHandlerThread
+     * can handle the rest. */
+    struct sigaction sa;
+    sa.sa_handler = sigHandler;
+    if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH");
+#endif
 
     /* Register a SIGSEGV handler to detect stack overflows. */
     detectStackOverflow();

From 9e0f5f803f6cbfe9925cef69a0e58cbf5375bfaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Sat, 8 Jun 2019 00:41:19 +0200
Subject: [PATCH 1426/2196] Daemon: warn when an untrusted user cannot override
 a setting

In a daemon-based Nix setup, some options cannot be overridden by a
client unless the client's user is considered trusted.

Currently, if an untrusted user tries to override one of those
options, we are silently ignoring it.

This can be pretty confusing in certain situations.

e.g. a user thinks he disabled the sandbox when in reality he did not.

We are now sending a warning message letting know the user some options
have been ignored.

Related to #1761.
---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 8d63b8f362e..973f64a3446 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 else if (setSubstituters(settings.extraSubstituters))
                     ;
                 else
-                    debug("ignoring untrusted setting '%s'", name);
+                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name);
             } catch (UsageError & e) {
                 warn(e.what());
             }

From 34fa8ce9179b17cc4cd13ae49b69fccf393271a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 15 Jun 2019 16:34:06 +0200
Subject: [PATCH 1427/2196] nix: Support -j flag

---
 src/libmain/common-args.cc | 9 +++++++++
 src/libmain/shared.cc      | 4 ----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 4c35a419959..9e1d7cee60e 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName)
             }
         });
 
+    mkFlag()
+        .longName("max-jobs")
+        .shortName('j')
+        .label("jobs")
+        .description("maximum number of parallel builds")
+        .handler([=](std::string s) {
+            settings.set("max-jobs", s);
+        });
+
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 4ed34e54dc5..cd752f4678a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -175,10 +175,6 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         .description("build from source if substitution fails")
         .set(&(bool&) settings.tryFallback, true);
 
-    mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) {
-        settings.set("max-jobs", s);
-    });
-
     auto intSettingAlias = [&](char shortName, const std::string & longName,
         const std::string & description, const std::string & dest) {
         mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {

From 5064971ded3e2cc7c6b80d3166270a87a9650c09 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 May 2019 21:15:45 +0200
Subject: [PATCH 1428/2196] Fix test failures when $TMPDIR changes

(cherry picked from commit c38c726eb5d447c7e9d894d57cd05ac46c173ddd)
---
 tests/build-dry.sh    | 6 +++---
 tests/nix-copy-ssh.sh | 2 +-
 tests/nix-shell.sh    | 8 ++++----
 tests/placeholders.sh | 2 --
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/build-dry.sh b/tests/build-dry.sh
index 610e6070c5d..e72533e7061 100644
--- a/tests/build-dry.sh
+++ b/tests/build-dry.sh
@@ -8,13 +8,13 @@ clearStore
 clearCache
 
 # Ensure this builds successfully first
-nix build -f dependencies.nix
+nix build --no-link -f dependencies.nix
 
 clearStore
 clearCache
 
 # Try --dry-run using old command first
-nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now new command:
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
@@ -27,7 +27,7 @@ clearCache
 # Try --dry-run using new command first
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now old command:
-nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
 fi
 
 ###################################################
diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh
index 6aba667a45a..eb801548d2f 100644
--- a/tests/nix-copy-ssh.sh
+++ b/tests/nix-copy-ssh.sh
@@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2
 chmod -R u+w "$remoteRoot" || true
 rm -rf "$remoteRoot"
 
-outPath=$(nix-build dependencies.nix)
+outPath=$(nix-build --no-out-link dependencies.nix)
 
 nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
 
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 6024ea39975..ee502dddb95 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run
 # Test nix-shell on a .drv symlink
 
 # Legacy: absolute path and .drv extension required
-nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv
-[[ $(nix-shell --pure $PWD/shell.drv --run \
+nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv
+[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # New behaviour: just needs to resolve to a derivation in the store
-nix-instantiate shell.nix -A shellDrv --indirect --add-root shell
-[[ $(nix-shell --pure shell --run \
+nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell
+[[ $(nix-shell --pure $TEST_ROOT/shell --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # Test nix-shell -p
diff --git a/tests/placeholders.sh b/tests/placeholders.sh
index 071cfe2dc89..cd1bb7bc2aa 100644
--- a/tests/placeholders.sh
+++ b/tests/placeholders.sh
@@ -18,5 +18,3 @@ nix-build --no-out-link -E '
     ";
   }
 '
-
-echo XYZZY

From 26bc876ae6e7ceff8b37ba5c2c1043ce216cf384 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 15 Jun 2019 16:45:00 +0200
Subject: [PATCH 1429/2196] nix: Add -L alias for --print-build-logs

---
 src/nix/main.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 4f87ad72b65..25e321b86bc 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -45,6 +45,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
         mkFlag()
             .longName("print-build-logs")
+            .shortName('L')
             .description("print full build logs on stderr")
             .set(&printBuildLogs, true);
 

From b693029ca076abf8a4084a053a122ffcdf4d61b2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 16 Jun 2019 09:43:20 +0200
Subject: [PATCH 1430/2196] Style fix

---
 src/nix-daemon/nix-daemon.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 973f64a3446..e88aaf63644 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 else if (setSubstituters(settings.extraSubstituters))
                     ;
                 else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name);
+                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
             } catch (UsageError & e) {
                 warn(e.what());
             }

From e84c26564507388f2e8b7c7f2b00eb2b57856da1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 17 May 2019 22:29:15 +0200
Subject: [PATCH 1431/2196] Run builds in a pseudo-terminal

This allows many programs (e.g. gcc, clang, cmake) to print colorized
log output (assuming $TERM is set to a value like "xterm").

There are other ways to get colors, in particular setting
CLICOLOR_FORCE, but they're less widely supported and can break
programs that parse tool output.
---
 src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 46 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0bd7388097c..9de7613f8d7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -38,6 +38,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <cstring>
+#include <termios.h>
 
 #include <pwd.h>
 #include <grp.h>
@@ -1558,8 +1559,8 @@ void DerivationGoal::buildDone()
     if (hook) {
         hook->builderOut.readSide = -1;
         hook->fromHook.readSide = -1;
-    }
-    else builderOut.readSide = -1;
+    } else
+        builderOut.readSide = -1;
 
     /* Close the log file. */
     closeLogFile();
@@ -2181,7 +2182,43 @@ void DerivationGoal::startBuilder()
     Path logFile = openLogFile();
 
     /* Create a pipe to get the output of the builder. */
-    builderOut.create();
+    //builderOut.create();
+
+    builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
+    if (!builderOut.readSide)
+        throw SysError("opening pseudoterminal master");
+
+    std::string slaveName(ptsname(builderOut.readSide.get()));
+
+    if (chmod(slaveName.c_str(), 0600))
+        throw SysError("changing mode of pseudoterminal slave");
+
+    if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0))
+        throw SysError("changing owner of pseudoterminal slave");
+
+    #if 0
+    // Mount the pt in the sandbox so that the "tty" command works.
+    // FIXME: this doesn't work with the new devpts in the sandbox.
+    if (useChroot)
+        dirsInChroot[slaveName] = {slaveName, false};
+    #endif
+
+    if (unlockpt(builderOut.readSide.get()))
+        throw SysError("unlocking pseudoterminal");
+
+    builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
+    if (!builderOut.writeSide)
+        throw SysError("opening pseudoterminal slave");
+
+    // Put the pt into raw mode to prevent \n -> \r\n translation.
+    struct termios term;
+    if (tcgetattr(builderOut.writeSide.get(), &term))
+        throw SysError("getting pseudoterminal attributes");
+
+    cfmakeraw(&term);
+
+    if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
+        throw SysError("putting pseudoterminal into raw mode");
 
     result.startTime = time(0);
 
@@ -4361,14 +4398,15 @@ void Worker::waitForInput()
         for (auto & k : fds2) {
             if (FD_ISSET(k, &fds)) {
                 ssize_t rd = read(k, buffer.data(), buffer.size());
-                if (rd == -1) {
-                    if (errno != EINTR)
-                        throw SysError(format("reading from %1%")
-                            % goal->getName());
-                } else if (rd == 0) {
+                // FIXME: is there a cleaner way to handle pt close
+                // than EIO? Is this even standard?
+                if (rd == 0 || (rd == -1 && errno == EIO)) {
                     debug(format("%1%: got EOF") % goal->getName());
                     goal->handleEOF(k);
                     j->fds.erase(k);
+                } else if (rd == -1) {
+                    if (errno != EINTR)
+                        throw SysError("%s: read failed", goal->getName());
                 } else {
                     printMsg(lvlVomit, format("%1%: read %2% bytes")
                         % goal->getName() % rd);

From 82ca6ef390b752faf1bf27b22daa34b29a4b00d4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 19 May 2019 16:56:08 +0200
Subject: [PATCH 1432/2196] Set $TERM

---
 src/libstore/build.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9de7613f8d7..5a75403cfc9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2443,6 +2443,9 @@ void DerivationGoal::initEnv()
        may change that in the future. So tell the builder which file
        descriptor to use for that. */
     env["NIX_LOG_FD"] = "2";
+
+    /* Trigger colored output in various tools. */
+    env["TERM"] = "xterm-256color";
 }
 
 

From 2743bf0bb1727056ce6d6d6be4e38c3251aac1f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 16 Jun 2019 20:02:40 +0200
Subject: [PATCH 1433/2196] Hopefully fix macOS tests

---
 src/libstore/build.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5a75403cfc9..cccf0c7c687 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2190,11 +2190,13 @@ void DerivationGoal::startBuilder()
 
     std::string slaveName(ptsname(builderOut.readSide.get()));
 
-    if (chmod(slaveName.c_str(), 0600))
-        throw SysError("changing mode of pseudoterminal slave");
+    if (buildUser) {
+        if (chmod(slaveName.c_str(), 0600))
+            throw SysError("changing mode of pseudoterminal slave");
 
-    if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0))
-        throw SysError("changing owner of pseudoterminal slave");
+        if (chown(slaveName.c_str(), buildUser->getUID(), 0))
+            throw SysError("changing owner of pseudoterminal slave");
+    }
 
     #if 0
     // Mount the pt in the sandbox so that the "tty" command works.

From 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jun 2019 08:08:04 +0200
Subject: [PATCH 1434/2196] Another attempt at getting pseudoterminals to work
 on macOS

---
 src/libstore/build.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cccf0c7c687..5b38bcf3c5e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2196,6 +2196,9 @@ void DerivationGoal::startBuilder()
 
         if (chown(slaveName.c_str(), buildUser->getUID(), 0))
             throw SysError("changing owner of pseudoterminal slave");
+    } else {
+        if (grantpt(builderOut.readSide.get()))
+            throw SysError("granting access to pseudoterminal slave");
     }
 
     #if 0

From 74a65d313f5c9f4d504ea9e2caadfe0ff83776d5 Mon Sep 17 00:00:00 2001
From: Bruno Bieth <biethb@gmail.com>
Date: Thu, 20 Jun 2019 14:47:45 +0200
Subject: [PATCH 1435/2196] Nix uses the CPP SDK, not Java

---
 doc/manual/packages/s3-substituter.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index e7589ffdb03..1722090efec 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -113,7 +113,7 @@ the S3 URL:</para>
   exactly <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 
@@ -138,7 +138,7 @@ the S3 URL:</para>
   be <uri>s3://example-nix-cache</uri>.</para>
 
   <para>Nix will use the <link
-  xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
+  xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default
   credential provider chain</link> for authenticating requests to
   Amazon S3.</para>
 

From 99cec651c9c1d95656e182871f0bb186f798077c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Jun 2019 16:52:11 +0200
Subject: [PATCH 1436/2196] Add more fromTOML tests

---
 tests/lang/eval-okay-fromTOML.exp |  2 +-
 tests/lang/eval-okay-fromTOML.nix | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp
index 392ff7a728a..3c16e1fa146 100644
--- a/tests/lang/eval-okay-fromTOML.exp
+++ b/tests/lang/eval-okay-fromTOML.exp
@@ -1 +1 @@
-[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
+[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix
index 5626ef3382f..dd9ad9ea122 100644
--- a/tests/lang/eval-okay-fromTOML.nix
+++ b/tests/lang/eval-okay-fromTOML.nix
@@ -147,6 +147,25 @@
     name = "Nail"
     sku = 284758393
     color = "gray"
+
+    [[fruit]]
+      name = "apple"
+
+      [fruit.physical]
+        color = "red"
+        shape = "round"
+
+      [[fruit.variety]]
+        name = "red delicious"
+
+      [[fruit.variety]]
+        name = "granny smith"
+
+    [[fruit]]
+      name = "banana"
+
+      [[fruit.variety]]
+        name = "plantain"
   '')
 
   (builtins.fromTOML ''

From 94f11d0a61bba15ff8a93f2de8d1883783c8c508 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Jun 2019 17:09:09 +0200
Subject: [PATCH 1437/2196] Fix abort in fromTOML

Fixes #2969.
---
 src/libexpr/primops/fromTOML.cc   | 13 +++++++++++++
 tests/lang/eval-okay-fromTOML.exp |  2 +-
 tests/lang/eval-okay-fromTOML.nix |  7 +++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 4128de05d0c..a84e569e944 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
                 visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
         }
 
+        // Handle cases like 'a = [[{ a = true }]]', which IMHO should be
+        // parsed as a array containing an array containing a table,
+        // but instead are parsed as an array containing a table array
+        // containing a table.
+        else if (auto t2 = t->as_table_array()) {
+            size_t size = t2->get().size();
+
+            state.mkList(v, size);
+
+            for (size_t j = 0; j < size; ++j)
+                visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
+        }
+
         else if (t->is_value()) {
             if (auto val = t->as<int64_t>())
                 mkInt(v, val->get());
diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp
index 3c16e1fa146..d0dd3af2c81 100644
--- a/tests/lang/eval-okay-fromTOML.exp
+++ b/tests/lang/eval-okay-fromTOML.exp
@@ -1 +1 @@
-[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ]
+[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ]
diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix
index dd9ad9ea122..96393268994 100644
--- a/tests/lang/eval-okay-fromTOML.nix
+++ b/tests/lang/eval-okay-fromTOML.nix
@@ -198,4 +198,11 @@
     "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
     "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
   '')
+
+  (builtins.fromTOML ''
+    a = [[{ b = true }]]
+    c = [ [ { d = true } ] ]
+    e = [[123]]
+  '')
+
 ]

From dc29e9fb47f9f98a851dc88b2bd3cae4b5c1fe6b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 25 Feb 2019 23:20:50 +0800
Subject: [PATCH 1438/2196] downloadCached: Return ETag

(cherry picked from commit 529add316c5356a8060c35f987643b7bf5c796dc)
---
 src/libexpr/common-eval-args.cc |  2 +-
 src/libexpr/parser.y            |  2 +-
 src/libexpr/primops.cc          |  2 +-
 src/libstore/download.cc        | 17 +++++++++++++----
 src/libstore/download.hh        | 12 ++++++++++--
 src/nix-channel/nix-channel.cc  |  6 +++---
 6 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 3e0c78f280f..37c74a94b29 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s))
-        return getDownloader()->downloadCached(state.store, s, true);
+        return getDownloader()->downloadCached(state.store, s, true).path;
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 78a50390718..29860fb0576 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -677,7 +677,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, getDownloader()->downloadCached(store, elem.second, true) };
+            res = { true, getDownloader()->downloadCached(store, elem.second, true).path };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 06f577f36fc..55a1bde117c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2083,7 +2083,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash);
+    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path;
 
     if (state.allowedPaths)
         state.allowedPaths->insert(res);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 22382ab1d6e..cb77cdc77df 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -790,7 +790,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
+CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
 {
     auto url = resolveUri(url_);
 
@@ -802,8 +802,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     Path expectedStorePath;
     if (expectedHash) {
         expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
-        if (store->isValidPath(expectedStorePath))
-            return store->toRealPath(expectedStorePath);
+        if (store->isValidPath(expectedStorePath)) {
+            CachedDownloadResult result;
+            result.path = store->toRealPath(expectedStorePath);
+            return result;
+        }
     }
 
     Path cacheDir = getCacheDir() + "/nix/tarballs";
@@ -822,6 +825,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 
     bool skip = false;
 
+    CachedDownloadResult result;
+
     if (pathExists(fileLink) && pathExists(dataFile)) {
         storePath = readLink(fileLink);
         store->addTempRoot(storePath);
@@ -833,6 +838,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                     skip = true;
                     if (effectiveUrl)
                         *effectiveUrl = url_;
+                    result.etag = ss[1];
                 } else if (!ss[1].empty()) {
                     debug(format("verifying previous ETag '%1%'") % ss[1]);
                     expectedETag = ss[1];
@@ -850,6 +856,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             auto res = download(request);
             if (effectiveUrl)
                 *effectiveUrl = res.effectiveUrl;
+            result.etag = res.etag;
 
             if (!res.cached) {
                 ValidPathInfo info;
@@ -871,6 +878,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
             printError(format("warning: %1%; using cached result") % e.msg());
+            result.etag = expectedETag;
         }
     }
 
@@ -904,7 +912,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             url, expectedHash.to_string(), gotHash.to_string());
     }
 
-    return store->toRealPath(storePath);
+    result.path = store->toRealPath(storePath);
+    return result;
 }
 
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index f0228f7d053..8acfe4e1a39 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -41,6 +41,12 @@ struct DownloadResult
     uint64_t bodySize = 0;
 };
 
+struct CachedDownloadResult
+{
+    Path path;
+    std::optional<std::string> etag;
+};
+
 class Store;
 
 struct Downloader
@@ -64,8 +70,10 @@ struct Downloader
        and is more recent than ‘tarball-ttl’ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
-    Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "",
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl);
+    CachedDownloadResult downloadCached(
+        ref<Store> store, const string & uri, bool unpack, string name = "",
+        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr,
+        int ttl = settings.tarballTtl);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 8b66cc7e314..7b23088a2c5 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -88,7 +88,7 @@ static void update(const StringSet & channelNames)
         // definition from a consistent location if the redirect changes mid-download.
         std::string effectiveUrl;
         auto dl = getDownloader();
-        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0);
+        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path;
         url = chomp(std::move(effectiveUrl));
 
         // If the URL contains a version number, append it to the name
@@ -123,10 +123,10 @@ static void update(const StringSet & channelNames)
             // Download the channel tarball.
             auto fullURL = url + "/nixexprs.tar.xz";
             try {
-                filename = dl->downloadCached(store, fullURL, false);
+                filename = dl->downloadCached(store, fullURL, false).path;
             } catch (DownloadError & e) {
                 fullURL = url + "/nixexprs.tar.bz2";
-                filename = dl->downloadCached(store, fullURL, false);
+                filename = dl->downloadCached(store, fullURL, false).path;
             }
             chomp(filename);
         }

From b43e1e186e50bedc30d05d063dffbd751c1ff161 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 May 2019 15:38:24 +0200
Subject: [PATCH 1439/2196] CachedDownloadResult: Include store path

Also, make fetchGit and fetchMercurial update allowedPaths properly.

(Maybe the evaluator, rather than the caller of the evaluator, should
apply toRealPath(), but that's a bigger change.)

(cherry picked from commit 5c34d665386f4053d666b0899ecca0639e500fbd)
---
 src/libexpr/primops/fetchGit.cc       | 2 +-
 src/libexpr/primops/fetchMercurial.cc | 2 +-
 src/libstore/download.cc              | 2 ++
 src/libstore/download.hh              | 3 +++
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index aaf02c856d4..cf7ccca418e 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -235,7 +235,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(gitInfo.storePath);
+        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 66f49f37432..d576138f543 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -214,7 +214,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(hgInfo.storePath);
+        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index cb77cdc77df..975cfd97d30 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -804,6 +804,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
         expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
         if (store->isValidPath(expectedStorePath)) {
             CachedDownloadResult result;
+            result.storePath = expectedStorePath;
             result.path = store->toRealPath(expectedStorePath);
             return result;
         }
@@ -912,6 +913,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
             url, expectedHash.to_string(), gotHash.to_string());
     }
 
+    result.storePath = storePath;
     result.path = store->toRealPath(storePath);
     return result;
 }
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 8acfe4e1a39..aa8c34be2d6 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -43,6 +43,9 @@ struct DownloadResult
 
 struct CachedDownloadResult
 {
+    // Note: 'storePath' may be different from 'path' when using a
+    // chroot store.
+    Path storePath;
     Path path;
     std::optional<std::string> etag;
 };

From 2fef4dd29673ef383b695480c9b8212f2e8d4711 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Jun 2019 21:06:37 +0200
Subject: [PATCH 1440/2196] Downloader: Propagate exceptions from
 decompressionSink->finish()

(cherry picked from commit 15fa70cd1b853f5e62662b99ccb9ef3da6cfadff)
---
 src/libstore/download.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 975cfd97d30..8e94646d49a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -327,8 +327,13 @@ struct CurlDownloader : public Downloader
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
                 request.verb(), request.uri, code, httpStatus, result.bodySize);
 
-            if (decompressionSink)
-                decompressionSink->finish();
+            if (decompressionSink) {
+                try {
+                    decompressionSink->finish();
+                } catch (...) {
+                    writeException = std::current_exception();
+                }
+            }
 
             if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) {
                 code = CURLE_OK;

From 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Jun 2019 21:48:52 +0200
Subject: [PATCH 1441/2196] Fix 'error 9 while decompressing xz file'

Once we've started writing data to a Sink, we can't restart a download
request, because then we end up writing duplicate data to the
Sink. Therefore we shouldn't handle retries in Downloader but at a
higher level (in particular, in copyStorePath()).

Fixes #2952.

(cherry picked from commit a67cf5a3585c41dd9f219a2c7aa9cf67fa69520b)
---
 src/libstore/binary-cache-store.cc      | 18 +++--
 src/libstore/download.cc                | 67 ++++--------------
 src/libstore/download.hh                | 11 ++-
 src/libstore/http-binary-cache-store.cc | 57 ++++++++++-----
 src/libstore/store-api.cc               | 94 +++++++++++++------------
 src/libutil/retry.hh                    | 38 ++++++++++
 src/libutil/types.hh                    |  2 +
 7 files changed, 162 insertions(+), 125 deletions(-)
 create mode 100644 src/libutil/retry.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4527ee6ba66..8b736056e01 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -10,6 +10,8 @@
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
 #include "json.hh"
+#include "retry.hh"
+#include "download.hh"
 
 #include <chrono>
 
@@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 
 std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
-    StringSink sink;
-    try {
-        getFile(path, sink);
-    } catch (NoSuchBinaryCacheFile &) {
-        return nullptr;
-    }
-    return sink.s;
+    return retry<std::shared_ptr<std::string>>(downloadSettings.tries, [&]() -> std::shared_ptr<std::string> {
+        StringSink sink;
+        try {
+            getFile(path, sink);
+        } catch (NoSuchBinaryCacheFile &) {
+            return nullptr;
+        }
+        return sink.s;
+    });
 }
 
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 8e94646d49a..43d231e3036 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -8,6 +8,7 @@
 #include "compression.hh"
 #include "pathlocks.hh"
 #include "finally.hh"
+#include "retry.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -19,11 +20,9 @@
 #include <curl/curl.h>
 
 #include <algorithm>
-#include <cmath>
 #include <cstring>
 #include <iostream>
 #include <queue>
-#include <random>
 #include <thread>
 
 using namespace std::string_literals;
@@ -62,9 +61,6 @@ struct CurlDownloader : public Downloader
 {
     CURLM * curlm = 0;
 
-    std::random_device rd;
-    std::mt19937 mt19937;
-
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
         CurlDownloader & downloader;
@@ -77,12 +73,6 @@ struct CurlDownloader : public Downloader
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
 
-        unsigned int attempt = 0;
-
-        /* Don't start this download until the specified time point
-           has been reached. */
-        std::chrono::steady_clock::time_point embargo;
-
         struct curl_slist * requestHeaders = 0;
 
         std::string encoding;
@@ -401,9 +391,7 @@ struct CurlDownloader : public Downloader
                     }
                 }
 
-                attempt++;
-
-                auto exc =
+                fail(
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
                     ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
@@ -414,31 +402,15 @@ struct CurlDownloader : public Downloader
                         )
                     : DownloadError(err,
                         fmt("unable to %s '%s': %s (%d)",
-                            request.verb(), request.uri, curl_easy_strerror(code), code));
-
-                /* If this is a transient error, then maybe retry the
-                   download after a while. */
-                if (err == Transient && attempt < request.tries) {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    printError(format("warning: %s; retrying in %d ms") % exc.what() % ms);
-                    embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
-                    downloader.enqueueItem(shared_from_this());
-                }
-                else
-                    fail(exc);
+                            request.verb(), request.uri, curl_easy_strerror(code), code)));
             }
         }
     };
 
     struct State
     {
-        struct EmbargoComparator {
-            bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
-                return i1->embargo > i2->embargo;
-            }
-        };
         bool quit = false;
-        std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
+        std::vector<std::shared_ptr<DownloadItem>> incoming;
     };
 
     Sync<State> state_;
@@ -451,7 +423,6 @@ struct CurlDownloader : public Downloader
     std::thread workerThread;
 
     CurlDownloader()
-        : mt19937(rd())
     {
         static std::once_flag globalInit;
         std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
@@ -545,9 +516,7 @@ struct CurlDownloader : public Downloader
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
-            /* Add new curl requests from the incoming requests queue,
-               except for requests that are embargoed (waiting for a
-               retry timeout to expire). */
+            /* Add new curl requests from the incoming requests queue. */
             if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
                 char buf[1024];
                 auto res = read(extraFDs[0].fd, buf, sizeof(buf));
@@ -556,22 +525,9 @@ struct CurlDownloader : public Downloader
             }
 
             std::vector<std::shared_ptr<DownloadItem>> incoming;
-            auto now = std::chrono::steady_clock::now();
-
             {
                 auto state(state_.lock());
-                while (!state->incoming.empty()) {
-                    auto item = state->incoming.top();
-                    if (item->embargo <= now) {
-                        incoming.push_back(item);
-                        state->incoming.pop();
-                    } else {
-                        if (nextWakeup == std::chrono::steady_clock::time_point()
-                            || item->embargo < nextWakeup)
-                            nextWakeup = item->embargo;
-                        break;
-                    }
-                }
+                std::swap(state->incoming, incoming);
                 quit = state->quit;
             }
 
@@ -598,7 +554,7 @@ struct CurlDownloader : public Downloader
 
         {
             auto state(state_.lock());
-            while (!state->incoming.empty()) state->incoming.pop();
+            state->incoming.clear();
             state->quit = true;
         }
     }
@@ -614,7 +570,7 @@ struct CurlDownloader : public Downloader
             auto state(state_.lock());
             if (state->quit)
                 throw nix::Error("cannot enqueue download request because the download thread is shutting down");
-            state->incoming.push(item);
+            state->incoming.push_back(item);
         }
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
@@ -697,7 +653,9 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
 
 DownloadResult Downloader::download(const DownloadRequest & request)
 {
-    return enqueueDownload(request).get();
+    return retry<DownloadResult>(request.tries, [&]() {
+        return enqueueDownload(request).get();
+    });
 }
 
 void Downloader::download(DownloadRequest && request, Sink & sink)
@@ -883,7 +841,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
             writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
-            printError(format("warning: %1%; using cached result") % e.msg());
+            warn("%s; using cached result", e.msg());
             result.etag = expectedETag;
         }
     }
@@ -933,5 +891,4 @@ bool isUri(const string & s)
     return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
 }
 
-
 }
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index aa8c34be2d6..752f918575f 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -62,11 +62,13 @@ struct Downloader
 
     std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
 
-    /* Synchronously download a file. */
+    /* Synchronously download a file. The request will be retried in
+       case of transient failures. */
     DownloadResult download(const DownloadRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
-       invoked on the thread of the caller. */
+       invoked on the thread of the caller. The request will not be
+       retried in case of transient failures. */
     void download(DownloadRequest && request, Sink & sink);
 
     /* Check if the specified file is already in ~/.cache/nix/tarballs
@@ -95,6 +97,11 @@ public:
     DownloadError(Downloader::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
+
+    bool isTransient() override
+    {
+        return error == Downloader::Error::Transient;
+    }
 };
 
 bool isUri(const string & s);
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 8da0e2f9d82..5133dba5720 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -2,6 +2,7 @@
 #include "download.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
+#include "retry.hh"
 
 namespace nix {
 
@@ -114,7 +115,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
-        request.tries = 8;
         return request;
     }
 
@@ -137,21 +137,46 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     {
         checkEnabled();
 
-        auto request(makeRequest(path));
-
-        getDownloader()->enqueueDownload(request,
-            {[callback, this](std::future<DownloadResult> result) {
-                try {
-                    callback(result.get().data);
-                } catch (DownloadError & e) {
-                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                        return callback(std::shared_ptr<std::string>());
-                    maybeDisable();
-                    callback.rethrow();
-                } catch (...) {
-                    callback.rethrow();
-                }
-            }});
+        struct State
+        {
+            DownloadRequest request;
+            std::function<void()> tryDownload;
+            unsigned int attempt = 0;
+            State(DownloadRequest && request) : request(request) {}
+        };
+
+        auto state = std::make_shared<State>(makeRequest(path));
+
+        state->tryDownload = [callback, state, this]() {
+            getDownloader()->enqueueDownload(state->request,
+                {[callback, state, this](std::future<DownloadResult> result) {
+                    try {
+                        callback(result.get().data);
+                    } catch (DownloadError & e) {
+                        if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                            return callback(std::shared_ptr<std::string>());
+                        ++state->attempt;
+                        if (state->attempt < state->request.tries && e.isTransient()) {
+                            auto ms = retrySleepTime(state->attempt);
+                            warn("%s; retrying in %d ms", e.what(), ms);
+                            /* We can't sleep here because that would
+                               block the download thread. So use a
+                               separate thread for sleeping. */
+                            std::thread([state, ms]() {
+                                std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+                                state->tryDownload();
+                            }).detach();
+                        } else {
+                            maybeDisable();
+                            callback.rethrow();
+                        }
+                    } catch (...) {
+                        callback.rethrow();
+                    }
+                }});
+        };
+
+        state->tryDownload();
     }
 
 };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c13ff11564e..28ad7c019a9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,10 +6,11 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
+#include "retry.hh"
+#include "download.hh"
 
 #include <future>
 
-
 namespace nix {
 
 
@@ -572,54 +573,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    auto srcUri = srcStore->getUri();
-    auto dstUri = dstStore->getUri();
-
-    Activity act(*logger, lvlInfo, actCopyPath,
-        srcUri == "local" || srcUri == "daemon"
-          ? fmt("copying path '%s' to '%s'", storePath, dstUri)
-          : dstUri == "local" || dstUri == "daemon"
-            ? fmt("copying path '%s' from '%s'", storePath, srcUri)
-            : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
-        {storePath, srcUri, dstUri});
-    PushActivity pact(act.id);
-
-    auto info = srcStore->queryPathInfo(storePath);
-
-    uint64_t total = 0;
-
-    if (!info->narHash) {
-        StringSink sink;
-        srcStore->narFromPath({storePath}, sink);
-        auto info2 = make_ref<ValidPathInfo>(*info);
-        info2->narHash = hashString(htSHA256, *sink.s);
-        if (!info->narSize) info2->narSize = sink.s->size();
-        if (info->ultimate) info2->ultimate = false;
-        info = info2;
-
-        StringSource source(*sink.s);
-        dstStore->addToStore(*info, source, repair, checkSigs);
-        return;
-    }
+    retry<void>(downloadSettings.tries, [&]() {
 
-    if (info->ultimate) {
-        auto info2 = make_ref<ValidPathInfo>(*info);
-        info2->ultimate = false;
-        info = info2;
-    }
+        auto srcUri = srcStore->getUri();
+        auto dstUri = dstStore->getUri();
+
+        Activity act(*logger, lvlInfo, actCopyPath,
+            srcUri == "local" || srcUri == "daemon"
+              ? fmt("copying path '%s' to '%s'", storePath, dstUri)
+              : dstUri == "local" || dstUri == "daemon"
+                ? fmt("copying path '%s' from '%s'", storePath, srcUri)
+                : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
+            {storePath, srcUri, dstUri});
+        PushActivity pact(act.id);
+
+        auto info = srcStore->queryPathInfo(storePath);
+
+        uint64_t total = 0;
+
+        if (!info->narHash) {
+            StringSink sink;
+            srcStore->narFromPath({storePath}, sink);
+            auto info2 = make_ref<ValidPathInfo>(*info);
+            info2->narHash = hashString(htSHA256, *sink.s);
+            if (!info->narSize) info2->narSize = sink.s->size();
+            if (info->ultimate) info2->ultimate = false;
+            info = info2;
 
-    auto source = sinkToSource([&](Sink & sink) {
-        LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
-            sink(data, len);
-            total += len;
-            act.progress(total, info->narSize);
+            StringSource source(*sink.s);
+            dstStore->addToStore(*info, source, repair, checkSigs);
+            return;
+        }
+
+        if (info->ultimate) {
+            auto info2 = make_ref<ValidPathInfo>(*info);
+            info2->ultimate = false;
+            info = info2;
+        }
+
+        auto source = sinkToSource([&](Sink & sink) {
+            LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
+                sink(data, len);
+                total += len;
+                act.progress(total, info->narSize);
+            });
+            srcStore->narFromPath({storePath}, wrapperSink);
+        }, [&]() {
+            throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
         });
-        srcStore->narFromPath({storePath}, wrapperSink);
-    }, [&]() {
-        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
-    });
 
-    dstStore->addToStore(*info, *source, repair, checkSigs);
+        dstStore->addToStore(*info, *source, repair, checkSigs);
+    });
 }
 
 
diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh
new file mode 100644
index 00000000000..b45cb37f736
--- /dev/null
+++ b/src/libutil/retry.hh
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "logging.hh"
+
+#include <functional>
+#include <cmath>
+#include <random>
+#include <thread>
+
+namespace nix {
+
+inline unsigned int retrySleepTime(unsigned int attempt)
+{
+    std::random_device rd;
+    std::mt19937 mt19937;
+    return 250.0 * std::pow(2.0f,
+        attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937));
+}
+
+template<typename C>
+C retry(unsigned int attempts, std::function<C()> && f)
+{
+    unsigned int attempt = 0;
+    while (true) {
+        try {
+            return f();
+        } catch (BaseError & e) {
+            ++attempt;
+            if (attempt >= attempts || !e.isTransient())
+                throw;
+            auto ms = retrySleepTime(attempt);
+            warn("%s; retrying in %d ms", e.what(), ms);
+            std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+        }
+    }
+}
+
+}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 92bf469b5c6..88e3243f47a 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -109,6 +109,8 @@ public:
     const string & msg() const { return err; }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
+
+    virtual bool isTransient() { return false; }
 };
 
 #define MakeError(newClass, superClass) \

From 7b9c68766d513260d5262d5782b46384834cdb33 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jun 2019 08:43:45 +0200
Subject: [PATCH 1442/2196] Add '--no-net' convenience flag

This flag

* Disables substituters.

* Sets the tarball-ttl to infinity (ensuring e.g. that the flake
  registry and any downloaded flakes are considered current).

* Disables retrying downloads and sets the connection timeout to the
  minimum. (So it doesn't completely disable downloads at the moment.)

(cherry picked from commit 8ea842260b4fd93315d35c5ba94b1ff99ab391d8)
---
 src/libmain/common-args.cc              | 11 +++++++++++
 src/libstore/download.cc                | 18 +-----------------
 src/libstore/download.hh                | 23 ++++++++++++++++++++++-
 src/libstore/globals.hh                 |  2 +-
 src/libstore/http-binary-cache-store.cc |  1 -
 5 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 9e1d7cee60e..0486932b20e 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -1,5 +1,6 @@
 #include "common-args.hh"
 #include "globals.hh"
+#include "download.hh"
 
 namespace nix {
 
@@ -44,6 +45,16 @@ MixCommonArgs::MixCommonArgs(const string & programName)
             settings.set("max-jobs", s);
         });
 
+    mkFlag()
+        .longName("no-net")
+        .description("disable substituters and consider all previously downloaded files up-to-date")
+        .handler([]() {
+            settings.useSubstitutes = false;
+            settings.tarballTtl = std::numeric_limits<unsigned int>::max();
+            downloadSettings.tries = 0;
+            downloadSettings.connectTimeout = 1;
+        });
+
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 43d231e3036..342a8aa2146 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -29,23 +29,7 @@ using namespace std::string_literals;
 
 namespace nix {
 
-struct DownloadSettings : Config
-{
-    Setting<bool> enableHttp2{this, true, "http2",
-        "Whether to enable HTTP/2 support."};
-
-    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
-        "String appended to the user agent in HTTP requests."};
-
-    Setting<size_t> httpConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
-        {"binary-caches-parallel-connections"}};
-
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
-};
-
-static DownloadSettings downloadSettings;
+DownloadSettings downloadSettings;
 
 static GlobalConfig::Register r1(&downloadSettings);
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 752f918575f..dae082ab9ae 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -9,13 +9,34 @@
 
 namespace nix {
 
+struct DownloadSettings : Config
+{
+    Setting<bool> enableHttp2{this, true, "http2",
+        "Whether to enable HTTP/2 support."};
+
+    Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
+        "String appended to the user agent in HTTP requests."};
+
+    Setting<size_t> httpConnections{this, 25, "http-connections",
+        "Number of parallel HTTP connections.",
+        {"binary-caches-parallel-connections"}};
+
+    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
+        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+
+    Setting<unsigned int> tries{this, 5, "download-attempts",
+        "How often Nix will attempt to download a file before giving up."};
+};
+
+extern DownloadSettings downloadSettings;
+
 struct DownloadRequest
 {
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = 5;
+    size_t tries = downloadSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 53efc6a90fb..0af8215d1fd 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -255,7 +255,7 @@ public:
         "Secret keys with which to sign local builds."};
 
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
-        "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
+        "How long downloaded files are considered up-to-date."};
 
     Setting<bool> requireSigs{this, true, "require-sigs",
         "Whether to check that any non-content-addressed path added to the "
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 5133dba5720..5633b4355d2 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -85,7 +85,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         try {
             DownloadRequest request(cacheUri + "/" + path);
             request.head = true;
-            request.tries = 5;
             getDownloader()->download(request);
             return true;
         } catch (DownloadError & e) {

From f8b30338ac231262bdf19844f044f0572c460048 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 May 2019 23:36:29 +0200
Subject: [PATCH 1443/2196] Refactor downloadCached() interface

(cherry picked from commit df3f5a78d5ab0a1f2dc9d288b271b38a9b8b33b5)
---
 src/libexpr/common-eval-args.cc |  8 +++---
 src/libexpr/parser.y            |  4 ++-
 src/libexpr/primops.cc          | 22 ++++++++---------
 src/libstore/download.cc        | 44 ++++++++++++++++-----------------
 src/libstore/download.hh        | 20 +++++++++++----
 src/nix-channel/nix-channel.cc  | 14 +++++------
 6 files changed, 63 insertions(+), 49 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 37c74a94b29..13950ab8d16 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 
 Path lookupFileArg(EvalState & state, string s)
 {
-    if (isUri(s))
-        return getDownloader()->downloadCached(state.store, s, true).path;
-    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+    if (isUri(s)) {
+        CachedDownloadRequest request(s);
+        request.unpack = true;
+        return getDownloader()->downloadCached(state.store, request).path;
+    } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
     } else
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 29860fb0576..967c88d9bc8 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -677,7 +677,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            res = { true, getDownloader()->downloadCached(store, elem.second, true).path };
+            CachedDownloadRequest request(elem.second);
+            request.unpack = true;
+            res = { true, getDownloader()->downloadCached(store, request).path };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 55a1bde117c..070e72f3a96 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     const string & who, bool unpack, const std::string & defaultName)
 {
-    string url;
-    Hash expectedHash;
-    string name = defaultName;
+    CachedDownloadRequest request("");
+    request.unpack = unpack;
+    request.name = defaultName;
 
     state.forceValue(*args[0]);
 
@@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         for (auto & attr : *args[0]->attrs) {
             string n(attr.name);
             if (n == "url")
-                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+                request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+                request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
             else if (n == "name")
-                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+                request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
         }
 
-        if (url.empty())
+        if (request.uri.empty())
             throw EvalError(format("'url' argument required, at %1%") % pos);
 
     } else
-        url = state.forceStringNoCtx(*args[0], pos);
+        request.uri = state.forceStringNoCtx(*args[0], pos);
 
-    state.checkURI(url);
+    state.checkURI(request.uri);
 
-    if (evalSettings.pureEval && !expectedHash)
+    if (evalSettings.pureEval && !request.expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path;
+    Path res = getDownloader()->downloadCached(state.store, request).path;
 
     if (state.allowedPaths)
         state.allowedPaths->insert(res);
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 342a8aa2146..892c1b21dc0 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -293,10 +293,10 @@ struct CurlDownloader : public Downloader
             long httpStatus = 0;
             curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
 
-            char * effectiveUrlCStr;
-            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr);
-            if (effectiveUrlCStr)
-                result.effectiveUrl = effectiveUrlCStr;
+            char * effectiveUriCStr;
+            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
+            if (effectiveUriCStr)
+                result.effectiveUri = effectiveUriCStr;
 
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
                 request.verb(), request.uri, code, httpStatus, result.bodySize);
@@ -737,18 +737,20 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
+CachedDownloadResult Downloader::downloadCached(
+    ref<Store> store, const CachedDownloadRequest & request)
 {
-    auto url = resolveUri(url_);
+    auto url = resolveUri(request.uri);
 
+    auto name = request.name;
     if (name == "") {
         auto p = url.rfind('/');
         if (p != string::npos) name = string(url, p + 1);
     }
 
     Path expectedStorePath;
-    if (expectedHash) {
-        expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
+    if (request.expectedHash) {
+        expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
         if (store->isValidPath(expectedStorePath)) {
             CachedDownloadResult result;
             result.storePath = expectedStorePath;
@@ -782,10 +784,9 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
             if (ss.size() >= 3 && ss[0] == url) {
                 time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) {
+                if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) {
                     skip = true;
-                    if (effectiveUrl)
-                        *effectiveUrl = url_;
+                    result.effectiveUri = request.uri;
                     result.etag = ss[1];
                 } else if (!ss[1].empty()) {
                     debug(format("verifying previous ETag '%1%'") % ss[1]);
@@ -799,18 +800,17 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
     if (!skip) {
 
         try {
-            DownloadRequest request(url);
-            request.expectedETag = expectedETag;
-            auto res = download(request);
-            if (effectiveUrl)
-                *effectiveUrl = res.effectiveUrl;
+            DownloadRequest request2(url);
+            request2.expectedETag = expectedETag;
+            auto res = download(request2);
+            result.effectiveUri = res.effectiveUri;
             result.etag = res.etag;
 
             if (!res.cached) {
                 ValidPathInfo info;
                 StringSink sink;
                 dumpString(*res.data, sink);
-                Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
+                Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
                 info.path = store->makeFixedOutputPath(false, hash, name);
                 info.narHash = hashString(htSHA256, *sink.s);
                 info.narSize = sink.s->size();
@@ -830,7 +830,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
         }
     }
 
-    if (unpack) {
+    if (request.unpack) {
         Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
         PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
         Path unpackedStorePath;
@@ -853,11 +853,11 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string &
     }
 
     if (expectedStorePath != "" && storePath != expectedStorePath) {
-        Hash gotHash = unpack
-            ? hashPath(expectedHash.type, store->toRealPath(storePath)).first
-            : hashFile(expectedHash.type, store->toRealPath(storePath));
+        Hash gotHash = request.unpack
+            ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
+            : hashFile(request.expectedHash.type, store->toRealPath(storePath));
         throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, expectedHash.to_string(), gotHash.to_string());
+            url, request.expectedHash.to_string(), gotHash.to_string());
     }
 
     result.storePath = storePath;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index dae082ab9ae..9e965b506d0 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -57,11 +57,23 @@ struct DownloadResult
 {
     bool cached = false;
     std::string etag;
-    std::string effectiveUrl;
+    std::string effectiveUri;
     std::shared_ptr<std::string> data;
     uint64_t bodySize = 0;
 };
 
+struct CachedDownloadRequest
+{
+    std::string uri;
+    bool unpack = false;
+    std::string name;
+    Hash expectedHash;
+    unsigned int ttl = settings.tarballTtl;
+
+    CachedDownloadRequest(const std::string & uri)
+        : uri(uri) { }
+};
+
 struct CachedDownloadResult
 {
     // Note: 'storePath' may be different from 'path' when using a
@@ -69,6 +81,7 @@ struct CachedDownloadResult
     Path storePath;
     Path path;
     std::optional<std::string> etag;
+    std::string effectiveUri;
 };
 
 class Store;
@@ -96,10 +109,7 @@ struct Downloader
        and is more recent than ‘tarball-ttl’ seconds. Otherwise,
        use the recorded ETag to verify if the server has a more
        recent version, and if so, download it to the Nix store. */
-    CachedDownloadResult downloadCached(
-        ref<Store> store, const string & uri, bool unpack, string name = "",
-        const Hash & expectedHash = Hash(), string * effectiveUri = nullptr,
-        int ttl = settings.tarballTtl);
+    CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 7b23088a2c5..bd1371dbae3 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -86,10 +86,12 @@ static void update(const StringSet & channelNames)
         // We want to download the url to a file to see if it's a tarball while also checking if we
         // got redirected in the process, so that we can grab the various parts of a nix channel
         // definition from a consistent location if the redirect changes mid-download.
-        std::string effectiveUrl;
+        CachedDownloadRequest request(url);
+        request.ttl = 0;
         auto dl = getDownloader();
-        auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path;
-        url = chomp(std::move(effectiveUrl));
+        auto result = dl->downloadCached(store, request);
+        auto filename = result.path;
+        url = chomp(result.effectiveUri);
 
         // If the URL contains a version number, append it to the name
         // attribute (so that "nix-env -q" on the channels profile
@@ -121,12 +123,10 @@ static void update(const StringSet & channelNames)
             }
 
             // Download the channel tarball.
-            auto fullURL = url + "/nixexprs.tar.xz";
             try {
-                filename = dl->downloadCached(store, fullURL, false).path;
+                filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path;
             } catch (DownloadError & e) {
-                fullURL = url + "/nixexprs.tar.bz2";
-                filename = dl->downloadCached(store, fullURL, false).path;
+                filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path;
             }
             chomp(filename);
         }

From 64ec087f582cec33733f4102ab42a3e8f291758a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 21 Jun 2019 12:49:18 +0200
Subject: [PATCH 1444/2196] Fix 32-bit overflow with --no-net

--no-net causes tarballTtl to be set to the largest 32-bit integer,
which causes comparison like 'time + tarballTtl < other_time' to
fail on 32-bit systems. So cast them to 64-bit first.

https://hydra.nixos.org/build/95076624
(cherry picked from commit 29ccb2e9697ee2184012dd13854e487928ae4441)
---
 src/libexpr/primops/fetchGit.cc       |  2 +-
 src/libexpr/primops/fetchMercurial.cc | 13 +++++--------
 src/libstore/download.cc              |  2 +-
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index cf7ccca418e..3dcf3e9ff86 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -116,7 +116,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
            git fetch to update the local ref to the remote ref. */
         struct stat st;
         doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            st.st_mtime + settings.tarballTtl <= now;
+            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
     }
     if (doFetch)
     {
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index d576138f543..a907d0e1cd8 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
     time_t now = time(0);
     struct stat st;
     if (stat(stampFile.c_str(), &st) != 0 ||
-        st.st_mtime + settings.tarballTtl <= now)
+        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
     {
         /* Except that if this is a commit hash that we already have,
            we don't have to pull again. */
@@ -96,17 +96,14 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 try {
                     runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
                 }
-                catch (ExecError & e){
+                catch (ExecError & e) {
                     string transJournal = cacheDir + "/.hg/store/journal";
                     /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal))
-                    {
+                    if (pathExists(transJournal)) {
                         runProgram("hg", true, { "recover", "-R", cacheDir });
                         runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    }
-                    else 
-                    {
-                        throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status)));
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
                     }
                 }
             } else {
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 892c1b21dc0..6ce9525c38d 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -784,7 +784,7 @@ CachedDownloadResult Downloader::downloadCached(
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
             if (ss.size() >= 3 && ss[0] == url) {
                 time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) {
+                if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
                     skip = true;
                     result.effectiveUri = request.uri;
                     result.etag = ss[1];

From 5600b070a766b83200a68d3632793917cf19a550 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jun 2019 09:12:03 +0200
Subject: [PATCH 1445/2196] Add "warning" verbosity level

This ensures that "nix" shows warnings. Previously these were hidden
because they were at "info" level.

(cherry picked from commit 615a9d031d22a6aee64f8511e15685e47b6f8796)
---
 src/libutil/logging.cc | 3 ++-
 src/libutil/logging.hh | 1 +
 src/nix/main.cc        | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 799c6e1ae44..b379306f6ec 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger();
 
 void Logger::warn(const std::string & msg)
 {
-    log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
+    log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
 }
 
 class SimpleLogger : public Logger
@@ -46,6 +46,7 @@ class SimpleLogger : public Logger
             char c;
             switch (lvl) {
             case lvlError: c = '3'; break;
+            case lvlWarn: c = '4'; break;
             case lvlInfo: c = '5'; break;
             case lvlTalkative: case lvlChatty: c = '6'; break;
             default: c = '7';
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 678703102e9..5f221944594 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -6,6 +6,7 @@ namespace nix {
 
 typedef enum {
     lvlError = 0,
+    lvlWarn,
     lvlInfo,
     lvlTalkative,
     lvlChatty,
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 25e321b86bc..19bb7a54315 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -93,7 +93,7 @@ void mainWrapped(int argc, char * * argv)
         if (legacy) return legacy(argc, argv);
     }
 
-    verbosity = lvlError;
+    verbosity = lvlWarn;
     settings.verboseBuild = false;
 
     NixArgs args;

From 09dde33c19129b949a064d35e248a3e318178b08 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Jun 2019 09:57:22 +0200
Subject: [PATCH 1446/2196] Automatically use --no-net if there are no network
 interfaces

(cherry picked from commit 04a59769963fe2a28d10ba15de743fe499333c80)
---
 src/libmain/common-args.cc | 11 --------
 src/nix/main.cc            | 55 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 0486932b20e..9e1d7cee60e 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -1,6 +1,5 @@
 #include "common-args.hh"
 #include "globals.hh"
-#include "download.hh"
 
 namespace nix {
 
@@ -45,16 +44,6 @@ MixCommonArgs::MixCommonArgs(const string & programName)
             settings.set("max-jobs", s);
         });
 
-    mkFlag()
-        .longName("no-net")
-        .description("disable substituters and consider all previously downloaded files up-to-date")
-        .handler([]() {
-            settings.useSubstitutes = false;
-            settings.tarballTtl = std::numeric_limits<unsigned int>::max();
-            downloadSettings.tries = 0;
-            downloadSettings.connectTimeout = 1;
-        });
-
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 19bb7a54315..73c4b8db112 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -8,19 +8,52 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
+#include "download.hh"
 #include "finally.hh"
 
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <ifaddrs.h>
+#include <netdb.h>
+
 extern std::string chrootHelperName;
 
 void chrootHelper(int argc, char * * argv);
 
 namespace nix {
 
+/* Check if we have a non-loopback/link-local network interface. */
+static bool haveInternet()
+{
+    struct ifaddrs * addrs;
+
+    if (getifaddrs(&addrs))
+        return true;
+
+    Finally free([&]() { freeifaddrs(addrs); });
+
+    for (auto i = addrs; i; i = i->ifa_next) {
+        if (!i->ifa_addr) continue;
+        if (i->ifa_addr->sa_family == AF_INET) {
+            if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) {
+                return true;
+            }
+        } else if (i->ifa_addr->sa_family == AF_INET6) {
+            if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) &&
+                !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr))
+                return true;
+        }
+    }
+
+    return false;
+}
+
 std::string programPath;
 
 struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     bool printBuildLogs = false;
+    bool useNet = true;
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
@@ -53,6 +86,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName("version")
             .description("show version information")
             .handler([&]() { printVersion(programName); });
+
+        mkFlag()
+            .longName("no-net")
+            .description("disable substituters and consider all previously downloaded files up-to-date")
+            .handler([&]() { useNet = false; });
     }
 
     void printFlags(std::ostream & out) override
@@ -108,6 +146,23 @@ void mainWrapped(int argc, char * * argv)
 
     startProgressBar(args.printBuildLogs);
 
+    if (args.useNet && !haveInternet()) {
+        warn("you don't have Internet access; disabling some network-dependent features");
+        args.useNet = false;
+    }
+
+    if (!args.useNet) {
+        // FIXME: should check for command line overrides only.
+        if (!settings.useSubstitutes.overriden)
+            settings.useSubstitutes = false;
+        if (!settings.tarballTtl.overriden)
+            settings.tarballTtl = std::numeric_limits<unsigned int>::max();
+        if (!downloadSettings.tries.overriden)
+            downloadSettings.tries = 0;
+        if (!downloadSettings.connectTimeout.overriden)
+            downloadSettings.connectTimeout = 1;
+    }
+
     args.command->prepare();
     args.command->run();
 }

From 88571219d97f2bdfdbafcff25ef6ee424b0b008f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Jun 2019 13:27:16 +0200
Subject: [PATCH 1447/2196] nix-channel: Don't fetch binary-cache-url

This has been ignored since the Perl->C++ rewrite.
---
 corepkgs/unpack-channel.nix            |  8 ++------
 doc/manual/command-ref/nix-channel.xml | 23 +++++------------------
 src/libstore/store-api.hh              |  3 +--
 src/nix-channel/nix-channel.cc         |  9 ---------
 4 files changed, 8 insertions(+), 35 deletions(-)

diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix
index a654db40e62..d39a2063781 100644
--- a/corepkgs/unpack-channel.nix
+++ b/corepkgs/unpack-channel.nix
@@ -18,21 +18,17 @@ let
       if [ * != $channelName ]; then
         mv * $out/$channelName
       fi
-      if [ -n "$binaryCacheURL" ]; then
-        mkdir $out/binary-caches
-        echo -n "$binaryCacheURL" > $out/binary-caches/$channelName
-      fi
     '';
 
 in
 
-{ name, channelName, src, binaryCacheURL ? "" }:
+{ name, channelName, src }:
 
 derivation {
   system = builtins.currentSystem;
   builder = shell;
   args = [ "-e" builder ];
-  inherit name channelName src binaryCacheURL;
+  inherit name channelName src;
 
   PATH = "${nixBinDir}:${coreutils}";
 
diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
index ff4021a765e..5a2866e6bc4 100644
--- a/doc/manual/command-ref/nix-channel.xml
+++ b/doc/manual/command-ref/nix-channel.xml
@@ -31,12 +31,11 @@
 
 <refsection><title>Description</title>
 
-<para>A Nix channel is a mechanism that allows you to automatically stay
-up-to-date with a set of pre-built Nix expressions.  A Nix channel is
-just a URL that points to a place containing both a set of Nix
-expressions and a pointer to a binary cache.  <phrase
-condition="manual">See also <xref linkend="sec-channels"
-/>.</phrase></para>
+<para>A Nix channel is a mechanism that allows you to automatically
+stay up-to-date with a set of pre-built Nix expressions.  A Nix
+channel is just a URL that points to a place containing a set of Nix
+expressions.  <phrase condition="manual">See also <xref
+linkend="sec-channels" />.</phrase></para>
 
 <para>This command has the following operations:
 
@@ -172,18 +171,6 @@ following files:</para>
 
   </varlistentry>
 
-  <varlistentry><term><filename>binary-cache-url</filename></term>
-
-    <listitem><para>A file containing the URL to a binary cache (such
-    as <uri>https://cache.nixos.org</uri>). Nix will automatically
-    check this cache for pre-built binaries, if the user has
-    sufficient rights to add binary caches. For instance, in a
-    multi-user Nix setup, the binary caches provided by the channels
-    of the root user are used automatically, but caches corresponding
-    to the channels of non-root users are ignored.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 </refsection>
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7a1b31d0ff5..59967737670 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -766,8 +766,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
     const std::string & stateDir = settings.nixStateDir);
 
 /* Return the default substituter stores, defined by the
-   ‘substituters’ option and various legacy options like
-   ‘binary-caches’. */
+   ‘substituters’ option and various legacy options. */
 std::list<ref<Store>> getDefaultSubstituters();
 
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index bd1371dbae3..06eb3d23ba8 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -113,15 +113,6 @@ static void update(const StringSet & channelNames)
         }
 
         if (!unpacked) {
-            // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
-            // Check if the channel advertises a binary cache.
-            DownloadRequest request(url + "/binary-cache-url");
-            try {
-                auto dlRes = dl->download(request);
-                extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";";
-            } catch (DownloadError & e) {
-            }
-
             // Download the channel tarball.
             try {
                 filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path;

From 324a5dc92f8e50e6b637c5e67dea48c80be10837 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Jun 2019 21:57:18 +0200
Subject: [PATCH 1448/2196] ProgressBar: Fix updating

'updateCV.notify_one()' does nothing if the update thread is not
waiting for updateCV (in particular this happens when it is sleeping
on quitCV). So also set a variable to ensure that the update isn't
lost.
---
 src/nix/progress-bar.cc | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index e7104540816..b1c1d87de1a 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -62,6 +62,7 @@ class ProgressBar : public Logger
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
 
         bool active = true;
+        bool haveUpdate = true;
     };
 
     Sync<State> state_;
@@ -83,7 +84,8 @@ class ProgressBar : public Logger
         updateThread = std::thread([&]() {
             auto state(state_.lock());
             while (state->active) {
-                state.wait(updateCV);
+                if (!state->haveUpdate)
+                    state.wait(updateCV);
                 draw(*state);
                 state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
@@ -178,7 +180,7 @@ class ProgressBar : public Logger
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
-        update();
+        update(*state);
     }
 
     /* Check whether an activity has an ancestore with the specified
@@ -213,7 +215,7 @@ class ProgressBar : public Logger
             state->its.erase(i);
         }
 
-        update();
+        update(*state);
     }
 
     void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
@@ -223,7 +225,7 @@ class ProgressBar : public Logger
         if (type == resFileLinked) {
             state->filesLinked++;
             state->bytesLinked += getI(fields, 0);
-            update();
+            update(*state);
         }
 
         else if (type == resBuildLogLine) {
@@ -239,26 +241,26 @@ class ProgressBar : public Logger
                     info.lastLine = lastLine;
                     state->activities.emplace_back(info);
                     i->second = std::prev(state->activities.end());
-                    update();
+                    update(*state);
                 }
             }
         }
 
         else if (type == resUntrustedPath) {
             state->untrustedPaths++;
-            update();
+            update(*state);
         }
 
         else if (type == resCorruptedPath) {
             state->corruptedPaths++;
-            update();
+            update(*state);
         }
 
         else if (type == resSetPhase) {
             auto i = state->its.find(act);
             assert(i != state->its.end());
             i->second->phase = getS(fields, 0);
-            update();
+            update(*state);
         }
 
         else if (type == resProgress) {
@@ -269,7 +271,7 @@ class ProgressBar : public Logger
             actInfo.expected = getI(fields, 1);
             actInfo.running = getI(fields, 2);
             actInfo.failed = getI(fields, 3);
-            update();
+            update(*state);
         }
 
         else if (type == resSetExpected) {
@@ -281,17 +283,19 @@ class ProgressBar : public Logger
             state->activitiesByType[type].expected -= j;
             j = getI(fields, 1);
             state->activitiesByType[type].expected += j;
-            update();
+            update(*state);
         }
     }
 
-    void update()
+    void update(State & state)
     {
+        state.haveUpdate = true;
         updateCV.notify_one();
     }
 
     void draw(State & state)
     {
+        state.haveUpdate = false;
         if (!state.active) return;
 
         std::string line;

From ec0087df0a2da5c68363b3f4509d4545deed97a0 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 27 Jun 2019 14:22:53 -0400
Subject: [PATCH 1449/2196] =?UTF-8?q?Don=E2=80=99t=20use=20entire=20/etc/n?=
 =?UTF-8?q?sswitch.conf=20file?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The default nsswitch.conf(5) file in most distros can handle many
different things including host name, user names, groups, etc. In Nix,
we want to limit the amount of impurities that come from these things.
As a result, we should only allow nss to be used for gethostbyname(3)
and getservent(3).

/cc @Ericson2314
---
 src/libstore/build.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5b38bcf3c5e..813d7e2c2d0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2728,7 +2728,13 @@ void DerivationGoal::runChild()
                on. */
             if (fixedOutput) {
                 ss.push_back("/etc/resolv.conf");
-                ss.push_back("/etc/nsswitch.conf");
+
+                // Only use nss functions to resolve hosts and
+                // services. Don’t use it for anything else that may
+                // be configured for this system. This limits the
+                // potential impurities introduced in fixed outputs.
+                writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
+
                 ss.push_back("/etc/services");
                 ss.push_back("/etc/hosts");
                 if (pathExists("/var/run/nscd/socket"))

From 6847c9278840edc97f1ef85b5fafac4338fb1b37 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Jun 2019 15:38:23 +0200
Subject: [PATCH 1450/2196] Fix macOS build failure

Issue #2976.
---
 src/nix/main.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 73c4b8db112..a80fd0ea62f 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -39,8 +39,8 @@ static bool haveInternet()
                 return true;
             }
         } else if (i->ifa_addr->sa_family == AF_INET6) {
-            if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) &&
-                !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr))
+            if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) &&
+                !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr))
                 return true;
         }
     }

From ec58ba38c55a3ab61fdb1da7d746251e4887ea2c Mon Sep 17 00:00:00 2001
From: Aniket Deshpande <anicake@gmail.com>
Date: Fri, 28 Jun 2019 19:43:22 +0530
Subject: [PATCH 1451/2196] Fix `http2 = false` having no effect. Fixes #2971.

Setting `http2 = false` in nix config (e.g. /etc/nix/nix.conf)
had no effect, and `nix-env -vvvvv -i hello` still downloaded .nar
packages using HTTP/2.

In `src/libstore/download.cc`, the `CURL_HTTP_VERSION_2TLS` option was
being explicitly set when `downloadSettings.enableHttp2` was `true`,
but, `CURL_HTTP_VERSION_1_1` option was not being explicitly set when
`downloadSettings.enableHttp2` was `false`.

This may be because `https://curl.haxx.se/libcurl/c/libcurl-env.html` states:
"You have to set this option if you want to use libcurl's HTTP/2 support."
but, also, in the changelog, states:
"DEFAULT
Since curl 7.62.0: CURL_HTTP_VERSION_2TLS
Before that: CURL_HTTP_VERSION_1_1"

So, the default setting for `libcurl` is HTTP/2 for version >= 7.62.0.

In this commit, option `CURLOPT_HTTP_VERSION` is explicitly set to
`CURL_HTTP_VERSION_1_1` when `downloadSettings.enableHttp2` nix config
setting is `false`.

This can be tested by running `nix-env -vvvvv -i hello | grep HTTP`
---
 src/libstore/download.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 6ce9525c38d..0c5a73ea3c5 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -244,6 +244,8 @@ struct CurlDownloader : public Downloader
             #if LIBCURL_VERSION_NUM >= 0x072f00
             if (downloadSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
+            else
+                curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             #endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);

From 97baf32fbca13101f58d30bb3a601302c3d560f3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Tue, 2 Jul 2019 00:12:12 +0200
Subject: [PATCH 1452/2196] build: add exit code for hash and check mismatches

Makes it easier to identify the failure reason in other tooling, eg.
differentiate between a non-deterministic --check vs a failed build.

	$ nix-build '<nix/fetchurl.nix>' --argstr url http://example.org --argstr sha256 0000000000000000000000000000000000000000000000000000
	hash mismatch in fixed-output derivation '/nix/store/nzi9ck45rwlxzcwr25is7qlf3hs5xl83-example.org':
	  wanted: sha256:0000000000000000000000000000000000000000000000000000
	  got:    sha256:08y4734bm2zahw75b16bcmcg587vvyvh0n11gwiyir70divwp1rm
	$ echo $?
	102

	$ nix-build -E 'with import <nixpkgs> {}; runCommand "foo" {} "date +%s > $out"' --check
	warning: rewriting hashes in '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo'; cross fingers
	error: derivation '/nix/store/mggc8dz13ackb49qca6m23zq4fpq132q-foo.drv' may not be deterministic: output '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo' differs
	$ echo $?
	104
---
 src/libstore/build.cc    | 16 +++++++++++++---
 src/libstore/download.cc |  3 ++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5b38bcf3c5e..c076ea8b000 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -266,6 +266,12 @@ class Worker
     /* Set if at least one derivation had a timeout. */
     bool timedOut;
 
+    /* Set if at least one derivation fails with a hash mismatch. */
+    bool hashMismatch;
+
+    /* Set if at least one derivation is not deterministic in check mode. */
+    bool checkMismatch;
+
     LocalStore & store;
 
     std::unique_ptr<HookInstance> hook;
@@ -3213,6 +3219,7 @@ void DerivationGoal::registerOutputs()
 
                 /* Throw an error after registering the path as
                    valid. */
+                worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         dest, h.to_string(), h2.to_string()));
@@ -3255,6 +3262,7 @@ void DerivationGoal::registerOutputs()
             if (!worker.store.isValidPath(path)) continue;
             auto info = *worker.store.queryPathInfo(path);
             if (hash.first != info.narHash) {
+                worker.checkMismatch = true;
                 if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
@@ -3266,10 +3274,10 @@ void DerivationGoal::registerOutputs()
                         buildUser ? buildUser->getGID() : getgid(),
                         path, dst, drvPath, tmpDir);
 
-                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
+                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
                         % drvPath % path % dst);
                 } else
-                    throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs")
+                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs")
                         % drvPath % path);
             }
 
@@ -4101,6 +4109,8 @@ Worker::Worker(LocalStore & store)
     lastWokenUp = steady_time_point::min();
     permanentFailure = false;
     timedOut = false;
+    hashMismatch = false;
+    checkMismatch = false;
 }
 
 
@@ -4461,7 +4471,7 @@ void Worker::waitForInput()
 
 unsigned int Worker::exitStatus()
 {
-    return timedOut ? 101 : (permanentFailure ? 100 : 1);
+    return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1)));
 }
 
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 0c5a73ea3c5..7a2af237ee8 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -855,10 +855,11 @@ CachedDownloadResult Downloader::downloadCached(
     }
 
     if (expectedStorePath != "" && storePath != expectedStorePath) {
+        unsigned int statusCode = 102;
         Hash gotHash = request.unpack
             ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
             : hashFile(request.expectedHash.type, store->toRealPath(storePath));
-        throw nix::Error("hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+        throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
             url, request.expectedHash.to_string(), gotHash.to_string());
     }
 

From cbf84bcce7d3fddb353d3aa0f012f5cbdbb77629 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 11 May 2019 23:14:19 +0200
Subject: [PATCH 1453/2196] build: use binary mask for build status flags

If multiple builds with fail with different errors it will be reflected
in the status code.

eg.

	103 => timeout + hash mismatch
	105 => timeout + check mismatch
	106 => hash mismatch + check mismatch
	107 => timeout + hash mismatch + check mismatch
---
 src/libstore/build.cc | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index c076ea8b000..abfae3c7cf6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4471,7 +4471,15 @@ void Worker::waitForInput()
 
 unsigned int Worker::exitStatus()
 {
-    return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1)));
+    unsigned int mask = 0;
+    if (timedOut)
+        mask |= 1;
+    if (hashMismatch)
+        mask |= 2;
+    if (checkMismatch)
+        mask |= 4;
+
+    return mask ? 100 + mask : 1;
 }
 
 

From 99ee3755dd12e18f7085b8319a27798b0e5b7de9 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 11 May 2019 23:32:53 +0200
Subject: [PATCH 1454/2196] build: add tests for --check status codes

---
 tests/check.nix |  5 +++++
 tests/check.sh  | 23 +++++++++++++++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/tests/check.nix b/tests/check.nix
index 08aac2fb0a7..56c82e565a8 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -10,6 +10,11 @@ with import ./config.nix;
       '';
   };
 
+  hashmismatch = import <nix/fetchurl.nix> {
+    url = "file://" + toString ./dummy;
+    sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
+  };
+
   fetchurl = import <nix/fetchurl.nix> {
     url = "file://" + toString ./lang/eval-okay-xml.exp.xml;
     sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v";
diff --git a/tests/check.sh b/tests/check.sh
index b05e40ffbee..aa1e5cd26f8 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -6,14 +6,16 @@ nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
 nix-build check.nix -A nondeterministic --no-out-link
-(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log)
+nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
+[ "$status" = "104" ]
 grep 'may not be deterministic' $TEST_ROOT/log
 
 clearStore
 
 nix-build dependencies.nix --no-out-link --repeat 3
 
-(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log)
+nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$?
+[ "$status" = "1" ]
 grep 'differs from previous round' $TEST_ROOT/log
 
 path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
@@ -23,10 +25,23 @@ echo foo > $path
 chmod -w $path
 
 nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
-
 # Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
 [[ $(cat $path) = foo ]]
 
 nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
-
 [[ $(cat $path) != foo ]]
+
+nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+[ "$status" = "102" ]
+
+echo -n > ./dummy
+nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+echo 'Hello World' > ./dummy
+
+nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+[ "$status" = "102" ]
+
+# Multiple failures with --keep-going
+nix-build check.nix -A nondeterministic --no-out-link
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+[ "$status" = "106" ]

From 1ac399dd115d5c86629389e0cdfefa0d654fc90a Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Mon, 13 May 2019 21:03:36 +0200
Subject: [PATCH 1455/2196] nix-store: document exit codes

---
 doc/manual/command-ref/nix-store.xml | 35 ++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index d73cb92ee22..bf12d06f164 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -215,6 +215,41 @@ printed.)</para>
 
 </variablelist>
 
+<para>Special exit codes:</para>
+
+<variablelist>
+
+  <varlistentry><term><literal>100</literal></term>
+    <listitem><para>Generic build failure, the builder process
+    returned with a non-zero exit code.</para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>101</literal></term>
+    <listitem><para>Build timeout, the build was aborted because it
+    did not complete within the specified <link
+    linkend='conf-timeout'><literal>timeout</literal></link>.
+    </para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>102</literal></term>
+    <listitem><para>Hash mismatch, the build output was rejected
+    because it does not match the specified <link
+    linkend="fixed-output-drvs"><varname>outputHash</varname></link>.
+    </para></listitem>
+  </varlistentry>
+
+  <varlistentry><term><literal>104</literal></term>
+    <listitem><para>Not deterministic, the build succeeded in check
+    mode but the resulting output is not binary reproducable.</para>
+    </listitem>
+  </varlistentry>
+
+</variablelist>
+
+<para>With the <option>--keep-going</option> flag it's possible for
+multiple build failures to occur, in this case the 1xx status codes
+are or combined.</para>
+
 </refsection>
 
 

From a52c331edba6e8c67469f53dda0be2903d18fa8c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 15 Jun 2019 15:28:32 +0200
Subject: [PATCH 1456/2196] build: replace 100 offset for build exit codes

---
 doc/manual/command-ref/nix-store.xml | 11 +++++++++--
 src/libstore/build.cc                | 24 +++++++++++++++++++-----
 tests/check.sh                       |  4 ++--
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index bf12d06f164..2ecc63db7c2 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -247,8 +247,15 @@ printed.)</para>
 </variablelist>
 
 <para>With the <option>--keep-going</option> flag it's possible for
-multiple build failures to occur, in this case the 1xx status codes
-are or combined.</para>
+multiple failures to occur, in this case the 1xx status codes are or combined
+using binary or. <screen>
+1100100
+   ^^^^
+   |||`- timeout
+   ||`-- output hash mismatch
+   |`--- build failure
+   `---- not deterministic
+</screen></para>
 
 </refsection>
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index abfae3c7cf6..350ac409285 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4471,15 +4471,29 @@ void Worker::waitForInput()
 
 unsigned int Worker::exitStatus()
 {
+    /*
+     * 1100100
+     *    ^^^^
+     *    |||`- timeout
+     *    ||`-- output hash mismatch
+     *    |`--- build failure
+     *    `---- not deterministic
+     */
     unsigned int mask = 0;
+    bool buildFailure = permanentFailure || timedOut || hashMismatch;
+    if (buildFailure)
+        mask |= 0x04;  // 100
     if (timedOut)
-        mask |= 1;
+        mask |= 0x01;  // 101
     if (hashMismatch)
-        mask |= 2;
-    if (checkMismatch)
-        mask |= 4;
+        mask |= 0x02;  // 102
+    if (checkMismatch) {
+        mask |= 0x08;  // 104
+    }
 
-    return mask ? 100 + mask : 1;
+    if (mask)
+        mask |= 0x60;
+    return mask ? mask : 1;
 }
 
 
diff --git a/tests/check.sh b/tests/check.sh
index aa1e5cd26f8..bc23a6634ca 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -7,8 +7,8 @@ nix-build dependencies.nix --no-out-link --check
 
 nix-build check.nix -A nondeterministic --no-out-link
 nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
-[ "$status" = "104" ]
 grep 'may not be deterministic' $TEST_ROOT/log
+[ "$status" = "104" ]
 
 clearStore
 
@@ -44,4 +44,4 @@ nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' ||
 # Multiple failures with --keep-going
 nix-build check.nix -A nondeterministic --no-out-link
 nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
-[ "$status" = "106" ]
+[ "$status" = "110" ]

From 17d3ec3405eb114f89dc42d065afa887161f6149 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 27 Jun 2019 14:21:22 -0400
Subject: [PATCH 1457/2196] checkStoreName: give more precise/verbose error
 information

$ sudo ./inst/bin/nix-instantiate -E '"${./.git}"'
error: The path name '.git' is invalid: it is illegal to start the
name with a period. Path names are alphanumeric and can include the
symbols +-._?= and must not begin with a period. Note: If '.git' is a
source file and you cannot rename it on disk,
builtins.path { name = ... } can be used to give it an alternative
name.
---
 src/libstore/store-api.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 28ad7c019a9..92f01fd2e0c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -86,18 +86,25 @@ string storePathToHash(const Path & path)
 void checkStoreName(const string & name)
 {
     string validChars = "+-._?=";
+
+    auto baseError = format("The path name '%2%' is invalid: %3%. "
+        "Path names are alphanumeric and can include the symbols %1% "
+        "and must not begin with a period. "
+        "Note: If '%2%' is a source file and you cannot rename it on "
+        "disk, builtins.path { name = ... } can be used to give it an "
+        "alternative name.") % validChars % name;
+
     /* Disallow names starting with a dot for possible security
        reasons (e.g., "." and ".."). */
     if (string(name, 0, 1) == ".")
-        throw Error(format("illegal name: '%1%'") % name);
+        throw Error(baseError % "it is illegal to start the name with a period");
     for (auto & i : name)
         if (!((i >= 'A' && i <= 'Z') ||
               (i >= 'a' && i <= 'z') ||
               (i >= '0' && i <= '9') ||
               validChars.find(i) != string::npos))
         {
-            throw Error(format("invalid character '%1%' in name '%2%'")
-                % i % name);
+            throw Error(baseError % (format("the '%1%' character is invalid") % i));
         }
 }
 

From c8205a3413217ccf8a6a1f7e064b06a5b86c3253 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 2 Jul 2019 09:04:04 -0400
Subject: [PATCH 1458/2196] builtins.fetchGit: document absolute ref support

---
 doc/manual/expressions/builtins.xml        | 15 +++++++++++++++
 doc/manual/release-notes/release-notes.xml |  1 +
 doc/manual/release-notes/rl-2.3.xml        | 22 ++++++++++++++++++++++
 3 files changed, 38 insertions(+)
 create mode 100644 doc/manual/release-notes/rl-2.3.xml

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 412622714d4..dcd7aaf243c 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -419,6 +419,13 @@ stdenv.mkDerivation { … }
               This is often a branch or tag name. Defaults to
               <literal>HEAD</literal>.
             </para>
+
+            <para>
+              By default, the <varname>ref</varname> value is prefixed
+              with <literal>refs/heads/</literal>. As of Nix 2.3.0
+              Nix will not prefix <literal>refs/heads/</literal> if
+              <varname>ref</varname> starts with <literal>refs/</literal>.
+            </para>
           </listitem>
         </varlistentry>
       </variablelist>
@@ -432,6 +439,14 @@ stdenv.mkDerivation { … }
 }</programlisting>
       </example>
 
+      <example>
+        <title>Fetching an arbitrary ref</title>
+        <programlisting>builtins.fetchGit {
+  url = "https://gitub.com/NixOS/nix.git";
+  ref = "refs/heads/0.5-release";
+}</programlisting>
+      </example>
+
       <example>
         <title>Fetching a repository's specific commit on an arbitrary branch</title>
         <para>
diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml
index e8ff586fa43..2655d68e354 100644
--- a/doc/manual/release-notes/release-notes.xml
+++ b/doc/manual/release-notes/release-notes.xml
@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-2.3.xml" />
 <xi:include href="rl-2.2.xml" />
 <xi:include href="rl-2.1.xml" />
 <xi:include href="rl-2.0.xml" />
diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
new file mode 100644
index 00000000000..428213b360b
--- /dev/null
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -0,0 +1,22 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-2.3">
+
+<title>Release 2.3 (????-??-??)</title>
+
+<para>This release contains the following changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para><function>builtins.fetchGit</function>'s <varname>ref</varname>
+    argument now allows specifying an absolute remote ref.
+    Nix will automatically prefix <varname>ref</varname> with
+    <literal>refs/heads</literal> only if <varname>ref</varname> doesn't
+    already begin with <literal>refs/</literal>.
+    </para>
+  </listitem>
+</itemizedlist>
+</section>

From a3c77c15369703975f7c3e65ebb829580754bc1e Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 14 Mar 2019 22:39:49 +0100
Subject: [PATCH 1459/2196] nix-store: document --add-fixed

---
 doc/manual/command-ref/nix-store.xml | 54 ++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index d73cb92ee22..625598310ff 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -883,6 +883,60 @@ $ nix-store --add ./foo.c
 
 </refsection>
 
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--add-fixed</option></title>
+
+<refsection><title>Synopsis</title>
+
+<cmdsynopsis>
+  <command>nix-store</command>
+  <arg><option>--recursive</option></arg>
+  <arg choice='plain'><option>--add-fixed</option></arg>
+  <arg choice='plain'><replaceable>algorithm</replaceable></arg>
+  <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+</cmdsynopsis>
+
+</refsection>
+
+<refsection><title>Description</title>
+
+<para>The operation <option>--add-fixed</option> adds the specified paths to
+the Nix store.  Unlike <option>--add</option> paths are registered using the
+specified hashing algorithm, resulting in the same output path as a fixed output
+derivation.  This can be used for sources that are not available from a public
+url or broke since the download expression was written.
+</para>
+
+<para>This operation has the following options:
+
+<variablelist>
+
+  <varlistentry><term><option>--recursive</option></term>
+
+    <listitem><para>
+      Use recursive instead of flat hashing mode, used when adding directories
+      to the store.
+    </para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</refsection>
+
+<refsection><title>Example</title>
+
+<screen>
+$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
+/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen>
+
+</refsection>
+
+</refsection>
+
 
 
 <!--######################################################################-->

From 68bdd83dc88ec55c6c51fa92e84e7d7d408c554a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 2 Jul 2019 11:18:36 -0400
Subject: [PATCH 1460/2196] timeout: test for error code

---
 tests/timeout.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/timeout.sh b/tests/timeout.sh
index 39ecf0a1a30..eea9b5731da 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -2,10 +2,14 @@
 
 source common.sh
 
-failed=0
-messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`"
-if [ $failed -ne 0 ]; then
-    echo "error: 'nix-store' succeeded; should have timed out"
+
+set +e
+messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1)
+status=$?
+set -e
+
+if [ $status -ne 101 ]; then
+    echo "error: 'nix-store' exited with '$status'; should have exited 101"
     exit 1
 fi
 

From 96cd3d607374cb4bab27095a62a6f3a832518c5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Tue, 2 Jul 2019 03:11:52 +0200
Subject: [PATCH 1461/2196] autoconf: Change quotes in description.

The unbalanced single-quotes cause many editor syntax highlighters
to interpret the rest of the file as a string literal, making it easier
to make syntax mistakes in absence of proper highlighting.
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index f5b1614f19f..398bac0476c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -42,7 +42,7 @@ esac
 
 AC_MSG_RESULT($system)
 AC_SUBST(system)
-AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
+AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 
 
 # State should be stored in /nix/var, unless the user overrides it explicitly.

From 00a450026f71306bef6acc150af4aa5ffe75801c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Tue, 2 Jul 2019 03:12:58 +0200
Subject: [PATCH 1462/2196] autoconf: Detect boost, require version, set
 CXXFLAGS.

This turns previous compiler errors complaining about missing files
into proper ./configure time errors telling the user which version
of boost is required.
---
 configure.ac | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/configure.ac b/configure.ac
index 398bac0476c..426d3b67407 100644
--- a/configure.ac
+++ b/configure.ac
@@ -145,6 +145,13 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
 AC_SUBST(storedir)
 
 
+# Look for boost, a required dependency.
+# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags,
+# and CPPFLAGS are not passed to the C++ compiler automatically.
+# Thus we append the returned CPPFLAGS to the CXXFLAGS here.
+AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])])
+
+
 # Look for OpenSSL, a required dependency.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 

From 1f97b16b1d9d93e083a4f1436ba002e073cbe379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Mon, 1 Jul 2019 18:31:54 +0200
Subject: [PATCH 1463/2196] autoconf: Work around editline not being found on
 Ubuntu 16.04.

And probably other Linux distributions with long-term support releases.

Also update manual stating what version is needed;
I checked that 1.14 is the oldest version with which current nix compiles,
and added autoconf feature checks for some functions added in that release
that nix uses.
---
 configure.ac                                     | 11 ++++++++++-
 doc/manual/installation/prerequisites-source.xml |  4 ++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 426d3b67407..28be50575ef 100644
--- a/configure.ac
+++ b/configure.ac
@@ -171,7 +171,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
 # Look for editline, a required dependency.
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
+# The the libeditline.pc file was added only in libeditline >= 1.15.2,
+# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607,
+# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for
+# editline.h when the pkg-config approach fails.
+PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [
+  AC_CHECK_HEADERS([editline.h], [true],
+    [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])])
+  AC_SEARCH_LIBS([readline read_history], [editline], [],
+    [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
+])
 
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium],
diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index e87d0de21ef..cbaa4d52527 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -62,6 +62,10 @@
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>
 
+  <listitem><para>The <literal>editline</literal> library of version
+  1.14.0 or higher. It can be obtained from the its repository
+  <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem>
+
   <listitem><para>The <command>xmllint</command> and
   <command>xsltproc</command> programs to build this manual and the
   man-pages.  These are part of the <literal>libxml2</literal> and

From 57daa860e8ed8432937aeecdcf6b9e952b0481b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Mon, 1 Jul 2019 19:04:03 +0200
Subject: [PATCH 1464/2196] autoconf: Fix C++17 detection not working on Ubuntu
 16.04.

And probably many other distributions.

Until now, ./configure would fail silently printing a warning

    ./configure: line 4621: AX_CXX_COMPILE_STDCXX_17: command not found

and then continuing, later failing with a C++ #error saying that some C++11
feature isn't supported (it didn't even get to the C++17 features).

This is because older distributions don't come with the
`AX_CXX_COMPILE_STDCXX_17` m4 macro.

This commit vendors that macro accordingly.

Now ./configure complains correctly:

    configure: error: *** A compiler with support for C++17 language features is required.

On Ubuntu 16.04, ./configure completes if a newer compiler is used, e.g. with
gcc-7 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
using:

    ./bootstrap.sh
    ./configure CXX=g++-7 --disable-doc-gen --with-boost=$(nix-build --no-link '<nixpkgs>' -A boost.dev)
---
 configure.ac                                  |   3 +-
 .../installation/prerequisites-source.xml     |   2 +-
 m4/ax_cxx_compile_stdcxx.m4                   | 951 ++++++++++++++++++
 m4/ax_cxx_compile_stdcxx_17.m4                |  35 +
 4 files changed, 989 insertions(+), 2 deletions(-)
 create mode 100644 m4/ax_cxx_compile_stdcxx.m4
 create mode 100644 m4/ax_cxx_compile_stdcxx_17.m4

diff --git a/configure.ac b/configure.ac
index 28be50575ef..c12f9d5ff95 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,5 @@
 AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"]))
+AC_CONFIG_MACRO_DIRS([m4])
 AC_CONFIG_SRCDIR(README.md)
 AC_CONFIG_AUX_DIR(config)
 
@@ -62,7 +63,7 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_17
+AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
 
 AC_CHECK_TOOL([AR], [ar])
 
diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index cbaa4d52527..e7bdcf966cf 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -13,7 +13,7 @@
   <listitem><para>Bash Shell. The <literal>./configure</literal> script
   relies on bashisms, so Bash is required.</para></listitem>
 
-  <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem>
+  <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem>
 
   <listitem><para><command>pkg-config</command> to locate
   dependencies.  If your distribution does not provide it, you can get
diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4
new file mode 100644
index 00000000000..43087b2e688
--- /dev/null
+++ b/m4/ax_cxx_compile_stdcxx.m4
@@ -0,0 +1,951 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the specified
+#   version of the C++ standard.  If necessary, add switches to CXX and
+#   CXXCPP to enable support.  VERSION may be '11' (for the C++11 standard)
+#   or '14' (for the C++14 standard).
+#
+#   The second argument, if specified, indicates whether you insist on an
+#   extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
+#   -std=c++11).  If neither is specified, you get whatever works, with
+#   preference for an extended mode.
+#
+#   The third argument, if specified 'mandatory' or if left unspecified,
+#   indicates that baseline support for the specified C++ standard is
+#   required and that the macro should error out if no mode with that
+#   support is found.  If specified 'optional', then configuration proceeds
+#   regardless, after defining HAVE_CXX${VERSION} if and only if a
+#   supporting mode is found.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
+#   Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
+#   Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
+#   Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
+#   Copyright (c) 2015 Paul Norman <penorman@mac.com>
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#   Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
+#   Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 11
+
+dnl  This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
+dnl  (serial version number 13).
+
+AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
+  m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
+        [$1], [14], [ax_cxx_compile_alternatives="14 1y"],
+        [$1], [17], [ax_cxx_compile_alternatives="17 1z"],
+        [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$2], [], [],
+        [$2], [ext], [],
+        [$2], [noext], [],
+        [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [optional], [ax_cxx_compile_cxx$1_required=false],
+        [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
+  AC_LANG_PUSH([C++])dnl
+  ac_success=no
+
+  m4_if([$2], [noext], [], [dnl
+  if test x$ac_success = xno; then
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      switch="-std=gnu++${alternative}"
+      cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+      AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                     $cachevar,
+        [ac_save_CXX="$CXX"
+         CXX="$CXX $switch"
+         AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+          [eval $cachevar=yes],
+          [eval $cachevar=no])
+         CXX="$ac_save_CXX"])
+      if eval test x\$$cachevar = xyes; then
+        CXX="$CXX $switch"
+        if test -n "$CXXCPP" ; then
+          CXXCPP="$CXXCPP $switch"
+        fi
+        ac_success=yes
+        break
+      fi
+    done
+  fi])
+
+  m4_if([$2], [ext], [], [dnl
+  if test x$ac_success = xno; then
+    dnl HP's aCC needs +std=c++11 according to:
+    dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
+    dnl Cray's crayCC needs "-h std=c++11"
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do
+        cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+        AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                       $cachevar,
+          [ac_save_CXX="$CXX"
+           CXX="$CXX $switch"
+           AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+            [eval $cachevar=yes],
+            [eval $cachevar=no])
+           CXX="$ac_save_CXX"])
+        if eval test x\$$cachevar = xyes; then
+          CXX="$CXX $switch"
+          if test -n "$CXXCPP" ; then
+            CXXCPP="$CXXCPP $switch"
+          fi
+          ac_success=yes
+          break
+        fi
+      done
+      if test x$ac_success = xyes; then
+        break
+      fi
+    done
+  fi])
+  AC_LANG_POP([C++])
+  if test x$ax_cxx_compile_cxx$1_required = xtrue; then
+    if test x$ac_success = xno; then
+      AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.])
+    fi
+  fi
+  if test x$ac_success = xno; then
+    HAVE_CXX$1=0
+    AC_MSG_NOTICE([No compiler with C++$1 support was found])
+  else
+    HAVE_CXX$1=1
+    AC_DEFINE(HAVE_CXX$1,1,
+              [define if the compiler supports basic C++$1 syntax])
+  fi
+  AC_SUBST(HAVE_CXX$1)
+])
+
+
+dnl  Test body for checking C++11 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+)
+
+
+dnl  Test body for checking C++14 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+)
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
+)
+
+dnl  Tests for new features in C++11
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
+
+// If the compiler admits that it is not ready for C++11, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201103L
+
+#error "This is not a C++11 compiler"
+
+#else
+
+namespace cxx11
+{
+
+  namespace test_static_assert
+  {
+
+    template <typename T>
+    struct check
+    {
+      static_assert(sizeof(int) <= sizeof(T), "not big enough");
+    };
+
+  }
+
+  namespace test_final_override
+  {
+
+    struct Base
+    {
+      virtual ~Base() {}
+      virtual void f() {}
+    };
+
+    struct Derived : public Base
+    {
+      virtual ~Derived() override {}
+      virtual void f() override {}
+    };
+
+  }
+
+  namespace test_double_right_angle_brackets
+  {
+
+    template < typename T >
+    struct check {};
+
+    typedef check<void> single_type;
+    typedef check<check<void>> double_type;
+    typedef check<check<check<void>>> triple_type;
+    typedef check<check<check<check<void>>>> quadruple_type;
+
+  }
+
+  namespace test_decltype
+  {
+
+    int
+    f()
+    {
+      int a = 1;
+      decltype(a) b = 2;
+      return a + b;
+    }
+
+  }
+
+  namespace test_type_deduction
+  {
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static const bool value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static const bool value = true;
+    };
+
+    template < typename T1, typename T2 >
+    auto
+    add(T1 a1, T2 a2) -> decltype(a1 + a2)
+    {
+      return a1 + a2;
+    }
+
+    int
+    test(const int c, volatile int v)
+    {
+      static_assert(is_same<int, decltype(0)>::value == true, "");
+      static_assert(is_same<int, decltype(c)>::value == false, "");
+      static_assert(is_same<int, decltype(v)>::value == false, "");
+      auto ac = c;
+      auto av = v;
+      auto sumi = ac + av + 'x';
+      auto sumf = ac + av + 1.0;
+      static_assert(is_same<int, decltype(ac)>::value == true, "");
+      static_assert(is_same<int, decltype(av)>::value == true, "");
+      static_assert(is_same<int, decltype(sumi)>::value == true, "");
+      static_assert(is_same<int, decltype(sumf)>::value == false, "");
+      static_assert(is_same<int, decltype(add(c, v))>::value == true, "");
+      return (sumf > 0.0) ? sumi : add(c, v);
+    }
+
+  }
+
+  namespace test_noexcept
+  {
+
+    int f() { return 0; }
+    int g() noexcept { return 0; }
+
+    static_assert(noexcept(f()) == false, "");
+    static_assert(noexcept(g()) == true, "");
+
+  }
+
+  namespace test_constexpr
+  {
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c_r(const CharT *const s, const unsigned long acc) noexcept
+    {
+      return *s ? strlen_c_r(s + 1, acc + 1) : acc;
+    }
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c(const CharT *const s) noexcept
+    {
+      return strlen_c_r(s, 0UL);
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("1") == 1UL, "");
+    static_assert(strlen_c("example") == 7UL, "");
+    static_assert(strlen_c("another\0example") == 7UL, "");
+
+  }
+
+  namespace test_rvalue_references
+  {
+
+    template < int N >
+    struct answer
+    {
+      static constexpr int value = N;
+    };
+
+    answer<1> f(int&)       { return answer<1>(); }
+    answer<2> f(const int&) { return answer<2>(); }
+    answer<3> f(int&&)      { return answer<3>(); }
+
+    void
+    test()
+    {
+      int i = 0;
+      const int c = 0;
+      static_assert(decltype(f(i))::value == 1, "");
+      static_assert(decltype(f(c))::value == 2, "");
+      static_assert(decltype(f(0))::value == 3, "");
+    }
+
+  }
+
+  namespace test_uniform_initialization
+  {
+
+    struct test
+    {
+      static const int zero {};
+      static const int one {1};
+    };
+
+    static_assert(test::zero == 0, "");
+    static_assert(test::one == 1, "");
+
+  }
+
+  namespace test_lambdas
+  {
+
+    void
+    test1()
+    {
+      auto lambda1 = [](){};
+      auto lambda2 = lambda1;
+      lambda1();
+      lambda2();
+    }
+
+    int
+    test2()
+    {
+      auto a = [](int i, int j){ return i + j; }(1, 2);
+      auto b = []() -> int { return '0'; }();
+      auto c = [=](){ return a + b; }();
+      auto d = [&](){ return c; }();
+      auto e = [a, &b](int x) mutable {
+        const auto identity = [](int y){ return y; };
+        for (auto i = 0; i < a; ++i)
+          a += b--;
+        return x + identity(a + b);
+      }(0);
+      return a + b + c + d + e;
+    }
+
+    int
+    test3()
+    {
+      const auto nullary = [](){ return 0; };
+      const auto unary = [](int x){ return x; };
+      using nullary_t = decltype(nullary);
+      using unary_t = decltype(unary);
+      const auto higher1st = [](nullary_t f){ return f(); };
+      const auto higher2nd = [unary](nullary_t f1){
+        return [unary, f1](unary_t f2){ return f2(unary(f1())); };
+      };
+      return higher1st(nullary) + higher2nd(nullary)(unary);
+    }
+
+  }
+
+  namespace test_variadic_templates
+  {
+
+    template <int...>
+    struct sum;
+
+    template <int N0, int... N1toN>
+    struct sum<N0, N1toN...>
+    {
+      static constexpr auto value = N0 + sum<N1toN...>::value;
+    };
+
+    template <>
+    struct sum<>
+    {
+      static constexpr auto value = 0;
+    };
+
+    static_assert(sum<>::value == 0, "");
+    static_assert(sum<1>::value == 1, "");
+    static_assert(sum<23>::value == 23, "");
+    static_assert(sum<1, 2>::value == 3, "");
+    static_assert(sum<5, 5, 11>::value == 21, "");
+    static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, "");
+
+  }
+
+  // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae
+  // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function
+  // because of this.
+  namespace test_template_alias_sfinae
+  {
+
+    struct foo {};
+
+    template<typename T>
+    using member = typename T::member_type;
+
+    template<typename T>
+    void func(...) {}
+
+    template<typename T>
+    void func(member<T>*) {}
+
+    void test();
+
+    void test() { func<foo>(0); }
+
+  }
+
+}  // namespace cxx11
+
+#endif  // __cplusplus >= 201103L
+
+]])
+
+
+dnl  Tests for new features in C++14
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[
+
+// If the compiler admits that it is not ready for C++14, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201402L
+
+#error "This is not a C++14 compiler"
+
+#else
+
+namespace cxx14
+{
+
+  namespace test_polymorphic_lambdas
+  {
+
+    int
+    test()
+    {
+      const auto lambda = [](auto&&... args){
+        const auto istiny = [](auto x){
+          return (sizeof(x) == 1UL) ? 1 : 0;
+        };
+        const int aretiny[] = { istiny(args)... };
+        return aretiny[0];
+      };
+      return lambda(1, 1L, 1.0f, '1');
+    }
+
+  }
+
+  namespace test_binary_literals
+  {
+
+    constexpr auto ivii = 0b0000000000101010;
+    static_assert(ivii == 42, "wrong value");
+
+  }
+
+  namespace test_generalized_constexpr
+  {
+
+    template < typename CharT >
+    constexpr unsigned long
+    strlen_c(const CharT *const s) noexcept
+    {
+      auto length = 0UL;
+      for (auto p = s; *p; ++p)
+        ++length;
+      return length;
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("x") == 1UL, "");
+    static_assert(strlen_c("test") == 4UL, "");
+    static_assert(strlen_c("another\0test") == 7UL, "");
+
+  }
+
+  namespace test_lambda_init_capture
+  {
+
+    int
+    test()
+    {
+      auto x = 0;
+      const auto lambda1 = [a = x](int b){ return a + b; };
+      const auto lambda2 = [a = lambda1(x)](){ return a; };
+      return lambda2();
+    }
+
+  }
+
+  namespace test_digit_separators
+  {
+
+    constexpr auto ten_million = 100'000'000;
+    static_assert(ten_million == 100000000, "");
+
+  }
+
+  namespace test_return_type_deduction
+  {
+
+    auto f(int& x) { return x; }
+    decltype(auto) g(int& x) { return x; }
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static constexpr auto value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static constexpr auto value = true;
+    };
+
+    int
+    test()
+    {
+      auto x = 0;
+      static_assert(is_same<int, decltype(f(x))>::value, "");
+      static_assert(is_same<int&, decltype(g(x))>::value, "");
+      return x;
+    }
+
+  }
+
+}  // namespace cxx14
+
+#endif  // __cplusplus >= 201402L
+
+]])
+
+
+dnl  Tests for new features in C++17
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[
+
+// If the compiler admits that it is not ready for C++17, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201703L
+
+#error "This is not a C++17 compiler"
+
+#else
+
+#include <initializer_list>
+#include <utility>
+#include <type_traits>
+
+namespace cxx17
+{
+
+  namespace test_constexpr_lambdas
+  {
+
+    constexpr int foo = [](){return 42;}();
+
+  }
+
+  namespace test::nested_namespace::definitions
+  {
+
+  }
+
+  namespace test_fold_expression
+  {
+
+    template<typename... Args>
+    int multiply(Args... args)
+    {
+      return (args * ... * 1);
+    }
+
+    template<typename... Args>
+    bool all(Args... args)
+    {
+      return (args && ...);
+    }
+
+  }
+
+  namespace test_extended_static_assert
+  {
+
+    static_assert (true);
+
+  }
+
+  namespace test_auto_brace_init_list
+  {
+
+    auto foo = {5};
+    auto bar {5};
+
+    static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value);
+    static_assert(std::is_same<int, decltype(bar)>::value);
+  }
+
+  namespace test_typename_in_template_template_parameter
+  {
+
+    template<template<typename> typename X> struct D;
+
+  }
+
+  namespace test_fallthrough_nodiscard_maybe_unused_attributes
+  {
+
+    int f1()
+    {
+      return 42;
+    }
+
+    [[nodiscard]] int f2()
+    {
+      [[maybe_unused]] auto unused = f1();
+
+      switch (f1())
+      {
+      case 17:
+        f1();
+        [[fallthrough]];
+      case 42:
+        f1();
+      }
+      return f1();
+    }
+
+  }
+
+  namespace test_extended_aggregate_initialization
+  {
+
+    struct base1
+    {
+      int b1, b2 = 42;
+    };
+
+    struct base2
+    {
+      base2() {
+        b3 = 42;
+      }
+      int b3;
+    };
+
+    struct derived : base1, base2
+    {
+        int d;
+    };
+
+    derived d1 {{1, 2}, {}, 4};  // full initialization
+    derived d2 {{}, {}, 4};      // value-initialized bases
+
+  }
+
+  namespace test_general_range_based_for_loop
+  {
+
+    struct iter
+    {
+      int i;
+
+      int& operator* ()
+      {
+        return i;
+      }
+
+      const int& operator* () const
+      {
+        return i;
+      }
+
+      iter& operator++()
+      {
+        ++i;
+        return *this;
+      }
+    };
+
+    struct sentinel
+    {
+      int i;
+    };
+
+    bool operator== (const iter& i, const sentinel& s)
+    {
+      return i.i == s.i;
+    }
+
+    bool operator!= (const iter& i, const sentinel& s)
+    {
+      return !(i == s);
+    }
+
+    struct range
+    {
+      iter begin() const
+      {
+        return {0};
+      }
+
+      sentinel end() const
+      {
+        return {5};
+      }
+    };
+
+    void f()
+    {
+      range r {};
+
+      for (auto i : r)
+      {
+        [[maybe_unused]] auto v = i;
+      }
+    }
+
+  }
+
+  namespace test_lambda_capture_asterisk_this_by_value
+  {
+
+    struct t
+    {
+      int i;
+      int foo()
+      {
+        return [*this]()
+        {
+          return i;
+        }();
+      }
+    };
+
+  }
+
+  namespace test_enum_class_construction
+  {
+
+    enum class byte : unsigned char
+    {};
+
+    byte foo {42};
+
+  }
+
+  namespace test_constexpr_if
+  {
+
+    template <bool cond>
+    int f ()
+    {
+      if constexpr(cond)
+      {
+        return 13;
+      }
+      else
+      {
+        return 42;
+      }
+    }
+
+  }
+
+  namespace test_selection_statement_with_initializer
+  {
+
+    int f()
+    {
+      return 13;
+    }
+
+    int f2()
+    {
+      if (auto i = f(); i > 0)
+      {
+        return 3;
+      }
+
+      switch (auto i = f(); i + 4)
+      {
+      case 17:
+        return 2;
+
+      default:
+        return 1;
+      }
+    }
+
+  }
+
+  namespace test_template_argument_deduction_for_class_templates
+  {
+
+    template <typename T1, typename T2>
+    struct pair
+    {
+      pair (T1 p1, T2 p2)
+        : m1 {p1},
+          m2 {p2}
+      {}
+
+      T1 m1;
+      T2 m2;
+    };
+
+    void f()
+    {
+      [[maybe_unused]] auto p = pair{13, 42u};
+    }
+
+  }
+
+  namespace test_non_type_auto_template_parameters
+  {
+
+    template <auto n>
+    struct B
+    {};
+
+    B<5> b1;
+    B<'a'> b2;
+
+  }
+
+  namespace test_structured_bindings
+  {
+
+    int arr[2] = { 1, 2 };
+    std::pair<int, int> pr = { 1, 2 };
+
+    auto f1() -> int(&)[2]
+    {
+      return arr;
+    }
+
+    auto f2() -> std::pair<int, int>&
+    {
+      return pr;
+    }
+
+    struct S
+    {
+      int x1 : 2;
+      volatile double y1;
+    };
+
+    S f3()
+    {
+      return {};
+    }
+
+    auto [ x1, y1 ] = f1();
+    auto& [ xr1, yr1 ] = f1();
+    auto [ x2, y2 ] = f2();
+    auto& [ xr2, yr2 ] = f2();
+    const auto [ x3, y3 ] = f3();
+
+  }
+
+  namespace test_exception_spec_type_system
+  {
+
+    struct Good {};
+    struct Bad {};
+
+    void g1() noexcept;
+    void g2();
+
+    template<typename T>
+    Bad
+    f(T*, T*);
+
+    template<typename T1, typename T2>
+    Good
+    f(T1*, T2*);
+
+    static_assert (std::is_same_v<Good, decltype(f(g1, g2))>);
+
+  }
+
+  namespace test_inline_variables
+  {
+
+    template<class T> void f(T)
+    {}
+
+    template<class T> inline T g(T)
+    {
+      return T{};
+    }
+
+    template<> inline void f<>(int)
+    {}
+
+    template<> int g<>(int)
+    {
+      return 5;
+    }
+
+  }
+
+}  // namespace cxx17
+
+#endif  // __cplusplus < 201703L
+
+]])
diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4
new file mode 100644
index 00000000000..a6834171739
--- /dev/null
+++ b/m4/ax_cxx_compile_stdcxx_17.m4
@@ -0,0 +1,35 @@
+# =============================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html
+# =============================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the C++17
+#   standard; if necessary, add switches to CXX and CXXCPP to enable
+#   support.
+#
+#   This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX
+#   macro with the version set to C++17.  The two optional arguments are
+#   forwarded literally as the second and third argument respectively.
+#   Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for
+#   more information.  If you want to use this macro, you also need to
+#   download the ax_cxx_compile_stdcxx.m4 file.
+#
+# LICENSE
+#
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#   Copyright (c) 2016 Krzesimir Nowak <qdlacz@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 2
+
+AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX])
+AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])])

From fe068eca00d6a2dd2ea9ee08924dc3898c37fb6c Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <siarheit@google.com>
Date: Tue, 24 Jan 2017 22:42:15 +0000
Subject: [PATCH 1465/2196] mk: add support for passing LDFLAGS to libs and
 bins

autotools-based systems usually allow user to
append own LDFLAGS like
    LDFLAGS="-Wl,-O1 -Wl,--as-needed -Wl,--hash-style=gnu"
at ./configure stage

This change plumbs LDFLAGS through similar to existing CXXFLAGS variable.

Signed-off-by: Sergei Trofimovich <siarheit@google.com>
---
 Makefile.config.in | 1 +
 mk/libraries.mk    | 4 ++--
 mk/programs.mk     | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 59730b64638..eabf1a426eb 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -5,6 +5,7 @@ CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
+LDFLAGS = @LDFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_READLINE = @HAVE_READLINE@
diff --git a/mk/libraries.mk b/mk/libraries.mk
index 3953446cba3..307e29b9d05 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -91,7 +91,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT)
 
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
+	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
     ifneq ($(OS), Darwin)
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
@@ -105,7 +105,7 @@ define build-library
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
     $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
+	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
     ifneq ($(OS), Darwin)
diff --git a/mk/programs.mk b/mk/programs.mk
index 2fbda12bd15..d93df446808 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
   $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
@@ -46,7 +46,7 @@ define build-program
     _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
 
     $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
   else
 

From 20129bd83d57316cc0b69bec7abeec43011c56d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Tue, 2 Jul 2019 22:28:30 +0200
Subject: [PATCH 1466/2196] autoconf: Fix AC_STRUCT_DIRENT_D_TYPE being used
 before AC_PROG_CC.

That was incorrect, because checking the dirent type already requires
a working compiler.

It had the effect that setting e.g. `: ${CFLAGS=""}` before `AC_PROG_CC`
as per `AC_PROG_CC`'s documentation would have no effect, because
`AC_STRUCT_DIRENT_D_TYPE` would automatically set CFLASGS.

(In a followup commit `: ${CFLAGS=""}` will be used, so it's important
to get this working first.)
---
 configure.ac | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/configure.ac b/configure.ac
index c12f9d5ff95..3a486fab8e7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,14 +50,6 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Solaris-specific stuff.
-AC_STRUCT_DIRENT_D_TYPE
-if test "$sys_name" = sunos; then
-    # Solaris requires -lsocket -lnsl for network functions
-    LIBS="-lsocket -lnsl $LIBS"
-fi
-
-
 CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
@@ -71,6 +63,14 @@ AC_CHECK_TOOL([AR], [ar])
 AC_SYS_LARGEFILE
 
 
+# Solaris-specific stuff.
+AC_STRUCT_DIRENT_D_TYPE
+if test "$sys_name" = sunos; then
+    # Solaris requires -lsocket -lnsl for network functions
+    LIBS="-lsocket -lnsl $LIBS"
+fi
+
+
 # Check for pubsetbuf.
 AC_MSG_CHECKING([for pubsetbuf])
 AC_LANG_PUSH(C++)

From 717e821b99797845e1bef47d862f8cb0fb69cfc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Tue, 2 Jul 2019 22:30:15 +0200
Subject: [PATCH 1467/2196] autoconf: Allow overriding CFLAGS/CXXFLAGS from
 outside.

As is normal for autoconf-based projects.

For example, it is a common use case to do

    ./configure CXXFLAGS=-O0

This did not work for nix until now, because the `CXXFLAGS=` declaration
would unconditionally erase what the user had specified.

The custom `OPTIMIZE` flag is removed, but the default `-O3` is retained;
autoconf would default to `-g -O2` by default otherwise as documented on:

https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C-Compiler.html
https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C_002b_002b-Compiler.html
---
 Makefile          | 7 -------
 configure.ac      | 6 ++++--
 perl/Makefile     | 7 -------
 perl/configure.ac | 6 ++++--
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 45a3338ed21..9ac82fda657 100644
--- a/Makefile
+++ b/Makefile
@@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h
 
 -include Makefile.config
 
-OPTIMIZE = 1
-
-ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CFLAGS += -O3
-  GLOBAL_CXXFLAGS += -O3
-endif
-
 include mk/lib.mk
diff --git a/configure.ac b/configure.ac
index 3a486fab8e7..571bb54d8a1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,8 +50,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-CFLAGS=
-CXXFLAGS=
+# Set default flags for nix (as per AC_PROG_CC/CXX docs),
+# while still allowing the user to override them from the command line.
+: ${CFLAGS="-O3"}
+: ${CXXFLAGS="-O3"}
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
diff --git a/perl/Makefile b/perl/Makefile
index 284c7502249..f36f5d0e9d8 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 
-OPTIMIZE = 1
-
-ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CFLAGS += -O3
-  GLOBAL_CXXFLAGS += -O3
-endif
-
 include mk/lib.mk
diff --git a/perl/configure.ac b/perl/configure.ac
index 966700695ff..e8e3610a860 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -2,8 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 
-CFLAGS=
-CXXFLAGS=
+# Set default flags for nix (as per AC_PROG_CC/CXX docs),
+# while still allowing the user to override them from the command line.
+: ${CFLAGS="-O3"}
+: ${CXXFLAGS="-O3"}
 AC_PROG_CC
 AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11

From b49c3a9db516a87d38476966eff9994c13c5acf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 01:34:21 +0200
Subject: [PATCH 1468/2196] Makefile.config.in: Remove HAVE_READLINE.

It was forgotten to be removed with
commit c5f23f10a84f568874321c04984b1a14d2dce978
and so it until now stayed unsubstituted as `HAVE_READLINE = @HAVE_READLINE@`
in Makefile.config.
---
 Makefile.config.in | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index eabf1a426eb..936a4f4039d 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -8,7 +8,6 @@ CXXFLAGS = @CXXFLAGS@
 LDFLAGS = @LDFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
-HAVE_READLINE = @HAVE_READLINE@
 HAVE_SECCOMP = @HAVE_SECCOMP@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@

From d203c554faa00fec55377d6640c8fb335a611a09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 01:46:07 +0200
Subject: [PATCH 1469/2196] Fix C++ compatibility with older editline versions.

For example, Ubuntu 16.04 and many similar long-term-support distros
have older versions.
---
 src/nix/repl.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index d8f81214906..f857b2e89c2 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -9,7 +9,14 @@
 #include <readline/history.h>
 #include <readline/readline.h>
 #else
+// editline < 1.15.2 don't wrap their API for C++ usage
+// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461).
+// This results in linker errors due to to name-mangling of editline C symbols.
+// For compatibility with these versions, we wrap the API here
+// (wrapping multiple times on newer versions is no problem).
+extern "C" {
 #include <editline.h>
+}
 #endif
 
 #include "shared.hh"

From a96006d97fc87c5073f9a39db841625bdb7c401c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 02:47:27 +0200
Subject: [PATCH 1470/2196] Get BOOST_LDFLAGS from autoconf, fix Ubuntu 16.04
 build.

Our use of boost::coroutine2 depends on -lboost_context,
which in turn depends on `-lboost_thread`, which in turn depends
on `-lboost_system`.

I suspect that this builds on nix only because of low-level hacks
like NIX_LDFLAGS.

This commit passes the proper linker flags, thus fixing bootstrap
builds on non-nix distributions like Ubuntu 16.04.

With these changes, I can build Nix on Ubuntu 16.04 using:

    ./bootstrap.sh
    ./configure --prefix=$HOME/editline-prefix \
      --disable-doc-gen \
      CXX=g++-7 \
      --with-boost=$HOME/boost-prefix \
      EDITLINE_CFLAGS=-I$HOME/editline-prefix/include \
      EDITLINE_LIBS=-leditline \
      LDFLAGS=-L$HOME/editline-prefix/lib
    make

where

* g++-7 comes from gcc-7 from
  https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test,
* editline 1.14 from https://github.com/troglobit/editline/releases/tag/1.14.0
	was installed into `$HOME/editline-prefix`
  (because Ubuntu 16.04's `editline` is too old to have the function nix uses),
* boost 1.66 from
	https://www.boost.org/doc/libs/1_66_0/more/getting_started/unix-variants.html
	was installed into $HOME/boost-prefix (because Ubuntu 16.04 only has 1.58)
---
 Makefile.config.in   | 1 +
 configure.ac         | 3 +++
 src/libutil/local.mk | 2 +-
 src/nix/local.mk     | 2 +-
 4 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 936a4f4039d..7e3b35b9819 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -9,6 +9,7 @@ LDFLAGS = @LDFLAGS@
 ENABLE_S3 = @ENABLE_S3@
 HAVE_SODIUM = @HAVE_SODIUM@
 HAVE_SECCOMP = @HAVE_SECCOMP@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
diff --git a/configure.ac b/configure.ac
index 571bb54d8a1..a52830b3835 100644
--- a/configure.ac
+++ b/configure.ac
@@ -153,6 +153,9 @@ AC_SUBST(storedir)
 # and CPPFLAGS are not passed to the C++ compiler automatically.
 # Thus we append the returned CPPFLAGS to the CXXFLAGS here.
 AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])])
+# For unknown reasons, setting this directly in the ACTION-IF-FOUND above
+# ends up with LDFLAGS being empty, so we set it afterwards.
+LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
 
 
 # Look for OpenSSL, a required dependency.
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 3ccc23fd5c1..e41a67d1f9e 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,4 +6,4 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
diff --git a/src/nix/local.mk b/src/nix/local.mk
index ca4604d566c..c09efd1fc89 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -17,7 +17,7 @@ nix_SOURCES := \
 
 nix_LIBS = libexpr libmain libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS)
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \

From c3db9e6f8fd06d691be04cdd95a6bb21a400481d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 03:34:17 +0200
Subject: [PATCH 1471/2196] autoconf: Check if --nonet works. Fixes #967 #506.

Also give a helpful error message on what package the user likely
has to install to make it work.
---
 configure.ac | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/configure.ac b/configure.ac
index a52830b3835..f510f8a9782 100644
--- a/configure.ac
+++ b/configure.ac
@@ -258,6 +258,15 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed)
+if test "$doc_generate" = yes; then
+  AC_MSG_CHECKING([if docbook can run without networking])
+  if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then
+    AC_MSG_RESULT(yes)
+  else
+    AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?])
+  fi
+fi
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])

From cd8bc06e8786018ddb16cea4cb10971b63d0efd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 04:20:37 +0200
Subject: [PATCH 1472/2196] autoconf: Add comment on use of `false`.

This is to avoid confusion as in commit
a0d29040f79b365598fe75d01f72d29ab538206b.
---
 configure.ac | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index f510f8a9782..1e3a8091cd5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -119,10 +119,12 @@ if test -z "$$1"; then
 fi
 ])
 
+# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program
+# is not required in all cases. (e.g. not when building from a release tarball).
+# The use of `false` isn't ideal, because when used somewhere in the build it
+# will silently fail without error message; this should be improved.
 NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
-AC_PATH_PROG(xmllint, xmllint, false)
-AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(sed, sed)
@@ -258,8 +260,12 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
-# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed)
 if test "$doc_generate" = yes; then
+  # Programs required to build the manual
+  NEED_PROG(xmllint, xmllint)
+  NEED_PROG(xsltproc, xsltproc)
+
+  # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed)
   AC_MSG_CHECKING([if docbook can run without networking])
   if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then
     AC_MSG_RESULT(yes)

From 82b7f0e840983879a510245903ff7c917276f65d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Wed, 3 Jul 2019 04:28:06 +0200
Subject: [PATCH 1473/2196] autoconf: Implement release tarball detection.
 Fixes #257.

This should finally allow us to address all cases of build errors due to
differences between release tarballs and building from git.

See also https://github.com/NixOS/nix/issues/506#issuecomment-507312587
---
 configure.ac | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1e3a8091cd5..8bc14d476d5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5,6 +5,19 @@ AC_CONFIG_AUX_DIR(config)
 
 AC_PROG_SED
 
+# Detect building from release tarballs (we need less dependencies then)
+# by checking for the `.dist-files` file present only in release tarballs
+# (created in release.nix).
+# For example, we don't need to build the manual or run bison/flex
+# parser generators for release tarballs, as those have them pre-generated.
+AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)])
+if test -f .dist-files; then
+  AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs)
+  building_from_release_tarball=yes;
+else
+  AC_MSG_RESULT(no)
+fi
+
 # Construct a Nix system name (like "i686-linux").
 AC_CANONICAL_HOST
 AC_MSG_CHECKING([for the canonical Nix system name])
@@ -119,14 +132,13 @@ if test -z "$$1"; then
 fi
 ])
 
-# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program
-# is not required in all cases. (e.g. not when building from a release tarball).
-# The use of `false` isn't ideal, because when used somewhere in the build it
-# will silently fail without error message; this should be improved.
 NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
-AC_PATH_PROG(flex, flex, false)
-AC_PATH_PROG(bison, bison, false)
+if test "x$building_from_release_tarball" != x"yes"; then
+  # Parser generators are pre-built in release tarballs.
+  NEED_PROG(flex, flex)
+  NEED_PROG(bison, bison)
+fi
 NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 NEED_PROG(bzip2, bzip2)
@@ -260,7 +272,8 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
-if test "$doc_generate" = yes; then
+# The manual is pre-generated in release tarballs.
+if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then
   # Programs required to build the manual
   NEED_PROG(xmllint, xmllint)
   NEED_PROG(xsltproc, xsltproc)

From e486d8d40e626a20e06d792db8cc5ac5aba9a5b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jul 2019 00:33:51 +0200
Subject: [PATCH 1474/2196] Revert 82b7f0e840983879a510245903ff7c917276f65d,
 cd8bc06e8786018ddb16cea4cb10971b63d0efd2,
 c3db9e6f8fd06d691be04cdd95a6bb21a400481d

This breaks the tarball job: https://hydra.nixos.org/build/95714570
---
 configure.ac | 36 ++++--------------------------------
 1 file changed, 4 insertions(+), 32 deletions(-)

diff --git a/configure.ac b/configure.ac
index 8bc14d476d5..a52830b3835 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5,19 +5,6 @@ AC_CONFIG_AUX_DIR(config)
 
 AC_PROG_SED
 
-# Detect building from release tarballs (we need less dependencies then)
-# by checking for the `.dist-files` file present only in release tarballs
-# (created in release.nix).
-# For example, we don't need to build the manual or run bison/flex
-# parser generators for release tarballs, as those have them pre-generated.
-AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)])
-if test -f .dist-files; then
-  AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs)
-  building_from_release_tarball=yes;
-else
-  AC_MSG_RESULT(no)
-fi
-
 # Construct a Nix system name (like "i686-linux").
 AC_CANONICAL_HOST
 AC_MSG_CHECKING([for the canonical Nix system name])
@@ -134,11 +121,10 @@ fi
 
 NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
-if test "x$building_from_release_tarball" != x"yes"; then
-  # Parser generators are pre-built in release tarballs.
-  NEED_PROG(flex, flex)
-  NEED_PROG(bison, bison)
-fi
+AC_PATH_PROG(xmllint, xmllint, false)
+AC_PATH_PROG(xsltproc, xsltproc, false)
+AC_PATH_PROG(flex, flex, false)
+AC_PATH_PROG(bison, bison, false)
 NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 NEED_PROG(bzip2, bzip2)
@@ -272,20 +258,6 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
-# The manual is pre-generated in release tarballs.
-if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then
-  # Programs required to build the manual
-  NEED_PROG(xmllint, xmllint)
-  NEED_PROG(xsltproc, xsltproc)
-
-  # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed)
-  AC_MSG_CHECKING([if docbook can run without networking])
-  if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then
-    AC_MSG_RESULT(yes)
-  else
-    AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?])
-  fi
-fi
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])

From 648bdf153d2e4d3c6687d8e1780b0dfd0aa98cda Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 21 May 2019 22:29:23 -0400
Subject: [PATCH 1475/2196] tarball-ttl: document
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Incorporates text from Niklas Hambüchen in #2978

Closes #1115
---
 doc/manual/command-ref/conf-file.xml | 25 +++++++++++++++++++++++++
 doc/manual/expressions/builtins.xml  |  4 ++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 24fbf28cff2..09aad2e0553 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -864,6 +864,31 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term>
+
+    <listitem>
+      <para>Default: <literal>3600</literal> seconds.</para>
+
+      <para>The number of seconds a downloaded tarball is considered
+      fresh. If the cached tarball is stale, Nix will check whether
+      it is still up to date using the ETag header. Nix will download
+      a new version if the ETag header is unsupported, or the
+      cached ETag doesn't match.
+      </para>
+
+      <para>Setting the TTL to <literal>0</literal> forces Nix to always
+      check if the tarball is up to date.</para>
+
+      <para>Nix caches tarballs in
+      <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para>
+
+      <para>Files fetched via <envar>NIX_PATH</envar>,
+      <function>fetchGit</function>, <function>fetchMercurial</function>,
+      <function>fetchTarball</function>, and <function>fetchurl</function>
+      respect this TTL.
+      </para>
+    </listitem>
+  </varlistentry>
 
   <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term>
 
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index a87639a075a..deffcb59442 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -347,7 +347,7 @@ stdenv.mkDerivation { … }
     You can change the cache timeout either on the command line with
     <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
     in the Nix configuration file with this option:
-    <literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>.
+    <literal><xref linkend="conf-tarball-ttl" /> <replaceable>number of seconds to cache</replaceable></literal>.
     </para>
 
     <para>Note that when obtaining the hash with <varname>nix-prefetch-url
@@ -498,7 +498,7 @@ stdenv.mkDerivation { … }
            fetch the latest version of a remote branch.
         </para>
         <note><para>Nix will refetch the branch in accordance to
-        <option>tarball-ttl</option>.</para></note>
+        <xref linkend="conf-tarball-ttl" />.</para></note>
         <note><para>This behavior is disabled in
         <emphasis>Pure evaluation mode</emphasis>.</para></note>
         <programlisting>builtins.fetchGit {

From aa739e783993864aa6e0c8a4820e6b59f4626d92 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 Jul 2019 10:29:41 +0200
Subject: [PATCH 1476/2196] nix copy: Rename --substitute to
 --substitute-on-destination

'--substitute' was being shadowed by the regular '--substitute' (the
short-hand for '--option substitute true').

Fixes #2983.
---
 src/nix/copy.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 96bd453d87b..12a9f9cd337 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -36,7 +36,7 @@ struct CmdCopy : StorePathsCommand
             .set(&checkSigs, NoCheckSigs);
 
         mkFlag()
-            .longName("substitute")
+            .longName("substitute-on-destination")
             .shortName('s')
             .description("whether to try substitutes on the destination store (only supported by SSH)")
             .set(&substitute, Substitute);

From 03f09e1d18c39706b917964f14f9e40513e4aaea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 Jul 2019 19:46:15 +0200
Subject: [PATCH 1477/2196] Revert "Fix 'error 9 while decompressing xz file'"

This reverts commit 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91.
---
 src/libstore/binary-cache-store.cc      | 18 ++---
 src/libstore/download.cc                | 67 ++++++++++++++----
 src/libstore/download.hh                | 11 +--
 src/libstore/http-binary-cache-store.cc | 57 +++++----------
 src/libstore/store-api.cc               | 94 ++++++++++++-------------
 src/libutil/retry.hh                    | 38 ----------
 src/libutil/types.hh                    |  2 -
 7 files changed, 125 insertions(+), 162 deletions(-)
 delete mode 100644 src/libutil/retry.hh

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8b736056e01..4527ee6ba66 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -10,8 +10,6 @@
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
 #include "json.hh"
-#include "retry.hh"
-#include "download.hh"
 
 #include <chrono>
 
@@ -81,15 +79,13 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 
 std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
-    return retry<std::shared_ptr<std::string>>(downloadSettings.tries, [&]() -> std::shared_ptr<std::string> {
-        StringSink sink;
-        try {
-            getFile(path, sink);
-        } catch (NoSuchBinaryCacheFile &) {
-            return nullptr;
-        }
-        return sink.s;
-    });
+    StringSink sink;
+    try {
+        getFile(path, sink);
+    } catch (NoSuchBinaryCacheFile &) {
+        return nullptr;
+    }
+    return sink.s;
 }
 
 Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 7a2af237ee8..2267b5d3517 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -8,7 +8,6 @@
 #include "compression.hh"
 #include "pathlocks.hh"
 #include "finally.hh"
-#include "retry.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -20,9 +19,11 @@
 #include <curl/curl.h>
 
 #include <algorithm>
+#include <cmath>
 #include <cstring>
 #include <iostream>
 #include <queue>
+#include <random>
 #include <thread>
 
 using namespace std::string_literals;
@@ -45,6 +46,9 @@ struct CurlDownloader : public Downloader
 {
     CURLM * curlm = 0;
 
+    std::random_device rd;
+    std::mt19937 mt19937;
+
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
         CurlDownloader & downloader;
@@ -57,6 +61,12 @@ struct CurlDownloader : public Downloader
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
 
+        unsigned int attempt = 0;
+
+        /* Don't start this download until the specified time point
+           has been reached. */
+        std::chrono::steady_clock::time_point embargo;
+
         struct curl_slist * requestHeaders = 0;
 
         std::string encoding;
@@ -377,7 +387,9 @@ struct CurlDownloader : public Downloader
                     }
                 }
 
-                fail(
+                attempt++;
+
+                auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
                     ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
@@ -388,15 +400,31 @@ struct CurlDownloader : public Downloader
                         )
                     : DownloadError(err,
                         fmt("unable to %s '%s': %s (%d)",
-                            request.verb(), request.uri, curl_easy_strerror(code), code)));
+                            request.verb(), request.uri, curl_easy_strerror(code), code));
+
+                /* If this is a transient error, then maybe retry the
+                   download after a while. */
+                if (err == Transient && attempt < request.tries) {
+                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
+                    printError(format("warning: %s; retrying in %d ms") % exc.what() % ms);
+                    embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
+                    downloader.enqueueItem(shared_from_this());
+                }
+                else
+                    fail(exc);
             }
         }
     };
 
     struct State
     {
+        struct EmbargoComparator {
+            bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
+                return i1->embargo > i2->embargo;
+            }
+        };
         bool quit = false;
-        std::vector<std::shared_ptr<DownloadItem>> incoming;
+        std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
     };
 
     Sync<State> state_;
@@ -409,6 +437,7 @@ struct CurlDownloader : public Downloader
     std::thread workerThread;
 
     CurlDownloader()
+        : mt19937(rd())
     {
         static std::once_flag globalInit;
         std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
@@ -502,7 +531,9 @@ struct CurlDownloader : public Downloader
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
-            /* Add new curl requests from the incoming requests queue. */
+            /* Add new curl requests from the incoming requests queue,
+               except for requests that are embargoed (waiting for a
+               retry timeout to expire). */
             if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
                 char buf[1024];
                 auto res = read(extraFDs[0].fd, buf, sizeof(buf));
@@ -511,9 +542,22 @@ struct CurlDownloader : public Downloader
             }
 
             std::vector<std::shared_ptr<DownloadItem>> incoming;
+            auto now = std::chrono::steady_clock::now();
+
             {
                 auto state(state_.lock());
-                std::swap(state->incoming, incoming);
+                while (!state->incoming.empty()) {
+                    auto item = state->incoming.top();
+                    if (item->embargo <= now) {
+                        incoming.push_back(item);
+                        state->incoming.pop();
+                    } else {
+                        if (nextWakeup == std::chrono::steady_clock::time_point()
+                            || item->embargo < nextWakeup)
+                            nextWakeup = item->embargo;
+                        break;
+                    }
+                }
                 quit = state->quit;
             }
 
@@ -540,7 +584,7 @@ struct CurlDownloader : public Downloader
 
         {
             auto state(state_.lock());
-            state->incoming.clear();
+            while (!state->incoming.empty()) state->incoming.pop();
             state->quit = true;
         }
     }
@@ -556,7 +600,7 @@ struct CurlDownloader : public Downloader
             auto state(state_.lock());
             if (state->quit)
                 throw nix::Error("cannot enqueue download request because the download thread is shutting down");
-            state->incoming.push_back(item);
+            state->incoming.push(item);
         }
         writeFull(wakeupPipe.writeSide.get(), " ");
     }
@@ -639,9 +683,7 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
 
 DownloadResult Downloader::download(const DownloadRequest & request)
 {
-    return retry<DownloadResult>(request.tries, [&]() {
-        return enqueueDownload(request).get();
-    });
+    return enqueueDownload(request).get();
 }
 
 void Downloader::download(DownloadRequest && request, Sink & sink)
@@ -827,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached(
             writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
-            warn("%s; using cached result", e.msg());
+            printError(format("warning: %1%; using cached result") % e.msg());
             result.etag = expectedETag;
         }
     }
@@ -878,4 +920,5 @@ bool isUri(const string & s)
     return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
 }
 
+
 }
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 9e965b506d0..3b7fff3ba4c 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -96,13 +96,11 @@ struct Downloader
 
     std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
 
-    /* Synchronously download a file. The request will be retried in
-       case of transient failures. */
+    /* Synchronously download a file. */
     DownloadResult download(const DownloadRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
-       invoked on the thread of the caller. The request will not be
-       retried in case of transient failures. */
+       invoked on the thread of the caller. */
     void download(DownloadRequest && request, Sink & sink);
 
     /* Check if the specified file is already in ~/.cache/nix/tarballs
@@ -128,11 +126,6 @@ public:
     DownloadError(Downloader::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
-
-    bool isTransient() override
-    {
-        return error == Downloader::Error::Transient;
-    }
 };
 
 bool isUri(const string & s);
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 5633b4355d2..3d17a89660e 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -2,7 +2,6 @@
 #include "download.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
-#include "retry.hh"
 
 namespace nix {
 
@@ -114,6 +113,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
+        request.tries = 8;
         return request;
     }
 
@@ -136,46 +136,21 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     {
         checkEnabled();
 
-        struct State
-        {
-            DownloadRequest request;
-            std::function<void()> tryDownload;
-            unsigned int attempt = 0;
-            State(DownloadRequest && request) : request(request) {}
-        };
-
-        auto state = std::make_shared<State>(makeRequest(path));
-
-        state->tryDownload = [callback, state, this]() {
-            getDownloader()->enqueueDownload(state->request,
-                {[callback, state, this](std::future<DownloadResult> result) {
-                    try {
-                        callback(result.get().data);
-                    } catch (DownloadError & e) {
-                        if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                            return callback(std::shared_ptr<std::string>());
-                        ++state->attempt;
-                        if (state->attempt < state->request.tries && e.isTransient()) {
-                            auto ms = retrySleepTime(state->attempt);
-                            warn("%s; retrying in %d ms", e.what(), ms);
-                            /* We can't sleep here because that would
-                               block the download thread. So use a
-                               separate thread for sleeping. */
-                            std::thread([state, ms]() {
-                                std::this_thread::sleep_for(std::chrono::milliseconds(ms));
-                                state->tryDownload();
-                            }).detach();
-                        } else {
-                            maybeDisable();
-                            callback.rethrow();
-                        }
-                    } catch (...) {
-                        callback.rethrow();
-                    }
-                }});
-        };
-
-        state->tryDownload();
+        auto request(makeRequest(path));
+
+        getDownloader()->enqueueDownload(request,
+            {[callback, this](std::future<DownloadResult> result) {
+                try {
+                    callback(result.get().data);
+                } catch (DownloadError & e) {
+                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                        return callback(std::shared_ptr<std::string>());
+                    maybeDisable();
+                    callback.rethrow();
+                } catch (...) {
+                    callback.rethrow();
+                }
+            }});
     }
 
 };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 92f01fd2e0c..f5608d3849f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,11 +6,10 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
-#include "retry.hh"
-#include "download.hh"
 
 #include <future>
 
+
 namespace nix {
 
 
@@ -580,57 +579,54 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    retry<void>(downloadSettings.tries, [&]() {
-
-        auto srcUri = srcStore->getUri();
-        auto dstUri = dstStore->getUri();
-
-        Activity act(*logger, lvlInfo, actCopyPath,
-            srcUri == "local" || srcUri == "daemon"
-              ? fmt("copying path '%s' to '%s'", storePath, dstUri)
-              : dstUri == "local" || dstUri == "daemon"
-                ? fmt("copying path '%s' from '%s'", storePath, srcUri)
-                : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
-            {storePath, srcUri, dstUri});
-        PushActivity pact(act.id);
-
-        auto info = srcStore->queryPathInfo(storePath);
-
-        uint64_t total = 0;
-
-        if (!info->narHash) {
-            StringSink sink;
-            srcStore->narFromPath({storePath}, sink);
-            auto info2 = make_ref<ValidPathInfo>(*info);
-            info2->narHash = hashString(htSHA256, *sink.s);
-            if (!info->narSize) info2->narSize = sink.s->size();
-            if (info->ultimate) info2->ultimate = false;
-            info = info2;
-
-            StringSource source(*sink.s);
-            dstStore->addToStore(*info, source, repair, checkSigs);
-            return;
-        }
+    auto srcUri = srcStore->getUri();
+    auto dstUri = dstStore->getUri();
+
+    Activity act(*logger, lvlInfo, actCopyPath,
+        srcUri == "local" || srcUri == "daemon"
+          ? fmt("copying path '%s' to '%s'", storePath, dstUri)
+          : dstUri == "local" || dstUri == "daemon"
+            ? fmt("copying path '%s' from '%s'", storePath, srcUri)
+            : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
+        {storePath, srcUri, dstUri});
+    PushActivity pact(act.id);
+
+    auto info = srcStore->queryPathInfo(storePath);
+
+    uint64_t total = 0;
+
+    if (!info->narHash) {
+        StringSink sink;
+        srcStore->narFromPath({storePath}, sink);
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->narHash = hashString(htSHA256, *sink.s);
+        if (!info->narSize) info2->narSize = sink.s->size();
+        if (info->ultimate) info2->ultimate = false;
+        info = info2;
+
+        StringSource source(*sink.s);
+        dstStore->addToStore(*info, source, repair, checkSigs);
+        return;
+    }
 
-        if (info->ultimate) {
-            auto info2 = make_ref<ValidPathInfo>(*info);
-            info2->ultimate = false;
-            info = info2;
-        }
+    if (info->ultimate) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->ultimate = false;
+        info = info2;
+    }
 
-        auto source = sinkToSource([&](Sink & sink) {
-            LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
-                sink(data, len);
-                total += len;
-                act.progress(total, info->narSize);
-            });
-            srcStore->narFromPath({storePath}, wrapperSink);
-        }, [&]() {
-            throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
+    auto source = sinkToSource([&](Sink & sink) {
+        LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
+            sink(data, len);
+            total += len;
+            act.progress(total, info->narSize);
         });
-
-        dstStore->addToStore(*info, *source, repair, checkSigs);
+        srcStore->narFromPath({storePath}, wrapperSink);
+    }, [&]() {
+        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
     });
+
+    dstStore->addToStore(*info, *source, repair, checkSigs);
 }
 
 
diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh
deleted file mode 100644
index b45cb37f736..00000000000
--- a/src/libutil/retry.hh
+++ /dev/null
@@ -1,38 +0,0 @@
-#pragma once
-
-#include "logging.hh"
-
-#include <functional>
-#include <cmath>
-#include <random>
-#include <thread>
-
-namespace nix {
-
-inline unsigned int retrySleepTime(unsigned int attempt)
-{
-    std::random_device rd;
-    std::mt19937 mt19937;
-    return 250.0 * std::pow(2.0f,
-        attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937));
-}
-
-template<typename C>
-C retry(unsigned int attempts, std::function<C()> && f)
-{
-    unsigned int attempt = 0;
-    while (true) {
-        try {
-            return f();
-        } catch (BaseError & e) {
-            ++attempt;
-            if (attempt >= attempts || !e.isTransient())
-                throw;
-            auto ms = retrySleepTime(attempt);
-            warn("%s; retrying in %d ms", e.what(), ms);
-            std::this_thread::sleep_for(std::chrono::milliseconds(ms));
-        }
-    }
-}
-
-}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 88e3243f47a..92bf469b5c6 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -109,8 +109,6 @@ public:
     const string & msg() const { return err; }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
-
-    virtual bool isTransient() { return false; }
 };
 
 #define MakeError(newClass, superClass) \

From f76b2a7fdd7dabad7891400248f5cc778182ebf3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 Jul 2019 22:27:50 +0200
Subject: [PATCH 1478/2196] Downloader: Use warn()

---
 src/libstore/download.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 2267b5d3517..195eccc2a12 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -406,7 +406,7 @@ struct CurlDownloader : public Downloader
                    download after a while. */
                 if (err == Transient && attempt < request.tries) {
                     int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    printError(format("warning: %s; retrying in %d ms") % exc.what() % ms);
+                    warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     downloader.enqueueItem(shared_from_this());
                 }
@@ -579,7 +579,7 @@ struct CurlDownloader : public Downloader
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            printError(format("unexpected error in download thread: %s") % e.what());
+            printError("unexpected error in download thread: %s", e.what());
         }
 
         {
@@ -869,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached(
             writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
         } catch (DownloadError & e) {
             if (storePath.empty()) throw;
-            printError(format("warning: %1%; using cached result") % e.msg());
+            warn("warning: %s; using cached result", e.msg());
             result.etag = expectedETag;
         }
     }

From 00f6fafad61db4537268e1ffa636fd0d2ae86059 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 Jul 2019 23:05:04 +0200
Subject: [PATCH 1479/2196] HttpBinaryCacheStore: Use default number of retries
 for NARs

---
 src/libstore/http-binary-cache-store.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 3d17a89660e..df2fb93320f 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -113,7 +113,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     DownloadRequest makeRequest(const std::string & path)
     {
         DownloadRequest request(cacheUri + "/" + path);
-        request.tries = 8;
         return request;
     }
 

From 53247d6b116905e7233b1efd6c14845e20d27442 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 10 Jul 2019 23:12:17 +0200
Subject: [PATCH 1480/2196] Resume NAR downloads

This is a much simpler fix to the 'error 9 while decompressing xz
file' problem than 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91. We just
do a ranged HTTP request starting after the data that we previously
wrote into the sink.

Fixes #2952, #379.
---
 src/libstore/download.cc | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 195eccc2a12..91087eebcfc 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -71,6 +71,10 @@ struct CurlDownloader : public Downloader
 
         std::string encoding;
 
+        bool acceptRanges = false;
+
+        curl_off_t writtenToSink = 0;
+
         DownloadItem(CurlDownloader & downloader,
             const DownloadRequest & request,
             Callback<DownloadResult> callback)
@@ -81,9 +85,10 @@ struct CurlDownloader : public Downloader
                 {request.uri}, request.parentAct)
             , callback(callback)
             , finalSink([this](const unsigned char * data, size_t len) {
-                if (this->request.dataCallback)
+                if (this->request.dataCallback) {
+                    writtenToSink += len;
                     this->request.dataCallback((char *) data, len);
-                else
+                } else
                     this->result.data->append((char *) data, len);
               })
         {
@@ -161,6 +166,7 @@ struct CurlDownloader : public Downloader
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
                 result.bodySize = 0;
+                acceptRanges = false;
                 encoding = "";
             } else {
                 auto i = line.find(':');
@@ -178,7 +184,9 @@ struct CurlDownloader : public Downloader
                             return 0;
                         }
                     } else if (name == "content-encoding")
-                        encoding = trim(string(line, i + 1));;
+                        encoding = trim(string(line, i + 1));
+                    else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
+                        acceptRanges = true;
                 }
             }
             return realSize;
@@ -296,6 +304,9 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
+            if (writtenToSink)
+                curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
+
             result.data = std::make_shared<std::string>();
             result.bodySize = 0;
         }
@@ -330,7 +341,7 @@ struct CurlDownloader : public Downloader
                 failEx(writeException);
 
             else if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
@@ -403,10 +414,20 @@ struct CurlDownloader : public Downloader
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
                 /* If this is a transient error, then maybe retry the
-                   download after a while. */
-                if (err == Transient && attempt < request.tries) {
+                   download after a while. If we're writing to a
+                   sink, we can only retry if the server supports
+                   ranged requests. */
+                if (err == Transient
+                    && attempt < request.tries
+                    && (!this->request.dataCallback
+                        || writtenToSink == 0
+                        || (acceptRanges && encoding.empty())))
+                {
                     int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    warn("%s; retrying in %d ms", exc.what(), ms);
+                    if (writtenToSink)
+                        warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
+                    else
+                        warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     downloader.enqueueItem(shared_from_this());
                 }

From 5e0a64229b3d244d46b3b5f9fae964d01958c15e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 18 Jul 2019 10:57:26 +0200
Subject: [PATCH 1481/2196] Add Open Collective

---
 .github/FUNDING.yml | 2 +-
 README.md           | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
index bbaabf93c7a..15792e7132e 100644
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,3 +1,3 @@
 # These are supported funding model platforms
 
-custom: https://nixos.org/nixos/foundation.html
+open_collective: nixos
diff --git a/README.md b/README.md
index 3173c6c448a..48cb1685c7b 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
+
 Nix, the purely functional package manager
 ------------------------------------------
 

From cf6172f05e9446b2b85dd7fa2e73cb93f56620e2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 28 Mar 2019 16:04:02 -0400
Subject: [PATCH 1482/2196] docs: document balancing cores and max-jobs

---
 .../advanced-topics/advanced-topics.xml       |   1 +
 doc/manual/advanced-topics/cores-vs-jobs.xml  | 121 ++++++++++++++++++
 doc/manual/command-ref/conf-file.xml          |   8 +-
 3 files changed, 128 insertions(+), 2 deletions(-)
 create mode 100644 doc/manual/advanced-topics/cores-vs-jobs.xml

diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml
index c304367aaf8..12a82625429 100644
--- a/doc/manual/advanced-topics/advanced-topics.xml
+++ b/doc/manual/advanced-topics/advanced-topics.xml
@@ -7,6 +7,7 @@
 <title>Advanced Topics</title>
 
 <xi:include href="distributed-builds.xml" />
+<xi:include href="cores-vs-jobs.xml" />
 <xi:include href="diff-hook.xml" />
 
 </part>
diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml
new file mode 100644
index 00000000000..eba645faf87
--- /dev/null
+++ b/doc/manual/advanced-topics/cores-vs-jobs.xml
@@ -0,0 +1,121 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="chap-tuning-cores-and-jobs">
+
+<title>Tuning Cores and Jobs</title>
+
+<para>Nix has two relevant settings with regards to how your CPU cores
+will be utilized: <xref linkend="conf-cores" /> and
+<xref linkend="conf-max-jobs" />. This chapter will talk about what
+they are, how they interact, and their configuration trade-offs.</para>
+
+<variablelist>
+  <varlistentry>
+    <term><xref linkend="conf-max-jobs" /></term>
+    <listitem><para>
+      Dictates how many separate derivations will be built at the same
+      time. If you set this to zero, the local machine will do no
+      builds. Nix will still substitute from binary caches, and build
+      remotely if remote builders are configured.
+    </para></listitem>
+  </varlistentry>
+  <varlistentry>
+    <term><xref linkend="conf-cores" /></term>
+    <listitem><para>
+      Suggests how many cores each derivation should use. Similar to
+      <command>make -j</command>.
+    </para></listitem>
+  </varlistentry>
+</variablelist>
+
+<para>The <xref linkend="conf-cores" /> setting determines the value of
+<envar>NIX_BUILD_CORES</envar>. <envar>NIX_BUILD_CORES</envar> is equal
+to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
+equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
+will be the total number of cores in the system.</para>
+
+<para>The total number of consumed cores is a simple multiplication,
+<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+
+<para>The balance on how to set these two independent variables depends
+upon each builder's workload and hardware. Here are a few example
+scenarios on a machine with 24 cores:</para>
+
+<table>
+  <caption>Balancing 24 Build Cores</caption>
+  <thead>
+    <tr>
+      <th><xref linkend="conf-max-jobs" /></th>
+      <th><xref linkend="conf-cores" /></th>
+      <th><envar>NIX_BUILD_CORES</envar></th>
+      <th>Maximum Processes</th>
+      <th>Result</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>1</td>
+      <td>24</td>
+      <td>24</td>
+      <td>24</td>
+      <td>
+        One derivation will be built at a time, each one can use 24
+        cores. Undersold if a job can’t use 24 cores.
+      </td>
+    </tr>
+
+    <tr>
+      <td>4</td>
+      <td>6</td>
+      <td>6</td>
+      <td>24</td>
+      <td>
+        Four derivations will be built at once, each given access to
+        six cores.
+      </td>
+    </tr>
+    <tr>
+      <td>12</td>
+      <td>6</td>
+      <td>6</td>
+      <td>72</td>
+      <td>
+        12 derivations will be built at once, each given access to six
+        cores. This configuration is over-sold. If all 12 derivations
+        being built simultaneously try to use all six cores, the
+        machine's performance will be degraded due to extensive context
+        switching between the 12 builds.
+      </td>
+    </tr>
+    <tr>
+      <td>24</td>
+      <td>1</td>
+      <td>1</td>
+      <td>24</td>
+      <td>
+        24 derivations can build at the same time, each using a single
+        core. Never oversold, but derivations which require many cores
+        will be very slow to compile.
+      </td>
+    </tr>
+    <tr>
+      <td>24</td>
+      <td>0</td>
+      <td>24</td>
+      <td>576</td>
+      <td>
+        24 derivations can build at the same time, each using all the
+        available cores of the machine. Very likely to be oversold,
+        and very likely to suffer context switches.
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+<para>It is up to the derivations' build script to respect
+host's requested cores-per-build by following the value of the
+<envar>NIX_BUILD_CORES</envar> environment variable.</para>
+
+</chapter>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 09aad2e0553..4f7393e2e7f 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -238,8 +238,9 @@ false</literal>.</para>
     linkend='opt-cores'>--cores</option> command line switch and
     defaults to <literal>1</literal>.  The value <literal>0</literal>
     means that the builder should use all available CPU cores in the
-    system.</para></listitem>
+    system.</para>
 
+    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term>
@@ -498,7 +499,10 @@ builtins.fetchurl {
     regardless).  It can be
     overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
-    command line switch.</para></listitem>
+    command line switch.</para>
+
+    <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para>
+    </listitem>
   </varlistentry>
 
   <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term>

From 9031a6838ccf10b0db0d91a89e14d71a1adf6235 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Tue, 23 Jul 2019 15:21:09 +0200
Subject: [PATCH 1483/2196] Remove .github/FUNDING.yml

The configuration is now done through the shared configuration repo:

https://github.com/nixos/.github
---
 .github/FUNDING.yml | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 .github/FUNDING.yml

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
deleted file mode 100644
index 15792e7132e..00000000000
--- a/.github/FUNDING.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-# These are supported funding model platforms
-
-open_collective: nixos

From c82a856b36bcd008edeb9c4c981b974ae642afea Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 25 Jul 2019 09:39:44 -0400
Subject: [PATCH 1484/2196] Add default for USER when unset

uses $(id -u -n) when USER is unset, this is needed on some weird
setups in Docker. Fixes #971
---
 scripts/install-nix-from-closure.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index fc999d336d1..35926f3dac9 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then
     echo "$0: incomplete installer (.reginfo is missing)" >&2
 fi
 
-if [ -z "$USER" ]; then
+if [ -z "$USER" ] && ! USER=$(id -u -n); then
     echo "$0: \$USER is not set" >&2
     exit 1
 fi

From 03addc3b0a79cb7a58ae5651d1915af7383a220c Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 25 Jul 2019 09:43:24 -0400
Subject: [PATCH 1485/2196] Use $HOME instead of $USER

$USER/.nix-profile will not be a path. I think $HOME/.nix-profile was
the origininal intent.

/cc @Grahamc
---
 tests/install-darwin.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh
index c99ce84acab..9933eba9443 100755
--- a/tests/install-darwin.sh
+++ b/tests/install-darwin.sh
@@ -34,7 +34,7 @@ cleanup() {
     sudo rm -rf /etc/nix \
          /nix \
          /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \
-         "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels"
+         "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels"
 }
 
 verify() {

From d171090530f4a2a79efec2c385bee1a10844c706 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 25 Jul 2019 09:37:57 -0400
Subject: [PATCH 1486/2196] =?UTF-8?q?Disable=20CLONE=5FNEWUSER=20when=20it?=
 =?UTF-8?q?=E2=80=99s=20unavailable?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some kernels disable "unpriveleged user namespaces". This is
unfortunate, but we can still use mount namespaces. Anyway, since each
builder has its own nixbld user, we already have most of the benefits
of user namespaces.
---
 src/libstore/build.cc | 14 ++++++++++++--
 src/nix/run.cc        |  5 ++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf6428e1246..c10005839bf 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2302,10 +2302,20 @@ void DerivationGoal::startBuilder()
                 flags |= CLONE_NEWNET;
 
             pid_t child = clone(childEntry, stack + stackSize, flags, this);
-            if (child == -1 && errno == EINVAL)
+            if (child == -1 && errno == EINVAL) {
                 /* Fallback for Linux < 2.13 where CLONE_NEWPID and
                    CLONE_PARENT are not allowed together. */
-                child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this);
+                flags &= ~CLONE_NEWPID;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
+            if (child == -1 && (errno == EPERM || errno == EINVAL)) {
+                /* Some distros patch Linux to not allow unpriveleged
+                 * user namespaces. If we get EPERM or EINVAL, try
+                 * without CLONE_NEWUSER and see if that works.
+                 */
+                flags &= ~CLONE_NEWUSER;
+                child = clone(childEntry, stack + stackSize, flags, this);
+            }
             if (child == -1) throw SysError("cloning builder process");
 
             writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 35b76334587..90b76d6663e 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -199,7 +199,10 @@ void chrootHelper(int argc, char * * argv)
     uid_t gid = getgid();
 
     if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
-        throw SysError("setting up a private mount namespace");
+        /* Try with just CLONE_NEWNS in case user namespaces are
+           specifically disabled. */
+        if (unshare(CLONE_NEWNS) == -1)
+            throw SysError("setting up a private mount namespace");
 
     /* Bind-mount realStoreDir on /nix/store. If the latter mount
        point doesn't already exists, we have to create a chroot

From 11d853462925d0b57fe956962e07edf5751fd4c3 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 25 Jul 2019 14:29:58 -0400
Subject: [PATCH 1487/2196] Use sandbox fallback when cloning fails in builder
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When sandbox-fallback = true (the default), the Nix builder will fall
back to disabled sandbox mode when the kernel doesn’t allow users to
set it up. This prevents hard errors from occuring in tricky places,
especially the initial installer. To restore the previous behavior,
users can set:

  sandbox-fallback = false

in their /etc/nix/nix.conf configuration.
---
 src/libstore/build.cc   | 12 +++++++++++-
 src/libstore/globals.hh |  3 +++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index c10005839bf..dd08ce7d771 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2316,13 +2316,22 @@ void DerivationGoal::startBuilder()
                 flags &= ~CLONE_NEWUSER;
                 child = clone(childEntry, stack + stackSize, flags, this);
             }
+            /* Otherwise exit with EPERM so we can handle this in the
+               parent. This is only done when sandbox-fallback is set
+               to true (the default). */
+            if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
+                _exit(EPERM);
             if (child == -1) throw SysError("cloning builder process");
 
             writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
             _exit(0);
         }, options);
 
-        if (helper.wait() != 0)
+        int res = helper.wait();
+        if (res == EPERM && settings.sandboxFallback) {
+            useChroot = false;
+            goto fallback;
+        } else if (res != 0)
             throw Error("unable to start build process");
 
         userNamespaceSync.readSide = -1;
@@ -2353,6 +2362,7 @@ void DerivationGoal::startBuilder()
     } else
 #endif
     {
+    fallback:
         options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0af8215d1fd..cc9534b2794 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -209,6 +209,9 @@ public:
         "The paths to make available inside the build sandbox.",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
+    Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
+        "Whether to disable sandboxing when the kernel doesn't allow it."};
+
     Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
         "Additional paths to make available inside the build sandbox.",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};

From cd933b22d2041b7efc348dcc09ff255967ffc663 Mon Sep 17 00:00:00 2001
From: Tom McLaughlin <pyro777@gmail.com>
Date: Sat, 27 Jul 2019 19:40:19 -0700
Subject: [PATCH 1488/2196] Add pname and version to nix-env -q --json

---
 src/nix-env/nix-env.cc | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 56ed75daee4..87b2e43f063 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -860,7 +860,10 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
     for (auto & i : elems) {
         JSONObject pkgObj = topObj.object(i.attrPath);
 
-        pkgObj.attr("name", i.queryName());
+        auto drvName = DrvName(i.queryName());
+        pkgObj.attr("name", drvName.fullName);
+        pkgObj.attr("pname", drvName.name);
+        pkgObj.attr("version", drvName.version);
         pkgObj.attr("system", i.querySystem());
 
         JSONObject metaObj = pkgObj.object("meta");
@@ -1026,10 +1029,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             else if (printAttrPath)
                 columns.push_back(i.attrPath);
 
-            if (xmlOutput)
-                attrs["name"] = i.queryName();
-            else if (printName)
+            if (xmlOutput) {
+                auto drvName = DrvName(i.queryName());
+                attrs["name"] = drvName.fullName;
+                attrs["pname"] = drvName.name;
+                attrs["version"] = drvName.version;
+            } else if (printName) {
                 columns.push_back(i.queryName());
+            }
 
             if (compareVersions) {
                 /* Compare this element against the versions of the

From 89865144c3ba0162cd37bcbe49b3095e9bab4164 Mon Sep 17 00:00:00 2001
From: Bas van Dijk <v.dijk.bas@gmail.com>
Date: Tue, 30 Jul 2019 11:22:55 +0200
Subject: [PATCH 1489/2196] Allow builtins.pathExists to check the existence of
 /nix/store paths

This makes it consitent with builtins.readDir.
---
 src/libexpr/primops.cc      | 10 ++++++++--
 tests/import-derivation.nix |  5 ++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 070e72f3a96..350dba47409 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -832,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[0], context);
-    if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+    try {
+        state.realiseContext(context);
+    } catch (InvalidPathError & e) {
+        throw EvalError(format(
+                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%")
+            % path % e.path % pos);
+    }
+
     try {
         mkBool(v, pathExists(state.checkSourcePath(path)));
     } catch (SysError & e) {
diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix
index 91adcd288f6..44fa9a45d7e 100644
--- a/tests/import-derivation.nix
+++ b/tests/import-derivation.nix
@@ -10,7 +10,10 @@ let
       '';
   };
 
-  value = import bar;
+  value =
+    # Test that pathExists can check the existence of /nix/store paths
+    assert builtins.pathExists bar;
+    import bar;
 
 in
 

From ee1e3132cab9d7de14837f2b04b115e544ae1622 Mon Sep 17 00:00:00 2001
From: Bas van Dijk <v.dijk.bas@gmail.com>
Date: Tue, 30 Jul 2019 11:29:03 +0200
Subject: [PATCH 1490/2196] Disable findRuntimeRoots on darwin when running
 tests because lsof is slow

See: https://github.com/NixOS/nix/issues/3011
---
 src/libstore/gc.cc | 25 +++++++++++++++----------
 tests/common.sh.in |  1 +
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 26e2b0dca7c..83cdf13a6a6 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -444,17 +444,22 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
     }
 
 #if !defined(__linux__)
-    try {
-        std::regex lsofRegex(R"(^n(/.*)$)");
-        auto lsofLines =
-            tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
-        for (const auto & line : lsofLines) {
-            std::smatch match;
-            if (std::regex_match(line, match, lsofRegex))
-                unchecked[match[1]].emplace("{lsof}");
+    // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
+    // See: https://github.com/NixOS/nix/issues/3011
+    // Because of this we disable lsof when running the tests.
+    if (getEnv("_NIX_TEST_NO_LSOF") == "") {
+        try {
+            std::regex lsofRegex(R"(^n(/.*)$)");
+            auto lsofLines =
+                tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
+            for (const auto & line : lsofLines) {
+                std::smatch match;
+                if (std::regex_match(line, match, lsofRegex))
+                    unchecked[match[1]].emplace("{lsof}");
+            }
+        } catch (ExecError & e) {
+            /* lsof not installed, lsof failed */
         }
-    } catch (ExecError & e) {
-        /* lsof not installed, lsof failed */
     }
 #endif
 
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 6a523ca9d83..15d7b1ef911 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -16,6 +16,7 @@ if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
 fi
 export _NIX_IN_TEST=$TEST_ROOT/shared
+export _NIX_TEST_NO_LSOF=1
 export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home

From 9a0855bbb6546e792848e551e79f8efc40782eeb Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 30 Jul 2019 17:52:42 -0400
Subject: [PATCH 1491/2196] =?UTF-8?q?Don=E2=80=99t=20rely=20on=20EPERM?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

startProcess does not appear to send the exit code to the helper
correctly. Not sure why this is, but it is probably safe to just
fallback on all sandbox errors.
---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dd08ce7d771..0f71e751151 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2320,7 +2320,7 @@ void DerivationGoal::startBuilder()
                parent. This is only done when sandbox-fallback is set
                to true (the default). */
             if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
-                _exit(EPERM);
+                _exit(1);
             if (child == -1) throw SysError("cloning builder process");
 
             writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n");
@@ -2328,7 +2328,7 @@ void DerivationGoal::startBuilder()
         }, options);
 
         int res = helper.wait();
-        if (res == EPERM && settings.sandboxFallback) {
+        if (res != 0 && settings.sandboxFallback) {
             useChroot = false;
             goto fallback;
         } else if (res != 0)

From 320126aeebc89f8c34ee1668ebacf1fe4b512ed8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 2 Aug 2019 14:04:09 +0200
Subject: [PATCH 1492/2196] Tweak min-free/max-free descriptions

---
 doc/manual/command-ref/conf-file.xml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 4f7393e2e7f..0a97719ef06 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -483,8 +483,10 @@ builtins.fetchurl {
 
   <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term>
 
-    <listitem><para>This option defines after how many free bytes to stop collecting
-    garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem>
+    <listitem><para>When a garbage collection is triggered by the
+    <literal>min-free</literal> option, it stops as soon as
+    <literal>max-free</literal> bytes are available. The default is
+    infinity (i.e. delete all garbage).</para></listitem>
 
   </varlistentry>
 
@@ -528,9 +530,11 @@ builtins.fetchurl {
   <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term>
 
     <listitem>
-      <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix
-        will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk.
-        A value of <literal>0</literal> (the default) means that this feature is disabled.</para>
+      <para>When free disk space in <filename>/nix/store</filename>
+      drops below <literal>min-free</literal> during a build, Nix
+      performs a garbage-collection until <literal>max-free</literal>
+      bytes are available or there is no more garbage.  A value of
+      <literal>0</literal> (the default) disables this feature.</para>
     </listitem>
 
   </varlistentry>

From 7c5596734f65b30b250ea73a423b40a4ce513fdf Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Thu, 11 Jul 2019 20:23:03 +0200
Subject: [PATCH 1493/2196] Add a post-build-hook

Passing `--post-build-hook /foo/bar` to a nix-* command will cause
`/foo/bar` to be executed after each build with the following
environment variables set:

    DRV_PATH=/nix/store/drv-that-has-been-built.drv
    OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev

This can be useful in particular to upload all the builded artifacts to
the cache (including the ones that don't appear in the runtime closure
of the final derivation or are built because of IFD).

This new feature prints the stderr/stdout output to the `nix-build`
and `nix build` client, and the output is printed in a Nix 2
compatible format:

    [nix]$ ./inst/bin/nix-build ./test.nix
    these derivations will be built:
      /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv
    building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'...
    hello!
    bye!
    running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation
    post-build-hook: + sleep 1
    post-build-hook: + printf 'very important stuff'
    /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation

    [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix
    my-example-derivation> hello!
    my-example-derivation> bye!
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation
    my-example-derivation (post)> + sleep 1
    my-example-derivation (post)> + printf 'very important stuff'
    [1 built, 0.0 MiB DL]

Co-authored-by: Graham Christensen <graham@grahamc.com>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 .../advanced-topics/advanced-topics.xml       |   1 +
 .../advanced-topics/post-build-hook.xml       | 157 ++++++++++++++++++
 doc/manual/command-ref/conf-file.xml          |  55 ++++++
 src/libstore/build.cc                         |  55 ++++++
 src/libstore/globals.hh                       |   3 +
 src/libutil/logging.hh                        |   2 +
 src/libutil/util.cc                           |  23 ++-
 src/libutil/util.hh                           |   2 +
 src/nix/progress-bar.cc                       |  16 +-
 tests/dependencies.nix                        |   1 +
 tests/local.mk                                |   3 +-
 tests/post-hook.sh                            |  15 ++
 tests/push_to_store.sh                        |   4 +
 13 files changed, 333 insertions(+), 4 deletions(-)
 create mode 100644 doc/manual/advanced-topics/post-build-hook.xml
 create mode 100644 tests/post-hook.sh
 create mode 100755 tests/push_to_store.sh

diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml
index c304367aaf8..1b8841ad21c 100644
--- a/doc/manual/advanced-topics/advanced-topics.xml
+++ b/doc/manual/advanced-topics/advanced-topics.xml
@@ -8,5 +8,6 @@
 
 <xi:include href="distributed-builds.xml" />
 <xi:include href="diff-hook.xml" />
+<xi:include href="post-build-hook.xml" />
 
 </part>
diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
new file mode 100644
index 00000000000..4335b308b9d
--- /dev/null
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -0,0 +1,157 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      xml:id="chap-post-build-hook"
+      version="5.0"
+      >
+
+<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
+
+
+<section xml:id="chap-post-build-hook-caveats">
+  <title>Implementation Caveats</title>
+  <para>Here we use the post-build hook to upload to a binary cache.
+  This is a simple and working example, but it is not suitable for all
+  use cases.</para>
+
+  <para>The post build hook program runs after each executed build,
+  and blocks the build loop. The build loop exits if the hook program
+  fails.</para>
+
+  <para>Concretely, this implementation will make Nix slow or unusable
+  when the internet is slow or unreliable.</para>
+
+  <para>A more advanced implementation might pass the store paths to a
+  user-supplied daemon or queue for processing the store paths outside
+  of the build loop.</para>
+</section>
+
+<section>
+  <title>Prerequisites</title>
+
+  <para>
+    This tutorial assumes you have configured an S3-compatible binary cache
+    according to the instructions at
+    <xref linkend="ssec-s3-substituter-authenticated-writes" />, and
+    that the <literal>root</literal> user's default AWS profile can
+    upload to the bucket.
+  </para>
+</section>
+
+<section>
+  <title>Set up a Signing Key</title>
+  <para>Use <command>nix-store --generate-binary-cache-key</command> to
+  create our public and private signing keys. We will sign paths
+  with the private key, and distribute the public key for verifying
+  the authenticity of the paths.</para>
+
+  <screen>
+# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
+# cat /etc/nix/key.public
+example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</screen>
+
+<para>Then, add the public key and the cache URL to your
+<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" />
+and <xref linkend="conf-substituters" /> like:</para>
+
+<programlisting>
+substituters = https://cache.nixos.org/ s3://example-nix-cache
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
+</programlisting>
+
+<para>we will restart the Nix daemon a later step.</para>
+</section>
+
+<section>
+  <title>Implementing the build hook</title>
+  <para>Write the following script to
+  <filename>/etc/nix/upload-to-cache.sh</filename>:
+  </para>
+
+  <programlisting>
+#!/bin/sh
+
+set -eu
+
+echo "Signing paths" $OUT_PATHS
+nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+echo "Uploading paths" $OUT_PATHS
+exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
+</programlisting>
+
+  <note>
+    <title>Should <literal>$OUT_PATHS</literal> be quoted?</title>
+    <para>
+      The <literal>$OUT_PATHS</literal> variable is a space-separated
+      list of Nix store paths. In this case, we expect and want the
+      shell to perform word splitting to make each output path its
+      own argument to <command>nix sign-paths</command>. Nix guarantees
+      the paths will only contain characters which are safe for word
+      splitting, and free of any globs.
+    </para>
+  </note>
+  <para>
+    Then make sure the hook program is executable by the <literal>root</literal> user:
+    <screen>
+# chmod +x /etc/nix/upload-to-cache.sh
+</screen></para>
+</section>
+
+<section>
+  <title>Updating Nix Configuration</title>
+
+  <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook,
+  by adding the following configuration snippet at the end:</para>
+
+  <programlisting>
+post-build-hook = /etc/nix/upload-to-cache.sh
+</programlisting>
+
+<para>Then, restart the <command>nix-daemon</command>.</para>
+</section>
+
+<section>
+  <title>Testing</title>
+
+  <para>Build any derivation, for example:</para>
+
+  <screen>
+$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
+these derivations will be built:
+  /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
+building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
+running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
+post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+  <para>Then delete the path from the store, and try substituting it from the binary cache:</para>
+  <screen>
+$ rm ./result
+$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+</screen>
+
+<para>Now, copy the path back from the cache:</para>
+<screen>
+$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
+warning: you did not specify '--add-root'; the result might be removed by the garbage collector
+/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
+</screen>
+</section>
+<section>
+  <title>Conclusion</title>
+  <para>
+    We now have a Nix installation configured to automatically sign and
+    upload every local build to a remote binary cache.
+  </para>
+
+  <para>
+    Before deploying this to production, be sure to consider the
+    implementation caveats in <xref linkend="chap-post-build-hook-caveats" />.
+  </para>
+</section>
+</chapter>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 09aad2e0553..d2c9c7502eb 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -656,6 +656,61 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-post-build-hook">
+    <term><literal>post-build-hook</literal></term>
+    <listitem>
+      <para>Optional. The path to a program to execute after each build.</para>
+
+      <para>This option is only settable in the global
+      <filename>nix.conf</filename>, or on the command line by trusted
+      users.</para>
+
+      <para>When using the nix-daemon, the daemon executes the hook as
+      <literal>root</literal>. If the nix-daemon is not involved, the
+      hook runs as the user executing the nix-build.</para>
+
+      <itemizedlist>
+        <listitem><para>The hook executes after an evaluation-time build.</para></listitem>
+        <listitem><para>The hook does not execute on substituted paths.</para></listitem>
+        <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
+        <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
+      </itemizedlist>
+
+      <para>The program executes with no arguments. The program's environment
+      contains the following environment variables:</para>
+
+      <variablelist>
+        <varlistentry>
+          <term><envar>DRV_PATH</envar></term>
+          <listitem>
+            <para>The derivation for the built paths.</para>
+            <para>Example:
+            <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
+            </para>
+          </listitem>
+        </varlistentry>
+
+        <varlistentry>
+          <term><envar>OUT_PATHS</envar></term>
+          <listitem>
+            <para>Output paths of the built derivation, separated by a space (<literal> </literal>) character.</para>
+            <para>Example:
+            <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+            /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+            /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+            /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+            /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+
+      <para>See <xref linkend="chap-post-build-hook" /> for an example
+      implementation.</para>
+
+    </listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
 
     <listitem><para>How many times to repeat builds to check whether
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf6428e1246..7494eec419e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{drvPath});
+            PushActivity pact(act.id);
+            auto outputPaths = drv->outputPaths();
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", drvPath);
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (const unsigned char * data, size_t len) override {
+                    for (size_t i = 0; i < len; i++) {
+                        auto c = data[i];
+
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    if (settings.verboseBuild) {
+                        printError("post-build-hook: " + currentLine);
+                    } else {
+                        act.result(resPostBuildLogLine, currentLine);
+                    }
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
+
         if (buildMode == bmCheck) {
             done(BuildResult::Built);
             return;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0af8215d1fd..7dea6892143 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -315,6 +315,9 @@ public:
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 
+    Setting<std::string> postBuildHook{this, "", "post-build-hook",
+        "A program to run just after each succesful build."};
+
     Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         "Path to the netrc file used to obtain usernames/passwords for downloads."};
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5f221944594..5df03da74e0 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -26,6 +26,7 @@ typedef enum {
     actVerifyPaths = 107,
     actSubstitute = 108,
     actQueryPathInfo = 109,
+    actPostBuildHook = 110,
 } ActivityType;
 
 typedef enum {
@@ -36,6 +37,7 @@ typedef enum {
     resSetPhase = 104,
     resProgress = 105,
     resSetExpected = 106,
+    resPostBuildLogLine = 107,
 } ResultType;
 
 typedef uint64_t ActivityId;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 17aee2d5c3d..44fa7248255 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -84,6 +84,15 @@ void clearEnv()
         unsetenv(name.first.c_str());
 }
 
+void replaceEnv(std::map<std::string, std::string> newEnv)
+{
+    clearEnv();
+    for (auto newEnvVar : newEnv)
+    {
+        setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
+    }
+}
+
 
 Path absPath(Path path, Path dir)
 {
@@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options)
     if (options.standardOut) out.create();
     if (source) in.create();
 
+    ProcessOptions processOptions;
+    // vfork implies that the environment of the main process and the fork will
+    // be shared (technically this is undefined, but in practice that's the
+    // case), so we can't use it if we alter the environment
+    if (options.environment)
+        processOptions.allowVfork = false;
+
     /* Fork. */
     Pid pid = startProcess([&]() {
+        if (options.environment)
+            replaceEnv(*options.environment);
         if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
+        if (options.mergeStderrToStdout)
+            if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+                throw SysError("cannot dup stdout into stderr");
         if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
 
@@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options)
             execv(options.program.c_str(), stringsToCharPtrs(args_).data());
 
         throw SysError("executing '%1%'", options.program);
-    });
+    }, processOptions);
 
     out.writeSide = -1;
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fce3cab8def..b538a0b41ce 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -270,12 +270,14 @@ struct RunOptions
     std::optional<uid_t> uid;
     std::optional<uid_t> gid;
     std::optional<Path> chdir;
+    std::optional<std::map<std::string, std::string>> environment;
     Path program;
     bool searchPath = true;
     Strings args;
     std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
+    bool mergeStderrToStdout = false;
     bool _killStderr = false;
 
     RunOptions(const Path & program, const Strings & args)
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index b1c1d87de1a..c0bcfb0c91b 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -170,6 +170,14 @@ class ProgressBar : public Logger
                 name, sub);
         }
 
+        if (type == actPostBuildHook) {
+            auto name = storePathToName(getS(fields, 0));
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
+            i->name = DrvName(name).name;
+        }
+
         if (type == actQueryPathInfo) {
             auto name = storePathToName(getS(fields, 0));
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
@@ -228,14 +236,18 @@ class ProgressBar : public Logger
             update(*state);
         }
 
-        else if (type == resBuildLogLine) {
+        else if (type == resBuildLogLine || type == resPostBuildLogLine) {
             auto lastLine = trim(getS(fields, 0));
             if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
                 ActInfo info = *i->second;
                 if (printBuildLogs) {
-                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
+                    auto suffix = "> ";
+                    if (type == resPostBuildLogLine) {
+                        suffix = " (post)> ";
+                    }
+                    log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine);
                 } else {
                     state->activities.erase(i->second);
                     info.lastLine = lastLine;
diff --git a/tests/dependencies.nix b/tests/dependencies.nix
index 687237add82..eca4b2964cf 100644
--- a/tests/dependencies.nix
+++ b/tests/dependencies.nix
@@ -17,6 +17,7 @@ let {
     builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
     input1 = input1 + "/.";
     input2 = "${input2}/.";
+    input1_drv = input1;
     meta.description = "Random test package";
   };
 
diff --git a/tests/local.mk b/tests/local.mk
index 1ff68348b3c..8daaa859f9e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -26,7 +26,8 @@ nix_tests = \
   check.sh \
   plugins.sh \
   search.sh \
-  nix-copy-ssh.sh
+  nix-copy-ssh.sh \
+  post-hook.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/post-hook.sh b/tests/post-hook.sh
new file mode 100644
index 00000000000..3f82d0c5e66
--- /dev/null
+++ b/tests/post-hook.sh
@@ -0,0 +1,15 @@
+source common.sh
+
+clearStore
+
+export REMOTE_STORE=$TEST_ROOT/remote_store
+
+# Build the dependencies and push them to the remote store
+nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh
+
+clearStore
+
+# Ensure that we the remote store contains both the runtime and buildtime
+# closure of what we've just built
+nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix
+nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv
diff --git a/tests/push_to_store.sh b/tests/push_to_store.sh
new file mode 100755
index 00000000000..d97eb095dd7
--- /dev/null
+++ b/tests/push_to_store.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo Pushing "$@" to "$REMOTE_STORE"
+echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs

From ec415d7166d607c92cf8f1af688f86e4b4731dff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 2 Aug 2019 17:07:33 +0200
Subject: [PATCH 1494/2196] Add a test for auto-GC

This currently fails because we're using POSIX file locks. So when the
garbage collector opens and closes its own temproots file, it causes
the lock to be released and then deleted by another GC instance.
---
 src/libstore/gc.cc      |  9 +++++--
 src/libstore/globals.hh |  3 +++
 tests/gc-auto.sh        | 59 +++++++++++++++++++++++++++++++++++++++++
 tests/local.mk          |  4 ++-
 4 files changed, 72 insertions(+), 3 deletions(-)
 create mode 100644 tests/gc-auto.sh

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 83cdf13a6a6..eeb2378393a 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -871,7 +871,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void LocalStore::autoGC(bool sync)
 {
-    auto getAvail = [this]() {
+    static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
+
+    auto getAvail = [this]() -> uint64_t {
+        if (!fakeFreeSpaceFile.empty())
+            return std::stoll(readFile(fakeFreeSpaceFile));
+
         struct statvfs st;
         if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
@@ -892,7 +897,7 @@ void LocalStore::autoGC(bool sync)
 
         auto now = std::chrono::steady_clock::now();
 
-        if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
+        if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return;
 
         auto avail = getAvail();
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 0af8215d1fd..0c0a9ec544e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -342,6 +342,9 @@ public:
     Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
         "Stop deleting garbage when free disk space is above the specified amount."};
 
+    Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
+        "Number of seconds between checking free disk space."};
+
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 };
diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
new file mode 100644
index 00000000000..1e91282d0f9
--- /dev/null
+++ b/tests/gc-auto.sh
@@ -0,0 +1,59 @@
+source common.sh
+
+clearStore
+
+garbage1=$(nix add-to-store --name garbage1 ./tarball.sh)
+garbage2=$(nix add-to-store --name garbage2 ./tarball.sh)
+garbage3=$(nix add-to-store --name garbage3 ./tarball.sh)
+
+fake_free=$TEST_ROOT/fake-free
+export _NIX_TEST_FREE_SPACE_FILE=$fake_free
+echo 1100 > $fake_free
+
+expr=$(cat <<EOF
+with import ./config.nix; mkDerivation {
+  name = "gc-A";
+  buildCommand = ''
+    [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
+    mkdir \$out
+    echo foo > \$out/bar
+    echo 1...
+    sleep 2
+    echo 100 > $fake_free
+    echo 2...
+    sleep 2
+    echo 3...
+    [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
+  '';
+}
+EOF
+)
+
+nix build -o $TEST_ROOT/result-A -L "($expr)" \
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
+pid=$!
+
+expr2=$(cat <<EOF
+with import ./config.nix; mkDerivation {
+  name = "gc-B";
+  buildCommand = ''
+    mkdir \$out
+    echo foo > \$out/bar
+    echo 1...
+    sleep 2
+    echo 100 > $fake_free
+    echo 2...
+    sleep 2
+    echo 3...
+  '';
+}
+EOF
+)
+
+nix build -o $TEST_ROOT/result-B -L "($expr2)" \
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1
+
+wait "$pid"
+
+[[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
+[[ foo = $(cat $TEST_ROOT/result-B/bar) ]]
diff --git a/tests/local.mk b/tests/local.mk
index 1ff68348b3c..261e4db0de8 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -3,7 +3,9 @@ check:
 
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
-  gc.sh gc-concurrent.sh \
+  gc.sh \
+  gc-concurrent.sh \
+  gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh check-refs.sh filter-source.sh \
   remote-store.sh export.sh export-graph.sh \

From e349f2c0a370e4dfd09ae51c2cae4db08a834ad5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Oct 2018 15:03:03 +0200
Subject: [PATCH 1495/2196] Use BSD instead of POSIX file locks

POSIX file locks are essentially incompatible with multithreading. BSD
locks have much saner semantics. We need this now that there can be
multiple concurrent LocalStore::buildPaths() invocations.
---
 src/libstore/build.cc       |  17 -----
 src/libstore/gc.cc          |   4 +-
 src/libstore/local-store.hh |   2 +-
 src/libstore/pathlocks.cc   | 127 ++++++++++++------------------------
 src/libstore/pathlocks.hh   |   4 --
 5 files changed, 46 insertions(+), 108 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index cf6428e1246..a28619ab907 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3984,17 +3984,6 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
-    /* If the store path is already locked (probably by a
-       DerivationGoal), then put this goal to sleep. Note: we don't
-       acquire a lock here since that breaks addToStore(), so below we
-       handle an AlreadyLocked exception from addToStore(). The check
-       here is just an optimisation to prevent having to redo a
-       download due to a locked path. */
-    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -4034,12 +4023,6 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
-    } catch (AlreadyLocked & e) {
-        /* Probably a DerivationGoal is already building this store
-           path. Sleep for a while and try again. */
-        state = &SubstitutionGoal::init;
-        worker.waitForAWhile(shared_from_this());
-        return;
     } catch (std::exception & e) {
         printError(e.what());
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index eeb2378393a..2c791efbe67 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots";
    read.  To be precise: when they try to create a new temporary root
    file, they will block until the garbage collector has finished /
    yielded the GC lock. */
-int LocalStore::openGCLock(LockType lockType)
+AutoCloseFD LocalStore::openGCLock(LockType lockType)
 {
     Path fnGCLock = (format("%1%/%2%")
         % stateDir % gcLockName).str();
@@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType)
        process that can open the file for reading can DoS the
        collector. */
 
-    return fdGCLock.release();
+    return fdGCLock;
 }
 
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 6b655647b03..af8b84bf5d7 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -263,7 +263,7 @@ private:
     bool isActiveTempFile(const GCState & state,
         const Path & path, const string & suffix);
 
-    int openGCLock(LockType lockType);
+    AutoCloseFD openGCLock(LockType lockType);
 
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 08d1efdbeb0..b6d8547c70e 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -7,7 +7,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/file.h>
 
 
 namespace nix {
@@ -40,17 +40,14 @@ void deleteLockFile(const Path & path, int fd)
 
 bool lockFile(int fd, LockType lockType, bool wait)
 {
-    struct flock lock;
-    if (lockType == ltRead) lock.l_type = F_RDLCK;
-    else if (lockType == ltWrite) lock.l_type = F_WRLCK;
-    else if (lockType == ltNone) lock.l_type = F_UNLCK;
+    int type;
+    if (lockType == ltRead) type = LOCK_SH;
+    else if (lockType == ltWrite) type = LOCK_EX;
+    else if (lockType == ltNone) type = LOCK_UN;
     else abort();
-    lock.l_whence = SEEK_SET;
-    lock.l_start = 0;
-    lock.l_len = 0; /* entire file */
 
     if (wait) {
-        while (fcntl(fd, F_SETLKW, &lock) != 0) {
+        while (flock(fd, type) != 0) {
             checkInterrupt();
             if (errno != EINTR)
                 throw SysError(format("acquiring/releasing lock"));
@@ -58,9 +55,9 @@ bool lockFile(int fd, LockType lockType, bool wait)
                 return false;
         }
     } else {
-        while (fcntl(fd, F_SETLK, &lock) != 0) {
+        while (flock(fd, type | LOCK_NB) != 0) {
             checkInterrupt();
-            if (errno == EACCES || errno == EAGAIN) return false;
+            if (errno == EWOULDBLOCK) return false;
             if (errno != EINTR)
                 throw SysError(format("acquiring/releasing lock"));
         }
@@ -70,14 +67,6 @@ bool lockFile(int fd, LockType lockType, bool wait)
 }
 
 
-/* This enables us to check whether are not already holding a lock on
-   a file ourselves.  POSIX locks (fcntl) suck in this respect: if we
-   close a descriptor, the previous lock will be closed as well.  And
-   there is no way to query whether we already have a lock (F_GETLK
-   only works on locks held by other processes). */
-static Sync<StringSet> lockedPaths_;
-
-
 PathLocks::PathLocks()
     : deletePaths(false)
 {
@@ -91,7 +80,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
 }
 
 
-bool PathLocks::lockPaths(const PathSet & _paths,
+bool PathLocks::lockPaths(const PathSet & paths,
     const string & waitMsg, bool wait)
 {
     assert(fds.empty());
@@ -99,75 +88,54 @@ bool PathLocks::lockPaths(const PathSet & _paths,
     /* Note that `fds' is built incrementally so that the destructor
        will only release those locks that we have already acquired. */
 
-    /* Sort the paths.  This assures that locks are always acquired in
-       the same order, thus preventing deadlocks. */
-    Paths paths(_paths.begin(), _paths.end());
-    paths.sort();
-
-    /* Acquire the lock for each path. */
+    /* Acquire the lock for each path in sorted order. This ensures
+       that locks are always acquired in the same order, thus
+       preventing deadlocks. */
     for (auto & path : paths) {
         checkInterrupt();
         Path lockPath = path + ".lock";
 
         debug(format("locking path '%1%'") % path);
 
-        {
-            auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath)) {
-                if (!wait) return false;
-                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
-            }
-            lockedPaths->insert(lockPath);
-        }
-
-        try {
+        AutoCloseFD fd;
 
-            AutoCloseFD fd;
+        while (1) {
 
-            while (1) {
+            /* Open/create the lock file. */
+            fd = openLockFile(lockPath, true);
 
-                /* Open/create the lock file. */
-                fd = openLockFile(lockPath, true);
-
-                /* Acquire an exclusive lock. */
-                if (!lockFile(fd.get(), ltWrite, false)) {
-                    if (wait) {
-                        if (waitMsg != "") printError(waitMsg);
-                        lockFile(fd.get(), ltWrite, true);
-                    } else {
-                        /* Failed to lock this path; release all other
-                           locks. */
-                        unlock();
-                        lockedPaths_.lock()->erase(lockPath);
-                        return false;
-                    }
+            /* Acquire an exclusive lock. */
+            if (!lockFile(fd.get(), ltWrite, false)) {
+                if (wait) {
+                    if (waitMsg != "") printError(waitMsg);
+                    lockFile(fd.get(), ltWrite, true);
+                } else {
+                    /* Failed to lock this path; release all other
+                       locks. */
+                    unlock();
+                    return false;
                 }
-
-                debug(format("lock acquired on '%1%'") % lockPath);
-
-                /* Check that the lock file hasn't become stale (i.e.,
-                   hasn't been unlinked). */
-                struct stat st;
-                if (fstat(fd.get(), &st) == -1)
-                    throw SysError(format("statting lock file '%1%'") % lockPath);
-                if (st.st_size != 0)
-                    /* This lock file has been unlinked, so we're holding
-                       a lock on a deleted file.  This means that other
-                       processes may create and acquire a lock on
-                       `lockPath', and proceed.  So we must retry. */
-                    debug(format("open lock file '%1%' has become stale") % lockPath);
-                else
-                    break;
             }
 
-            /* Use borrow so that the descriptor isn't closed. */
-            fds.push_back(FDPair(fd.release(), lockPath));
-
-        } catch (...) {
-            lockedPaths_.lock()->erase(lockPath);
-            throw;
+            debug(format("lock acquired on '%1%'") % lockPath);
+
+            /* Check that the lock file hasn't become stale (i.e.,
+               hasn't been unlinked). */
+            struct stat st;
+            if (fstat(fd.get(), &st) == -1)
+                throw SysError(format("statting lock file '%1%'") % lockPath);
+            if (st.st_size != 0)
+                /* This lock file has been unlinked, so we're holding
+                   a lock on a deleted file.  This means that other
+                   processes may create and acquire a lock on
+                   `lockPath', and proceed.  So we must retry. */
+                debug(format("open lock file '%1%' has become stale") % lockPath);
+            else
+                break;
         }
 
+        /* Use borrow so that the descriptor isn't closed. */
+        fds.push_back(FDPair(fd.release(), lockPath));
     }
 
     return true;
@@ -189,8 +157,6 @@ void PathLocks::unlock()
     for (auto & i : fds) {
         if (deletePaths) deleteLockFile(i.second, i.first);
 
-        lockedPaths_.lock()->erase(i.second);
-
         if (close(i.first) == -1)
             printError(
                 format("error (ignored): cannot close lock file on '%1%'") % i.second);
@@ -208,11 +174,4 @@ void PathLocks::setDeletion(bool deletePaths)
 }
 
 
-bool pathIsLockedByMe(const Path & path)
-{
-    Path lockPath = path + ".lock";
-    return lockedPaths_.lock()->count(lockPath);
-}
-
-
 }
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index db51f950a32..411da022295 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
-MakeError(AlreadyLocked, Error);
-
 class PathLocks
 {
 private:
@@ -37,6 +35,4 @@ public:
     void setDeletion(bool deletePaths);
 };
 
-bool pathIsLockedByMe(const Path & path);
-
 }

From a2597d5f27bc6cfa26343be2f481c8a9d1e22753 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 2 Aug 2019 18:37:55 +0200
Subject: [PATCH 1496/2196] Simplify

With BSD locks we don't have to guard against reading our own
temproots.
---
 src/libstore/gc.cc | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 2c791efbe67..366dbfb0a65 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-        if (path != fnTempRoots) {
-
-            /* Try to acquire a write lock without blocking.  This can
-               only succeed if the owning process has died.  In that case
-               we don't care about its temporary roots. */
-            if (lockFile(fd->get(), ltWrite, false)) {
-                printError(format("removing stale temporary roots file '%1%'") % path);
-                unlink(path.c_str());
-                writeFull(fd->get(), "d");
-                continue;
-            }
-
-            /* Acquire a read lock.  This will prevent the owning process
-               from upgrading to a write lock, therefore it will block in
-               addTempRoot(). */
-            debug(format("waiting for read lock on '%1%'") % path);
-            lockFile(fd->get(), ltRead, true);
-
+        /* Try to acquire a write lock without blocking.  This can
+           only succeed if the owning process has died.  In that case
+           we don't care about its temporary roots. */
+        if (lockFile(fd->get(), ltWrite, false)) {
+            printError(format("removing stale temporary roots file '%1%'") % path);
+            unlink(path.c_str());
+            writeFull(fd->get(), "d");
+            continue;
         }
 
+        /* Acquire a read lock.  This will prevent the owning process
+           from upgrading to a write lock, therefore it will block in
+           addTempRoot(). */
+        debug(format("waiting for read lock on '%1%'") % path);
+        lockFile(fd->get(), ltRead, true);
+
         /* Read the entire file. */
         string contents = readFile(fd->get());
 

From 399b6f3c46077e10a7047e8216fc1a67425a768a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Oct 2018 14:46:30 +0200
Subject: [PATCH 1497/2196] nix-store --verify: Don't repair while holding the
 GC lock

---
 src/libstore/local-store.cc | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 485fdd69193..63b11467eb9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1210,7 +1210,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     bool errors = false;
 
-    /* Acquire the global GC lock to prevent a garbage collection. */
+    /* Acquire the global GC lock to get a consistent snapshot of
+       existing and valid paths. */
     AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
     PathSet store;
@@ -1221,13 +1222,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
 
+    fdGCLock = -1;
+
     for (auto & i : validPaths2)
         verifyPath(i, store, done, validPaths, repair, errors);
 
-    /* Release the GC lock so that checking content hashes (which can
-       take ages) doesn't block the GC or builds. */
-    fdGCLock = -1;
-
     /* Optionally, check the content hashes (slow). */
     if (checkContents) {
         printInfo("checking hashes...");

From 363a2f68261af73aefe4edead9c0526030751a27 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 6 Aug 2019 14:26:43 -0400
Subject: [PATCH 1498/2196] post-build-hook: docs fixup

---
 doc/manual/advanced-topics/post-build-hook.xml | 7 +++++--
 doc/manual/command-ref/conf-file.xml           | 3 ++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
index 4335b308b9d..3dc43ee795b 100644
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -74,6 +74,8 @@ trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDS
 #!/bin/sh
 
 set -eu
+set -f # disable globbing
+export IFS=' '
 
 echo "Signing paths" $OUT_PATHS
 nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
@@ -88,8 +90,9 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
       list of Nix store paths. In this case, we expect and want the
       shell to perform word splitting to make each output path its
       own argument to <command>nix sign-paths</command>. Nix guarantees
-      the paths will only contain characters which are safe for word
-      splitting, and free of any globs.
+      the paths will not contain any spaces, however a store path
+      might contain glob characters. The <command>set -f</command>
+      disables globbing in the shell.
     </para>
   </note>
   <para>
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index d2c9c7502eb..e818a74cd40 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -674,6 +674,7 @@ password <replaceable>my-password</replaceable>
         <listitem><para>The hook does not execute on substituted paths.</para></listitem>
         <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
         <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
+        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para>
       </itemizedlist>
 
       <para>The program executes with no arguments. The program's environment
@@ -693,7 +694,7 @@ password <replaceable>my-password</replaceable>
         <varlistentry>
           <term><envar>OUT_PATHS</envar></term>
           <listitem>
-            <para>Output paths of the built derivation, separated by a space (<literal> </literal>) character.</para>
+            <para>Output paths of the built derivation, separated by a space character.</para>
             <para>Example:
             <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
             /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc

From c3fefd1a6efec457395a187d15f435447dee6f3b Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 7 Aug 2019 07:34:11 -0500
Subject: [PATCH 1499/2196] pathlocks: add include to fcntl.h for O_CLOEXEC

---
 src/libstore/pathlocks.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index b6d8547c70e..2635e3940af 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -5,6 +5,7 @@
 #include <cerrno>
 #include <cstdlib>
 
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/file.h>

From 1eeaf99cf89630f84b4a1b6882a0f98019725d9e Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 7 Aug 2019 14:45:07 -0400
Subject: [PATCH 1500/2196] fixup: docs for post-build-hook

---
 doc/manual/command-ref/conf-file.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index f3c721c783f..c7b540892f9 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -682,7 +682,7 @@ password <replaceable>my-password</replaceable>
         <listitem><para>The hook does not execute on substituted paths.</para></listitem>
         <listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
         <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
-        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para>
+        <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem>
       </itemizedlist>
 
       <para>The program executes with no arguments. The program's environment

From 2053ac774797f237a5335c4232bc1ed8902686cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Aug 2019 12:18:46 +0200
Subject: [PATCH 1501/2196] Rename file for consistency

---
 tests/post-hook.sh                           | 2 +-
 tests/{push_to_store.sh => push-to-store.sh} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{push_to_store.sh => push-to-store.sh} (100%)

diff --git a/tests/post-hook.sh b/tests/post-hook.sh
index 3f82d0c5e66..54d605d63e8 100644
--- a/tests/post-hook.sh
+++ b/tests/post-hook.sh
@@ -5,7 +5,7 @@ clearStore
 export REMOTE_STORE=$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store
-nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh
+nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh
 
 clearStore
 
diff --git a/tests/push_to_store.sh b/tests/push-to-store.sh
similarity index 100%
rename from tests/push_to_store.sh
rename to tests/push-to-store.sh

From 05a10dd835923092b54fcad2276d54bf164c1b7f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 8 Aug 2019 14:39:05 +0200
Subject: [PATCH 1502/2196] tests/post-hook.sh: Don't put result link in cwd

---
 tests/post-hook.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/post-hook.sh b/tests/post-hook.sh
index 54d605d63e8..a026572154d 100644
--- a/tests/post-hook.sh
+++ b/tests/post-hook.sh
@@ -5,7 +5,7 @@ clearStore
 export REMOTE_STORE=$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store
-nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh
+nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh
 
 clearStore
 

From a02457db715a80de7630d823fd8d7230a4bd451f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 7 Aug 2019 15:14:40 -0400
Subject: [PATCH 1503/2196] conf: stalled-download-timeout: make tunable

Make curl's low speed limit configurable via stalled-download-timeout.
Before, this limit was five minutes without receiving a single byte.
This is much too long as if the remote end may not have even
acknowledged the HTTP request.
---
 doc/manual/command-ref/conf-file.xml | 8 ++++++++
 src/libstore/download.cc             | 4 +---
 src/libstore/download.hh             | 3 +++
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c7b540892f9..1325a3f2225 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -873,6 +873,14 @@ password <replaceable>my-password</replaceable>
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term>
+    <listitem>
+      <para>The timeout (in seconds) for receiving data from servers
+      during download. Nix cancels idle downloads after this timeout's
+      duration.</para>
+    </listitem>
+  </varlistentry>
+
   <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term>
 
     <listitem><para>A list of URLs of substituters, separated by
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 91087eebcfc..c322d267d4f 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader
             return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
         }
 
-        long lowSpeedTimeout = 300;
-
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout);
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 3b7fff3ba4c..c6838184694 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -24,6 +24,9 @@ struct DownloadSettings : Config
     Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
         "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
 
+    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
+        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};
 };

From ee9c988a1b2e3c511b8613e698a0f9632ab1538f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 12 Apr 2019 12:31:33 -0400
Subject: [PATCH 1504/2196] Track function start and ends for flame graphs

With this patch, and this file I called `log.py`:

    #!/usr/bin/env nix-shell
    #!nix-shell -i python3 -p python3 --pure

    import sys
    from pprint import pprint

    stack = []
    timestack = []

    for line in open(sys.argv[1]):
        components = line.strip().split(" ", 2)
        if components[0] != "function-trace":
            continue

        direction = components[1]
        components = components[2].rsplit(" ", 2)

        loc = components[0]
        _at = components[1]
        time = int(components[2])

        if direction == "entered":
            stack.append(loc)
            timestack.append(time)
        elif direction == "exited":
            dur = time - timestack.pop()
            vst = ";".join(stack)
            print(f"{vst} {dur}")
            stack.pop()

and:

    nix-instantiate --trace-function-calls -vvvv ../nixpkgs/pkgs/top-level/release.nix -A unstable > log.matthewbauer 2>&1
    ./log.py ./log.matthewbauer > log.matthewbauer.folded
    flamegraph.pl --title matthewbauer-post-pr log.matthewbauer.folded > log.matthewbauer.folded.svg

I can make flame graphs like: http://gsc.io/log.matthewbauer.folded.svg

---

Includes test cases around function call failures and tryEval. Uses
RAII so the finish is always called at the end of the function.
---
 contrib/stack-collapse.py            | 39 +++++++++++++
 doc/manual/command-ref/conf-file.xml | 28 +++++++++
 src/libexpr/eval.cc                  |  8 ++-
 src/libexpr/eval.hh                  |  4 ++
 src/libexpr/function-trace.hh        | 24 ++++++++
 tests/function-trace.sh              | 86 ++++++++++++++++++++++++++++
 tests/local.mk                       |  3 +-
 7 files changed, 189 insertions(+), 3 deletions(-)
 create mode 100755 contrib/stack-collapse.py
 create mode 100644 src/libexpr/function-trace.hh
 create mode 100755 tests/function-trace.sh

diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py
new file mode 100755
index 00000000000..ee77e158945
--- /dev/null
+++ b/contrib/stack-collapse.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3 --pure
+
+# To be used with `--trace-function-calls` and `-vvvv` and
+# `flamegraph.pl`.
+#
+# For example:
+#
+# nix-instantiate --trace-function-calls -vvvv '<nixpkgs>' -A hello 2> nix-function-calls.trace
+# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded
+# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg"
+
+import sys
+from pprint import pprint
+import fileinput
+
+stack = []
+timestack = []
+
+for line in fileinput.input():
+    components = line.strip().split(" ", 2)
+    if components[0] != "function-trace":
+        continue
+
+    direction = components[1]
+    components = components[2].rsplit(" ", 2)
+
+    loc = components[0]
+    _at = components[1]
+    time = int(components[2])
+
+    if direction == "entered":
+        stack.append(loc)
+        timestack.append(time)
+    elif direction == "exited":
+        dur = time - timestack.pop()
+        vst = ";".join(stack)
+        print(f"{vst} {dur}")
+        stack.pop()
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index c7b540892f9..407e5d10b6a 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -973,6 +973,34 @@ requiredSystemFeatures = [ "kvm" ];
 
   </varlistentry>
 
+  <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term>
+
+    <listitem>
+
+      <para>Default: <literal>false</literal>.</para>
+
+      <para>If set to <literal>true</literal>, the Nix evaluator will
+      trace every function call. Nix will print a log message at the
+      "vomit" level for every function entrance and function exit.</para>
+
+      <informalexample><screen>
+function-trace entered undefined position at 1565795816999559622
+function-trace exited undefined position at 1565795816999581277
+function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+</screen></informalexample>
+
+      <para>The <literal>undefined position</literal> means the function
+      call is a builtin.</para>
+
+      <para>Use the <literal>contrib/stack-collapse.py</literal> script
+      distributed with the Nix source code to convert the trace logs
+      in to a format suitable for <command>flamegraph.pl</command>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
   <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term>
 
     <listitem><para>A whitespace-separated list of public keys. When
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d8e10d9f20e..9f4b6b411a7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -9,6 +9,7 @@
 #include "json.hh"
 
 #include <algorithm>
+#include <chrono>
 #include <cstring>
 #include <unistd.h>
 #include <sys/time.h>
@@ -16,7 +17,6 @@
 #include <iostream>
 #include <fstream>
 
-#include <sys/time.h>
 #include <sys/resource.h>
 
 #if HAVE_BOEHMGC
@@ -1094,9 +1094,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     }
 }
 
-
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
+    std::optional<FunctionCallTrace> trace;
+    if (evalSettings.traceFunctionCalls) {
+        trace.emplace(pos);
+    }
+
     forceValue(fun, pos);
 
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index a314e01e0a7..22472fd726b 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -6,6 +6,7 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
+#include "function-trace.hh"
 
 #include <map>
 #include <unordered_map>
@@ -349,6 +350,9 @@ struct EvalSettings : Config
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
 };
 
 extern EvalSettings evalSettings;
diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh
new file mode 100644
index 00000000000..8234b760312
--- /dev/null
+++ b/src/libexpr/function-trace.hh
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "eval.hh"
+#include <sys/time.h>
+
+namespace nix {
+
+struct FunctionCallTrace
+{
+    const Pos & pos;
+
+    FunctionCallTrace(const Pos & pos) : pos(pos) {
+        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+        vomit("function-trace entered %1% at %2%", pos, ns.count());
+    }
+
+    ~FunctionCallTrace() {
+        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+        vomit("function-trace exited %1% at %2%", pos, ns.count());
+    }
+};
+}
diff --git a/tests/function-trace.sh b/tests/function-trace.sh
new file mode 100755
index 00000000000..f7d93b4356e
--- /dev/null
+++ b/tests/function-trace.sh
@@ -0,0 +1,86 @@
+source common.sh
+
+set +x
+
+expect_trace() {
+    expr="$1"
+    expect="$2"
+    actual=$(
+        nix-instantiate \
+            --trace-function-calls \
+            -vvvv \
+            --expr "$expr" 2>&1 \
+            | grep "function-trace" \
+            | sed -e 's/ [0-9]*$//'
+    );
+
+    echo -n "Tracing expression '$expr'"
+    set +e
+    msg=$(diff -swB \
+               <(echo "$expect") \
+               <(echo "$actual")
+    );
+    result=$?
+    set -e
+    if [ $result -eq 0 ]; then
+        echo " ok."
+    else
+        echo " failed. difference:"
+        echo "$msg"
+        return $result
+    fi
+}
+
+# failure inside a tryEval
+expect_trace 'builtins.tryEval (throw "example")' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace entered (string):1:19 at
+function-trace exited (string):1:19 at
+function-trace exited (string):1:1 at
+"
+
+# Missing argument to a formal function
+expect_trace '({ x }: x) { }' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+"
+
+# Too many arguments to a formal function
+expect_trace '({ x }: x) { x = "x"; y = "y"; }' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+"
+
+# Not enough arguments to a lambda
+expect_trace '(x: y: x + y) 1' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+"
+
+# Too many arguments to a lambda
+expect_trace '(x: x) 1 2' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+"
+
+# Not a function
+expect_trace '1 2' "
+function-trace entered undefined position at
+function-trace exited undefined position at
+function-trace entered (string):1:1 at
+function-trace exited (string):1:1 at
+"
+
+set -e
diff --git a/tests/local.mk b/tests/local.mk
index ef359c631d6..187f96ea206 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -29,7 +29,8 @@ nix_tests = \
   plugins.sh \
   search.sh \
   nix-copy-ssh.sh \
-  post-hook.sh
+  post-hook.sh \
+  function-trace.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

From 653c40778462b5921fbbe2abffd1a2822f08ec57 Mon Sep 17 00:00:00 2001
From: Daniel Diaz <diaz.carrete@facebook.com>
Date: Fri, 14 Jun 2019 00:05:31 +0200
Subject: [PATCH 1505/2196] Expanded documentation for .nix-defexpr

---
 doc/manual/command-ref/nix-env.xml | 40 +++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 56c466268ea..693f23f7f33 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -221,31 +221,53 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   <varlistentry><term><filename>~/.nix-defexpr</filename></term>
 
-    <listitem><para>A directory that contains the default Nix
+    <listitem><para>The source for the default Nix
     expressions used by the <option>--install</option>,
     <option>--upgrade</option>, and <option>--query
-    --available</option> operations to obtain derivations.  The
+    --available</option> operations to obtain derivations. The
     <option>--file</option> option may be used to override this
     default.</para>
 
-    <para>The Nix expressions in this directory are combined into a
-    single set, with each file as an attribute that has the name of
-    the file.  Thus, if <filename>~/.nix-defexpr</filename> contains
-    two files, <filename>foo</filename> and <filename>bar</filename>,
+    <para>If <filename>~/.nix-defexpr</filename> is a file,
+    it is loaded as a Nix expression. If the expression
+    is a set, it is used as the default Nix expression.
+    If the expression is a function, an empty set is passed
+    as argument and the return value is used as
+    the default Nix expression.</para>
+
+    <para>If <filename>~/.nix-defexpr</filename> is a directory
+    containing a <filename>default.nix</filename> file, that file
+    is loaded as in the above paragraph.</para>
+
+    <para>If <filename>~/.nix-defexpr</filename> is a directory without
+    a <filename>default.nix</filename> file, then its contents
+    (both files and subdirectories) are loaded as Nix expressions.
+    The expressions are combined into a single set, each expression
+    under an attribute with the same name as the original file
+    or subdirectory.
+    </para>
+
+    <para>For example, if <filename>~/.nix-defexpr</filename> contains
+    two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>,
     then the default Nix expression will essentially be
 
 <programlisting>
 {
-  foo = import ~/.nix-defexpr/foo;
-  bar = import ~/.nix-defexpr/bar;
+  foo = import ~/.nix-defexpr/foo.nix;
+  bar = import ~/.nix-defexpr/bar.nix;
 }</programlisting>
 
     </para>
 
+    <para>The file <filename>manifest.nix</filename> is always ignored.
+    Subdirectories without a <filename>default.nix</filename> file
+    are traversed recursively in search of more Nix expressions,
+    but the names of these intermediate directories are not
+    added to the attribute paths of the default Nix expression.</para>
+
     <para>The command <command>nix-channel</command> places symlinks
     to the downloaded Nix expressions from each subscribed channel in
     this directory.</para>
-
     </listitem>
 
   </varlistentry>

From 91b00b145f0c50e346d0250168cbcbcba7aa3b40 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 12 Aug 2019 21:03:48 +0200
Subject: [PATCH 1506/2196] libutil: add SizedSource

Introduce the SizeSource which allows to bound how much data is being
read from a source. It also contains a drainAll() function to discard
the rest of the source, useful to keep the nix protocol in sync.
---
 src/libutil/serialise.hh | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 969e4dff383..a344a5ac752 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -179,6 +179,36 @@ struct TeeSource : Source
     }
 };
 
+/* A reader that consumes the original Source until 'size'. */
+struct SizedSource : Source
+{
+    Source & orig;
+    size_t remain;
+    SizedSource(Source & orig, size_t size)
+        : orig(orig), remain(size) { }
+    size_t read(unsigned char * data, size_t len)
+    {
+        if (this->remain <= 0) {
+            throw EndOfFile("sized: unexpected end-of-file");
+        }
+        len = std::min(len, this->remain);
+        size_t n = this->orig.read(data, len);
+        this->remain -= n;
+        return n;
+    }
+
+    /* Consume the original source until no remain data is left to consume. */
+    size_t drainAll()
+    {
+        std::vector<unsigned char> buf(8192);
+        size_t sum = 0;
+        while (this->remain > 0) {
+            size_t n = read(buf.data(), buf.size());
+            sum += n;
+        }
+        return sum;
+    }
+};
 
 /* Convert a function into a sink. */
 struct LambdaSink : Sink

From b226b5cd976ca71abb3c0861b56d5e5940430924 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 12 Aug 2019 17:19:43 +0200
Subject: [PATCH 1507/2196] nix-store: fix out of sync protocol

If a NAR is already in the store, addToStore doesn't read the source
which makes the protocol go out of sync. This happens for example when
two client try to nix-copy-closure the same derivation at the same time.
---
 src/nix-store/nix-store.cc | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index f324056bb3a..0cbceb02f31 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs)
                 info.sigs = readStrings<StringSet>(in);
                 in >> info.ca;
 
-                // FIXME: race if addToStore doesn't read source?
-                store->addToStore(info, in, NoRepair, NoCheckSigs);
+                if (info.narSize == 0) {
+                    throw Error("narInfo is too old and missing the narSize field");
+                }
+
+                SizedSource sizedSource(in, info.narSize);
+
+                store->addToStore(info, sizedSource, NoRepair, NoCheckSigs);
+
+                // consume all the data that has been sent before continuing.
+                sizedSource.drainAll();
 
                 out << 1; // indicate success
 

From 1dbaf119486fecbdf57ecc2f877307030b665f69 Mon Sep 17 00:00:00 2001
From: Toon Nolten <toon.nolten@student.kuleuven.be>
Date: Sat, 17 Aug 2019 16:33:35 +0200
Subject: [PATCH 1508/2196] Fix nix-env documentation for --delete-generations

The documentation for `--delete-generations` had an erroneous fullstop
and as it turns out inaccurate information on the `+No.` syntax.
---
 doc/manual/command-ref/nix-env.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 693f23f7f33..9d13d3014e1 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1370,10 +1370,10 @@ profile.  The generations can be a list of generation numbers, the
 special value <literal>old</literal> to delete all non-current
 generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
-generation that was active at that point in time), or a value such as.
-<literal>+5</literal> to only keep the specified items older than the
-current generation. Periodically deleting old generations is important
-to make garbage collection effective.</para>
+generation that was active at that point in time), or a value such as
+<literal>+5</literal> to only keep the specified number of generations.
+Periodically deleting old generations is important to make garbage collection
+effective.</para>
 
 </refsection>
 

From 0463d5e36ffdbb60f851bd3b34756eff84151b28 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Thu, 22 Aug 2019 23:38:52 -0400
Subject: [PATCH 1509/2196] Allow empty /nix directory in multi-user installer

With macOS catalina, we can no longer modify the root system
volume (#2925). macOS provides a system configuration file in
synthetic.conf(5) to create empty root directories. This can be used
to mount /nix to a separate volume. As a result, this directory will
need to already exist prior to installation. Instead, check for
/nix/store and /nix/var for a live Nix installation.
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 9b757e7da3f..3d8b035eb7d 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -330,7 +330,7 @@ EOF
         fi
     done
 
-    if [ -d /nix ]; then
+    if [ -d /nix/store ] || [ -d /nix/var ]; then
         failure <<EOF
 There are some relics of a previous installation of Nix at /nix, and
 this scripts assumes Nix is _not_ yet installed. Please delete the old

From 92ddce4f463dc3cf2327c832d45f62d985d5e09f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 23 Aug 2019 15:50:54 -0400
Subject: [PATCH 1510/2196] operators: document exact precedenc, split up
 similar operators

---
 doc/manual/expressions/language-operators.xml | 103 ++++++++++++++++--
 1 file changed, 95 insertions(+), 8 deletions(-)

diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml
index f1f75093461..1b14bc6580e 100644
--- a/doc/manual/expressions/language-operators.xml
+++ b/doc/manual/expressions/language-operators.xml
@@ -15,13 +15,16 @@ weakest binding).</para>
   <tgroup cols='3'>
     <thead>
       <row>
+        <entry>Name</entry>
         <entry>Syntax</entry>
         <entry>Associativity</entry>
         <entry>Description</entry>
+        <entry>Precedence</entry>
       </row>
     </thead>
     <tbody>
       <row>
+        <entry>Select</entry>
         <entry><replaceable>e</replaceable> <literal>.</literal>
         <replaceable>attrpath</replaceable>
         [ <literal>or</literal> <replaceable>def</replaceable> ]
@@ -33,19 +36,25 @@ weakest binding).</para>
         dot-separated list of attribute names.)  If the attribute
         doesn’t exist, return <replaceable>def</replaceable> if
         provided, otherwise abort evaluation.</entry>
+        <entry>1</entry>
       </row>
       <row>
+        <entry>Application</entry>
         <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Call function <replaceable>e1</replaceable> with
         argument <replaceable>e2</replaceable>.</entry>
+        <entry>2</entry>
       </row>
       <row>
+        <entry>Arithmetic Negation</entry>
         <entry><literal>-</literal> <replaceable>e</replaceable></entry>
         <entry>none</entry>
         <entry>Arithmetic negation.</entry>
+        <entry>3</entry>
       </row>
       <row>
+        <entry>Has Attribute</entry>
         <entry><replaceable>e</replaceable> <literal>?</literal>
         <replaceable>attrpath</replaceable></entry>
         <entry>none</entry>
@@ -53,34 +62,69 @@ weakest binding).</para>
         the attribute denoted by <replaceable>attrpath</replaceable>;
         return <literal>true</literal> or
         <literal>false</literal>.</entry>
+        <entry>4</entry>
       </row>
       <row>
+        <entry>List Concatenation</entry>
         <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry>
         <entry>right</entry>
         <entry>List concatenation.</entry>
+        <entry>5</entry>
       </row>
       <row>
+        <entry>Multiplication</entry>
         <entry>
           <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>,
+        </entry>
+        <entry>left</entry>
+        <entry>Arithmetic multiplication.</entry>
+        <entry>6</entry>
+      </row>
+      <row>
+        <entry>Division</entry>
+        <entry>
           <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>left</entry>
-        <entry>Arithmetic multiplication and division.</entry>
+        <entry>Arithmetic division.</entry>
+        <entry>6</entry>
+      </row>
+      <row>
+        <entry>Addition</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>left</entry>
+        <entry>Arithmetic addition.</entry>
+        <entry>7</entry>
       </row>
       <row>
+        <entry>Subtraction</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>,
           <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>left</entry>
-        <entry>Arithmetic addition and subtraction.  String or path concatenation (only by <literal>+</literal>).</entry>
+        <entry>Arithmetic subtraction.</entry>
+        <entry>7</entry>
       </row>
       <row>
+        <entry>String Concatenation</entry>
+        <entry>
+          <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable>
+        </entry>
+        <entry>left</entry>
+        <entry>String concatenation.</entry>
+        <entry>7</entry>
+      </row>
+      <row>
+        <entry>Not</entry>
         <entry><literal>!</literal> <replaceable>e</replaceable></entry>
         <entry>none</entry>
         <entry>Boolean negation.</entry>
+        <entry>8</entry>
       </row>
       <row>
+        <entry>Update</entry>
         <entry><replaceable>e1</replaceable> <literal>//</literal>
         <replaceable>e2</replaceable></entry>
         <entry>right</entry>
@@ -89,47 +133,90 @@ weakest binding).</para>
         <replaceable>e2</replaceable> (with the latter taking
         precedence over the former in case of equally named
         attributes).</entry>
+        <entry>9</entry>
       </row>
       <row>
+        <entry>Less Than</entry>
         <entry>
           <replaceable>e1</replaceable> <literal>&lt;</literal> <replaceable>e2</replaceable>,
-          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>,
-          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>,
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Less Than or Equal To</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>&lt;=</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Greater Than</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>&gt;</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Greater Than or Equal To</entry>
+        <entry>
           <replaceable>e1</replaceable> <literal>&gt;=</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>none</entry>
         <entry>Arithmetic comparison.</entry>
+        <entry>10</entry>
+      </row>
+      <row>
+        <entry>Equality</entry>
+        <entry>
+          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>
+        </entry>
+        <entry>none</entry>
+        <entry>Equality.</entry>
+        <entry>11</entry>
       </row>
       <row>
+        <entry>Inequality</entry>
         <entry>
-          <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>,
           <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable>
         </entry>
         <entry>none</entry>
-        <entry>Equality and inequality.</entry>
+        <entry>Inequality.</entry>
+        <entry>11</entry>
       </row>
       <row>
+        <entry>Logical And</entry>
         <entry><replaceable>e1</replaceable> <literal>&amp;&amp;</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Logical AND.</entry>
+        <entry>12</entry>
       </row>
       <row>
+        <entry>Logical Or</entry>
         <entry><replaceable>e1</replaceable> <literal>||</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>
         <entry>Logical OR.</entry>
+        <entry>13</entry>
       </row>
       <row>
+        <entry>Logical Implication</entry>
         <entry><replaceable>e1</replaceable> <literal>-></literal>
         <replaceable>e2</replaceable></entry>
         <entry>none</entry>
         <entry>Logical implication (equivalent to
         <literal>!<replaceable>e1</replaceable> ||
         <replaceable>e2</replaceable></literal>).</entry>
+        <entry>14</entry>
       </row>
     </tbody>
   </tgroup>
 </table>
 
-</section>
\ No newline at end of file
+</section>

From 5c06a8d3283139140e765b5f10ad7102a6a3e964 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 23 Aug 2019 20:24:39 -0400
Subject: [PATCH 1511/2196] Reset tmpDirInSandbox for unsandboxed

---
 src/libstore/build.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0f71e751151..96e9b8edda0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2330,6 +2330,7 @@ void DerivationGoal::startBuilder()
         int res = helper.wait();
         if (res != 0 && settings.sandboxFallback) {
             useChroot = false;
+            tmpDirInSandbox = tmpDir;
             goto fallback;
         } else if (res != 0)
             throw Error("unable to start build process");

From 6dab42a551a769f304c2a0e66e8771d7289502f2 Mon Sep 17 00:00:00 2001
From: Venkateswara Rao Mandela <venkat.mandela@gmail.com>
Date: Sat, 15 Jun 2019 00:27:26 +0530
Subject: [PATCH 1512/2196] installer: handle network proxy in systemd install

If a network proxy configuration is detected, setup an override
systemd unit file for nix-daemon service with the non-empty
proxy variables.

Proxy detection is performed by looking for http/https/ftp proxy and no
proxy variables in user environment
---
 scripts/install-systemd-multi-user.sh | 34 +++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 mode change 100644 => 100755 scripts/install-systemd-multi-user.sh

diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh
old mode 100644
new mode 100755
index 04bc539a109..bef3ac4f991
--- a/scripts/install-systemd-multi-user.sh
+++ b/scripts/install-systemd-multi-user.sh
@@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
 readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
 
+
+# Path for the systemd override unit file to contain the proxy settings
+readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
+
+create_systemd_override() {
+     header "Configuring proxy for the nix-daemon service"
+    _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)"
+    cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE"
+[Service]
+$1
+EOF
+}
+
+# Gather all non-empty proxy environment variables into a string
+create_systemd_proxy_env() {
+    vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY"
+    for v in $vars; do
+        if [ "x${!v:-}" != "x" ]; then
+            echo "Environment=${v}=${!v}"
+        fi
+    done
+}
+
+handle_network_proxy() {
+    # Create a systemd unit override with proxy environment variables
+    # if any proxy environment variables are not empty.
+    PROXY_ENV_STRING=$(create_systemd_proxy_env)
+    if [ -n "${PROXY_ENV_STRING}" ]; then
+        create_systemd_override "${PROXY_ENV_STRING}"
+    fi
+}
+
 poly_validate_assumptions() {
     if [ "$(uname -s)" != "Linux" ]; then
         failure "This script is for use with Linux!"
@@ -47,6 +79,8 @@ poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon socket service" \
           systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC"
 
+    handle_network_proxy
+
     _sudo "to load the systemd unit for nix-daemon" \
           systemctl daemon-reload
 

From 057af1dbd802987df7990d965e08db185820a193 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 23 Aug 2019 16:02:49 -0400
Subject: [PATCH 1513/2196] docs: document the installer's use of proxy env
 vars

---
 doc/manual/installation/env-variables.xml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml
index d1ee0bb2e09..e2b8fc867cd 100644
--- a/doc/manual/installation/env-variables.xml
+++ b/doc/manual/installation/env-variables.xml
@@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon
 </screen>
 </section>
 
+<section xml:id="sec-installer-proxy-settings">
+
+<title>Proxy Environment Variables</title>
+
+<para>The Nix installer has special handling for these proxy-related
+environment variables:
+<varname>http_proxy</varname>, <varname>https_proxy</varname>,
+<varname>ftp_proxy</varname>, <varname>no_proxy</varname>,
+<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>,
+<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>.
+</para>
+<para>If any of these variables are set when running the Nix installer,
+then the installer will create an override file at
+<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename>
+so <command>nix-daemon</command> will use them.
+</para>
+</section>
+
 </section>
 </chapter>

From d45922472434783bdc29610479a735dd236229a3 Mon Sep 17 00:00:00 2001
From: Piotr Szubiakowski <pszubiak@eso.org>
Date: Tue, 27 Aug 2019 10:35:35 +0200
Subject: [PATCH 1514/2196] nix-daemon.service: add install section.

Signed-off-by: Piotr Szubiakowski <pszubiak@eso.org>
---
 misc/systemd/nix-daemon.service.in | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in
index 5fc04a3f571..25655204d4d 100644
--- a/misc/systemd/nix-daemon.service.in
+++ b/misc/systemd/nix-daemon.service.in
@@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
+
+[Install]
+WantedBy=multi-user.target

From 171d784404fefa7c91e2082f92ec92c39cc1aa8e Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 27 Aug 2019 06:55:22 -0400
Subject: [PATCH 1515/2196] docs: operators: Make OR and AND capitalized

---
 doc/manual/expressions/language-operators.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml
index 1b14bc6580e..4f11bf52938 100644
--- a/doc/manual/expressions/language-operators.xml
+++ b/doc/manual/expressions/language-operators.xml
@@ -190,7 +190,7 @@ weakest binding).</para>
         <entry>11</entry>
       </row>
       <row>
-        <entry>Logical And</entry>
+        <entry>Logical AND</entry>
         <entry><replaceable>e1</replaceable> <literal>&amp;&amp;</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>
@@ -198,7 +198,7 @@ weakest binding).</para>
         <entry>12</entry>
       </row>
       <row>
-        <entry>Logical Or</entry>
+        <entry>Logical OR</entry>
         <entry><replaceable>e1</replaceable> <literal>||</literal>
         <replaceable>e2</replaceable></entry>
         <entry>left</entry>

From 800fba1037ad97ff5e6d9acf9e820a30f0922f59 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 27 Aug 2019 10:57:09 -0400
Subject: [PATCH 1516/2196] Use wait4path on org.nixos.nix-daemon.plist

When using a volume, the nix-daemon path may not exist. To avoid this
issue, we must use the wait4path tool. This should solve one of the
issues in multi-user on macOS Catalina.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 92ed12fa509..6bc448d285e 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -14,7 +14,12 @@
     <key>RunAtLoad</key>
     <true/>
     <key>Program</key>
-    <string>@bindir@/nix-daemon</string>
+    <string>/bin/sh</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>-c</string>
+      <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; @bindir@/nix-daemon</string>
+    </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>

From 45b3dc325a1d49f86182195cf22fc96ddc05c644 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Aug 2019 17:00:04 +0200
Subject: [PATCH 1517/2196] Add 2.3 release notes

---
 doc/manual/release-notes/rl-2.3.xml | 70 ++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index 428213b360b..b0f7c8ec53d 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -4,9 +4,23 @@
       version="5.0"
       xml:id="ssec-relnotes-2.3">
 
-<title>Release 2.3 (????-??-??)</title>
+<title>Release 2.3 (2019-08-??)</title>
 
-<para>This release contains the following changes:</para>
+<para>This is primarily a bug fix release. However, it makes some
+incompatible changes:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix now uses BSD file locks instead of POSIX file
+    locks. Since previous releases used POSIX file locks, you should
+    not use Nix 2.2 and previous releases at the same time on a Nix
+    store.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>It also has the following changes:</para>
 
 <itemizedlist>
 
@@ -18,5 +32,57 @@
     already begin with <literal>refs/</literal>.
     </para>
   </listitem>
+
+  <listitem>
+    <para>The installer now enables sandboxing by default on
+    Linux. The <literal>max-jobs</literal> setting now defaults to
+    1.</para>
+  </listitem>
+
+  <listitem>
+    <para>New builtin functions:
+    <literal>builtins.isPath</literal>,
+    <literal>builtins.hashFile</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix</command>: Add
+    <option>--print-build-logs</option> (<option>-L</option>) flag to
+    print build log output to stderr rather than showing the last log
+    line in the progress bar. To distinguish between concurrent
+    builds, log lines are prefixed by the name of the package.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>Builds are now executed in a pseudo-terminal, and the
+    <envar>TERM</envar> evnironment variable is set to
+    <literal>xterm-256color</literal>. This allows many programs
+    (e.g. <command>gcc</command>, <command>clang</command>,
+    <command>cmake</command>) to print colorized log output.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add <option>--no-net</option> convenience flag. This flag
+    disables substituters; sets the <literal>tarball-ttl</literal>
+    setting to infinity (ensuring that any previously downloaded files
+    are considered current); and disables retrying downloads and sets
+    the connection timeout to the minimum. This flag is enabled
+    automatically if there are no configured non-loopback network
+    interfaces.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add a <literal>post-build-hook</literal> setting to run a
+    program after a build has succeeded.</para>
+  </listitem>
+
+  <listitem>
+    <para>Add a <literal>trace-function-calls</literal> setting to log
+    the duration of Nix function calls to stderr.</para>
+  </listitem>
+
 </itemizedlist>
+
 </section>

From 73728874ab89d77bce3b103acb239d4d5eec14b1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Aug 2019 17:01:54 +0200
Subject: [PATCH 1518/2196] Hopefully fix post-hook test on macOS

https://hydra.nixos.org/build/99262744
---
 tests/push-to-store.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh
index d97eb095dd7..6aadb916ba0 100755
--- a/tests/push-to-store.sh
+++ b/tests/push-to-store.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
 echo Pushing "$@" to "$REMOTE_STORE"
-echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs
+printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs

From fdff96501f7f1762446a930e3a27b56a4fc912c2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Aug 2019 21:18:00 +0200
Subject: [PATCH 1519/2196] Update release notes

---
 doc/manual/release-notes/rl-2.3.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index b0f7c8ec53d..a1f4a4b756f 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -83,6 +83,11 @@ incompatible changes:</para>
     the duration of Nix function calls to stderr.</para>
   </listitem>
 
+  <listitem>
+    <para>On Linux, sandboxing is now disabled by default on systems
+    that don’t have the necessary kernel support.</para>
+  </listitem>
+
 </itemizedlist>
 
 </section>

From ceefddafe839a0c9ac553367e16f965dda7fb086 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Aug 2019 22:18:34 +0200
Subject: [PATCH 1520/2196] Compress binary tarballs using xz

Fixes https://github.com/NixOS/nix/issues/240.

Apparently 'tar -xf' can decompress xz files on macOS nowadays.
---
 release.nix        | 6 +++---
 scripts/install.in | 7 +++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/release.nix b/release.nix
index 78b39108f85..bec95573083 100644
--- a/release.nix
+++ b/release.nix
@@ -165,10 +165,10 @@ let
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
           dir=nix-${version}-${system}
-          fn=$out/$dir.tar.bz2
+          fn=$out/$dir.tar.xz
           mkdir -p $out/nix-support
           echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-          tar cvfj $fn \
+          tar cvfJ $fn \
             --owner=0 --group=0 --mode=u+rw,uga+r \
             --absolute-names \
             --hard-dereference \
@@ -295,7 +295,7 @@ let
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
+              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
               [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
             } \
             --replace '@nixVersion@' ${build.x86_64-linux.src.version}
diff --git a/scripts/install.in b/scripts/install.in
index 4857638c026..902758b138a 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -30,12 +30,11 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
+url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
-tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
+tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
-require_util bzcat "decompress the binary tarball"
 require_util tar "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
@@ -57,7 +56,7 @@ fi
 
 unpack=$tmpDir/unpack
 mkdir -p "$unpack"
-< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
+tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 
 script=$(echo "$unpack"/*/install)
 

From 5fa8b3f96577db65e080aaef65ad1cae462356cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 27 Aug 2019 22:38:48 +0200
Subject: [PATCH 1521/2196] Update the libboost hack

This cuts about 46 MiB from the closure.
---
 release.nix | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index bec95573083..ab677f3bd46 100644
--- a/release.nix
+++ b/release.nix
@@ -72,7 +72,12 @@ let
           # https://github.com/NixOS/nixpkgs/issues/45462
           ''
             mkdir -p $out/lib
-            cp ${boost}/lib/libboost_context* $out/lib
+            cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+            rm -f $out/lib/*.a
+            chmod u+w $out/lib/*.so.*
+            ${lib.optionalString stdenv.isLinux ''
+              patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+            ''}
           '';
 
         configureFlags = configureFlags ++

From 7298a38a07d62d2077ca30a3ff619e32edb4eb1e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Aug 2019 16:29:44 +0200
Subject: [PATCH 1522/2196] Don't send certain setting overrides to the daemon

These are already handled separately. This fixes warnings like

  warning: ignoring the user-specified setting 'max-jobs', because it is a restricted setting and you are not a trusted user

when using the -j flag.
---
 src/libstore/remote-store.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 15faf78a526..1c2e23f9cd5 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -191,6 +191,13 @@ void RemoteStore::setOptions(Connection & conn)
     if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
         std::map<std::string, Config::SettingInfo> overrides;
         globalConfig.getSettings(overrides, true);
+        overrides.erase(settings.keepFailed.name);
+        overrides.erase(settings.keepGoing.name);
+        overrides.erase(settings.tryFallback.name);
+        overrides.erase(settings.maxBuildJobs.name);
+        overrides.erase(settings.maxSilentTime.name);
+        overrides.erase(settings.buildCores.name);
+        overrides.erase(settings.useSubstitutes.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;

From 693e68e09c9a17ca72b074bd2e575bf435647b45 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 28 Aug 2019 12:26:08 -0400
Subject: [PATCH 1523/2196] Set maximum name length in Nix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously we allowed any length of name for Nix derivations. This is
bad because different file systems have different max lengths. To make
things predictable, I have picked a max. This was done by trying to
build this derivation:

  derivation {
    name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    builder = "/no-such-path";
    system = "x86_64-linux";
  }

Take off one a and it will not lead to file name too long. That ends
up being 212 a’s. An even smaller max could be picked if we want to
support more file systems.

Working backwards, this is why:

aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-${name}.drv.chroot

> 255 - 32 - 1 - 4 - 7 = 211
---
 src/libstore/store-api.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f5608d3849f..3bb9db0b723 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -97,6 +97,10 @@ void checkStoreName(const string & name)
        reasons (e.g., "." and ".."). */
     if (string(name, 0, 1) == ".")
         throw Error(baseError % "it is illegal to start the name with a period");
+    /* Disallow names longer than 211 characters. ext4’s max is 256,
+       but we need extra space for the hash and .chroot extensions. */
+    if (name.length() > 211)
+        throw Error(baseError % "name must be less than 212 characters");
     for (auto & i : name)
         if (!((i >= 'A' && i <= 'Z') ||
               (i >= 'a' && i <= 'z') ||

From c128031492b7134f67065b66a89f9183e79e99f9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Aug 2019 22:04:16 +0200
Subject: [PATCH 1524/2196] Fix macOS build

https://hydra.nixos.org/build/99500938
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index ab677f3bd46..0fe6835186b 100644
--- a/release.nix
+++ b/release.nix
@@ -74,8 +74,8 @@ let
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
-            chmod u+w $out/lib/*.so.*
             ${lib.optionalString stdenv.isLinux ''
+              chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
           '';

From b6120d26a84a69755c049ca5115b4388239d4540 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 28 Aug 2019 22:19:31 +0200
Subject: [PATCH 1525/2196] gc-auto.sh: Increase verbosity

---
 tests/gc-auto.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 1e91282d0f9..4b1f6374a49 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -29,7 +29,7 @@ with import ./config.nix; mkDerivation {
 EOF
 )
 
-nix build -o $TEST_ROOT/result-A -L "($expr)" \
+nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
 pid=$!
 
@@ -50,7 +50,7 @@ with import ./config.nix; mkDerivation {
 EOF
 )
 
-nix build -o $TEST_ROOT/result-B -L "($expr2)" \
+nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1
 
 wait "$pid"

From f27e53f77e39d1247c47b48acb63af3d24ca74bf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Aug 2019 12:09:58 +0200
Subject: [PATCH 1526/2196] Cleanup

---
 src/libstore/gc.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 366dbfb0a65..f736016797a 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -920,11 +920,11 @@ void LocalStore::autoGC(bool sync)
                     promise.set_value();
                 });
 
-                printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
-
                 GCOptions options;
                 options.maxFreed = settings.maxFree - avail;
 
+                printInfo("running auto-GC to free %d bytes", options.maxFreed);
+
                 GCResults results;
 
                 collectGarbage(options, results);

From ecb0a23d51e46e2e0180850bdc8ad6cb7cc3d13f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Aug 2019 12:10:01 +0200
Subject: [PATCH 1527/2196] Add some more instrumentation

---
 tests/gc-auto.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 4b1f6374a49..49894bc68ef 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -6,6 +6,9 @@ garbage1=$(nix add-to-store --name garbage1 ./tarball.sh)
 garbage2=$(nix add-to-store --name garbage2 ./tarball.sh)
 garbage3=$(nix add-to-store --name garbage3 ./tarball.sh)
 
+ls -l $garbage3
+du $garbage3
+
 fake_free=$TEST_ROOT/fake-free
 export _NIX_TEST_FREE_SPACE_FILE=$fake_free
 echo 1100 > $fake_free

From 31f5ecfaa5bce2282f50daed1e4f477c64705fb3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Aug 2019 12:35:15 +0200
Subject: [PATCH 1528/2196] Maybe fix #3058

---
 tests/gc-auto.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 49894bc68ef..68b7219f238 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -2,12 +2,12 @@ source common.sh
 
 clearStore
 
-garbage1=$(nix add-to-store --name garbage1 ./tarball.sh)
-garbage2=$(nix add-to-store --name garbage2 ./tarball.sh)
-garbage3=$(nix add-to-store --name garbage3 ./tarball.sh)
+garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh)
+garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh)
+garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh)
 
 ls -l $garbage3
-du $garbage3
+POSIXLY_CORRECT=1 du $garbage3
 
 fake_free=$TEST_ROOT/fake-free
 export _NIX_TEST_FREE_SPACE_FILE=$fake_free

From 5bdac86be2eb6d541784c4f149bfa06b59ef63a2 Mon Sep 17 00:00:00 2001
From: toonn <toonn@toonn.io>
Date: Thu, 29 Aug 2019 13:46:38 +0200
Subject: [PATCH 1529/2196] Reword to clarify newer generations are left alone

My attempt at clarifying the docs resulted in a false explanation. This
is now fixed and I added an example to eliminate all possible confusion.
---
 doc/manual/command-ref/nix-env.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 9d13d3014e1..c8c01f9a44f 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1371,7 +1371,10 @@ special value <literal>old</literal> to delete all non-current
 generations,  a value such as <literal>30d</literal> to delete all
 generations older than the specified number of days (except for the
 generation that was active at that point in time), or a value such as
-<literal>+5</literal> to only keep the specified number of generations.
+<literal>+5</literal> to keep the last <literal>5</literal> generations
+ignoring any newer than current, e.g., if <literal>30</literal> is the current
+generation <literal>+5</literal> will delete generation <literal>25</literal>
+and all older generations.
 Periodically deleting old generations is important to make garbage collection
 effective.</para>
 

From a2c4fcd5e9782dc8d2998773380c7171ee53b813 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 29 Aug 2019 14:49:58 +0200
Subject: [PATCH 1530/2196] Don't rely on st_blocks

It doesn't seem very reliable on ZFS.
---
 src/libstore/gc.cc  | 7 +++----
 src/libutil/util.cc | 2 +-
 tests/gc-auto.sh    | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index f736016797a..a166f4ee248 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -690,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state)
             throw SysError(format("statting '%1%'") % path);
 
         if (st.st_nlink != 1) {
-            unsigned long long size = st.st_blocks * 512ULL;
-            actualSize += size;
-            unsharedSize += (st.st_nlink - 1) * size;
+            actualSize += st.st_size;
+            unsharedSize += (st.st_nlink - 1) * st.st_size;
             continue;
         }
 
@@ -701,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (unlink(path.c_str()) == -1)
             throw SysError(format("deleting '%1%'") % path);
 
-        state.results.bytesFreed += st.st_blocks * 512ULL;
+        state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 44fa7248255..1b744999153 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -397,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
     }
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
-        bytesFreed += st.st_blocks * 512;
+        bytesFreed += st.st_size;
 
     if (S_ISDIR(st.st_mode)) {
         /* Make the directory accessible. */
diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 68b7219f238..54ead227ef6 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -22,7 +22,7 @@ with import ./config.nix; mkDerivation {
     echo foo > \$out/bar
     echo 1...
     sleep 2
-    echo 100 > $fake_free
+    echo 200 > $fake_free
     echo 2...
     sleep 2
     echo 3...
@@ -44,7 +44,7 @@ with import ./config.nix; mkDerivation {
     echo foo > \$out/bar
     echo 1...
     sleep 2
-    echo 100 > $fake_free
+    echo 200 > $fake_free
     echo 2...
     sleep 2
     echo 3...

From 87c604c1f04ff31a68f2fa73fe5a2ed9f6ec7ac7 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Mon, 2 Sep 2019 18:35:10 -0400
Subject: [PATCH 1531/2196] Fix launchd program args

launchd has some weird syntx. Apparently the program needs to be in
the ProgramArguments, as Program appears to be ignored.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 6bc448d285e..b340610e900 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -13,10 +13,9 @@
     <true/>
     <key>RunAtLoad</key>
     <true/>
-    <key>Program</key>
-    <string>/bin/sh</string>
     <key>ProgramArguments</key>
     <array>
+      <string>/bin/sh</string>
       <string>-c</string>
       <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; @bindir@/nix-daemon</string>
     </array>

From e4ea3e03066a760c8cd462108af99aebaaa44c1b Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <infinisil@icloud.com>
Date: Tue, 3 Sep 2019 07:32:44 +0200
Subject: [PATCH 1532/2196] docs: Note that tryEval doesn't do deep evaluation

---
 doc/manual/expressions/builtins.xml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 4c1d618e951..f73d2e1ae99 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1607,12 +1607,18 @@ stdenv.mkDerivation (rec {
     <term><function>builtins.tryEval</function>
     <replaceable>e</replaceable></term>
 
-    <listitem><para>Try to evaluate <replaceable>e</replaceable>.
+    <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>.
     Return a set containing the attributes <literal>success</literal>
     (<literal>true</literal> if <replaceable>e</replaceable> evaluated
     successfully, <literal>false</literal> if an error was thrown) and
     <literal>value</literal>, equalling <replaceable>e</replaceable>
-    if successful and <literal>false</literal> otherwise.
+    if successful and <literal>false</literal> otherwise. Note that this
+    doesn't evaluate <replaceable>e</replaceable> deeply, so
+    <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success
+    </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq
+    </literal> one can get the expected result: <literal>let e = { x = throw "";
+    }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be
+    <literal>false</literal>.
     </para></listitem>
 
   </varlistentry>

From 8c4ea7a4516c517a0dd37b446bf5c1a6b157064c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 12:51:14 +0200
Subject: [PATCH 1533/2196] Downloader: Remove a possible double call to
 Callback

---
 src/libstore/download.cc | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index c322d267d4f..a7d059465b0 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -342,15 +342,9 @@ struct CurlDownloader : public Downloader
                 (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
+                act.progress(result.bodySize, result.bodySize);
                 done = true;
-
-                try {
-                    act.progress(result.bodySize, result.bodySize);
-                    callback(std::move(result));
-                } catch (...) {
-                    done = true;
-                    callback.rethrow();
-                }
+                callback(std::move(result));
             }
 
             else {

From 7348653ff4fc4e9b2dc24943aabdb57179b1c75a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 12:51:35 +0200
Subject: [PATCH 1534/2196] Ensure that Callback is called only once

Also, make Callback movable but uncopyable.
---
 src/libstore/binary-cache-store.cc      |  8 +++++---
 src/libstore/download.cc                |  6 +++---
 src/libstore/http-binary-cache-store.cc | 12 +++++++-----
 src/libstore/store-api.cc               |  8 +++++---
 src/libutil/util.hh                     | 19 ++++++++++++++++---
 5 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4527ee6ba66..e56be625de4 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -249,21 +249,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
 
     auto narInfoFile = narInfoFileFor(storePath);
 
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
     getFile(narInfoFile,
         {[=](std::future<std::shared_ptr<std::string>> fut) {
             try {
                 auto data = fut.get();
 
-                if (!data) return callback(nullptr);
+                if (!data) return (*callbackPtr)(nullptr);
 
                 stats.narInfoRead++;
 
-                callback((std::shared_ptr<ValidPathInfo>)
+                (*callbackPtr)((std::shared_ptr<ValidPathInfo>)
                     std::make_shared<NarInfo>(*this, *data, narInfoFile));
 
                 (void) act; // force Activity into this lambda to ensure it stays alive
             } catch (...) {
-                callback.rethrow();
+                callbackPtr->rethrow();
             }
         }});
 }
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index a7d059465b0..cdf56e09d69 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -77,13 +77,13 @@ struct CurlDownloader : public Downloader
 
         DownloadItem(CurlDownloader & downloader,
             const DownloadRequest & request,
-            Callback<DownloadResult> callback)
+            Callback<DownloadResult> && callback)
             : downloader(downloader)
             , request(request)
             , act(*logger, lvlTalkative, actDownload,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
-            , callback(callback)
+            , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
                 if (this->request.dataCallback) {
                     writtenToSink += len;
@@ -665,7 +665,7 @@ struct CurlDownloader : public Downloader
             return;
         }
 
-        enqueueItem(std::make_shared<DownloadItem>(*this, request, callback));
+        enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
     }
 };
 
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index df2fb93320f..e631d95f0fd 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -137,17 +137,19 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
         auto request(makeRequest(path));
 
+        auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
         getDownloader()->enqueueDownload(request,
-            {[callback, this](std::future<DownloadResult> result) {
+            {[callbackPtr, this](std::future<DownloadResult> result) {
                 try {
-                    callback(result.get().data);
+                    (*callbackPtr)(result.get().data);
                 } catch (DownloadError & e) {
                     if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
-                        return callback(std::shared_ptr<std::string>());
+                        return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
-                    callback.rethrow();
+                    callbackPtr->rethrow();
                 } catch (...) {
-                    callback.rethrow();
+                    callbackPtr->rethrow();
                 }
             }});
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 3bb9db0b723..88a5b2f448b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -365,8 +365,10 @@ void Store::queryPathInfo(const Path & storePath,
 
     } catch (...) { return callback.rethrow(); }
 
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
     queryPathInfoUncached(storePath,
-        {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) {
+        {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -386,8 +388,8 @@ void Store::queryPathInfo(const Path & storePath,
                     throw InvalidPath("path '%s' is not valid", storePath);
                 }
 
-                callback(ref<ValidPathInfo>(info));
-            } catch (...) { callback.rethrow(); }
+                (*callbackPtr)(ref<ValidPathInfo>(info));
+            } catch (...) { callbackPtr->rethrow(); }
         }});
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b538a0b41ce..686e81d3f89 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -445,21 +445,34 @@ string get(const T & map, const string & key, const string & def = "")
    type T or an exception. (We abuse std::future<T> to pass the value or
    exception.) */
 template<typename T>
-struct Callback
+class Callback
 {
     std::function<void(std::future<T>)> fun;
+    std::atomic_flag done = ATOMIC_FLAG_INIT;
+
+public:
 
     Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
 
-    void operator()(T && t) const
+    Callback(Callback && callback) : fun(std::move(callback.fun))
+    {
+        auto prev = callback.done.test_and_set();
+        if (prev) done.test_and_set();
+    }
+
+    void operator()(T && t)
     {
+        auto prev = done.test_and_set();
+        assert(!prev);
         std::promise<T> promise;
         promise.set_value(std::move(t));
         fun(promise.get_future());
     }
 
-    void rethrow(const std::exception_ptr & exc = std::current_exception()) const
+    void rethrow(const std::exception_ptr & exc = std::current_exception())
     {
+        auto prev = done.test_and_set();
+        assert(!prev);
         std::promise<T> promise;
         promise.set_exception(exc);
         fun(promise.get_future());

From f186000367978fbe590343f47951232f42a30bec Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 13:00:55 +0200
Subject: [PATCH 1535/2196] Add some noexcepts

This is to assert that callback functions should never throw (since
the context in which they're called may not be able to handle the
exception).
---
 src/libstore/binary-cache-store.cc      | 4 ++--
 src/libstore/binary-cache-store.hh      | 4 ++--
 src/libstore/http-binary-cache-store.cc | 2 +-
 src/libstore/legacy-ssh-store.cc        | 2 +-
 src/libstore/local-store.cc             | 2 +-
 src/libstore/local-store.hh             | 2 +-
 src/libstore/remote-store.cc            | 2 +-
 src/libstore/remote-store.hh            | 2 +-
 src/libstore/store-api.cc               | 9 +++++----
 src/libstore/store-api.hh               | 4 ++--
 src/libutil/util.hh                     | 4 ++--
 11 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index e56be625de4..10cde8704bb 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -55,7 +55,7 @@ void BinaryCacheStore::init()
 }
 
 void BinaryCacheStore::getFile(const std::string & path,
-    Callback<std::shared_ptr<std::string>> callback)
+    Callback<std::shared_ptr<std::string>> callback) noexcept
 {
     try {
         callback(getFile(path));
@@ -240,7 +240,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-    Callback<std::shared_ptr<ValidPathInfo>> callback)
+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
 {
     auto uri = getUri();
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 953f3b90af4..af108880cba 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -47,7 +47,7 @@ public:
     /* Fetch the specified file and call the specified callback with
        the result. A subclass may implement this asynchronously. */
     virtual void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback);
+        Callback<std::shared_ptr<std::string>> callback) noexcept;
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
@@ -73,7 +73,7 @@ public:
     bool isValidPathUncached(const Path & path) override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
 
     Path queryPathFromHashPart(const string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index e631d95f0fd..779f89e68d9 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -131,7 +131,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     }
 
     void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) override
+        Callback<std::shared_ptr<std::string>> callback) noexcept override
     {
         checkEnabled();
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 7c9bc2b68ba..d5fbdd25aa4 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -88,7 +88,7 @@ struct LegacySSHStore : public Store
     }
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override
     {
         try {
             auto conn(connections->get());
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 63b11467eb9..2fcf08491c2 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -629,7 +629,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
 
 void LocalStore::queryPathInfoUncached(const Path & path,
-    Callback<std::shared_ptr<ValidPathInfo>> callback)
+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
 {
     try {
         auto info = std::make_shared<ValidPathInfo>();
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index af8b84bf5d7..3ae34c4035c 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,7 +127,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1c2e23f9cd5..e38fe49a7f9 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -349,7 +349,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
 
 void RemoteStore::queryPathInfoUncached(const Path & path,
-    Callback<std::shared_ptr<ValidPathInfo>> callback)
+    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
 {
     try {
         std::shared_ptr<ValidPathInfo> info;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 80f18ab715d..82fbec092fe 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -41,7 +41,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) override;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 88a5b2f448b..5f63c53b562 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -329,13 +329,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 
 
 void Store::queryPathInfo(const Path & storePath,
-    Callback<ref<ValidPathInfo>> callback)
+    Callback<ref<ValidPathInfo>> callback) noexcept
 {
-    assertStorePath(storePath);
-
-    auto hashPart = storePathToHash(storePath);
+    std::string hashPart;
 
     try {
+        assertStorePath(storePath);
+
+        hashPart = storePathToHash(storePath);
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 59967737670..7fb56860209 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -360,12 +360,12 @@ public:
 
     /* Asynchronous version of queryPathInfo(). */
     void queryPathInfo(const Path & path,
-        Callback<ref<ValidPathInfo>> callback);
+        Callback<ref<ValidPathInfo>> callback) noexcept;
 
 protected:
 
     virtual void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) = 0;
+        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
 
 public:
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 686e81d3f89..07c3d28ff2d 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -460,7 +460,7 @@ public:
         if (prev) done.test_and_set();
     }
 
-    void operator()(T && t)
+    void operator()(T && t) noexcept
     {
         auto prev = done.test_and_set();
         assert(!prev);
@@ -469,7 +469,7 @@ public:
         fun(promise.get_future());
     }
 
-    void rethrow(const std::exception_ptr & exc = std::current_exception())
+    void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
     {
         auto prev = done.test_and_set();
         assert(!prev);

From cec50290bf5537d574fd94557bd3918f2bea8a30 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 15:45:32 +0200
Subject: [PATCH 1536/2196] gc-auto.sh: Add some more instrumentation

---
 tests/gc-auto.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 54ead227ef6..5c8be7c0ecf 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -17,6 +17,7 @@ expr=$(cat <<EOF
 with import ./config.nix; mkDerivation {
   name = "gc-A";
   buildCommand = ''
+    set -x
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
     mkdir \$out
     echo foo > \$out/bar
@@ -32,14 +33,11 @@ with import ./config.nix; mkDerivation {
 EOF
 )
 
-nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
-    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid=$!
-
 expr2=$(cat <<EOF
 with import ./config.nix; mkDerivation {
   name = "gc-B";
   buildCommand = ''
+    set -x
     mkdir \$out
     echo foo > \$out/bar
     echo 1...
@@ -53,6 +51,10 @@ with import ./config.nix; mkDerivation {
 EOF
 )
 
+nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
+pid=$!
+
 nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1
 

From e07ec8d27e08bf23eccab079b044a6f1b37f3ac9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 16:02:12 +0200
Subject: [PATCH 1537/2196] Support allowSubstitutes attribute in structured
 attribute derivations

Hopefully fixes #3081 (didn't test).
---
 src/libstore/build.cc              | 2 +-
 src/libstore/derivations.cc        | 6 ------
 src/libstore/derivations.hh        | 2 --
 src/libstore/misc.cc               | 4 +++-
 src/libstore/parsed-derivations.cc | 5 +++++
 src/libstore/parsed-derivations.hh | 2 ++
 6 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index be52b66a7de..ab725e8e923 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1197,7 +1197,7 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes && drv->substitutesAllowed())
+    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
         for (auto & i : invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 3961126fff9..23fcfb281a7 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const
 }
 
 
-bool BasicDerivation::substitutesAllowed() const
-{
-    return get(env, "allowSubstitutes", "1") == "1";
-}
-
-
 bool BasicDerivation::isBuiltin() const
 {
     return string(builder, 0, 8) == "builtin:";
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 9753e796db5..8e02c9bc57c 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -56,8 +56,6 @@ struct BasicDerivation
        the given derivation. */
     Path findOutput(const string & id) const;
 
-    bool substitutesAllowed() const;
-
     bool isBuiltin() const;
 
     /* Return true iff this is a fixed-output derivation. */
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index adcce026fa1..dddf134300d 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -1,4 +1,5 @@
 #include "derivations.hh"
+#include "parsed-derivations.hh"
 #include "globals.hh"
 #include "local-store.hh"
 #include "store-api.hh"
@@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets,
             }
 
             Derivation drv = derivationFromPath(i2.first);
+            ParsedDerivation parsedDrv(i2.first, drv);
 
             PathSet invalid;
             for (auto & j : drv.outputs)
@@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets,
                     invalid.insert(j.second.path);
             if (invalid.empty()) return;
 
-            if (settings.useSubstitutes && drv.substitutesAllowed()) {
+            if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 17fde00a016..87be8a24ead 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const
     return getBoolAttr("preferLocalBuild") && canBuildLocally();
 }
 
+bool ParsedDerivation::substitutesAllowed() const
+{
+    return getBoolAttr("allowSubstitutes", true);
+}
+
 }
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index ed07dc652e8..9bde4b4dcfc 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -30,6 +30,8 @@ public:
     bool canBuildLocally() const;
 
     bool willBuildLocally() const;
+
+    bool substitutesAllowed() const;
 };
 
 }

From 08ee36495087e3d6f573615c050497ec2298cafa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Sep 2019 18:11:43 +0200
Subject: [PATCH 1538/2196] gc-auto.sh: More test fixes

---
 tests/gc-auto.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 5c8be7c0ecf..87f1d6798ef 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -23,10 +23,13 @@ with import ./config.nix; mkDerivation {
     echo foo > \$out/bar
     echo 1...
     sleep 2
-    echo 200 > $fake_free
+    echo 200 > ${fake_free}.tmp1
+    mv ${fake_free}.tmp1 $fake_free
     echo 2...
-    sleep 2
+    sleep 1
     echo 3...
+    sleep 1
+    echo 4...
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
   '';
 }
@@ -42,10 +45,13 @@ with import ./config.nix; mkDerivation {
     echo foo > \$out/bar
     echo 1...
     sleep 2
-    echo 200 > $fake_free
+    echo 200 > ${fake_free}.tmp2
+    mv ${fake_free}.tmp2 $fake_free
     echo 2...
-    sleep 2
+    sleep 1
     echo 3...
+    sleep 1
+    echo 4...
   '';
 }
 EOF

From 5fad9d01c242ffa9d83cddeeacb9abbaa3848c7c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 12:52:54 +0200
Subject: [PATCH 1539/2196] gc-auto.sh: Increase sleep time

---
 tests/gc-auto.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 87f1d6798ef..de1e2cfe405 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -26,9 +26,9 @@ with import ./config.nix; mkDerivation {
     echo 200 > ${fake_free}.tmp1
     mv ${fake_free}.tmp1 $fake_free
     echo 2...
-    sleep 1
+    sleep 2
     echo 3...
-    sleep 1
+    sleep 2
     echo 4...
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
   '';
@@ -48,9 +48,9 @@ with import ./config.nix; mkDerivation {
     echo 200 > ${fake_free}.tmp2
     mv ${fake_free}.tmp2 $fake_free
     echo 2...
-    sleep 1
+    sleep 2
     echo 3...
-    sleep 1
+    sleep 2
     echo 4...
   '';
 }

From b774845af7a645b44bff69cf9f655c47fe4b9fb2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 12:53:22 +0200
Subject: [PATCH 1540/2196] Set release date

---
 doc/manual/release-notes/rl-2.3.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index a1f4a4b756f..291b56448a0 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -4,7 +4,7 @@
       version="5.0"
       xml:id="ssec-relnotes-2.3">
 
-<title>Release 2.3 (2019-08-??)</title>
+<title>Release 2.3 (2019-09-04)</title>
 
 <para>This is primarily a bug fix release. However, it makes some
 incompatible changes:</para>

From 252c78b288426bae2f379e8320d396b7eb40253b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 15:59:07 +0200
Subject: [PATCH 1541/2196] Tweak release notes

---
 doc/manual/release-notes/rl-2.3.xml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index 291b56448a0..80fbd63966a 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -13,9 +13,8 @@ incompatible changes:</para>
 
   <listitem>
     <para>Nix now uses BSD file locks instead of POSIX file
-    locks. Since previous releases used POSIX file locks, you should
-    not use Nix 2.2 and previous releases at the same time on a Nix
-    store.</para>
+    locks. Because of this, you should not use Nix 2.3 and previous
+    releases at the same time on a Nix store.</para>
   </listitem>
 
 </itemizedlist>
@@ -47,9 +46,9 @@ incompatible changes:</para>
   </listitem>
 
   <listitem>
-    <para><command>nix</command>: Add
+    <para>The <command>nix</command> command has a new
     <option>--print-build-logs</option> (<option>-L</option>) flag to
-    print build log output to stderr rather than showing the last log
+    print build log output to stderr, rather than showing the last log
     line in the progress bar. To distinguish between concurrent
     builds, log lines are prefixed by the name of the package.
     </para>
@@ -57,7 +56,7 @@ incompatible changes:</para>
 
   <listitem>
     <para>Builds are now executed in a pseudo-terminal, and the
-    <envar>TERM</envar> evnironment variable is set to
+    <envar>TERM</envar> environment variable is set to
     <literal>xterm-256color</literal>. This allows many programs
     (e.g. <command>gcc</command>, <command>clang</command>,
     <command>cmake</command>) to print colorized log output.</para>

From d20f814cdef55fd9d7cecf464b32fb8b5efecb1c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 15:59:33 +0200
Subject: [PATCH 1542/2196] Bump version

---
 .version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.version b/.version
index c0943d3e98d..7208c218290 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.3
\ No newline at end of file
+2.4
\ No newline at end of file

From 5dafde28dbec379678da6d033cdc5c48856babf5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 19:24:35 +0200
Subject: [PATCH 1543/2196] BinaryCacheStore: Add index-debug-info option

This integrates the functionality of the index-debuginfo program in
nixos-channel-scripts to maintain an index of DWARF debuginfo files in
a format usable by dwarffs. Thus the debug info index is updated by
Hydra rather than by the channel mirroring script.

Example usage:

  $ nix copy --to 'file:///tmp/binary-cache?index-debug-info=true' /nix/store/vr9mhcch3fljzzkjld3kvkggvpq38cva-nix-2.2.2-debug

  $ cat /tmp/binary-cache/debuginfo/036b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug
  {"archive":"../nar/0313h2kdhk4v73xna9ysiksp2v8xrsk5xsw79mmwr3rg7byb4ka8.nar.xz","member":"lib/debug/.build-id/03/6b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug"}

Fixes #3083.
---
 src/libstore/binary-cache-store.cc       | 80 ++++++++++++++++++++----
 src/libstore/binary-cache-store.hh       |  1 +
 src/libstore/local-binary-cache-store.cc |  2 +
 3 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 10cde8704bb..8e6f1f55d71 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -10,10 +10,13 @@
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
 #include "json.hh"
+#include "thread-pool.hh"
 
 #include <chrono>
-
 #include <future>
+#include <regex>
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
     auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
 
+    auto narAccessor = makeNarAccessor(nar);
+
+    if (accessor_)
+        accessor_->addToCache(info.path, *nar, narAccessor);
+
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             JSONObject jsonRoot(jsonOut);
             jsonRoot.attr("version", 1);
 
-            auto narAccessor = makeNarAccessor(nar);
-
-            if (accessor_)
-                accessor_->addToCache(info.path, *nar, narAccessor);
-
             {
                 auto res = jsonRoot.placeholder("root");
                 listNar(res, narAccessor, "", true);
@@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
         upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
     }
 
-    else {
-        if (accessor_)
-            accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
-    }
-
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
@@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
         % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
-    /* Atomically write the NAR file. */
     narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "br" ? ".br" :
            "");
+
+    /* Optionally maintain an index of DWARF debug info files
+       consisting of JSON files named 'debuginfo/<build-id>' that
+       specify the NAR file and member containing the debug info. */
+    if (writeDebugInfo) {
+
+        std::string buildIdDir = "/lib/debug/.build-id";
+
+        if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+
+            ThreadPool threadPool(25);
+
+            auto doFile = [&](std::string member, std::string key, std::string target) {
+                checkInterrupt();
+
+                nlohmann::json json;
+                json["archive"] = target;
+                json["member"] = member;
+
+                // FIXME: or should we overwrite? The previous link may point
+                // to a GC'ed file, so overwriting might be useful...
+                if (fileExists(key)) return;
+
+                printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
+
+                upsertFile(key, json.dump(), "application/json");
+            };
+
+            std::regex regex1("^[0-9a-f]{2}$");
+            std::regex regex2("^[0-9a-f]{38}\\.debug$");
+
+            for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
+                auto dir = buildIdDir + "/" + s1;
+
+                if (narAccessor->stat(dir).type != FSAccessor::tDirectory
+                    || !std::regex_match(s1, regex1))
+                    continue;
+
+                for (auto & s2 : narAccessor->readDirectory(dir)) {
+                    auto debugPath = dir + "/" + s2;
+
+                    if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
+                        || !std::regex_match(s2, regex2))
+                        continue;
+
+                    auto buildId = s1 + s2;
+
+                    std::string key = "debuginfo/" + buildId;
+                    std::string target = "../" + narInfo->url;
+
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                }
+            }
+
+            threadPool.process();
+        }
+    }
+
+    /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
         upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index af108880cba..c772922949a 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -17,6 +17,7 @@ public:
 
     const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{this, false, "parallel-compression",
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index b7001795be4..9f99ee76d84 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -63,6 +63,8 @@ class LocalBinaryCacheStore : public BinaryCacheStore
 void LocalBinaryCacheStore::init()
 {
     createDirs(binaryCacheDir + "/nar");
+    if (writeDebugInfo)
+        createDirs(binaryCacheDir + "/debuginfo");
     BinaryCacheStore::init();
 }
 

From b9d6c6f000d2ba3efd7c124e46982425962f4fbb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 21:44:40 +0200
Subject: [PATCH 1544/2196] upload-release.pl: Fix sshfs call

---
 maintainers/upload-release.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 1cdf5ed16dc..77534babba2 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -44,7 +44,7 @@ sub fetch {
 
 File::Path::make_path($releasesDir);
 if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die;
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
 }
 
 my $releaseDir = "$releasesDir/nix/$releaseName";

From a56b51a0ba7b0d6fdff7fd0127a118185b146f4f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Sep 2019 14:14:03 +0200
Subject: [PATCH 1545/2196] Disable OpenSSL lock callback on OpenSSL >= 1.1.1

---
 src/libmain/shared.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 0afddfb785d..d3dbfbc4459 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -80,6 +80,7 @@ string getArg(const string & opt,
 }
 
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
 /* OpenSSL is not thread-safe by default - it will randomly crash
    unless the user supplies a mutex locking function. So let's do
    that. */
@@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
     else
         opensslLocks[type].unlock();
 }
+#endif
 
 
 static void sigHandler(int signo) { }
@@ -105,9 +107,11 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
     /* Initialise OpenSSL locking. */
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
+#endif
 
     loadConfFile();
 

From 92ede15dd902f7c1d2771c194b8bb73fe406840f Mon Sep 17 00:00:00 2001
From: Julien Tanguy <julien.tanguy@jhome.fr>
Date: Wed, 11 Sep 2019 14:11:37 +0200
Subject: [PATCH 1546/2196] docs: Fix a typo in github in an example

---
 doc/manual/expressions/builtins.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 4c1d618e951..39407c9ac38 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -448,7 +448,7 @@ stdenv.mkDerivation { … }
       <example>
         <title>Fetching an arbitrary ref</title>
         <programlisting>builtins.fetchGit {
-  url = "https://gitub.com/NixOS/nix.git";
+  url = "https://github.com/NixOS/nix.git";
   ref = "refs/heads/0.5-release";
 }</programlisting>
       </example>

From ae244af242ca3621e5a3b9196f27d9fcbf297266 Mon Sep 17 00:00:00 2001
From: Julien Tanguy <julien.tanguy@jhome.fr>
Date: Wed, 11 Sep 2019 14:18:47 +0200
Subject: [PATCH 1547/2196] docs: Use the explicit ref for fetchGit with a tag

With the merge of #2582, the syntax "tags/1.9" for refs does not work
anymore.
However, the new syntax "refs/tags/1.9" seems to support annotated tags,
such as "refs/tags/2.0".

Closes #2385.
---
 doc/manual/expressions/builtins.xml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 39407c9ac38..191fa7810fb 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -499,11 +499,8 @@ stdenv.mkDerivation { … }
         <title>Fetching a tag</title>
         <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
-  ref = "tags/1.9";
+  ref = "refs/tags/1.9";
 }</programlisting>
-        <note><para>Due to a bug (<link
-        xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
-        only non-annotated tags can be fetched.</para></note>
       </example>
 
       <example>

From 6b83174ffffbdfc3f876d94d5178e0b83f675cae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Sep 2019 20:05:27 +0200
Subject: [PATCH 1548/2196] std::uncaught_exception() ->
 std::uncaught_exceptions()

The former is deprecated in C++17. Fixes a clang warning.
---
 src/libstore/remote-store.cc | 2 +-
 src/libutil/json.hh          | 2 +-
 src/libutil/util.cc          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index e38fe49a7f9..89b69941497 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -228,7 +228,7 @@ struct ConnectionHandle
 
     ~ConnectionHandle()
     {
-        if (!daemonException && std::uncaught_exception()) {
+        if (!daemonException && std::uncaught_exceptions()) {
             handle.markBad();
             debug("closing daemon connection because of an exception");
         }
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
index 02a39917fb5..45a22f01121 100644
--- a/src/libutil/json.hh
+++ b/src/libutil/json.hh
@@ -170,7 +170,7 @@ public:
 
     ~JSONPlaceholder()
     {
-        assert(!first || std::uncaught_exception());
+        assert(!first || std::uncaught_exceptions());
     }
 
     template<typename T>
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1b744999153..e65f8ee56c3 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1169,7 +1169,7 @@ void _interrupted()
     /* Block user interrupts while an exception is being handled.
        Throwing an exception while another exception is being handled
        kills the program! */
-    if (!interruptThrown && !std::uncaught_exception()) {
+    if (!interruptThrown && !std::uncaught_exceptions()) {
         interruptThrown = true;
         throw Interrupted("interrupted by the user");
     }

From 9533d85ce009c65722a2addc55b64ee51f15dea3 Mon Sep 17 00:00:00 2001
From: Alexandre Esteves <alexandre.fmp.esteves@gmail.com>
Date: Tue, 17 Sep 2019 23:40:49 +0100
Subject: [PATCH 1549/2196] Move 'builtins.splitVersion' to position respecting
 alphabetical order

---
 doc/manual/expressions/builtins.xml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 191fa7810fb..9bd86e12055 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -170,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-splitVersion'>
-    <term><function>builtins.splitVersion</function>
-    <replaceable>s</replaceable></term>
-
-    <listitem><para>Split a string representing a version into its
-    components, by the same version splitting logic underlying the
-    version comparison in <link linkend="ssec-version-comparisons">
-    <command>nix-env -u</command></link>.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id='builtin-concatLists'>
     <term><function>builtins.concatLists</function>
     <replaceable>lists</replaceable></term>
@@ -1272,6 +1260,19 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
     </para></listitem>
   </varlistentry>
 
+
+  <varlistentry xml:id='builtin-splitVersion'>
+    <term><function>builtins.splitVersion</function>
+    <replaceable>s</replaceable></term>
+
+    <listitem><para>Split a string representing a version into its
+    components, by the same version splitting logic underlying the
+    version comparison in <link linkend="ssec-version-comparisons">
+    <command>nix-env -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id='builtin-stringLength'>
     <term><function>builtins.stringLength</function>
     <replaceable>e</replaceable></term>

From 619cc4af855fab7b0400586a4fd40745b23e72ad Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 18 Sep 2019 11:40:11 +0000
Subject: [PATCH 1550/2196] function-trace: always show the trace

If the user invokes nix with --trace-function-calls it means that they
want to see the trace.
---
 contrib/stack-collapse.py     | 5 ++---
 src/libexpr/function-trace.hh | 4 ++--
 tests/function-trace.sh       | 1 -
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py
index ee77e158945..f5602c95c48 100755
--- a/contrib/stack-collapse.py
+++ b/contrib/stack-collapse.py
@@ -1,12 +1,11 @@
 #!/usr/bin/env nix-shell
 #!nix-shell -i python3 -p python3 --pure
 
-# To be used with `--trace-function-calls` and `-vvvv` and
-# `flamegraph.pl`.
+# To be used with `--trace-function-calls` and `flamegraph.pl`.
 #
 # For example:
 #
-# nix-instantiate --trace-function-calls -vvvv '<nixpkgs>' -A hello 2> nix-function-calls.trace
+# nix-instantiate --trace-function-calls '<nixpkgs>' -A hello 2> nix-function-calls.trace
 # ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded
 # nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg"
 
diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh
index 8234b760312..8b0ec848d5a 100644
--- a/src/libexpr/function-trace.hh
+++ b/src/libexpr/function-trace.hh
@@ -12,13 +12,13 @@ struct FunctionCallTrace
     FunctionCallTrace(const Pos & pos) : pos(pos) {
         auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
         auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        vomit("function-trace entered %1% at %2%", pos, ns.count());
+        printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
     }
 
     ~FunctionCallTrace() {
         auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
         auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        vomit("function-trace exited %1% at %2%", pos, ns.count());
+        printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
     }
 };
 }
diff --git a/tests/function-trace.sh b/tests/function-trace.sh
index f7d93b4356e..182a4d5c287 100755
--- a/tests/function-trace.sh
+++ b/tests/function-trace.sh
@@ -8,7 +8,6 @@ expect_trace() {
     actual=$(
         nix-instantiate \
             --trace-function-calls \
-            -vvvv \
             --expr "$expr" 2>&1 \
             | grep "function-trace" \
             | sed -e 's/ [0-9]*$//'

From c6a542f22a165f711fc0730bfcfbd936ad1c3dfb Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sat, 21 Sep 2019 18:53:15 -0400
Subject: [PATCH 1551/2196] Fix spelling in comment

---
 src/libstore/build.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ab725e8e923..9a6729f9e52 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2364,7 +2364,7 @@ void DerivationGoal::startBuilder()
                 child = clone(childEntry, stack + stackSize, flags, this);
             }
             if (child == -1 && (errno == EPERM || errno == EINVAL)) {
-                /* Some distros patch Linux to not allow unpriveleged
+                /* Some distros patch Linux to not allow unprivileged
                  * user namespaces. If we get EPERM or EINVAL, try
                  * without CLONE_NEWUSER and see if that works.
                  */
@@ -2410,8 +2410,7 @@ void DerivationGoal::startBuilder()
         writeFile("/proc/" + std::to_string(pid) + "/gid_map",
             (format("%d %d 1") % sandboxGid % hostGid).str());
 
-        /* Signal the builder that we've updated its user
-           namespace. */
+        /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
         userNamespaceSync.writeSide = -1;
 

From e63c9e73e3e5d1f31fa5065c9ff59f442dd07d0e Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Sun, 22 Sep 2019 10:57:20 +0000
Subject: [PATCH 1552/2196] libstore: don't forward --show-trace

---
 src/libstore/remote-store.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1c2e23f9cd5..76107db54ba 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -198,6 +198,7 @@ void RemoteStore::setOptions(Connection & conn)
         overrides.erase(settings.maxSilentTime.name);
         overrides.erase(settings.buildCores.name);
         overrides.erase(settings.useSubstitutes.name);
+        overrides.erase(settings.showTrace.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;

From bd79c1f6f6391786772a8a79962abe22f374cca4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 22 Sep 2019 21:29:33 +0200
Subject: [PATCH 1553/2196] Don't catch exceptions by value

(cherry picked from commit 893be6f5e36abb58bbaa9c49055a5218114dd514)
---
 src/libexpr/json-to-value.cc    | 4 ++--
 src/libexpr/primops/fetchGit.cc | 2 +-
 src/libstore/local-store.cc     | 4 ++--
 src/nix-build/nix-build.cc      | 2 +-
 src/nix/edit.cc                 | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 3f601795778..8bae986f97f 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v)
                 mkFloat(v, stod(tmp_number));
             else
                 mkInt(v, stol(tmp_number));
-        } catch (std::invalid_argument e) {
+        } catch (std::invalid_argument & e) {
             throw JSONParseError("invalid JSON number");
-        } catch (std::out_of_range e) {
+        } catch (std::out_of_range & e) {
             throw JSONParseError("out-of-range JSON number");
         }
     }
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 6229fef8d02..90f600284b2 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -38,7 +38,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         try {
             runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError e) {
+        } catch (ExecError & e) {
             if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
             clean = false;
         }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2fcf08491c2..a2af51d0ed5 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -879,8 +879,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
                     info->references,
                     narInfo ? narInfo->fileSize : 0,
                     info->narSize};
-            } catch (InvalidPath) {
-            } catch (SubstituterDisabled) {
+            } catch (InvalidPath &) {
+            } catch (SubstituterDisabled &) {
             } catch (Error & e) {
                 if (settings.tryFallback)
                     printError(e.what());
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index c6a4d416648..33ad28704a6 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -280,7 +280,7 @@ static void _main(int argc, char * * argv)
                 auto absolute = i;
                 try {
                     absolute = canonPath(absPath(i), true);
-                } catch (Error e) {};
+                } catch (Error & e) {};
                 if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
                 drvs.push_back(DrvInfo(*state, store, absolute));
             else
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index c9671f76d0f..a6169f1183c 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -55,7 +55,7 @@ struct CmdEdit : InstallableCommand
         int lineno;
         try {
             lineno = std::stoi(std::string(pos, colon + 1));
-        } catch (std::invalid_argument e) {
+        } catch (std::invalid_argument & e) {
             throw Error("cannot parse line number '%s'", pos);
         }
 

From 3a022d45993b6fa8c7bf03517a3a3d1a2ab15f4a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 20 Sep 2019 13:48:53 +0200
Subject: [PATCH 1554/2196] Shut up some warnings

(cherry picked from commit 99e8e58f2de9941353b47ed14fbe4ed76d635519)
---
 src/libstore/download.hh    | 2 ++
 src/libstore/fs-accessor.hh | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index c6838184694..68565bf462a 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -91,6 +91,8 @@ class Store;
 
 struct Downloader
 {
+    virtual ~Downloader() { }
+
     /* Enqueue a download request, returning a future to the result of
        the download. The future may throw a DownloadError
        exception. */
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index f703e1d1540..64780a6daf4 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -19,6 +19,8 @@ public:
         uint64_t narOffset = 0; // regular files only
     };
 
+    virtual ~FSAccessor() { }
+
     virtual Stat stat(const Path & path) = 0;
 
     virtual StringSet readDirectory(const Path & path) = 0;

From 10bfc5c0d09d5508e3dab4c32f3368caeb5f7f56 Mon Sep 17 00:00:00 2001
From: Joseph Lucas <lucas.jojo@gmail.com>
Date: Mon, 23 Sep 2019 13:18:59 +0000
Subject: [PATCH 1555/2196] Update garbage-collection.xml readability

1. remove a typo space
2. Simplify negative style by using affirmative style
---
 doc/manual/packages/garbage-collection.xml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml
index a1b0ef22a11..b506f22b030 100644
--- a/doc/manual/packages/garbage-collection.xml
+++ b/doc/manual/packages/garbage-collection.xml
@@ -52,12 +52,13 @@ garbage collector as follows:
 <screen>
 $ nix-store --gc</screen>
 
-The behaviour of the gargage collector is affected by the <literal>keep-
-derivations</literal> (default: true) and <literal>keep-outputs</literal>
+The behaviour of the gargage collector is affected by the 
+<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal>
 (default: false) options in the Nix configuration file. The defaults will ensure
-that all derivations that are not build-time dependencies of garbage collector roots
-will be collected but that all output paths that are not runtime dependencies
-will be collected. (This is usually what you want, but while you are developing
+that all derivations that are build-time dependencies of garbage collector roots
+will be kept and that all output paths that are runtime dependencies
+will be kept as well. All other derivations or paths will be collected. 
+(This is usually what you want, but while you are developing
 it may make sense to keep outputs to ensure that rebuild times are quick.)
 
 If you are feeling uncertain, you can also first view what files would

From 168a8879165dd0deab1a93d343a6003146f37031 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 12 Jun 2019 10:34:13 +0200
Subject: [PATCH 1556/2196] Fix fetchTarball with chroot stores

Fixes #2405.
---
 src/libexpr/primops.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 350dba47409..d4c60f870ed 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2089,12 +2089,12 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !request.expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, request).path;
+    auto res = getDownloader()->downloadCached(state.store, request);
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(res);
+        state.allowedPaths->insert(res.path);
 
-    mkString(v, res, PathSet({res}));
+    mkString(v, res.storePath, PathSet({res.storePath}));
 }
 
 

From 74b4737d8f0e1922ef5314a158271acf81cd79f8 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 1 Oct 2019 21:07:07 -0400
Subject: [PATCH 1557/2196] Add libatomic for 32-bit ARM

Fixes #3113
---
 configure.ac | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/configure.ac b/configure.ac
index a52830b3835..96a2065b113 100644
--- a/configure.ac
+++ b/configure.ac
@@ -157,6 +157,11 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix
 # ends up with LDFLAGS being empty, so we set it afterwards.
 LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
 
+# Boost atomic needs GCC libatomic on 32-bit ARM
+case "$host_cpu" in
+  armv5*|armv6*|armv7*) LIBS="-latomic $LIBS"
+esac
+
 
 # Look for OpenSSL, a required dependency.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])

From b1c34152feac76c9baded58732068541b7a586f1 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 1 Oct 2019 21:22:18 -0400
Subject: [PATCH 1558/2196] Use more robust test for libatomics

Taken from Mesa configure script:

https://github.com/mesa3d/mesa/blob/17.2/configure.ac#L405-L427
---
 configure.ac | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index 96a2065b113..ebe6d4267f4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -157,11 +157,18 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix
 # ends up with LDFLAGS being empty, so we set it afterwards.
 LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
 
-# Boost atomic needs GCC libatomic on 32-bit ARM
-case "$host_cpu" in
-  armv5*|armv6*|armv7*) LIBS="-latomic $LIBS"
-esac
-
+# On some platforms, new-style atomics need a helper library
+AC_MSG_CHECKING(whether -latomic is needed)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#include <stdint.h>
+uint64_t v;
+int main() {
+    return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE);
+}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
+AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
+if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
+    LIBS="-latomic $LIBS"
+fi
 
 # Look for OpenSSL, a required dependency.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])

From 15e70c662e86c42658cdbc76b54da0f3f2be3c30 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 2 Oct 2019 16:26:15 +0200
Subject: [PATCH 1559/2196] Fix indentation

---
 src/nix-build/nix-build.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 33ad28704a6..2b36846cd71 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -282,12 +282,12 @@ static void _main(int argc, char * * argv)
                     absolute = canonPath(absPath(i), true);
                 } catch (Error & e) {};
                 if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
-                drvs.push_back(DrvInfo(*state, store, absolute));
-            else
-                /* If we're in a #! script, interpret filenames
-                   relative to the script. */
-                exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
-                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+                    drvs.push_back(DrvInfo(*state, store, absolute));
+                else
+                    /* If we're in a #! script, interpret filenames
+                       relative to the script. */
+                    exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
+                                        inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
             }
         }
 

From 93b1ce1ac522ff92626da5cdd689008063935844 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 4 Oct 2019 16:34:59 +0200
Subject: [PATCH 1560/2196] Revert "std::uncaught_exception() ->
 std::uncaught_exceptions()"

This reverts commit 6b83174ffffbdfc3f876d94d5178e0b83f675cae because
it doesn't work on macOS yet.

https://hydra.nixos.org/build/102617587
---
 src/libstore/remote-store.cc | 2 +-
 src/libutil/json.hh          | 2 +-
 src/libutil/util.cc          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b9e7a80bab6..b7f202a6bbc 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -229,7 +229,7 @@ struct ConnectionHandle
 
     ~ConnectionHandle()
     {
-        if (!daemonException && std::uncaught_exceptions()) {
+        if (!daemonException && std::uncaught_exception()) {
             handle.markBad();
             debug("closing daemon connection because of an exception");
         }
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
index 45a22f01121..02a39917fb5 100644
--- a/src/libutil/json.hh
+++ b/src/libutil/json.hh
@@ -170,7 +170,7 @@ public:
 
     ~JSONPlaceholder()
     {
-        assert(!first || std::uncaught_exceptions());
+        assert(!first || std::uncaught_exception());
     }
 
     template<typename T>
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e65f8ee56c3..1b744999153 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1169,7 +1169,7 @@ void _interrupted()
     /* Block user interrupts while an exception is being handled.
        Throwing an exception while another exception is being handled
        kills the program! */
-    if (!interruptThrown && !std::uncaught_exceptions()) {
+    if (!interruptThrown && !std::uncaught_exception()) {
         interruptThrown = true;
         throw Interrupted("interrupted by the user");
     }

From c5bd564c69ec1fb4f2ffc1091653d99ebc87e284 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sun, 6 Oct 2019 16:54:56 -0400
Subject: [PATCH 1561/2196] nix doctor: add more logging output to checks

When running nix doctor on a healthy system, it just prints the store URI and
nothing else. This makes it unclear whether the system is in a good state and
what check(s) it actually ran, since some of the checks are optional depending
on the store type.

This commit updates nix doctor to print an colored log message for every check
that it does, and explicitly state whether that check was a PASS or FAIL to make
it clear to the user whether the system passed its checkup with the doctor.

Fixes #3084
---
 src/nix/doctor.cc | 67 +++++++++++++++++++++++++++++------------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 7b544461947..795236c5f50 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -1,11 +1,17 @@
+#include <sstream>
+
 #include "command.hh"
+#include "logging.hh"
 #include "serve-protocol.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "util.hh"
 #include "worker-protocol.hh"
 
 using namespace nix;
 
+namespace {
+
 std::string formatProtocol(unsigned int proto)
 {
     if (proto) {
@@ -16,6 +22,18 @@ std::string formatProtocol(unsigned int proto)
     return "unknown";
 }
 
+bool checkPass(const std::string & msg) {
+    logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg);
+    return true;
+}
+
+bool checkFail(const std::string & msg) {
+    logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg);
+    return false;
+}
+
+}
+
 struct CmdDoctor : StoreCommand
 {
     bool success = true;
@@ -27,13 +45,12 @@ struct CmdDoctor : StoreCommand
 
     std::string description() override
     {
-        return "check your system for potential problems";
+        return "check your system for potential problems and print a PASS or FAIL for each check.";
     }
 
     void run(ref<Store> store) override
     {
-        std::cout << "Store uri: " << store->getUri() << std::endl;
-        std::cout << std::endl;
+        logger->log("Running checks against store uri: " + store->getUri());
 
         auto type = getStoreType();
 
@@ -56,15 +73,14 @@ struct CmdDoctor : StoreCommand
                 dirs.insert(dirOf(canonPath(dir + "/nix-env", true)));
 
         if (dirs.size() != 1) {
-            std::cout << "Warning: multiple versions of nix found in PATH." << std::endl;
-            std::cout << std::endl;
+            std::stringstream ss;
+            ss << "Multiple versions of nix found in PATH:\n";
             for (auto & dir : dirs)
-                std::cout << "  " << dir << std::endl;
-            std::cout << std::endl;
-            return false;
+                ss << "  " << dir << "\n";
+            return checkFail(ss.str());
         }
 
-        return true;
+        return checkPass("PATH contains only one nix version.");
     }
 
     bool checkProfileRoots(ref<Store> store)
@@ -87,17 +103,17 @@ struct CmdDoctor : StoreCommand
         }
 
         if (!dirs.empty()) {
-            std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl;
-            std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl;
-            std::cout << "garbage collected, resulting in broken symlinks." << std::endl;
-            std::cout << std::endl;
+            std::stringstream ss;
+            ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n"
+               << "The generation this profile points to might not have a gcroot and could be\n"
+               << "garbage collected, resulting in broken symlinks.\n\n";
             for (auto & dir : dirs)
-                std::cout << "  " << dir << std::endl;
-            std::cout << std::endl;
-            return false;
+                ss << "  " << dir << "\n";
+            ss << "\n";
+            return checkFail(ss.str());
         }
 
-        return true;
+        return checkPass("All profiles are gcroots.");
     }
 
     bool checkStoreProtocol(unsigned int storeProto)
@@ -107,17 +123,16 @@ struct CmdDoctor : StoreCommand
             : PROTOCOL_VERSION;
 
         if (clientProto != storeProto) {
-            std::cout << "Warning: protocol version of this client does not match the store." << std::endl;
-            std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl;
-            std::cout << "sync with the daemon." << std::endl;
-            std::cout << std::endl;
-            std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl;
-            std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl;
-            std::cout << std::endl;
-            return false;
+            std::stringstream ss;
+            ss << "Warning: protocol version of this client does not match the store.\n"
+               << "While this is not necessarily a problem it's recommended to keep the client in\n"
+               << "sync with the daemon.\n\n"
+               << "Client protocol: " << formatProtocol(clientProto) << "\n"
+               << "Store protocol: " << formatProtocol(storeProto) << "\n\n";
+            return checkFail(ss.str());
         }
 
-        return true;
+        return checkPass("Client protocol matches store protocol.");
     }
 };
 

From 6f6cb5e3880d0c7a1dd2bc13c2e0be8ce0ae9fa1 Mon Sep 17 00:00:00 2001
From: Sam Doshi <sam@metal-fish.co.uk>
Date: Mon, 7 Oct 2019 11:40:42 +0100
Subject: [PATCH 1562/2196] nix search: remove verbose example

---
 src/nix/search.cc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index e086de2260a..764dcb70a5d 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -80,10 +80,6 @@ struct CmdSearch : SourceExprCommand, MixJSON
             Example{
                 "To search for git and frontend or gui:",
                 "nix search git 'frontend|gui'"
-            },
-            Example{
-                "To display the description of the found packages:",
-                "nix search git --verbose"
             }
         };
     }

From 00a567588e43b51944a9b91c1e890bda27773aed Mon Sep 17 00:00:00 2001
From: Danny Bautista <pyrolagus@nerdpol.ch>
Date: Tue, 8 Oct 2019 14:02:40 -0400
Subject: [PATCH 1563/2196] Fix typos in the Nix Manual.

---
 doc/manual/command-ref/conf-file.xml               | 2 +-
 doc/manual/command-ref/nix-env.xml                 | 2 +-
 doc/manual/command-ref/nix-prefetch-url.xml        | 2 +-
 doc/manual/expressions/builtins.xml                | 4 ++--
 doc/manual/expressions/simple-building-testing.xml | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 320e15339d9..48dce7c950d 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -433,7 +433,7 @@ builtins.fetchurl {
   <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivation
+    are not stored in Nix user environments.  That is, the derivations of
     any build-time-only dependencies may be garbage-collected.</para>
 
     <para>If <literal>true</literal>, when you add a Nix derivation to
diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index c8c01f9a44f..d257a5e49c7 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -659,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 <literal>gcc-3.3.1</literal> are split into two parts: the package
 name (<literal>gcc</literal>), and the version
 (<literal>3.3.1</literal>).  The version part starts after the first
-dash not following by a letter.  <varname>x</varname> is considered an
+dash not followed by a letter.  <varname>x</varname> is considered an
 upgrade of <varname>y</varname> if their package names match, and the
 version of <varname>y</varname> is higher that that of
 <varname>x</varname>.</para>
diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml
index 8ef748c74be..621ded72ec2 100644
--- a/doc/manual/command-ref/nix-prefetch-url.xml
+++ b/doc/manual/command-ref/nix-prefetch-url.xml
@@ -53,7 +53,7 @@ avoided.</para>
 <para>If <replaceable>hash</replaceable> is specified, then a download
 is not performed if the Nix store already contains a file with the
 same hash and base name.  Otherwise, the file is downloaded, and an
-error if signaled if the actual hash of the file does not match the
+error is signaled if the actual hash of the file does not match the
 specified hash.</para>
 
 <para>This command prints the hash on standard output.  Additionally,
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index e164d14416b..465fa1e0b00 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -289,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
     <listitem><para>Return element <replaceable>n</replaceable> from
     the list <replaceable>xs</replaceable>.  Elements are counted
-    starting from 0.  A fatal error occurs in the index is out of
+    starting from 0.  A fatal error occurs if the index is out of
     bounds.</para></listitem>
 
   </varlistentry>
@@ -1466,7 +1466,7 @@ in foo</programlisting>
       <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
       <listitem><para>An integer.</para></listitem>
       <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem>
-      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>.</para></listitem>
+      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem>
       <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem>
     </itemizedlist>
     </listitem>
diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
index 0348c082b20..7326a3e76ad 100644
--- a/doc/manual/expressions/simple-building-testing.xml
+++ b/doc/manual/expressions/simple-building-testing.xml
@@ -43,7 +43,7 @@ use <command>nix-build</command>’s <option
 linkend='opt-out-link'>-o</option> switch to give the symlink another
 name.</para>
 
-<para>Nix has a transactional semantics.  Once a build finishes
+<para>Nix has transactional semantics.  Once a build finishes
 successfully, Nix makes a note of this in its database: it registers
 that the path denoted by <envar>out</envar> is now
 <quote>valid</quote>.  If you try to build the derivation again, Nix

From 0847f2f1b3145a62dc34707ba788275ce6b6fc57 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 8 Oct 2019 21:17:27 -0400
Subject: [PATCH 1564/2196] Copy instead of linking launch agent

On Catalina, the /nix filesystem might not be mounted at start time.
To avoid this service not starting, we need to keep the launch agent
outside of the Nix store. A wait4pid will hold for our /nix dir to be
mounted.

Fixes #3125.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 2 +-
 scripts/install-darwin-multi-user.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index b340610e900..39d7757a4e0 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -17,7 +17,7 @@
     <array>
       <string>/bin/sh</string>
       <string>-c</string>
-      <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; @bindir@/nix-daemon</string>
+      <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
     </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>
diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 87c4c2b0582..49076bd5c03 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -39,7 +39,7 @@ EOF
 
 poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
+          cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist

From d4e51aac080e73982e16b43088f9ae404f07136e Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 8 Oct 2019 21:35:09 -0400
Subject: [PATCH 1565/2196] Make preexisting Nix install a warning, not a
 failure

In the multi-user install script, we originally made sure no previous
references to Nix existed. This prevented any previous installs from
contaminating the new install. However, some users need the ability to
repair their existing Nix installation without uninstalling all
references to Nix. This change allows users with existing Nix
installations to use the installer, while still outputing a warning
message on the dangers of this. As a result, the multi-user install
script work much more like the single-user install script has worked
in the past.

This is a requirement for macOS Catalina users now that
/Library/LaunchDaemons/org.nixos.nix-daemon.plisg is not managed by
the Nix store. If there is ever a change to the .plist, all users will
need to rerun this install script to get the new changes. Otherwise,
changes to the launch daemon will require manual interventions.
---
 scripts/install-multi-user.sh | 104 ++--------------------------------
 1 file changed, 5 insertions(+), 99 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a41309e930b..fd0f54184aa 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -278,73 +278,9 @@ EOF
     fi
 
     if type nix-env 2> /dev/null >&2; then
-        failure <<EOF
-Nix already appears to be installed, and this tool assumes it is
-_not_ yet installed.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ "${NIX_REMOTE:-}" != "" ]; then
-        failure <<EOF
-For some reason, \$NIX_REMOTE is set. It really should not be set
-before this installer runs, and it hints that Nix is currently
-installed. Please delete the old Nix installation and start again.
-
-Note: You might need to close your shell window and open a new shell
-to clear the variable.
-EOF
-    fi
-
-    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
-        failure <<EOF
-It looks like \$SSL_CERT_FILE is set to a path that used to be part of
-the old Nix installation. Please unset that variable and try again:
-
-  $ unset SSL_CERT_FILE
-
-EOF
-    fi
-
-    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        if [ -f "$file" ]; then
-            if grep -l "^[^#].*.nix-profile" "$file"; then
-                failure <<EOF
-I found a reference to a ".nix-profile" in $file.
-This has a high chance of breaking a new nix installation. It was most
-likely put there by a previous Nix installer.
-
-Please remove this reference and try running this again. You should
-also look for similar references in:
-
- - ~/.bash_profile
- - ~/.bash_login
- - ~/.profile
-
-or other shell init files that you may have.
-
-$(uninstall_directions)
-EOF
-            fi
-        fi
-    done
-
-    if [ -d /nix/store ] || [ -d /nix/var ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ -d /etc/nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /etc/nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
+        warning <<EOF
+Nix already appears to be installed. This installer may run into issues.
+If an error occurs, try manually uninstalling, then rerunning this script.
 
 $(uninstall_directions)
 EOF
@@ -352,7 +288,7 @@ EOF
 
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-        failure <<EOF
+            failure <<EOF
 When this script runs, it backs up the current $profile_target to
 $profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
 
@@ -364,38 +300,10 @@ in case.
 2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
 it has anything nix-related in it. If it does, something is probably
 quite wrong. Please open an issue or get in touch immediately.
-
-3. Take care to make sure that $profile_target doesn't look like it has
-anything nix-related in it. If it does, and $profile_target _did not_,
-run:
-
-  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-and try again.
-EOF
-        fi
-
-        if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then
-            failure <<EOF
-It looks like $profile_target already has some Nix configuration in
-there. There should be no reason to run this again. If you're having
-trouble, please open an issue.
 EOF
         fi
     done
 
-    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
-    for danger_path in "${danger_paths[@]}"; do
-        if _sudo "making sure that $danger_path doesn't exist" \
-           test -e "$danger_path"; then
-            failure <<EOF
-I found a file at $danger_path, which is a relic of a previous
-installation. You must first delete this file before continuing.
-
-$(uninstall_directions)
-EOF
-        fi
-    done
 }
 
 setup_report() {
@@ -772,9 +680,7 @@ main() {
     welcome_to_nix
     chat_about_sudo
 
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
-        validate_starting_assumptions
-    fi
+    validate_starting_assumptions
 
     setup_report
 

From 65f6d5db6f5ef2724a3dc03e1773c510123287f1 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 8 Oct 2019 22:41:05 -0400
Subject: [PATCH 1566/2196] =?UTF-8?q?Don=E2=80=99t=20source=20bashrc=20in?=
 =?UTF-8?q?=20pure=20mode?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pure mode should not try to source the user’s bashrc file. These may
have many impurities that the user does not expect to get into their
shell.

Fixes #3090
---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 2b36846cd71..3e1081c05c6 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -412,7 +412,7 @@ static void _main(int argc, char * * argv)
         auto rcfile = (Path) tmpDir + "/rc";
         writeFile(rcfile, fmt(
                 (keepTmp ? "" : "rm -rf '%1%'; "s) +
-                "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; "
+                (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;")
                 "%2%"
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "

From 199e888785bd23073e44e56f6c74b95dc7c10ffa Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 8 Oct 2019 23:12:31 -0400
Subject: [PATCH 1567/2196] Handle empty sandbox_shell

Previously, SANDBOX_SHELL was set to empty when unavailable. This
caused issues when actually generating the sandbox. Instead, just set
SANDBOX_SHELL when --with-sandbox-shell= is non-empty. Alternative
implementation to https://github.com/NixOS/nix/pull/3038.
---
 src/libstore/local.mk | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 89fc918c30f..d690fea28c2 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -39,9 +39,12 @@ libstore_CXXFLAGS = \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_MAN_DIR=\"$(mandir)\" \
- -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 
+ifneq ($(sandbox_shell),)
+libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
+endif
+
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh
 
 $(d)/build.cc:

From 8c4a5e7ba1990348b9497c0fc4dc236dda3e7986 Mon Sep 17 00:00:00 2001
From: Dan Callahan <dan.callahan@gmail.com>
Date: Wed, 9 Oct 2019 08:51:49 +0100
Subject: [PATCH 1568/2196] Make nix-daemon.plist less fragile on macOS

We're calling `wait4path` on the full, resolved `@bindir@/nix-daemon` path.

That means we're hardcoding something like:

    /bin/wait4path /nix/store/zs9c5xhp3zv9p23qnjxp87nl5injsi1i-nix-2.3/bin/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon

That seems unnecessarily fragile.

It might be better to wait4path on the path we intend to call.
---
 misc/launchd/org.nixos.nix-daemon.plist.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in
index 39d7757a4e0..9f26296a9d7 100644
--- a/misc/launchd/org.nixos.nix-daemon.plist.in
+++ b/misc/launchd/org.nixos.nix-daemon.plist.in
@@ -17,7 +17,7 @@
     <array>
       <string>/bin/sh</string>
       <string>-c</string>
-      <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
+      <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
     </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>

From 64d88729005841ff3ccfa75134f9aa5e5828da6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 14:46:44 +0200
Subject: [PATCH 1569/2196] nix-build: Fix compilation

---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 3e1081c05c6..085650e4f29 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -412,7 +412,7 @@ static void _main(int argc, char * * argv)
         auto rcfile = (Path) tmpDir + "/rc";
         writeFile(rcfile, fmt(
                 (keepTmp ? "" : "rm -rf '%1%'; "s) +
-                (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;")
+                (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") +
                 "%2%"
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "

From 7c74f075f4a7274ad38c90085cc269a19a977438 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 14:46:58 +0200
Subject: [PATCH 1570/2196] nix search: Don't quietly ignore errors

---
 src/nix/search.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index 764dcb70a5d..eb75493e475 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -261,6 +261,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                    https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */
                 if (!jsonCacheFile)
                     throw Error("error writing to %s", tmpFile);
+                throw;
             }
 
             if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)

From e6e61f0a54dac0174df996e93fcfedcac7769ab4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 15:36:51 +0200
Subject: [PATCH 1571/2196] getSourceExpr(): Handle channels

Fixes #1892.
Fixes #1865.
Fixes #3119.
---
 src/nix/installables.cc | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 0c1ad3ab3db..52f9778fc6d 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -45,25 +45,26 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
         std::unordered_set<std::string> seen;
 
-        for (auto & i : searchPath) {
-            if (i.first == "") continue;
-            if (seen.count(i.first)) continue;
-            seen.insert(i.first);
-#if 0
-            auto res = state.resolveSearchPathElem(i);
-            if (!res.first) continue;
-            if (!pathExists(res.second)) continue;
-            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)),
-                state.getBuiltin("import"),
-                mkString(*state.allocValue(), res.second));
-#endif
+        auto addEntry = [&](const std::string & name) {
+            if (name == "") return;
+            if (!seen.insert(name).second) return;
             Value * v1 = state.allocValue();
             mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath"));
             Value * v2 = state.allocValue();
-            mkApp(*v2, *v1, mkString(*state.allocValue(), i.first));
-            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)),
+            mkApp(*v2, *v1, mkString(*state.allocValue(), name));
+            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)),
                 state.getBuiltin("import"), *v2);
-        }
+        };
+
+        for (auto & i : searchPath)
+            /* Hack to handle channels. */
+            if (i.first.empty() && pathExists(i.second + "/manifest.nix")) {
+                for (auto & j : readDirectory(i.second))
+                    if (j.name != "manifest.nix"
+                        && pathExists(fmt("%s/%s/default.nix", i.second, j.name)))
+                        addEntry(j.name);
+            } else
+                addEntry(i.first);
 
         vSourceExpr->attrs->sort();
     }

From 99b73fb5071db9fd757c9927fc3fde34e2abac63 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 15:51:52 +0200
Subject: [PATCH 1572/2196] OCD performance fix: {find,count}+insert => insert

---
 src/libexpr/eval.cc                           | 24 +++++++------------
 src/libexpr/get-drvs.cc                       |  3 +--
 src/libexpr/parser.y                          |  3 +--
 src/libexpr/primops.cc                        |  6 ++---
 src/libexpr/value-to-xml.cc                   |  5 ++--
 src/libstore/build.cc                         | 11 ++++-----
 src/libstore/builtins/buildenv.cc             |  3 +--
 src/libstore/local-store.cc                   |  3 +--
 src/libstore/misc.cc                          |  9 +++----
 src/libstore/references.cc                    |  3 +--
 src/libstore/store-api.cc                     |  3 +--
 src/nix-env/nix-env.cc                        |  7 ++----
 src/nix-store/dotgraph.cc                     |  7 ++----
 src/nix-store/nix-store.cc                    |  3 +--
 src/nix/verify.cc                             |  3 +--
 .../resolve-system-dependencies.cc            |  4 +---
 16 files changed, 32 insertions(+), 65 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9f4b6b411a7..f60e8d3ab7e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -46,11 +46,10 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
 {
     checkInterrupt();
 
-    if (active.find(&v) != active.end()) {
+    if (!active.insert(&v).second) {
         str << "<CYCLE>";
         return;
     }
-    active.insert(&v);
 
     switch (v.type) {
     case tInt:
@@ -1446,8 +1445,7 @@ void EvalState::forceValueDeep(Value & v)
     std::function<void(Value & v)> recurse;
 
     recurse = [&](Value & v) {
-        if (seen.find(&v) != seen.end()) return;
-        seen.insert(&v);
+        if (!seen.insert(&v).second) return;
 
         forceValue(v);
 
@@ -1865,8 +1863,7 @@ size_t valueSize(Value & v)
     std::set<const void *> seen;
 
     auto doString = [&](const char * s) -> size_t {
-        if (seen.find(s) != seen.end()) return 0;
-        seen.insert(s);
+        if (!seen.insert(s).second) return 0;
         return strlen(s) + 1;
     };
 
@@ -1874,8 +1871,7 @@ size_t valueSize(Value & v)
     std::function<size_t(Env & v)> doEnv;
 
     doValue = [&](Value & v) -> size_t {
-        if (seen.find(&v) != seen.end()) return 0;
-        seen.insert(&v);
+        if (!seen.insert(&v).second) return 0;
 
         size_t sz = sizeof(Value);
 
@@ -1890,8 +1886,7 @@ size_t valueSize(Value & v)
             sz += doString(v.path);
             break;
         case tAttrs:
-            if (seen.find(v.attrs) == seen.end()) {
-                seen.insert(v.attrs);
+            if (seen.insert(v.attrs).second) {
                 sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity();
                 for (auto & i : *v.attrs)
                     sz += doValue(*i.value);
@@ -1900,8 +1895,7 @@ size_t valueSize(Value & v)
         case tList1:
         case tList2:
         case tListN:
-            if (seen.find(v.listElems()) == seen.end()) {
-                seen.insert(v.listElems());
+            if (seen.insert(v.listElems()).second) {
                 sz += v.listSize() * sizeof(Value *);
                 for (size_t n = 0; n < v.listSize(); ++n)
                     sz += doValue(*v.listElems()[n]);
@@ -1922,8 +1916,7 @@ size_t valueSize(Value & v)
             sz += doValue(*v.primOpApp.right);
             break;
         case tExternal:
-            if (seen.find(v.external) != seen.end()) break;
-            seen.insert(v.external);
+            if (!seen.insert(v.external).second) break;
             sz += v.external->valueSize(seen);
             break;
         default:
@@ -1934,8 +1927,7 @@ size_t valueSize(Value & v)
     };
 
     doEnv = [&](Env & env) -> size_t {
-        if (seen.find(&env) != seen.end()) return 0;
-        seen.insert(&env);
+        if (!seen.insert(&env).second) return 0;
 
         size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 21a4d7917fc..4e22a1d776f 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -277,8 +277,7 @@ static bool getDerivation(EvalState & state, Value & v,
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (done.find(v.attrs) != done.end()) return false;
-        done.insert(v.attrs);
+        if (!done.insert(v.attrs).second) return false;
 
         DrvInfo drv(state, attrPath, v.attrs);
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 967c88d9bc8..1dcd532adf7 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -138,11 +138,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
-    if (formals->argNames.find(formal.name) != formals->argNames.end())
+    if (!formals->argNames.insert(formal.name).second)
         throw ParseError(format("duplicate formal function argument '%1%' at %2%")
             % formal.name % pos);
     formals->formals.push_front(formal);
-    formals->argNames.insert(formal.name);
 }
 
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d4c60f870ed..deaffa605fe 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -396,8 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
         state.forceValue(*key->value);
 
-        if (doneKeys.find(key->value) != doneKeys.end()) continue;
-        doneKeys.insert(key->value);
+        if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
 
         /* Call the `operator' function with `e' as argument. */
@@ -1273,13 +1272,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
-        if (seen.find(sym) == seen.end()) {
+        if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
                 throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
-            seen.insert(sym);
         }
     }
 
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index 00b1918a82a..1f0b1541d6e 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);
 
-                if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
-                    drvsSeen.insert(drvPath);
+                if (drvPath != "" && drvsSeen.insert(drvPath).second)
                     showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
-                } else
+                else
                     doc.writeEmptyElement("repeated");
             }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9a6729f9e52..0e0f8a545a0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -568,10 +568,9 @@ UserLock::UserLock()
 
         {
             auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(fnUserLock))
+            if (!lockedPaths->insert(fnUserLock).second)
                 /* We already have a lock on this one. */
                 continue;
-            lockedPaths->insert(fnUserLock);
         }
 
         try {
@@ -620,8 +619,8 @@ UserLock::UserLock()
 UserLock::~UserLock()
 {
     auto lockedPaths(lockedPaths_.lock());
-    assert(lockedPaths->count(fnUserLock));
-    lockedPaths->erase(fnUserLock);
+    auto erased = lockedPaths->erase(fnUserLock);
+    assert(erased);
 }
 
 
@@ -1125,10 +1124,8 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs)
         needRestart = true;
     } else
         for (auto & i : outputs)
-            if (wantedOutputs.find(i) == wantedOutputs.end()) {
-                wantedOutputs.insert(i);
+            if (wantedOutputs.insert(i).second)
                 needRestart = true;
-            }
 }
 
 
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 74e70666469..0965938862e 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -123,8 +123,7 @@ static Path out;
 
 static void addPkg(const Path & pkgDir, int priority)
 {
-    if (done.count(pkgDir)) return;
-    done.insert(pkgDir);
+    if (!done.insert(pkgDir).second) return;
     createLinks(pkgDir, out, priority);
 
     try {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a2af51d0ed5..859094e6109 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1292,8 +1292,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
 {
     checkInterrupt();
 
-    if (done.find(path) != done.end()) return;
-    done.insert(path);
+    if (!done.insert(path).second) return;
 
     if (!isStorePath(path)) {
         printError(format("path '%1%' is not in the Nix store") % path);
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index dddf134300d..05b93d4c9b0 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -29,8 +29,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
         {
             auto state(state_.lock());
             if (state->exc) return;
-            if (state->paths.count(path)) return;
-            state->paths.insert(path);
+            if (!state->paths.insert(path).second) return;
             state->pending++;
         }
 
@@ -175,8 +174,7 @@ void Store::queryMissing(const PathSet & targets,
 
         {
             auto state(state_.lock());
-            if (state->done.count(path)) return;
-            state->done.insert(path);
+            if (!state->done.insert(path).second) return;
         }
 
         DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path);
@@ -252,8 +250,7 @@ Paths Store::topoSortPaths(const PathSet & paths)
         if (parents.find(path) != parents.end())
             throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
 
-        if (visited.find(path) != visited.end()) return;
-        visited.insert(path);
+        if (!visited.insert(path).second) return;
         parents.insert(path);
 
         PathSet references;
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 5b7eb1f846a..0dcc264c364 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -36,11 +36,10 @@ static void search(const unsigned char * s, size_t len,
             }
         if (!match) continue;
         string ref((const char *) s + i, refLength);
-        if (hashes.find(ref) != hashes.end()) {
+        if (hashes.erase(ref)) {
             debug(format("found reference to '%1%' at offset '%2%'")
                   % ref % i);
             seen.insert(ref);
-            hashes.erase(ref);
         }
         ++i;
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 5f63c53b562..be13fa49aa4 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -954,8 +954,7 @@ std::list<ref<Store>> getDefaultSubstituters()
         StringSet done;
 
         auto addStore = [&](const std::string & uri) {
-            if (done.count(uri)) return;
-            done.insert(uri);
+            if (!done.insert(uri).second) return;
             try {
                 stores.push_back(openStore(uri));
             } catch (Error & e) {
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 87b2e43f063..48686ce721e 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -124,11 +124,10 @@ static void getAllExprs(EvalState & state,
             string attrName = i;
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
-            if (attrs.find(attrName) != attrs.end()) {
+            if (!attrs.insert(attrName).second) {
                 printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2);
                 continue;
             }
-            attrs.insert(attrName);
             /* Load the expression on demand. */
             Value & vFun = state.getBuiltin("import");
             Value & vArg(*state.allocValue());
@@ -307,10 +306,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
         /* Insert only those elements in the final list that we
            haven't inserted before. */
         for (auto & j : matches)
-            if (done.find(j.second) == done.end()) {
-                done.insert(j.second);
+            if (done.insert(j.second).second)
                 elems.push_back(j.first);
-            }
     }
 
     checkSelectorUse(selectors);
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index abdfa5e58f9..d448654fe3a 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -71,9 +71,7 @@ void printClosure(const Path & nePath, const StoreExpr & fs)
         Path path = *(workList.begin());
         workList.erase(path);
 
-        if (doneSet.find(path) == doneSet.end()) {
-            doneSet.insert(path);
-
+        if (doneSet.insert(path).second) {
             ClosureElems::const_iterator elem = fs.closure.elems.find(path);
             if (elem == fs.closure.elems.end())
                 throw Error(format("bad closure, missing path '%1%'") % path);
@@ -104,8 +102,7 @@ void printDotGraph(ref<Store> store, const PathSet & roots)
         Path path = *(workList.begin());
         workList.erase(path);
 
-        if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
+        if (!doneSet.insert(path).second) continue;
 
         cout << makeNode(path, symbolicName(path), "#ff0000");
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 0cbceb02f31..19384499d3b 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -242,11 +242,10 @@ const string treeNull = "    ";
 static void printTree(const Path & path,
     const string & firstPad, const string & tailPad, PathSet & done)
 {
-    if (done.find(path) != done.end()) {
+    if (!done.insert(path).second) {
         cout << format("%1%%2% [...]\n") % firstPad % path;
         return;
     }
-    done.insert(path);
 
     cout << format("%1%%2%\n") % firstPad % path;
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 7ef571561a0..2e0d5cffe08 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -118,8 +118,7 @@ struct CmdVerify : StorePathsCommand
 
                         auto doSigs = [&](StringSet sigs) {
                             for (auto sig : sigs) {
-                                if (sigsSeen.count(sig)) continue;
-                                sigsSeen.insert(sig);
+                                if (!sigsSeen.insert(sig).second) continue;
                                 if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig))
                                     validSigs++;
                             }
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index e02cfc03ec9..45e8b679670 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -117,9 +117,7 @@ Path resolveSymlink(const Path & path)
 std::set<string> resolveTree(const Path & path, PathSet & deps)
 {
     std::set<string> results;
-    if (deps.count(path))
-        return {};
-    deps.insert(path);
+    if (!deps.insert(path).second) return {};
     for (auto & lib : runResolver(path)) {
         results.insert(lib);
         for (auto & p : resolveTree(lib, deps)) {

From 926d3e5bb0ca696ab59352b25d0a818707a28186 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 22:57:37 +0200
Subject: [PATCH 1573/2196] Fix Bison 2.4 warning

---
 src/libexpr/parser.y | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 1dcd532adf7..93834bec664 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -1,5 +1,5 @@
 %glr-parser
-%pure-parser
+%define api.pure
 %locations
 %define parse.error verbose
 %defines

From 55bba8e4f51758e6c0556b24743e6946c96f6d92 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 23:04:11 +0200
Subject: [PATCH 1574/2196] Make std::uncaught_exception warning less noisy

---
 src/libutil/json.cc | 5 +++++
 src/libutil/json.hh | 5 +----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libutil/json.cc b/src/libutil/json.cc
index 0a6fb65f060..74e37b4c442 100644
--- a/src/libutil/json.cc
+++ b/src/libutil/json.cc
@@ -171,4 +171,9 @@ JSONObject JSONPlaceholder::object()
     return JSONObject(state);
 }
 
+JSONPlaceholder::~JSONPlaceholder()
+{
+    assert(!first || std::uncaught_exception());
+}
+
 }
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
index 02a39917fb5..83213ca6664 100644
--- a/src/libutil/json.hh
+++ b/src/libutil/json.hh
@@ -168,10 +168,7 @@ public:
     {
     }
 
-    ~JSONPlaceholder()
-    {
-        assert(!first || std::uncaught_exception());
-    }
+    ~JSONPlaceholder();
 
     template<typename T>
     void write(const T & v)

From 4331eeb13d241dfe2d2e6a01c53915c556cac94f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 23:25:06 +0200
Subject: [PATCH 1575/2196] Filter ANSI escape sequences in -L output

Otherwise, builds like NixOS VM tests may leave the terminal in a
weird state and do resets.
---
 src/nix/progress-bar.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index c0bcfb0c91b..5c05d6b227e 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -120,7 +120,7 @@ class ProgressBar : public Logger
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {
-            writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n");
+            writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n");
             draw(state);
         } else {
             auto s2 = s + ANSI_NORMAL "\n";

From 5a303093dcae1e5ce9212616ef18f2ca51020b0d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 18:01:21 +0200
Subject: [PATCH 1576/2196] Remove world-writability from per-user directories

'nix-daemon' now creates subdirectories for users when they first
connect.

Fixes #509 (CVE-2019-17365).
Should also fix #3127.
---
 nix.spec.in                      |  2 +-
 scripts/install-multi-user.sh    |  9 ++++-----
 scripts/nix-profile-daemon.sh.in | 13 -------------
 scripts/nix-profile.sh.in        | 14 --------------
 src/libstore/local-store.cc      | 24 ++++++++++++++++++++----
 src/libstore/local-store.hh      |  2 ++
 src/libstore/store-api.hh        |  3 +++
 src/nix-daemon/nix-daemon.cc     |  9 ++++++---
 tests/nix-profile.sh             |  2 --
 tests/remote-store.sh            |  4 ++++
 tests/user-envs.sh               |  2 +-
 11 files changed, 41 insertions(+), 43 deletions(-)

diff --git a/nix.spec.in b/nix.spec.in
index 477768c6a68..6b9e3763738 100644
--- a/nix.spec.in
+++ b/nix.spec.in
@@ -106,7 +106,7 @@ chmod 1775 $RPM_BUILD_ROOT/nix/store
 for d in profiles gcroots;
 do
   mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-  chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
+  chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
 done
 
 # fix permission of nix profile
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index fd0f54184aa..258469a0b10 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -437,16 +437,15 @@ create_build_users() {
 }
 
 create_directories() {
+    # FIXME: remove all of this because it duplicates LocalStore::LocalStore().
+
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
           mkdir -pv -m 1775 /nix/store
 
-    _sudo "to make the basic directory structure of Nix (part 4)" \
+    _sudo "to make the basic directory structure of Nix (part 3)" \
           chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
 
     _sudo "to set up the root user's profile (part 1)" \
diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 23da5e8559e..3e138ac4249 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -5,12 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1
 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
-# Set up the per-user profile.
-mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
-if ! test -O "$NIX_USER_PROFILE_DIR"; then
-    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
-fi
-
 if test -w $HOME; then
   if ! test -L $HOME/.nix-profile; then
       if test "$USER" != root; then
@@ -26,13 +20,6 @@ if test -w $HOME; then
       echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
   fi
 
-  # Create the per-user garbage collector roots directory.
-  NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER
-  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
-  if ! test -O "$NIX_USER_GCROOTS_DIR"; then
-      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
-  fi
-
   # Set up a default Nix expression from which to install stuff.
   if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
       rm -f $HOME/.nix-defexpr
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 85f1d6e5dae..7f9b5877a8d 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -9,12 +9,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
 
-    mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR"
-
-    if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then
-        echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2
-    fi
-
     if [ -w "$HOME" ]; then
         if ! [ -L "$NIX_LINK" ]; then
             echo "Nix: creating $NIX_LINK" >&2
@@ -33,14 +27,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
             echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels"
         fi
 
-        # Create the per-user garbage collector roots directory.
-        __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER"
-        mkdir -m 0755 -p "$__user_gcroots"
-        if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then
-            echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2
-        fi
-        unset __user_gcroots
-
         # Set up a default Nix expression from which to install stuff.
         __nix_defexpr="$HOME"/.nix-defexpr
         [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 859094e6109..685d43c9cd4 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -74,10 +74,11 @@ LocalStore::LocalStore(const Params & params)
        multi-user install. */
     if (getuid() == 0 && settings.buildUsersGroup != "") {
 
-        Path perUserDir = profilesDir + "/per-user";
-        createDirs(perUserDir);
-        if (chmod(perUserDir.c_str(), 01777) == -1)
-            throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir);
+        for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
+            createDirs(perUserDir);
+            if (chmod(perUserDir.c_str(), 0755) == -1)
+                throw SysError("could not set permissions on '%s' to 755", perUserDir);
+        }
 
         mode_t perm = 01775;
 
@@ -1432,4 +1433,19 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 }
 
 
+void LocalStore::createUser(const std::string & userName, uid_t userId)
+{
+    for (auto & dir : {
+        fmt("%s/profiles/per-user/%s", stateDir, userName),
+        fmt("%s/gcroots/per-user/%s", stateDir, userName)
+    }) {
+        createDirs(dir);
+        if (chmod(dir.c_str(), 0700) == -1)
+            throw SysError("changing permissions of directory '%s'", dir);
+        if (chown(dir.c_str(), userId, -1) == -1)
+            throw SysError("changing owner of directory '%s'", dir);
+    }
+}
+
+
 }
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 3ae34c4035c..379a06af87d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -293,6 +293,8 @@ private:
 
     Path getRealStoreDir() override { return realStoreDir; }
 
+    void createUser(const std::string & userName, uid_t userId) override;
+
     friend class DerivationGoal;
     friend class SubstitutionGoal;
 };
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 7fb56860209..ba899075567 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -628,6 +628,9 @@ public:
         return storePath;
     }
 
+    virtual void createUser(const std::string & userName, uid_t userId)
+    { }
+
 protected:
 
     Stats stats;
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index e88aaf63644..cd18489b0cd 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -742,7 +742,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 }
 
 
-static void processConnection(bool trusted)
+static void processConnection(bool trusted,
+    const std::string & userName, uid_t userId)
 {
     MonitorFdHup monitor(from.fd);
 
@@ -793,6 +794,8 @@ static void processConnection(bool trusted)
         params["path-info-cache-size"] = "0";
         auto store = openStore(settings.storeUri, params);
 
+        store->createUser(userName, userId);
+
         tunnelLogger->stopWork();
         to.flush();
 
@@ -1053,7 +1056,7 @@ static void daemonLoop(char * * argv)
                 /* Handle the connection. */
                 from.fd = remote.get();
                 to.fd = remote.get();
-                processConnection(trusted);
+                processConnection(trusted, user, peer.uid);
 
                 exit(0);
             }, options);
@@ -1133,7 +1136,7 @@ static int _main(int argc, char * * argv)
                     }
                 }
             } else {
-                processConnection(true);
+                processConnection(true, "root", 0);
             }
         } else {
             daemonLoop(argv);
diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh
index b244815e290..5b17fe3faf5 100644
--- a/tests/nix-profile.sh
+++ b/tests/nix-profile.sh
@@ -10,5 +10,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency
 
 [ -L $TEST_HOME/.nix-profile ]
 [ -e $TEST_HOME/.nix-channels ]
-[ -e $TEST_ROOT/profile-var/nix/gcroots/per-user/$user ]
-[ -e $TEST_ROOT/profile-var/nix/profiles/per-user/$user ]
diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index f2f2806d022..77437658ead 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -13,3 +13,7 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2
 nix-store --gc --max-freed 1K
 
 killDaemon
+
+user=$(whoami)
+[ -e $NIX_STATE_DIR/gcroots/per-user/$user ]
+[ -e $NIX_STATE_DIR/profiles/per-user/$user ]
diff --git a/tests/user-envs.sh b/tests/user-envs.sh
index ba63923113d..aebf6a2a2b8 100644
--- a/tests/user-envs.sh
+++ b/tests/user-envs.sh
@@ -20,7 +20,7 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1
 
 # Query descriptions.
 nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly
-rm -f $HOME/.nix-defexpr
+rm -rf $HOME/.nix-defexpr
 ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr
 nix-env -qa '*' --description | grep -q silly
 

From c43d9f6131102f2761f22b1ec26f345d357f169c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 18:05:42 +0200
Subject: [PATCH 1577/2196] Remove some redundant initialization

---
 scripts/install-multi-user.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 258469a0b10..40013eaafc4 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -448,12 +448,6 @@ create_directories() {
     _sudo "to make the basic directory structure of Nix (part 3)" \
           chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
 
-    _sudo "to set up the root user's profile (part 1)" \
-          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
-
-    _sudo "to set up the root user's profile (part 2)" \
-          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
-
     _sudo "to place the default nix daemon configuration (part 1)" \
           mkdir -pv -m 0555 /etc/nix
 }

From 26762ceb8629af95300c0cc8c372a99282060dc1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 18:30:31 +0200
Subject: [PATCH 1578/2196] nix-profile.sh: Don't create .nix-channels

This is already done by the installer, so no need to do it again.
---
 scripts/nix-profile-daemon.sh.in | 5 -----
 scripts/nix-profile.sh.in        | 5 -----
 tests/nix-profile.sh             | 1 -
 3 files changed, 11 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 3e138ac4249..2133a6d3b18 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -15,11 +15,6 @@ if test -w $HOME; then
       fi
   fi
 
-  # Subscribe the root user to the NixOS channel by default.
-  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
-      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
-  fi
-
   # Set up a default Nix expression from which to install stuff.
   if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
       rm -f $HOME/.nix-defexpr
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 7f9b5877a8d..8e70c68b285 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -22,11 +22,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
             fi
         fi
 
-        # Subscribe the user to the unstable Nixpkgs channel by default.
-        if [ ! -e "$HOME/.nix-channels" ]; then
-            echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels"
-        fi
-
         # Set up a default Nix expression from which to install stuff.
         __nix_defexpr="$HOME"/.nix-defexpr
         [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh
index 5b17fe3faf5..b808cf1b463 100644
--- a/tests/nix-profile.sh
+++ b/tests/nix-profile.sh
@@ -9,4 +9,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set"
 USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency
 
 [ -L $TEST_HOME/.nix-profile ]
-[ -e $TEST_HOME/.nix-channels ]

From 9348f9291e5d9e4ba3c4347ea1b235640f54fd79 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 19:21:07 +0200
Subject: [PATCH 1579/2196] nix-env: Create ~/.nix-profile automatically

---
 scripts/nix-profile-daemon.sh.in |  9 ---------
 scripts/nix-profile.sh.in        | 12 ------------
 src/libstore/local-store.cc      | 15 ++++++++-------
 src/libutil/util.cc              | 10 ++++++++++
 src/libutil/util.hh              |  2 ++
 src/nix-channel/nix-channel.cc   |  8 +-------
 src/nix-env/nix-env.cc           | 11 ++++++++---
 tests/nix-channel.sh             |  4 ++--
 tests/nix-profile.sh             |  2 --
 9 files changed, 31 insertions(+), 42 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 2133a6d3b18..aa78e6fccf8 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -6,15 +6,6 @@ export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 if test -w $HOME; then
-  if ! test -L $HOME/.nix-profile; then
-      if test "$USER" != root; then
-          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
-      else
-          # Root installs in the system-wide profile by default.
-          ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile
-      fi
-  fi
-
   # Set up a default Nix expression from which to install stuff.
   if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
       rm -f $HOME/.nix-defexpr
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 8e70c68b285..087f6ee57ce 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -10,18 +10,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
 
     if [ -w "$HOME" ]; then
-        if ! [ -L "$NIX_LINK" ]; then
-            echo "Nix: creating $NIX_LINK" >&2
-            if [ "$USER" != root ]; then
-                if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then
-                    echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2
-                fi
-            else
-                # Root installs in the system-wide profile by default.
-                ln -s @localstatedir@/nix/profiles/default "$NIX_LINK"
-            fi
-        fi
-
         # Set up a default Nix expression from which to install stuff.
         __nix_defexpr="$HOME"/.nix-defexpr
         [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 685d43c9cd4..2bc04da469d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -70,16 +70,17 @@ LocalStore::LocalStore(const Params & params)
         createSymlink(profilesDir, gcRootsDir + "/profiles");
     }
 
+    for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
+        createDirs(perUserDir);
+        if (chmod(perUserDir.c_str(), 0755) == -1)
+            throw SysError("could not set permissions on '%s' to 755", perUserDir);
+    }
+
+    createUser(getUserName(), getuid());
+
     /* Optionally, create directories and set permissions for a
        multi-user install. */
     if (getuid() == 0 && settings.buildUsersGroup != "") {
-
-        for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
-            createDirs(perUserDir);
-            if (chmod(perUserDir.c_str(), 0755) == -1)
-                throw SysError("could not set permissions on '%s' to 755", perUserDir);
-        }
-
         mode_t perm = 01775;
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1b744999153..6f3bf7ae86e 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -475,6 +475,16 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 }
 
 
+std::string getUserName()
+{
+    auto pw = getpwuid(geteuid());
+    std::string name = pw ? pw->pw_name : getEnv("USER", "");
+    if (name.empty())
+        throw Error("cannot figure out user name");
+    return name;
+}
+
+
 static Lazy<Path> getHome2([]() {
     Path homeDir = getEnv("HOME");
     if (homeDir.empty()) {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 07c3d28ff2d..f057fdb2c04 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -126,6 +126,8 @@ void deletePath(const Path & path, unsigned long long & bytesFreed);
 Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
     bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
 
+std::string getUserName();
+
 /* Return $HOME or the user's home directory from /etc/passwd. */
 Path getHome();
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 06eb3d23ba8..70aa5c96669 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -159,13 +159,7 @@ static int _main(int argc, char ** argv)
         nixDefExpr = home + "/.nix-defexpr";
 
         // Figure out the name of the channels profile.
-        ;
-        auto pw = getpwuid(geteuid());
-        std::string name = pw ? pw->pw_name : getEnv("USER", "");
-        if (name.empty())
-            throw Error("cannot figure out user name");
-        profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels";
-        createDirs(dirOf(profile));
+        profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName());
 
         enum {
             cNone,
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 48686ce721e..1b412f0bd4e 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1422,9 +1422,14 @@ static int _main(int argc, char * * argv)
 
         if (globals.profile == "") {
             Path profileLink = getHome() + "/.nix-profile";
-            globals.profile = pathExists(profileLink)
-                ? absPath(readLink(profileLink), dirOf(profileLink))
-                : canonPath(settings.nixStateDir + "/profiles/default");
+            if (!pathExists(profileLink)) {
+                replaceSymlink(
+                    getuid() == 0
+                    ? settings.nixStateDir + "/profiles/default"
+                    : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
+                    profileLink);
+            }
+            globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
         }
 
         op(globals, opFlags, opArgs);
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 55f1695c441..93f837befce 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -36,7 +36,7 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
 
 # Do an install.
 nix-env -i dependencies
-[ -e $TEST_ROOT/var/nix/profiles/default/foobar ]
+[ -e $TEST_HOME/.nix-profile/foobar ]
 
 clearProfiles
 rm -f $TEST_HOME/.nix-channels
@@ -55,5 +55,5 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
 
 # Do an install.
 nix-env -i dependencies
-[ -e $TEST_ROOT/var/nix/profiles/default/foobar ]
+[ -e $TEST_HOME/.nix-profile/foobar ]
 
diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh
index b808cf1b463..e2e0d109080 100644
--- a/tests/nix-profile.sh
+++ b/tests/nix-profile.sh
@@ -7,5 +7,3 @@ rm -rf $TEST_HOME $TEST_ROOT/profile-var
 mkdir -p $TEST_HOME
 USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set"
 USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency
-
-[ -L $TEST_HOME/.nix-profile ]

From 61a6176acaa8522cbcf091a34a663ef45307fef7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 19:38:01 +0200
Subject: [PATCH 1580/2196] nix-profile.sh: Remove coreutils dependency

---
 scripts/nix-profile.sh.in | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 087f6ee57ce..ed4d1a6393f 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -1,6 +1,4 @@
 if [ -n "$HOME" ] && [ -n "$USER" ]; then
-    __savedpath="$PATH"
-    export PATH=@coreutils@
 
     # Set up the per-user profile.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix
@@ -47,6 +45,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 
-    export PATH="$NIX_LINK/bin:$__savedpath"
-    unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR
+    export PATH="$NIX_LINK/bin:$PATH"
+    unset NIX_LINK NIX_USER_PROFILE_DIR
 fi

From c9159f86cc9a2fc07e2ab1217c2d8a8824123df4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 19:50:46 +0200
Subject: [PATCH 1581/2196] nix-env: Create ~/.nix-defexpr automatically

---
 scripts/nix-profile-daemon.sh.in | 12 ------------
 scripts/nix-profile.sh.in        | 11 -----------
 src/nix-env/nix-env.cc           | 19 ++++++++++++-------
 3 files changed, 12 insertions(+), 30 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index aa78e6fccf8..47655080a67 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -5,18 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1
 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
-if test -w $HOME; then
-  # Set up a default Nix expression from which to install stuff.
-  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
-      rm -f $HOME/.nix-defexpr
-      mkdir -p $HOME/.nix-defexpr
-      if [ "$USER" != root ]; then
-          ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
-      fi
-  fi
-fi
-
-
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index ed4d1a6393f..e15f7cd46b7 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -7,17 +7,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
 
-    if [ -w "$HOME" ]; then
-        # Set up a default Nix expression from which to install stuff.
-        __nix_defexpr="$HOME"/.nix-defexpr
-        [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
-        mkdir -m 0755 -p "$__nix_defexpr"
-        if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then
-            ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root
-        fi
-        unset __nix_defexpr
-    fi
-
     # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
     # paths work when the user has fetched the Nixpkgs channel.
     export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 1b412f0bd4e..2aeb8ebb012 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -192,12 +192,6 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
 }
 
 
-static Path getDefNixExprPath()
-{
-    return getHome() + "/.nix-defexpr";
-}
-
-
 static long getPriority(EvalState & state, DrvInfo & drv)
 {
     return drv.queryMetaInt("priority", 0);
@@ -1327,9 +1321,20 @@ static int _main(int argc, char * * argv)
         Globals globals;
 
         globals.instSource.type = srcUnknown;
-        globals.instSource.nixExprPath = getDefNixExprPath();
+        globals.instSource.nixExprPath = getHome() + "/.nix-defexpr";
         globals.instSource.systemFilter = "*";
 
+        if (!pathExists(globals.instSource.nixExprPath)) {
+            createDirs(globals.instSource.nixExprPath);
+            replaceSymlink(
+                fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()),
+                globals.instSource.nixExprPath + "/channels");
+            if (getuid() != 0)
+                replaceSymlink(
+                    fmt("%s/profiles/per-user/root/channels", settings.nixStateDir),
+                    globals.instSource.nixExprPath + "/channels_root");
+        }
+
         globals.dryRun = false;
         globals.preserveInstalled = false;
         globals.removeAll = false;

From d7bae5680fc26303acb9a9ee1a202f537841a624 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 19:55:05 +0200
Subject: [PATCH 1582/2196] Go back to 755 permission on per-user directories

700 is pointless since the store is world-readable anyway. And
per-user/root/channels must be world-readable.
---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2bc04da469d..fbf5dc35c80 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1441,7 +1441,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
         fmt("%s/gcroots/per-user/%s", stateDir, userName)
     }) {
         createDirs(dir);
-        if (chmod(dir.c_str(), 0700) == -1)
+        if (chmod(dir.c_str(), 0755) == -1)
             throw SysError("changing permissions of directory '%s'", dir);
         if (chown(dir.c_str(), userId, -1) == -1)
             throw SysError("changing owner of directory '%s'", dir);

From 9277e72cb0aac72100c01334fdf25ea79d19052e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 21:04:21 +0200
Subject: [PATCH 1583/2196] Typo

---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 40013eaafc4..d060e5165a8 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -490,7 +490,7 @@ EOF
 We will:
 
  - make sure your computer doesn't already have Nix files
-   (if it does, I  will tell you how to clean them up.)
+   (if it does, I will tell you how to clean them up.)
  - create local users (see the list above for the users we'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT

From 20eec802ff11dd2b152715cd5c81b756d318219d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 9 Oct 2019 23:33:28 +0200
Subject: [PATCH 1584/2196] Force per-user group to a known value

---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index fbf5dc35c80..00f3a31a24c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
         createDirs(dir);
         if (chmod(dir.c_str(), 0755) == -1)
             throw SysError("changing permissions of directory '%s'", dir);
-        if (chown(dir.c_str(), userId, -1) == -1)
+        if (chown(dir.c_str(), userId, 0) == -1)
             throw SysError("changing owner of directory '%s'", dir);
     }
 }

From bda64a2b0f79346012332ed18f5a76388e6d9bae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Oct 2019 00:12:30 +0200
Subject: [PATCH 1585/2196] Doh

https://hydra.nixos.org/build/102803044
---
 src/libstore/local-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 00f3a31a24c..d86f7b1d274 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
         createDirs(dir);
         if (chmod(dir.c_str(), 0755) == -1)
             throw SysError("changing permissions of directory '%s'", dir);
-        if (chown(dir.c_str(), userId, 0) == -1)
+        if (chown(dir.c_str(), userId, getgid()) == -1)
             throw SysError("changing owner of directory '%s'", dir);
     }
 }

From c3aaf3b8da1a925c569389f13a861816a781a3c8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Oct 2019 09:14:05 +0200
Subject: [PATCH 1586/2196] nix-env: Ignore failures creating ~/.nix-profile
 and ~/.nix-defexpr

https://hydra.nixos.org/build/102803093
---
 src/nix-env/nix-env.cc | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 2aeb8ebb012..199dc92aa1c 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1325,14 +1325,16 @@ static int _main(int argc, char * * argv)
         globals.instSource.systemFilter = "*";
 
         if (!pathExists(globals.instSource.nixExprPath)) {
-            createDirs(globals.instSource.nixExprPath);
-            replaceSymlink(
-                fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()),
-                globals.instSource.nixExprPath + "/channels");
-            if (getuid() != 0)
+            try {
+                createDirs(globals.instSource.nixExprPath);
                 replaceSymlink(
-                    fmt("%s/profiles/per-user/root/channels", settings.nixStateDir),
-                    globals.instSource.nixExprPath + "/channels_root");
+                    fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()),
+                    globals.instSource.nixExprPath + "/channels");
+                if (getuid() != 0)
+                    replaceSymlink(
+                        fmt("%s/profiles/per-user/root/channels", settings.nixStateDir),
+                        globals.instSource.nixExprPath + "/channels_root");
+            } catch (Error &) { }
         }
 
         globals.dryRun = false;
@@ -1427,14 +1429,18 @@ static int _main(int argc, char * * argv)
 
         if (globals.profile == "") {
             Path profileLink = getHome() + "/.nix-profile";
-            if (!pathExists(profileLink)) {
-                replaceSymlink(
-                    getuid() == 0
-                    ? settings.nixStateDir + "/profiles/default"
-                    : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
-                    profileLink);
+            try {
+                if (!pathExists(profileLink)) {
+                    replaceSymlink(
+                        getuid() == 0
+                        ? settings.nixStateDir + "/profiles/default"
+                        : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
+                        profileLink);
+                }
+                globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
+            } catch (Error &) {
+                globals.profile = profileLink;
             }
-            globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
         }
 
         op(globals, opFlags, opArgs);

From 95cf23ee7c5b0fd69b21811989a5668f4261fd51 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 10 Oct 2019 15:03:01 +0200
Subject: [PATCH 1587/2196] nix verify: Fix uninitialized variable

---
 src/nix/verify.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 2e0d5cffe08..74d9673b616 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -13,7 +13,7 @@ struct CmdVerify : StorePathsCommand
     bool noContents = false;
     bool noTrust = false;
     Strings substituterUris;
-    size_t sigsNeeded;
+    size_t sigsNeeded = 0;
 
     CmdVerify()
     {
@@ -113,7 +113,7 @@ struct CmdVerify : StorePathsCommand
                     else {
 
                         StringSet sigsSeen;
-                        size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1;
+                        size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1);
                         size_t validSigs = 0;
 
                         auto doSigs = [&](StringSet sigs) {

From 906d56a96b442d4dd8f924c1ce0d1eec0e214af3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 11 Oct 2019 18:48:59 +0200
Subject: [PATCH 1588/2196] ssh-ng: Don't set CPU affinity on the remote

Fixes #3138.
---
 src/libstore/remote-store.cc | 2 +-
 src/libstore/remote-store.hh | 5 +++++
 src/libstore/ssh-store.cc    | 3 +++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b7f202a6bbc..f34369d8fe0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -151,7 +151,7 @@ void RemoteStore::initConnection(Connection & conn)
         conn.to << PROTOCOL_VERSION;
 
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
-            int cpu = settings.lockCPU ? lockToCurrentCPU() : -1;
+            int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
             if (cpu != -1)
                 conn.to << 1 << cpu;
             else
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 82fbec092fe..1f375dd7152 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -29,6 +29,8 @@ public:
     const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
             "max-connection-age", "number of seconds to reuse a connection"};
 
+    virtual bool sameMachine() = 0;
+
     RemoteStore(const Params & params);
 
     /* Implementations of abstract store API methods. */
@@ -146,6 +148,9 @@ public:
 
     std::string getUri() override;
 
+    bool sameMachine()
+    { return true; }
+
 private:
 
     ref<RemoteStore::Connection> openConnection() override;
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 39205ae2ce1..93e11738991 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -35,6 +35,9 @@ class SSHStore : public RemoteStore
         return uriScheme + host;
     }
 
+    bool sameMachine()
+    { return false; }
+
     void narFromPath(const Path & path, Sink & sink) override;
 
     ref<FSAccessor> getFSAccessor() override;

From 499b0388759db0f9f385da402a4bba551268aa99 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sat, 12 Oct 2019 19:02:57 -0400
Subject: [PATCH 1589/2196] Fix sandbox fallback settings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The tmpDirInSandbox is different when in sandboxed vs. non-sandboxed.
Since we don’t know ahead of time here whether sandboxing is enabled,
we need to reset all of the env vars we’ve set previously. This fixes
the issue encountered in https://github.com/NixOS/nixpkgs/issues/70856.
---
 src/libstore/build.cc | 60 ++++++++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0e0f8a545a0..273f253635c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -956,6 +956,9 @@ class DerivationGoal : public Goal
     /* Fill in the environment for the builder. */
     void initEnv();
 
+    /* Setup tmp dir location. */
+    void initTmpDir();
+
     /* Write a JSON file containing the derivation attributes. */
     void writeStructuredAttrs();
 
@@ -2383,6 +2386,7 @@ void DerivationGoal::startBuilder()
         if (res != 0 && settings.sandboxFallback) {
             useChroot = false;
             tmpDirInSandbox = tmpDir;
+            initTmpDir();
             goto fallback;
         } else if (res != 0)
             throw Error("unable to start build process");
@@ -2438,32 +2442,7 @@ void DerivationGoal::startBuilder()
 }
 
 
-void DerivationGoal::initEnv()
-{
-    env.clear();
-
-    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
-       PATH is not set.  We don't want this, so we fill it in with some dummy
-       value. */
-    env["PATH"] = "/path-not-set";
-
-    /* Set HOME to a non-existing path to prevent certain programs from using
-       /etc/passwd (or NIS, or whatever) to locate the home directory (for
-       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
-       if HOME is not set, but they will just assume that the settings file
-       they are looking for does not exist if HOME is set but points to some
-       non-existing path. */
-    env["HOME"] = homeDir;
-
-    /* Tell the builder where the Nix store is.  Usually they
-       shouldn't care, but this is useful for purity checking (e.g.,
-       the compiler or linker might only want to accept paths to files
-       in the store or in the build directory). */
-    env["NIX_STORE"] = worker.store.storeDir;
-
-    /* The maximum number of cores to utilize for parallel building. */
-    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
+void DerivationGoal::initTmpDir() {
     /* In non-structured mode, add all bindings specified in the
        derivation via the environment, except those listed in the
        passAsFile attribute. Those are passed as file names pointing
@@ -2501,6 +2480,35 @@ void DerivationGoal::initEnv()
        inode of the current directory doesn't appear in .. (because
        getdents returns the inode of the mount point). */
     env["PWD"] = tmpDirInSandbox;
+}
+
+void DerivationGoal::initEnv()
+{
+    env.clear();
+
+    /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
+       PATH is not set.  We don't want this, so we fill it in with some dummy
+       value. */
+    env["PATH"] = "/path-not-set";
+
+    /* Set HOME to a non-existing path to prevent certain programs from using
+       /etc/passwd (or NIS, or whatever) to locate the home directory (for
+       example, wget looks for ~/.wgetrc).  I.e., these tools use /etc/passwd
+       if HOME is not set, but they will just assume that the settings file
+       they are looking for does not exist if HOME is set but points to some
+       non-existing path. */
+    env["HOME"] = homeDir;
+
+    /* Tell the builder where the Nix store is.  Usually they
+       shouldn't care, but this is useful for purity checking (e.g.,
+       the compiler or linker might only want to accept paths to files
+       in the store or in the build directory). */
+    env["NIX_STORE"] = worker.store.storeDir;
+
+    /* The maximum number of cores to utilize for parallel building. */
+    env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+
+    initTmpDir();
 
     /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
        derivation, tell the builder, so that for instance `fetchurl'

From 96c84937c49435525d0733437aa88902b2c8caf6 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Sun, 13 Oct 2019 16:41:49 -0400
Subject: [PATCH 1590/2196] Move tmpDirInSandbox to initTmpDir

---
 src/libstore/build.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 273f253635c..03a069ae2c1 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1961,13 +1961,6 @@ void DerivationGoal::startBuilder()
     auto drvName = storePathToName(drvPath);
     tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700);
 
-    /* In a sandbox, for determinism, always use the same temporary
-       directory. */
-#if __linux__
-    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
-    tmpDirInSandbox = tmpDir;
-#endif
     chownToBuilder(tmpDir);
 
     /* Substitute output placeholders with the actual output paths. */
@@ -2385,7 +2378,6 @@ void DerivationGoal::startBuilder()
         int res = helper.wait();
         if (res != 0 && settings.sandboxFallback) {
             useChroot = false;
-            tmpDirInSandbox = tmpDir;
             initTmpDir();
             goto fallback;
         } else if (res != 0)
@@ -2443,6 +2435,14 @@ void DerivationGoal::startBuilder()
 
 
 void DerivationGoal::initTmpDir() {
+    /* In a sandbox, for determinism, always use the same temporary
+       directory. */
+#if __linux__
+    tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
+#else
+    tmpDirInSandbox = tmpDir;
+#endif
+
     /* In non-structured mode, add all bindings specified in the
        derivation via the environment, except those listed in the
        passAsFile attribute. Those are passed as file names pointing

From 7c568d4c6e3291f01833aa9cf3779dfdcdaa4505 Mon Sep 17 00:00:00 2001
From: xbreak <xbreak@alphaware.se>
Date: Sat, 12 Oct 2019 13:00:19 +0000
Subject: [PATCH 1591/2196] Downloader: Warn if no trusted CA file has been
 configured

---
 src/libstore/download.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index cdf56e09d69..e5f8556b709 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -285,6 +285,7 @@ struct CurlDownloader : public Downloader
             }
 
             if (request.verifyTLS) {
+                debug("verify TLS: Nix CA file = '%s'", settings.caFile);
                 if (settings.caFile != "")
                     curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
             } else {

From f0ec4b4ce478a8d2760203e8192275b88c770e1c Mon Sep 17 00:00:00 2001
From: Steven Shaw <steven@steshaw.org>
Date: Sat, 19 Oct 2019 13:25:18 +1000
Subject: [PATCH 1592/2196] Fix unset variable in installer

---
 scripts/install-nix-from-closure.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 35926f3dac9..3f15818547a 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -141,11 +141,9 @@ if [ -z "$_NIX_INSTALLER_TEST" ]; then
 fi
 
 added=
+p=$HOME/.nix-profile/etc/profile.d/nix.sh
 if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
-
     # Make the shell source nix.sh during login.
-    p=$HOME/.nix-profile/etc/profile.d/nix.sh
-
     for i in .bash_profile .bash_login .profile; do
         fn="$HOME/$i"
         if [ -w "$fn" ]; then
@@ -157,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
-
 fi
 
 if [ -z "$added" ]; then

From d8730fb86facadbef22d3df7f8a743a56e7ed53c Mon Sep 17 00:00:00 2001
From: Ersin Akinci <me@ersinakinci.com>
Date: Sun, 20 Oct 2019 19:08:05 -0700
Subject: [PATCH 1593/2196] Document import <path> syntax

---
 doc/manual/expressions/builtins.xml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 465fa1e0b00..21da8b87b28 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -733,13 +733,21 @@ builtins.genList (x: x * x) 5
   <varlistentry xml:id='builtin-import'>
     <term><function>import</function>
     <replaceable>path</replaceable></term>
+    <term><function>import</function>
+    <replaceable>&lt;path&gt;</replaceable></term>
     <term><function>builtins.import</function>
     <replaceable>path</replaceable></term>
+    <term><function>builtins.import</function>
+    <replaceable>&lt;path&gt;</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
     </replaceable> is a directory, the file <filename>default.nix
-    </filename> in that directory is loaded.  Evaluation aborts if the
+    </filename> in that directory is loaded.  If the <replaceable>
+    &lt;path&gt;</replaceable> syntax is used, the path will be resolved
+    relative to those listed in the <envar>NIX_PATH</envar> environment
+    variable (see the section on <envar linkend="env-NIX_PATH">NIX_PATH</envar>
+    for details on how the resolution works). Evaluation aborts if the
     file doesn’t exist or contains an incorrect Nix expression.
     <function>import</function> implements Nix’s module system: you
     can put any Nix expression (such as a set or a function) in a

From 389a2cebed7cd72bda524ece0a56af2888cd80b6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Oct 2019 13:14:39 +0200
Subject: [PATCH 1594/2196] SourceExprCommand::getSourceExpr(): Allocate more
 space

Fixes #3140.
---
 src/nix/installables.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 52f9778fc6d..0e8bba39de2 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -39,7 +39,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
         auto searchPath = state.getSearchPath();
 
-        state.mkAttrs(*vSourceExpr, searchPath.size() + 1);
+        state.mkAttrs(*vSourceExpr, 1024);
 
         mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true);
 

From aabf5c86c9df1b4e1616a4cf06ee14a6edf2e5e1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Oct 2019 17:45:09 +0200
Subject: [PATCH 1595/2196] Add experimental-features setting

Experimental features are now opt-in. There is currently one
experimental feature: "nix-command" (which enables the "nix"
command. This will allow us to merge experimental features more
quickly, without committing to supporting them indefinitely.

Typical usage:

$ nix build --experimental-features 'nix-command flakes' nixpkgs#hello

(cherry picked from commit 8e478c234100cf03ea1b777d4bd42a9be7be9e8c,
without the "flakes" feature)
---
 src/libstore/globals.cc | 7 +++++++
 src/libstore/globals.hh | 5 +++++
 src/nix/main.cc         | 2 ++
 tests/init.sh           | 1 +
 4 files changed, 15 insertions(+)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 1c2c08715a1..249c3667357 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures()
     return features;
 }
 
+void Settings::requireExperimentalFeature(const std::string & name)
+{
+    auto & f = experimentalFeatures.get();
+    if (std::find(f.begin(), f.end(), name) == f.end())
+        throw Error("experimental Nix feature '%s' is disabled", name);
+}
+
 const string nixVersion = PACKAGE_VERSION;
 
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index ab1c09aa25d..1221e4db7ab 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -353,6 +353,11 @@ public:
 
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
+
+    Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
+        "Experimental Nix features to enable."};
+
+    void requireExperimentalFeature(const std::string & name);
 };
 
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index a80fd0ea62f..22f5e8c4c8b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -138,6 +138,8 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
+    settings.requireExperimentalFeature("nix-command");
+
     initPlugins();
 
     if (!args.command) args.showHelpAndExit();
diff --git a/tests/init.sh b/tests/init.sh
index 19a12c1e2d9..6a119aad0a4 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -17,6 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
+experimental-features = nix-command
 include nix.conf.extra
 EOF
 

From 0abb3ad53795aa3a4792d30e5721a337f0eddfb7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 30 Mar 2018 00:56:13 +0200
Subject: [PATCH 1596/2196] Allow content-addressable paths to have references

This adds a command 'nix make-content-addressable' that rewrites the
specified store paths into content-addressable paths. The advantage of
such paths is that 1) they can be imported without signatures; 2) they
can enable deduplication in cases where derivation changes do not
cause output changes (apart from store path hashes).

For example,

  $ nix make-content-addressable -r nixpkgs.cowsay
  rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49'
  ...
  rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16'

We can then copy the resulting closure to another store without
signatures:

  $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16

In order to support self-references in content-addressable paths,
these paths are hashed "modulo" self-references, meaning that
self-references are zeroed out during hashing. Somewhat annoyingly,
this means that the NAR hash stored in the Nix database is no longer
necessarily equal to the output of "nix hash-path"; for
content-addressable paths, you need to pass the --modulo flag:

  $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16  | jq -r .[].narHash
  sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16
  1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67
  0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw
---
 src/libstore/build.cc               | 19 +------
 src/libstore/local-store.cc         | 21 ++++++--
 src/libstore/references.cc          | 62 ++++++++++++++++++++++
 src/libstore/references.hh          | 29 +++++++++-
 src/libstore/store-api.cc           | 36 ++++++++-----
 src/libstore/store-api.hh           |  3 +-
 src/libutil/hash.cc                 | 20 ++-----
 src/libutil/hash.hh                 | 11 ++--
 src/libutil/serialise.hh            |  2 +-
 src/libutil/util.cc                 | 13 +++++
 src/libutil/util.hh                 | 14 +++++
 src/nix/command.cc                  |  2 +-
 src/nix/command.hh                  |  4 ++
 src/nix/hash.cc                     | 22 +++++++-
 src/nix/make-content-addressable.cc | 82 +++++++++++++++++++++++++++++
 src/nix/verify.cc                   | 12 +++--
 16 files changed, 288 insertions(+), 64 deletions(-)
 create mode 100644 src/nix/make-content-addressable.cc

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0e0f8a545a0..68c2f2ce361 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -727,23 +727,6 @@ HookInstance::~HookInstance()
 //////////////////////////////////////////////////////////////////////
 
 
-typedef map<std::string, std::string> StringRewrites;
-
-
-std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
-{
-    for (auto & i : rewrites) {
-        size_t j = 0;
-        while ((j = s.find(i.first, j)) != string::npos)
-            s.replace(j, i.first.size(), i.second);
-    }
-    return s;
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
@@ -865,7 +848,7 @@ class DerivationGoal : public Goal
 #endif
 
     /* Hash rewriting. */
-    StringRewrites inputRewrites, outputRewrites;
+    StringMap inputRewrites, outputRewrites;
     typedef map<Path, Path> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d86f7b1d274..1280ecab56c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -5,6 +5,7 @@
 #include "worker-protocol.hh"
 #include "derivations.hh"
 #include "nar-info.hh"
+#include "references.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
-            HashSink hashSink(htSHA256);
+            std::unique_ptr<AbstractHashSink> hashSink;
+            if (info.ca == "")
+                hashSink = std::make_unique<HashSink>(htSHA256);
+            else
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
-                hashSink(data, n);
+                (*hashSink)(data, n);
                 return n;
             });
 
             restorePath(realPath, wrapperSource);
 
-            auto hashResult = hashSink.finish();
+            auto hashResult = hashSink->finish();
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
@@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
-                HashResult current = hashPath(info->narHash.type, toRealPath(i));
+
+                std::unique_ptr<AbstractHashSink> hashSink;
+                if (info->ca == "")
+                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                else
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
+
+                dumpPath(toRealPath(i), *hashSink);
+                auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError(format("path '%1%' was modified! "
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 0dcc264c364..605ca98154f 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path,
 }
 
 
+RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
+    : from(from), to(to), nextSink(nextSink)
+{
+    assert(from.size() == to.size());
+}
+
+void RewritingSink::operator () (const unsigned char * data, size_t len)
+{
+    std::string s(prev);
+    s.append((const char *) data, len);
+
+    size_t j = 0;
+    while ((j = s.find(from, j)) != string::npos) {
+        matches.push_back(pos + j);
+        s.replace(j, from.size(), to);
+    }
+
+    prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
+
+    auto consumed = s.size() - prev.size();
+
+    pos += consumed;
+
+    if (consumed) nextSink((unsigned char *) s.data(), consumed);
+}
+
+void RewritingSink::flush()
+{
+    if (prev.empty()) return;
+    pos += prev.size();
+    nextSink((unsigned char *) prev.data(), prev.size());
+    prev.clear();
+}
+
+HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
+    : hashSink(ht)
+    , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
+{
+}
+
+void HashModuloSink::operator () (const unsigned char * data, size_t len)
+{
+    rewritingSink(data, len);
+}
+
+HashResult HashModuloSink::finish()
+{
+    rewritingSink.flush();
+
+    /* Hash the positions of the self-references. This ensures that a
+       NAR with self-references and a NAR with some of the
+       self-references already zeroed out do not produce a hash
+       collision. FIXME: proof. */
+    for (auto & pos : rewritingSink.matches) {
+        auto s = fmt("|%d", pos);
+        hashSink((unsigned char *) s.data(), s.size());
+    }
+
+    auto h = hashSink.finish();
+    return {h.first, rewritingSink.pos};
+}
+
 }
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index 013809d122f..c38bdd720d3 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -7,5 +7,32 @@ namespace nix {
 
 PathSet scanForReferences(const Path & path, const PathSet & refs,
     HashResult & hash);
-    
+
+struct RewritingSink : Sink
+{
+    std::string from, to, prev;
+    Sink & nextSink;
+    uint64_t pos = 0;
+
+    std::vector<uint64_t> matches;
+
+    RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
+
+    void operator () (const unsigned char * data, size_t len) override;
+
+    void flush();
+};
+
+struct HashModuloSink : AbstractHashSink
+{
+    HashSink hashSink;
+    RewritingSink rewritingSink;
+
+    HashModuloSink(HashType ht, const std::string & modulus);
+
+    void operator () (const unsigned char * data, size_t len) override;
+
+    HashResult finish() override;
+};
+
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index be13fa49aa4..1a1b24e3b0c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id,
 }
 
 
+static std::string makeType(string && type, const PathSet & references)
+{
+    for (auto & i : references) {
+        type += ":";
+        type += i;
+    }
+    return type;
+}
+
+
 Path Store::makeFixedOutputPath(bool recursive,
-    const Hash & hash, const string & name) const
+    const Hash & hash, const string & name, const PathSet & references) const
 {
-    return hash.type == htSHA256 && recursive
-        ? makeStorePath("source", hash, name)
-        : makeStorePath("output:out", hashString(htSHA256,
+    if (hash.type == htSHA256 && recursive) {
+        return makeStorePath(makeType("source", references), hash, name);
+    } else {
+        assert(references.empty());
+        return makeStorePath("output:out", hashString(htSHA256,
                 "fixed:out:" + (recursive ? (string) "r:" : "") +
-                hash.to_string(Base16) + ":"),
-            name);
+                hash.to_string(Base16) + ":"), name);
+    }
 }
 
 
@@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
     /* Stuff the references (if any) into the type.  This is a bit
        hacky, but we can't put them in `s' since that would be
        ambiguous. */
-    string type = "text";
-    for (auto & i : references) {
-        type += ":";
-        type += i;
-    }
-    return makeStorePath(type, hash, name);
+    return makeStorePath(makeType("text", references), hash, name);
 }
 
 
@@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     else if (hasPrefix(ca, "fixed:")) {
         bool recursive = ca.compare(6, 2, "r:") == 0;
         Hash hash(std::string(ca, recursive ? 8 : 6));
-        if (references.empty() &&
-            store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
+        auto refs = references;
+        replaceInSet(refs, path, std::string("self"));
+        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
             return true;
         else
             warn();
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ba899075567..250e1a4bcc1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -303,7 +303,8 @@ public:
         const Hash & hash, const string & name) const;
 
     Path makeFixedOutputPath(bool recursive,
-        const Hash & hash, const string & name) const;
+        const Hash & hash, const string & name,
+        const PathSet & references = {}) const;
 
     Path makeTextPath(const string & name, const Hash & hash,
         const PathSet & references) const;
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 1c14ebb187c..7caee1da7f8 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s)
 
 Hash hashFile(HashType ht, const Path & path)
 {
-    Ctx ctx;
-    Hash hash(ht);
-    start(ht, ctx);
-
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file '%1%'") % path);
-
-    std::vector<unsigned char> buf(8192);
-    ssize_t n;
-    while ((n = read(fd.get(), buf.data(), buf.size()))) {
-        checkInterrupt();
-        if (n == -1) throw SysError(format("reading file '%1%'") % path);
-        update(ht, ctx, buf.data(), n);
-    }
-
-    finish(ht, ctx, hash.hash);
-    return hash;
+    HashSink sink(ht);
+    readFile(path, sink);
+    return sink.finish().first;
 }
 
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 2dbc3b63081..ffa43ecf5a1 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -111,7 +111,12 @@ string printHashType(HashType ht);
 
 union Ctx;
 
-class HashSink : public BufferedSink
+struct AbstractHashSink : virtual Sink
+{
+    virtual HashResult finish() = 0;
+};
+
+class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
     HashType ht;
@@ -122,8 +127,8 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    void write(const unsigned char * data, size_t len);
-    HashResult finish();
+    void write(const unsigned char * data, size_t len) override;
+    HashResult finish() override;
     HashResult currentHash();
 };
 
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index a344a5ac752..0120aeecb23 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -24,7 +24,7 @@ struct Sink
 
 
 /* A buffered abstract sink. */
-struct BufferedSink : Sink
+struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
     std::unique_ptr<unsigned char[]> buffer;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6f3bf7ae86e..2e416edeffe 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s,
 }
 
 
+std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
+{
+    auto s = _s;
+    for (auto & i : rewrites) {
+        if (i.first == i.second) continue;
+        size_t j = 0;
+        while ((j = s.find(i.first, j)) != string::npos)
+            s.replace(j, i.first.size(), i.second);
+    }
+    return s;
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index f057fdb2c04..3493e80b5eb 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -362,6 +362,20 @@ string replaceStrings(const std::string & s,
     const std::string & from, const std::string & to);
 
 
+std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
+
+
+/* If a set contains 'from', remove it and insert 'to'. */
+template<typename T>
+void replaceInSet(std::set<T> & set, const T & from, const T & to)
+{
+    auto i = set.find(from);
+    if (i == set.end()) return;
+    set.erase(i);
+    set.insert(to);
+}
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 3d7d582d6f5..532f331a757 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -129,7 +129,7 @@ void StorePathsCommand::run(ref<Store> store)
     }
 
     else {
-        for (auto & p : toStorePaths(store, NoBuild, installables))
+        for (auto & p : toStorePaths(store, realiseMode, installables))
             storePaths.push_back(p);
 
         if (recursive) {
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 97a6fee7fd2..45ad1cd2a28 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -139,6 +139,10 @@ private:
     bool recursive = false;
     bool all = false;
 
+protected:
+
+    RealiseMode realiseMode = NoBuild;
+
 public:
 
     StorePathsCommand(bool recursive = false);
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index af4105e2890..9c06e611609 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -2,6 +2,8 @@
 #include "hash.hh"
 #include "legacy.hh"
 #include "shared.hh"
+#include "references.hh"
+#include "archive.hh"
 
 using namespace nix;
 
@@ -13,6 +15,7 @@ struct CmdHash : Command
     bool truncate = false;
     HashType ht = htSHA256;
     std::vector<std::string> paths;
+    std::experimental::optional<std::string> modulus;
 
     CmdHash(Mode mode) : mode(mode)
     {
@@ -23,6 +26,11 @@ struct CmdHash : Command
         mkFlag()
             .longName("type")
             .mkHashTypeFlag(&ht);
+        mkFlag()
+            .longName("modulo")
+            .description("compute hash modulo specified string")
+            .labels({"modulus"})
+            .dest(&modulus);
         expectArgs("paths", &paths);
     }
 
@@ -41,7 +49,19 @@ struct CmdHash : Command
     void run() override
     {
         for (auto path : paths) {
-            Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
+
+            std::unique_ptr<AbstractHashSink> hashSink;
+            if (modulus)
+                hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
+            else
+                hashSink = std::make_unique<HashSink>(ht);
+
+            if (mode == mFile)
+                readFile(path, *hashSink);
+            else
+                dumpPath(path, *hashSink);
+
+            Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
                 h.to_string(base, base == SRI);
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
new file mode 100644
index 00000000000..14ed06413b7
--- /dev/null
+++ b/src/nix/make-content-addressable.cc
@@ -0,0 +1,82 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "references.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressable : StorePathsCommand
+{
+    CmdMakeContentAddressable()
+    {
+        realiseMode = Build;
+    }
+
+    std::string name() override
+    {
+        return "make-content-addressable";
+    }
+
+    std::string description() override
+    {
+        return "test";
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));
+
+        paths.reverse();
+
+        std::map<Path, Path> remappings;
+
+        for (auto & path : paths) {
+            auto oldInfo = store->queryPathInfo(path);
+            auto oldHashPart = storePathToHash(path);
+            auto name = storePathToName(path);
+
+            StringSink sink;
+            store->narFromPath(path, sink);
+
+            StringMap rewrites;
+
+            ValidPathInfo info;
+            for (auto & ref : oldInfo->references) {
+                if (ref == path)
+                    info.references.insert("self");
+                else {
+                    auto replacement = get(remappings, ref, ref);
+                    // FIXME: warn about unremapped paths?
+                    info.references.insert(replacement);
+                    if (replacement != ref)
+                        rewrites[storePathToHash(ref)] = storePathToHash(replacement);
+                }
+            }
+
+            *sink.s = rewriteStrings(*sink.s, rewrites);
+
+            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+            hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
+
+            info.narHash = hashModuloSink.finish().first;
+            info.narSize = sink.s->size();
+            replaceInSet(info.references, path, std::string("self"));
+            info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references);
+            replaceInSet(info.references, std::string("self"), info.path);
+            info.ca = makeFixedOutputCA(true, info.narHash);
+
+            printError("rewrote '%s' to '%s'", path, info.path);
+
+            auto source = sinkToSource([&](Sink & nextSink) {
+                RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink);
+                rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                rsink2.flush();
+            });
+
+            store->addToStore(info, *source);
+
+            remappings[path] = info.path;
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdMakeContentAddressable>());
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 74d9673b616..4b0f80c62b4 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -3,6 +3,7 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "thread-pool.hh"
+#include "references.hh"
 
 #include <atomic>
 
@@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand
 
                 if (!noContents) {
 
-                    HashSink sink(info->narHash.type);
-                    store->narFromPath(info->path, sink);
+                    std::unique_ptr<AbstractHashSink> hashSink;
+                    if (info->ca == "")
+                        hashSink = std::make_unique<HashSink>(info->narHash.type);
+                    else
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
 
-                    auto hash = sink.finish();
+                    store->narFromPath(info->path, *hashSink);
+
+                    auto hash = hashSink->finish();
 
                     if (hash.first != info->narHash) {
                         corrupted++;

From d77970fde7fac66cf8c5cdb7688906ba2f5d5362 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Oct 2019 17:49:16 +0200
Subject: [PATCH 1597/2196] Fix build

---
 src/nix/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 9c06e611609..53a9fd20437 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -15,7 +15,7 @@ struct CmdHash : Command
     bool truncate = false;
     HashType ht = htSHA256;
     std::vector<std::string> paths;
-    std::experimental::optional<std::string> modulus;
+    std::optional<std::string> modulus;
 
     CmdHash(Mode mode) : mode(mode)
     {

From e68736936a11c76725915e628b8d41d9351bbcf2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Oct 2019 17:58:17 +0200
Subject: [PATCH 1598/2196] nix make-content-addressable: Add examples

---
 src/nix/make-content-addressable.cc | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 14ed06413b7..16344ee14c5 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -18,9 +18,22 @@ struct CmdMakeContentAddressable : StorePathsCommand
 
     std::string description() override
     {
-        return "test";
+        return "rewrite a path or closure to content-addressable form";
     }
 
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To create a content-addressable representation of GNU Hello (but not its dependencies):",
+                "nix make-content-addressable nixpkgs.hello"
+            },
+            Example{
+                "To compute a content-addressable representation of the current NixOS system closure:",
+                "nix make-content-addressable -r /run/current-system"
+            },
+        };
+    }
     void run(ref<Store> store, Paths storePaths) override
     {
         auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));

From 629b9b0049363e091b76b7f60a8357d9f94733cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Oct 2019 18:05:31 +0200
Subject: [PATCH 1599/2196] Mark content-addressable paths with references as
 experimental

---
 src/libstore/local-store.cc | 5 ++++-
 src/nix/hash.cc             | 2 ++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1280ecab56c..49061f27d13 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1013,8 +1013,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             std::unique_ptr<AbstractHashSink> hashSink;
             if (info.ca == "")
                 hashSink = std::make_unique<HashSink>(htSHA256);
-            else
+            else {
+                if (!info.references.empty())
+                    settings.requireExperimentalFeature("ca-references");
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
+            }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 53a9fd20437..d7451376c4a 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -26,11 +26,13 @@ struct CmdHash : Command
         mkFlag()
             .longName("type")
             .mkHashTypeFlag(&ht);
+        #if 0
         mkFlag()
             .longName("modulo")
             .description("compute hash modulo specified string")
             .labels({"modulus"})
             .dest(&modulus);
+        #endif
         expectArgs("paths", &paths);
     }
 

From 9be7787ec04d604a4e65a22a6b9ae56567aaf618 Mon Sep 17 00:00:00 2001
From: Ersin Akinci <me@ersinakinci.com>
Date: Mon, 21 Oct 2019 13:12:41 -0700
Subject: [PATCH 1600/2196] Revert "Document import <path> syntax"

This reverts commit d8730fb86facadbef22d3df7f8a743a56e7ed53c.
---
 doc/manual/expressions/builtins.xml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 21da8b87b28..465fa1e0b00 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -733,21 +733,13 @@ builtins.genList (x: x * x) 5
   <varlistentry xml:id='builtin-import'>
     <term><function>import</function>
     <replaceable>path</replaceable></term>
-    <term><function>import</function>
-    <replaceable>&lt;path&gt;</replaceable></term>
     <term><function>builtins.import</function>
     <replaceable>path</replaceable></term>
-    <term><function>builtins.import</function>
-    <replaceable>&lt;path&gt;</replaceable></term>
 
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
     </replaceable> is a directory, the file <filename>default.nix
-    </filename> in that directory is loaded.  If the <replaceable>
-    &lt;path&gt;</replaceable> syntax is used, the path will be resolved
-    relative to those listed in the <envar>NIX_PATH</envar> environment
-    variable (see the section on <envar linkend="env-NIX_PATH">NIX_PATH</envar>
-    for details on how the resolution works). Evaluation aborts if the
+    </filename> in that directory is loaded.  Evaluation aborts if the
     file doesn’t exist or contains an incorrect Nix expression.
     <function>import</function> implements Nix’s module system: you
     can put any Nix expression (such as a set or a function) in a

From b7a936224ee6e1ad9c842f618bbd5bc1864ebd54 Mon Sep 17 00:00:00 2001
From: Ersin Akinci <me@ersinakinci.com>
Date: Mon, 21 Oct 2019 14:11:26 -0700
Subject: [PATCH 1601/2196] Add hint about path in builtins.import

---
 doc/manual/expressions/builtins.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 465fa1e0b00..8b53cd57854 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -746,6 +746,10 @@ builtins.genList (x: x * x) 5
     separate file, and use it from Nix expressions in other
     files.</para>
 
+    <note><para>Unlike some languages, <function>import</function> is a regular
+    function in Nix. Paths using the angle bracket syntax (e.g., <function>import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal
+    values (see <xref linkend='ssec-values' />).</para></note>
+
     <para>A Nix expression loaded by <function>import</function> must
     not contain any <emphasis>free variables</emphasis> (identifiers
     that are not defined in the Nix expression itself and are not

From f107a270026596e525bdf8df0e2d331783ec4a10 Mon Sep 17 00:00:00 2001
From: Ersin Akinci <me@ersinakinci.com>
Date: Mon, 21 Oct 2019 14:16:55 -0700
Subject: [PATCH 1602/2196] Tweak path hint

---
 doc/manual/expressions/builtins.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 8b53cd57854..f39b393bf71 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -747,7 +747,8 @@ builtins.genList (x: x * x) 5
     files.</para>
 
     <note><para>Unlike some languages, <function>import</function> is a regular
-    function in Nix. Paths using the angle bracket syntax (e.g., <function>import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal
+    function in Nix. Paths using the angle bracket syntax (e.g., <function>
+    import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal path
     values (see <xref linkend='ssec-values' />).</para></note>
 
     <para>A Nix expression loaded by <function>import</function> must

From c92ea927e508f3c06b6b3ee68e1f0565264cac2c Mon Sep 17 00:00:00 2001
From: Chaz Schlarp <schlarpc@gmail.com>
Date: Tue, 22 Oct 2019 16:04:49 -0700
Subject: [PATCH 1603/2196] Remove superfluous IAM action for S3 cache

`s3:ListObjects` isn't a real IAM action, but _is_ the name of an S3 API method. `s3:ListBucket` is the relevant action for that method.

https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html
---
 doc/manual/packages/s3-substituter.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml
index 1722090efec..868b5a66dc3 100644
--- a/doc/manual/packages/s3-substituter.xml
+++ b/doc/manual/packages/s3-substituter.xml
@@ -159,7 +159,6 @@ the S3 URL:</para>
         "s3:ListBucket",
         "s3:ListBucketMultipartUploads",
         "s3:ListMultipartUploadParts",
-        "s3:ListObjects",
         "s3:PutObject"
       ],
       "Resource": [

From 73ff84f6a820ac1c6e0fe502692432c8945fd8b0 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 23 Oct 2019 15:29:16 +0200
Subject: [PATCH 1604/2196] nix repl: add :edit command

This allows to have a repl-centric workflow to working on nixpkgs.

Usage:

    :edit <package> - heuristic that find the package file path

    :edit <path> - just open the editor on the file path

Once invoked, `nix repl` will open $EDITOR on that file path. Once the
editor exits, `nix repl` will automatically reload itself.
---
 src/nix/repl.cc | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index f857b2e89c2..79f365cdb60 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -22,6 +22,7 @@ extern "C" {
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "attr-path.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
 #include "get-drvs.hh"
@@ -440,6 +441,7 @@ bool NixRepl::processLine(string line)
              << "  <x> = <expr>  Bind expression to variable\n"
              << "  :a <expr>     Add attributes from resulting set to scope\n"
              << "  :b <expr>     Build derivation\n"
+             << "  :e <expr>     Open the derivation in $EDITOR\n"
              << "  :i <expr>     Build derivation, then install result into current profile\n"
              << "  :l <path>     Load Nix expression and add it to scope\n"
              << "  :p <expr>     Evaluate and print expression recursively\n"
@@ -466,6 +468,61 @@ bool NixRepl::processLine(string line)
         reloadFiles();
     }
 
+    else if (command == ":e" || command == ":edit") {
+        Value v;
+        evalString(arg, v);
+
+        std::string filename;
+        int lineno = 0;
+
+        if (v.type == tPath || v.type == tString) {
+            PathSet context;
+            filename = state.coerceToString(noPos, v, context);
+            lineno = 0;
+        } else {
+            // assume it's a derivation
+            Value * v2;
+            try {
+                auto dummyArgs = state.allocBindings(0);
+                v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
+            } catch (Error &) {
+                throw Error("package '%s' has no source location information", arg);
+            }
+
+            auto pos = state.forceString(*v2);
+            debug("position is %s", pos);
+
+            auto colon = pos.rfind(':');
+            if (colon == std::string::npos)
+                throw Error("cannot parse meta.position attribute '%s'", pos);
+
+            filename = std::string(pos, 0, colon);
+            try {
+                lineno = std::stoi(std::string(pos, colon + 1));
+            } catch (std::invalid_argument & e) {
+                throw Error("cannot parse line number '%s'", pos);
+            }
+
+        }
+
+        // Open in EDITOR
+        auto editor = getEnv("EDITOR", "cat");
+        auto args = tokenizeString<Strings>(editor);
+        if (lineno > 0 && (
+            editor.find("emacs") != std::string::npos ||
+            editor.find("nano") != std::string::npos ||
+            editor.find("vim") != std::string::npos))
+            args.push_back(fmt("+%d", lineno));
+        args.push_back(filename);
+        editor = args.front();
+        args.pop_front();
+        runProgram(editor, args);
+
+        // Reload right after exiting the editor
+        state.resetFileCache();
+        reloadFiles();
+    }
+
     else if (command == ":t") {
         Value v;
         evalString(arg, v);

From 207a537343ace1d39ac125059456d832a5237f2e Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 23 Oct 2019 16:48:28 +0200
Subject: [PATCH 1605/2196] libutil: add editorFor heuristic

---
 src/libutil/args.cc | 13 +++++++++++++
 src/libutil/args.hh |  3 +++
 src/nix/edit.cc     | 17 +++++------------
 src/nix/repl.cc     | 11 ++---------
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 7af2a1bf731..35ec3e4ab75 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -178,6 +178,19 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
+Strings editorFor(std::string filename, int lineno)
+{
+    auto editor = getEnv("EDITOR", "cat");
+    auto args = tokenizeString<Strings>(editor);
+    if (lineno > 0 && (
+        editor.find("emacs") != std::string::npos ||
+        editor.find("nano") != std::string::npos ||
+        editor.find("vim") != std::string::npos))
+        args.push_back(fmt("+%d", lineno));
+    args.push_back(filename);
+    return args;
+}
+
 std::string renderLabels(const Strings & labels)
 {
     std::string res;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index ad5fcca3941..22702c2d89b 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -190,6 +190,9 @@ public:
 
 Strings argvToStrings(int argc, char * * argv);
 
+/* Helper function to generate args that invoke $EDITOR on filename:lineno */
+Strings editorFor(std::string filename, int lineno);
+
 /* Helper function for rendering argument labels. */
 std::string renderLabels(const Strings & labels);
 
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index a6169f1183c..3a27b9ccac0 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -59,22 +59,15 @@ struct CmdEdit : InstallableCommand
             throw Error("cannot parse line number '%s'", pos);
         }
 
-        auto editor = getEnv("EDITOR", "cat");
-
-        auto args = tokenizeString<Strings>(editor);
-
-        if (editor.find("emacs") != std::string::npos ||
-            editor.find("nano") != std::string::npos ||
-            editor.find("vim") != std::string::npos)
-            args.push_back(fmt("+%d", lineno));
-
-        args.push_back(filename);
-
         stopProgressBar();
 
+        auto args = editorFor(filename, lineno);
+
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("cannot run editor '%s'", editor);
+        std::string command;
+        for (const auto &arg : args) command += " '" + arg + "'";
+        throw SysError("cannot run command%s", command);
     }
 };
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 79f365cdb60..d4334cf7f73 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -506,15 +506,8 @@ bool NixRepl::processLine(string line)
         }
 
         // Open in EDITOR
-        auto editor = getEnv("EDITOR", "cat");
-        auto args = tokenizeString<Strings>(editor);
-        if (lineno > 0 && (
-            editor.find("emacs") != std::string::npos ||
-            editor.find("nano") != std::string::npos ||
-            editor.find("vim") != std::string::npos))
-            args.push_back(fmt("+%d", lineno));
-        args.push_back(filename);
-        editor = args.front();
+        auto args = editorFor(filename, lineno);
+        auto editor = args.front();
         args.pop_front();
         runProgram(editor, args);
 

From 59c72497696eaafa294c34699795788d24d68c68 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 23 Oct 2019 17:21:10 +0200
Subject: [PATCH 1606/2196] libexpr: add findDerivationFilename

extract the derivation to filename:lineno heuristic
---
 src/libexpr/attr-path.cc | 28 ++++++++++++++++++++++++++++
 src/libexpr/attr-path.hh |  5 +++++
 src/nix/edit.cc          | 23 ++---------------------
 src/nix/repl.cc          | 23 +----------------------
 4 files changed, 36 insertions(+), 43 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index b0f80db32a8..7a6d8dfd01a 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -93,4 +93,32 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 }
 
 
+std::tuple<std::string, int> findDerivationFilename(EvalState & state, Value & v, std::string what)
+{
+    Value * v2;
+    try {
+        auto dummyArgs = state.allocBindings(0);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
+    } catch (Error &) {
+        throw Error("package '%s' has no source location information", what);
+    }
+
+    auto pos = state.forceString(*v2);
+
+    auto colon = pos.rfind(':');
+    if (colon == std::string::npos)
+        throw Error("cannot parse meta.position attribute '%s'", pos);
+
+    std::string filename(pos, 0, colon);
+    int lineno;
+    try {
+        lineno = std::stoi(std::string(pos, colon + 1));
+    } catch (std::invalid_argument & e) {
+        throw Error("cannot parse line number '%s'", pos);
+    }
+
+    return std::make_tuple(filename, lineno);
+}
+
+
 }
diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh
index 46a34195093..dca94cc7832 100644
--- a/src/libexpr/attr-path.hh
+++ b/src/libexpr/attr-path.hh
@@ -4,10 +4,15 @@
 
 #include <string>
 #include <map>
+#include <tuple>
 
 namespace nix {
 
 Value * findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
+/* Heuristic to find the filename and lineno or a derivation. */
+std::tuple<std::string, int> findDerivationFilename(EvalState & state,
+    Value & v, std::string what);
+
 }
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 3a27b9ccac0..a4aa40bed8f 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -36,28 +36,9 @@ struct CmdEdit : InstallableCommand
 
         auto v = installable->toValue(*state);
 
-        Value * v2;
-        try {
-            auto dummyArgs = state->allocBindings(0);
-            v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v);
-        } catch (Error &) {
-            throw Error("package '%s' has no source location information", installable->what());
-        }
-
-        auto pos = state->forceString(*v2);
-        debug("position is %s", pos);
-
-        auto colon = pos.rfind(':');
-        if (colon == std::string::npos)
-            throw Error("cannot parse meta.position attribute '%s'", pos);
-
-        std::string filename(pos, 0, colon);
+        std::string filename;
         int lineno;
-        try {
-            lineno = std::stoi(std::string(pos, colon + 1));
-        } catch (std::invalid_argument & e) {
-            throw Error("cannot parse line number '%s'", pos);
-        }
+        std::tie(filename, lineno) = findDerivationFilename(*state, *v, installable->what());
 
         stopProgressBar();
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index d4334cf7f73..5cfb408f7b5 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -481,28 +481,7 @@ bool NixRepl::processLine(string line)
             lineno = 0;
         } else {
             // assume it's a derivation
-            Value * v2;
-            try {
-                auto dummyArgs = state.allocBindings(0);
-                v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
-            } catch (Error &) {
-                throw Error("package '%s' has no source location information", arg);
-            }
-
-            auto pos = state.forceString(*v2);
-            debug("position is %s", pos);
-
-            auto colon = pos.rfind(':');
-            if (colon == std::string::npos)
-                throw Error("cannot parse meta.position attribute '%s'", pos);
-
-            filename = std::string(pos, 0, colon);
-            try {
-                lineno = std::stoi(std::string(pos, colon + 1));
-            } catch (std::invalid_argument & e) {
-                throw Error("cannot parse line number '%s'", pos);
-            }
-
+            std::tie(filename, lineno) = findDerivationFilename(state, v, arg);
         }
 
         // Open in EDITOR

From 2f96a89646c6e55e2f1bbb80805dcbbe60fa94ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 23 Oct 2019 21:24:00 +0200
Subject: [PATCH 1607/2196] install-multi-user.sh: Remove unused variables

https://hydra.nixos.org/build/104119659
---
 scripts/install-multi-user.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index d060e5165a8..13762cba389 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -19,9 +19,6 @@ readonly BLUE_UL='\033[38;4;34m'
 readonly GREEN='\033[38;32m'
 readonly GREEN_UL='\033[38;4;32m'
 readonly RED='\033[38;31m'
-readonly RED_UL='\033[38;4;31m'
-readonly YELLOW='\033[38;33m'
-readonly YELLOW_UL='\033[38;4;33m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"

From 70cab0587dcbc2d0600c07c96b477a2ea2fcd6f3 Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Fri, 25 Oct 2019 07:23:05 -0400
Subject: [PATCH 1608/2196] Switch to nixpkgs 19.09

---
 release.nix | 2 +-
 shell.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 0fe6835186b..8fcf04680de 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
diff --git a/shell.nix b/shell.nix
index 8167f87a292..9c504f024e9 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From e583df52800b4baa1564b027fe3b83a21756c2cc Mon Sep 17 00:00:00 2001
From: Robin Gloster <mail@glob.in>
Date: Sun, 27 Oct 2019 10:15:51 +0100
Subject: [PATCH 1609/2196] builtins.toJSON: fix __toString usage

---
 src/libexpr/eval.cc             | 23 +++++++++++++++++------
 src/libexpr/eval.hh             |  4 ++++
 src/libexpr/value-to-json.cc    |  7 ++++++-
 tests/lang/eval-okay-tojson.exp |  2 +-
 tests/lang/eval-okay-tojson.nix |  1 +
 5 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d8e10d9f20e..8f728b906c2 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1565,6 +1565,19 @@ bool EvalState::isDerivation(Value & v)
 }
 
 
+std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
+    PathSet & context, bool coerceMore, bool copyToStore)
+{
+    auto i = v.attrs->find(sToString);
+    if (i != v.attrs->end()) {
+        Value v1;
+        callFunction(*i->value, v, v1, pos);
+        return coerceToString(pos, v1, context, coerceMore, copyToStore);
+    }
+
+    return {};
+}
+
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
@@ -1583,13 +1596,11 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     }
 
     if (v.type == tAttrs) {
-        auto i = v.attrs->find(sToString);
-        if (i != v.attrs->end()) {
-            Value v1;
-            callFunction(*i->value, v, v1, pos);
-            return coerceToString(pos, v1, context, coerceMore, copyToStore);
+        auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
+        if (maybeString) {
+            return *maybeString;
         }
-        i = v.attrs->find(sOutPath);
+        auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index a314e01e0a7..82e6ca8779a 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -8,6 +8,7 @@
 #include "config.hh"
 
 #include <map>
+#include <optional>
 #include <unordered_map>
 
 
@@ -195,6 +196,9 @@ public:
        set with attribute `type = "derivation"'). */
     bool isDerivation(Value & v);
 
+    std::optional<string> tryAttrsToString(const Pos & pos, Value & v,
+        PathSet & context, bool coerceMore = false, bool copyToStore = true);
+
     /* String coercion.  Converts strings, paths and derivations to a
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index 72e413e4491..5fe8570adeb 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -40,7 +40,12 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case tAttrs: {
-            Bindings::iterator i = v.attrs->find(state.sOutPath);
+            auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
+            if (maybeString) {
+                out.write(*maybeString);
+                break;
+            }
+            auto i = v.attrs->find(state.sOutPath);
             if (i == v.attrs->end()) {
                 auto obj(out.object());
                 StringSet names;
diff --git a/tests/lang/eval-okay-tojson.exp b/tests/lang/eval-okay-tojson.exp
index 33588493f75..e92aae3235f 100644
--- a/tests/lang/eval-okay-tojson.exp
+++ b/tests/lang/eval-okay-tojson.exp
@@ -1 +1 @@
-"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44}"
+"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44,\"k\":\"foo\"}"
diff --git a/tests/lang/eval-okay-tojson.nix b/tests/lang/eval-okay-tojson.nix
index c046ba4ae59..ce67943bead 100644
--- a/tests/lang/eval-okay-tojson.nix
+++ b/tests/lang/eval-okay-tojson.nix
@@ -9,4 +9,5 @@ builtins.toJSON
     h = [ "a" [ "b" { "foo\nbar" = {}; } ] ];
     i = 1 + 2;
     j = 1.44;
+    k = { __toString = self: self.a; a = "foo"; };
   }

From 9d612c393abc3a73590650d24bcfe2ee57792872 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 27 Oct 2019 09:34:33 +0100
Subject: [PATCH 1610/2196] Add inNixShell = true to nix-shell auto-call

This is an alternative to the IN_NIX_SHELL environment variable,
allowing the expression to adapt itself to nix-shell without
triggering those adaptations when used as a dependency of another
shell.

Closes #3147
---
 src/nix-build/nix-build.cc | 16 +++++++++++++++-
 tests/nix-shell.sh         |  8 ++++----
 tests/shell.nix            |  3 ++-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 085650e4f29..357986980a0 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -245,7 +245,21 @@ static void _main(int argc, char * * argv)
     auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
     state->repair = repair;
 
-    Bindings & autoArgs = *myArgs.getAutoArgs(*state);
+    Bindings & autoArgs = *[&](){ 
+            Bindings *userAutoArgs = myArgs.getAutoArgs(*state);
+            if (runEnv) {
+                Bindings * res = state->allocBindings(userAutoArgs->size() + 1);
+                Value * tru = state->allocValue();
+                mkBool(*tru, true);
+                res->push_back(Attr(state->symbols.create("inNixShell"), tru));
+                for (auto & i : *userAutoArgs) {
+                    res->push_back(i);
+                }
+                res->sort();
+                return res;
+            }
+            else return userAutoArgs;
+        }();
 
     if (packages) {
         std::ostringstream joined;
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index ee502dddb95..235e2a5ffbf 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -7,9 +7,9 @@ export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
 export NIX_BUILD_SHELL=$SHELL
 output=$(nix-shell --pure shell.nix -A shellDrv --run \
-    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"')
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"')
 
-[ "$output" = " - foo - bar" ]
+[ "$output" = " - foo - bar - true" ]
 
 # Test --keep
 output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \
@@ -19,10 +19,10 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run
 
 # Test nix-shell on a .drv
 [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
-    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]]
 
 [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
-    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
+    'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]]
 
 # Test nix-shell on a .drv symlink
 
diff --git a/tests/shell.nix b/tests/shell.nix
index eb39f9039a8..6cb4f082b44 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -1,4 +1,4 @@
-{ }:
+{ inNixShell ? false }:
 
 with import ./config.nix;
 
@@ -22,6 +22,7 @@ let pkgs = rec {
     name = "shellDrv";
     builder = "/does/not/exist";
     VAR_FROM_NIX = "bar";
+    TEST_inNixShell = if inNixShell then "true" else "false";
     inherit stdenv;
   };
 

From 3913afdd69a5bd065d6096dd569397a836ac6857 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 27 Oct 2019 17:58:25 +0100
Subject: [PATCH 1611/2196] Simplification

---
 src/nix-build/nix-build.cc | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 357986980a0..1c5d7168189 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -245,21 +245,17 @@ static void _main(int argc, char * * argv)
     auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
     state->repair = repair;
 
-    Bindings & autoArgs = *[&](){ 
-            Bindings *userAutoArgs = myArgs.getAutoArgs(*state);
-            if (runEnv) {
-                Bindings * res = state->allocBindings(userAutoArgs->size() + 1);
-                Value * tru = state->allocValue();
-                mkBool(*tru, true);
-                res->push_back(Attr(state->symbols.create("inNixShell"), tru));
-                for (auto & i : *userAutoArgs) {
-                    res->push_back(i);
-                }
-                res->sort();
-                return res;
-            }
-            else return userAutoArgs;
-        }();
+    auto autoArgs = myArgs.getAutoArgs(*state);
+
+    if (runEnv) {
+        auto newArgs = state->allocBindings(autoArgs->size() + 1);
+        auto tru = state->allocValue();
+        mkBool(*tru, true);
+        newArgs->push_back(Attr(state->symbols.create("inNixShell"), tru));
+        for (auto & i : *autoArgs) newArgs->push_back(i);
+        newArgs->sort();
+        autoArgs = newArgs;
+    }
 
     if (packages) {
         std::ostringstream joined;
@@ -313,9 +309,9 @@ static void _main(int argc, char * * argv)
         state->eval(e, vRoot);
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot));
+            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot));
             state->forceValue(v);
-            getDerivations(*state, v, "", autoArgs, drvs, false);
+            getDerivations(*state, v, "", *autoArgs, drvs, false);
         }
     }
 

From f1c0b2c0e16154e738f342f2defa42c4d9348c4c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 1 Oct 2018 17:05:55 +0200
Subject: [PATCH 1612/2196] Add O(1)-memory copyPath() function

---
 src/libutil/archive.cc | 9 +++++++++
 src/libutil/archive.hh | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 3aa12027097..db544a212ab 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -375,4 +375,13 @@ void copyNAR(Source & source, Sink & sink)
 }
 
 
+void copyPath(const Path & from, const Path & to)
+{
+    auto source = sinkToSource([&](Sink & sink) {
+        dumpPath(from, sink);
+    });
+    restorePath(to, *source);
+}
+
+
 }
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 25be426c1a4..768fe25368d 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -77,6 +77,8 @@ void restorePath(const Path & path, Source & source);
 /* Read a NAR from 'source' and write it to 'sink'. */
 void copyNAR(Source & source, Sink & sink);
 
+void copyPath(const Path & from, const Path & to);
+
 
 extern const std::string narVersionMagic1;
 

From f7ce80f90ab364058e28989af68ba993fd8180a9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 Oct 2018 11:22:13 +0200
Subject: [PATCH 1613/2196] Factor out linkOrCopy()

---
 src/libstore/build.cc | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 68c2f2ce361..886c864a3cc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1874,6 +1874,21 @@ static void preloadNSS() {
     });
 }
 
+
+void linkOrCopy(const Path & from, const Path & to)
+{
+    if (link(from.c_str(), to.c_str()) == -1) {
+        /* Hard-linking fails if we exceed the maximum link count on a
+           file (e.g. 32000 of ext3), which is quite possible after a
+           'nix-store --optimise'. FIXME: actually, why don't we just
+           bind-mount in this case? */
+        if (errno != EMLINK)
+            throw SysError("linking '%s' to '%s'", to, from);
+        copyPath(from, to);
+    }
+}
+
+
 void DerivationGoal::startBuilder()
 {
     /* Right platform? */
@@ -2117,22 +2132,8 @@ void DerivationGoal::startBuilder()
                 throw SysError(format("getting attributes of path '%1%'") % i);
             if (S_ISDIR(st.st_mode))
                 dirsInChroot[i] = r;
-            else {
-                Path p = chrootRootDir + i;
-                debug("linking '%1%' to '%2%'", p, r);
-                if (link(r.c_str(), p.c_str()) == -1) {
-                    /* Hard-linking fails if we exceed the maximum
-                       link count on a file (e.g. 32000 of ext3),
-                       which is quite possible after a `nix-store
-                       --optimise'. */
-                    if (errno != EMLINK)
-                        throw SysError(format("linking '%1%' to '%2%'") % p % i);
-                    StringSink sink;
-                    dumpPath(r, sink);
-                    StringSource source(*sink.s);
-                    restorePath(p, source);
-                }
-            }
+            else
+                linkOrCopy(r, chrootRootDir + i);
         }
 
         /* If we're repairing, checking or rebuilding part of a

From ec448f8bb694b6f9546e49fe6a79b86ff2b2f90a Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 28 Oct 2019 21:22:38 +0100
Subject: [PATCH 1614/2196] libexpr: findDerivationFilename return Pos instead
 of tuple

---
 src/libexpr/attr-path.cc | 8 +++++---
 src/libexpr/attr-path.hh | 6 ++----
 src/nix/edit.cc          | 6 +++---
 src/nix/repl.cc          | 4 +++-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 7a6d8dfd01a..c86dbeebb16 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -93,7 +93,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 }
 
 
-std::tuple<std::string, int> findDerivationFilename(EvalState & state, Value & v, std::string what)
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
@@ -110,14 +110,16 @@ std::tuple<std::string, int> findDerivationFilename(EvalState & state, Value & v
         throw Error("cannot parse meta.position attribute '%s'", pos);
 
     std::string filename(pos, 0, colon);
-    int lineno;
+    unsigned int lineno;
     try {
         lineno = std::stoi(std::string(pos, colon + 1));
     } catch (std::invalid_argument & e) {
         throw Error("cannot parse line number '%s'", pos);
     }
 
-    return std::make_tuple(filename, lineno);
+    Symbol file = state.symbols.create(filename);
+
+    return { file, lineno, 0 };
 }
 
 
diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh
index dca94cc7832..716e5ba2746 100644
--- a/src/libexpr/attr-path.hh
+++ b/src/libexpr/attr-path.hh
@@ -4,15 +4,13 @@
 
 #include <string>
 #include <map>
-#include <tuple>
 
 namespace nix {
 
 Value * findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
-/* Heuristic to find the filename and lineno or a derivation. */
-std::tuple<std::string, int> findDerivationFilename(EvalState & state,
-    Value & v, std::string what);
+/* Heuristic to find the filename and lineno or a nix value. */
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
 }
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index a4aa40bed8f..d0607747d4e 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -36,9 +36,9 @@ struct CmdEdit : InstallableCommand
 
         auto v = installable->toValue(*state);
 
-        std::string filename;
-        int lineno;
-        std::tie(filename, lineno) = findDerivationFilename(*state, *v, installable->what());
+        Pos pos = findDerivationFilename(*state, *v, installable->what());
+        std::string filename(pos.file);
+        int lineno(pos.line);
 
         stopProgressBar();
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 5cfb408f7b5..ed67c285f6c 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -481,7 +481,9 @@ bool NixRepl::processLine(string line)
             lineno = 0;
         } else {
             // assume it's a derivation
-            std::tie(filename, lineno) = findDerivationFilename(state, v, arg);
+            Pos pos = findDerivationFilename(state, v, arg);
+            filename = pos.file;
+            lineno = pos.line;
         }
 
         // Open in EDITOR

From 3774fe55fd6c96e80cc91e13fe0a231ce836ac47 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 28 Oct 2019 21:36:34 +0100
Subject: [PATCH 1615/2196] editorFor: take a pos object instead

---
 src/libutil/args.cc |  8 ++++----
 src/libutil/args.hh |  3 ++-
 src/nix/edit.cc     |  4 +---
 src/nix/repl.cc     | 13 +++++--------
 4 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 35ec3e4ab75..b7baad37536 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -178,16 +178,16 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
-Strings editorFor(std::string filename, int lineno)
+Strings editorFor(Pos pos)
 {
     auto editor = getEnv("EDITOR", "cat");
     auto args = tokenizeString<Strings>(editor);
-    if (lineno > 0 && (
+    if (pos.line > 0 && (
         editor.find("emacs") != std::string::npos ||
         editor.find("nano") != std::string::npos ||
         editor.find("vim") != std::string::npos))
-        args.push_back(fmt("+%d", lineno));
-    args.push_back(filename);
+        args.push_back(fmt("+%d", pos.line));
+    args.push_back(pos.file);
     return args;
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 22702c2d89b..1e29bd4fae1 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -5,6 +5,7 @@
 #include <memory>
 
 #include "util.hh"
+#include "nixexpr.hh"
 
 namespace nix {
 
@@ -191,7 +192,7 @@ public:
 Strings argvToStrings(int argc, char * * argv);
 
 /* Helper function to generate args that invoke $EDITOR on filename:lineno */
-Strings editorFor(std::string filename, int lineno);
+Strings editorFor(Pos pos);
 
 /* Helper function for rendering argument labels. */
 std::string renderLabels(const Strings & labels);
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index d0607747d4e..553765f137a 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -37,12 +37,10 @@ struct CmdEdit : InstallableCommand
         auto v = installable->toValue(*state);
 
         Pos pos = findDerivationFilename(*state, *v, installable->what());
-        std::string filename(pos.file);
-        int lineno(pos.line);
 
         stopProgressBar();
 
-        auto args = editorFor(filename, lineno);
+        auto args = editorFor(pos);
 
         execvp(args.front().c_str(), stringsToCharPtrs(args).data());
 
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index ed67c285f6c..683a117f3a0 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -472,22 +472,19 @@ bool NixRepl::processLine(string line)
         Value v;
         evalString(arg, v);
 
-        std::string filename;
-        int lineno = 0;
+        Pos pos;
 
         if (v.type == tPath || v.type == tString) {
             PathSet context;
-            filename = state.coerceToString(noPos, v, context);
-            lineno = 0;
+            auto filename = state.coerceToString(noPos, v, context);
+            pos.file = state.symbols.create(filename);
         } else {
             // assume it's a derivation
-            Pos pos = findDerivationFilename(state, v, arg);
-            filename = pos.file;
-            lineno = pos.line;
+            pos = findDerivationFilename(state, v, arg);
         }
 
         // Open in EDITOR
-        auto args = editorFor(filename, lineno);
+        auto args = editorFor(pos);
         auto editor = args.front();
         args.pop_front();
         runProgram(editor, args);

From d407f4d15f86aca585e0edebc5bc74ab8b1bebd1 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 28 Oct 2019 21:37:22 +0100
Subject: [PATCH 1616/2196] nix repl: also handle lambda edit

---
 src/nix/repl.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 683a117f3a0..35c7aec6678 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -478,6 +478,8 @@ bool NixRepl::processLine(string line)
             PathSet context;
             auto filename = state.coerceToString(noPos, v, context);
             pos.file = state.symbols.create(filename);
+        } else if (v.type == tLambda) {
+            pos = v.lambda.fun->pos;
         } else {
             // assume it's a derivation
             pos = findDerivationFilename(state, v, arg);

From 9a2505965667267f03a8385926f3b31a47732ed5 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 28 Oct 2019 21:40:02 +0100
Subject: [PATCH 1617/2196] findDerivationFilename: add FIXME

---
 src/libexpr/attr-path.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index c86dbeebb16..06b472d8bfa 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -103,6 +103,8 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
         throw Error("package '%s' has no source location information", what);
     }
 
+    // FIXME: is it possible to extract the Pos object instead of doing this
+    //        toString + parsing?
     auto pos = state.forceString(*v2);
 
     auto colon = pos.rfind(':');

From 95c727caef110c03900a77133eb3e873fca7f019 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 Oct 2018 11:24:54 +0200
Subject: [PATCH 1618/2196] Remove the check against concurrent builds in the
 same process

---
 src/libstore/build.cc | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 886c864a3cc..dfae8024aa7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4138,9 +4138,6 @@ void SubstitutionGoal::handleEOF(int fd)
 //////////////////////////////////////////////////////////////////////
 
 
-static bool working = false;
-
-
 Worker::Worker(LocalStore & store)
     : act(*logger, actRealise)
     , actDerivations(*logger, actBuilds)
@@ -4148,8 +4145,6 @@ Worker::Worker(LocalStore & store)
     , store(store)
 {
     /* Debugging: prevent recursive workers. */
-    if (working) abort();
-    working = true;
     nrLocalBuilds = 0;
     lastWokenUp = steady_time_point::min();
     permanentFailure = false;
@@ -4161,8 +4156,6 @@ Worker::Worker(LocalStore & store)
 
 Worker::~Worker()
 {
-    working = false;
-
     /* Explicitly get rid of all strong pointers now.  After this all
        goals that refer to this worker should be gone.  (Otherwise we
        are in trouble, since goals may call childTerminated() etc. in

From 2d37e88319441958b7eed876042a106e79c7c85d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 24 Sep 2018 13:53:44 +0200
Subject: [PATCH 1619/2196] Move most of the daemon implementation to libstore

---
 src/libstore/daemon.cc          | 798 +++++++++++++++++++++++++++++++
 src/libstore/daemon.hh          |  12 +
 src/libstore/worker-protocol.hh |   3 +
 src/nix-daemon/nix-daemon.cc    | 802 +-------------------------------
 4 files changed, 821 insertions(+), 794 deletions(-)
 create mode 100644 src/libstore/daemon.cc
 create mode 100644 src/libstore/daemon.hh

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
new file mode 100644
index 00000000000..2e1d10bcca7
--- /dev/null
+++ b/src/libstore/daemon.cc
@@ -0,0 +1,798 @@
+#include "daemon.hh"
+#include "monitor-fd.hh"
+#include "worker-protocol.hh"
+#include "store-api.hh"
+#include "local-store.hh"
+#include "finally.hh"
+#include "affinity.hh"
+#include "archive.hh"
+#include "derivations.hh"
+#include "args.hh"
+
+namespace nix::daemon {
+
+Sink & operator << (Sink & sink, const Logger::Fields & fields)
+{
+    sink << fields.size();
+    for (auto & f : fields) {
+        sink << f.type;
+        if (f.type == Logger::Field::tInt)
+            sink << f.i;
+        else if (f.type == Logger::Field::tString)
+            sink << f.s;
+        else abort();
+    }
+    return sink;
+}
+
+/* Logger that forwards log messages to the client, *if* we're in a
+   state where the protocol allows it (i.e., when canSendStderr is
+   true). */
+struct TunnelLogger : public Logger
+{
+    FdSink & to;
+
+    struct State
+    {
+        bool canSendStderr = false;
+        std::vector<std::string> pendingMsgs;
+    };
+
+    Sync<State> state_;
+
+    unsigned int clientVersion;
+
+    TunnelLogger(FdSink & to, unsigned int clientVersion)
+        : to(to), clientVersion(clientVersion) { }
+
+    void enqueueMsg(const std::string & s)
+    {
+        auto state(state_.lock());
+
+        if (state->canSendStderr) {
+            assert(state->pendingMsgs.empty());
+            try {
+                to(s);
+                to.flush();
+            } catch (...) {
+                /* Write failed; that means that the other side is
+                   gone. */
+                state->canSendStderr = false;
+                throw;
+            }
+        } else
+            state->pendingMsgs.push_back(s);
+    }
+
+    void log(Verbosity lvl, const FormatOrString & fs) override
+    {
+        if (lvl > verbosity) return;
+
+        StringSink buf;
+        buf << STDERR_NEXT << (fs.s + "\n");
+        enqueueMsg(*buf.s);
+    }
+
+    /* startWork() means that we're starting an operation for which we
+      want to send out stderr to the client. */
+    void startWork()
+    {
+        auto state(state_.lock());
+        state->canSendStderr = true;
+
+        for (auto & msg : state->pendingMsgs)
+            to(msg);
+
+        state->pendingMsgs.clear();
+
+        to.flush();
+    }
+
+    /* stopWork() means that we're done; stop sending stderr to the
+       client. */
+    void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
+    {
+        auto state(state_.lock());
+
+        state->canSendStderr = false;
+
+        if (success)
+            to << STDERR_LAST;
+        else {
+            to << STDERR_ERROR << msg;
+            if (status != 0) to << status;
+        }
+    }
+
+    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
+        const std::string & s, const Fields & fields, ActivityId parent) override
+    {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) {
+            if (!s.empty())
+                log(lvl, s + "...");
+            return;
+        }
+
+        StringSink buf;
+        buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
+        enqueueMsg(*buf.s);
+    }
+
+    void stopActivity(ActivityId act) override
+    {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
+        StringSink buf;
+        buf << STDERR_STOP_ACTIVITY << act;
+        enqueueMsg(*buf.s);
+    }
+
+    void result(ActivityId act, ResultType type, const Fields & fields) override
+    {
+        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
+        StringSink buf;
+        buf << STDERR_RESULT << act << type << fields;
+        enqueueMsg(*buf.s);
+    }
+};
+
+struct TunnelSink : Sink
+{
+    Sink & to;
+    TunnelSink(Sink & to) : to(to) { }
+    virtual void operator () (const unsigned char * data, size_t len)
+    {
+        to << STDERR_WRITE;
+        writeString(data, len, to);
+    }
+};
+
+struct TunnelSource : BufferedSource
+{
+    Source & from;
+    BufferedSink & to;
+    TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
+    size_t readUnbuffered(unsigned char * data, size_t len) override
+    {
+        to << STDERR_READ << len;
+        to.flush();
+        size_t n = readString(data, len, from);
+        if (n == 0) throw EndOfFile("unexpected end-of-file");
+        return n;
+    }
+};
+
+/* If the NAR archive contains a single file at top-level, then save
+   the contents of the file to `s'.  Otherwise barf. */
+struct RetrieveRegularNARSink : ParseSink
+{
+    bool regular;
+    string s;
+
+    RetrieveRegularNARSink() : regular(true) { }
+
+    void createDirectory(const Path & path)
+    {
+        regular = false;
+    }
+
+    void receiveContents(unsigned char * data, unsigned int len)
+    {
+        s.append((const char *) data, len);
+    }
+
+    void createSymlink(const Path & path, const string & target)
+    {
+        regular = false;
+    }
+};
+
+static void performOp(TunnelLogger * logger, ref<Store> store,
+    bool trusted, unsigned int clientVersion,
+    Source & from, BufferedSink & to, unsigned int op)
+{
+    switch (op) {
+
+    case wopIsValidPath: {
+        /* 'readStorePath' could raise an error leading to the connection
+           being closed.  To be able to recover from an invalid path error,
+           call 'startWork' early, and do 'assertStorePath' afterwards so
+           that the 'Error' exception handler doesn't close the
+           connection.  */
+        Path path = readString(from);
+        logger->startWork();
+        store->assertStorePath(path);
+        bool result = store->isValidPath(path);
+        logger->stopWork();
+        to << result;
+        break;
+    }
+
+    case wopQueryValidPaths: {
+        PathSet paths = readStorePaths<PathSet>(*store, from);
+        logger->startWork();
+        PathSet res = store->queryValidPaths(paths);
+        logger->stopWork();
+        to << res;
+        break;
+    }
+
+    case wopHasSubstitutes: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        PathSet res = store->querySubstitutablePaths({path});
+        logger->stopWork();
+        to << (res.find(path) != res.end());
+        break;
+    }
+
+    case wopQuerySubstitutablePaths: {
+        PathSet paths = readStorePaths<PathSet>(*store, from);
+        logger->startWork();
+        PathSet res = store->querySubstitutablePaths(paths);
+        logger->stopWork();
+        to << res;
+        break;
+    }
+
+    case wopQueryPathHash: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        auto hash = store->queryPathInfo(path)->narHash;
+        logger->stopWork();
+        to << hash.to_string(Base16, false);
+        break;
+    }
+
+    case wopQueryReferences:
+    case wopQueryReferrers:
+    case wopQueryValidDerivers:
+    case wopQueryDerivationOutputs: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        PathSet paths;
+        if (op == wopQueryReferences)
+            paths = store->queryPathInfo(path)->references;
+        else if (op == wopQueryReferrers)
+            store->queryReferrers(path, paths);
+        else if (op == wopQueryValidDerivers)
+            paths = store->queryValidDerivers(path);
+        else paths = store->queryDerivationOutputs(path);
+        logger->stopWork();
+        to << paths;
+        break;
+    }
+
+    case wopQueryDerivationOutputNames: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        StringSet names;
+        names = store->queryDerivationOutputNames(path);
+        logger->stopWork();
+        to << names;
+        break;
+    }
+
+    case wopQueryDeriver: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        auto deriver = store->queryPathInfo(path)->deriver;
+        logger->stopWork();
+        to << deriver;
+        break;
+    }
+
+    case wopQueryPathFromHashPart: {
+        string hashPart = readString(from);
+        logger->startWork();
+        Path path = store->queryPathFromHashPart(hashPart);
+        logger->stopWork();
+        to << path;
+        break;
+    }
+
+    case wopAddToStore: {
+        bool fixed, recursive;
+        std::string s, baseName;
+        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+        /* Compatibility hack. */
+        if (!fixed) {
+            s = "sha256";
+            recursive = true;
+        }
+        HashType hashAlgo = parseHashType(s);
+
+        TeeSource savedNAR(from);
+        RetrieveRegularNARSink savedRegular;
+
+        if (recursive) {
+            /* Get the entire NAR dump from the client and save it to
+               a string so that we can pass it to
+               addToStoreFromDump(). */
+            ParseSink sink; /* null sink; just parse the NAR */
+            parseDump(sink, savedNAR);
+        } else
+            parseDump(savedRegular, from);
+
+        logger->startWork();
+        if (!savedRegular.regular) throw Error("regular file expected");
+
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+        if (!store2) throw Error("operation is only supported by LocalStore");
+
+        Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        logger->stopWork();
+
+        to << path;
+        break;
+    }
+
+    case wopAddTextToStore: {
+        string suffix = readString(from);
+        string s = readString(from);
+        PathSet refs = readStorePaths<PathSet>(*store, from);
+        logger->startWork();
+        Path path = store->addTextToStore(suffix, s, refs, NoRepair);
+        logger->stopWork();
+        to << path;
+        break;
+    }
+
+    case wopExportPath: {
+        Path path = readStorePath(*store, from);
+        readInt(from); // obsolete
+        logger->startWork();
+        TunnelSink sink(to);
+        store->exportPath(path, sink);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopImportPaths: {
+        logger->startWork();
+        TunnelSource source(from, to);
+        Paths paths = store->importPaths(source, nullptr,
+            trusted ? NoCheckSigs : CheckSigs);
+        logger->stopWork();
+        to << paths;
+        break;
+    }
+
+    case wopBuildPaths: {
+        PathSet drvs = readStorePaths<PathSet>(*store, from);
+        BuildMode mode = bmNormal;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
+            mode = (BuildMode) readInt(from);
+
+            /* Repairing is not atomic, so disallowed for "untrusted"
+               clients.  */
+            if (mode == bmRepair && !trusted)
+                throw Error("repairing is not allowed because you are not in 'trusted-users'");
+        }
+        logger->startWork();
+        store->buildPaths(drvs, mode);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopBuildDerivation: {
+        Path drvPath = readStorePath(*store, from);
+        BasicDerivation drv;
+        readDerivation(from, *store, drv);
+        BuildMode buildMode = (BuildMode) readInt(from);
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to build derivations");
+        auto res = store->buildDerivation(drvPath, drv, buildMode);
+        logger->stopWork();
+        to << res.status << res.errorMsg;
+        break;
+    }
+
+    case wopEnsurePath: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        store->ensurePath(path);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopAddTempRoot: {
+        Path path = readStorePath(*store, from);
+        logger->startWork();
+        store->addTempRoot(path);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopAddIndirectRoot: {
+        Path path = absPath(readString(from));
+        logger->startWork();
+        store->addIndirectRoot(path);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopSyncWithGC: {
+        logger->startWork();
+        store->syncWithGC();
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopFindRoots: {
+        logger->startWork();
+        Roots roots = store->findRoots(!trusted);
+        logger->stopWork();
+
+        size_t size = 0;
+        for (auto & i : roots)
+            size += i.second.size();
+
+        to << size;
+
+        for (auto & [target, links] : roots)
+            for (auto & link : links)
+                to << link << target;
+
+        break;
+    }
+
+    case wopCollectGarbage: {
+        GCOptions options;
+        options.action = (GCOptions::GCAction) readInt(from);
+        options.pathsToDelete = readStorePaths<PathSet>(*store, from);
+        from >> options.ignoreLiveness >> options.maxFreed;
+        // obsolete fields
+        readInt(from);
+        readInt(from);
+        readInt(from);
+
+        GCResults results;
+
+        logger->startWork();
+        if (options.ignoreLiveness)
+            throw Error("you are not allowed to ignore liveness");
+        store->collectGarbage(options, results);
+        logger->stopWork();
+
+        to << results.paths << results.bytesFreed << 0 /* obsolete */;
+
+        break;
+    }
+
+    case wopSetOptions: {
+        settings.keepFailed = readInt(from);
+        settings.keepGoing = readInt(from);
+        settings.tryFallback = readInt(from);
+        verbosity = (Verbosity) readInt(from);
+        settings.maxBuildJobs.assign(readInt(from));
+        settings.maxSilentTime = readInt(from);
+        readInt(from); // obsolete useBuildHook
+        settings.verboseBuild = lvlError == (Verbosity) readInt(from);
+        readInt(from); // obsolete logType
+        readInt(from); // obsolete printBuildTrace
+        settings.buildCores = readInt(from);
+        settings.useSubstitutes  = readInt(from);
+
+        StringMap overrides;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
+            unsigned int n = readInt(from);
+            for (unsigned int i = 0; i < n; i++) {
+                string name = readString(from);
+                string value = readString(from);
+                overrides.emplace(name, value);
+            }
+        }
+
+        logger->startWork();
+
+        for (auto & i : overrides) {
+            auto & name(i.first);
+            auto & value(i.second);
+
+            auto setSubstituters = [&](Setting<Strings> & res) {
+                if (name != res.name && res.aliases.count(name) == 0)
+                    return false;
+                StringSet trusted = settings.trustedSubstituters;
+                for (auto & s : settings.substituters.get())
+                    trusted.insert(s);
+                Strings subs;
+                auto ss = tokenizeString<Strings>(value);
+                for (auto & s : ss)
+                    if (trusted.count(s))
+                        subs.push_back(s);
+                    else
+                        warn("ignoring untrusted substituter '%s'", s);
+                res = subs;
+                return true;
+            };
+
+            try {
+                if (name == "ssh-auth-sock") // obsolete
+                    ;
+                else if (trusted
+                    || name == settings.buildTimeout.name
+                    || name == "connect-timeout"
+                    || (name == "builders" && value == ""))
+                    settings.set(name, value);
+                else if (setSubstituters(settings.substituters))
+                    ;
+                else if (setSubstituters(settings.extraSubstituters))
+                    ;
+                else
+                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
+            } catch (UsageError & e) {
+                warn(e.what());
+            }
+        }
+
+        logger->stopWork();
+        break;
+    }
+
+    case wopQuerySubstitutablePathInfo: {
+        Path path = absPath(readString(from));
+        logger->startWork();
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos({path}, infos);
+        logger->stopWork();
+        SubstitutablePathInfos::iterator i = infos.find(path);
+        if (i == infos.end())
+            to << 0;
+        else {
+            to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize;
+        }
+        break;
+    }
+
+    case wopQuerySubstitutablePathInfos: {
+        PathSet paths = readStorePaths<PathSet>(*store, from);
+        logger->startWork();
+        SubstitutablePathInfos infos;
+        store->querySubstitutablePathInfos(paths, infos);
+        logger->stopWork();
+        to << infos.size();
+        for (auto & i : infos) {
+            to << i.first << i.second.deriver << i.second.references
+               << i.second.downloadSize << i.second.narSize;
+        }
+        break;
+    }
+
+    case wopQueryAllValidPaths: {
+        logger->startWork();
+        PathSet paths = store->queryAllValidPaths();
+        logger->stopWork();
+        to << paths;
+        break;
+    }
+
+    case wopQueryPathInfo: {
+        Path path = readStorePath(*store, from);
+        std::shared_ptr<const ValidPathInfo> info;
+        logger->startWork();
+        try {
+            info = store->queryPathInfo(path);
+        } catch (InvalidPath &) {
+            if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw;
+        }
+        logger->stopWork();
+        if (info) {
+            if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
+                to << 1;
+            to << info->deriver << info->narHash.to_string(Base16, false) << info->references
+               << info->registrationTime << info->narSize;
+            if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
+                to << info->ultimate
+                   << info->sigs
+                   << info->ca;
+            }
+        } else {
+            assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
+            to << 0;
+        }
+        break;
+    }
+
+    case wopOptimiseStore:
+        logger->startWork();
+        store->optimiseStore();
+        logger->stopWork();
+        to << 1;
+        break;
+
+    case wopVerifyStore: {
+        bool checkContents, repair;
+        from >> checkContents >> repair;
+        logger->startWork();
+        if (repair && !trusted)
+            throw Error("you are not privileged to repair paths");
+        bool errors = store->verifyStore(checkContents, (RepairFlag) repair);
+        logger->stopWork();
+        to << errors;
+        break;
+    }
+
+    case wopAddSignatures: {
+        Path path = readStorePath(*store, from);
+        StringSet sigs = readStrings<StringSet>(from);
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to add signatures");
+        store->addSignatures(path, sigs);
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
+    case wopNarFromPath: {
+        auto path = readStorePath(*store, from);
+        logger->startWork();
+        logger->stopWork();
+        dumpPath(path, to);
+        break;
+    }
+
+    case wopAddToStoreNar: {
+        bool repair, dontCheckSigs;
+        ValidPathInfo info;
+        info.path = readStorePath(*store, from);
+        from >> info.deriver;
+        if (!info.deriver.empty())
+            store->assertStorePath(info.deriver);
+        info.narHash = Hash(readString(from), htSHA256);
+        info.references = readStorePaths<PathSet>(*store, from);
+        from >> info.registrationTime >> info.narSize >> info.ultimate;
+        info.sigs = readStrings<StringSet>(from);
+        from >> info.ca >> repair >> dontCheckSigs;
+        if (!trusted && dontCheckSigs)
+            dontCheckSigs = false;
+        if (!trusted)
+            info.ultimate = false;
+
+        std::string saved;
+        std::unique_ptr<Source> source;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+            source = std::make_unique<TunnelSource>(from, to);
+        else {
+            TeeSink tee(from);
+            parseDump(tee, tee.source);
+            saved = std::move(*tee.source.data);
+            source = std::make_unique<StringSource>(saved);
+        }
+
+        logger->startWork();
+
+        // FIXME: race if addToStore doesn't read source?
+        store->addToStore(info, *source, (RepairFlag) repair,
+            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+
+        logger->stopWork();
+        break;
+    }
+
+    case wopQueryMissing: {
+        PathSet targets = readStorePaths<PathSet>(*store, from);
+        logger->startWork();
+        PathSet willBuild, willSubstitute, unknown;
+        unsigned long long downloadSize, narSize;
+        store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
+        logger->stopWork();
+        to << willBuild << willSubstitute << unknown << downloadSize << narSize;
+        break;
+    }
+
+    default:
+        throw Error(format("invalid operation %1%") % op);
+    }
+}
+
+void processConnection(
+    FdSource & from,
+    FdSink & to,
+    bool trusted,
+    const std::string & userName,
+    uid_t userId)
+{
+    MonitorFdHup monitor(from.fd);
+
+    /* Exchange the greeting. */
+    unsigned int magic = readInt(from);
+    if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
+    to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
+    to.flush();
+    unsigned int clientVersion = readInt(from);
+
+    if (clientVersion < 0x10a)
+        throw Error("the Nix client version is too old");
+
+    auto tunnelLogger = new TunnelLogger(to, clientVersion);
+    auto prevLogger = nix::logger;
+    logger = tunnelLogger;
+
+    unsigned int opCount = 0;
+
+    Finally finally([&]() {
+        _isInterrupted = false;
+        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
+    });
+
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
+        setAffinityTo(readInt(from));
+
+    readInt(from); // obsolete reserveSpace
+
+    /* Send startup error messages to the client. */
+    tunnelLogger->startWork();
+
+    try {
+
+        /* If we can't accept clientVersion, then throw an error
+           *here* (not above). */
+
+#if 0
+        /* Prevent users from doing something very dangerous. */
+        if (geteuid() == 0 &&
+            querySetting("build-users-group", "") == "")
+            throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
+#endif
+
+        /* Open the store. */
+        Store::Params params; // FIXME: get params from somewhere
+        // Disable caching since the client already does that.
+        params["path-info-cache-size"] = "0";
+        auto store = openStore(settings.storeUri, params);
+
+        store->createUser(userName, userId);
+
+        tunnelLogger->stopWork();
+        to.flush();
+
+        /* Process client requests. */
+        while (true) {
+            WorkerOp op;
+            try {
+                op = (WorkerOp) readInt(from);
+            } catch (Interrupted & e) {
+                break;
+            } catch (EndOfFile & e) {
+                break;
+            }
+
+            opCount++;
+
+            try {
+                performOp(tunnelLogger, store, trusted, clientVersion, from, to, op);
+            } catch (Error & e) {
+                /* If we're not in a state where we can send replies, then
+                   something went wrong processing the input of the
+                   client.  This can happen especially if I/O errors occur
+                   during addTextToStore() / importPath().  If that
+                   happens, just send the error message and exit. */
+                bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
+                tunnelLogger->stopWork(false, e.msg(), e.status);
+                if (!errorAllowed) throw;
+            } catch (std::bad_alloc & e) {
+                tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
+                throw;
+            }
+
+            to.flush();
+
+            assert(!tunnelLogger->state_.lock()->canSendStderr);
+        };
+
+    } catch (std::exception & e) {
+        tunnelLogger->stopWork(false, e.what(), 1);
+        to.flush();
+        return;
+    }
+}
+
+}
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
new file mode 100644
index 00000000000..6d4015e34d1
--- /dev/null
+++ b/src/libstore/daemon.hh
@@ -0,0 +1,12 @@
+#include "serialise.hh"
+
+namespace nix::daemon {
+
+void processConnection(
+    FdSource & from,
+    FdSink & to,
+    bool trusted,
+    const std::string & userName,
+    uid_t userId);
+
+}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 5ebdfaf134d..a03edd10f6f 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -62,6 +62,9 @@ typedef enum {
 #define STDERR_RESULT         0x52534c54
 
 
+class Store;
+class Source;
+
 Path readStorePath(Store & store, Source & from);
 template<class T> T readStorePaths(Store & store, Source & from);
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index cd18489b0cd..799d2bacfaa 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -2,14 +2,12 @@
 #include "local-store.hh"
 #include "util.hh"
 #include "serialise.hh"
-#include "worker-protocol.hh"
 #include "archive.hh"
-#include "affinity.hh"
 #include "globals.hh"
-#include "monitor-fd.hh"
 #include "derivations.hh"
 #include "finally.hh"
 #include "legacy.hh"
+#include "daemon.hh"
 
 #include <algorithm>
 
@@ -32,6 +30,7 @@
 #endif
 
 using namespace nix;
+using namespace nix::daemon;
 
 #ifndef __linux__
 #define SPLICE_F_MOVE 0
@@ -53,793 +52,6 @@ static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t
 }
 #endif
 
-static FdSource from(STDIN_FILENO);
-static FdSink to(STDOUT_FILENO);
-
-
-Sink & operator << (Sink & sink, const Logger::Fields & fields)
-{
-    sink << fields.size();
-    for (auto & f : fields) {
-        sink << f.type;
-        if (f.type == Logger::Field::tInt)
-            sink << f.i;
-        else if (f.type == Logger::Field::tString)
-            sink << f.s;
-        else abort();
-    }
-    return sink;
-}
-
-
-/* Logger that forwards log messages to the client, *if* we're in a
-   state where the protocol allows it (i.e., when canSendStderr is
-   true). */
-struct TunnelLogger : public Logger
-{
-    struct State
-    {
-        bool canSendStderr = false;
-        std::vector<std::string> pendingMsgs;
-    };
-
-    Sync<State> state_;
-
-    unsigned int clientVersion;
-
-    TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { }
-
-    void enqueueMsg(const std::string & s)
-    {
-        auto state(state_.lock());
-
-        if (state->canSendStderr) {
-            assert(state->pendingMsgs.empty());
-            try {
-                to(s);
-                to.flush();
-            } catch (...) {
-                /* Write failed; that means that the other side is
-                   gone. */
-                state->canSendStderr = false;
-                throw;
-            }
-        } else
-            state->pendingMsgs.push_back(s);
-    }
-
-    void log(Verbosity lvl, const FormatOrString & fs) override
-    {
-        if (lvl > verbosity) return;
-
-        StringSink buf;
-        buf << STDERR_NEXT << (fs.s + "\n");
-        enqueueMsg(*buf.s);
-    }
-
-    /* startWork() means that we're starting an operation for which we
-      want to send out stderr to the client. */
-    void startWork()
-    {
-        auto state(state_.lock());
-        state->canSendStderr = true;
-
-        for (auto & msg : state->pendingMsgs)
-            to(msg);
-
-        state->pendingMsgs.clear();
-
-        to.flush();
-    }
-
-    /* stopWork() means that we're done; stop sending stderr to the
-       client. */
-    void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
-    {
-        auto state(state_.lock());
-
-        state->canSendStderr = false;
-
-        if (success)
-            to << STDERR_LAST;
-        else {
-            to << STDERR_ERROR << msg;
-            if (status != 0) to << status;
-        }
-    }
-
-    void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
-        const std::string & s, const Fields & fields, ActivityId parent) override
-    {
-        if (GET_PROTOCOL_MINOR(clientVersion) < 20) {
-            if (!s.empty())
-                log(lvl, s + "...");
-            return;
-        }
-
-        StringSink buf;
-        buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
-        enqueueMsg(*buf.s);
-    }
-
-    void stopActivity(ActivityId act) override
-    {
-        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
-        StringSink buf;
-        buf << STDERR_STOP_ACTIVITY << act;
-        enqueueMsg(*buf.s);
-    }
-
-    void result(ActivityId act, ResultType type, const Fields & fields) override
-    {
-        if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
-        StringSink buf;
-        buf << STDERR_RESULT << act << type << fields;
-        enqueueMsg(*buf.s);
-    }
-};
-
-
-struct TunnelSink : Sink
-{
-    Sink & to;
-    TunnelSink(Sink & to) : to(to) { }
-    virtual void operator () (const unsigned char * data, size_t len)
-    {
-        to << STDERR_WRITE;
-        writeString(data, len, to);
-    }
-};
-
-
-struct TunnelSource : BufferedSource
-{
-    Source & from;
-    TunnelSource(Source & from) : from(from) { }
-protected:
-    size_t readUnbuffered(unsigned char * data, size_t len) override
-    {
-        to << STDERR_READ << len;
-        to.flush();
-        size_t n = readString(data, len, from);
-        if (n == 0) throw EndOfFile("unexpected end-of-file");
-        return n;
-    }
-};
-
-
-/* If the NAR archive contains a single file at top-level, then save
-   the contents of the file to `s'.  Otherwise barf. */
-struct RetrieveRegularNARSink : ParseSink
-{
-    bool regular;
-    string s;
-
-    RetrieveRegularNARSink() : regular(true) { }
-
-    void createDirectory(const Path & path)
-    {
-        regular = false;
-    }
-
-    void receiveContents(unsigned char * data, unsigned int len)
-    {
-        s.append((const char *) data, len);
-    }
-
-    void createSymlink(const Path & path, const string & target)
-    {
-        regular = false;
-    }
-};
-
-
-static void performOp(TunnelLogger * logger, ref<Store> store,
-    bool trusted, unsigned int clientVersion,
-    Source & from, Sink & to, unsigned int op)
-{
-    switch (op) {
-
-    case wopIsValidPath: {
-        /* 'readStorePath' could raise an error leading to the connection
-           being closed.  To be able to recover from an invalid path error,
-           call 'startWork' early, and do 'assertStorePath' afterwards so
-           that the 'Error' exception handler doesn't close the
-           connection.  */
-        Path path = readString(from);
-        logger->startWork();
-        store->assertStorePath(path);
-        bool result = store->isValidPath(path);
-        logger->stopWork();
-        to << result;
-        break;
-    }
-
-    case wopQueryValidPaths: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
-        logger->startWork();
-        PathSet res = store->queryValidPaths(paths);
-        logger->stopWork();
-        to << res;
-        break;
-    }
-
-    case wopHasSubstitutes: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        PathSet res = store->querySubstitutablePaths({path});
-        logger->stopWork();
-        to << (res.find(path) != res.end());
-        break;
-    }
-
-    case wopQuerySubstitutablePaths: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
-        logger->startWork();
-        PathSet res = store->querySubstitutablePaths(paths);
-        logger->stopWork();
-        to << res;
-        break;
-    }
-
-    case wopQueryPathHash: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        auto hash = store->queryPathInfo(path)->narHash;
-        logger->stopWork();
-        to << hash.to_string(Base16, false);
-        break;
-    }
-
-    case wopQueryReferences:
-    case wopQueryReferrers:
-    case wopQueryValidDerivers:
-    case wopQueryDerivationOutputs: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        PathSet paths;
-        if (op == wopQueryReferences)
-            paths = store->queryPathInfo(path)->references;
-        else if (op == wopQueryReferrers)
-            store->queryReferrers(path, paths);
-        else if (op == wopQueryValidDerivers)
-            paths = store->queryValidDerivers(path);
-        else paths = store->queryDerivationOutputs(path);
-        logger->stopWork();
-        to << paths;
-        break;
-    }
-
-    case wopQueryDerivationOutputNames: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        StringSet names;
-        names = store->queryDerivationOutputNames(path);
-        logger->stopWork();
-        to << names;
-        break;
-    }
-
-    case wopQueryDeriver: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        auto deriver = store->queryPathInfo(path)->deriver;
-        logger->stopWork();
-        to << deriver;
-        break;
-    }
-
-    case wopQueryPathFromHashPart: {
-        string hashPart = readString(from);
-        logger->startWork();
-        Path path = store->queryPathFromHashPart(hashPart);
-        logger->stopWork();
-        to << path;
-        break;
-    }
-
-    case wopAddToStore: {
-        bool fixed, recursive;
-        std::string s, baseName;
-        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-        /* Compatibility hack. */
-        if (!fixed) {
-            s = "sha256";
-            recursive = true;
-        }
-        HashType hashAlgo = parseHashType(s);
-
-        TeeSource savedNAR(from);
-        RetrieveRegularNARSink savedRegular;
-
-        if (recursive) {
-            /* Get the entire NAR dump from the client and save it to
-               a string so that we can pass it to
-               addToStoreFromDump(). */
-            ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNAR);
-        } else
-            parseDump(savedRegular, from);
-
-        logger->startWork();
-        if (!savedRegular.regular) throw Error("regular file expected");
-
-        auto store2 = store.dynamic_pointer_cast<LocalStore>();
-        if (!store2) throw Error("operation is only supported by LocalStore");
-
-        Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
-        logger->stopWork();
-
-        to << path;
-        break;
-    }
-
-    case wopAddTextToStore: {
-        string suffix = readString(from);
-        string s = readString(from);
-        PathSet refs = readStorePaths<PathSet>(*store, from);
-        logger->startWork();
-        Path path = store->addTextToStore(suffix, s, refs, NoRepair);
-        logger->stopWork();
-        to << path;
-        break;
-    }
-
-    case wopExportPath: {
-        Path path = readStorePath(*store, from);
-        readInt(from); // obsolete
-        logger->startWork();
-        TunnelSink sink(to);
-        store->exportPath(path, sink);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopImportPaths: {
-        logger->startWork();
-        TunnelSource source(from);
-        Paths paths = store->importPaths(source, nullptr,
-            trusted ? NoCheckSigs : CheckSigs);
-        logger->stopWork();
-        to << paths;
-        break;
-    }
-
-    case wopBuildPaths: {
-        PathSet drvs = readStorePaths<PathSet>(*store, from);
-        BuildMode mode = bmNormal;
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
-            mode = (BuildMode) readInt(from);
-
-            /* Repairing is not atomic, so disallowed for "untrusted"
-               clients.  */
-            if (mode == bmRepair && !trusted)
-                throw Error("repairing is not allowed because you are not in 'trusted-users'");
-        }
-        logger->startWork();
-        store->buildPaths(drvs, mode);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopBuildDerivation: {
-        Path drvPath = readStorePath(*store, from);
-        BasicDerivation drv;
-        readDerivation(from, *store, drv);
-        BuildMode buildMode = (BuildMode) readInt(from);
-        logger->startWork();
-        if (!trusted)
-            throw Error("you are not privileged to build derivations");
-        auto res = store->buildDerivation(drvPath, drv, buildMode);
-        logger->stopWork();
-        to << res.status << res.errorMsg;
-        break;
-    }
-
-    case wopEnsurePath: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        store->ensurePath(path);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopAddTempRoot: {
-        Path path = readStorePath(*store, from);
-        logger->startWork();
-        store->addTempRoot(path);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopAddIndirectRoot: {
-        Path path = absPath(readString(from));
-        logger->startWork();
-        store->addIndirectRoot(path);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopSyncWithGC: {
-        logger->startWork();
-        store->syncWithGC();
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopFindRoots: {
-        logger->startWork();
-        Roots roots = store->findRoots(!trusted);
-        logger->stopWork();
-
-        size_t size = 0;
-        for (auto & i : roots)
-            size += i.second.size();
-
-        to << size;
-
-        for (auto & [target, links] : roots)
-            for (auto & link : links)
-                to << link << target;
-
-        break;
-    }
-
-    case wopCollectGarbage: {
-        GCOptions options;
-        options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = readStorePaths<PathSet>(*store, from);
-        from >> options.ignoreLiveness >> options.maxFreed;
-        // obsolete fields
-        readInt(from);
-        readInt(from);
-        readInt(from);
-
-        GCResults results;
-
-        logger->startWork();
-        if (options.ignoreLiveness)
-            throw Error("you are not allowed to ignore liveness");
-        store->collectGarbage(options, results);
-        logger->stopWork();
-
-        to << results.paths << results.bytesFreed << 0 /* obsolete */;
-
-        break;
-    }
-
-    case wopSetOptions: {
-        settings.keepFailed = readInt(from);
-        settings.keepGoing = readInt(from);
-        settings.tryFallback = readInt(from);
-        verbosity = (Verbosity) readInt(from);
-        settings.maxBuildJobs.assign(readInt(from));
-        settings.maxSilentTime = readInt(from);
-        readInt(from); // obsolete useBuildHook
-        settings.verboseBuild = lvlError == (Verbosity) readInt(from);
-        readInt(from); // obsolete logType
-        readInt(from); // obsolete printBuildTrace
-        settings.buildCores = readInt(from);
-        settings.useSubstitutes  = readInt(from);
-
-        StringMap overrides;
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
-            unsigned int n = readInt(from);
-            for (unsigned int i = 0; i < n; i++) {
-                string name = readString(from);
-                string value = readString(from);
-                overrides.emplace(name, value);
-            }
-        }
-
-        logger->startWork();
-
-        for (auto & i : overrides) {
-            auto & name(i.first);
-            auto & value(i.second);
-
-            auto setSubstituters = [&](Setting<Strings> & res) {
-                if (name != res.name && res.aliases.count(name) == 0)
-                    return false;
-                StringSet trusted = settings.trustedSubstituters;
-                for (auto & s : settings.substituters.get())
-                    trusted.insert(s);
-                Strings subs;
-                auto ss = tokenizeString<Strings>(value);
-                for (auto & s : ss)
-                    if (trusted.count(s))
-                        subs.push_back(s);
-                    else
-                        warn("ignoring untrusted substituter '%s'", s);
-                res = subs;
-                return true;
-            };
-
-            try {
-                if (name == "ssh-auth-sock") // obsolete
-                    ;
-                else if (trusted
-                    || name == settings.buildTimeout.name
-                    || name == "connect-timeout"
-                    || (name == "builders" && value == ""))
-                    settings.set(name, value);
-                else if (setSubstituters(settings.substituters))
-                    ;
-                else if (setSubstituters(settings.extraSubstituters))
-                    ;
-                else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
-            } catch (UsageError & e) {
-                warn(e.what());
-            }
-        }
-
-        logger->stopWork();
-        break;
-    }
-
-    case wopQuerySubstitutablePathInfo: {
-        Path path = absPath(readString(from));
-        logger->startWork();
-        SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos({path}, infos);
-        logger->stopWork();
-        SubstitutablePathInfos::iterator i = infos.find(path);
-        if (i == infos.end())
-            to << 0;
-        else {
-            to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize;
-        }
-        break;
-    }
-
-    case wopQuerySubstitutablePathInfos: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
-        logger->startWork();
-        SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos(paths, infos);
-        logger->stopWork();
-        to << infos.size();
-        for (auto & i : infos) {
-            to << i.first << i.second.deriver << i.second.references
-               << i.second.downloadSize << i.second.narSize;
-        }
-        break;
-    }
-
-    case wopQueryAllValidPaths: {
-        logger->startWork();
-        PathSet paths = store->queryAllValidPaths();
-        logger->stopWork();
-        to << paths;
-        break;
-    }
-
-    case wopQueryPathInfo: {
-        Path path = readStorePath(*store, from);
-        std::shared_ptr<const ValidPathInfo> info;
-        logger->startWork();
-        try {
-            info = store->queryPathInfo(path);
-        } catch (InvalidPath &) {
-            if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw;
-        }
-        logger->stopWork();
-        if (info) {
-            if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
-                to << 1;
-            to << info->deriver << info->narHash.to_string(Base16, false) << info->references
-               << info->registrationTime << info->narSize;
-            if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
-                to << info->ultimate
-                   << info->sigs
-                   << info->ca;
-            }
-        } else {
-            assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
-            to << 0;
-        }
-        break;
-    }
-
-    case wopOptimiseStore:
-        logger->startWork();
-        store->optimiseStore();
-        logger->stopWork();
-        to << 1;
-        break;
-
-    case wopVerifyStore: {
-        bool checkContents, repair;
-        from >> checkContents >> repair;
-        logger->startWork();
-        if (repair && !trusted)
-            throw Error("you are not privileged to repair paths");
-        bool errors = store->verifyStore(checkContents, (RepairFlag) repair);
-        logger->stopWork();
-        to << errors;
-        break;
-    }
-
-    case wopAddSignatures: {
-        Path path = readStorePath(*store, from);
-        StringSet sigs = readStrings<StringSet>(from);
-        logger->startWork();
-        if (!trusted)
-            throw Error("you are not privileged to add signatures");
-        store->addSignatures(path, sigs);
-        logger->stopWork();
-        to << 1;
-        break;
-    }
-
-    case wopNarFromPath: {
-        auto path = readStorePath(*store, from);
-        logger->startWork();
-        logger->stopWork();
-        dumpPath(path, to);
-        break;
-    }
-
-    case wopAddToStoreNar: {
-        bool repair, dontCheckSigs;
-        ValidPathInfo info;
-        info.path = readStorePath(*store, from);
-        from >> info.deriver;
-        if (!info.deriver.empty())
-            store->assertStorePath(info.deriver);
-        info.narHash = Hash(readString(from), htSHA256);
-        info.references = readStorePaths<PathSet>(*store, from);
-        from >> info.registrationTime >> info.narSize >> info.ultimate;
-        info.sigs = readStrings<StringSet>(from);
-        from >> info.ca >> repair >> dontCheckSigs;
-        if (!trusted && dontCheckSigs)
-            dontCheckSigs = false;
-        if (!trusted)
-            info.ultimate = false;
-
-        std::string saved;
-        std::unique_ptr<Source> source;
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
-            source = std::make_unique<TunnelSource>(from);
-        else {
-            TeeSink tee(from);
-            parseDump(tee, tee.source);
-            saved = std::move(*tee.source.data);
-            source = std::make_unique<StringSource>(saved);
-        }
-
-        logger->startWork();
-
-        // FIXME: race if addToStore doesn't read source?
-        store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
-
-        logger->stopWork();
-        break;
-    }
-
-    case wopQueryMissing: {
-        PathSet targets = readStorePaths<PathSet>(*store, from);
-        logger->startWork();
-        PathSet willBuild, willSubstitute, unknown;
-        unsigned long long downloadSize, narSize;
-        store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
-        logger->stopWork();
-        to << willBuild << willSubstitute << unknown << downloadSize << narSize;
-        break;
-    }
-
-    default:
-        throw Error(format("invalid operation %1%") % op);
-    }
-}
-
-
-static void processConnection(bool trusted,
-    const std::string & userName, uid_t userId)
-{
-    MonitorFdHup monitor(from.fd);
-
-    /* Exchange the greeting. */
-    unsigned int magic = readInt(from);
-    if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
-    to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
-    to.flush();
-    unsigned int clientVersion = readInt(from);
-
-    if (clientVersion < 0x10a)
-        throw Error("the Nix client version is too old");
-
-    auto tunnelLogger = new TunnelLogger(clientVersion);
-    auto prevLogger = nix::logger;
-    logger = tunnelLogger;
-
-    unsigned int opCount = 0;
-
-    Finally finally([&]() {
-        _isInterrupted = false;
-        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
-    });
-
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
-        setAffinityTo(readInt(from));
-
-    readInt(from); // obsolete reserveSpace
-
-    /* Send startup error messages to the client. */
-    tunnelLogger->startWork();
-
-    try {
-
-        /* If we can't accept clientVersion, then throw an error
-           *here* (not above). */
-
-#if 0
-        /* Prevent users from doing something very dangerous. */
-        if (geteuid() == 0 &&
-            querySetting("build-users-group", "") == "")
-            throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-
-        /* Open the store. */
-        Store::Params params; // FIXME: get params from somewhere
-        // Disable caching since the client already does that.
-        params["path-info-cache-size"] = "0";
-        auto store = openStore(settings.storeUri, params);
-
-        store->createUser(userName, userId);
-
-        tunnelLogger->stopWork();
-        to.flush();
-
-        /* Process client requests. */
-        while (true) {
-            WorkerOp op;
-            try {
-                op = (WorkerOp) readInt(from);
-            } catch (Interrupted & e) {
-                break;
-            } catch (EndOfFile & e) {
-                break;
-            }
-
-            opCount++;
-
-            try {
-                performOp(tunnelLogger, store, trusted, clientVersion, from, to, op);
-            } catch (Error & e) {
-                /* If we're not in a state where we can send replies, then
-                   something went wrong processing the input of the
-                   client.  This can happen especially if I/O errors occur
-                   during addTextToStore() / importPath().  If that
-                   happens, just send the error message and exit. */
-                bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
-                tunnelLogger->stopWork(false, e.msg(), e.status);
-                if (!errorAllowed) throw;
-            } catch (std::bad_alloc & e) {
-                tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
-                throw;
-            }
-
-            to.flush();
-
-            assert(!tunnelLogger->state_.lock()->canSendStderr);
-        };
-
-    } catch (std::exception & e) {
-        tunnelLogger->stopWork(false, e.what(), 1);
-        to.flush();
-        return;
-    }
-}
-
 
 static void sigChldHandler(int sigNo)
 {
@@ -1054,9 +266,9 @@ static void daemonLoop(char * * argv)
                 }
 
                 /* Handle the connection. */
-                from.fd = remote.get();
-                to.fd = remote.get();
-                processConnection(trusted, user, peer.uid);
+                FdSource from(remote.get());
+                FdSink to(remote.get());
+                processConnection(from, to, trusted, user, peer.uid);
 
                 exit(0);
             }, options);
@@ -1136,7 +348,9 @@ static int _main(int argc, char * * argv)
                     }
                 }
             } else {
-                processConnection(true, "root", 0);
+                FdSource from(STDIN_FILENO);
+                FdSink to(STDOUT_FILENO);
+                processConnection(from, to, true, "root", 0);
             }
         } else {
             daemonLoop(argv);

From 63b99af85a1e280876da3d807e4bc96a5c5dde39 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Sep 2018 12:36:11 +0200
Subject: [PATCH 1620/2196] Move Unix domain socket creation to libutil

Also drop multithread-unfriendly hacks like doing a temporary
chmod/umask.
---
 src/libutil/util.cc          | 31 +++++++++++++++++++++++++
 src/libutil/util.hh          |  4 ++++
 src/nix-daemon/nix-daemon.cc | 44 +++---------------------------------
 3 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 2e416edeffe..2a127288554 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -21,7 +21,9 @@
 #include <pwd.h>
 #include <sys/ioctl.h>
 #include <sys/types.h>
+#include <sys/socket.h>
 #include <sys/wait.h>
+#include <sys/un.h>
 #include <unistd.h>
 
 #ifdef __APPLE__
@@ -1562,4 +1564,33 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
     return std::unique_ptr<InterruptCallback>(res.release());
 }
 
+
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
+{
+    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
+    if (!fdSocket)
+        throw SysError("cannot create Unix domain socket");
+
+    closeOnExec(fdSocket.get());
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+    if (path.size() >= sizeof(addr.sun_path))
+        throw Error("socket path '%1%' is too long", path);
+    strcpy(addr.sun_path, path.c_str());
+
+    unlink(path.c_str());
+
+    if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
+        throw SysError("cannot bind to socket '%1%'", path);
+
+    if (chmod(path.c_str(), mode) == -1)
+        throw SysError("changing permissions on '%1%'", path);
+
+    if (listen(fdSocket.get(), 5) == -1)
+        throw SysError("cannot listen on socket '%1%'", path);
+
+    return fdSocket;
+}
+
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3493e80b5eb..0bb17126839 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -553,4 +553,8 @@ typedef std::function<bool(const Path & path)> PathFilter;
 extern PathFilter defaultPathFilter;
 
 
+/* Create a Unix domain socket in listen mode. */
+AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
+
+
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 799d2bacfaa..67bf4dfeb3e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -156,53 +156,15 @@ static void daemonLoop(char * * argv)
         if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1")
             throw Error("unexpected systemd environment variables");
         fdSocket = SD_LISTEN_FDS_START;
+        closeOnExec(fdSocket.get());
     }
 
     /* Otherwise, create and bind to a Unix domain socket. */
     else {
-
-        /* Create and bind to a Unix domain socket. */
-        fdSocket = socket(PF_UNIX, SOCK_STREAM, 0);
-        if (!fdSocket)
-            throw SysError("cannot create Unix domain socket");
-
-        string socketPath = settings.nixDaemonSocketFile;
-
-        createDirs(dirOf(socketPath));
-
-        /* Urgh, sockaddr_un allows path names of only 108 characters.
-           So chdir to the socket directory so that we can pass a
-           relative path name. */
-        if (chdir(dirOf(socketPath).c_str()) == -1)
-            throw SysError("cannot change current directory");
-        Path socketPathRel = "./" + baseNameOf(socketPath);
-
-        struct sockaddr_un addr;
-        addr.sun_family = AF_UNIX;
-        if (socketPathRel.size() >= sizeof(addr.sun_path))
-            throw Error(format("socket path '%1%' is too long") % socketPathRel);
-        strcpy(addr.sun_path, socketPathRel.c_str());
-
-        unlink(socketPath.c_str());
-
-        /* Make sure that the socket is created with 0666 permission
-           (everybody can connect --- provided they have access to the
-           directory containing the socket). */
-        mode_t oldMode = umask(0111);
-        int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr));
-        umask(oldMode);
-        if (res == -1)
-            throw SysError(format("cannot bind to socket '%1%'") % socketPath);
-
-        if (chdir("/") == -1) /* back to the root */
-            throw SysError("cannot change current directory");
-
-        if (listen(fdSocket.get(), 5) == -1)
-            throw SysError(format("cannot listen on socket '%1%'") % socketPath);
+        createDirs(dirOf(settings.nixDaemonSocketFile));
+        fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
     }
 
-    closeOnExec(fdSocket.get());
-
     /* Loop accepting connections. */
     while (1) {
 

From 05819d013fb39eb9902d952978fff7f69e0e326e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Sep 2018 12:49:20 +0200
Subject: [PATCH 1621/2196] Don't create a Store in processConnection()

---
 src/libstore/daemon.cc       |  7 +------
 src/libstore/daemon.hh       |  2 ++
 src/nix-daemon/nix-daemon.cc | 13 +++++++++++--
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 2e1d10bcca7..cc1436f90ca 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -694,6 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 }
 
 void processConnection(
+    ref<Store> store,
     FdSource & from,
     FdSink & to,
     bool trusted,
@@ -743,12 +744,6 @@ void processConnection(
             throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
 #endif
 
-        /* Open the store. */
-        Store::Params params; // FIXME: get params from somewhere
-        // Disable caching since the client already does that.
-        params["path-info-cache-size"] = "0";
-        auto store = openStore(settings.storeUri, params);
-
         store->createUser(userName, userId);
 
         tunnelLogger->stopWork();
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index 6d4015e34d1..fa26ba9c222 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -1,8 +1,10 @@
 #include "serialise.hh"
+#include "store-api.hh"
 
 namespace nix::daemon {
 
 void processConnection(
+    ref<Store> store,
     FdSource & from,
     FdSink & to,
     bool trusted,
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 67bf4dfeb3e..da17ddcba07 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -140,6 +140,15 @@ static PeerInfo getPeerInfo(int remote)
 #define SD_LISTEN_FDS_START 3
 
 
+static ref<Store> openUncachedStore()
+{
+    Store::Params params; // FIXME: get params from somewhere
+    // Disable caching since the client already does that.
+    params["path-info-cache-size"] = "0";
+    return openStore(settings.storeUri, params);
+}
+
+
 static void daemonLoop(char * * argv)
 {
     if (chdir("/") == -1)
@@ -230,7 +239,7 @@ static void daemonLoop(char * * argv)
                 /* Handle the connection. */
                 FdSource from(remote.get());
                 FdSink to(remote.get());
-                processConnection(from, to, trusted, user, peer.uid);
+                processConnection(openUncachedStore(), from, to, trusted, user, peer.uid);
 
                 exit(0);
             }, options);
@@ -312,7 +321,7 @@ static int _main(int argc, char * * argv)
             } else {
                 FdSource from(STDIN_FILENO);
                 FdSink to(STDOUT_FILENO);
-                processConnection(from, to, true, "root", 0);
+                processConnection(openUncachedStore(), from, to, true, "root", 0);
             }
         } else {
             daemonLoop(argv);

From 992a2ad4757f43c02f3d06771c632a408d90ded0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Sep 2018 18:55:37 +0200
Subject: [PATCH 1622/2196] Move addToStoreFromDump to Store

---
 src/libstore/daemon.cc      | 5 +----
 src/libstore/local-store.hh | 2 +-
 src/libstore/store-api.hh   | 7 +++++++
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index cc1436f90ca..64147027b26 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -316,10 +316,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        auto store2 = store.dynamic_pointer_cast<LocalStore>();
-        if (!store2) throw Error("operation is only supported by LocalStore");
-
-        Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         logger->stopWork();
 
         to << path;
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 379a06af87d..77253fa4a4a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -157,7 +157,7 @@ public:
        true) or simply the contents of a regular file (if recursive ==
        false). */
     Path addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair);
+        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
     Path addTextToStore(const string & name, const string & s,
         const PathSet & references, RepairFlag repair) override;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 250e1a4bcc1..115b23cbf74 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -422,6 +422,13 @@ public:
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
+    // FIXME: remove?
+    virtual Path addToStoreFromDump(const string & dump, const string & name,
+        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
+    {
+        throw Error("addToStoreFromDump() is not supported by this store");
+    }
+
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */
     virtual Path addTextToStore(const string & name, const string & s,

From e5319a87ce75bbd2dd88f57c3b470a396195e849 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 25 Sep 2018 18:54:16 +0200
Subject: [PATCH 1623/2196] queryPathInfoUncached(): Return const ValidPathInfo

---
 src/libstore/binary-cache-store.cc  |  2 +-
 src/libstore/binary-cache-store.hh  |  2 +-
 src/libstore/legacy-ssh-store.cc    |  2 +-
 src/libstore/local-store.cc         |  2 +-
 src/libstore/local-store.hh         |  2 +-
 src/libstore/misc.cc                |  2 +-
 src/libstore/nar-info-disk-cache.cc |  4 ++--
 src/libstore/nar-info-disk-cache.hh |  2 +-
 src/libstore/remote-store.cc        |  2 +-
 src/libstore/remote-store.hh        |  2 +-
 src/libstore/store-api.cc           | 16 ++++++++--------
 src/libstore/store-api.hh           |  6 +++---
 12 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8e6f1f55d71..7e3e5ff8847 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -296,7 +296,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     auto uri = getUri();
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index c772922949a..2d7cd1947bf 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -74,7 +74,7 @@ public:
     bool isValidPathUncached(const Path & path) override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
     Path queryPathFromHashPart(const string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index d5fbdd25aa4..1b8b5908cfa 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -88,7 +88,7 @@ struct LegacySSHStore : public Store
     }
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
     {
         try {
             auto conn(connections->get());
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 49061f27d13..6bbe5433ce5 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -632,7 +632,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
 
 void LocalStore::queryPathInfoUncached(const Path & path,
-    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
         auto info = std::make_shared<ValidPathInfo>();
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 77253fa4a4a..5aa6b051964 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -127,7 +127,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 05b93d4c9b0..2fceb9b9ac3 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -33,7 +33,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
             state->pending++;
         }
 
-        queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) {
+        queryPathInfo(path, {[&, path](std::future<ref<const ValidPathInfo>> fut) {
             // FIXME: calls to isValidPath() should be async
 
             try {
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 32ad7f2b27f..5bf98219585 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -219,7 +219,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
     void upsertNarInfo(
         const std::string & uri, const std::string & hashPart,
-        std::shared_ptr<ValidPathInfo> info) override
+        std::shared_ptr<const ValidPathInfo> info) override
     {
         retrySQLite<void>([&]() {
             auto state(_state.lock());
@@ -228,7 +228,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
             if (info) {
 
-                auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
+                auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
                 assert(hashPart == storePathToHash(info->path));
 
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index 88d909732db..11e6c55ca9c 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -21,7 +21,7 @@ public:
 
     virtual void upsertNarInfo(
         const std::string & uri, const std::string & hashPart,
-        std::shared_ptr<ValidPathInfo> info) = 0;
+        std::shared_ptr<const ValidPathInfo> info) = 0;
 };
 
 /* Return a singleton cache object that can be used concurrently by
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f34369d8fe0..2a89b7c98dc 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -350,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
 
 void RemoteStore::queryPathInfoUncached(const Path & path,
-    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
         std::shared_ptr<ValidPathInfo> info;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 1f375dd7152..84de4c4601c 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -43,7 +43,7 @@ public:
     PathSet queryAllValidPaths() override;
 
     void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
     void queryReferrers(const Path & path, PathSet & referrers) override;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1a1b24e3b0c..54430d3baac 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -320,10 +320,10 @@ bool Store::isValidPathUncached(const Path & path)
 
 ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 {
-    std::promise<ref<ValidPathInfo>> promise;
+    std::promise<ref<const ValidPathInfo>> promise;
 
     queryPathInfo(storePath,
-        {[&](std::future<ref<ValidPathInfo>> result) {
+        {[&](std::future<ref<const ValidPathInfo>> result) {
             try {
                 promise.set_value(result.get());
             } catch (...) {
@@ -336,7 +336,7 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 
 
 void Store::queryPathInfo(const Path & storePath,
-    Callback<ref<ValidPathInfo>> callback) noexcept
+    Callback<ref<const ValidPathInfo>> callback) noexcept
 {
     std::string hashPart;
 
@@ -351,7 +351,7 @@ void Store::queryPathInfo(const Path & storePath,
                 stats.narInfoReadAverted++;
                 if (!*res)
                     throw InvalidPath(format("path '%s' is not valid") % storePath);
-                return callback(ref<ValidPathInfo>(*res));
+                return callback(ref<const ValidPathInfo>(*res));
             }
         }
 
@@ -367,7 +367,7 @@ void Store::queryPathInfo(const Path & storePath,
                         (res.second->path != storePath && storePathToName(storePath) != ""))
                         throw InvalidPath(format("path '%s' is not valid") % storePath);
                 }
-                return callback(ref<ValidPathInfo>(res.second));
+                return callback(ref<const ValidPathInfo>(res.second));
             }
         }
 
@@ -376,7 +376,7 @@ void Store::queryPathInfo(const Path & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     queryPathInfoUncached(storePath,
-        {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) {
+        {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -396,7 +396,7 @@ void Store::queryPathInfo(const Path & storePath,
                     throw InvalidPath("path '%s' is not valid", storePath);
                 }
 
-                (*callbackPtr)(ref<ValidPathInfo>(info));
+                (*callbackPtr)(ref<const ValidPathInfo>(info));
             } catch (...) { callbackPtr->rethrow(); }
         }});
 }
@@ -418,7 +418,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
 
     auto doQuery = [&](const Path & path ) {
         checkInterrupt();
-        queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) {
+        queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
             auto state(state_.lock());
             try {
                 auto info = fut.get();
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 115b23cbf74..e9e6e0dd2a8 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -258,7 +258,7 @@ protected:
 
     struct State
     {
-        LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache;
+        LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
     };
 
     Sync<State> state;
@@ -361,12 +361,12 @@ public:
 
     /* Asynchronous version of queryPathInfo(). */
     void queryPathInfo(const Path & path,
-        Callback<ref<ValidPathInfo>> callback) noexcept;
+        Callback<ref<const ValidPathInfo>> callback) noexcept;
 
 protected:
 
     virtual void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0;
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
 
 public:
 

From 0e9b72e097e027ff7f78e0f596a3ade606df9253 Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 30 Oct 2019 14:38:02 -0500
Subject: [PATCH 1624/2196] minor: fix mismatch of struct/class forward decl of
 'Source'

Fixes the following warning and the indicate potential issue:

src/libstore/worker-protocol.hh:66:1: warning: class 'Source' was previously declared as a struct; this is valid, but may result in linker errors
under the Microsoft C++ ABI [-Wmismatched-tags]

(cherry picked from commit 6e1bb04870b1b723282d32182af286646f13bf3c)
---
 src/libstore/worker-protocol.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index a03edd10f6f..6762b609ddb 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -63,7 +63,7 @@ typedef enum {
 
 
 class Store;
-class Source;
+struct Source;
 
 Path readStorePath(Store & store, Source & from);
 template<class T> T readStorePaths(Store & store, Source & from);

From 99aac72a16544637e9ab2e14c7b28f64b4638c8e Mon Sep 17 00:00:00 2001
From: Kevin Stock <kevin@kevinstock.org>
Date: Wed, 30 Oct 2019 16:53:04 -0400
Subject: [PATCH 1625/2196] docs: fix upper bound on number of consumed cores

---
 doc/manual/advanced-topics/cores-vs-jobs.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml
index eba645faf87..4d58ac7c5e4 100644
--- a/doc/manual/advanced-topics/cores-vs-jobs.xml
+++ b/doc/manual/advanced-topics/cores-vs-jobs.xml
@@ -36,8 +36,8 @@ to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
 equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
 will be the total number of cores in the system.</para>
 
-<para>The total number of consumed cores is a simple multiplication,
-<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+<para>The maximum number of consumed cores is a simple multiplication,
+<xref linkend="conf-max-jobs" /> * <envar>NIX_BUILD_CORES</envar>.</para>
 
 <para>The balance on how to set these two independent variables depends
 upon each builder's workload and hardware. Here are a few example

From 80d5ec6ff4388b258278ee507b94694dc34d19f5 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sun, 27 Oct 2019 13:45:02 -0400
Subject: [PATCH 1626/2196] Minor updates to inline comments

Add missing docstring on InstallableCommand. Also, some of these were wrapped
when they're right next to a line longer than the unwrapped line, so we can just
unwrap them to save vertical space.
---
 src/libexpr/primops/fetchGit.cc |  4 +---
 src/libstore/build.cc           | 12 ++++--------
 src/libstore/local-store.cc     |  3 +--
 src/nix-store/nix-store.cc      |  6 ++----
 src/nix/command.hh              |  3 ++-
 5 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 90f600284b2..7ef3b382398 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -45,9 +45,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         if (!clean) {
 
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
+            /* This is an unclean working tree. So copy all tracked files. */
             GitInfo gitInfo;
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dfae8024aa7..ada1cf8ec4d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3264,8 +3264,7 @@ void DerivationGoal::registerOutputs()
             i.second.parseHashInfo(recursive, h);
 
             if (!recursive) {
-                /* The output path should be a regular file without
-                   execute permission. */
+                /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         format("output path '%1%' should be a non-executable regular file") % path);
@@ -3343,8 +3342,7 @@ void DerivationGoal::registerOutputs()
                         % drvPath % path);
             }
 
-            /* Since we verified the build, it's now ultimately
-               trusted. */
+            /* Since we verified the build, it's now ultimately trusted. */
             if (!info.ultimate) {
                 info.ultimate = true;
                 worker.store.signPathInfo(info);
@@ -3354,8 +3352,7 @@ void DerivationGoal::registerOutputs()
             continue;
         }
 
-        /* For debugging, print out the referenced and unreferenced
-           paths. */
+        /* For debugging, print out the referenced and unreferenced paths. */
         for (auto & i : inputPaths) {
             PathSet::iterator j = references.find(i);
             if (j == references.end())
@@ -3413,8 +3410,7 @@ void DerivationGoal::registerOutputs()
             }
     }
 
-    /* If this is the first round of several, then move the output out
-       of the way. */
+    /* If this is the first round of several, then move the output out of the way. */
     if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
         for (auto & i : drv->outputs) {
             Path prev = i.second.path + checkSuffix;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 6bbe5433ce5..b5b0f1ba745 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -937,8 +937,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
            not be valid yet. */
         for (auto & i : infos)
             if (isDerivation(i.path)) {
-                // FIXME: inefficient; we already loaded the
-                // derivation in addValidPath().
+                // FIXME: inefficient; we already loaded the derivation in addValidPath().
                 Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path));
                 checkDerivationOutputs(i.path, drv);
             }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 19384499d3b..dfa473db5e3 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -632,8 +632,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
 }
 
 
-/* Dump a path as a Nix archive.  The archive is written to standard
-   output. */
+/* Dump a path as a Nix archive.  The archive is written to stdout */
 static void opDump(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -646,8 +645,7 @@ static void opDump(Strings opFlags, Strings opArgs)
 }
 
 
-/* Restore a value from a Nix archive.  The archive is read from
-   standard input. */
+/* Restore a value from a Nix archive.  The archive is read from stdin. */
 static void opRestore(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 45ad1cd2a28..fad404c7319 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -34,7 +34,7 @@ struct Command : virtual Args
 
 class Store;
 
-/* A command that require a Nix store. */
+/* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
     StoreCommand();
@@ -115,6 +115,7 @@ private:
     std::vector<std::string> _installables;
 };
 
+/* A command that operates on exactly one "installable" */
 struct InstallableCommand : virtual Args, SourceExprCommand
 {
     std::shared_ptr<Installable> installable;

From f1782642d363330b9ce8607d942dcbb47414bf82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 31 Oct 2019 16:34:59 +0000
Subject: [PATCH 1627/2196] travis: enable linux builds

Also disable email to not notify the whole NixOS community about build failures
---
 .travis.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 99218a963cb..ee4ea1ac65a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,2 +1,8 @@
-os: osx
-script: ./tests/install-darwin.sh
+matrix:
+  include:
+    - language: osx
+      script: ./tests/install-darwin.sh
+    - language: nix
+      script: nix-build release.nix -A build.x86_64-linux
+notifications:
+  email: false

From b811bd2172bc9796f42b87996f455d4e5e4382ba Mon Sep 17 00:00:00 2001
From: ng0 <ng0@n0.is>
Date: Fri, 1 Nov 2019 14:09:42 +0000
Subject: [PATCH 1628/2196] include netinet/in.h in src/nix/main.cc

Fixes #3186
---
 src/nix/main.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 22f5e8c4c8b..1c9d909d86b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -15,6 +15,7 @@
 #include <sys/socket.h>
 #include <ifaddrs.h>
 #include <netdb.h>
+#include <netinet/in.h>
 
 extern std::string chrootHelperName;
 

From f1d4ba2afd1169b6c15248e50e30b592e514bb21 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 1 Nov 2019 13:25:15 -0400
Subject: [PATCH 1629/2196] Update man to show that nix-shell allows --arg

---
 doc/manual/command-ref/opt-common.xml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index b8a2f260e8f..28ec42bbcd3 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -243,9 +243,10 @@
 <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
 
   <listitem><para>This option is accepted by
-  <command>nix-env</command>, <command>nix-instantiate</command> and
-  <command>nix-build</command>.  When evaluating Nix expressions, the
-  expression evaluator will automatically try to call functions that
+  <command>nix-env</command>, <command>nix-instantiate</command>,
+  <command>nix-shell</command and <command>nix-build</command>.
+  When evaluating Nix expressions, the expression evaluator will
+  automatically try to call functions that
   it encounters.  It can automatically call functions for which every
   argument has a <link linkend='ss-functions'>default value</link>
   (e.g., <literal>{ <replaceable>argName</replaceable> ?

From 3e85c57a6cbf46d5f0fe8a89b368a43abd26daba Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 1 Nov 2019 13:27:40 -0400
Subject: [PATCH 1630/2196] Pass --static flag to pkg-config when necessary

---
 configure.ac | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/configure.ac b/configure.ac
index ebe6d4267f4..20969c14f87 100644
--- a/configure.ac
+++ b/configure.ac
@@ -170,6 +170,18 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LIBS="-latomic $LIBS"
 fi
 
+PKG_PROG_PKG_CONFIG
+
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+  PKG_CONFIG="$PKG_CONFIG --static"
+fi
+
 # Look for OpenSSL, a required dependency.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
@@ -302,16 +314,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
-AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
-  [Build shared libraries for Nix [default=yes]]),
-  shared=$enableval, shared=yes)
-if test "$shared" = yes; then
-  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
-else
-  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
-fi
-
-
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'

From 808cb6444e33c413fad220d2cf9c565fa68614f0 Mon Sep 17 00:00:00 2001
From: Kevin Stock <kevin@kevinstock.org>
Date: Sat, 2 Nov 2019 17:55:53 -0400
Subject: [PATCH 1631/2196] docs: xref doesn't render in title

The `post-build-hook` text currently appears in the index, but not on the actual title line of the section, this follows the pattern used in a previous section to get a reference into a title.
---
 doc/manual/advanced-topics/post-build-hook.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
index 3dc43ee795b..08a7a772f43 100644
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -5,7 +5,7 @@
       version="5.0"
       >
 
-<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<title>Using the <option linkend="conf-post-build-hook">post-build-hook</option></title>
 <subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
 
 

From dcd7a260639c5130368e5969a34aa9bb2365df17 Mon Sep 17 00:00:00 2001
From: kraem <ebrin.ronnie@protonmail.com>
Date: Sun, 3 Nov 2019 18:47:28 +0100
Subject: [PATCH 1632/2196] Update nix eval --help msg to not include
 deprecated command

---
 src/nix/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index b7058361cbe..daefea7578b 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -41,7 +41,7 @@ struct CmdEval : MixJSON, InstallableCommand
             },
             Example{
                 "To get the current version of Nixpkgs:",
-                "nix eval --raw nixpkgs.lib.nixpkgsVersion"
+                "nix eval --raw nixpkgs.lib.version"
             },
             Example{
                 "To print the store path of the Hello package:",

From c935ad3f025d5c3d8026711a1eb50b2917b61d59 Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald@gigawatt.nl>
Date: Sun, 3 Nov 2019 21:46:59 +0000
Subject: [PATCH 1633/2196] Fix progress bar when nix-prefetch-url is piped.

The intent of the code was that if the window size cannot be determined,
it would be treated as having the maximum possible size. Because of a
missing assignment, it was actually treated as having a width of 0.

The reason the width could not be determined was because it was obtained
from stdout, not stderr, even though the printing was done to stderr.

This commit addresses both issues.
---
 src/libutil/util.cc     | 2 +-
 src/nix/progress-bar.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 2a127288554..d29802bcdcb 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1467,7 +1467,7 @@ static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 static void updateWindowSize()
 {
     struct winsize ws;
-    if (ioctl(1, TIOCGWINSZ, &ws) == 0) {
+    if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
         auto windowSize_(windowSize.lock());
         windowSize_->first = ws.ws_row;
         windowSize_->second = ws.ws_col;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 5c05d6b227e..661966733d3 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -341,7 +341,7 @@ class ProgressBar : public Logger
         }
 
         auto width = getWindowSize().second;
-        if (width <= 0) std::numeric_limits<decltype(width)>::max();
+        if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
 
         writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
     }

From cea05e5ee758daad40047db0b861980d80da2e85 Mon Sep 17 00:00:00 2001
From: Kevin Stock <kevin@kevinstock.org>
Date: Mon, 4 Nov 2019 16:23:03 -0500
Subject: [PATCH 1634/2196] docs: correct default location of log directory

---
 doc/manual/command-ref/env-common.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 6a3aaae717e..696d68c3451 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -122,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><envar>NIX_LOG_DIR</envar></term>
 
   <listitem><para>Overrides the location of the Nix log directory
-  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem>
 
 </varlistentry>
 

From 1b600ecd14049985dc1001d22d6495810ca418a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Nov 2019 10:25:09 +0100
Subject: [PATCH 1635/2196] Don't use SOCK_CLOEXEC on macOS

https://hydra.nixos.org/build/105428308
---
 src/libutil/util.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index d29802bcdcb..34c1d96dcb8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1567,7 +1567,11 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
 
 AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
 {
-    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
+    AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
+        #ifdef SOCK_CLOEXEC
+        | SOCK_CLOEXEC
+        #endif
+        , 0);
     if (!fdSocket)
         throw SysError("cannot create Unix domain socket");
 

From e1725ba946aa89271bd8ef83f9ecc2e2dc6eb5df Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Nov 2019 11:12:25 +0100
Subject: [PATCH 1636/2196] Fix VM tests

---
 tests/nix-copy-closure.nix | 4 ++--
 tests/remote-builds.nix    | 4 ++--
 tests/setuid.nix           | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index 0dc147fb34e..bb5db74107f 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -8,11 +8,11 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
 
   nodes =
     { client =
-        { config, pkgs, ... }:
+        { config, lib, pkgs, ... }:
         { virtualisation.writableStore = true;
           virtualisation.pathsInNixDB = [ pkgA ];
           nix.package = nix;
-          nix.binaryCaches = [ ];
+          nix.binaryCaches = lib.mkForce [ ];
         };
 
       server =
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index b867f13b499..18d4908300e 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -40,7 +40,7 @@ in
       builder2 = builder;
 
       client =
-        { config, pkgs, ... }:
+        { config, lib, pkgs, ... }:
         { nix.maxJobs = 0; # force remote building
           nix.distributedBuilds = true;
           nix.buildMachines =
@@ -60,7 +60,7 @@ in
           virtualisation.writableStore = true;
           virtualisation.pathsInNixDB = [ config.system.build.extraUtils ];
           nix.package = nix;
-          nix.binaryCaches = [ ];
+          nix.binaryCaches = lib.mkForce [ ];
           programs.ssh.extraConfig = "ConnectTimeout 30";
         };
     };
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 77e83c8d6c2..63d3c05cb8e 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -10,7 +10,7 @@ makeTest {
     { config, lib, pkgs, ... }:
     { virtualisation.writableStore = true;
       nix.package = nix;
-      nix.binaryCaches = [ ];
+      nix.binaryCaches = lib.mkForce [ ];
       nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ];
       virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ];
     };

From 81a9b93689c76a9b90134bcc5ad8a6b851590599 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Nov 2019 11:21:32 +0100
Subject: [PATCH 1637/2196] Fix manual build

---
 doc/manual/command-ref/opt-common.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 28ec42bbcd3..6a8107be91c 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -244,7 +244,7 @@
 
   <listitem><para>This option is accepted by
   <command>nix-env</command>, <command>nix-instantiate</command>,
-  <command>nix-shell</command and <command>nix-build</command>.
+  <command>nix-shell</command> and <command>nix-build</command>.
   When evaluating Nix expressions, the expression evaluator will
   automatically try to call functions that
   it encounters.  It can automatically call functions for which every

From b4e260d887441fde9ab568dff7c21a77d7cff904 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Nov 2019 16:00:30 +0100
Subject: [PATCH 1638/2196] Disable shellcheck

It's broken at the moment: https://hydra.nixos.org/build/105746055

Also it pulls in GHC which is a pretty big dependency.
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 8fcf04680de..e512a7671d1 100644
--- a/release.nix
+++ b/release.nix
@@ -128,7 +128,7 @@ let
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''

From b874272f7a121a859e7c05abe4d7818bc852eba8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Oct 2019 14:43:09 +0100
Subject: [PATCH 1639/2196] Make --enable-gc the default

---
 configure.ac       | 4 ++--
 release-common.nix | 4 +---
 release.nix        | 2 --
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/configure.ac b/configure.ac
index ebe6d4267f4..4a6eaac8a39 100644
--- a/configure.ac
+++ b/configure.ac
@@ -255,8 +255,8 @@ fi
 
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
-  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]),
-  gc=$enableval, gc=no)
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
+  gc=$enableval, gc=yes)
 if test "$gc" = yes; then
   PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
   CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
diff --git a/release-common.nix b/release-common.nix
index 2e8a951b9cd..11944a63124 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -30,9 +30,7 @@ rec {
   });
 
   configureFlags =
-    [
-      "--enable-gc"
-    ] ++ lib.optionals stdenv.isLinux [
+    lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
diff --git a/release.nix b/release.nix
index e512a7671d1..0b74de39072 100644
--- a/release.nix
+++ b/release.nix
@@ -25,8 +25,6 @@ let
 
         buildInputs = tarballDeps ++ buildDeps;
 
-        configureFlags = "--enable-gc";
-
         postUnpack = ''
           (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
           cat $sourceRoot/.dist-files

From c4d7c76b641d82b2696fef73ce0ac160043c18da Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 Oct 2018 16:01:26 +0200
Subject: [PATCH 1640/2196] Recursive Nix support

This allows Nix builders to call Nix to build derivations, with some
limitations.

Example:

  let nixpkgs = fetchTarball channel:nixos-18.03; in

  with import <nixpkgs> {};

  runCommand "foo"
    {
      buildInputs = [ nix jq ];
      NIX_PATH = "nixpkgs=${nixpkgs}";
    }
    ''
      hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')

      $hello/bin/hello

      mkdir -p $out/bin
      ln -s $hello/bin/hello $out/bin/hello

      nix path-info -r --json $hello | jq .
    ''

This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.

  # ll ./result/bin/
  lrwxrwxrwx 1 root root 63 Jan  1  1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello

  # nix-store -qR ./result
  /nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
  /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
  /nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo

This is implemented as follows:

* Before running the outer builder, Nix creates a Unix domain socket
  '.nix-socket' in the builder's temporary directory and sets
  $NIX_REMOTE to point to it. It starts a thread to process
  connections to this socket. (Thus you don't need to have nix-daemon
  running.)

* The daemon thread uses a wrapper store (RestrictedStore) to keep
  track of paths added through recursive Nix calls, to implement some
  restrictions (see below), and to do some censorship (e.g. for
  purity, queryPathInfo() won't return impure information such as
  signatures and timestamps).

* After the build finishes, the output paths are scanned for
  references to the paths added through recursive Nix calls (in
  addition to the inputs closure). Thus, in the example above, $out
  has a reference to $hello.

The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing

  nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10

is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.

Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.

When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.

Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
---
 src/libstore/build.cc        | 403 +++++++++++++++++++++++++++++++++--
 src/libstore/daemon.cc       | 153 ++++++++-----
 src/libstore/daemon.hh       |   6 +-
 src/nix-daemon/nix-daemon.cc |   8 +-
 4 files changed, 492 insertions(+), 78 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ada1cf8ec4d..2fd27cfa05f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -13,6 +13,7 @@
 #include "nar-info.hh"
 #include "parsed-derivations.hh"
 #include "machines.hh"
+#include "daemon.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -33,6 +34,7 @@
 #include <sys/select.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
+#include <sys/un.h>
 #include <fcntl.h>
 #include <netdb.h>
 #include <unistd.h>
@@ -765,9 +767,6 @@ class DerivationGoal : public Goal
        immediate input paths). */
     PathSet inputPaths;
 
-    /* Referenceable paths (i.e., input and output paths). */
-    PathSet allPaths;
-
     /* Outputs that are already valid.  If we're repairing, these are
        the outputs that are valid *and* not corrupt. */
     PathSet validPaths;
@@ -805,9 +804,13 @@ class DerivationGoal : public Goal
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
 
-    /* Pipe for synchronising updates to the builder user namespace. */
+    /* Pipe for synchronising updates to the builder namespaces. */
     Pipe userNamespaceSync;
 
+    /* The mount namespace of the builder, used to add additional
+       paths to the sandbox as a result of recursive Nix calls. */
+    AutoCloseFD sandboxMountNamespace;
+
     /* The build hook. */
     std::unique_ptr<HookInstance> hook;
 
@@ -886,6 +889,26 @@ class DerivationGoal : public Goal
     /* The remote machine on which we're building. */
     std::string machineName;
 
+    /* The recursive Nix daemon socket. */
+    AutoCloseFD daemonSocket;
+
+    /* The daemon main thread. */
+    std::thread daemonThread;
+
+    /* Paths that were added via recursive Nix calls. */
+    PathSet addedPaths;
+
+    /* Recursive Nix calls are only allowed to build or realize paths
+       in the original input closure or added via a recursive Nix call
+       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+       /nix/store/<bla> is some arbitrary path in a binary cache). */
+    bool isAllowed(const Path & path)
+    {
+        return inputPaths.count(path) || addedPaths.count(path);
+    }
+
+    friend class RestrictedStore;
+
 public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
@@ -942,6 +965,14 @@ class DerivationGoal : public Goal
     /* Write a JSON file containing the derivation attributes. */
     void writeStructuredAttrs();
 
+    void startDaemon();
+
+    void stopDaemon();
+
+    /* Add 'path' to the set of paths that may be referenced by the
+       outputs, and make it appear in the sandbox. */
+    void addDependency(const Path & path);
+
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
 
@@ -1043,6 +1074,7 @@ DerivationGoal::~DerivationGoal()
     /* Careful: we should never ever throw an exception from a
        destructor. */
     try { killChild(); } catch (...) { ignoreException(); }
+    try { stopDaemon(); } catch (...) { ignoreException(); }
     try { deleteTmpDir(false); } catch (...) { ignoreException(); }
     try { closeLogFile(); } catch (...) { ignoreException(); }
 }
@@ -1332,12 +1364,6 @@ void DerivationGoal::inputsRealised()
     /* Gather information necessary for computing the closure and/or
        running the build hook. */
 
-    /* The outputs are referenceable paths. */
-    for (auto & i : drv->outputs) {
-        debug(format("building path '%1%'") % i.second.path);
-        allPaths.insert(i.second.path);
-    }
-
     /* Determine the full set of input paths. */
 
     /* First, the input derivations. */
@@ -1362,8 +1388,6 @@ void DerivationGoal::inputsRealised()
 
     debug(format("added input paths %1%") % showPaths(inputPaths));
 
-    allPaths.insert(inputPaths.begin(), inputPaths.end());
-
     /* Is this a fixed-output derivation? */
     fixedOutput = drv->isFixedOutput();
 
@@ -1527,6 +1551,8 @@ void DerivationGoal::buildDone()
        uid and then messing around with our output. */
     Finally releaseBuildUser([&]() { buildUser.reset(); });
 
+    sandboxMountNamespace = -1;
+
     /* Since we got an EOF on the logger pipe, the builder is presumed
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe, so just to be sure,
@@ -1558,6 +1584,9 @@ void DerivationGoal::buildDone()
        root. */
     if (buildUser) buildUser->kill();
 
+    /* Terminate the recursive Nix daemon. */
+    stopDaemon();
+
     bool diskFull = false;
 
     try {
@@ -2217,6 +2246,10 @@ void DerivationGoal::startBuilder()
         }
     }
 
+    /* Fire up a Nix daemon to process recursive Nix calls from the
+       builder. */
+    startDaemon();
+
     /* Run the builder. */
     printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder);
 
@@ -2391,6 +2424,12 @@ void DerivationGoal::startBuilder()
         writeFile("/proc/" + std::to_string(pid) + "/gid_map",
             (format("%d %d 1") % sandboxGid % hostGid).str());
 
+        /* Save the mount namespace of the child. We have to do this
+           *before* the child does a chroot. */
+        sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
+        if (sandboxMountNamespace.get() == -1)
+            throw SysError("getting sandbox mount namespace");
+
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
         userNamespaceSync.writeSide = -1;
@@ -2621,6 +2660,302 @@ void DerivationGoal::writeStructuredAttrs()
 }
 
 
+/* A wrapper around LocalStore that only allows building/querying of
+   paths that are in the input closures of the build or were added via
+   recursive Nix calls. */
+struct RestrictedStore : public LocalFSStore
+{
+    ref<LocalStore> next;
+
+    DerivationGoal & goal;
+
+    RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
+        : Store(params), LocalFSStore(params), next(next), goal(goal)
+    { }
+
+    Path getRealStoreDir() override
+    { return next->realStoreDir; }
+
+    std::string getUri() override
+    { return next->getUri(); }
+
+    PathSet queryAllValidPaths() override
+    {
+        PathSet paths;
+        for (auto & p : goal.inputPaths) paths.insert(p);
+        for (auto & p : goal.addedPaths) paths.insert(p);
+        return paths;
+    }
+
+    void queryPathInfoUncached(const Path & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+    {
+        if (goal.isAllowed(path)) {
+            try {
+                /* Censor impure information. */
+                auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
+                info->deriver.clear();
+                info->registrationTime = 0;
+                info->ultimate = false;
+                info->sigs.clear();
+                callback(info);
+            } catch (InvalidPath &) {
+                callback(nullptr);
+            }
+        } else
+            callback(nullptr);
+    };
+
+    void queryReferrers(const Path & path, PathSet & referrers) override
+    { }
+
+    PathSet queryDerivationOutputs(const Path & path) override
+    { throw Error("queryDerivationOutputs"); }
+
+    StringSet queryDerivationOutputNames(const Path & path) override
+    { throw Error("queryDerivationOutputNames"); }
+
+    Path queryPathFromHashPart(const string & hashPart) override
+    { throw Error("queryPathFromHashPart"); }
+
+    Path addToStore(const string & name, const Path & srcPath,
+        bool recursive = true, HashType hashAlgo = htSHA256,
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
+    { throw Error("addToStore");
+    }
+
+    Path addToStoreFromDump(const string & dump, const string & name,
+        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+    {
+        auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
+        goal.addDependency(path);
+        return path;
+    }
+
+    Path addTextToStore(const string & name, const string & s,
+        const PathSet & references, RepairFlag repair = NoRepair) override
+    {
+        auto path = next->addTextToStore(name, s, references, repair);
+        goal.addDependency(path);
+        return path;
+    }
+
+    void narFromPath(const Path & path, Sink & sink) override 
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path);
+        LocalFSStore::narFromPath(path, sink);
+    }
+
+    void ensurePath(const Path & path) override
+    {
+        if (!goal.isAllowed(path))
+            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path);
+        /* Nothing to be done; 'path' must already be valid. */
+    }
+
+    void buildPaths(const PathSet & paths, BuildMode buildMode) override
+    {
+        if (buildMode != bmNormal) throw Error("unsupported build mode");
+
+        PathSet newPaths;
+
+        for (auto & path : paths) {
+            DrvPathWithOutputs i = parseDrvPathWithOutputs(path);
+            if (isDerivation(i.first)) {
+                if (!goal.isAllowed(i.first))
+                    throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first);
+                auto drv = derivationFromPath(i.first);
+                for (auto & output : drv.outputs)
+                    if (wantOutput(output.first, i.second))
+                        newPaths.insert(output.second.path);
+            } else if (!goal.isAllowed(path))
+                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path);
+        }
+
+        next->buildPaths(paths, buildMode);
+
+        PathSet closure;
+        next->computeFSClosure(newPaths, closure);
+        for (auto & path : closure)
+            goal.addDependency(path);
+    }
+
+    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode = bmNormal) override
+    { unsupported("buildDerivation"); }
+
+    void addTempRoot(const Path & path)
+    { }
+
+    void addIndirectRoot(const Path & path)
+    { }
+
+    Roots findRoots()
+    { return Roots(); }
+
+    void collectGarbage(const GCOptions & options, GCResults & results)
+    { }
+
+    void addSignatures(const Path & storePath, const StringSet & sigs)
+    { unsupported("addSignatures"); }
+
+    void queryMissing(const PathSet & targets,
+        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+        unsigned long long & downloadSize, unsigned long long & narSize)
+    {
+        /* This is slightly impure since it leaks information to the
+           client about what paths will be built/substituted or are
+           already present. Probably not a big deal. */
+
+        PathSet allowed;
+        for (auto & path : targets) {
+            DrvPathWithOutputs i = parseDrvPathWithOutputs(path);
+            if (goal.isAllowed(i.first))
+                allowed.insert(i.first);
+            else
+                unknown.insert(i.first);
+        }
+
+        next->queryMissing(allowed, willBuild, willSubstitute,
+            unknown, downloadSize, narSize);
+    }
+};
+
+
+void DerivationGoal::startDaemon()
+{
+    Store::Params params;
+    params["path-info-cache-size"] = "0";
+    params["store"] = worker.store.storeDir;
+    params["root"] = worker.store.rootDir;
+    params["state"] = "/no-such-path";
+    params["log"] = "/no-such-path";
+    auto store = make_ref<RestrictedStore>(params,
+        ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
+        *this);
+
+    addedPaths.clear();
+
+    auto socketName = ".nix-socket";
+    Path socketPath = tmpDir + "/" + socketName;
+    env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
+
+    daemonSocket = createUnixDomainSocket(socketPath, 0600);
+
+    chownToBuilder(socketPath);
+
+    daemonThread = std::thread([this, store]() {
+
+        while (true) {
+
+            /* Accept a connection. */
+            struct sockaddr_un remoteAddr;
+            socklen_t remoteAddrLen = sizeof(remoteAddr);
+
+            AutoCloseFD remote = accept(daemonSocket.get(),
+                (struct sockaddr *) &remoteAddr, &remoteAddrLen);
+            if (!remote) {
+                if (errno == EINTR) continue;
+                if (errno == EINVAL) break;
+                throw SysError("accepting connection");
+            }
+
+            closeOnExec(remote.get());
+
+            debug("received daemon connection");
+
+            // FIXME: process on a separate thread.
+            FdSource from(remote.get());
+            FdSink to(remote.get());
+            try {
+                daemon::processConnection(store, from, to,
+                    daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
+            } catch (SysError &) {
+                ignoreException();
+            }
+
+            debug("terminated daemon connection");
+        }
+
+        debug("daemon shutting down");
+    });
+}
+
+
+void DerivationGoal::stopDaemon()
+{
+    if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
+        throw SysError("shutting down daemon socket");
+
+    if (daemonThread.joinable())
+        daemonThread.join();
+
+    daemonSocket = -1;
+}
+
+
+void DerivationGoal::addDependency(const Path & path)
+{
+    worker.store.assertStorePath(path);
+
+    if (isAllowed(path)) return;
+
+    addedPaths.insert(path);
+
+    /* If we're doing a sandbox build, then we have to make the path
+       appear in the sandbox. */
+    if (useChroot) {
+
+        debug("materialising '%s' in the sandbox", path);
+
+        #if __linux__
+
+            Path source = worker.store.toRealPath(path);
+            Path target = chrootRootDir + path;
+            debug("bind-mounting %s -> %s", target, source);
+
+            if (pathExists(target))
+                throw Error("store path '%s' already exists in the sandbox", path);
+
+            struct stat st;
+            if (lstat(source.c_str(), &st))
+                throw SysError("getting attributes of path '%s'", source);
+
+            if (S_ISDIR(st.st_mode)) {
+
+                /* Bind-mount the path into the sandbox. This requires
+                   entering its mount namespace, which is not possible
+                   in multithreaded programs. So we do this in a
+                   child process.*/
+                Pid child(startProcess([&]() {
+
+                    if (setns(sandboxMountNamespace.get(), 0) == -1)
+                        throw SysError("entering sandbox mount namespace");
+
+                    createDirs(target);
+
+                    if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
+                        throw SysError("bind mount from '%s' to '%s' failed", source, target);
+
+                    _exit(0);
+                }));
+
+                int status = child.wait();
+                if (status != 0)
+                    throw Error("could not add path '%s' to sandbox", path);
+
+            } else
+                linkOrCopy(source, target);
+
+        #else
+            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path);
+        #endif
+
+    }
+}
+
+
 void DerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;
@@ -2756,15 +3091,30 @@ void DerivationGoal::runChild()
                outside of the namespace.  Making a subtree private is
                local to the namespace, though, so setting MS_PRIVATE
                does not affect the outside world. */
-            if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) {
-                throw SysError("unable to make '/' private mount");
-            }
+            if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
+                throw SysError("unable to make '/' private");
 
             /* Bind-mount chroot directory to itself, to treat it as a
                different filesystem from /, as needed for pivot_root. */
             if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
                 throw SysError(format("unable to bind mount '%1%'") % chrootRootDir);
 
+            /* Bind-mount the sandbox's Nix store onto itself so that
+               we can mark it as a "shared" subtree, allowing bind
+               mounts made in *this* mount namespace to be propagated
+               into the child namespace created by the
+               unshare(CLONE_NEWNS) call below.
+
+               Marking chrootRootDir as MS_SHARED causes pivot_root()
+               to fail with EINVAL. Don't know why. */
+            Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
+
+            if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
+                throw SysError("unable to bind mount the Nix store", chrootStoreDir);
+
+            if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
+                throw SysError("unable to make '%s' shared", chrootStoreDir);
+
             /* Set up a nearly empty /dev, unless the user asked to
                bind-mount the host /dev. */
             Strings ss;
@@ -2866,6 +3216,19 @@ void DerivationGoal::runChild()
                 }
             }
 
+            /* Unshare this mount namespace. This is necessary because
+               pivot_root() below changes the root of the mount
+               namespace. This means that the call to setns() in
+               addDependency() would hide the host's filesystem,
+               making it impossible to bind-mount paths from the host
+               Nix store into the sandbox. Therefore, we save the
+               pre-pivot_root namespace in
+               sandboxMountNamespace. Since we made /nix/store a
+               shared subtree above, this allows addDependency() to
+               make paths appear in the sandbox. */
+            if (unshare(CLONE_NEWNS) == -1)
+                throw SysError("unsharing mount namespace");
+
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError(format("cannot change directory to '%1%'") % chrootRootDir);
@@ -3182,6 +3545,14 @@ void DerivationGoal::registerOutputs()
 
     std::exception_ptr delayedException;
 
+    /* The paths that can be referenced are the input closures, the
+       output paths, and any paths that have been built via recursive
+       Nix calls. */
+    PathSet referenceablePaths;
+    for (auto & p : inputPaths) referenceablePaths.insert(p);
+    for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
+    for (auto & p : addedPaths) referenceablePaths.insert(p);
+
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
@@ -3317,7 +3688,7 @@ void DerivationGoal::registerOutputs()
            verify later on whether nobody has messed with the store. */
         debug("scanning for references inside '%1%'", path);
         HashResult hash;
-        PathSet references = scanForReferences(actualPath, allPaths, hash);
+        PathSet references = scanForReferences(actualPath, referenceablePaths, hash);
 
         if (buildMode == bmCheck) {
             if (!worker.store.isValidPath(path)) continue;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 64147027b26..d3c17e7721b 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -186,8 +186,75 @@ struct RetrieveRegularNARSink : ParseSink
     }
 };
 
+struct ClientSettings
+{
+    bool keepFailed;
+    bool keepGoing;
+    bool tryFallback;
+    Verbosity verbosity;
+    unsigned int maxBuildJobs;
+    time_t maxSilentTime;
+    bool verboseBuild;
+    unsigned int buildCores;
+    bool useSubstitutes;
+    StringMap overrides;
+
+    void apply(TrustedFlag trusted)
+    {
+        settings.keepFailed = keepFailed;
+        settings.keepGoing = keepGoing;
+        settings.tryFallback = tryFallback;
+        nix::verbosity = verbosity;
+        settings.maxBuildJobs.assign(maxBuildJobs);
+        settings.maxSilentTime = maxSilentTime;
+        settings.verboseBuild = verboseBuild;
+        settings.buildCores = buildCores;
+        settings.useSubstitutes = useSubstitutes;
+
+        for (auto & i : overrides) {
+            auto & name(i.first);
+            auto & value(i.second);
+
+            auto setSubstituters = [&](Setting<Strings> & res) {
+                if (name != res.name && res.aliases.count(name) == 0)
+                    return false;
+                StringSet trusted = settings.trustedSubstituters;
+                for (auto & s : settings.substituters.get())
+                    trusted.insert(s);
+                Strings subs;
+                auto ss = tokenizeString<Strings>(value);
+                for (auto & s : ss)
+                    if (trusted.count(s))
+                        subs.push_back(s);
+                    else
+                        warn("ignoring untrusted substituter '%s'", s);
+                res = subs;
+                return true;
+            };
+
+            try {
+                if (name == "ssh-auth-sock") // obsolete
+                    ;
+                else if (trusted
+                    || name == settings.buildTimeout.name
+                    || name == "connect-timeout"
+                    || (name == "builders" && value == ""))
+                    settings.set(name, value);
+                else if (setSubstituters(settings.substituters))
+                    ;
+                else if (setSubstituters(settings.extraSubstituters))
+                    ;
+                else
+                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
+            } catch (UsageError & e) {
+                warn(e.what());
+            }
+        }
+    }
+};
+
 static void performOp(TunnelLogger * logger, ref<Store> store,
-    bool trusted, unsigned int clientVersion,
+    TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
     Source & from, BufferedSink & to, unsigned int op)
 {
     switch (op) {
@@ -464,70 +531,37 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopSetOptions: {
-        settings.keepFailed = readInt(from);
-        settings.keepGoing = readInt(from);
-        settings.tryFallback = readInt(from);
-        verbosity = (Verbosity) readInt(from);
-        settings.maxBuildJobs.assign(readInt(from));
-        settings.maxSilentTime = readInt(from);
+
+        ClientSettings clientSettings;
+
+        clientSettings.keepFailed = readInt(from);
+        clientSettings.keepGoing = readInt(from);
+        clientSettings.tryFallback = readInt(from);
+        clientSettings.verbosity = (Verbosity) readInt(from);
+        clientSettings.maxBuildJobs = readInt(from);
+        clientSettings.maxSilentTime = readInt(from);
         readInt(from); // obsolete useBuildHook
-        settings.verboseBuild = lvlError == (Verbosity) readInt(from);
+        clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from);
         readInt(from); // obsolete logType
         readInt(from); // obsolete printBuildTrace
-        settings.buildCores = readInt(from);
-        settings.useSubstitutes  = readInt(from);
+        clientSettings.buildCores = readInt(from);
+        clientSettings.useSubstitutes = readInt(from);
 
-        StringMap overrides;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
             unsigned int n = readInt(from);
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                overrides.emplace(name, value);
+                clientSettings.overrides.emplace(name, value);
             }
         }
 
         logger->startWork();
 
-        for (auto & i : overrides) {
-            auto & name(i.first);
-            auto & value(i.second);
-
-            auto setSubstituters = [&](Setting<Strings> & res) {
-                if (name != res.name && res.aliases.count(name) == 0)
-                    return false;
-                StringSet trusted = settings.trustedSubstituters;
-                for (auto & s : settings.substituters.get())
-                    trusted.insert(s);
-                Strings subs;
-                auto ss = tokenizeString<Strings>(value);
-                for (auto & s : ss)
-                    if (trusted.count(s))
-                        subs.push_back(s);
-                    else
-                        warn("ignoring untrusted substituter '%s'", s);
-                res = subs;
-                return true;
-            };
-
-            try {
-                if (name == "ssh-auth-sock") // obsolete
-                    ;
-                else if (trusted
-                    || name == settings.buildTimeout.name
-                    || name == "connect-timeout"
-                    || (name == "builders" && value == ""))
-                    settings.set(name, value);
-                else if (setSubstituters(settings.substituters))
-                    ;
-                else if (setSubstituters(settings.extraSubstituters))
-                    ;
-                else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
-            } catch (UsageError & e) {
-                warn(e.what());
-            }
-        }
+        // FIXME: use some setting in recursive mode. Will need to use
+        // non-global variables.
+        if (!recursive)
+            clientSettings.apply(trusted);
 
         logger->stopWork();
         break;
@@ -694,11 +728,12 @@ void processConnection(
     ref<Store> store,
     FdSource & from,
     FdSink & to,
-    bool trusted,
+    TrustedFlag trusted,
+    RecursiveFlag recursive,
     const std::string & userName,
     uid_t userId)
 {
-    MonitorFdHup monitor(from.fd);
+    auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
 
     /* Exchange the greeting. */
     unsigned int magic = readInt(from);
@@ -712,7 +747,9 @@ void processConnection(
 
     auto tunnelLogger = new TunnelLogger(to, clientVersion);
     auto prevLogger = nix::logger;
-    logger = tunnelLogger;
+    // FIXME
+    if (!recursive)
+        logger = tunnelLogger;
 
     unsigned int opCount = 0;
 
@@ -721,8 +758,10 @@ void processConnection(
         prevLogger->log(lvlDebug, fmt("%d operations", opCount));
     });
 
-    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
-        setAffinityTo(readInt(from));
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
+        auto affinity = readInt(from);
+        setAffinityTo(affinity);
+    }
 
     readInt(from); // obsolete reserveSpace
 
@@ -760,7 +799,7 @@ void processConnection(
             opCount++;
 
             try {
-                performOp(tunnelLogger, store, trusted, clientVersion, from, to, op);
+                performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op);
             } catch (Error & e) {
                 /* If we're not in a state where we can send replies, then
                    something went wrong processing the input of the
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index fa26ba9c222..26693201325 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -3,11 +3,15 @@
 
 namespace nix::daemon {
 
+enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
+enum RecursiveFlag : bool { NotRecursive = false, Recursive = true };
+
 void processConnection(
     ref<Store> store,
     FdSource & from,
     FdSink & to,
-    bool trusted,
+    TrustedFlag trusted,
+    RecursiveFlag recursive,
     const std::string & userName,
     uid_t userId);
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index da17ddcba07..9314b616108 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -192,7 +192,7 @@ static void daemonLoop(char * * argv)
 
             closeOnExec(remote.get());
 
-            bool trusted = false;
+            TrustedFlag trusted = NotTrusted;
             PeerInfo peer = getPeerInfo(remote.get());
 
             struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0;
@@ -205,7 +205,7 @@ static void daemonLoop(char * * argv)
             Strings allowedUsers = settings.allowedUsers;
 
             if (matchUser(user, group, trustedUsers))
-                trusted = true;
+                trusted = Trusted;
 
             if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup)
                 throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
@@ -239,7 +239,7 @@ static void daemonLoop(char * * argv)
                 /* Handle the connection. */
                 FdSource from(remote.get());
                 FdSink to(remote.get());
-                processConnection(openUncachedStore(), from, to, trusted, user, peer.uid);
+                processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid);
 
                 exit(0);
             }, options);
@@ -321,7 +321,7 @@ static int _main(int argc, char * * argv)
             } else {
                 FdSource from(STDIN_FILENO);
                 FdSink to(STDOUT_FILENO);
-                processConnection(openUncachedStore(), from, to, true, "root", 0);
+                processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0);
             }
         } else {
             daemonLoop(argv);

From c921074c1921d4bf392991a44c5644877f9c567a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Nov 2019 12:55:05 +0100
Subject: [PATCH 1641/2196] RestrictedStore: Implement addToStore()

---
 src/libstore/build.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2fd27cfa05f..e8b23d3961a 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2721,7 +2721,14 @@ struct RestrictedStore : public LocalFSStore
     Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
-    { throw Error("addToStore");
+    { throw Error("addToStore"); }
+
+    void addToStore(const ValidPathInfo & info, Source & narSource,
+        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
+        std::shared_ptr<FSAccessor> accessor = 0) override
+    {
+        next->addToStore(info, narSource, repair, checkSigs, accessor);
+        goal.addDependency(info.path);
     }
 
     Path addToStoreFromDump(const string & dump, const string & name,

From 2af9561316606892ce0337939c01f3c0d43d75bd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Nov 2019 13:23:58 +0100
Subject: [PATCH 1642/2196] Add a test for recursive Nix

---
 tests/local.mk     |  3 ++-
 tests/recursive.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 tests/recursive.sh

diff --git a/tests/local.mk b/tests/local.mk
index 187f96ea206..affdc3cc306 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -30,7 +30,8 @@ nix_tests = \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
-  function-trace.sh
+  function-trace.sh \
+  recursive.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
diff --git a/tests/recursive.sh b/tests/recursive.sh
new file mode 100644
index 00000000000..25204caf138
--- /dev/null
+++ b/tests/recursive.sh
@@ -0,0 +1,67 @@
+source common.sh
+
+clearStore
+
+export unreachable=$(nix add-to-store ./recursive.sh)
+
+nix build -o $TEST_ROOT/result -L '(
+  with import ./config.nix;
+  with import <nix/config.nix>;
+  mkDerivation {
+    name = "recursive";
+    dummy = builtins.toFile "dummy" "bla bla";
+    SHELL = shell;
+
+    # Note: this is a string without context.
+    unreachable = builtins.getEnv "unreachable";
+
+    buildCommand = '\'\''
+      mkdir $out
+      PATH=${nixBinDir}:$PATH
+      opts="--experimental-features nix-command"
+
+      # Check that we can query/build paths in our input closure.
+      nix $opts path-info $dummy
+      nix $opts build $dummy
+
+      # Make sure we cannot query/build paths not in out input closure.
+      [[ -e $unreachable ]]
+      (! nix $opts path-info $unreachable)
+      (! nix $opts build $unreachable)
+
+      # Add something to the store.
+      echo foobar > foobar
+      foobar=$(nix $opts add-to-store ./foobar)
+
+      nix $opts path-info $foobar
+      nix $opts build $foobar
+
+      # Add it to our closure.
+      ln -s $foobar $out/foobar
+
+      [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
+
+      # Build a derivation.
+      nix $opts build -L '\''(
+        derivation {
+          name = "inner1";
+          builder = builtins.getEnv "SHELL";
+          system = builtins.getEnv "system";
+          fnord = builtins.toFile "fnord" "fnord";
+          args = [ "-c" "echo $fnord blaat > $out" ];
+        }
+      )'\''
+
+      [[ $(nix $opts path-info --json ./result) =~ fnord ]]
+
+      ln -s $(nix $opts path-info ./result) $out/inner1
+    '\'\'';
+  })
+'
+
+[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]]
+
+# Make sure the recursively created paths are in the closure.
+nix path-info -r $TEST_ROOT/result | grep foobar
+nix path-info -r $TEST_ROOT/result | grep fnord
+nix path-info -r $TEST_ROOT/result | grep inner1

From c119ab9db0edf65379593883c5ed5253549ebbd0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Nov 2019 13:48:58 +0100
Subject: [PATCH 1643/2196] Enable recursive Nix using a feature

Derivations that want to use recursion should now set

  requiredSystemFeatures = [ "recursive-nix" ];

to make the daemon socket appear.

Also, Nix should be configured with "experimental-features =
recursive-nix".
---
 src/libstore/build.cc   | 5 ++++-
 src/libstore/globals.cc | 2 +-
 tests/recursive.sh      | 4 +++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e8b23d3961a..e77512ca488 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2248,7 +2248,8 @@ void DerivationGoal::startBuilder()
 
     /* Fire up a Nix daemon to process recursive Nix calls from the
        builder. */
-    startDaemon();
+    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
+        startDaemon();
 
     /* Run the builder. */
     printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder);
@@ -2832,6 +2833,8 @@ struct RestrictedStore : public LocalFSStore
 
 void DerivationGoal::startDaemon()
 {
+    settings.requireExperimentalFeature("recursive-nix");
+
     Store::Params params;
     params["path-info-cache-size"] = "0";
     params["store"] = worker.store.storeDir;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 249c3667357..042aaea95fa 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -95,7 +95,7 @@ StringSet Settings::getDefaultSystemFeatures()
     /* For backwards compatibility, accept some "features" that are
        used in Nixpkgs to route builds to certain machines but don't
        actually require anything special on the machines. */
-    StringSet features{"nixos-test", "benchmark", "big-parallel"};
+    StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"};
 
     #if __linux__
     if (access("/dev/kvm", R_OK | W_OK) == 0)
diff --git a/tests/recursive.sh b/tests/recursive.sh
index 25204caf138..62896c01590 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -4,7 +4,7 @@ clearStore
 
 export unreachable=$(nix add-to-store ./recursive.sh)
 
-nix build -o $TEST_ROOT/result -L '(
+nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(
   with import ./config.nix;
   with import <nix/config.nix>;
   mkDerivation {
@@ -15,6 +15,8 @@ nix build -o $TEST_ROOT/result -L '(
     # Note: this is a string without context.
     unreachable = builtins.getEnv "unreachable";
 
+    requiredSystemFeatures = [ "recursive-nix" ];
+
     buildCommand = '\'\''
       mkdir $out
       PATH=${nixBinDir}:$PATH

From 69326f3637f1560407711838e7298d736274ffd4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 Nov 2019 14:27:28 +0100
Subject: [PATCH 1644/2196] Recursive Nix: Handle concurrent client connections

---
 src/libstore/build.cc | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e77512ca488..63557b560c7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -895,6 +895,9 @@ class DerivationGoal : public Goal
     /* The daemon main thread. */
     std::thread daemonThread;
 
+    /* The daemon worker threads. */
+    std::vector<std::thread> daemonWorkerThreads;
+
     /* Paths that were added via recursive Nix calls. */
     PathSet addedPaths;
 
@@ -2875,17 +2878,19 @@ void DerivationGoal::startDaemon()
 
             debug("received daemon connection");
 
-            // FIXME: process on a separate thread.
-            FdSource from(remote.get());
-            FdSink to(remote.get());
-            try {
-                daemon::processConnection(store, from, to,
-                    daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
-            } catch (SysError &) {
-                ignoreException();
-            }
+            auto workerThread = std::thread([this, store, remote{std::move(remote)}]() {
+                FdSource from(remote.get());
+                FdSink to(remote.get());
+                try {
+                    daemon::processConnection(store, from, to,
+                        daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
+                    debug("terminated daemon connection");
+                } catch (SysError &) {
+                    ignoreException();
+                }
+            });
 
-            debug("terminated daemon connection");
+            daemonWorkerThreads.push_back(std::move(workerThread));
         }
 
         debug("daemon shutting down");
@@ -2901,6 +2906,12 @@ void DerivationGoal::stopDaemon()
     if (daemonThread.joinable())
         daemonThread.join();
 
+    // FIXME: should prune worker threads more quickly.
+    // FIXME: shutdown the client socket to speed up worker termination.
+    for (auto & thread : daemonWorkerThreads)
+        thread.join();
+    daemonWorkerThreads.clear();
+
     daemonSocket = -1;
 }
 

From 7614a127a0ef52868c35fa9d6ae6398b7297d408 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 10:35:31 +0100
Subject: [PATCH 1645/2196] Fix binaryTarball test

---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 0b74de39072..c5bee782a3b 100644
--- a/release.nix
+++ b/release.nix
@@ -260,7 +260,7 @@ let
             x86_64-linux = "${build.x86_64-linux}";
           }
           EOF
-          su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
           (! [ -L /home/alice/.profile-1-link ])
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
 

From 35732a95bcdc0a4b4492845205e6283fcc88fd0d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 10:36:06 +0100
Subject: [PATCH 1646/2196] Disable the evalNixpkgs test

It constantly OOMs.

https://hydra.nixos.org/build/105784912
---
 release.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index c5bee782a3b..b137f1d2d16 100644
--- a/release.nix
+++ b/release.nix
@@ -269,6 +269,7 @@ let
           umount /nix
         ''); # */
 
+    /*
     tests.evalNixpkgs =
       import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
         inherit nixpkgs;
@@ -276,6 +277,7 @@ let
         nix = build.x86_64-linux;
         officialRelease = false;
       };
+    */
 
     tests.evalNixOS =
       pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
@@ -324,7 +326,7 @@ let
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
-          tests.evalNixpkgs
+          #tests.evalNixpkgs
           tests.evalNixOS
           installerScript
         ];

From 39a2e166dd316cf3bf248cb109dde3a95d753541 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 16:53:02 +0100
Subject: [PATCH 1647/2196] Cleanup

---
 src/libstore/build.cc        | 2 +-
 src/libutil/util.cc          | 2 +-
 src/libutil/xml-writer.cc    | 4 ++--
 src/nix-daemon/nix-daemon.cc | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ada1cf8ec4d..c82f6074869 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -23,8 +23,8 @@
 #include <chrono>
 #include <regex>
 #include <queue>
+#include <climits>
 
-#include <limits.h>
 #include <sys/time.h>
 #include <sys/wait.h>
 #include <sys/types.h>
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 34c1d96dcb8..79716c5a7b8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -10,6 +10,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <climits>
 #include <iostream>
 #include <sstream>
 #include <thread>
@@ -17,7 +18,6 @@
 
 #include <fcntl.h>
 #include <grp.h>
-#include <limits.h>
 #include <pwd.h>
 #include <sys/ioctl.h>
 #include <sys/types.h>
diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc
index e5cc2e9fc71..68857e34dc1 100644
--- a/src/libutil/xml-writer.cc
+++ b/src/libutil/xml-writer.cc
@@ -1,10 +1,10 @@
-#include <assert.h>
+#include <cassert>
 
 #include "xml-writer.hh"
 
 
 namespace nix {
-    
+
 
 XMLWriter::XMLWriter(bool indent, std::ostream & output)
     : output(output), indent(indent)
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index da17ddcba07..65ae88fafe4 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -10,8 +10,9 @@
 #include "daemon.hh"
 
 #include <algorithm>
-
+#include <climits>
 #include <cstring>
+
 #include <unistd.h>
 #include <signal.h>
 #include <sys/types.h>
@@ -23,7 +24,6 @@
 #include <pwd.h>
 #include <grp.h>
 #include <fcntl.h>
-#include <limits.h>
 
 #if __APPLE__ || __FreeBSD__
 #include <sys/ucred.h>

From 52ffe2797a6a943bac968e92ad443a567dfdf80e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Thu, 7 Nov 2019 00:11:57 +0100
Subject: [PATCH 1648/2196] doc: Document `--dry-run` option for `nix-build`

---
 doc/manual/command-ref/nix-build.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml
index 40fe7a43f10..c1b783c87d6 100644
--- a/doc/manual/command-ref/nix-build.xml
+++ b/doc/manual/command-ref/nix-build.xml
@@ -30,6 +30,7 @@
       <replaceable>attrPath</replaceable>
     </arg>
     <arg><option>--no-out-link</option></arg>
+    <arg><option>--dry-run</option></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--out-link</option></arg>
@@ -98,6 +99,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--dry-run</option></term>
+   <listitem><para>Show what store paths would be built or downloaded</para></listitem>
+  </varlistentry>
+
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /
   <option>-o</option> <replaceable>outlink</replaceable></term>
 

From 5ff4d77f5585f9aad068fcc473393095e0640ae4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 16:54:18 +0100
Subject: [PATCH 1649/2196] Precompile headers

This cuts 'make install -j6' on my laptop from 170s to 134s.
---
 Makefile                  |  4 +--
 configure.ac              |  1 -
 mk/lib.mk                 |  1 +
 mk/patterns.mk            |  4 +--
 mk/precompiled-headers.mk | 42 +++++++++++++++++++++++
 precompiled-headers.h     | 71 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 118 insertions(+), 5 deletions(-)
 create mode 100644 mk/precompiled-headers.mk
 create mode 100644 precompiled-headers.h

diff --git a/Makefile b/Makefile
index 9ac82fda657..c441a933e15 100644
--- a/Makefile
+++ b/Makefile
@@ -15,8 +15,8 @@ makefiles = \
   tests/local.mk \
   tests/plugins/local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h
-
 -include Makefile.config
 
 include mk/lib.mk
+
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17
diff --git a/configure.ac b/configure.ac
index 4a6eaac8a39..c67f1e7395e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -57,7 +57,6 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
 
 AC_CHECK_TOOL([AR], [ar])
 
diff --git a/mk/lib.mk b/mk/lib.mk
index 1da51d87973..b0a36cf8dd4 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -89,6 +89,7 @@ include mk/jars.mk
 include mk/patterns.mk
 include mk/templates.mk
 include mk/tests.mk
+include mk/precompiled-headers.mk
 
 
 # Include all sub-Makefiles.
diff --git a/mk/patterns.mk b/mk/patterns.mk
index 3219d9629fb..7319f4cddee 100644
--- a/mk/patterns.mk
+++ b/mk/patterns.mk
@@ -1,10 +1,10 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
new file mode 100644
index 00000000000..779389b262d
--- /dev/null
+++ b/mk/precompiled-headers.mk
@@ -0,0 +1,42 @@
+PRECOMPILE_HEADERS ?= 1
+
+print-var-help += \
+  echo "  PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";
+
+GCH = $(buildprefix)precompiled-headers.h.gch
+
+$(GCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS)
+
+PCH = $(buildprefix)precompiled-headers.h.pch
+
+$(PCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS)
+
+clean-files += $(GCH) $(PCH)
+
+ifeq ($(PRECOMPILE_HEADERS), 1)
+
+  ifeq ($(CXX), g++)
+
+    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(GCH)
+
+  else ifeq ($(CXX), clang++)
+
+    GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(PCH)
+
+  else
+
+    $(error Don't know how to precompile headers on $(CXX))
+
+  endif
+
+endif
diff --git a/precompiled-headers.h b/precompiled-headers.h
new file mode 100644
index 00000000000..c5790bec4d4
--- /dev/null
+++ b/precompiled-headers.h
@@ -0,0 +1,71 @@
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cctype>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <future>
+#include <iostream>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <optional>
+#include <queue>
+#include <random>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include <boost/format.hpp>
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <grp.h>
+#include <net/if.h>
+#include <netdb.h>
+#include <netinet/ip.h>
+#include <pwd.h>
+#include <sched.h>
+#include <signal.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/param.h>
+#include <sys/personality.h>
+#include <sys/resource.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <sys/wait.h>
+#include <termios.h>
+#include <unistd.h>
+
+#include "util.hh"
+#include "args.hh"

From f5b7991e59d8c9ae38fb4c9074a5847942caab73 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 16:55:37 +0100
Subject: [PATCH 1650/2196] Revert "autoconf: Allow overriding CFLAGS/CXXFLAGS
 from outside."

This reverts commit 717e821b99797845e1bef47d862f8cb0fb69cfc9. It's
much more convenient to do 'make OPTIMIZE=0'.
---
 Makefile          | 8 ++++++++
 configure.ac      | 6 ++----
 perl/Makefile     | 7 +++++++
 perl/configure.ac | 6 ++----
 4 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index c441a933e15..fe397e018b2 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,14 @@ makefiles = \
 
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
+endif
+
 include mk/lib.mk
 
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17
diff --git a/configure.ac b/configure.ac
index c67f1e7395e..26f16b8460a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,10 +50,8 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
diff --git a/perl/Makefile b/perl/Makefile
index f36f5d0e9d8..284c7502249 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -4,4 +4,11 @@ GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CFLAGS += -O3
+  GLOBAL_CXXFLAGS += -O3
+endif
+
 include mk/lib.mk
diff --git a/perl/configure.ac b/perl/configure.ac
index e8e3610a860..966700695ff 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -2,10 +2,8 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AX_CXX_COMPILE_STDCXX_11

From 04bf9acd22c5e3f399f66d7edb0863e8394e542e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2019 17:30:48 +0100
Subject: [PATCH 1651/2196] Remove #include

---
 src/libstore/download.cc | 4 ++++
 src/libstore/download.hh | 8 ++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index cdf56e09d69..8fe278d02e4 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -34,6 +34,10 @@ DownloadSettings downloadSettings;
 
 static GlobalConfig::Register r1(&downloadSettings);
 
+CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
+    : uri(uri), ttl(settings.tarballTtl)
+{ }
+
 std::string resolveUri(const std::string & uri)
 {
     if (uri.compare(0, 8, "channel:") == 0)
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 68565bf462a..5a131c70450 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -2,7 +2,7 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "globals.hh"
+#include "config.hh"
 
 #include <string>
 #include <future>
@@ -71,10 +71,10 @@ struct CachedDownloadRequest
     bool unpack = false;
     std::string name;
     Hash expectedHash;
-    unsigned int ttl = settings.tarballTtl;
+    unsigned int ttl;
 
-    CachedDownloadRequest(const std::string & uri)
-        : uri(uri) { }
+    CachedDownloadRequest(const std::string & uri);
+    CachedDownloadRequest() = delete;
 };
 
 struct CachedDownloadResult

From 99af822004418b25b6d04eca67949c62e770c16b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Nov 2019 10:14:00 +0100
Subject: [PATCH 1652/2196] Disable the evalNixOS test

It also OOMs.

https://hydra.nixos.org/build/105942679
---
 release.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index b137f1d2d16..61ca4323c64 100644
--- a/release.nix
+++ b/release.nix
@@ -277,7 +277,6 @@ let
         nix = build.x86_64-linux;
         officialRelease = false;
       };
-    */
 
     tests.evalNixOS =
       pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
@@ -289,6 +288,7 @@ let
 
           touch $out
         '';
+    */
 
 
     installerScript =
@@ -327,7 +327,7 @@ let
           tests.nix-copy-closure
           tests.binaryTarball
           #tests.evalNixpkgs
-          tests.evalNixOS
+          #tests.evalNixOS
           installerScript
         ];
     };

From 6d2605500fd99091cea2985499957a0aace0d81b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Nov 2019 11:53:28 +0100
Subject: [PATCH 1653/2196] Fix macOS build

---
 precompiled-headers.h | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/precompiled-headers.h b/precompiled-headers.h
index c5790bec4d4..e0d885b230c 100644
--- a/precompiled-headers.h
+++ b/precompiled-headers.h
@@ -43,23 +43,13 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <grp.h>
-#include <net/if.h>
 #include <netdb.h>
-#include <netinet/ip.h>
 #include <pwd.h>
-#include <sched.h>
 #include <signal.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/mount.h>
-#include <sys/param.h>
-#include <sys/personality.h>
 #include <sys/resource.h>
 #include <sys/select.h>
 #include <sys/socket.h>
-#include <sys/socket.h>
 #include <sys/stat.h>
-#include <sys/syscall.h>
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/utsname.h>

From e5bf81256c8a7ee9d725d4093853c76e0a562c66 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Nov 2019 12:18:37 +0100
Subject: [PATCH 1654/2196] Fix Perl bindings

---
 Makefile          |  1 +
 mk/lib.mk         |  1 -
 perl/Makefile     |  3 ++-
 perl/configure.ac | 14 +++++++-------
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index fe397e018b2..866c0961eb6 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,5 @@
 makefiles = \
+  mk/precompiled-headers.mk \
   local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
diff --git a/mk/lib.mk b/mk/lib.mk
index b0a36cf8dd4..1da51d87973 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -89,7 +89,6 @@ include mk/jars.mk
 include mk/patterns.mk
 include mk/templates.mk
 include mk/tests.mk
-include mk/precompiled-headers.mk
 
 
 # Include all sub-Makefiles.
diff --git a/perl/Makefile b/perl/Makefile
index 284c7502249..7ddb0cf692c 100644
--- a/perl/Makefile
+++ b/perl/Makefile
@@ -7,8 +7,9 @@ GLOBAL_CXXFLAGS += -g -Wall
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CFLAGS += -O3
   GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
 endif
 
 include mk/lib.mk
diff --git a/perl/configure.ac b/perl/configure.ac
index 966700695ff..c3769e1426c 100644
--- a/perl/configure.ac
+++ b/perl/configure.ac
@@ -6,7 +6,6 @@ CFLAGS=
 CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX_11
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -69,14 +68,15 @@ AC_SUBST(perlFlags)
 
 PKG_CHECK_MODULES([NIX], [nix-store])
 
-NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate])
+NEED_PROG([NIX], [nix])
 
 # Get nix configure values
-nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixBinDir | tr -d \")
-nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLibexecDir | tr -d \")
-nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLocalstateDir | tr -d \")
-nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixSysconfDir | tr -d \")
-nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixStoreDir | tr -d \")
+export NIX_REMOTE=daemon
+nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixBinDir)
+nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLibexecDir)
+nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLocalstateDir)
+nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixSysconfDir)
+nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixStoreDir)
 AC_SUBST(nixbindir)
 AC_SUBST(nixlibexecdir)
 AC_SUBST(nixlocalstatedir)

From 4145cd2da002e1bd8affa0392c80118eabe58e3c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 22 Oct 2019 20:12:23 +0200
Subject: [PATCH 1655/2196] Use upstream nlohmann_json

---
 release-common.nix    |     1 +
 release.nix           |     2 +-
 src/nlohmann/json.hpp | 20406 ----------------------------------------
 3 files changed, 2 insertions(+), 20407 deletions(-)
 delete mode 100644 src/nlohmann/json.hpp

diff --git a/release-common.nix b/release-common.nix
index 11944a63124..63f39f00548 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -50,6 +50,7 @@ rec {
       bzip2 xz brotli editline
       openssl pkgconfig sqlite boehmgc
       boost
+      nlohmann_json
 
       # Tests
       git
diff --git a/release.nix b/release.nix
index 61ca4323c64..1af6424dd8d 100644
--- a/release.nix
+++ b/release.nix
@@ -200,7 +200,7 @@ let
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp
deleted file mode 100644
index c9af0bed36d..00000000000
--- a/src/nlohmann/json.hpp
+++ /dev/null
@@ -1,20406 +0,0 @@
-/*
-    __ _____ _____ _____
- __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 3.5.0
-|_____|_____|_____|_|___|  https://github.com/nlohmann/json
-
-Licensed under the MIT License <http://opensource.org/licenses/MIT>.
-SPDX-License-Identifier: MIT
-Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>.
-
-Permission is hereby  granted, free of charge, to any  person obtaining a copy
-of this software and associated  documentation files (the "Software"), to deal
-in the Software  without restriction, including without  limitation the rights
-to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
-copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
-IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
-FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
-AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
-LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-*/
-
-#ifndef NLOHMANN_JSON_HPP
-#define NLOHMANN_JSON_HPP
-
-#define NLOHMANN_JSON_VERSION_MAJOR 3
-#define NLOHMANN_JSON_VERSION_MINOR 5
-#define NLOHMANN_JSON_VERSION_PATCH 0
-
-#include <algorithm> // all_of, find, for_each
-#include <cassert> // assert
-#include <ciso646> // and, not, or
-#include <cstddef> // nullptr_t, ptrdiff_t, size_t
-#include <functional> // hash, less
-#include <initializer_list> // initializer_list
-#include <iosfwd> // istream, ostream
-#include <iterator> // random_access_iterator_tag
-#include <numeric> // accumulate
-#include <string> // string, stoi, to_string
-#include <utility> // declval, forward, move, pair, swap
-
-// #include <nlohmann/json_fwd.hpp>
-#ifndef NLOHMANN_JSON_FWD_HPP
-#define NLOHMANN_JSON_FWD_HPP
-
-#include <cstdint> // int64_t, uint64_t
-#include <map> // map
-#include <memory> // allocator
-#include <string> // string
-#include <vector> // vector
-
-/*!
-@brief namespace for Niels Lohmann
-@see https://github.com/nlohmann
-@since version 1.0.0
-*/
-namespace nlohmann
-{
-/*!
-@brief default JSONSerializer template argument
-
-This serializer ignores the template arguments and uses ADL
-([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
-for serialization.
-*/
-template<typename T = void, typename SFINAE = void>
-struct adl_serializer;
-
-template<template<typename U, typename V, typename... Args> class ObjectType =
-         std::map,
-         template<typename U, typename... Args> class ArrayType = std::vector,
-         class StringType = std::string, class BooleanType = bool,
-         class NumberIntegerType = std::int64_t,
-         class NumberUnsignedType = std::uint64_t,
-         class NumberFloatType = double,
-         template<typename U> class AllocatorType = std::allocator,
-         template<typename T, typename SFINAE = void> class JSONSerializer =
-         adl_serializer>
-class basic_json;
-
-/*!
-@brief JSON Pointer
-
-A JSON pointer defines a string syntax for identifying a specific value
-within a JSON document. It can be used with functions `at` and
-`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
-
-@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
-
-@since version 2.0.0
-*/
-template<typename BasicJsonType>
-class json_pointer;
-
-/*!
-@brief default JSON class
-
-This type is the default specialization of the @ref basic_json class which
-uses the standard template types.
-
-@since version 1.0.0
-*/
-using json = basic_json<>;
-}  // namespace nlohmann
-
-#endif
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-
-// This file contains all internal macro definitions
-// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them
-
-// exclude unsupported compilers
-#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)
-    #if defined(__clang__)
-        #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
-            #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
-        #endif
-    #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
-        #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800
-            #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
-        #endif
-    #endif
-#endif
-
-// disable float-equal warnings on GCC/clang
-#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wfloat-equal"
-#endif
-
-// disable documentation warnings on clang
-#if defined(__clang__)
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wdocumentation"
-#endif
-
-// allow for portable deprecation warnings
-#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
-    #define JSON_DEPRECATED __attribute__((deprecated))
-#elif defined(_MSC_VER)
-    #define JSON_DEPRECATED __declspec(deprecated)
-#else
-    #define JSON_DEPRECATED
-#endif
-
-// allow to disable exceptions
-#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
-    #define JSON_THROW(exception) throw exception
-    #define JSON_TRY try
-    #define JSON_CATCH(exception) catch(exception)
-    #define JSON_INTERNAL_CATCH(exception) catch(exception)
-#else
-    #define JSON_THROW(exception) std::abort()
-    #define JSON_TRY if(true)
-    #define JSON_CATCH(exception) if(false)
-    #define JSON_INTERNAL_CATCH(exception) if(false)
-#endif
-
-// override exception macros
-#if defined(JSON_THROW_USER)
-    #undef JSON_THROW
-    #define JSON_THROW JSON_THROW_USER
-#endif
-#if defined(JSON_TRY_USER)
-    #undef JSON_TRY
-    #define JSON_TRY JSON_TRY_USER
-#endif
-#if defined(JSON_CATCH_USER)
-    #undef JSON_CATCH
-    #define JSON_CATCH JSON_CATCH_USER
-    #undef JSON_INTERNAL_CATCH
-    #define JSON_INTERNAL_CATCH JSON_CATCH_USER
-#endif
-#if defined(JSON_INTERNAL_CATCH_USER)
-    #undef JSON_INTERNAL_CATCH
-    #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER
-#endif
-
-// manual branch prediction
-#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
-    #define JSON_LIKELY(x)      __builtin_expect(!!(x), 1)
-    #define JSON_UNLIKELY(x)    __builtin_expect(!!(x), 0)
-#else
-    #define JSON_LIKELY(x)      x
-    #define JSON_UNLIKELY(x)    x
-#endif
-
-// C++ language standard detection
-#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
-    #define JSON_HAS_CPP_17
-    #define JSON_HAS_CPP_14
-#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1)
-    #define JSON_HAS_CPP_14
-#endif
-
-/*!
-@brief macro to briefly define a mapping between an enum and JSON
-@def NLOHMANN_JSON_SERIALIZE_ENUM
-@since version 3.4.0
-*/
-#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...)                                           \
-    template<typename BasicJsonType>                                                           \
-    inline void to_json(BasicJsonType& j, const ENUM_TYPE& e)                                  \
-    {                                                                                          \
-        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
-        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
-        auto it = std::find_if(std::begin(m), std::end(m),                                     \
-                               [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
-        {                                                                                      \
-            return ej_pair.first == e;                                                         \
-        });                                                                                    \
-        j = ((it != std::end(m)) ? it : std::begin(m))->second;                                \
-    }                                                                                          \
-    template<typename BasicJsonType>                                                           \
-    inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                \
-    {                                                                                          \
-        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!");         \
-        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                    \
-        auto it = std::find_if(std::begin(m), std::end(m),                                     \
-                               [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \
-        {                                                                                      \
-            return ej_pair.second == j;                                                        \
-        });                                                                                    \
-        e = ((it != std::end(m)) ? it : std::begin(m))->first;                                 \
-    }
-
-// Ugly macros to avoid uglier copy-paste when specializing basic_json. They
-// may be removed in the future once the class is split.
-
-#define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \
-    template<template<typename, typename, typename...> class ObjectType,   \
-             template<typename, typename...> class ArrayType,              \
-             class StringType, class BooleanType, class NumberIntegerType, \
-             class NumberUnsignedType, class NumberFloatType,              \
-             template<typename> class AllocatorType,                       \
-             template<typename, typename = void> class JSONSerializer>
-
-#define NLOHMANN_BASIC_JSON_TPL                                            \
-    basic_json<ObjectType, ArrayType, StringType, BooleanType,             \
-    NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
-    AllocatorType, JSONSerializer>
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-
-#include <ciso646> // not
-#include <cstddef> // size_t
-#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
-
-namespace nlohmann
-{
-namespace detail
-{
-// alias templates to reduce boilerplate
-template<bool B, typename T = void>
-using enable_if_t = typename std::enable_if<B, T>::type;
-
-template<typename T>
-using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
-
-// implementation of C++14 index_sequence and affiliates
-// source: https://stackoverflow.com/a/32223343
-template<std::size_t... Ints>
-struct index_sequence
-{
-    using type = index_sequence;
-    using value_type = std::size_t;
-    static constexpr std::size_t size() noexcept
-    {
-        return sizeof...(Ints);
-    }
-};
-
-template<class Sequence1, class Sequence2>
-struct merge_and_renumber;
-
-template<std::size_t... I1, std::size_t... I2>
-struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
-        : index_sequence < I1..., (sizeof...(I1) + I2)... > {};
-
-template<std::size_t N>
-struct make_index_sequence
-    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
-      typename make_index_sequence < N - N / 2 >::type > {};
-
-template<> struct make_index_sequence<0> : index_sequence<> {};
-template<> struct make_index_sequence<1> : index_sequence<0> {};
-
-template<typename... Ts>
-using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
-
-// dispatch utility (taken from ranges-v3)
-template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
-template<> struct priority_tag<0> {};
-
-// taken from ranges-v3
-template<typename T>
-struct static_const
-{
-    static constexpr T value{};
-};
-
-template<typename T>
-constexpr T static_const<T>::value;
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-
-#include <ciso646> // not
-#include <limits> // numeric_limits
-#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type
-#include <utility> // declval
-
-// #include <nlohmann/json_fwd.hpp>
-
-// #include <nlohmann/detail/iterators/iterator_traits.hpp>
-
-
-#include <iterator> // random_access_iterator_tag
-
-// #include <nlohmann/detail/meta/void_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template <typename ...Ts> struct make_void
-{
-    using type = void;
-};
-template <typename ...Ts> using void_t = typename make_void<Ts...>::type;
-} // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template <typename It, typename = void>
-struct iterator_types {};
-
-template <typename It>
-struct iterator_types <
-    It,
-    void_t<typename It::difference_type, typename It::value_type, typename It::pointer,
-    typename It::reference, typename It::iterator_category >>
-{
-    using difference_type = typename It::difference_type;
-    using value_type = typename It::value_type;
-    using pointer = typename It::pointer;
-    using reference = typename It::reference;
-    using iterator_category = typename It::iterator_category;
-};
-
-// This is required as some compilers implement std::iterator_traits in a way that
-// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.
-template <typename T, typename = void>
-struct iterator_traits
-{
-};
-
-template <typename T>
-struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>
-            : iterator_types<T>
-{
-};
-
-template <typename T>
-struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>
-{
-    using iterator_category = std::random_access_iterator_tag;
-    using value_type = T;
-    using difference_type = ptrdiff_t;
-    using pointer = T*;
-    using reference = T&;
-};
-}
-}
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-// #include <nlohmann/detail/meta/detected.hpp>
-
-
-#include <type_traits>
-
-// #include <nlohmann/detail/meta/void_t.hpp>
-
-
-// http://en.cppreference.com/w/cpp/experimental/is_detected
-namespace nlohmann
-{
-namespace detail
-{
-struct nonesuch
-{
-    nonesuch() = delete;
-    ~nonesuch() = delete;
-    nonesuch(nonesuch const&) = delete;
-    void operator=(nonesuch const&) = delete;
-};
-
-template <class Default,
-          class AlwaysVoid,
-          template <class...> class Op,
-          class... Args>
-struct detector
-{
-    using value_t = std::false_type;
-    using type = Default;
-};
-
-template <class Default, template <class...> class Op, class... Args>
-struct detector<Default, void_t<Op<Args...>>, Op, Args...>
-{
-    using value_t = std::true_type;
-    using type = Op<Args...>;
-};
-
-template <template <class...> class Op, class... Args>
-using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;
-
-template <template <class...> class Op, class... Args>
-using detected_t = typename detector<nonesuch, void, Op, Args...>::type;
-
-template <class Default, template <class...> class Op, class... Args>
-using detected_or = detector<Default, void, Op, Args...>;
-
-template <class Default, template <class...> class Op, class... Args>
-using detected_or_t = typename detected_or<Default, Op, Args...>::type;
-
-template <class Expected, template <class...> class Op, class... Args>
-using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;
-
-template <class To, template <class...> class Op, class... Args>
-using is_detected_convertible =
-    std::is_convertible<detected_t<Op, Args...>, To>;
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-
-namespace nlohmann
-{
-/*!
-@brief detail namespace with internal helper functions
-
-This namespace collects functions that should not be exposed,
-implementations of some @ref basic_json methods, and meta-programming helpers.
-
-@since version 2.1.0
-*/
-namespace detail
-{
-/////////////
-// helpers //
-/////////////
-
-// Note to maintainers:
-//
-// Every trait in this file expects a non CV-qualified type.
-// The only exceptions are in the 'aliases for detected' section
-// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))
-//
-// In this case, T has to be properly CV-qualified to constraint the function arguments
-// (e.g. to_json(BasicJsonType&, const T&))
-
-template<typename> struct is_basic_json : std::false_type {};
-
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
-
-//////////////////////////
-// aliases for detected //
-//////////////////////////
-
-template <typename T>
-using mapped_type_t = typename T::mapped_type;
-
-template <typename T>
-using key_type_t = typename T::key_type;
-
-template <typename T>
-using value_type_t = typename T::value_type;
-
-template <typename T>
-using difference_type_t = typename T::difference_type;
-
-template <typename T>
-using pointer_t = typename T::pointer;
-
-template <typename T>
-using reference_t = typename T::reference;
-
-template <typename T>
-using iterator_category_t = typename T::iterator_category;
-
-template <typename T>
-using iterator_t = typename T::iterator;
-
-template <typename T, typename... Args>
-using to_json_function = decltype(T::to_json(std::declval<Args>()...));
-
-template <typename T, typename... Args>
-using from_json_function = decltype(T::from_json(std::declval<Args>()...));
-
-template <typename T, typename U>
-using get_template_function = decltype(std::declval<T>().template get<U>());
-
-// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
-template <typename BasicJsonType, typename T, typename = void>
-struct has_from_json : std::false_type {};
-
-template <typename BasicJsonType, typename T>
-struct has_from_json<BasicJsonType, T,
-           enable_if_t<not is_basic_json<T>::value>>
-{
-    using serializer = typename BasicJsonType::template json_serializer<T, void>;
-
-    static constexpr bool value =
-        is_detected_exact<void, from_json_function, serializer,
-        const BasicJsonType&, T&>::value;
-};
-
-// This trait checks if JSONSerializer<T>::from_json(json const&) exists
-// this overload is used for non-default-constructible user-defined-types
-template <typename BasicJsonType, typename T, typename = void>
-struct has_non_default_from_json : std::false_type {};
-
-template<typename BasicJsonType, typename T>
-struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
-{
-    using serializer = typename BasicJsonType::template json_serializer<T, void>;
-
-    static constexpr bool value =
-        is_detected_exact<T, from_json_function, serializer,
-        const BasicJsonType&>::value;
-};
-
-// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
-// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.
-template <typename BasicJsonType, typename T, typename = void>
-struct has_to_json : std::false_type {};
-
-template <typename BasicJsonType, typename T>
-struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>>
-{
-    using serializer = typename BasicJsonType::template json_serializer<T, void>;
-
-    static constexpr bool value =
-        is_detected_exact<void, to_json_function, serializer, BasicJsonType&,
-        T>::value;
-};
-
-
-///////////////////
-// is_ functions //
-///////////////////
-
-template <typename T, typename = void>
-struct is_iterator_traits : std::false_type {};
-
-template <typename T>
-struct is_iterator_traits<iterator_traits<T>>
-{
-  private:
-    using traits = iterator_traits<T>;
-
-  public:
-    static constexpr auto value =
-        is_detected<value_type_t, traits>::value &&
-        is_detected<difference_type_t, traits>::value &&
-        is_detected<pointer_t, traits>::value &&
-        is_detected<iterator_category_t, traits>::value &&
-        is_detected<reference_t, traits>::value;
-};
-
-// source: https://stackoverflow.com/a/37193089/4116453
-
-template <typename T, typename = void>
-struct is_complete_type : std::false_type {};
-
-template <typename T>
-struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};
-
-template <typename BasicJsonType, typename CompatibleObjectType,
-          typename = void>
-struct is_compatible_object_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename CompatibleObjectType>
-struct is_compatible_object_type_impl <
-    BasicJsonType, CompatibleObjectType,
-    enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and
-    is_detected<key_type_t, CompatibleObjectType>::value >>
-{
-
-    using object_t = typename BasicJsonType::object_t;
-
-    // macOS's is_constructible does not play well with nonesuch...
-    static constexpr bool value =
-        std::is_constructible<typename object_t::key_type,
-        typename CompatibleObjectType::key_type>::value and
-        std::is_constructible<typename object_t::mapped_type,
-        typename CompatibleObjectType::mapped_type>::value;
-};
-
-template <typename BasicJsonType, typename CompatibleObjectType>
-struct is_compatible_object_type
-    : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};
-
-template <typename BasicJsonType, typename ConstructibleObjectType,
-          typename = void>
-struct is_constructible_object_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename ConstructibleObjectType>
-struct is_constructible_object_type_impl <
-    BasicJsonType, ConstructibleObjectType,
-    enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and
-    is_detected<key_type_t, ConstructibleObjectType>::value >>
-{
-    using object_t = typename BasicJsonType::object_t;
-
-    static constexpr bool value =
-        (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and
-         std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or
-        (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or
-         has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value);
-};
-
-template <typename BasicJsonType, typename ConstructibleObjectType>
-struct is_constructible_object_type
-    : is_constructible_object_type_impl<BasicJsonType,
-      ConstructibleObjectType> {};
-
-template <typename BasicJsonType, typename CompatibleStringType,
-          typename = void>
-struct is_compatible_string_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename CompatibleStringType>
-struct is_compatible_string_type_impl <
-    BasicJsonType, CompatibleStringType,
-    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
-    value_type_t, CompatibleStringType>::value >>
-{
-    static constexpr auto value =
-        std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;
-};
-
-template <typename BasicJsonType, typename ConstructibleStringType>
-struct is_compatible_string_type
-    : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
-
-template <typename BasicJsonType, typename ConstructibleStringType,
-          typename = void>
-struct is_constructible_string_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename ConstructibleStringType>
-struct is_constructible_string_type_impl <
-    BasicJsonType, ConstructibleStringType,
-    enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type,
-    value_type_t, ConstructibleStringType>::value >>
-{
-    static constexpr auto value =
-        std::is_constructible<ConstructibleStringType,
-        typename BasicJsonType::string_t>::value;
-};
-
-template <typename BasicJsonType, typename ConstructibleStringType>
-struct is_constructible_string_type
-    : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {};
-
-template <typename BasicJsonType, typename CompatibleArrayType, typename = void>
-struct is_compatible_array_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename CompatibleArrayType>
-struct is_compatible_array_type_impl <
-    BasicJsonType, CompatibleArrayType,
-    enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and
-    is_detected<iterator_t, CompatibleArrayType>::value and
-// This is needed because json_reverse_iterator has a ::iterator type...
-// Therefore it is detected as a CompatibleArrayType.
-// The real fix would be to have an Iterable concept.
-    not is_iterator_traits<
-    iterator_traits<CompatibleArrayType>>::value >>
-{
-    static constexpr bool value =
-        std::is_constructible<BasicJsonType,
-        typename CompatibleArrayType::value_type>::value;
-};
-
-template <typename BasicJsonType, typename CompatibleArrayType>
-struct is_compatible_array_type
-    : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};
-
-template <typename BasicJsonType, typename ConstructibleArrayType, typename = void>
-struct is_constructible_array_type_impl : std::false_type {};
-
-template <typename BasicJsonType, typename ConstructibleArrayType>
-struct is_constructible_array_type_impl <
-    BasicJsonType, ConstructibleArrayType,
-    enable_if_t<std::is_same<ConstructibleArrayType,
-    typename BasicJsonType::value_type>::value >>
-            : std::true_type {};
-
-template <typename BasicJsonType, typename ConstructibleArrayType>
-struct is_constructible_array_type_impl <
-    BasicJsonType, ConstructibleArrayType,
-    enable_if_t<not std::is_same<ConstructibleArrayType,
-    typename BasicJsonType::value_type>::value and
-    is_detected<value_type_t, ConstructibleArrayType>::value and
-    is_detected<iterator_t, ConstructibleArrayType>::value and
-    is_complete_type<
-    detected_t<value_type_t, ConstructibleArrayType>>::value >>
-{
-    static constexpr bool value =
-        // This is needed because json_reverse_iterator has a ::iterator type,
-        // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`...
-        // Therefore it is detected as a ConstructibleArrayType.
-        // The real fix would be to have an Iterable concept.
-        not is_iterator_traits <
-        iterator_traits<ConstructibleArrayType >>::value and
-
-        (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or
-         has_from_json<BasicJsonType,
-         typename ConstructibleArrayType::value_type>::value or
-         has_non_default_from_json <
-         BasicJsonType, typename ConstructibleArrayType::value_type >::value);
-};
-
-template <typename BasicJsonType, typename ConstructibleArrayType>
-struct is_constructible_array_type
-    : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};
-
-template <typename RealIntegerType, typename CompatibleNumberIntegerType,
-          typename = void>
-struct is_compatible_integer_type_impl : std::false_type {};
-
-template <typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type_impl <
-    RealIntegerType, CompatibleNumberIntegerType,
-    enable_if_t<std::is_integral<RealIntegerType>::value and
-    std::is_integral<CompatibleNumberIntegerType>::value and
-    not std::is_same<bool, CompatibleNumberIntegerType>::value >>
-{
-    // is there an assert somewhere on overflows?
-    using RealLimits = std::numeric_limits<RealIntegerType>;
-    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
-
-    static constexpr auto value =
-        std::is_constructible<RealIntegerType,
-        CompatibleNumberIntegerType>::value and
-        CompatibleLimits::is_integer and
-        RealLimits::is_signed == CompatibleLimits::is_signed;
-};
-
-template <typename RealIntegerType, typename CompatibleNumberIntegerType>
-struct is_compatible_integer_type
-    : is_compatible_integer_type_impl<RealIntegerType,
-      CompatibleNumberIntegerType> {};
-
-template <typename BasicJsonType, typename CompatibleType, typename = void>
-struct is_compatible_type_impl: std::false_type {};
-
-template <typename BasicJsonType, typename CompatibleType>
-struct is_compatible_type_impl <
-    BasicJsonType, CompatibleType,
-    enable_if_t<is_complete_type<CompatibleType>::value >>
-{
-    static constexpr bool value =
-        has_to_json<BasicJsonType, CompatibleType>::value;
-};
-
-template <typename BasicJsonType, typename CompatibleType>
-struct is_compatible_type
-    : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-
-#include <exception> // exception
-#include <stdexcept> // runtime_error
-#include <string> // to_string
-
-// #include <nlohmann/detail/input/position_t.hpp>
-
-
-#include <cstddef> // size_t
-
-namespace nlohmann
-{
-namespace detail
-{
-/// struct to capture the start position of the current token
-struct position_t
-{
-    /// the total number of characters read
-    std::size_t chars_read_total = 0;
-    /// the number of characters read in the current line
-    std::size_t chars_read_current_line = 0;
-    /// the number of lines read
-    std::size_t lines_read = 0;
-
-    /// conversion to size_t to preserve SAX interface
-    constexpr operator size_t() const
-    {
-        return chars_read_total;
-    }
-};
-
-}
-}
-
-
-namespace nlohmann
-{
-namespace detail
-{
-////////////////
-// exceptions //
-////////////////
-
-/*!
-@brief general exception of the @ref basic_json class
-
-This class is an extension of `std::exception` objects with a member @a id for
-exception ids. It is used as the base class for all exceptions thrown by the
-@ref basic_json class. This class can hence be used as "wildcard" to catch
-exceptions.
-
-Subclasses:
-- @ref parse_error for exceptions indicating a parse error
-- @ref invalid_iterator for exceptions indicating errors with iterators
-- @ref type_error for exceptions indicating executing a member function with
-                  a wrong type
-- @ref out_of_range for exceptions indicating access out of the defined range
-- @ref other_error for exceptions indicating other library errors
-
-@internal
-@note To have nothrow-copy-constructible exceptions, we internally use
-      `std::runtime_error` which can cope with arbitrary-length error messages.
-      Intermediate strings are built with static functions and then passed to
-      the actual constructor.
-@endinternal
-
-@liveexample{The following code shows how arbitrary library exceptions can be
-caught.,exception}
-
-@since version 3.0.0
-*/
-class exception : public std::exception
-{
-  public:
-    /// returns the explanatory string
-    const char* what() const noexcept override
-    {
-        return m.what();
-    }
-
-    /// the id of the exception
-    const int id;
-
-  protected:
-    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
-
-    static std::string name(const std::string& ename, int id_)
-    {
-        return "[json.exception." + ename + "." + std::to_string(id_) + "] ";
-    }
-
-  private:
-    /// an exception object as storage for error messages
-    std::runtime_error m;
-};
-
-/*!
-@brief exception indicating a parse error
-
-This exception is thrown by the library when a parse error occurs. Parse errors
-can occur during the deserialization of JSON text, CBOR, MessagePack, as well
-as when using JSON Patch.
-
-Member @a byte holds the byte index of the last read character in the input
-file.
-
-Exceptions have ids 1xx.
-
-name / id                      | example message | description
------------------------------- | --------------- | -------------------------
-json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
-json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
-json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
-json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
-json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
-json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`.
-json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
-json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
-json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
-json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
-json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
-json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
-json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet).
-
-@note For an input with n bytes, 1 is the index of the first character and n+1
-      is the index of the terminating null byte or the end of file. This also
-      holds true when reading a byte vector (CBOR or MessagePack).
-
-@liveexample{The following code shows how a `parse_error` exception can be
-caught.,parse_error}
-
-@sa @ref exception for the base class of the library exceptions
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
-
-@since version 3.0.0
-*/
-class parse_error : public exception
-{
-  public:
-    /*!
-    @brief create a parse error exception
-    @param[in] id_       the id of the exception
-    @param[in] position  the position where the error occurred (or with
-                         chars_read_total=0 if the position cannot be
-                         determined)
-    @param[in] what_arg  the explanatory string
-    @return parse_error object
-    */
-    static parse_error create(int id_, const position_t& pos, const std::string& what_arg)
-    {
-        std::string w = exception::name("parse_error", id_) + "parse error" +
-                        position_string(pos) + ": " + what_arg;
-        return parse_error(id_, pos.chars_read_total, w.c_str());
-    }
-
-    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg)
-    {
-        std::string w = exception::name("parse_error", id_) + "parse error" +
-                        (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") +
-                        ": " + what_arg;
-        return parse_error(id_, byte_, w.c_str());
-    }
-
-    /*!
-    @brief byte index of the parse error
-
-    The byte index of the last read character in the input file.
-
-    @note For an input with n bytes, 1 is the index of the first character and
-          n+1 is the index of the terminating null byte or the end of file.
-          This also holds true when reading a byte vector (CBOR or MessagePack).
-    */
-    const std::size_t byte;
-
-  private:
-    parse_error(int id_, std::size_t byte_, const char* what_arg)
-        : exception(id_, what_arg), byte(byte_) {}
-
-    static std::string position_string(const position_t& pos)
-    {
-        return " at line " + std::to_string(pos.lines_read + 1) +
-               ", column " + std::to_string(pos.chars_read_current_line);
-    }
-};
-
-/*!
-@brief exception indicating errors with iterators
-
-This exception is thrown if iterators passed to a library function do not match
-the expected semantics.
-
-Exceptions have ids 2xx.
-
-name / id                           | example message | description
------------------------------------ | --------------- | -------------------------
-json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
-json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
-json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
-json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
-json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
-json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
-json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
-json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
-json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
-json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
-json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
-json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
-json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
-json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
-
-@liveexample{The following code shows how an `invalid_iterator` exception can be
-caught.,invalid_iterator}
-
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
-
-@since version 3.0.0
-*/
-class invalid_iterator : public exception
-{
-  public:
-    static invalid_iterator create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("invalid_iterator", id_) + what_arg;
-        return invalid_iterator(id_, w.c_str());
-    }
-
-  private:
-    invalid_iterator(int id_, const char* what_arg)
-        : exception(id_, what_arg) {}
-};
-
-/*!
-@brief exception indicating executing a member function with a wrong type
-
-This exception is thrown in case of a type error; that is, a library function is
-executed on a JSON value whose type does not match the expected semantics.
-
-Exceptions have ids 3xx.
-
-name / id                     | example message | description
------------------------------ | --------------- | -------------------------
-json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
-json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
-json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
-json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
-json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
-json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
-json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
-json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
-json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
-json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
-json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
-json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
-json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
-json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
-json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
-json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. |
-json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) |
-
-@liveexample{The following code shows how a `type_error` exception can be
-caught.,type_error}
-
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-@sa @ref other_error for exceptions indicating other library errors
-
-@since version 3.0.0
-*/
-class type_error : public exception
-{
-  public:
-    static type_error create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("type_error", id_) + what_arg;
-        return type_error(id_, w.c_str());
-    }
-
-  private:
-    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
-};
-
-/*!
-@brief exception indicating access out of the defined range
-
-This exception is thrown in case a library function is called on an input
-parameter that exceeds the expected range, for instance in case of array
-indices or nonexisting object keys.
-
-Exceptions have ids 4xx.
-
-name / id                       | example message | description
-------------------------------- | --------------- | -------------------------
-json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
-json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
-json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
-json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
-json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
-json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
-json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. |
-json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. |
-json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string |
-
-@liveexample{The following code shows how an `out_of_range` exception can be
-caught.,out_of_range}
-
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref other_error for exceptions indicating other library errors
-
-@since version 3.0.0
-*/
-class out_of_range : public exception
-{
-  public:
-    static out_of_range create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("out_of_range", id_) + what_arg;
-        return out_of_range(id_, w.c_str());
-    }
-
-  private:
-    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
-};
-
-/*!
-@brief exception indicating other library errors
-
-This exception is thrown in case of errors that cannot be classified with the
-other exception types.
-
-Exceptions have ids 5xx.
-
-name / id                      | example message | description
------------------------------- | --------------- | -------------------------
-json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
-
-@sa @ref exception for the base class of the library exceptions
-@sa @ref parse_error for exceptions indicating a parse error
-@sa @ref invalid_iterator for exceptions indicating errors with iterators
-@sa @ref type_error for exceptions indicating executing a member function with
-                    a wrong type
-@sa @ref out_of_range for exceptions indicating access out of the defined range
-
-@liveexample{The following code shows how an `other_error` exception can be
-caught.,other_error}
-
-@since version 3.0.0
-*/
-class other_error : public exception
-{
-  public:
-    static other_error create(int id_, const std::string& what_arg)
-    {
-        std::string w = exception::name("other_error", id_) + what_arg;
-        return other_error(id_, w.c_str());
-    }
-
-  private:
-    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-#include <array> // array
-#include <ciso646> // and
-#include <cstddef> // size_t
-#include <cstdint> // uint8_t
-
-namespace nlohmann
-{
-namespace detail
-{
-///////////////////////////
-// JSON type enumeration //
-///////////////////////////
-
-/*!
-@brief the JSON type enumeration
-
-This enumeration collects the different JSON types. It is internally used to
-distinguish the stored values, and the functions @ref basic_json::is_null(),
-@ref basic_json::is_object(), @ref basic_json::is_array(),
-@ref basic_json::is_string(), @ref basic_json::is_boolean(),
-@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
-@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
-@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
-@ref basic_json::is_structured() rely on it.
-
-@note There are three enumeration entries (number_integer, number_unsigned, and
-number_float), because the library distinguishes these three types for numbers:
-@ref basic_json::number_unsigned_t is used for unsigned integers,
-@ref basic_json::number_integer_t is used for signed integers, and
-@ref basic_json::number_float_t is used for floating-point numbers or to
-approximate integers which do not fit in the limits of their respective type.
-
-@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
-value with the default value for a given type
-
-@since version 1.0.0
-*/
-enum class value_t : std::uint8_t
-{
-    null,             ///< null value
-    object,           ///< object (unordered set of name/value pairs)
-    array,            ///< array (ordered collection of values)
-    string,           ///< string value
-    boolean,          ///< boolean value
-    number_integer,   ///< number value (signed integer)
-    number_unsigned,  ///< number value (unsigned integer)
-    number_float,     ///< number value (floating-point)
-    discarded         ///< discarded by the the parser callback function
-};
-
-/*!
-@brief comparison operator for JSON types
-
-Returns an ordering that is similar to Python:
-- order: null < boolean < number < object < array < string
-- furthermore, each type is not smaller than itself
-- discarded values are not comparable
-
-@since version 1.0.0
-*/
-inline bool operator<(const value_t lhs, const value_t rhs) noexcept
-{
-    static constexpr std::array<std::uint8_t, 8> order = {{
-            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
-            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
-        }
-    };
-
-    const auto l_index = static_cast<std::size_t>(lhs);
-    const auto r_index = static_cast<std::size_t>(rhs);
-    return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index];
-}
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/conversions/from_json.hpp>
-
-
-#include <algorithm> // transform
-#include <array> // array
-#include <ciso646> // and, not
-#include <forward_list> // forward_list
-#include <iterator> // inserter, front_inserter, end
-#include <map> // map
-#include <string> // string
-#include <tuple> // tuple, make_tuple
-#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible
-#include <unordered_map> // unordered_map
-#include <utility> // pair, declval
-#include <valarray> // valarray
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename std::nullptr_t& n)
-{
-    if (JSON_UNLIKELY(not j.is_null()))
-    {
-        JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name())));
-    }
-    n = nullptr;
-}
-
-// overloads for basic_json template parameters
-template<typename BasicJsonType, typename ArithmeticType,
-         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
-                     not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
-                     int> = 0>
-void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
-{
-    switch (static_cast<value_t>(j))
-    {
-        case value_t::number_unsigned:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
-            break;
-        }
-        case value_t::number_integer:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
-            break;
-        }
-        case value_t::number_float:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
-            break;
-        }
-
-        default:
-            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
-    }
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
-{
-    if (JSON_UNLIKELY(not j.is_boolean()))
-    {
-        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
-    }
-    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
-{
-    if (JSON_UNLIKELY(not j.is_string()))
-    {
-        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
-    }
-    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
-}
-
-template <
-    typename BasicJsonType, typename ConstructibleStringType,
-    enable_if_t <
-        is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and
-        not std::is_same<typename BasicJsonType::string_t,
-                         ConstructibleStringType>::value,
-        int > = 0 >
-void from_json(const BasicJsonType& j, ConstructibleStringType& s)
-{
-    if (JSON_UNLIKELY(not j.is_string()))
-    {
-        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
-    }
-
-    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType>
-void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
-{
-    get_arithmetic_value(j, val);
-}
-
-template<typename BasicJsonType, typename EnumType,
-         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
-void from_json(const BasicJsonType& j, EnumType& e)
-{
-    typename std::underlying_type<EnumType>::type val;
-    get_arithmetic_value(j, val);
-    e = static_cast<EnumType>(val);
-}
-
-// forward_list doesn't have an insert method
-template<typename BasicJsonType, typename T, typename Allocator,
-         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
-void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    std::transform(j.rbegin(), j.rend(),
-                   std::front_inserter(l), [](const BasicJsonType & i)
-    {
-        return i.template get<T>();
-    });
-}
-
-// valarray doesn't have an insert method
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
-void from_json(const BasicJsonType& j, std::valarray<T>& l)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    l.resize(j.size());
-    std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l));
-}
-
-template<typename BasicJsonType>
-void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)
-{
-    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
-}
-
-template <typename BasicJsonType, typename T, std::size_t N>
-auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
-                          priority_tag<2> /*unused*/)
--> decltype(j.template get<T>(), void())
-{
-    for (std::size_t i = 0; i < N; ++i)
-    {
-        arr[i] = j.at(i).template get<T>();
-    }
-}
-
-template<typename BasicJsonType, typename ConstructibleArrayType>
-auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
--> decltype(
-    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
-    j.template get<typename ConstructibleArrayType::value_type>(),
-    void())
-{
-    using std::end;
-
-    arr.reserve(j.size());
-    std::transform(j.begin(), j.end(),
-                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
-    {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename ConstructibleArrayType::value_type>();
-    });
-}
-
-template <typename BasicJsonType, typename ConstructibleArrayType>
-void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
-                          priority_tag<0> /*unused*/)
-{
-    using std::end;
-
-    std::transform(
-        j.begin(), j.end(), std::inserter(arr, end(arr)),
-        [](const BasicJsonType & i)
-    {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename ConstructibleArrayType::value_type>();
-    });
-}
-
-template <typename BasicJsonType, typename ConstructibleArrayType,
-          enable_if_t <
-              is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and
-              not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and
-              not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and
-              not is_basic_json<ConstructibleArrayType>::value,
-              int > = 0 >
-
-auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
--> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
-j.template get<typename ConstructibleArrayType::value_type>(),
-void())
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " +
-                                      std::string(j.type_name())));
-    }
-
-    from_json_array_impl(j, arr, priority_tag<3> {});
-}
-
-template<typename BasicJsonType, typename ConstructibleObjectType,
-         enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
-void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
-{
-    if (JSON_UNLIKELY(not j.is_object()))
-    {
-        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
-    }
-
-    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
-    using value_type = typename ConstructibleObjectType::value_type;
-    std::transform(
-        inner_object->begin(), inner_object->end(),
-        std::inserter(obj, obj.begin()),
-        [](typename BasicJsonType::object_t::value_type const & p)
-    {
-        return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
-    });
-}
-
-// overload for arithmetic types, not chosen for basic_json template arguments
-// (BooleanType, etc..); note: Is it really necessary to provide explicit
-// overloads for boolean_t etc. in case of a custom BooleanType which is not
-// an arithmetic type?
-template<typename BasicJsonType, typename ArithmeticType,
-         enable_if_t <
-             std::is_arithmetic<ArithmeticType>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
-             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
-             int> = 0>
-void from_json(const BasicJsonType& j, ArithmeticType& val)
-{
-    switch (static_cast<value_t>(j))
-    {
-        case value_t::number_unsigned:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
-            break;
-        }
-        case value_t::number_integer:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
-            break;
-        }
-        case value_t::number_float:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
-            break;
-        }
-        case value_t::boolean:
-        {
-            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
-            break;
-        }
-
-        default:
-            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
-    }
-}
-
-template<typename BasicJsonType, typename A1, typename A2>
-void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
-{
-    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
-}
-
-template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/)
-{
-    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
-}
-
-template<typename BasicJsonType, typename... Args>
-void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
-{
-    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
-}
-
-template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
-          typename = enable_if_t<not std::is_constructible<
-                                     typename BasicJsonType::string_t, Key>::value>>
-void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    for (const auto& p : j)
-    {
-        if (JSON_UNLIKELY(not p.is_array()))
-        {
-            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
-        }
-        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
-    }
-}
-
-template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
-          typename = enable_if_t<not std::is_constructible<
-                                     typename BasicJsonType::string_t, Key>::value>>
-void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
-{
-    if (JSON_UNLIKELY(not j.is_array()))
-    {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
-    }
-    for (const auto& p : j)
-    {
-        if (JSON_UNLIKELY(not p.is_array()))
-        {
-            JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name())));
-        }
-        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
-    }
-}
-
-struct from_json_fn
-{
-    template<typename BasicJsonType, typename T>
-    auto operator()(const BasicJsonType& j, T& val) const
-    noexcept(noexcept(from_json(j, val)))
-    -> decltype(from_json(j, val), void())
-    {
-        return from_json(j, val);
-    }
-};
-}  // namespace detail
-
-/// namespace to hold default `from_json` function
-/// to see why this is required:
-/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html
-namespace
-{
-constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
-} // namespace
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/conversions/to_json.hpp>
-
-
-#include <ciso646> // or, and, not
-#include <iterator> // begin, end
-#include <tuple> // tuple, get
-#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type
-#include <utility> // move, forward, declval, pair
-#include <valarray> // valarray
-#include <vector> // vector
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
-
-
-#include <cstddef> // size_t
-#include <string> // string, to_string
-#include <iterator> // input_iterator_tag
-#include <tuple> // tuple_size, get, tuple_element
-
-// #include <nlohmann/detail/value_t.hpp>
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template <typename IteratorType> class iteration_proxy_value
-{
-  public:
-    using difference_type = std::ptrdiff_t;
-    using value_type = iteration_proxy_value;
-    using pointer = value_type * ;
-    using reference = value_type & ;
-    using iterator_category = std::input_iterator_tag;
-
-  private:
-    /// the iterator
-    IteratorType anchor;
-    /// an index for arrays (used to create key names)
-    std::size_t array_index = 0;
-    /// last stringified array index
-    mutable std::size_t array_index_last = 0;
-    /// a string representation of the array index
-    mutable std::string array_index_str = "0";
-    /// an empty string (to return a reference for primitive values)
-    const std::string empty_str = "";
-
-  public:
-    explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {}
-
-    /// dereference operator (needed for range-based for)
-    iteration_proxy_value& operator*()
-    {
-        return *this;
-    }
-
-    /// increment operator (needed for range-based for)
-    iteration_proxy_value& operator++()
-    {
-        ++anchor;
-        ++array_index;
-
-        return *this;
-    }
-
-    /// equality operator (needed for InputIterator)
-    bool operator==(const iteration_proxy_value& o) const noexcept
-    {
-        return anchor == o.anchor;
-    }
-
-    /// inequality operator (needed for range-based for)
-    bool operator!=(const iteration_proxy_value& o) const noexcept
-    {
-        return anchor != o.anchor;
-    }
-
-    /// return key of the iterator
-    const std::string& key() const
-    {
-        assert(anchor.m_object != nullptr);
-
-        switch (anchor.m_object->type())
-        {
-            // use integer array index as key
-            case value_t::array:
-            {
-                if (array_index != array_index_last)
-                {
-                    array_index_str = std::to_string(array_index);
-                    array_index_last = array_index;
-                }
-                return array_index_str;
-            }
-
-            // use key from the object
-            case value_t::object:
-                return anchor.key();
-
-            // use an empty key for all primitive types
-            default:
-                return empty_str;
-        }
-    }
-
-    /// return value of the iterator
-    typename IteratorType::reference value() const
-    {
-        return anchor.value();
-    }
-};
-
-/// proxy class for the items() function
-template<typename IteratorType> class iteration_proxy
-{
-  private:
-    /// the container to iterate
-    typename IteratorType::reference container;
-
-  public:
-    /// construct iteration proxy from a container
-    explicit iteration_proxy(typename IteratorType::reference cont) noexcept
-        : container(cont) {}
-
-    /// return iterator begin (needed for range-based for)
-    iteration_proxy_value<IteratorType> begin() noexcept
-    {
-        return iteration_proxy_value<IteratorType>(container.begin());
-    }
-
-    /// return iterator end (needed for range-based for)
-    iteration_proxy_value<IteratorType> end() noexcept
-    {
-        return iteration_proxy_value<IteratorType>(container.end());
-    }
-};
-// Structured Bindings Support
-// For further reference see https://blog.tartanllama.xyz/structured-bindings/
-// And see https://github.com/nlohmann/json/pull/1391
-template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>
-auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())
-{
-    return i.key();
-}
-// Structured Bindings Support
-// For further reference see https://blog.tartanllama.xyz/structured-bindings/
-// And see https://github.com/nlohmann/json/pull/1391
-template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>
-auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())
-{
-    return i.value();
-}
-}  // namespace detail
-}  // namespace nlohmann
-
-// The Addition to the STD Namespace is required to add
-// Structured Bindings Support to the iteration_proxy_value class
-// For further reference see https://blog.tartanllama.xyz/structured-bindings/
-// And see https://github.com/nlohmann/json/pull/1391
-namespace std
-{
-template <typename IteratorType>
-class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>
-            : public std::integral_constant<std::size_t, 2> {};
-
-template <std::size_t N, typename IteratorType>
-class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>
-{
-  public:
-    using type = decltype(
-                     get<N>(std::declval <
-                            ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));
-};
-}
-
-namespace nlohmann
-{
-namespace detail
-{
-//////////////////
-// constructors //
-//////////////////
-
-template<value_t> struct external_constructor;
-
-template<>
-struct external_constructor<value_t::boolean>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
-    {
-        j.m_type = value_t::boolean;
-        j.m_value = b;
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::string>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
-    {
-        j.m_type = value_t::string;
-        j.m_value = s;
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
-    {
-        j.m_type = value_t::string;
-        j.m_value = std::move(s);
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType, typename CompatibleStringType,
-             enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,
-                         int> = 0>
-    static void construct(BasicJsonType& j, const CompatibleStringType& str)
-    {
-        j.m_type = value_t::string;
-        j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::number_float>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
-    {
-        j.m_type = value_t::number_float;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::number_unsigned>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
-    {
-        j.m_type = value_t::number_unsigned;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::number_integer>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
-    {
-        j.m_type = value_t::number_integer;
-        j.m_value = val;
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::array>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = arr;
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = std::move(arr);
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType, typename CompatibleArrayType,
-             enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,
-                         int> = 0>
-    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
-    {
-        using std::begin;
-        using std::end;
-        j.m_type = value_t::array;
-        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = value_t::array;
-        j.m_value.array->reserve(arr.size());
-        for (const bool x : arr)
-        {
-            j.m_value.array->push_back(x);
-        }
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType, typename T,
-             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
-    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
-    {
-        j.m_type = value_t::array;
-        j.m_value = value_t::array;
-        j.m_value.array->resize(arr.size());
-        std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());
-        j.assert_invariant();
-    }
-};
-
-template<>
-struct external_constructor<value_t::object>
-{
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
-    {
-        j.m_type = value_t::object;
-        j.m_value = obj;
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType>
-    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
-    {
-        j.m_type = value_t::object;
-        j.m_value = std::move(obj);
-        j.assert_invariant();
-    }
-
-    template<typename BasicJsonType, typename CompatibleObjectType,
-             enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0>
-    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
-    {
-        using std::begin;
-        using std::end;
-
-        j.m_type = value_t::object;
-        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
-        j.assert_invariant();
-    }
-};
-
-/////////////
-// to_json //
-/////////////
-
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
-void to_json(BasicJsonType& j, T b) noexcept
-{
-    external_constructor<value_t::boolean>::construct(j, b);
-}
-
-template<typename BasicJsonType, typename CompatibleString,
-         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>
-void to_json(BasicJsonType& j, const CompatibleString& s)
-{
-    external_constructor<value_t::string>::construct(j, s);
-}
-
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
-{
-    external_constructor<value_t::string>::construct(j, std::move(s));
-}
-
-template<typename BasicJsonType, typename FloatType,
-         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
-void to_json(BasicJsonType& j, FloatType val) noexcept
-{
-    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
-}
-
-template<typename BasicJsonType, typename CompatibleNumberUnsignedType,
-         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>
-void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
-{
-    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
-}
-
-template<typename BasicJsonType, typename CompatibleNumberIntegerType,
-         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>
-void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
-{
-    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
-}
-
-template<typename BasicJsonType, typename EnumType,
-         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
-void to_json(BasicJsonType& j, EnumType e) noexcept
-{
-    using underlying_type = typename std::underlying_type<EnumType>::type;
-    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
-}
-
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, const std::vector<bool>& e)
-{
-    external_constructor<value_t::array>::construct(j, e);
-}
-
-template <typename BasicJsonType, typename CompatibleArrayType,
-          enable_if_t<is_compatible_array_type<BasicJsonType,
-                      CompatibleArrayType>::value and
-                      not is_compatible_object_type<
-                          BasicJsonType, CompatibleArrayType>::value and
-                      not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and
-                      not is_basic_json<CompatibleArrayType>::value,
-                      int> = 0>
-void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
-{
-    external_constructor<value_t::array>::construct(j, arr);
-}
-
-template<typename BasicJsonType, typename T,
-         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
-void to_json(BasicJsonType& j, const std::valarray<T>& arr)
-{
-    external_constructor<value_t::array>::construct(j, std::move(arr));
-}
-
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
-{
-    external_constructor<value_t::array>::construct(j, std::move(arr));
-}
-
-template<typename BasicJsonType, typename CompatibleObjectType,
-         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0>
-void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
-{
-    external_constructor<value_t::object>::construct(j, obj);
-}
-
-template<typename BasicJsonType>
-void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
-{
-    external_constructor<value_t::object>::construct(j, std::move(obj));
-}
-
-template <
-    typename BasicJsonType, typename T, std::size_t N,
-    enable_if_t<not std::is_constructible<typename BasicJsonType::string_t,
-                const T(&)[N]>::value,
-                int> = 0 >
-void to_json(BasicJsonType& j, const T(&arr)[N])
-{
-    external_constructor<value_t::array>::construct(j, arr);
-}
-
-template<typename BasicJsonType, typename... Args>
-void to_json(BasicJsonType& j, const std::pair<Args...>& p)
-{
-    j = { p.first, p.second };
-}
-
-// for https://github.com/nlohmann/json/pull/1134
-template < typename BasicJsonType, typename T,
-           enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>
-void to_json(BasicJsonType& j, const T& b)
-{
-    j = { {b.key(), b.value()} };
-}
-
-template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
-void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)
-{
-    j = { std::get<Idx>(t)... };
-}
-
-template<typename BasicJsonType, typename... Args>
-void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
-{
-    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
-}
-
-struct to_json_fn
-{
-    template<typename BasicJsonType, typename T>
-    auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
-    -> decltype(to_json(j, std::forward<T>(val)), void())
-    {
-        return to_json(j, std::forward<T>(val));
-    }
-};
-}  // namespace detail
-
-/// namespace to hold default `to_json` function
-namespace
-{
-constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
-} // namespace
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/input_adapters.hpp>
-
-
-#include <cassert> // assert
-#include <cstddef> // size_t
-#include <cstring> // strlen
-#include <istream> // istream
-#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
-#include <memory> // shared_ptr, make_shared, addressof
-#include <numeric> // accumulate
-#include <string> // string, char_traits
-#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
-#include <utility> // pair, declval
-#include <cstdio> //FILE *
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-/// the supported input formats
-enum class input_format_t { json, cbor, msgpack, ubjson, bson };
-
-////////////////////
-// input adapters //
-////////////////////
-
-/*!
-@brief abstract input adapter interface
-
-Produces a stream of std::char_traits<char>::int_type characters from a
-std::istream, a buffer, or some other input type. Accepts the return of
-exactly one non-EOF character for future input. The int_type characters
-returned consist of all valid char values as positive values (typically
-unsigned char), plus an EOF value outside that range, specified by the value
-of the function std::char_traits<char>::eof(). This value is typically -1, but
-could be any arbitrary value which is not a valid char value.
-*/
-struct input_adapter_protocol
-{
-    /// get a character [0,255] or std::char_traits<char>::eof().
-    virtual std::char_traits<char>::int_type get_character() = 0;
-    virtual ~input_adapter_protocol() = default;
-};
-
-/// a type to simplify interfaces
-using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
-
-/*!
-Input adapter for stdio file access. This adapter read only 1 byte and do not use any
- buffer. This adapter is a very low level adapter.
-*/
-class file_input_adapter : public input_adapter_protocol
-{
-  public:
-    explicit file_input_adapter(std::FILE* f)  noexcept
-        : m_file(f)
-    {}
-
-    std::char_traits<char>::int_type get_character() noexcept override
-    {
-        return std::fgetc(m_file);
-    }
-  private:
-    /// the file pointer to read from
-    std::FILE* m_file;
-};
-
-
-/*!
-Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
-beginning of input. Does not support changing the underlying std::streambuf
-in mid-input. Maintains underlying std::istream and std::streambuf to support
-subsequent use of standard std::istream operations to process any input
-characters following those used in parsing the JSON input.  Clears the
-std::istream flags; any input errors (e.g., EOF) will be detected by the first
-subsequent call for input from the std::istream.
-*/
-class input_stream_adapter : public input_adapter_protocol
-{
-  public:
-    ~input_stream_adapter() override
-    {
-        // clear stream flags; we use underlying streambuf I/O, do not
-        // maintain ifstream flags, except eof
-        is.clear(is.rdstate() & std::ios::eofbit);
-    }
-
-    explicit input_stream_adapter(std::istream& i)
-        : is(i), sb(*i.rdbuf())
-    {}
-
-    // delete because of pointer members
-    input_stream_adapter(const input_stream_adapter&) = delete;
-    input_stream_adapter& operator=(input_stream_adapter&) = delete;
-    input_stream_adapter(input_stream_adapter&&) = delete;
-    input_stream_adapter& operator=(input_stream_adapter&&) = delete;
-
-    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
-    // ensure that std::char_traits<char>::eof() and the character 0xFF do not
-    // end up as the same value, eg. 0xFFFFFFFF.
-    std::char_traits<char>::int_type get_character() override
-    {
-        auto res = sb.sbumpc();
-        // set eof manually, as we don't use the istream interface.
-        if (res == EOF)
-        {
-            is.clear(is.rdstate() | std::ios::eofbit);
-        }
-        return res;
-    }
-
-  private:
-    /// the associated input stream
-    std::istream& is;
-    std::streambuf& sb;
-};
-
-/// input adapter for buffer input
-class input_buffer_adapter : public input_adapter_protocol
-{
-  public:
-    input_buffer_adapter(const char* b, const std::size_t l) noexcept
-        : cursor(b), limit(b + l)
-    {}
-
-    // delete because of pointer members
-    input_buffer_adapter(const input_buffer_adapter&) = delete;
-    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
-    input_buffer_adapter(input_buffer_adapter&&) = delete;
-    input_buffer_adapter& operator=(input_buffer_adapter&&) = delete;
-    ~input_buffer_adapter() override = default;
-
-    std::char_traits<char>::int_type get_character() noexcept override
-    {
-        if (JSON_LIKELY(cursor < limit))
-        {
-            return std::char_traits<char>::to_int_type(*(cursor++));
-        }
-
-        return std::char_traits<char>::eof();
-    }
-
-  private:
-    /// pointer to the current character
-    const char* cursor;
-    /// pointer past the last character
-    const char* const limit;
-};
-
-template<typename WideStringType, size_t T>
-struct wide_string_input_helper
-{
-    // UTF-32
-    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
-    {
-        utf8_bytes_index = 0;
-
-        if (current_wchar == str.size())
-        {
-            utf8_bytes[0] = std::char_traits<char>::eof();
-            utf8_bytes_filled = 1;
-        }
-        else
-        {
-            // get the current character
-            const auto wc = static_cast<int>(str[current_wchar++]);
-
-            // UTF-32 to UTF-8 encoding
-            if (wc < 0x80)
-            {
-                utf8_bytes[0] = wc;
-                utf8_bytes_filled = 1;
-            }
-            else if (wc <= 0x7FF)
-            {
-                utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F);
-                utf8_bytes[1] = 0x80 | (wc & 0x3F);
-                utf8_bytes_filled = 2;
-            }
-            else if (wc <= 0xFFFF)
-            {
-                utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F);
-                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
-                utf8_bytes[2] = 0x80 | (wc & 0x3F);
-                utf8_bytes_filled = 3;
-            }
-            else if (wc <= 0x10FFFF)
-            {
-                utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07);
-                utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F);
-                utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F);
-                utf8_bytes[3] = 0x80 | (wc & 0x3F);
-                utf8_bytes_filled = 4;
-            }
-            else
-            {
-                // unknown character
-                utf8_bytes[0] = wc;
-                utf8_bytes_filled = 1;
-            }
-        }
-    }
-};
-
-template<typename WideStringType>
-struct wide_string_input_helper<WideStringType, 2>
-{
-    // UTF-16
-    static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled)
-    {
-        utf8_bytes_index = 0;
-
-        if (current_wchar == str.size())
-        {
-            utf8_bytes[0] = std::char_traits<char>::eof();
-            utf8_bytes_filled = 1;
-        }
-        else
-        {
-            // get the current character
-            const auto wc = static_cast<int>(str[current_wchar++]);
-
-            // UTF-16 to UTF-8 encoding
-            if (wc < 0x80)
-            {
-                utf8_bytes[0] = wc;
-                utf8_bytes_filled = 1;
-            }
-            else if (wc <= 0x7FF)
-            {
-                utf8_bytes[0] = 0xC0 | ((wc >> 6));
-                utf8_bytes[1] = 0x80 | (wc & 0x3F);
-                utf8_bytes_filled = 2;
-            }
-            else if (0xD800 > wc or wc >= 0xE000)
-            {
-                utf8_bytes[0] = 0xE0 | ((wc >> 12));
-                utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
-                utf8_bytes[2] = 0x80 | (wc & 0x3F);
-                utf8_bytes_filled = 3;
-            }
-            else
-            {
-                if (current_wchar < str.size())
-                {
-                    const auto wc2 = static_cast<int>(str[current_wchar++]);
-                    const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF));
-                    utf8_bytes[0] = 0xf0 | (charcode >> 18);
-                    utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F);
-                    utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F);
-                    utf8_bytes[3] = 0x80 | (charcode & 0x3F);
-                    utf8_bytes_filled = 4;
-                }
-                else
-                {
-                    // unknown character
-                    ++current_wchar;
-                    utf8_bytes[0] = wc;
-                    utf8_bytes_filled = 1;
-                }
-            }
-        }
-    }
-};
-
-template<typename WideStringType>
-class wide_string_input_adapter : public input_adapter_protocol
-{
-  public:
-    explicit wide_string_input_adapter(const WideStringType& w)  noexcept
-        : str(w)
-    {}
-
-    std::char_traits<char>::int_type get_character() noexcept override
-    {
-        // check if buffer needs to be filled
-        if (utf8_bytes_index == utf8_bytes_filled)
-        {
-            fill_buffer<sizeof(typename WideStringType::value_type)>();
-
-            assert(utf8_bytes_filled > 0);
-            assert(utf8_bytes_index == 0);
-        }
-
-        // use buffer
-        assert(utf8_bytes_filled > 0);
-        assert(utf8_bytes_index < utf8_bytes_filled);
-        return utf8_bytes[utf8_bytes_index++];
-    }
-
-  private:
-    template<size_t T>
-    void fill_buffer()
-    {
-        wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);
-    }
-
-    /// the wstring to process
-    const WideStringType& str;
-
-    /// index of the current wchar in str
-    std::size_t current_wchar = 0;
-
-    /// a buffer for UTF-8 bytes
-    std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
-
-    /// index to the utf8_codes array for the next valid byte
-    std::size_t utf8_bytes_index = 0;
-    /// number of valid bytes in the utf8_codes array
-    std::size_t utf8_bytes_filled = 0;
-};
-
-class input_adapter
-{
-  public:
-    // native support
-    input_adapter(std::FILE* file)
-        : ia(std::make_shared<file_input_adapter>(file)) {}
-    /// input adapter for input stream
-    input_adapter(std::istream& i)
-        : ia(std::make_shared<input_stream_adapter>(i)) {}
-
-    /// input adapter for input stream
-    input_adapter(std::istream&& i)
-        : ia(std::make_shared<input_stream_adapter>(i)) {}
-
-    input_adapter(const std::wstring& ws)
-        : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {}
-
-    input_adapter(const std::u16string& ws)
-        : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {}
-
-    input_adapter(const std::u32string& ws)
-        : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {}
-
-    /// input adapter for buffer
-    template<typename CharT,
-             typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
-                 int>::type = 0>
-    input_adapter(CharT b, std::size_t l)
-        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
-
-    // derived support
-
-    /// input adapter for string literal
-    template<typename CharT,
-             typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
-                 int>::type = 0>
-    input_adapter(CharT b)
-        : input_adapter(reinterpret_cast<const char*>(b),
-                        std::strlen(reinterpret_cast<const char*>(b))) {}
-
-    /// input adapter for iterator range with contiguous storage
-    template<class IteratorType,
-             typename std::enable_if<
-                 std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
-                 int>::type = 0>
-    input_adapter(IteratorType first, IteratorType last)
-    {
-#ifndef NDEBUG
-        // assertion to check that the iterator range is indeed contiguous,
-        // see http://stackoverflow.com/a/35008842/266378 for more discussion
-        const auto is_contiguous = std::accumulate(
-                                       first, last, std::pair<bool, int>(true, 0),
-                                       [&first](std::pair<bool, int> res, decltype(*first) val)
-        {
-            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
-            return res;
-        }).first;
-        assert(is_contiguous);
-#endif
-
-        // assertion to check that each element is 1 byte long
-        static_assert(
-            sizeof(typename iterator_traits<IteratorType>::value_type) == 1,
-            "each element in the iterator range must have the size of 1 byte");
-
-        const auto len = static_cast<size_t>(std::distance(first, last));
-        if (JSON_LIKELY(len > 0))
-        {
-            // there is at least one element: use the address of first
-            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
-        }
-        else
-        {
-            // the address of first cannot be used: use nullptr
-            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
-        }
-    }
-
-    /// input adapter for array
-    template<class T, std::size_t N>
-    input_adapter(T (&array)[N])
-        : input_adapter(std::begin(array), std::end(array)) {}
-
-    /// input adapter for contiguous container
-    template<class ContiguousContainer, typename
-             std::enable_if<not std::is_pointer<ContiguousContainer>::value and
-                            std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
-                            int>::type = 0>
-    input_adapter(const ContiguousContainer& c)
-        : input_adapter(std::begin(c), std::end(c)) {}
-
-    operator input_adapter_t()
-    {
-        return ia;
-    }
-
-  private:
-    /// the actual adapter
-    input_adapter_t ia = nullptr;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/lexer.hpp>
-
-
-#include <clocale> // localeconv
-#include <cstddef> // size_t
-#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
-#include <cstdio> // snprintf
-#include <initializer_list> // initializer_list
-#include <string> // char_traits, string
-#include <vector> // vector
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/input/input_adapters.hpp>
-
-// #include <nlohmann/detail/input/position_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-///////////
-// lexer //
-///////////
-
-/*!
-@brief lexical analysis
-
-This class organizes the lexical analysis during JSON deserialization.
-*/
-template<typename BasicJsonType>
-class lexer
-{
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-
-  public:
-    /// token types for the parser
-    enum class token_type
-    {
-        uninitialized,    ///< indicating the scanner is uninitialized
-        literal_true,     ///< the `true` literal
-        literal_false,    ///< the `false` literal
-        literal_null,     ///< the `null` literal
-        value_string,     ///< a string -- use get_string() for actual value
-        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
-        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
-        value_float,      ///< an floating point number -- use get_number_float() for actual value
-        begin_array,      ///< the character for array begin `[`
-        begin_object,     ///< the character for object begin `{`
-        end_array,        ///< the character for array end `]`
-        end_object,       ///< the character for object end `}`
-        name_separator,   ///< the name separator `:`
-        value_separator,  ///< the value separator `,`
-        parse_error,      ///< indicating a parse error
-        end_of_input,     ///< indicating the end of the input buffer
-        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
-    };
-
-    /// return name of values of type token_type (only used for errors)
-    static const char* token_type_name(const token_type t) noexcept
-    {
-        switch (t)
-        {
-            case token_type::uninitialized:
-                return "<uninitialized>";
-            case token_type::literal_true:
-                return "true literal";
-            case token_type::literal_false:
-                return "false literal";
-            case token_type::literal_null:
-                return "null literal";
-            case token_type::value_string:
-                return "string literal";
-            case lexer::token_type::value_unsigned:
-            case lexer::token_type::value_integer:
-            case lexer::token_type::value_float:
-                return "number literal";
-            case token_type::begin_array:
-                return "'['";
-            case token_type::begin_object:
-                return "'{'";
-            case token_type::end_array:
-                return "']'";
-            case token_type::end_object:
-                return "'}'";
-            case token_type::name_separator:
-                return "':'";
-            case token_type::value_separator:
-                return "','";
-            case token_type::parse_error:
-                return "<parse error>";
-            case token_type::end_of_input:
-                return "end of input";
-            case token_type::literal_or_value:
-                return "'[', '{', or a literal";
-            // LCOV_EXCL_START
-            default: // catch non-enum values
-                return "unknown token";
-                // LCOV_EXCL_STOP
-        }
-    }
-
-    explicit lexer(detail::input_adapter_t&& adapter)
-        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
-
-    // delete because of pointer members
-    lexer(const lexer&) = delete;
-    lexer(lexer&&) = delete;
-    lexer& operator=(lexer&) = delete;
-    lexer& operator=(lexer&&) = delete;
-    ~lexer() = default;
-
-  private:
-    /////////////////////
-    // locales
-    /////////////////////
-
-    /// return the locale-dependent decimal point
-    static char get_decimal_point() noexcept
-    {
-        const auto loc = localeconv();
-        assert(loc != nullptr);
-        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
-    }
-
-    /////////////////////
-    // scan functions
-    /////////////////////
-
-    /*!
-    @brief get codepoint from 4 hex characters following `\u`
-
-    For input "\u c1 c2 c3 c4" the codepoint is:
-      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
-    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
-
-    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
-    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
-    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
-    between the ASCII value of the character and the desired integer value.
-
-    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
-            non-hex character)
-    */
-    int get_codepoint()
-    {
-        // this function only makes sense after reading `\u`
-        assert(current == 'u');
-        int codepoint = 0;
-
-        const auto factors = { 12, 8, 4, 0 };
-        for (const auto factor : factors)
-        {
-            get();
-
-            if (current >= '0' and current <= '9')
-            {
-                codepoint += ((current - 0x30) << factor);
-            }
-            else if (current >= 'A' and current <= 'F')
-            {
-                codepoint += ((current - 0x37) << factor);
-            }
-            else if (current >= 'a' and current <= 'f')
-            {
-                codepoint += ((current - 0x57) << factor);
-            }
-            else
-            {
-                return -1;
-            }
-        }
-
-        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
-        return codepoint;
-    }
-
-    /*!
-    @brief check if the next byte(s) are inside a given range
-
-    Adds the current byte and, for each passed range, reads a new byte and
-    checks if it is inside the range. If a violation was detected, set up an
-    error message and return false. Otherwise, return true.
-
-    @param[in] ranges  list of integers; interpreted as list of pairs of
-                       inclusive lower and upper bound, respectively
-
-    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,
-         1, 2, or 3 pairs. This precondition is enforced by an assertion.
-
-    @return true if and only if no range violation was detected
-    */
-    bool next_byte_in_range(std::initializer_list<int> ranges)
-    {
-        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
-        add(current);
-
-        for (auto range = ranges.begin(); range != ranges.end(); ++range)
-        {
-            get();
-            if (JSON_LIKELY(*range <= current and current <= *(++range)))
-            {
-                add(current);
-            }
-            else
-            {
-                error_message = "invalid string: ill-formed UTF-8 byte";
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    /*!
-    @brief scan a string literal
-
-    This function scans a string according to Sect. 7 of RFC 7159. While
-    scanning, bytes are escaped and copied into buffer token_buffer. Then the
-    function returns successfully, token_buffer is *not* null-terminated (as it
-    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
-    string.
-
-    @return token_type::value_string if string could be successfully scanned,
-            token_type::parse_error otherwise
-
-    @note In case of errors, variable error_message contains a textual
-          description.
-    */
-    token_type scan_string()
-    {
-        // reset token_buffer (ignore opening quote)
-        reset();
-
-        // we entered the function by reading an open quote
-        assert(current == '\"');
-
-        while (true)
-        {
-            // get next character
-            switch (get())
-            {
-                // end of file while parsing string
-                case std::char_traits<char>::eof():
-                {
-                    error_message = "invalid string: missing closing quote";
-                    return token_type::parse_error;
-                }
-
-                // closing quote
-                case '\"':
-                {
-                    return token_type::value_string;
-                }
-
-                // escapes
-                case '\\':
-                {
-                    switch (get())
-                    {
-                        // quotation mark
-                        case '\"':
-                            add('\"');
-                            break;
-                        // reverse solidus
-                        case '\\':
-                            add('\\');
-                            break;
-                        // solidus
-                        case '/':
-                            add('/');
-                            break;
-                        // backspace
-                        case 'b':
-                            add('\b');
-                            break;
-                        // form feed
-                        case 'f':
-                            add('\f');
-                            break;
-                        // line feed
-                        case 'n':
-                            add('\n');
-                            break;
-                        // carriage return
-                        case 'r':
-                            add('\r');
-                            break;
-                        // tab
-                        case 't':
-                            add('\t');
-                            break;
-
-                        // unicode escapes
-                        case 'u':
-                        {
-                            const int codepoint1 = get_codepoint();
-                            int codepoint = codepoint1; // start with codepoint1
-
-                            if (JSON_UNLIKELY(codepoint1 == -1))
-                            {
-                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                return token_type::parse_error;
-                            }
-
-                            // check if code point is a high surrogate
-                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
-                            {
-                                // expect next \uxxxx entry
-                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
-                                {
-                                    const int codepoint2 = get_codepoint();
-
-                                    if (JSON_UNLIKELY(codepoint2 == -1))
-                                    {
-                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                        return token_type::parse_error;
-                                    }
-
-                                    // check if codepoint2 is a low surrogate
-                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
-                                    {
-                                        // overwrite codepoint
-                                        codepoint =
-                                            // high surrogate occupies the most significant 22 bits
-                                            (codepoint1 << 10)
-                                            // low surrogate occupies the least significant 15 bits
-                                            + codepoint2
-                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
-                                            // in the result so we have to subtract with:
-                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
-                                            - 0x35FDC00;
-                                    }
-                                    else
-                                    {
-                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                        return token_type::parse_error;
-                                    }
-                                }
-                                else
-                                {
-                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                    return token_type::parse_error;
-                                }
-                            }
-                            else
-                            {
-                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
-                                {
-                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
-                                    return token_type::parse_error;
-                                }
-                            }
-
-                            // result of the above calculation yields a proper codepoint
-                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
-
-                            // translate codepoint into bytes
-                            if (codepoint < 0x80)
-                            {
-                                // 1-byte characters: 0xxxxxxx (ASCII)
-                                add(codepoint);
-                            }
-                            else if (codepoint <= 0x7FF)
-                            {
-                                // 2-byte characters: 110xxxxx 10xxxxxx
-                                add(0xC0 | (codepoint >> 6));
-                                add(0x80 | (codepoint & 0x3F));
-                            }
-                            else if (codepoint <= 0xFFFF)
-                            {
-                                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
-                                add(0xE0 | (codepoint >> 12));
-                                add(0x80 | ((codepoint >> 6) & 0x3F));
-                                add(0x80 | (codepoint & 0x3F));
-                            }
-                            else
-                            {
-                                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                                add(0xF0 | (codepoint >> 18));
-                                add(0x80 | ((codepoint >> 12) & 0x3F));
-                                add(0x80 | ((codepoint >> 6) & 0x3F));
-                                add(0x80 | (codepoint & 0x3F));
-                            }
-
-                            break;
-                        }
-
-                        // other characters after escape
-                        default:
-                            error_message = "invalid string: forbidden character after backslash";
-                            return token_type::parse_error;
-                    }
-
-                    break;
-                }
-
-                // invalid control characters
-                case 0x00:
-                {
-                    error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000";
-                    return token_type::parse_error;
-                }
-
-                case 0x01:
-                {
-                    error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001";
-                    return token_type::parse_error;
-                }
-
-                case 0x02:
-                {
-                    error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002";
-                    return token_type::parse_error;
-                }
-
-                case 0x03:
-                {
-                    error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003";
-                    return token_type::parse_error;
-                }
-
-                case 0x04:
-                {
-                    error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004";
-                    return token_type::parse_error;
-                }
-
-                case 0x05:
-                {
-                    error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005";
-                    return token_type::parse_error;
-                }
-
-                case 0x06:
-                {
-                    error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006";
-                    return token_type::parse_error;
-                }
-
-                case 0x07:
-                {
-                    error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007";
-                    return token_type::parse_error;
-                }
-
-                case 0x08:
-                {
-                    error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b";
-                    return token_type::parse_error;
-                }
-
-                case 0x09:
-                {
-                    error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t";
-                    return token_type::parse_error;
-                }
-
-                case 0x0A:
-                {
-                    error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n";
-                    return token_type::parse_error;
-                }
-
-                case 0x0B:
-                {
-                    error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B";
-                    return token_type::parse_error;
-                }
-
-                case 0x0C:
-                {
-                    error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f";
-                    return token_type::parse_error;
-                }
-
-                case 0x0D:
-                {
-                    error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r";
-                    return token_type::parse_error;
-                }
-
-                case 0x0E:
-                {
-                    error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E";
-                    return token_type::parse_error;
-                }
-
-                case 0x0F:
-                {
-                    error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F";
-                    return token_type::parse_error;
-                }
-
-                case 0x10:
-                {
-                    error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010";
-                    return token_type::parse_error;
-                }
-
-                case 0x11:
-                {
-                    error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011";
-                    return token_type::parse_error;
-                }
-
-                case 0x12:
-                {
-                    error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012";
-                    return token_type::parse_error;
-                }
-
-                case 0x13:
-                {
-                    error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013";
-                    return token_type::parse_error;
-                }
-
-                case 0x14:
-                {
-                    error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014";
-                    return token_type::parse_error;
-                }
-
-                case 0x15:
-                {
-                    error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015";
-                    return token_type::parse_error;
-                }
-
-                case 0x16:
-                {
-                    error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016";
-                    return token_type::parse_error;
-                }
-
-                case 0x17:
-                {
-                    error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017";
-                    return token_type::parse_error;
-                }
-
-                case 0x18:
-                {
-                    error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018";
-                    return token_type::parse_error;
-                }
-
-                case 0x19:
-                {
-                    error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019";
-                    return token_type::parse_error;
-                }
-
-                case 0x1A:
-                {
-                    error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A";
-                    return token_type::parse_error;
-                }
-
-                case 0x1B:
-                {
-                    error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B";
-                    return token_type::parse_error;
-                }
-
-                case 0x1C:
-                {
-                    error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C";
-                    return token_type::parse_error;
-                }
-
-                case 0x1D:
-                {
-                    error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D";
-                    return token_type::parse_error;
-                }
-
-                case 0x1E:
-                {
-                    error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E";
-                    return token_type::parse_error;
-                }
-
-                case 0x1F:
-                {
-                    error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F";
-                    return token_type::parse_error;
-                }
-
-                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
-                case 0x20:
-                case 0x21:
-                case 0x23:
-                case 0x24:
-                case 0x25:
-                case 0x26:
-                case 0x27:
-                case 0x28:
-                case 0x29:
-                case 0x2A:
-                case 0x2B:
-                case 0x2C:
-                case 0x2D:
-                case 0x2E:
-                case 0x2F:
-                case 0x30:
-                case 0x31:
-                case 0x32:
-                case 0x33:
-                case 0x34:
-                case 0x35:
-                case 0x36:
-                case 0x37:
-                case 0x38:
-                case 0x39:
-                case 0x3A:
-                case 0x3B:
-                case 0x3C:
-                case 0x3D:
-                case 0x3E:
-                case 0x3F:
-                case 0x40:
-                case 0x41:
-                case 0x42:
-                case 0x43:
-                case 0x44:
-                case 0x45:
-                case 0x46:
-                case 0x47:
-                case 0x48:
-                case 0x49:
-                case 0x4A:
-                case 0x4B:
-                case 0x4C:
-                case 0x4D:
-                case 0x4E:
-                case 0x4F:
-                case 0x50:
-                case 0x51:
-                case 0x52:
-                case 0x53:
-                case 0x54:
-                case 0x55:
-                case 0x56:
-                case 0x57:
-                case 0x58:
-                case 0x59:
-                case 0x5A:
-                case 0x5B:
-                case 0x5D:
-                case 0x5E:
-                case 0x5F:
-                case 0x60:
-                case 0x61:
-                case 0x62:
-                case 0x63:
-                case 0x64:
-                case 0x65:
-                case 0x66:
-                case 0x67:
-                case 0x68:
-                case 0x69:
-                case 0x6A:
-                case 0x6B:
-                case 0x6C:
-                case 0x6D:
-                case 0x6E:
-                case 0x6F:
-                case 0x70:
-                case 0x71:
-                case 0x72:
-                case 0x73:
-                case 0x74:
-                case 0x75:
-                case 0x76:
-                case 0x77:
-                case 0x78:
-                case 0x79:
-                case 0x7A:
-                case 0x7B:
-                case 0x7C:
-                case 0x7D:
-                case 0x7E:
-                case 0x7F:
-                {
-                    add(current);
-                    break;
-                }
-
-                // U+0080..U+07FF: bytes C2..DF 80..BF
-                case 0xC2:
-                case 0xC3:
-                case 0xC4:
-                case 0xC5:
-                case 0xC6:
-                case 0xC7:
-                case 0xC8:
-                case 0xC9:
-                case 0xCA:
-                case 0xCB:
-                case 0xCC:
-                case 0xCD:
-                case 0xCE:
-                case 0xCF:
-                case 0xD0:
-                case 0xD1:
-                case 0xD2:
-                case 0xD3:
-                case 0xD4:
-                case 0xD5:
-                case 0xD6:
-                case 0xD7:
-                case 0xD8:
-                case 0xD9:
-                case 0xDA:
-                case 0xDB:
-                case 0xDC:
-                case 0xDD:
-                case 0xDE:
-                case 0xDF:
-                {
-                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
-                case 0xE0:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
-                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
-                case 0xE1:
-                case 0xE2:
-                case 0xE3:
-                case 0xE4:
-                case 0xE5:
-                case 0xE6:
-                case 0xE7:
-                case 0xE8:
-                case 0xE9:
-                case 0xEA:
-                case 0xEB:
-                case 0xEC:
-                case 0xEE:
-                case 0xEF:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
-                case 0xED:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
-                case 0xF0:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
-                case 0xF1:
-                case 0xF2:
-                case 0xF3:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
-                case 0xF4:
-                {
-                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
-                    {
-                        return token_type::parse_error;
-                    }
-                    break;
-                }
-
-                // remaining bytes (80..C1 and F5..FF) are ill-formed
-                default:
-                {
-                    error_message = "invalid string: ill-formed UTF-8 byte";
-                    return token_type::parse_error;
-                }
-            }
-        }
-    }
-
-    static void strtof(float& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtof(str, endptr);
-    }
-
-    static void strtof(double& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtod(str, endptr);
-    }
-
-    static void strtof(long double& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtold(str, endptr);
-    }
-
-    /*!
-    @brief scan a number literal
-
-    This function scans a string according to Sect. 6 of RFC 7159.
-
-    The function is realized with a deterministic finite state machine derived
-    from the grammar described in RFC 7159. Starting in state "init", the
-    input is read and used to determined the next state. Only state "done"
-    accepts the number. State "error" is a trap state to model errors. In the
-    table below, "anything" means any character but the ones listed before.
-
-    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
-    ---------|----------|----------|----------|---------|---------|----------|-----------
-    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
-    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
-    zero     | done     | done     | exponent | done    | done    | decimal1 | done
-    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
-    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
-    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
-    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
-    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
-    any2     | any2     | any2     | done     | done    | done    | done     | done
-
-    The state machine is realized with one label per state (prefixed with
-    "scan_number_") and `goto` statements between them. The state machine
-    contains cycles, but any cycle can be left when EOF is read. Therefore,
-    the function is guaranteed to terminate.
-
-    During scanning, the read bytes are stored in token_buffer. This string is
-    then converted to a signed integer, an unsigned integer, or a
-    floating-point number.
-
-    @return token_type::value_unsigned, token_type::value_integer, or
-            token_type::value_float if number could be successfully scanned,
-            token_type::parse_error otherwise
-
-    @note The scanner is independent of the current locale. Internally, the
-          locale's decimal point is used instead of `.` to work with the
-          locale-dependent converters.
-    */
-    token_type scan_number()  // lgtm [cpp/use-of-goto]
-    {
-        // reset token_buffer to store the number's bytes
-        reset();
-
-        // the type of the parsed number; initially set to unsigned; will be
-        // changed if minus sign, decimal point or exponent is read
-        token_type number_type = token_type::value_unsigned;
-
-        // state (init): we just found out we need to scan a number
-        switch (current)
-        {
-            case '-':
-            {
-                add(current);
-                goto scan_number_minus;
-            }
-
-            case '0':
-            {
-                add(current);
-                goto scan_number_zero;
-            }
-
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
-
-            // LCOV_EXCL_START
-            default:
-            {
-                // all other characters are rejected outside scan_number()
-                assert(false);
-            }
-                // LCOV_EXCL_STOP
-        }
-
-scan_number_minus:
-        // state: we just parsed a leading minus sign
-        number_type = token_type::value_integer;
-        switch (get())
-        {
-            case '0':
-            {
-                add(current);
-                goto scan_number_zero;
-            }
-
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
-
-            default:
-            {
-                error_message = "invalid number; expected digit after '-'";
-                return token_type::parse_error;
-            }
-        }
-
-scan_number_zero:
-        // state: we just parse a zero (maybe with a leading minus sign)
-        switch (get())
-        {
-            case '.':
-            {
-                add(decimal_point_char);
-                goto scan_number_decimal1;
-            }
-
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
-
-            default:
-                goto scan_number_done;
-        }
-
-scan_number_any1:
-        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any1;
-            }
-
-            case '.':
-            {
-                add(decimal_point_char);
-                goto scan_number_decimal1;
-            }
-
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
-
-            default:
-                goto scan_number_done;
-        }
-
-scan_number_decimal1:
-        // state: we just parsed a decimal point
-        number_type = token_type::value_float;
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_decimal2;
-            }
-
-            default:
-            {
-                error_message = "invalid number; expected digit after '.'";
-                return token_type::parse_error;
-            }
-        }
-
-scan_number_decimal2:
-        // we just parsed at least one number after a decimal point
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_decimal2;
-            }
-
-            case 'e':
-            case 'E':
-            {
-                add(current);
-                goto scan_number_exponent;
-            }
-
-            default:
-                goto scan_number_done;
-        }
-
-scan_number_exponent:
-        // we just parsed an exponent
-        number_type = token_type::value_float;
-        switch (get())
-        {
-            case '+':
-            case '-':
-            {
-                add(current);
-                goto scan_number_sign;
-            }
-
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any2;
-            }
-
-            default:
-            {
-                error_message =
-                    "invalid number; expected '+', '-', or digit after exponent";
-                return token_type::parse_error;
-            }
-        }
-
-scan_number_sign:
-        // we just parsed an exponent sign
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any2;
-            }
-
-            default:
-            {
-                error_message = "invalid number; expected digit after exponent sign";
-                return token_type::parse_error;
-            }
-        }
-
-scan_number_any2:
-        // we just parsed a number after the exponent or exponent sign
-        switch (get())
-        {
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-            {
-                add(current);
-                goto scan_number_any2;
-            }
-
-            default:
-                goto scan_number_done;
-        }
-
-scan_number_done:
-        // unget the character after the number (we only read it to know that
-        // we are done scanning a number)
-        unget();
-
-        char* endptr = nullptr;
-        errno = 0;
-
-        // try to parse integers first and fall back to floats
-        if (number_type == token_type::value_unsigned)
-        {
-            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
-
-            // we checked the number format before
-            assert(endptr == token_buffer.data() + token_buffer.size());
-
-            if (errno == 0)
-            {
-                value_unsigned = static_cast<number_unsigned_t>(x);
-                if (value_unsigned == x)
-                {
-                    return token_type::value_unsigned;
-                }
-            }
-        }
-        else if (number_type == token_type::value_integer)
-        {
-            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
-
-            // we checked the number format before
-            assert(endptr == token_buffer.data() + token_buffer.size());
-
-            if (errno == 0)
-            {
-                value_integer = static_cast<number_integer_t>(x);
-                if (value_integer == x)
-                {
-                    return token_type::value_integer;
-                }
-            }
-        }
-
-        // this code is reached if we parse a floating-point number or if an
-        // integer conversion above failed
-        strtof(value_float, token_buffer.data(), &endptr);
-
-        // we checked the number format before
-        assert(endptr == token_buffer.data() + token_buffer.size());
-
-        return token_type::value_float;
-    }
-
-    /*!
-    @param[in] literal_text  the literal text to expect
-    @param[in] length        the length of the passed literal text
-    @param[in] return_type   the token type to return on success
-    */
-    token_type scan_literal(const char* literal_text, const std::size_t length,
-                            token_type return_type)
-    {
-        assert(current == literal_text[0]);
-        for (std::size_t i = 1; i < length; ++i)
-        {
-            if (JSON_UNLIKELY(get() != literal_text[i]))
-            {
-                error_message = "invalid literal";
-                return token_type::parse_error;
-            }
-        }
-        return return_type;
-    }
-
-    /////////////////////
-    // input management
-    /////////////////////
-
-    /// reset token_buffer; current character is beginning of token
-    void reset() noexcept
-    {
-        token_buffer.clear();
-        token_string.clear();
-        token_string.push_back(std::char_traits<char>::to_char_type(current));
-    }
-
-    /*
-    @brief get next character from the input
-
-    This function provides the interface to the used input adapter. It does
-    not throw in case the input reached EOF, but returns a
-    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters
-    for use in error messages.
-
-    @return character read from the input
-    */
-    std::char_traits<char>::int_type get()
-    {
-        ++position.chars_read_total;
-        ++position.chars_read_current_line;
-
-        if (next_unget)
-        {
-            // just reset the next_unget variable and work with current
-            next_unget = false;
-        }
-        else
-        {
-            current = ia->get_character();
-        }
-
-        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
-        {
-            token_string.push_back(std::char_traits<char>::to_char_type(current));
-        }
-
-        if (current == '\n')
-        {
-            ++position.lines_read;
-            ++position.chars_read_current_line = 0;
-        }
-
-        return current;
-    }
-
-    /*!
-    @brief unget current character (read it again on next get)
-
-    We implement unget by setting variable next_unget to true. The input is not
-    changed - we just simulate ungetting by modifying chars_read_total,
-    chars_read_current_line, and token_string. The next call to get() will
-    behave as if the unget character is read again.
-    */
-    void unget()
-    {
-        next_unget = true;
-
-        --position.chars_read_total;
-
-        // in case we "unget" a newline, we have to also decrement the lines_read
-        if (position.chars_read_current_line == 0)
-        {
-            if (position.lines_read > 0)
-            {
-                --position.lines_read;
-            }
-        }
-        else
-        {
-            --position.chars_read_current_line;
-        }
-
-        if (JSON_LIKELY(current != std::char_traits<char>::eof()))
-        {
-            assert(token_string.size() != 0);
-            token_string.pop_back();
-        }
-    }
-
-    /// add a character to token_buffer
-    void add(int c)
-    {
-        token_buffer.push_back(std::char_traits<char>::to_char_type(c));
-    }
-
-  public:
-    /////////////////////
-    // value getters
-    /////////////////////
-
-    /// return integer value
-    constexpr number_integer_t get_number_integer() const noexcept
-    {
-        return value_integer;
-    }
-
-    /// return unsigned integer value
-    constexpr number_unsigned_t get_number_unsigned() const noexcept
-    {
-        return value_unsigned;
-    }
-
-    /// return floating-point value
-    constexpr number_float_t get_number_float() const noexcept
-    {
-        return value_float;
-    }
-
-    /// return current string value (implicitly resets the token; useful only once)
-    string_t& get_string()
-    {
-        return token_buffer;
-    }
-
-    /////////////////////
-    // diagnostics
-    /////////////////////
-
-    /// return position of last read token
-    constexpr position_t get_position() const noexcept
-    {
-        return position;
-    }
-
-    /// return the last read token (for errors only).  Will never contain EOF
-    /// (an arbitrary value that is not a valid char value, often -1), because
-    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
-    std::string get_token_string() const
-    {
-        // escape control characters
-        std::string result;
-        for (const auto c : token_string)
-        {
-            if ('\x00' <= c and c <= '\x1F')
-            {
-                // escape control characters
-                char cs[9];
-                (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c));
-                result += cs;
-            }
-            else
-            {
-                // add character as is
-                result.push_back(c);
-            }
-        }
-
-        return result;
-    }
-
-    /// return syntax error message
-    constexpr const char* get_error_message() const noexcept
-    {
-        return error_message;
-    }
-
-    /////////////////////
-    // actual scanner
-    /////////////////////
-
-    /*!
-    @brief skip the UTF-8 byte order mark
-    @return true iff there is no BOM or the correct BOM has been skipped
-    */
-    bool skip_bom()
-    {
-        if (get() == 0xEF)
-        {
-            // check if we completely parse the BOM
-            return get() == 0xBB and get() == 0xBF;
-        }
-
-        // the first character is not the beginning of the BOM; unget it to
-        // process is later
-        unget();
-        return true;
-    }
-
-    token_type scan()
-    {
-        // initially, skip the BOM
-        if (position.chars_read_total == 0 and not skip_bom())
-        {
-            error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
-            return token_type::parse_error;
-        }
-
-        // read next character and ignore whitespace
-        do
-        {
-            get();
-        }
-        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
-
-        switch (current)
-        {
-            // structural characters
-            case '[':
-                return token_type::begin_array;
-            case ']':
-                return token_type::end_array;
-            case '{':
-                return token_type::begin_object;
-            case '}':
-                return token_type::end_object;
-            case ':':
-                return token_type::name_separator;
-            case ',':
-                return token_type::value_separator;
-
-            // literals
-            case 't':
-                return scan_literal("true", 4, token_type::literal_true);
-            case 'f':
-                return scan_literal("false", 5, token_type::literal_false);
-            case 'n':
-                return scan_literal("null", 4, token_type::literal_null);
-
-            // string
-            case '\"':
-                return scan_string();
-
-            // number
-            case '-':
-            case '0':
-            case '1':
-            case '2':
-            case '3':
-            case '4':
-            case '5':
-            case '6':
-            case '7':
-            case '8':
-            case '9':
-                return scan_number();
-
-            // end of input (the null byte is needed when parsing from
-            // string literals)
-            case '\0':
-            case std::char_traits<char>::eof():
-                return token_type::end_of_input;
-
-            // error
-            default:
-                error_message = "invalid literal";
-                return token_type::parse_error;
-        }
-    }
-
-  private:
-    /// input adapter
-    detail::input_adapter_t ia = nullptr;
-
-    /// the current character
-    std::char_traits<char>::int_type current = std::char_traits<char>::eof();
-
-    /// whether the next get() call should just return current
-    bool next_unget = false;
-
-    /// the start position of the current token
-    position_t position;
-
-    /// raw input token string (for error messages)
-    std::vector<char> token_string {};
-
-    /// buffer for variable-length tokens (numbers, strings)
-    string_t token_buffer {};
-
-    /// a description of occurred lexer errors
-    const char* error_message = "";
-
-    // number values
-    number_integer_t value_integer = 0;
-    number_unsigned_t value_unsigned = 0;
-    number_float_t value_float = 0;
-
-    /// the decimal point
-    const char decimal_point_char = '.';
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/parser.hpp>
-
-
-#include <cassert> // assert
-#include <cmath> // isfinite
-#include <cstdint> // uint8_t
-#include <functional> // function
-#include <string> // string
-#include <utility> // move
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/meta/is_sax.hpp>
-
-
-#include <cstdint> // size_t
-#include <utility> // declval
-
-// #include <nlohmann/detail/meta/detected.hpp>
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template <typename T>
-using null_function_t = decltype(std::declval<T&>().null());
-
-template <typename T>
-using boolean_function_t =
-    decltype(std::declval<T&>().boolean(std::declval<bool>()));
-
-template <typename T, typename Integer>
-using number_integer_function_t =
-    decltype(std::declval<T&>().number_integer(std::declval<Integer>()));
-
-template <typename T, typename Unsigned>
-using number_unsigned_function_t =
-    decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));
-
-template <typename T, typename Float, typename String>
-using number_float_function_t = decltype(std::declval<T&>().number_float(
-                                    std::declval<Float>(), std::declval<const String&>()));
-
-template <typename T, typename String>
-using string_function_t =
-    decltype(std::declval<T&>().string(std::declval<String&>()));
-
-template <typename T>
-using start_object_function_t =
-    decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));
-
-template <typename T, typename String>
-using key_function_t =
-    decltype(std::declval<T&>().key(std::declval<String&>()));
-
-template <typename T>
-using end_object_function_t = decltype(std::declval<T&>().end_object());
-
-template <typename T>
-using start_array_function_t =
-    decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));
-
-template <typename T>
-using end_array_function_t = decltype(std::declval<T&>().end_array());
-
-template <typename T, typename Exception>
-using parse_error_function_t = decltype(std::declval<T&>().parse_error(
-        std::declval<std::size_t>(), std::declval<const std::string&>(),
-        std::declval<const Exception&>()));
-
-template <typename SAX, typename BasicJsonType>
-struct is_sax
-{
-  private:
-    static_assert(is_basic_json<BasicJsonType>::value,
-                  "BasicJsonType must be of type basic_json<...>");
-
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using exception_t = typename BasicJsonType::exception;
-
-  public:
-    static constexpr bool value =
-        is_detected_exact<bool, null_function_t, SAX>::value &&
-        is_detected_exact<bool, boolean_function_t, SAX>::value &&
-        is_detected_exact<bool, number_integer_function_t, SAX,
-        number_integer_t>::value &&
-        is_detected_exact<bool, number_unsigned_function_t, SAX,
-        number_unsigned_t>::value &&
-        is_detected_exact<bool, number_float_function_t, SAX, number_float_t,
-        string_t>::value &&
-        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
-        is_detected_exact<bool, start_object_function_t, SAX>::value &&
-        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
-        is_detected_exact<bool, end_object_function_t, SAX>::value &&
-        is_detected_exact<bool, start_array_function_t, SAX>::value &&
-        is_detected_exact<bool, end_array_function_t, SAX>::value &&
-        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
-};
-
-template <typename SAX, typename BasicJsonType>
-struct is_sax_static_asserts
-{
-  private:
-    static_assert(is_basic_json<BasicJsonType>::value,
-                  "BasicJsonType must be of type basic_json<...>");
-
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using exception_t = typename BasicJsonType::exception;
-
-  public:
-    static_assert(is_detected_exact<bool, null_function_t, SAX>::value,
-                  "Missing/invalid function: bool null()");
-    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
-                  "Missing/invalid function: bool boolean(bool)");
-    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
-                  "Missing/invalid function: bool boolean(bool)");
-    static_assert(
-        is_detected_exact<bool, number_integer_function_t, SAX,
-        number_integer_t>::value,
-        "Missing/invalid function: bool number_integer(number_integer_t)");
-    static_assert(
-        is_detected_exact<bool, number_unsigned_function_t, SAX,
-        number_unsigned_t>::value,
-        "Missing/invalid function: bool number_unsigned(number_unsigned_t)");
-    static_assert(is_detected_exact<bool, number_float_function_t, SAX,
-                  number_float_t, string_t>::value,
-                  "Missing/invalid function: bool number_float(number_float_t, const string_t&)");
-    static_assert(
-        is_detected_exact<bool, string_function_t, SAX, string_t>::value,
-        "Missing/invalid function: bool string(string_t&)");
-    static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,
-                  "Missing/invalid function: bool start_object(std::size_t)");
-    static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,
-                  "Missing/invalid function: bool key(string_t&)");
-    static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,
-                  "Missing/invalid function: bool end_object()");
-    static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,
-                  "Missing/invalid function: bool start_array(std::size_t)");
-    static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,
-                  "Missing/invalid function: bool end_array()");
-    static_assert(
-        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,
-        "Missing/invalid function: bool parse_error(std::size_t, const "
-        "std::string&, const exception&)");
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/input_adapters.hpp>
-
-// #include <nlohmann/detail/input/json_sax.hpp>
-
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-// #include <nlohmann/detail/input/parser.hpp>
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-
-namespace nlohmann
-{
-
-/*!
-@brief SAX interface
-
-This class describes the SAX interface used by @ref nlohmann::json::sax_parse.
-Each function is called in different situations while the input is parsed. The
-boolean return value informs the parser whether to continue processing the
-input.
-*/
-template<typename BasicJsonType>
-struct json_sax
-{
-    /// type for (signed) integers
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    /// type for unsigned integers
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    /// type for floating-point numbers
-    using number_float_t = typename BasicJsonType::number_float_t;
-    /// type for strings
-    using string_t = typename BasicJsonType::string_t;
-
-    /*!
-    @brief a null value was read
-    @return whether parsing should proceed
-    */
-    virtual bool null() = 0;
-
-    /*!
-    @brief a boolean value was read
-    @param[in] val  boolean value
-    @return whether parsing should proceed
-    */
-    virtual bool boolean(bool val) = 0;
-
-    /*!
-    @brief an integer number was read
-    @param[in] val  integer value
-    @return whether parsing should proceed
-    */
-    virtual bool number_integer(number_integer_t val) = 0;
-
-    /*!
-    @brief an unsigned integer number was read
-    @param[in] val  unsigned integer value
-    @return whether parsing should proceed
-    */
-    virtual bool number_unsigned(number_unsigned_t val) = 0;
-
-    /*!
-    @brief an floating-point number was read
-    @param[in] val  floating-point value
-    @param[in] s    raw token value
-    @return whether parsing should proceed
-    */
-    virtual bool number_float(number_float_t val, const string_t& s) = 0;
-
-    /*!
-    @brief a string was read
-    @param[in] val  string value
-    @return whether parsing should proceed
-    @note It is safe to move the passed string.
-    */
-    virtual bool string(string_t& val) = 0;
-
-    /*!
-    @brief the beginning of an object was read
-    @param[in] elements  number of object elements or -1 if unknown
-    @return whether parsing should proceed
-    @note binary formats may report the number of elements
-    */
-    virtual bool start_object(std::size_t elements) = 0;
-
-    /*!
-    @brief an object key was read
-    @param[in] val  object key
-    @return whether parsing should proceed
-    @note It is safe to move the passed string.
-    */
-    virtual bool key(string_t& val) = 0;
-
-    /*!
-    @brief the end of an object was read
-    @return whether parsing should proceed
-    */
-    virtual bool end_object() = 0;
-
-    /*!
-    @brief the beginning of an array was read
-    @param[in] elements  number of array elements or -1 if unknown
-    @return whether parsing should proceed
-    @note binary formats may report the number of elements
-    */
-    virtual bool start_array(std::size_t elements) = 0;
-
-    /*!
-    @brief the end of an array was read
-    @return whether parsing should proceed
-    */
-    virtual bool end_array() = 0;
-
-    /*!
-    @brief a parse error occurred
-    @param[in] position    the position in the input where the error occurs
-    @param[in] last_token  the last read token
-    @param[in] ex          an exception object describing the error
-    @return whether parsing should proceed (must return false)
-    */
-    virtual bool parse_error(std::size_t position,
-                             const std::string& last_token,
-                             const detail::exception& ex) = 0;
-
-    virtual ~json_sax() = default;
-};
-
-
-namespace detail
-{
-/*!
-@brief SAX implementation to create a JSON value from SAX events
-
-This class implements the @ref json_sax interface and processes the SAX events
-to create a JSON value which makes it basically a DOM parser. The structure or
-hierarchy of the JSON value is managed by the stack `ref_stack` which contains
-a pointer to the respective array or object for each recursion depth.
-
-After successful parsing, the value that is passed by reference to the
-constructor contains the parsed value.
-
-@tparam BasicJsonType  the JSON type
-*/
-template<typename BasicJsonType>
-class json_sax_dom_parser
-{
-  public:
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-
-    /*!
-    @param[in, out] r  reference to a JSON value that is manipulated while
-                       parsing
-    @param[in] allow_exceptions_  whether parse errors yield exceptions
-    */
-    explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)
-        : root(r), allow_exceptions(allow_exceptions_)
-    {}
-
-    bool null()
-    {
-        handle_value(nullptr);
-        return true;
-    }
-
-    bool boolean(bool val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_integer(number_integer_t val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_unsigned(number_unsigned_t val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_float(number_float_t val, const string_t& /*unused*/)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool string(string_t& val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool start_object(std::size_t len)
-    {
-        ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
-
-        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
-        {
-            JSON_THROW(out_of_range::create(408,
-                                            "excessive object size: " + std::to_string(len)));
-        }
-
-        return true;
-    }
-
-    bool key(string_t& val)
-    {
-        // add null at given key and store the reference for later
-        object_element = &(ref_stack.back()->m_value.object->operator[](val));
-        return true;
-    }
-
-    bool end_object()
-    {
-        ref_stack.pop_back();
-        return true;
-    }
-
-    bool start_array(std::size_t len)
-    {
-        ref_stack.push_back(handle_value(BasicJsonType::value_t::array));
-
-        if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
-        {
-            JSON_THROW(out_of_range::create(408,
-                                            "excessive array size: " + std::to_string(len)));
-        }
-
-        return true;
-    }
-
-    bool end_array()
-    {
-        ref_stack.pop_back();
-        return true;
-    }
-
-    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
-                     const detail::exception& ex)
-    {
-        errored = true;
-        if (allow_exceptions)
-        {
-            // determine the proper exception type from the id
-            switch ((ex.id / 100) % 100)
-            {
-                case 1:
-                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
-                case 4:
-                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
-                // LCOV_EXCL_START
-                case 2:
-                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
-                case 3:
-                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
-                case 5:
-                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
-                default:
-                    assert(false);
-                    // LCOV_EXCL_STOP
-            }
-        }
-        return false;
-    }
-
-    constexpr bool is_errored() const
-    {
-        return errored;
-    }
-
-  private:
-    /*!
-    @invariant If the ref stack is empty, then the passed value will be the new
-               root.
-    @invariant If the ref stack contains a value, then it is an array or an
-               object to which we can add elements
-    */
-    template<typename Value>
-    BasicJsonType* handle_value(Value&& v)
-    {
-        if (ref_stack.empty())
-        {
-            root = BasicJsonType(std::forward<Value>(v));
-            return &root;
-        }
-
-        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
-
-        if (ref_stack.back()->is_array())
-        {
-            ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));
-            return &(ref_stack.back()->m_value.array->back());
-        }
-        else
-        {
-            assert(object_element);
-            *object_element = BasicJsonType(std::forward<Value>(v));
-            return object_element;
-        }
-    }
-
-    /// the parsed JSON value
-    BasicJsonType& root;
-    /// stack to model hierarchy of values
-    std::vector<BasicJsonType*> ref_stack;
-    /// helper to hold the reference for the next object element
-    BasicJsonType* object_element = nullptr;
-    /// whether a syntax error occurred
-    bool errored = false;
-    /// whether to throw exceptions in case of errors
-    const bool allow_exceptions = true;
-};
-
-template<typename BasicJsonType>
-class json_sax_dom_callback_parser
-{
-  public:
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using parser_callback_t = typename BasicJsonType::parser_callback_t;
-    using parse_event_t = typename BasicJsonType::parse_event_t;
-
-    json_sax_dom_callback_parser(BasicJsonType& r,
-                                 const parser_callback_t cb,
-                                 const bool allow_exceptions_ = true)
-        : root(r), callback(cb), allow_exceptions(allow_exceptions_)
-    {
-        keep_stack.push_back(true);
-    }
-
-    bool null()
-    {
-        handle_value(nullptr);
-        return true;
-    }
-
-    bool boolean(bool val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_integer(number_integer_t val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_unsigned(number_unsigned_t val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool number_float(number_float_t val, const string_t& /*unused*/)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool string(string_t& val)
-    {
-        handle_value(val);
-        return true;
-    }
-
-    bool start_object(std::size_t len)
-    {
-        // check callback for object start
-        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);
-        keep_stack.push_back(keep);
-
-        auto val = handle_value(BasicJsonType::value_t::object, true);
-        ref_stack.push_back(val.second);
-
-        // check object limit
-        if (ref_stack.back())
-        {
-            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
-            {
-                JSON_THROW(out_of_range::create(408,
-                                                "excessive object size: " + std::to_string(len)));
-            }
-        }
-
-        return true;
-    }
-
-    bool key(string_t& val)
-    {
-        BasicJsonType k = BasicJsonType(val);
-
-        // check callback for key
-        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);
-        key_keep_stack.push_back(keep);
-
-        // add discarded value at given key and store the reference for later
-        if (keep and ref_stack.back())
-        {
-            object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);
-        }
-
-        return true;
-    }
-
-    bool end_object()
-    {
-        if (ref_stack.back())
-        {
-            if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))
-            {
-                // discard object
-                *ref_stack.back() = discarded;
-            }
-        }
-
-        assert(not ref_stack.empty());
-        assert(not keep_stack.empty());
-        ref_stack.pop_back();
-        keep_stack.pop_back();
-
-        if (not ref_stack.empty() and ref_stack.back())
-        {
-            // remove discarded value
-            if (ref_stack.back()->is_object())
-            {
-                for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
-                {
-                    if (it->is_discarded())
-                    {
-                        ref_stack.back()->erase(it);
-                        break;
-                    }
-                }
-            }
-        }
-
-        return true;
-    }
-
-    bool start_array(std::size_t len)
-    {
-        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);
-        keep_stack.push_back(keep);
-
-        auto val = handle_value(BasicJsonType::value_t::array, true);
-        ref_stack.push_back(val.second);
-
-        // check array limit
-        if (ref_stack.back())
-        {
-            if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size()))
-            {
-                JSON_THROW(out_of_range::create(408,
-                                                "excessive array size: " + std::to_string(len)));
-            }
-        }
-
-        return true;
-    }
-
-    bool end_array()
-    {
-        bool keep = true;
-
-        if (ref_stack.back())
-        {
-            keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
-            if (not keep)
-            {
-                // discard array
-                *ref_stack.back() = discarded;
-            }
-        }
-
-        assert(not ref_stack.empty());
-        assert(not keep_stack.empty());
-        ref_stack.pop_back();
-        keep_stack.pop_back();
-
-        // remove discarded value
-        if (not keep and not ref_stack.empty())
-        {
-            if (ref_stack.back()->is_array())
-            {
-                ref_stack.back()->m_value.array->pop_back();
-            }
-        }
-
-        return true;
-    }
-
-    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,
-                     const detail::exception& ex)
-    {
-        errored = true;
-        if (allow_exceptions)
-        {
-            // determine the proper exception type from the id
-            switch ((ex.id / 100) % 100)
-            {
-                case 1:
-                    JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex));
-                case 4:
-                    JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex));
-                // LCOV_EXCL_START
-                case 2:
-                    JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex));
-                case 3:
-                    JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex));
-                case 5:
-                    JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex));
-                default:
-                    assert(false);
-                    // LCOV_EXCL_STOP
-            }
-        }
-        return false;
-    }
-
-    constexpr bool is_errored() const
-    {
-        return errored;
-    }
-
-  private:
-    /*!
-    @param[in] v  value to add to the JSON value we build during parsing
-    @param[in] skip_callback  whether we should skip calling the callback
-               function; this is required after start_array() and
-               start_object() SAX events, because otherwise we would call the
-               callback function with an empty array or object, respectively.
-
-    @invariant If the ref stack is empty, then the passed value will be the new
-               root.
-    @invariant If the ref stack contains a value, then it is an array or an
-               object to which we can add elements
-
-    @return pair of boolean (whether value should be kept) and pointer (to the
-            passed value in the ref_stack hierarchy; nullptr if not kept)
-    */
-    template<typename Value>
-    std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)
-    {
-        assert(not keep_stack.empty());
-
-        // do not handle this value if we know it would be added to a discarded
-        // container
-        if (not keep_stack.back())
-        {
-            return {false, nullptr};
-        }
-
-        // create value
-        auto value = BasicJsonType(std::forward<Value>(v));
-
-        // check callback
-        const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);
-
-        // do not handle this value if we just learnt it shall be discarded
-        if (not keep)
-        {
-            return {false, nullptr};
-        }
-
-        if (ref_stack.empty())
-        {
-            root = std::move(value);
-            return {true, &root};
-        }
-
-        // skip this value if we already decided to skip the parent
-        // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)
-        if (not ref_stack.back())
-        {
-            return {false, nullptr};
-        }
-
-        // we now only expect arrays and objects
-        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
-
-        if (ref_stack.back()->is_array())
-        {
-            ref_stack.back()->m_value.array->push_back(std::move(value));
-            return {true, &(ref_stack.back()->m_value.array->back())};
-        }
-        else
-        {
-            // check if we should store an element for the current key
-            assert(not key_keep_stack.empty());
-            const bool store_element = key_keep_stack.back();
-            key_keep_stack.pop_back();
-
-            if (not store_element)
-            {
-                return {false, nullptr};
-            }
-
-            assert(object_element);
-            *object_element = std::move(value);
-            return {true, object_element};
-        }
-    }
-
-    /// the parsed JSON value
-    BasicJsonType& root;
-    /// stack to model hierarchy of values
-    std::vector<BasicJsonType*> ref_stack;
-    /// stack to manage which values to keep
-    std::vector<bool> keep_stack;
-    /// stack to manage which object keys to keep
-    std::vector<bool> key_keep_stack;
-    /// helper to hold the reference for the next object element
-    BasicJsonType* object_element = nullptr;
-    /// whether a syntax error occurred
-    bool errored = false;
-    /// callback function
-    const parser_callback_t callback = nullptr;
-    /// whether to throw exceptions in case of errors
-    const bool allow_exceptions = true;
-    /// a discarded value for the callback
-    BasicJsonType discarded = BasicJsonType::value_t::discarded;
-};
-
-template<typename BasicJsonType>
-class json_sax_acceptor
-{
-  public:
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-
-    bool null()
-    {
-        return true;
-    }
-
-    bool boolean(bool /*unused*/)
-    {
-        return true;
-    }
-
-    bool number_integer(number_integer_t /*unused*/)
-    {
-        return true;
-    }
-
-    bool number_unsigned(number_unsigned_t /*unused*/)
-    {
-        return true;
-    }
-
-    bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)
-    {
-        return true;
-    }
-
-    bool string(string_t& /*unused*/)
-    {
-        return true;
-    }
-
-    bool start_object(std::size_t  /*unused*/ = std::size_t(-1))
-    {
-        return true;
-    }
-
-    bool key(string_t& /*unused*/)
-    {
-        return true;
-    }
-
-    bool end_object()
-    {
-        return true;
-    }
-
-    bool start_array(std::size_t  /*unused*/ = std::size_t(-1))
-    {
-        return true;
-    }
-
-    bool end_array()
-    {
-        return true;
-    }
-
-    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)
-    {
-        return false;
-    }
-};
-}  // namespace detail
-
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/lexer.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-////////////
-// parser //
-////////////
-
-/*!
-@brief syntax analysis
-
-This class implements a recursive decent parser.
-*/
-template<typename BasicJsonType>
-class parser
-{
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using lexer_t = lexer<BasicJsonType>;
-    using token_type = typename lexer_t::token_type;
-
-  public:
-    enum class parse_event_t : uint8_t
-    {
-        /// the parser read `{` and started to process a JSON object
-        object_start,
-        /// the parser read `}` and finished processing a JSON object
-        object_end,
-        /// the parser read `[` and started to process a JSON array
-        array_start,
-        /// the parser read `]` and finished processing a JSON array
-        array_end,
-        /// the parser read a key of a value in an object
-        key,
-        /// the parser finished reading a JSON value
-        value
-    };
-
-    using parser_callback_t =
-        std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
-
-    /// a parser reading from an input adapter
-    explicit parser(detail::input_adapter_t&& adapter,
-                    const parser_callback_t cb = nullptr,
-                    const bool allow_exceptions_ = true)
-        : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_)
-    {
-        // read first token
-        get_token();
-    }
-
-    /*!
-    @brief public parser interface
-
-    @param[in] strict      whether to expect the last token to be EOF
-    @param[in,out] result  parsed JSON value
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-    */
-    void parse(const bool strict, BasicJsonType& result)
-    {
-        if (callback)
-        {
-            json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);
-            sax_parse_internal(&sdp);
-            result.assert_invariant();
-
-            // in strict mode, input must be completely read
-            if (strict and (get_token() != token_type::end_of_input))
-            {
-                sdp.parse_error(m_lexer.get_position(),
-                                m_lexer.get_token_string(),
-                                parse_error::create(101, m_lexer.get_position(),
-                                                    exception_message(token_type::end_of_input, "value")));
-            }
-
-            // in case of an error, return discarded value
-            if (sdp.is_errored())
-            {
-                result = value_t::discarded;
-                return;
-            }
-
-            // set top-level value to null if it was discarded by the callback
-            // function
-            if (result.is_discarded())
-            {
-                result = nullptr;
-            }
-        }
-        else
-        {
-            json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);
-            sax_parse_internal(&sdp);
-            result.assert_invariant();
-
-            // in strict mode, input must be completely read
-            if (strict and (get_token() != token_type::end_of_input))
-            {
-                sdp.parse_error(m_lexer.get_position(),
-                                m_lexer.get_token_string(),
-                                parse_error::create(101, m_lexer.get_position(),
-                                                    exception_message(token_type::end_of_input, "value")));
-            }
-
-            // in case of an error, return discarded value
-            if (sdp.is_errored())
-            {
-                result = value_t::discarded;
-                return;
-            }
-        }
-    }
-
-    /*!
-    @brief public accept interface
-
-    @param[in] strict  whether to expect the last token to be EOF
-    @return whether the input is a proper JSON text
-    */
-    bool accept(const bool strict = true)
-    {
-        json_sax_acceptor<BasicJsonType> sax_acceptor;
-        return sax_parse(&sax_acceptor, strict);
-    }
-
-    template <typename SAX>
-    bool sax_parse(SAX* sax, const bool strict = true)
-    {
-        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
-        const bool result = sax_parse_internal(sax);
-
-        // strict mode: next byte must be EOF
-        if (result and strict and (get_token() != token_type::end_of_input))
-        {
-            return sax->parse_error(m_lexer.get_position(),
-                                    m_lexer.get_token_string(),
-                                    parse_error::create(101, m_lexer.get_position(),
-                                            exception_message(token_type::end_of_input, "value")));
-        }
-
-        return result;
-    }
-
-  private:
-    template <typename SAX>
-    bool sax_parse_internal(SAX* sax)
-    {
-        // stack to remember the hierarchy of structured values we are parsing
-        // true = array; false = object
-        std::vector<bool> states;
-        // value to avoid a goto (see comment where set to true)
-        bool skip_to_state_evaluation = false;
-
-        while (true)
-        {
-            if (not skip_to_state_evaluation)
-            {
-                // invariant: get_token() was called before each iteration
-                switch (last_token)
-                {
-                    case token_type::begin_object:
-                    {
-                        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
-                        {
-                            return false;
-                        }
-
-                        // closing } -> we are done
-                        if (get_token() == token_type::end_object)
-                        {
-                            if (JSON_UNLIKELY(not sax->end_object()))
-                            {
-                                return false;
-                            }
-                            break;
-                        }
-
-                        // parse key
-                        if (JSON_UNLIKELY(last_token != token_type::value_string))
-                        {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(),
-                                                            exception_message(token_type::value_string, "object key")));
-                        }
-                        if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
-                        {
-                            return false;
-                        }
-
-                        // parse separator (:)
-                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
-                        {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(),
-                                                            exception_message(token_type::name_separator, "object separator")));
-                        }
-
-                        // remember we are now inside an object
-                        states.push_back(false);
-
-                        // parse values
-                        get_token();
-                        continue;
-                    }
-
-                    case token_type::begin_array:
-                    {
-                        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
-                        {
-                            return false;
-                        }
-
-                        // closing ] -> we are done
-                        if (get_token() == token_type::end_array)
-                        {
-                            if (JSON_UNLIKELY(not sax->end_array()))
-                            {
-                                return false;
-                            }
-                            break;
-                        }
-
-                        // remember we are now inside an array
-                        states.push_back(true);
-
-                        // parse values (no need to call get_token)
-                        continue;
-                    }
-
-                    case token_type::value_float:
-                    {
-                        const auto res = m_lexer.get_number_float();
-
-                        if (JSON_UNLIKELY(not std::isfinite(res)))
-                        {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
-                        }
-                        else
-                        {
-                            if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string())))
-                            {
-                                return false;
-                            }
-                            break;
-                        }
-                    }
-
-                    case token_type::literal_false:
-                    {
-                        if (JSON_UNLIKELY(not sax->boolean(false)))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::literal_null:
-                    {
-                        if (JSON_UNLIKELY(not sax->null()))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::literal_true:
-                    {
-                        if (JSON_UNLIKELY(not sax->boolean(true)))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::value_integer:
-                    {
-                        if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer())))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::value_string:
-                    {
-                        if (JSON_UNLIKELY(not sax->string(m_lexer.get_string())))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::value_unsigned:
-                    {
-                        if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned())))
-                        {
-                            return false;
-                        }
-                        break;
-                    }
-
-                    case token_type::parse_error:
-                    {
-                        // using "uninitialized" to avoid "expected" message
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(),
-                                                        exception_message(token_type::uninitialized, "value")));
-                    }
-
-                    default: // the last token was unexpected
-                    {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(),
-                                                        exception_message(token_type::literal_or_value, "value")));
-                    }
-                }
-            }
-            else
-            {
-                skip_to_state_evaluation = false;
-            }
-
-            // we reached this line after we successfully parsed a value
-            if (states.empty())
-            {
-                // empty stack: we reached the end of the hierarchy: done
-                return true;
-            }
-            else
-            {
-                if (states.back())  // array
-                {
-                    // comma -> next value
-                    if (get_token() == token_type::value_separator)
-                    {
-                        // parse a new value
-                        get_token();
-                        continue;
-                    }
-
-                    // closing ]
-                    if (JSON_LIKELY(last_token == token_type::end_array))
-                    {
-                        if (JSON_UNLIKELY(not sax->end_array()))
-                        {
-                            return false;
-                        }
-
-                        // We are done with this array. Before we can parse a
-                        // new value, we need to evaluate the new state first.
-                        // By setting skip_to_state_evaluation to false, we
-                        // are effectively jumping to the beginning of this if.
-                        assert(not states.empty());
-                        states.pop_back();
-                        skip_to_state_evaluation = true;
-                        continue;
-                    }
-                    else
-                    {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(),
-                                                        exception_message(token_type::end_array, "array")));
-                    }
-                }
-                else  // object
-                {
-                    // comma -> next value
-                    if (get_token() == token_type::value_separator)
-                    {
-                        // parse key
-                        if (JSON_UNLIKELY(get_token() != token_type::value_string))
-                        {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(),
-                                                            exception_message(token_type::value_string, "object key")));
-                        }
-                        else
-                        {
-                            if (JSON_UNLIKELY(not sax->key(m_lexer.get_string())))
-                            {
-                                return false;
-                            }
-                        }
-
-                        // parse separator (:)
-                        if (JSON_UNLIKELY(get_token() != token_type::name_separator))
-                        {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(),
-                                                            exception_message(token_type::name_separator, "object separator")));
-                        }
-
-                        // parse values
-                        get_token();
-                        continue;
-                    }
-
-                    // closing }
-                    if (JSON_LIKELY(last_token == token_type::end_object))
-                    {
-                        if (JSON_UNLIKELY(not sax->end_object()))
-                        {
-                            return false;
-                        }
-
-                        // We are done with this object. Before we can parse a
-                        // new value, we need to evaluate the new state first.
-                        // By setting skip_to_state_evaluation to false, we
-                        // are effectively jumping to the beginning of this if.
-                        assert(not states.empty());
-                        states.pop_back();
-                        skip_to_state_evaluation = true;
-                        continue;
-                    }
-                    else
-                    {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(),
-                                                        exception_message(token_type::end_object, "object")));
-                    }
-                }
-            }
-        }
-    }
-
-    /// get next token from lexer
-    token_type get_token()
-    {
-        return (last_token = m_lexer.scan());
-    }
-
-    std::string exception_message(const token_type expected, const std::string& context)
-    {
-        std::string error_msg = "syntax error ";
-
-        if (not context.empty())
-        {
-            error_msg += "while parsing " + context + " ";
-        }
-
-        error_msg += "- ";
-
-        if (last_token == token_type::parse_error)
-        {
-            error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
-                         m_lexer.get_token_string() + "'";
-        }
-        else
-        {
-            error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token));
-        }
-
-        if (expected != token_type::uninitialized)
-        {
-            error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
-        }
-
-        return error_msg;
-    }
-
-  private:
-    /// callback function
-    const parser_callback_t callback = nullptr;
-    /// the type of the last read token
-    token_type last_token = token_type::uninitialized;
-    /// the lexer
-    lexer_t m_lexer;
-    /// whether to throw exceptions in case of errors
-    const bool allow_exceptions = true;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
-
-
-#include <cstddef> // ptrdiff_t
-#include <limits>  // numeric_limits
-
-namespace nlohmann
-{
-namespace detail
-{
-/*
-@brief an iterator for primitive JSON types
-
-This class models an iterator for primitive JSON types (boolean, number,
-string). It's only purpose is to allow the iterator/const_iterator classes
-to "iterate" over primitive values. Internally, the iterator is modeled by
-a `difference_type` variable. Value begin_value (`0`) models the begin,
-end_value (`1`) models past the end.
-*/
-class primitive_iterator_t
-{
-  private:
-    using difference_type = std::ptrdiff_t;
-    static constexpr difference_type begin_value = 0;
-    static constexpr difference_type end_value = begin_value + 1;
-
-    /// iterator as signed integer type
-    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();
-
-  public:
-    constexpr difference_type get_value() const noexcept
-    {
-        return m_it;
-    }
-
-    /// set iterator to a defined beginning
-    void set_begin() noexcept
-    {
-        m_it = begin_value;
-    }
-
-    /// set iterator to a defined past the end
-    void set_end() noexcept
-    {
-        m_it = end_value;
-    }
-
-    /// return whether the iterator can be dereferenced
-    constexpr bool is_begin() const noexcept
-    {
-        return m_it == begin_value;
-    }
-
-    /// return whether the iterator is at end
-    constexpr bool is_end() const noexcept
-    {
-        return m_it == end_value;
-    }
-
-    friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-    {
-        return lhs.m_it == rhs.m_it;
-    }
-
-    friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-    {
-        return lhs.m_it < rhs.m_it;
-    }
-
-    primitive_iterator_t operator+(difference_type n) noexcept
-    {
-        auto result = *this;
-        result += n;
-        return result;
-    }
-
-    friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-    {
-        return lhs.m_it - rhs.m_it;
-    }
-
-    primitive_iterator_t& operator++() noexcept
-    {
-        ++m_it;
-        return *this;
-    }
-
-    primitive_iterator_t const operator++(int) noexcept
-    {
-        auto result = *this;
-        ++m_it;
-        return result;
-    }
-
-    primitive_iterator_t& operator--() noexcept
-    {
-        --m_it;
-        return *this;
-    }
-
-    primitive_iterator_t const operator--(int) noexcept
-    {
-        auto result = *this;
-        --m_it;
-        return result;
-    }
-
-    primitive_iterator_t& operator+=(difference_type n) noexcept
-    {
-        m_it += n;
-        return *this;
-    }
-
-    primitive_iterator_t& operator-=(difference_type n) noexcept
-    {
-        m_it -= n;
-        return *this;
-    }
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/iterators/internal_iterator.hpp>
-
-
-// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-/*!
-@brief an iterator value
-
-@note This structure could easily be a union, but MSVC currently does not allow
-unions members with complex constructors, see https://github.com/nlohmann/json/pull/105.
-*/
-template<typename BasicJsonType> struct internal_iterator
-{
-    /// iterator for JSON objects
-    typename BasicJsonType::object_t::iterator object_iterator {};
-    /// iterator for JSON arrays
-    typename BasicJsonType::array_t::iterator array_iterator {};
-    /// generic iterator for all other types
-    primitive_iterator_t primitive_iterator {};
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/iterators/iter_impl.hpp>
-
-
-#include <ciso646> // not
-#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next
-#include <type_traits> // conditional, is_const, remove_const
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/iterators/internal_iterator.hpp>
-
-// #include <nlohmann/detail/iterators/primitive_iterator.hpp>
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-// forward declare, to be able to friend it later on
-template<typename IteratorType> class iteration_proxy;
-template<typename IteratorType> class iteration_proxy_value;
-
-/*!
-@brief a template for a bidirectional iterator for the @ref basic_json class
-This class implements a both iterators (iterator and const_iterator) for the
-@ref basic_json class.
-@note An iterator is called *initialized* when a pointer to a JSON value has
-      been set (e.g., by a constructor or a copy assignment). If the iterator is
-      default-constructed, it is *uninitialized* and most methods are undefined.
-      **The library uses assertions to detect calls on uninitialized iterators.**
-@requirement The class satisfies the following concept requirements:
--
-[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
-  The iterator that can be moved can be moved in both directions (i.e.
-  incremented and decremented).
-@since version 1.0.0, simplified in version 2.0.9, change to bidirectional
-       iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)
-*/
-template<typename BasicJsonType>
-class iter_impl
-{
-    /// allow basic_json to access private members
-    friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
-    friend BasicJsonType;
-    friend iteration_proxy<iter_impl>;
-    friend iteration_proxy_value<iter_impl>;
-
-    using object_t = typename BasicJsonType::object_t;
-    using array_t = typename BasicJsonType::array_t;
-    // make sure BasicJsonType is basic_json or const basic_json
-    static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
-                  "iter_impl only accepts (const) basic_json");
-
-  public:
-
-    /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.
-    /// The C++ Standard has never required user-defined iterators to derive from std::iterator.
-    /// A user-defined iterator should provide publicly accessible typedefs named
-    /// iterator_category, value_type, difference_type, pointer, and reference.
-    /// Note that value_type is required to be non-const, even for constant iterators.
-    using iterator_category = std::bidirectional_iterator_tag;
-
-    /// the type of the values when the iterator is dereferenced
-    using value_type = typename BasicJsonType::value_type;
-    /// a type to represent differences between iterators
-    using difference_type = typename BasicJsonType::difference_type;
-    /// defines a pointer to the type iterated over (value_type)
-    using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,
-          typename BasicJsonType::const_pointer,
-          typename BasicJsonType::pointer>::type;
-    /// defines a reference to the type iterated over (value_type)
-    using reference =
-        typename std::conditional<std::is_const<BasicJsonType>::value,
-        typename BasicJsonType::const_reference,
-        typename BasicJsonType::reference>::type;
-
-    /// default constructor
-    iter_impl() = default;
-
-    /*!
-    @brief constructor for a given JSON instance
-    @param[in] object  pointer to a JSON object for this iterator
-    @pre object != nullptr
-    @post The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    explicit iter_impl(pointer object) noexcept : m_object(object)
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                m_it.object_iterator = typename object_t::iterator();
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_it.array_iterator = typename array_t::iterator();
-                break;
-            }
-
-            default:
-            {
-                m_it.primitive_iterator = primitive_iterator_t();
-                break;
-            }
-        }
-    }
-
-    /*!
-    @note The conventional copy constructor and copy assignment are implicitly
-          defined. Combined with the following converting constructor and
-          assignment, they support: (1) copy from iterator to iterator, (2)
-          copy from const iterator to const iterator, and (3) conversion from
-          iterator to const iterator. However conversion from const iterator
-          to iterator is not defined.
-    */
-
-    /*!
-    @brief converting constructor
-    @param[in] other  non-const iterator to copy from
-    @note It is not checked whether @a other is initialized.
-    */
-    iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
-        : m_object(other.m_object), m_it(other.m_it) {}
-
-    /*!
-    @brief converting assignment
-    @param[in,out] other  non-const iterator to copy from
-    @return const/non-const iterator
-    @note It is not checked whether @a other is initialized.
-    */
-    iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
-    {
-        m_object = other.m_object;
-        m_it = other.m_it;
-        return *this;
-    }
-
-  private:
-    /*!
-    @brief set the iterator to the first value
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    void set_begin() noexcept
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                m_it.object_iterator = m_object->m_value.object->begin();
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_it.array_iterator = m_object->m_value.array->begin();
-                break;
-            }
-
-            case value_t::null:
-            {
-                // set to end so begin()==end() is true: null is empty
-                m_it.primitive_iterator.set_end();
-                break;
-            }
-
-            default:
-            {
-                m_it.primitive_iterator.set_begin();
-                break;
-            }
-        }
-    }
-
-    /*!
-    @brief set the iterator past the last value
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    void set_end() noexcept
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                m_it.object_iterator = m_object->m_value.object->end();
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_it.array_iterator = m_object->m_value.array->end();
-                break;
-            }
-
-            default:
-            {
-                m_it.primitive_iterator.set_end();
-                break;
-            }
-        }
-    }
-
-  public:
-    /*!
-    @brief return a reference to the value pointed to by the iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    reference operator*() const
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                assert(m_it.object_iterator != m_object->m_value.object->end());
-                return m_it.object_iterator->second;
-            }
-
-            case value_t::array:
-            {
-                assert(m_it.array_iterator != m_object->m_value.array->end());
-                return *m_it.array_iterator;
-            }
-
-            case value_t::null:
-                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-
-            default:
-            {
-                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
-                {
-                    return *m_object;
-                }
-
-                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-            }
-        }
-    }
-
-    /*!
-    @brief dereference the iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    pointer operator->() const
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                assert(m_it.object_iterator != m_object->m_value.object->end());
-                return &(m_it.object_iterator->second);
-            }
-
-            case value_t::array:
-            {
-                assert(m_it.array_iterator != m_object->m_value.array->end());
-                return &*m_it.array_iterator;
-            }
-
-            default:
-            {
-                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
-                {
-                    return m_object;
-                }
-
-                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-            }
-        }
-    }
-
-    /*!
-    @brief post-increment (it++)
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl const operator++(int)
-    {
-        auto result = *this;
-        ++(*this);
-        return result;
-    }
-
-    /*!
-    @brief pre-increment (++it)
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl& operator++()
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                std::advance(m_it.object_iterator, 1);
-                break;
-            }
-
-            case value_t::array:
-            {
-                std::advance(m_it.array_iterator, 1);
-                break;
-            }
-
-            default:
-            {
-                ++m_it.primitive_iterator;
-                break;
-            }
-        }
-
-        return *this;
-    }
-
-    /*!
-    @brief post-decrement (it--)
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl const operator--(int)
-    {
-        auto result = *this;
-        --(*this);
-        return result;
-    }
-
-    /*!
-    @brief pre-decrement (--it)
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl& operator--()
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-            {
-                std::advance(m_it.object_iterator, -1);
-                break;
-            }
-
-            case value_t::array:
-            {
-                std::advance(m_it.array_iterator, -1);
-                break;
-            }
-
-            default:
-            {
-                --m_it.primitive_iterator;
-                break;
-            }
-        }
-
-        return *this;
-    }
-
-    /*!
-    @brief  comparison: equal
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator==(const iter_impl& other) const
-    {
-        // if objects are not the same, the comparison is undefined
-        if (JSON_UNLIKELY(m_object != other.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
-        }
-
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-                return (m_it.object_iterator == other.m_it.object_iterator);
-
-            case value_t::array:
-                return (m_it.array_iterator == other.m_it.array_iterator);
-
-            default:
-                return (m_it.primitive_iterator == other.m_it.primitive_iterator);
-        }
-    }
-
-    /*!
-    @brief  comparison: not equal
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator!=(const iter_impl& other) const
-    {
-        return not operator==(other);
-    }
-
-    /*!
-    @brief  comparison: smaller
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator<(const iter_impl& other) const
-    {
-        // if objects are not the same, the comparison is undefined
-        if (JSON_UNLIKELY(m_object != other.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
-        }
-
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-                JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
-
-            case value_t::array:
-                return (m_it.array_iterator < other.m_it.array_iterator);
-
-            default:
-                return (m_it.primitive_iterator < other.m_it.primitive_iterator);
-        }
-    }
-
-    /*!
-    @brief  comparison: less than or equal
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator<=(const iter_impl& other) const
-    {
-        return not other.operator < (*this);
-    }
-
-    /*!
-    @brief  comparison: greater than
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator>(const iter_impl& other) const
-    {
-        return not operator<=(other);
-    }
-
-    /*!
-    @brief  comparison: greater than or equal
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    bool operator>=(const iter_impl& other) const
-    {
-        return not operator<(other);
-    }
-
-    /*!
-    @brief  add to iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl& operator+=(difference_type i)
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
-
-            case value_t::array:
-            {
-                std::advance(m_it.array_iterator, i);
-                break;
-            }
-
-            default:
-            {
-                m_it.primitive_iterator += i;
-                break;
-            }
-        }
-
-        return *this;
-    }
-
-    /*!
-    @brief  subtract from iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl& operator-=(difference_type i)
-    {
-        return operator+=(-i);
-    }
-
-    /*!
-    @brief  add to iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl operator+(difference_type i) const
-    {
-        auto result = *this;
-        result += i;
-        return result;
-    }
-
-    /*!
-    @brief  addition of distance and iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    friend iter_impl operator+(difference_type i, const iter_impl& it)
-    {
-        auto result = it;
-        result += i;
-        return result;
-    }
-
-    /*!
-    @brief  subtract from iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    iter_impl operator-(difference_type i) const
-    {
-        auto result = *this;
-        result -= i;
-        return result;
-    }
-
-    /*!
-    @brief  return difference
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    difference_type operator-(const iter_impl& other) const
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
-
-            case value_t::array:
-                return m_it.array_iterator - other.m_it.array_iterator;
-
-            default:
-                return m_it.primitive_iterator - other.m_it.primitive_iterator;
-        }
-    }
-
-    /*!
-    @brief  access to successor
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    reference operator[](difference_type n) const
-    {
-        assert(m_object != nullptr);
-
-        switch (m_object->m_type)
-        {
-            case value_t::object:
-                JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
-
-            case value_t::array:
-                return *std::next(m_it.array_iterator, n);
-
-            case value_t::null:
-                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-
-            default:
-            {
-                if (JSON_LIKELY(m_it.primitive_iterator.get_value() == -n))
-                {
-                    return *m_object;
-                }
-
-                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-            }
-        }
-    }
-
-    /*!
-    @brief  return the key of an object iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    const typename object_t::key_type& key() const
-    {
-        assert(m_object != nullptr);
-
-        if (JSON_LIKELY(m_object->is_object()))
-        {
-            return m_it.object_iterator->first;
-        }
-
-        JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
-    }
-
-    /*!
-    @brief  return the value of an iterator
-    @pre The iterator is initialized; i.e. `m_object != nullptr`.
-    */
-    reference value() const
-    {
-        return operator*();
-    }
-
-  private:
-    /// associated JSON instance
-    pointer m_object = nullptr;
-    /// the actual iterator of the associated instance
-    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it;
-};
-}  // namespace detail
-} // namespace nlohmann
-// #include <nlohmann/detail/iterators/iteration_proxy.hpp>
-
-// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>
-
-
-#include <cstddef> // ptrdiff_t
-#include <iterator> // reverse_iterator
-#include <utility> // declval
-
-namespace nlohmann
-{
-namespace detail
-{
-//////////////////////
-// reverse_iterator //
-//////////////////////
-
-/*!
-@brief a template for a reverse iterator class
-
-@tparam Base the base iterator type to reverse. Valid types are @ref
-iterator (to create @ref reverse_iterator) and @ref const_iterator (to
-create @ref const_reverse_iterator).
-
-@requirement The class satisfies the following concept requirements:
--
-[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):
-  The iterator that can be moved can be moved in both directions (i.e.
-  incremented and decremented).
-- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):
-  It is possible to write to the pointed-to element (only if @a Base is
-  @ref iterator).
-
-@since version 1.0.0
-*/
-template<typename Base>
-class json_reverse_iterator : public std::reverse_iterator<Base>
-{
-  public:
-    using difference_type = std::ptrdiff_t;
-    /// shortcut to the reverse iterator adapter
-    using base_iterator = std::reverse_iterator<Base>;
-    /// the reference type for the pointed-to element
-    using reference = typename Base::reference;
-
-    /// create reverse iterator from iterator
-    explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
-        : base_iterator(it) {}
-
-    /// create reverse iterator from base class
-    explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
-
-    /// post-increment (it++)
-    json_reverse_iterator const operator++(int)
-    {
-        return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
-    }
-
-    /// pre-increment (++it)
-    json_reverse_iterator& operator++()
-    {
-        return static_cast<json_reverse_iterator&>(base_iterator::operator++());
-    }
-
-    /// post-decrement (it--)
-    json_reverse_iterator const operator--(int)
-    {
-        return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
-    }
-
-    /// pre-decrement (--it)
-    json_reverse_iterator& operator--()
-    {
-        return static_cast<json_reverse_iterator&>(base_iterator::operator--());
-    }
-
-    /// add to iterator
-    json_reverse_iterator& operator+=(difference_type i)
-    {
-        return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
-    }
-
-    /// add to iterator
-    json_reverse_iterator operator+(difference_type i) const
-    {
-        return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
-    }
-
-    /// subtract from iterator
-    json_reverse_iterator operator-(difference_type i) const
-    {
-        return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
-    }
-
-    /// return difference
-    difference_type operator-(const json_reverse_iterator& other) const
-    {
-        return base_iterator(*this) - base_iterator(other);
-    }
-
-    /// access to successor
-    reference operator[](difference_type n) const
-    {
-        return *(this->operator+(n));
-    }
-
-    /// return the key of an object iterator
-    auto key() const -> decltype(std::declval<Base>().key())
-    {
-        auto it = --this->base();
-        return it.key();
-    }
-
-    /// return the value of an iterator
-    reference value() const
-    {
-        auto it = --this->base();
-        return it.operator * ();
-    }
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/output/output_adapters.hpp>
-
-
-#include <algorithm> // copy
-#include <cstddef> // size_t
-#include <ios> // streamsize
-#include <iterator> // back_inserter
-#include <memory> // shared_ptr, make_shared
-#include <ostream> // basic_ostream
-#include <string> // basic_string
-#include <vector> // vector
-
-namespace nlohmann
-{
-namespace detail
-{
-/// abstract output adapter interface
-template<typename CharType> struct output_adapter_protocol
-{
-    virtual void write_character(CharType c) = 0;
-    virtual void write_characters(const CharType* s, std::size_t length) = 0;
-    virtual ~output_adapter_protocol() = default;
-};
-
-/// a type to simplify interfaces
-template<typename CharType>
-using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
-
-/// output adapter for byte vectors
-template<typename CharType>
-class output_vector_adapter : public output_adapter_protocol<CharType>
-{
-  public:
-    explicit output_vector_adapter(std::vector<CharType>& vec) noexcept
-        : v(vec)
-    {}
-
-    void write_character(CharType c) override
-    {
-        v.push_back(c);
-    }
-
-    void write_characters(const CharType* s, std::size_t length) override
-    {
-        std::copy(s, s + length, std::back_inserter(v));
-    }
-
-  private:
-    std::vector<CharType>& v;
-};
-
-/// output adapter for output streams
-template<typename CharType>
-class output_stream_adapter : public output_adapter_protocol<CharType>
-{
-  public:
-    explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept
-        : stream(s)
-    {}
-
-    void write_character(CharType c) override
-    {
-        stream.put(c);
-    }
-
-    void write_characters(const CharType* s, std::size_t length) override
-    {
-        stream.write(s, static_cast<std::streamsize>(length));
-    }
-
-  private:
-    std::basic_ostream<CharType>& stream;
-};
-
-/// output adapter for basic_string
-template<typename CharType, typename StringType = std::basic_string<CharType>>
-class output_string_adapter : public output_adapter_protocol<CharType>
-{
-  public:
-    explicit output_string_adapter(StringType& s) noexcept
-        : str(s)
-    {}
-
-    void write_character(CharType c) override
-    {
-        str.push_back(c);
-    }
-
-    void write_characters(const CharType* s, std::size_t length) override
-    {
-        str.append(s, length);
-    }
-
-  private:
-    StringType& str;
-};
-
-template<typename CharType, typename StringType = std::basic_string<CharType>>
-class output_adapter
-{
-  public:
-    output_adapter(std::vector<CharType>& vec)
-        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
-
-    output_adapter(std::basic_ostream<CharType>& s)
-        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
-
-    output_adapter(StringType& s)
-        : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}
-
-    operator output_adapter_t<CharType>()
-    {
-        return oa;
-    }
-
-  private:
-    output_adapter_t<CharType> oa = nullptr;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/input/binary_reader.hpp>
-
-
-#include <algorithm> // generate_n
-#include <array> // array
-#include <cassert> // assert
-#include <cmath> // ldexp
-#include <cstddef> // size_t
-#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
-#include <cstdio> // snprintf
-#include <cstring> // memcpy
-#include <iterator> // back_inserter
-#include <limits> // numeric_limits
-#include <string> // char_traits, string
-#include <utility> // make_pair, move
-
-// #include <nlohmann/detail/input/input_adapters.hpp>
-
-// #include <nlohmann/detail/input/json_sax.hpp>
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/meta/is_sax.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-///////////////////
-// binary reader //
-///////////////////
-
-/*!
-@brief deserialization of CBOR, MessagePack, and UBJSON values
-*/
-template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>>
-class binary_reader
-{
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using json_sax_t = SAX;
-
-  public:
-    /*!
-    @brief create a binary reader
-
-    @param[in] adapter  input adapter to read from
-    */
-    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
-    {
-        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
-        assert(ia);
-    }
-
-    /*!
-    @param[in] format  the binary format to parse
-    @param[in] sax_    a SAX event processor
-    @param[in] strict  whether to expect the input to be consumed completed
-
-    @return
-    */
-    bool sax_parse(const input_format_t format,
-                   json_sax_t* sax_,
-                   const bool strict = true)
-    {
-        sax = sax_;
-        bool result = false;
-
-        switch (format)
-        {
-            case input_format_t::bson:
-                result = parse_bson_internal();
-                break;
-
-            case input_format_t::cbor:
-                result = parse_cbor_internal();
-                break;
-
-            case input_format_t::msgpack:
-                result = parse_msgpack_internal();
-                break;
-
-            case input_format_t::ubjson:
-                result = parse_ubjson_internal();
-                break;
-
-            // LCOV_EXCL_START
-            default:
-                assert(false);
-                // LCOV_EXCL_STOP
-        }
-
-        // strict mode: next byte must be EOF
-        if (result and strict)
-        {
-            if (format == input_format_t::ubjson)
-            {
-                get_ignore_noop();
-            }
-            else
-            {
-                get();
-            }
-
-            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
-            {
-                return sax->parse_error(chars_read, get_token_string(),
-                                        parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value")));
-            }
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief determine system byte order
-
-    @return true if and only if system's byte order is little endian
-
-    @note from http://stackoverflow.com/a/1001328/266378
-    */
-    static constexpr bool little_endianess(int num = 1) noexcept
-    {
-        return (*reinterpret_cast<char*>(&num) == 1);
-    }
-
-  private:
-    //////////
-    // BSON //
-    //////////
-
-    /*!
-    @brief Reads in a BSON-object and passes it to the SAX-parser.
-    @return whether a valid BSON-value was passed to the SAX parser
-    */
-    bool parse_bson_internal()
-    {
-        std::int32_t document_size;
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
-
-        if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
-        {
-            return false;
-        }
-
-        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false)))
-        {
-            return false;
-        }
-
-        return sax->end_object();
-    }
-
-    /*!
-    @brief Parses a C-style string from the BSON input.
-    @param[in, out] result  A reference to the string variable where the read
-                            string is to be stored.
-    @return `true` if the \x00-byte indicating the end of the string was
-             encountered before the EOF; false` indicates an unexpected EOF.
-    */
-    bool get_bson_cstr(string_t& result)
-    {
-        auto out = std::back_inserter(result);
-        while (true)
-        {
-            get();
-            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring")))
-            {
-                return false;
-            }
-            if (current == 0x00)
-            {
-                return true;
-            }
-            *out++ = static_cast<char>(current);
-        }
-
-        return true;
-    }
-
-    /*!
-    @brief Parses a zero-terminated string of length @a len from the BSON
-           input.
-    @param[in] len  The length (including the zero-byte at the end) of the
-                    string to be read.
-    @param[in, out] result  A reference to the string variable where the read
-                            string is to be stored.
-    @tparam NumberType The type of the length @a len
-    @pre len >= 1
-    @return `true` if the string was successfully parsed
-    */
-    template<typename NumberType>
-    bool get_bson_string(const NumberType len, string_t& result)
-    {
-        if (JSON_UNLIKELY(len < 1))
-        {
-            auto last_token = get_token_string();
-            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string")));
-        }
-
-        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof();
-    }
-
-    /*!
-    @brief Read a BSON document element of the given @a element_type.
-    @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html
-    @param[in] element_type_parse_position The position in the input stream,
-               where the `element_type` was read.
-    @warning Not all BSON element types are supported yet. An unsupported
-             @a element_type will give rise to a parse_error.114:
-             Unsupported BSON record type 0x...
-    @return whether a valid BSON-object/array was passed to the SAX parser
-    */
-    bool parse_bson_element_internal(const int element_type,
-                                     const std::size_t element_type_parse_position)
-    {
-        switch (element_type)
-        {
-            case 0x01: // double
-            {
-                double number;
-                return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 0x02: // string
-            {
-                std::int32_t len;
-                string_t value;
-                return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value);
-            }
-
-            case 0x03: // object
-            {
-                return parse_bson_internal();
-            }
-
-            case 0x04: // array
-            {
-                return parse_bson_array();
-            }
-
-            case 0x08: // boolean
-            {
-                return sax->boolean(get() != 0);
-            }
-
-            case 0x0A: // null
-            {
-                return sax->null();
-            }
-
-            case 0x10: // int32
-            {
-                std::int32_t value;
-                return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value);
-            }
-
-            case 0x12: // int64
-            {
-                std::int64_t value;
-                return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value);
-            }
-
-            default: // anything else not supported (yet)
-            {
-                char cr[3];
-                (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type));
-                return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr)));
-            }
-        }
-    }
-
-    /*!
-    @brief Read a BSON element list (as specified in the BSON-spec)
-
-    The same binary layout is used for objects and arrays, hence it must be
-    indicated with the argument @a is_array which one is expected
-    (true --> array, false --> object).
-
-    @param[in] is_array Determines if the element list being read is to be
-                        treated as an object (@a is_array == false), or as an
-                        array (@a is_array == true).
-    @return whether a valid BSON-object/array was passed to the SAX parser
-    */
-    bool parse_bson_element_list(const bool is_array)
-    {
-        string_t key;
-        while (int element_type = get())
-        {
-            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list")))
-            {
-                return false;
-            }
-
-            const std::size_t element_type_parse_position = chars_read;
-            if (JSON_UNLIKELY(not get_bson_cstr(key)))
-            {
-                return false;
-            }
-
-            if (not is_array)
-            {
-                if (not sax->key(key))
-                {
-                    return false;
-                }
-            }
-
-            if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position)))
-            {
-                return false;
-            }
-
-            // get_bson_cstr only appends
-            key.clear();
-        }
-
-        return true;
-    }
-
-    /*!
-    @brief Reads an array from the BSON input and passes it to the SAX-parser.
-    @return whether a valid BSON-array was passed to the SAX parser
-    */
-    bool parse_bson_array()
-    {
-        std::int32_t document_size;
-        get_number<std::int32_t, true>(input_format_t::bson, document_size);
-
-        if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
-        {
-            return false;
-        }
-
-        if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true)))
-        {
-            return false;
-        }
-
-        return sax->end_array();
-    }
-
-    //////////
-    // CBOR //
-    //////////
-
-    /*!
-    @param[in] get_char  whether a new character should be retrieved from the
-                         input (true, default) or whether the last read
-                         character should be considered instead
-
-    @return whether a valid CBOR value was passed to the SAX parser
-    */
-    bool parse_cbor_internal(const bool get_char = true)
-    {
-        switch (get_char ? get() : current)
-        {
-            // EOF
-            case std::char_traits<char>::eof():
-                return unexpect_eof(input_format_t::cbor, "value");
-
-            // Integer 0x00..0x17 (0..23)
-            case 0x00:
-            case 0x01:
-            case 0x02:
-            case 0x03:
-            case 0x04:
-            case 0x05:
-            case 0x06:
-            case 0x07:
-            case 0x08:
-            case 0x09:
-            case 0x0A:
-            case 0x0B:
-            case 0x0C:
-            case 0x0D:
-            case 0x0E:
-            case 0x0F:
-            case 0x10:
-            case 0x11:
-            case 0x12:
-            case 0x13:
-            case 0x14:
-            case 0x15:
-            case 0x16:
-            case 0x17:
-                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
-
-            case 0x18: // Unsigned integer (one-byte uint8_t follows)
-            {
-                uint8_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
-            }
-
-            case 0x19: // Unsigned integer (two-byte uint16_t follows)
-            {
-                uint16_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
-            }
-
-            case 0x1A: // Unsigned integer (four-byte uint32_t follows)
-            {
-                uint32_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
-            }
-
-            case 0x1B: // Unsigned integer (eight-byte uint64_t follows)
-            {
-                uint64_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_unsigned(number);
-            }
-
-            // Negative integer -1-0x00..-1-0x17 (-1..-24)
-            case 0x20:
-            case 0x21:
-            case 0x22:
-            case 0x23:
-            case 0x24:
-            case 0x25:
-            case 0x26:
-            case 0x27:
-            case 0x28:
-            case 0x29:
-            case 0x2A:
-            case 0x2B:
-            case 0x2C:
-            case 0x2D:
-            case 0x2E:
-            case 0x2F:
-            case 0x30:
-            case 0x31:
-            case 0x32:
-            case 0x33:
-            case 0x34:
-            case 0x35:
-            case 0x36:
-            case 0x37:
-                return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current));
-
-            case 0x38: // Negative integer (one-byte uint8_t follows)
-            {
-                uint8_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
-            }
-
-            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
-            {
-                uint16_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
-            }
-
-            case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)
-            {
-                uint32_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number);
-            }
-
-            case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)
-            {
-                uint64_t number;
-                return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1)
-                        - static_cast<number_integer_t>(number));
-            }
-
-            // UTF-8 string (0x00..0x17 bytes follow)
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6A:
-            case 0x6B:
-            case 0x6C:
-            case 0x6D:
-            case 0x6E:
-            case 0x6F:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
-            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
-            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
-            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
-            case 0x7F: // UTF-8 string (indefinite length)
-            {
-                string_t s;
-                return get_cbor_string(s) and sax->string(s);
-            }
-
-            // array (0x00..0x17 data items follow)
-            case 0x80:
-            case 0x81:
-            case 0x82:
-            case 0x83:
-            case 0x84:
-            case 0x85:
-            case 0x86:
-            case 0x87:
-            case 0x88:
-            case 0x89:
-            case 0x8A:
-            case 0x8B:
-            case 0x8C:
-            case 0x8D:
-            case 0x8E:
-            case 0x8F:
-            case 0x90:
-            case 0x91:
-            case 0x92:
-            case 0x93:
-            case 0x94:
-            case 0x95:
-            case 0x96:
-            case 0x97:
-                return get_cbor_array(static_cast<std::size_t>(current & 0x1F));
-
-            case 0x98: // array (one-byte uint8_t for n follows)
-            {
-                uint8_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
-            }
-
-            case 0x99: // array (two-byte uint16_t for n follow)
-            {
-                uint16_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
-            }
-
-            case 0x9A: // array (four-byte uint32_t for n follow)
-            {
-                uint32_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
-            }
-
-            case 0x9B: // array (eight-byte uint64_t for n follow)
-            {
-                uint64_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len));
-            }
-
-            case 0x9F: // array (indefinite length)
-                return get_cbor_array(std::size_t(-1));
-
-            // map (0x00..0x17 pairs of data items follow)
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-                return get_cbor_object(static_cast<std::size_t>(current & 0x1F));
-
-            case 0xB8: // map (one-byte uint8_t for n follows)
-            {
-                uint8_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
-            }
-
-            case 0xB9: // map (two-byte uint16_t for n follow)
-            {
-                uint16_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
-            }
-
-            case 0xBA: // map (four-byte uint32_t for n follow)
-            {
-                uint32_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
-            }
-
-            case 0xBB: // map (eight-byte uint64_t for n follow)
-            {
-                uint64_t len;
-                return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len));
-            }
-
-            case 0xBF: // map (indefinite length)
-                return get_cbor_object(std::size_t(-1));
-
-            case 0xF4: // false
-                return sax->boolean(false);
-
-            case 0xF5: // true
-                return sax->boolean(true);
-
-            case 0xF6: // null
-                return sax->null();
-
-            case 0xF9: // Half-Precision Float (two-byte IEEE 754)
-            {
-                const int byte1_raw = get();
-                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
-                {
-                    return false;
-                }
-                const int byte2_raw = get();
-                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number")))
-                {
-                    return false;
-                }
-
-                const auto byte1 = static_cast<unsigned char>(byte1_raw);
-                const auto byte2 = static_cast<unsigned char>(byte2_raw);
-
-                // code from RFC 7049, Appendix D, Figure 3:
-                // As half-precision floating-point numbers were only added
-                // to IEEE 754 in 2008, today's programming platforms often
-                // still only have limited support for them. It is very
-                // easy to include at least decoding support for them even
-                // without such support. An example of a small decoder for
-                // half-precision floating-point numbers in the C language
-                // is shown in Fig. 3.
-                const int half = (byte1 << 8) + byte2;
-                const double val = [&half]
-                {
-                    const int exp = (half >> 10) & 0x1F;
-                    const int mant = half & 0x3FF;
-                    assert(0 <= exp and exp <= 32);
-                    assert(0 <= mant and mant <= 1024);
-                    switch (exp)
-                    {
-                        case 0:
-                            return std::ldexp(mant, -24);
-                        case 31:
-                            return (mant == 0)
-                            ? std::numeric_limits<double>::infinity()
-                            : std::numeric_limits<double>::quiet_NaN();
-                        default:
-                            return std::ldexp(mant + 1024, exp - 25);
-                    }
-                }();
-                return sax->number_float((half & 0x8000) != 0
-                                         ? static_cast<number_float_t>(-val)
-                                         : static_cast<number_float_t>(val), "");
-            }
-
-            case 0xFA: // Single-Precision Float (four-byte IEEE 754)
-            {
-                float number;
-                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
-            {
-                double number;
-                return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            default: // anything else (0xFF is handled inside the other types)
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value")));
-            }
-        }
-    }
-
-    /*!
-    @brief reads a CBOR string
-
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
-    Additionally, CBOR's strings with indefinite lengths are supported.
-
-    @param[out] result  created string
-
-    @return whether string creation completed
-    */
-    bool get_cbor_string(string_t& result)
-    {
-        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string")))
-        {
-            return false;
-        }
-
-        switch (current)
-        {
-            // UTF-8 string (0x00..0x17 bytes follow)
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6A:
-            case 0x6B:
-            case 0x6C:
-            case 0x6D:
-            case 0x6E:
-            case 0x6F:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
-            {
-                return get_string(input_format_t::cbor, current & 0x1F, result);
-            }
-
-            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
-            {
-                uint8_t len;
-                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
-            }
-
-            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-            {
-                uint16_t len;
-                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
-            }
-
-            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)
-            {
-                uint32_t len;
-                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
-            }
-
-            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)
-            {
-                uint64_t len;
-                return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result);
-            }
-
-            case 0x7F: // UTF-8 string (indefinite length)
-            {
-                while (get() != 0xFF)
-                {
-                    string_t chunk;
-                    if (not get_cbor_string(chunk))
-                    {
-                        return false;
-                    }
-                    result.append(chunk);
-                }
-                return true;
-            }
-
-            default:
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string")));
-            }
-        }
-    }
-
-    /*!
-    @param[in] len  the length of the array or std::size_t(-1) for an
-                    array of indefinite size
-    @return whether array creation completed
-    */
-    bool get_cbor_array(const std::size_t len)
-    {
-        if (JSON_UNLIKELY(not sax->start_array(len)))
-        {
-            return false;
-        }
-
-        if (len != std::size_t(-1))
-        {
-            for (std::size_t i = 0; i < len; ++i)
-            {
-                if (JSON_UNLIKELY(not parse_cbor_internal()))
-                {
-                    return false;
-                }
-            }
-        }
-        else
-        {
-            while (get() != 0xFF)
-            {
-                if (JSON_UNLIKELY(not parse_cbor_internal(false)))
-                {
-                    return false;
-                }
-            }
-        }
-
-        return sax->end_array();
-    }
-
-    /*!
-    @param[in] len  the length of the object or std::size_t(-1) for an
-                    object of indefinite size
-    @return whether object creation completed
-    */
-    bool get_cbor_object(const std::size_t len)
-    {
-        if (not JSON_UNLIKELY(sax->start_object(len)))
-        {
-            return false;
-        }
-
-        string_t key;
-        if (len != std::size_t(-1))
-        {
-            for (std::size_t i = 0; i < len; ++i)
-            {
-                get();
-                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
-                {
-                    return false;
-                }
-
-                if (JSON_UNLIKELY(not parse_cbor_internal()))
-                {
-                    return false;
-                }
-                key.clear();
-            }
-        }
-        else
-        {
-            while (get() != 0xFF)
-            {
-                if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key)))
-                {
-                    return false;
-                }
-
-                if (JSON_UNLIKELY(not parse_cbor_internal()))
-                {
-                    return false;
-                }
-                key.clear();
-            }
-        }
-
-        return sax->end_object();
-    }
-
-    /////////////
-    // MsgPack //
-    /////////////
-
-    /*!
-    @return whether a valid MessagePack value was passed to the SAX parser
-    */
-    bool parse_msgpack_internal()
-    {
-        switch (get())
-        {
-            // EOF
-            case std::char_traits<char>::eof():
-                return unexpect_eof(input_format_t::msgpack, "value");
-
-            // positive fixint
-            case 0x00:
-            case 0x01:
-            case 0x02:
-            case 0x03:
-            case 0x04:
-            case 0x05:
-            case 0x06:
-            case 0x07:
-            case 0x08:
-            case 0x09:
-            case 0x0A:
-            case 0x0B:
-            case 0x0C:
-            case 0x0D:
-            case 0x0E:
-            case 0x0F:
-            case 0x10:
-            case 0x11:
-            case 0x12:
-            case 0x13:
-            case 0x14:
-            case 0x15:
-            case 0x16:
-            case 0x17:
-            case 0x18:
-            case 0x19:
-            case 0x1A:
-            case 0x1B:
-            case 0x1C:
-            case 0x1D:
-            case 0x1E:
-            case 0x1F:
-            case 0x20:
-            case 0x21:
-            case 0x22:
-            case 0x23:
-            case 0x24:
-            case 0x25:
-            case 0x26:
-            case 0x27:
-            case 0x28:
-            case 0x29:
-            case 0x2A:
-            case 0x2B:
-            case 0x2C:
-            case 0x2D:
-            case 0x2E:
-            case 0x2F:
-            case 0x30:
-            case 0x31:
-            case 0x32:
-            case 0x33:
-            case 0x34:
-            case 0x35:
-            case 0x36:
-            case 0x37:
-            case 0x38:
-            case 0x39:
-            case 0x3A:
-            case 0x3B:
-            case 0x3C:
-            case 0x3D:
-            case 0x3E:
-            case 0x3F:
-            case 0x40:
-            case 0x41:
-            case 0x42:
-            case 0x43:
-            case 0x44:
-            case 0x45:
-            case 0x46:
-            case 0x47:
-            case 0x48:
-            case 0x49:
-            case 0x4A:
-            case 0x4B:
-            case 0x4C:
-            case 0x4D:
-            case 0x4E:
-            case 0x4F:
-            case 0x50:
-            case 0x51:
-            case 0x52:
-            case 0x53:
-            case 0x54:
-            case 0x55:
-            case 0x56:
-            case 0x57:
-            case 0x58:
-            case 0x59:
-            case 0x5A:
-            case 0x5B:
-            case 0x5C:
-            case 0x5D:
-            case 0x5E:
-            case 0x5F:
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6A:
-            case 0x6B:
-            case 0x6C:
-            case 0x6D:
-            case 0x6E:
-            case 0x6F:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
-            case 0x78:
-            case 0x79:
-            case 0x7A:
-            case 0x7B:
-            case 0x7C:
-            case 0x7D:
-            case 0x7E:
-            case 0x7F:
-                return sax->number_unsigned(static_cast<number_unsigned_t>(current));
-
-            // fixmap
-            case 0x80:
-            case 0x81:
-            case 0x82:
-            case 0x83:
-            case 0x84:
-            case 0x85:
-            case 0x86:
-            case 0x87:
-            case 0x88:
-            case 0x89:
-            case 0x8A:
-            case 0x8B:
-            case 0x8C:
-            case 0x8D:
-            case 0x8E:
-            case 0x8F:
-                return get_msgpack_object(static_cast<std::size_t>(current & 0x0F));
-
-            // fixarray
-            case 0x90:
-            case 0x91:
-            case 0x92:
-            case 0x93:
-            case 0x94:
-            case 0x95:
-            case 0x96:
-            case 0x97:
-            case 0x98:
-            case 0x99:
-            case 0x9A:
-            case 0x9B:
-            case 0x9C:
-            case 0x9D:
-            case 0x9E:
-            case 0x9F:
-                return get_msgpack_array(static_cast<std::size_t>(current & 0x0F));
-
-            // fixstr
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-            case 0xB8:
-            case 0xB9:
-            case 0xBA:
-            case 0xBB:
-            case 0xBC:
-            case 0xBD:
-            case 0xBE:
-            case 0xBF:
-            {
-                string_t s;
-                return get_msgpack_string(s) and sax->string(s);
-            }
-
-            case 0xC0: // nil
-                return sax->null();
-
-            case 0xC2: // false
-                return sax->boolean(false);
-
-            case 0xC3: // true
-                return sax->boolean(true);
-
-            case 0xCA: // float 32
-            {
-                float number;
-                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 0xCB: // float 64
-            {
-                double number;
-                return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 0xCC: // uint 8
-            {
-                uint8_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
-            }
-
-            case 0xCD: // uint 16
-            {
-                uint16_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
-            }
-
-            case 0xCE: // uint 32
-            {
-                uint32_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
-            }
-
-            case 0xCF: // uint 64
-            {
-                uint64_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number);
-            }
-
-            case 0xD0: // int 8
-            {
-                int8_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
-            }
-
-            case 0xD1: // int 16
-            {
-                int16_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
-            }
-
-            case 0xD2: // int 32
-            {
-                int32_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
-            }
-
-            case 0xD3: // int 64
-            {
-                int64_t number;
-                return get_number(input_format_t::msgpack, number) and sax->number_integer(number);
-            }
-
-            case 0xD9: // str 8
-            case 0xDA: // str 16
-            case 0xDB: // str 32
-            {
-                string_t s;
-                return get_msgpack_string(s) and sax->string(s);
-            }
-
-            case 0xDC: // array 16
-            {
-                uint16_t len;
-                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
-            }
-
-            case 0xDD: // array 32
-            {
-                uint32_t len;
-                return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len));
-            }
-
-            case 0xDE: // map 16
-            {
-                uint16_t len;
-                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
-            }
-
-            case 0xDF: // map 32
-            {
-                uint32_t len;
-                return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len));
-            }
-
-            // negative fixint
-            case 0xE0:
-            case 0xE1:
-            case 0xE2:
-            case 0xE3:
-            case 0xE4:
-            case 0xE5:
-            case 0xE6:
-            case 0xE7:
-            case 0xE8:
-            case 0xE9:
-            case 0xEA:
-            case 0xEB:
-            case 0xEC:
-            case 0xED:
-            case 0xEE:
-            case 0xEF:
-            case 0xF0:
-            case 0xF1:
-            case 0xF2:
-            case 0xF3:
-            case 0xF4:
-            case 0xF5:
-            case 0xF6:
-            case 0xF7:
-            case 0xF8:
-            case 0xF9:
-            case 0xFA:
-            case 0xFB:
-            case 0xFC:
-            case 0xFD:
-            case 0xFE:
-            case 0xFF:
-                return sax->number_integer(static_cast<int8_t>(current));
-
-            default: // anything else
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value")));
-            }
-        }
-    }
-
-    /*!
-    @brief reads a MessagePack string
-
-    This function first reads starting bytes to determine the expected
-    string length and then copies this number of bytes into a string.
-
-    @param[out] result  created string
-
-    @return whether string creation completed
-    */
-    bool get_msgpack_string(string_t& result)
-    {
-        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string")))
-        {
-            return false;
-        }
-
-        switch (current)
-        {
-            // fixstr
-            case 0xA0:
-            case 0xA1:
-            case 0xA2:
-            case 0xA3:
-            case 0xA4:
-            case 0xA5:
-            case 0xA6:
-            case 0xA7:
-            case 0xA8:
-            case 0xA9:
-            case 0xAA:
-            case 0xAB:
-            case 0xAC:
-            case 0xAD:
-            case 0xAE:
-            case 0xAF:
-            case 0xB0:
-            case 0xB1:
-            case 0xB2:
-            case 0xB3:
-            case 0xB4:
-            case 0xB5:
-            case 0xB6:
-            case 0xB7:
-            case 0xB8:
-            case 0xB9:
-            case 0xBA:
-            case 0xBB:
-            case 0xBC:
-            case 0xBD:
-            case 0xBE:
-            case 0xBF:
-            {
-                return get_string(input_format_t::msgpack, current & 0x1F, result);
-            }
-
-            case 0xD9: // str 8
-            {
-                uint8_t len;
-                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
-            }
-
-            case 0xDA: // str 16
-            {
-                uint16_t len;
-                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
-            }
-
-            case 0xDB: // str 32
-            {
-                uint32_t len;
-                return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result);
-            }
-
-            default:
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string")));
-            }
-        }
-    }
-
-    /*!
-    @param[in] len  the length of the array
-    @return whether array creation completed
-    */
-    bool get_msgpack_array(const std::size_t len)
-    {
-        if (JSON_UNLIKELY(not sax->start_array(len)))
-        {
-            return false;
-        }
-
-        for (std::size_t i = 0; i < len; ++i)
-        {
-            if (JSON_UNLIKELY(not parse_msgpack_internal()))
-            {
-                return false;
-            }
-        }
-
-        return sax->end_array();
-    }
-
-    /*!
-    @param[in] len  the length of the object
-    @return whether object creation completed
-    */
-    bool get_msgpack_object(const std::size_t len)
-    {
-        if (JSON_UNLIKELY(not sax->start_object(len)))
-        {
-            return false;
-        }
-
-        string_t key;
-        for (std::size_t i = 0; i < len; ++i)
-        {
-            get();
-            if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key)))
-            {
-                return false;
-            }
-
-            if (JSON_UNLIKELY(not parse_msgpack_internal()))
-            {
-                return false;
-            }
-            key.clear();
-        }
-
-        return sax->end_object();
-    }
-
-    ////////////
-    // UBJSON //
-    ////////////
-
-    /*!
-    @param[in] get_char  whether a new character should be retrieved from the
-                         input (true, default) or whether the last read
-                         character should be considered instead
-
-    @return whether a valid UBJSON value was passed to the SAX parser
-    */
-    bool parse_ubjson_internal(const bool get_char = true)
-    {
-        return get_ubjson_value(get_char ? get_ignore_noop() : current);
-    }
-
-    /*!
-    @brief reads a UBJSON string
-
-    This function is either called after reading the 'S' byte explicitly
-    indicating a string, or in case of an object key where the 'S' byte can be
-    left out.
-
-    @param[out] result   created string
-    @param[in] get_char  whether a new character should be retrieved from the
-                         input (true, default) or whether the last read
-                         character should be considered instead
-
-    @return whether string creation completed
-    */
-    bool get_ubjson_string(string_t& result, const bool get_char = true)
-    {
-        if (get_char)
-        {
-            get();  // TODO: may we ignore N here?
-        }
-
-        if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
-        {
-            return false;
-        }
-
-        switch (current)
-        {
-            case 'U':
-            {
-                uint8_t len;
-                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
-            }
-
-            case 'i':
-            {
-                int8_t len;
-                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
-            }
-
-            case 'I':
-            {
-                int16_t len;
-                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
-            }
-
-            case 'l':
-            {
-                int32_t len;
-                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
-            }
-
-            case 'L':
-            {
-                int64_t len;
-                return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result);
-            }
-
-            default:
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string")));
-        }
-    }
-
-    /*!
-    @param[out] result  determined size
-    @return whether size determination completed
-    */
-    bool get_ubjson_size_value(std::size_t& result)
-    {
-        switch (get_ignore_noop())
-        {
-            case 'U':
-            {
-                uint8_t number;
-                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
-                {
-                    return false;
-                }
-                result = static_cast<std::size_t>(number);
-                return true;
-            }
-
-            case 'i':
-            {
-                int8_t number;
-                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
-                {
-                    return false;
-                }
-                result = static_cast<std::size_t>(number);
-                return true;
-            }
-
-            case 'I':
-            {
-                int16_t number;
-                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
-                {
-                    return false;
-                }
-                result = static_cast<std::size_t>(number);
-                return true;
-            }
-
-            case 'l':
-            {
-                int32_t number;
-                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
-                {
-                    return false;
-                }
-                result = static_cast<std::size_t>(number);
-                return true;
-            }
-
-            case 'L':
-            {
-                int64_t number;
-                if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number)))
-                {
-                    return false;
-                }
-                result = static_cast<std::size_t>(number);
-                return true;
-            }
-
-            default:
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size")));
-            }
-        }
-    }
-
-    /*!
-    @brief determine the type and size for a container
-
-    In the optimized UBJSON format, a type and a size can be provided to allow
-    for a more compact representation.
-
-    @param[out] result  pair of the size and the type
-
-    @return whether pair creation completed
-    */
-    bool get_ubjson_size_type(std::pair<std::size_t, int>& result)
-    {
-        result.first = string_t::npos; // size
-        result.second = 0; // type
-
-        get_ignore_noop();
-
-        if (current == '$')
-        {
-            result.second = get();  // must not ignore 'N', because 'N' maybe the type
-            if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type")))
-            {
-                return false;
-            }
-
-            get_ignore_noop();
-            if (JSON_UNLIKELY(current != '#'))
-            {
-                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value")))
-                {
-                    return false;
-                }
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size")));
-            }
-
-            return get_ubjson_size_value(result.first);
-        }
-        else if (current == '#')
-        {
-            return get_ubjson_size_value(result.first);
-        }
-        return true;
-    }
-
-    /*!
-    @param prefix  the previously read or set type prefix
-    @return whether value creation completed
-    */
-    bool get_ubjson_value(const int prefix)
-    {
-        switch (prefix)
-        {
-            case std::char_traits<char>::eof():  // EOF
-                return unexpect_eof(input_format_t::ubjson, "value");
-
-            case 'T':  // true
-                return sax->boolean(true);
-            case 'F':  // false
-                return sax->boolean(false);
-
-            case 'Z':  // null
-                return sax->null();
-
-            case 'U':
-            {
-                uint8_t number;
-                return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number);
-            }
-
-            case 'i':
-            {
-                int8_t number;
-                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
-            }
-
-            case 'I':
-            {
-                int16_t number;
-                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
-            }
-
-            case 'l':
-            {
-                int32_t number;
-                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
-            }
-
-            case 'L':
-            {
-                int64_t number;
-                return get_number(input_format_t::ubjson, number) and sax->number_integer(number);
-            }
-
-            case 'd':
-            {
-                float number;
-                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 'D':
-            {
-                double number;
-                return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), "");
-            }
-
-            case 'C':  // char
-            {
-                get();
-                if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char")))
-                {
-                    return false;
-                }
-                if (JSON_UNLIKELY(current > 127))
-                {
-                    auto last_token = get_token_string();
-                    return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char")));
-                }
-                string_t s(1, static_cast<char>(current));
-                return sax->string(s);
-            }
-
-            case 'S':  // string
-            {
-                string_t s;
-                return get_ubjson_string(s) and sax->string(s);
-            }
-
-            case '[':  // array
-                return get_ubjson_array();
-
-            case '{':  // object
-                return get_ubjson_object();
-
-            default: // anything else
-            {
-                auto last_token = get_token_string();
-                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value")));
-            }
-        }
-    }
-
-    /*!
-    @return whether array creation completed
-    */
-    bool get_ubjson_array()
-    {
-        std::pair<std::size_t, int> size_and_type;
-        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
-        {
-            return false;
-        }
-
-        if (size_and_type.first != string_t::npos)
-        {
-            if (JSON_UNLIKELY(not sax->start_array(size_and_type.first)))
-            {
-                return false;
-            }
-
-            if (size_and_type.second != 0)
-            {
-                if (size_and_type.second != 'N')
-                {
-                    for (std::size_t i = 0; i < size_and_type.first; ++i)
-                    {
-                        if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
-                        {
-                            return false;
-                        }
-                    }
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                }
-            }
-        }
-        else
-        {
-            if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1))))
-            {
-                return false;
-            }
-
-            while (current != ']')
-            {
-                if (JSON_UNLIKELY(not parse_ubjson_internal(false)))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-            }
-        }
-
-        return sax->end_array();
-    }
-
-    /*!
-    @return whether object creation completed
-    */
-    bool get_ubjson_object()
-    {
-        std::pair<std::size_t, int> size_and_type;
-        if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type)))
-        {
-            return false;
-        }
-
-        string_t key;
-        if (size_and_type.first != string_t::npos)
-        {
-            if (JSON_UNLIKELY(not sax->start_object(size_and_type.first)))
-            {
-                return false;
-            }
-
-            if (size_and_type.second != 0)
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second)))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_UNLIKELY(not parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-        }
-        else
-        {
-            if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1))))
-            {
-                return false;
-            }
-
-            while (current != '}')
-            {
-                if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key)))
-                {
-                    return false;
-                }
-                if (JSON_UNLIKELY(not parse_ubjson_internal()))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-                key.clear();
-            }
-        }
-
-        return sax->end_object();
-    }
-
-    ///////////////////////
-    // Utility functions //
-    ///////////////////////
-
-    /*!
-    @brief get next character from the input
-
-    This function provides the interface to the used input adapter. It does
-    not throw in case the input reached EOF, but returns a -'ve valued
-    `std::char_traits<char>::eof()` in that case.
-
-    @return character read from the input
-    */
-    int get()
-    {
-        ++chars_read;
-        return (current = ia->get_character());
-    }
-
-    /*!
-    @return character read from the input after ignoring all 'N' entries
-    */
-    int get_ignore_noop()
-    {
-        do
-        {
-            get();
-        }
-        while (current == 'N');
-
-        return current;
-    }
-
-    /*
-    @brief read a number from the input
-
-    @tparam NumberType the type of the number
-    @param[in] format   the current format (for diagnostics)
-    @param[out] result  number of type @a NumberType
-
-    @return whether conversion completed
-
-    @note This function needs to respect the system's endianess, because
-          bytes in CBOR, MessagePack, and UBJSON are stored in network order
-          (big endian) and therefore need reordering on little endian systems.
-    */
-    template<typename NumberType, bool InputIsLittleEndian = false>
-    bool get_number(const input_format_t format, NumberType& result)
-    {
-        // step 1: read input into array with system's byte order
-        std::array<uint8_t, sizeof(NumberType)> vec;
-        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
-        {
-            get();
-            if (JSON_UNLIKELY(not unexpect_eof(format, "number")))
-            {
-                return false;
-            }
-
-            // reverse byte order prior to conversion if necessary
-            if (is_little_endian && !InputIsLittleEndian)
-            {
-                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
-            }
-            else
-            {
-                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
-            }
-        }
-
-        // step 2: convert array into number of type T and return
-        std::memcpy(&result, vec.data(), sizeof(NumberType));
-        return true;
-    }
-
-    /*!
-    @brief create a string by reading characters from the input
-
-    @tparam NumberType the type of the number
-    @param[in] format the current format (for diagnostics)
-    @param[in] len number of characters to read
-    @param[out] result string created by reading @a len bytes
-
-    @return whether string creation completed
-
-    @note We can not reserve @a len bytes for the result, because @a len
-          may be too large. Usually, @ref unexpect_eof() detects the end of
-          the input before we run out of string memory.
-    */
-    template<typename NumberType>
-    bool get_string(const input_format_t format,
-                    const NumberType len,
-                    string_t& result)
-    {
-        bool success = true;
-        std::generate_n(std::back_inserter(result), len, [this, &success, &format]()
-        {
-            get();
-            if (JSON_UNLIKELY(not unexpect_eof(format, "string")))
-            {
-                success = false;
-            }
-            return static_cast<char>(current);
-        });
-        return success;
-    }
-
-    /*!
-    @param[in] format   the current format (for diagnostics)
-    @param[in] context  further context information (for diagnostics)
-    @return whether the last read character is not EOF
-    */
-    bool unexpect_eof(const input_format_t format, const char* context) const
-    {
-        if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
-        {
-            return sax->parse_error(chars_read, "<end of file>",
-                                    parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context)));
-        }
-        return true;
-    }
-
-    /*!
-    @return a string representation of the last read byte
-    */
-    std::string get_token_string() const
-    {
-        char cr[3];
-        (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current));
-        return std::string{cr};
-    }
-
-    /*!
-    @param[in] format   the current format
-    @param[in] detail   a detailed error message
-    @param[in] context  further contect information
-    @return a message string to use in the parse_error exceptions
-    */
-    std::string exception_message(const input_format_t format,
-                                  const std::string& detail,
-                                  const std::string& context) const
-    {
-        std::string error_msg = "syntax error while parsing ";
-
-        switch (format)
-        {
-            case input_format_t::cbor:
-                error_msg += "CBOR";
-                break;
-
-            case input_format_t::msgpack:
-                error_msg += "MessagePack";
-                break;
-
-            case input_format_t::ubjson:
-                error_msg += "UBJSON";
-                break;
-
-            case input_format_t::bson:
-                error_msg += "BSON";
-                break;
-
-            // LCOV_EXCL_START
-            default:
-                assert(false);
-                // LCOV_EXCL_STOP
-        }
-
-        return error_msg + " " + context + ": " + detail;
-    }
-
-  private:
-    /// input adapter
-    input_adapter_t ia = nullptr;
-
-    /// the current character
-    int current = std::char_traits<char>::eof();
-
-    /// the number of characters read
-    std::size_t chars_read = 0;
-
-    /// whether we can assume little endianess
-    const bool is_little_endian = little_endianess();
-
-    /// the SAX parser
-    json_sax_t* sax = nullptr;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/output/binary_writer.hpp>
-
-
-#include <algorithm> // reverse
-#include <array> // array
-#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
-#include <cstring> // memcpy
-#include <limits> // numeric_limits
-
-// #include <nlohmann/detail/input/binary_reader.hpp>
-
-// #include <nlohmann/detail/output/output_adapters.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-///////////////////
-// binary writer //
-///////////////////
-
-/*!
-@brief serialization to CBOR and MessagePack values
-*/
-template<typename BasicJsonType, typename CharType>
-class binary_writer
-{
-    using string_t = typename BasicJsonType::string_t;
-
-  public:
-    /*!
-    @brief create a binary writer
-
-    @param[in] adapter  output adapter to write to
-    */
-    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
-    {
-        assert(oa);
-    }
-
-    /*!
-    @param[in] j  JSON value to serialize
-    @pre       j.type() == value_t::object
-    */
-    void write_bson(const BasicJsonType& j)
-    {
-        switch (j.type())
-        {
-            case value_t::object:
-            {
-                write_bson_object(*j.m_value.object);
-                break;
-            }
-
-            default:
-            {
-                JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
-            }
-        }
-    }
-
-    /*!
-    @param[in] j  JSON value to serialize
-    */
-    void write_cbor(const BasicJsonType& j)
-    {
-        switch (j.type())
-        {
-            case value_t::null:
-            {
-                oa->write_character(to_char_type(0xF6));
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                oa->write_character(j.m_value.boolean
-                                    ? to_char_type(0xF5)
-                                    : to_char_type(0xF4));
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // CBOR does not differentiate between positive signed
-                    // integers and unsigned integers. Therefore, we used the
-                    // code from the value_t::number_unsigned case here.
-                    if (j.m_value.number_integer <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x18));
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x19));
-                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x1A));
-                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                    }
-                    else
-                    {
-                        oa->write_character(to_char_type(0x1B));
-                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                    }
-                }
-                else
-                {
-                    // The conversions below encode the sign in the first
-                    // byte, and the value is converted to a positive number.
-                    const auto positive_number = -1 - j.m_value.number_integer;
-                    if (j.m_value.number_integer >= -24)
-                    {
-                        write_number(static_cast<uint8_t>(0x20 + positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x38));
-                        write_number(static_cast<uint8_t>(positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x39));
-                        write_number(static_cast<uint16_t>(positive_number));
-                    }
-                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        oa->write_character(to_char_type(0x3A));
-                        write_number(static_cast<uint32_t>(positive_number));
-                    }
-                    else
-                    {
-                        oa->write_character(to_char_type(0x3B));
-                        write_number(static_cast<uint64_t>(positive_number));
-                    }
-                }
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                if (j.m_value.number_unsigned <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x18));
-                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x19));
-                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x1A));
-                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
-                }
-                else
-                {
-                    oa->write_character(to_char_type(0x1B));
-                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
-                }
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
-                write_number(j.m_value.number_float);
-                break;
-            }
-
-            case value_t::string:
-            {
-                // step 1: write control byte and the string length
-                const auto N = j.m_value.string->size();
-                if (N <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(0x60 + N));
-                }
-                else if (N <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x78));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x79));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x7A));
-                    write_number(static_cast<uint32_t>(N));
-                }
-                // LCOV_EXCL_START
-                else if (N <= (std::numeric_limits<uint64_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x7B));
-                    write_number(static_cast<uint64_t>(N));
-                }
-                // LCOV_EXCL_STOP
-
-                // step 2: write the string
-                oa->write_characters(
-                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
-                    j.m_value.string->size());
-                break;
-            }
-
-            case value_t::array:
-            {
-                // step 1: write control byte and the array size
-                const auto N = j.m_value.array->size();
-                if (N <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(0x80 + N));
-                }
-                else if (N <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x98));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x99));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x9A));
-                    write_number(static_cast<uint32_t>(N));
-                }
-                // LCOV_EXCL_START
-                else if (N <= (std::numeric_limits<uint64_t>::max)())
-                {
-                    oa->write_character(to_char_type(0x9B));
-                    write_number(static_cast<uint64_t>(N));
-                }
-                // LCOV_EXCL_STOP
-
-                // step 2: write each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    write_cbor(el);
-                }
-                break;
-            }
-
-            case value_t::object:
-            {
-                // step 1: write control byte and the object size
-                const auto N = j.m_value.object->size();
-                if (N <= 0x17)
-                {
-                    write_number(static_cast<uint8_t>(0xA0 + N));
-                }
-                else if (N <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    oa->write_character(to_char_type(0xB8));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    oa->write_character(to_char_type(0xB9));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    oa->write_character(to_char_type(0xBA));
-                    write_number(static_cast<uint32_t>(N));
-                }
-                // LCOV_EXCL_START
-                else if (N <= (std::numeric_limits<uint64_t>::max)())
-                {
-                    oa->write_character(to_char_type(0xBB));
-                    write_number(static_cast<uint64_t>(N));
-                }
-                // LCOV_EXCL_STOP
-
-                // step 2: write each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    write_cbor(el.first);
-                    write_cbor(el.second);
-                }
-                break;
-            }
-
-            default:
-                break;
-        }
-    }
-
-    /*!
-    @param[in] j  JSON value to serialize
-    */
-    void write_msgpack(const BasicJsonType& j)
-    {
-        switch (j.type())
-        {
-            case value_t::null: // nil
-            {
-                oa->write_character(to_char_type(0xC0));
-                break;
-            }
-
-            case value_t::boolean: // true and false
-            {
-                oa->write_character(j.m_value.boolean
-                                    ? to_char_type(0xC3)
-                                    : to_char_type(0xC2));
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // MessagePack does not differentiate between positive
-                    // signed integers and unsigned integers. Therefore, we used
-                    // the code from the value_t::number_unsigned case here.
-                    if (j.m_value.number_unsigned < 128)
-                    {
-                        // positive fixnum
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        // uint 8
-                        oa->write_character(to_char_type(0xCC));
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        // uint 16
-                        oa->write_character(to_char_type(0xCD));
-                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        // uint 32
-                        oa->write_character(to_char_type(0xCE));
-                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
-                    {
-                        // uint 64
-                        oa->write_character(to_char_type(0xCF));
-                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                    }
-                }
-                else
-                {
-                    if (j.m_value.number_integer >= -32)
-                    {
-                        // negative fixnum
-                        write_number(static_cast<int8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
-                    {
-                        // int 8
-                        oa->write_character(to_char_type(0xD0));
-                        write_number(static_cast<int8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
-                    {
-                        // int 16
-                        oa->write_character(to_char_type(0xD1));
-                        write_number(static_cast<int16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
-                    {
-                        // int 32
-                        oa->write_character(to_char_type(0xD2));
-                        write_number(static_cast<int32_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
-                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
-                    {
-                        // int 64
-                        oa->write_character(to_char_type(0xD3));
-                        write_number(static_cast<int64_t>(j.m_value.number_integer));
-                    }
-                }
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                if (j.m_value.number_unsigned < 128)
-                {
-                    // positive fixnum
-                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    // uint 8
-                    oa->write_character(to_char_type(0xCC));
-                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    // uint 16
-                    oa->write_character(to_char_type(0xCD));
-                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    // uint 32
-                    oa->write_character(to_char_type(0xCE));
-                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                }
-                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
-                {
-                    // uint 64
-                    oa->write_character(to_char_type(0xCF));
-                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                }
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
-                write_number(j.m_value.number_float);
-                break;
-            }
-
-            case value_t::string:
-            {
-                // step 1: write control byte and the string length
-                const auto N = j.m_value.string->size();
-                if (N <= 31)
-                {
-                    // fixstr
-                    write_number(static_cast<uint8_t>(0xA0 | N));
-                }
-                else if (N <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    // str 8
-                    oa->write_character(to_char_type(0xD9));
-                    write_number(static_cast<uint8_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    // str 16
-                    oa->write_character(to_char_type(0xDA));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    // str 32
-                    oa->write_character(to_char_type(0xDB));
-                    write_number(static_cast<uint32_t>(N));
-                }
-
-                // step 2: write the string
-                oa->write_characters(
-                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
-                    j.m_value.string->size());
-                break;
-            }
-
-            case value_t::array:
-            {
-                // step 1: write control byte and the array size
-                const auto N = j.m_value.array->size();
-                if (N <= 15)
-                {
-                    // fixarray
-                    write_number(static_cast<uint8_t>(0x90 | N));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    // array 16
-                    oa->write_character(to_char_type(0xDC));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    // array 32
-                    oa->write_character(to_char_type(0xDD));
-                    write_number(static_cast<uint32_t>(N));
-                }
-
-                // step 2: write each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    write_msgpack(el);
-                }
-                break;
-            }
-
-            case value_t::object:
-            {
-                // step 1: write control byte and the object size
-                const auto N = j.m_value.object->size();
-                if (N <= 15)
-                {
-                    // fixmap
-                    write_number(static_cast<uint8_t>(0x80 | (N & 0xF)));
-                }
-                else if (N <= (std::numeric_limits<uint16_t>::max)())
-                {
-                    // map 16
-                    oa->write_character(to_char_type(0xDE));
-                    write_number(static_cast<uint16_t>(N));
-                }
-                else if (N <= (std::numeric_limits<uint32_t>::max)())
-                {
-                    // map 32
-                    oa->write_character(to_char_type(0xDF));
-                    write_number(static_cast<uint32_t>(N));
-                }
-
-                // step 2: write each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    write_msgpack(el.first);
-                    write_msgpack(el.second);
-                }
-                break;
-            }
-
-            default:
-                break;
-        }
-    }
-
-    /*!
-    @param[in] j  JSON value to serialize
-    @param[in] use_count   whether to use '#' prefixes (optimized format)
-    @param[in] use_type    whether to use '$' prefixes (optimized format)
-    @param[in] add_prefix  whether prefixes need to be used for this value
-    */
-    void write_ubjson(const BasicJsonType& j, const bool use_count,
-                      const bool use_type, const bool add_prefix = true)
-    {
-        switch (j.type())
-        {
-            case value_t::null:
-            {
-                if (add_prefix)
-                {
-                    oa->write_character(to_char_type('Z'));
-                }
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                if (add_prefix)
-                {
-                    oa->write_character(j.m_value.boolean
-                                        ? to_char_type('T')
-                                        : to_char_type('F'));
-                }
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix);
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix);
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix);
-                break;
-            }
-
-            case value_t::string:
-            {
-                if (add_prefix)
-                {
-                    oa->write_character(to_char_type('S'));
-                }
-                write_number_with_ubjson_prefix(j.m_value.string->size(), true);
-                oa->write_characters(
-                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
-                    j.m_value.string->size());
-                break;
-            }
-
-            case value_t::array:
-            {
-                if (add_prefix)
-                {
-                    oa->write_character(to_char_type('['));
-                }
-
-                bool prefix_required = true;
-                if (use_type and not j.m_value.array->empty())
-                {
-                    assert(use_count);
-                    const CharType first_prefix = ubjson_prefix(j.front());
-                    const bool same_prefix = std::all_of(j.begin() + 1, j.end(),
-                                                         [this, first_prefix](const BasicJsonType & v)
-                    {
-                        return ubjson_prefix(v) == first_prefix;
-                    });
-
-                    if (same_prefix)
-                    {
-                        prefix_required = false;
-                        oa->write_character(to_char_type('$'));
-                        oa->write_character(first_prefix);
-                    }
-                }
-
-                if (use_count)
-                {
-                    oa->write_character(to_char_type('#'));
-                    write_number_with_ubjson_prefix(j.m_value.array->size(), true);
-                }
-
-                for (const auto& el : *j.m_value.array)
-                {
-                    write_ubjson(el, use_count, use_type, prefix_required);
-                }
-
-                if (not use_count)
-                {
-                    oa->write_character(to_char_type(']'));
-                }
-
-                break;
-            }
-
-            case value_t::object:
-            {
-                if (add_prefix)
-                {
-                    oa->write_character(to_char_type('{'));
-                }
-
-                bool prefix_required = true;
-                if (use_type and not j.m_value.object->empty())
-                {
-                    assert(use_count);
-                    const CharType first_prefix = ubjson_prefix(j.front());
-                    const bool same_prefix = std::all_of(j.begin(), j.end(),
-                                                         [this, first_prefix](const BasicJsonType & v)
-                    {
-                        return ubjson_prefix(v) == first_prefix;
-                    });
-
-                    if (same_prefix)
-                    {
-                        prefix_required = false;
-                        oa->write_character(to_char_type('$'));
-                        oa->write_character(first_prefix);
-                    }
-                }
-
-                if (use_count)
-                {
-                    oa->write_character(to_char_type('#'));
-                    write_number_with_ubjson_prefix(j.m_value.object->size(), true);
-                }
-
-                for (const auto& el : *j.m_value.object)
-                {
-                    write_number_with_ubjson_prefix(el.first.size(), true);
-                    oa->write_characters(
-                        reinterpret_cast<const CharType*>(el.first.c_str()),
-                        el.first.size());
-                    write_ubjson(el.second, use_count, use_type, prefix_required);
-                }
-
-                if (not use_count)
-                {
-                    oa->write_character(to_char_type('}'));
-                }
-
-                break;
-            }
-
-            default:
-                break;
-        }
-    }
-
-  private:
-    //////////
-    // BSON //
-    //////////
-
-    /*!
-    @return The size of a BSON document entry header, including the id marker
-            and the entry name size (and its null-terminator).
-    */
-    static std::size_t calc_bson_entry_header_size(const string_t& name)
-    {
-        const auto it = name.find(static_cast<typename string_t::value_type>(0));
-        if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos))
-        {
-            JSON_THROW(out_of_range::create(409,
-                                            "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")"));
-        }
-
-        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
-    }
-
-    /*!
-    @brief Writes the given @a element_type and @a name to the output adapter
-    */
-    void write_bson_entry_header(const string_t& name,
-                                 const std::uint8_t element_type)
-    {
-        oa->write_character(to_char_type(element_type)); // boolean
-        oa->write_characters(
-            reinterpret_cast<const CharType*>(name.c_str()),
-            name.size() + 1u);
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and boolean value @a value
-    */
-    void write_bson_boolean(const string_t& name,
-                            const bool value)
-    {
-        write_bson_entry_header(name, 0x08);
-        oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and double value @a value
-    */
-    void write_bson_double(const string_t& name,
-                           const double value)
-    {
-        write_bson_entry_header(name, 0x01);
-        write_number<double, true>(value);
-    }
-
-    /*!
-    @return The size of the BSON-encoded string in @a value
-    */
-    static std::size_t calc_bson_string_size(const string_t& value)
-    {
-        return sizeof(std::int32_t) + value.size() + 1ul;
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and string value @a value
-    */
-    void write_bson_string(const string_t& name,
-                           const string_t& value)
-    {
-        write_bson_entry_header(name, 0x02);
-
-        write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul));
-        oa->write_characters(
-            reinterpret_cast<const CharType*>(value.c_str()),
-            value.size() + 1);
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and null value
-    */
-    void write_bson_null(const string_t& name)
-    {
-        write_bson_entry_header(name, 0x0A);
-    }
-
-    /*!
-    @return The size of the BSON-encoded integer @a value
-    */
-    static std::size_t calc_bson_integer_size(const std::int64_t value)
-    {
-        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
-        {
-            return sizeof(std::int32_t);
-        }
-        else
-        {
-            return sizeof(std::int64_t);
-        }
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and integer @a value
-    */
-    void write_bson_integer(const string_t& name,
-                            const std::int64_t value)
-    {
-        if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)())
-        {
-            write_bson_entry_header(name, 0x10); // int32
-            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
-        }
-        else
-        {
-            write_bson_entry_header(name, 0x12); // int64
-            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
-        }
-    }
-
-    /*!
-    @return The size of the BSON-encoded unsigned integer in @a j
-    */
-    static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept
-    {
-        return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
-               ? sizeof(std::int32_t)
-               : sizeof(std::int64_t);
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and unsigned @a value
-    */
-    void write_bson_unsigned(const string_t& name,
-                             const std::uint64_t value)
-    {
-        if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))
-        {
-            write_bson_entry_header(name, 0x10 /* int32 */);
-            write_number<std::int32_t, true>(static_cast<std::int32_t>(value));
-        }
-        else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
-        {
-            write_bson_entry_header(name, 0x12 /* int64 */);
-            write_number<std::int64_t, true>(static_cast<std::int64_t>(value));
-        }
-        else
-        {
-            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64"));
-        }
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and object @a value
-    */
-    void write_bson_object_entry(const string_t& name,
-                                 const typename BasicJsonType::object_t& value)
-    {
-        write_bson_entry_header(name, 0x03); // object
-        write_bson_object(value);
-    }
-
-    /*!
-    @return The size of the BSON-encoded array @a value
-    */
-    static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)
-    {
-        std::size_t embedded_document_size = 0ul;
-        std::size_t array_index = 0ul;
-
-        for (const auto& el : value)
-        {
-            embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el);
-        }
-
-        return sizeof(std::int32_t) + embedded_document_size + 1ul;
-    }
-
-    /*!
-    @brief Writes a BSON element with key @a name and array @a value
-    */
-    void write_bson_array(const string_t& name,
-                          const typename BasicJsonType::array_t& value)
-    {
-        write_bson_entry_header(name, 0x04); // array
-        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value)));
-
-        std::size_t array_index = 0ul;
-
-        for (const auto& el : value)
-        {
-            write_bson_element(std::to_string(array_index++), el);
-        }
-
-        oa->write_character(to_char_type(0x00));
-    }
-
-    /*!
-    @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
-    @return The calculated size for the BSON document entry for @a j with the given @a name.
-    */
-    static std::size_t calc_bson_element_size(const string_t& name,
-            const BasicJsonType& j)
-    {
-        const auto header_size = calc_bson_entry_header_size(name);
-        switch (j.type())
-        {
-            case value_t::object:
-                return header_size + calc_bson_object_size(*j.m_value.object);
-
-            case value_t::array:
-                return header_size + calc_bson_array_size(*j.m_value.array);
-
-            case value_t::boolean:
-                return header_size + 1ul;
-
-            case value_t::number_float:
-                return header_size + 8ul;
-
-            case value_t::number_integer:
-                return header_size + calc_bson_integer_size(j.m_value.number_integer);
-
-            case value_t::number_unsigned:
-                return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);
-
-            case value_t::string:
-                return header_size + calc_bson_string_size(*j.m_value.string);
-
-            case value_t::null:
-                return header_size + 0ul;
-
-            // LCOV_EXCL_START
-            default:
-                assert(false);
-                return 0ul;
-                // LCOV_EXCL_STOP
-        };
-    }
-
-    /*!
-    @brief Serializes the JSON value @a j to BSON and associates it with the
-           key @a name.
-    @param name The name to associate with the JSON entity @a j within the
-                current BSON document
-    @return The size of the BSON entry
-    */
-    void write_bson_element(const string_t& name,
-                            const BasicJsonType& j)
-    {
-        switch (j.type())
-        {
-            case value_t::object:
-                return write_bson_object_entry(name, *j.m_value.object);
-
-            case value_t::array:
-                return write_bson_array(name, *j.m_value.array);
-
-            case value_t::boolean:
-                return write_bson_boolean(name, j.m_value.boolean);
-
-            case value_t::number_float:
-                return write_bson_double(name, j.m_value.number_float);
-
-            case value_t::number_integer:
-                return write_bson_integer(name, j.m_value.number_integer);
-
-            case value_t::number_unsigned:
-                return write_bson_unsigned(name, j.m_value.number_unsigned);
-
-            case value_t::string:
-                return write_bson_string(name, *j.m_value.string);
-
-            case value_t::null:
-                return write_bson_null(name);
-
-            // LCOV_EXCL_START
-            default:
-                assert(false);
-                return;
-                // LCOV_EXCL_STOP
-        };
-    }
-
-    /*!
-    @brief Calculates the size of the BSON serialization of the given
-           JSON-object @a j.
-    @param[in] j  JSON value to serialize
-    @pre       j.type() == value_t::object
-    */
-    static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)
-    {
-        std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul,
-                                    [](size_t result, const typename BasicJsonType::object_t::value_type & el)
-        {
-            return result += calc_bson_element_size(el.first, el.second);
-        });
-
-        return sizeof(std::int32_t) + document_size + 1ul;
-    }
-
-    /*!
-    @param[in] j  JSON value to serialize
-    @pre       j.type() == value_t::object
-    */
-    void write_bson_object(const typename BasicJsonType::object_t& value)
-    {
-        write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value)));
-
-        for (const auto& el : value)
-        {
-            write_bson_element(el.first, el.second);
-        }
-
-        oa->write_character(to_char_type(0x00));
-    }
-
-    //////////
-    // CBOR //
-    //////////
-
-    static constexpr CharType get_cbor_float_prefix(float /*unused*/)
-    {
-        return to_char_type(0xFA);  // Single-Precision Float
-    }
-
-    static constexpr CharType get_cbor_float_prefix(double /*unused*/)
-    {
-        return to_char_type(0xFB);  // Double-Precision Float
-    }
-
-    /////////////
-    // MsgPack //
-    /////////////
-
-    static constexpr CharType get_msgpack_float_prefix(float /*unused*/)
-    {
-        return to_char_type(0xCA);  // float 32
-    }
-
-    static constexpr CharType get_msgpack_float_prefix(double /*unused*/)
-    {
-        return to_char_type(0xCB);  // float 64
-    }
-
-    ////////////
-    // UBJSON //
-    ////////////
-
-    // UBJSON: write number (floating point)
-    template<typename NumberType, typename std::enable_if<
-                 std::is_floating_point<NumberType>::value, int>::type = 0>
-    void write_number_with_ubjson_prefix(const NumberType n,
-                                         const bool add_prefix)
-    {
-        if (add_prefix)
-        {
-            oa->write_character(get_ubjson_float_prefix(n));
-        }
-        write_number(n);
-    }
-
-    // UBJSON: write number (unsigned integer)
-    template<typename NumberType, typename std::enable_if<
-                 std::is_unsigned<NumberType>::value, int>::type = 0>
-    void write_number_with_ubjson_prefix(const NumberType n,
-                                         const bool add_prefix)
-    {
-        if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)()))
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('i'));  // int8
-            }
-            write_number(static_cast<uint8_t>(n));
-        }
-        else if (n <= (std::numeric_limits<uint8_t>::max)())
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('U'));  // uint8
-            }
-            write_number(static_cast<uint8_t>(n));
-        }
-        else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)()))
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('I'));  // int16
-            }
-            write_number(static_cast<int16_t>(n));
-        }
-        else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)()))
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('l'));  // int32
-            }
-            write_number(static_cast<int32_t>(n));
-        }
-        else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)()))
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('L'));  // int64
-            }
-            write_number(static_cast<int64_t>(n));
-        }
-        else
-        {
-            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
-        }
-    }
-
-    // UBJSON: write number (signed integer)
-    template<typename NumberType, typename std::enable_if<
-                 std::is_signed<NumberType>::value and
-                 not std::is_floating_point<NumberType>::value, int>::type = 0>
-    void write_number_with_ubjson_prefix(const NumberType n,
-                                         const bool add_prefix)
-    {
-        if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)())
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('i'));  // int8
-            }
-            write_number(static_cast<int8_t>(n));
-        }
-        else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)()))
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('U'));  // uint8
-            }
-            write_number(static_cast<uint8_t>(n));
-        }
-        else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)())
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('I'));  // int16
-            }
-            write_number(static_cast<int16_t>(n));
-        }
-        else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)())
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('l'));  // int32
-            }
-            write_number(static_cast<int32_t>(n));
-        }
-        else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)())
-        {
-            if (add_prefix)
-            {
-                oa->write_character(to_char_type('L'));  // int64
-            }
-            write_number(static_cast<int64_t>(n));
-        }
-        // LCOV_EXCL_START
-        else
-        {
-            JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64"));
-        }
-        // LCOV_EXCL_STOP
-    }
-
-    /*!
-    @brief determine the type prefix of container values
-
-    @note This function does not need to be 100% accurate when it comes to
-          integer limits. In case a number exceeds the limits of int64_t,
-          this will be detected by a later call to function
-          write_number_with_ubjson_prefix. Therefore, we return 'L' for any
-          value that does not fit the previous limits.
-    */
-    CharType ubjson_prefix(const BasicJsonType& j) const noexcept
-    {
-        switch (j.type())
-        {
-            case value_t::null:
-                return 'Z';
-
-            case value_t::boolean:
-                return j.m_value.boolean ? 'T' : 'F';
-
-            case value_t::number_integer:
-            {
-                if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
-                {
-                    return 'i';
-                }
-                if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    return 'U';
-                }
-                if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
-                {
-                    return 'I';
-                }
-                if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
-                {
-                    return 'l';
-                }
-                // no check and assume int64_t (see note above)
-                return 'L';
-            }
-
-            case value_t::number_unsigned:
-            {
-                if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)())
-                {
-                    return 'i';
-                }
-                if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                {
-                    return 'U';
-                }
-                if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)())
-                {
-                    return 'I';
-                }
-                if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)())
-                {
-                    return 'l';
-                }
-                // no check and assume int64_t (see note above)
-                return 'L';
-            }
-
-            case value_t::number_float:
-                return get_ubjson_float_prefix(j.m_value.number_float);
-
-            case value_t::string:
-                return 'S';
-
-            case value_t::array:
-                return '[';
-
-            case value_t::object:
-                return '{';
-
-            default:  // discarded values
-                return 'N';
-        }
-    }
-
-    static constexpr CharType get_ubjson_float_prefix(float /*unused*/)
-    {
-        return 'd';  // float 32
-    }
-
-    static constexpr CharType get_ubjson_float_prefix(double /*unused*/)
-    {
-        return 'D';  // float 64
-    }
-
-    ///////////////////////
-    // Utility functions //
-    ///////////////////////
-
-    /*
-    @brief write a number to output input
-    @param[in] n number of type @a NumberType
-    @tparam NumberType the type of the number
-    @tparam OutputIsLittleEndian Set to true if output data is
-                                 required to be little endian
-
-    @note This function needs to respect the system's endianess, because bytes
-          in CBOR, MessagePack, and UBJSON are stored in network order (big
-          endian) and therefore need reordering on little endian systems.
-    */
-    template<typename NumberType, bool OutputIsLittleEndian = false>
-    void write_number(const NumberType n)
-    {
-        // step 1: write number to array of length NumberType
-        std::array<CharType, sizeof(NumberType)> vec;
-        std::memcpy(vec.data(), &n, sizeof(NumberType));
-
-        // step 2: write array to output (with possible reordering)
-        if (is_little_endian and not OutputIsLittleEndian)
-        {
-            // reverse byte order prior to conversion if necessary
-            std::reverse(vec.begin(), vec.end());
-        }
-
-        oa->write_characters(vec.data(), sizeof(NumberType));
-    }
-
-  public:
-    // The following to_char_type functions are implement the conversion
-    // between uint8_t and CharType. In case CharType is not unsigned,
-    // such a conversion is required to allow values greater than 128.
-    // See <https://github.com/nlohmann/json/issues/1286> for a discussion.
-    template < typename C = CharType,
-               enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr >
-    static constexpr CharType to_char_type(std::uint8_t x) noexcept
-    {
-        return *reinterpret_cast<char*>(&x);
-    }
-
-    template < typename C = CharType,
-               enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr >
-    static CharType to_char_type(std::uint8_t x) noexcept
-    {
-        static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t");
-        static_assert(std::is_pod<CharType>::value, "CharType must be POD");
-        CharType result;
-        std::memcpy(&result, &x, sizeof(x));
-        return result;
-    }
-
-    template<typename C = CharType,
-             enable_if_t<std::is_unsigned<C>::value>* = nullptr>
-    static constexpr CharType to_char_type(std::uint8_t x) noexcept
-    {
-        return x;
-    }
-
-    template < typename InputCharType, typename C = CharType,
-               enable_if_t <
-                   std::is_signed<C>::value and
-                   std::is_signed<char>::value and
-                   std::is_same<char, typename std::remove_cv<InputCharType>::type>::value
-                   > * = nullptr >
-    static constexpr CharType to_char_type(InputCharType x) noexcept
-    {
-        return x;
-    }
-
-  private:
-    /// whether we can assume little endianess
-    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
-
-    /// the output
-    output_adapter_t<CharType> oa = nullptr;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/output/serializer.hpp>
-
-
-#include <algorithm> // reverse, remove, fill, find, none_of
-#include <array> // array
-#include <cassert> // assert
-#include <ciso646> // and, or
-#include <clocale> // localeconv, lconv
-#include <cmath> // labs, isfinite, isnan, signbit
-#include <cstddef> // size_t, ptrdiff_t
-#include <cstdint> // uint8_t
-#include <cstdio> // snprintf
-#include <limits> // numeric_limits
-#include <string> // string
-#include <type_traits> // is_same
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/conversions/to_chars.hpp>
-
-
-#include <cassert> // assert
-#include <ciso646> // or, and, not
-#include <cmath>   // signbit, isfinite
-#include <cstdint> // intN_t, uintN_t
-#include <cstring> // memcpy, memmove
-
-namespace nlohmann
-{
-namespace detail
-{
-
-/*!
-@brief implements the Grisu2 algorithm for binary to decimal floating-point
-conversion.
-
-This implementation is a slightly modified version of the reference
-implementation which may be obtained from
-http://florian.loitsch.com/publications (bench.tar.gz).
-
-The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.
-
-For a detailed description of the algorithm see:
-
-[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with
-    Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming
-    Language Design and Implementation, PLDI 2010
-[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately",
-    Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language
-    Design and Implementation, PLDI 1996
-*/
-namespace dtoa_impl
-{
-
-template <typename Target, typename Source>
-Target reinterpret_bits(const Source source)
-{
-    static_assert(sizeof(Target) == sizeof(Source), "size mismatch");
-
-    Target target;
-    std::memcpy(&target, &source, sizeof(Source));
-    return target;
-}
-
-struct diyfp // f * 2^e
-{
-    static constexpr int kPrecision = 64; // = q
-
-    uint64_t f = 0;
-    int e = 0;
-
-    constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {}
-
-    /*!
-    @brief returns x - y
-    @pre x.e == y.e and x.f >= y.f
-    */
-    static diyfp sub(const diyfp& x, const diyfp& y) noexcept
-    {
-        assert(x.e == y.e);
-        assert(x.f >= y.f);
-
-        return {x.f - y.f, x.e};
-    }
-
-    /*!
-    @brief returns x * y
-    @note The result is rounded. (Only the upper q bits are returned.)
-    */
-    static diyfp mul(const diyfp& x, const diyfp& y) noexcept
-    {
-        static_assert(kPrecision == 64, "internal error");
-
-        // Computes:
-        //  f = round((x.f * y.f) / 2^q)
-        //  e = x.e + y.e + q
-
-        // Emulate the 64-bit * 64-bit multiplication:
-        //
-        // p = u * v
-        //   = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)
-        //   = (u_lo v_lo         ) + 2^32 ((u_lo v_hi         ) + (u_hi v_lo         )) + 2^64 (u_hi v_hi         )
-        //   = (p0                ) + 2^32 ((p1                ) + (p2                )) + 2^64 (p3                )
-        //   = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3                )
-        //   = (p0_lo             ) + 2^32 (p0_hi + p1_lo + p2_lo                      ) + 2^64 (p1_hi + p2_hi + p3)
-        //   = (p0_lo             ) + 2^32 (Q                                          ) + 2^64 (H                 )
-        //   = (p0_lo             ) + 2^32 (Q_lo + 2^32 Q_hi                           ) + 2^64 (H                 )
-        //
-        // (Since Q might be larger than 2^32 - 1)
-        //
-        //   = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)
-        //
-        // (Q_hi + H does not overflow a 64-bit int)
-        //
-        //   = p_lo + 2^64 p_hi
-
-        const uint64_t u_lo = x.f & 0xFFFFFFFF;
-        const uint64_t u_hi = x.f >> 32;
-        const uint64_t v_lo = y.f & 0xFFFFFFFF;
-        const uint64_t v_hi = y.f >> 32;
-
-        const uint64_t p0 = u_lo * v_lo;
-        const uint64_t p1 = u_lo * v_hi;
-        const uint64_t p2 = u_hi * v_lo;
-        const uint64_t p3 = u_hi * v_hi;
-
-        const uint64_t p0_hi = p0 >> 32;
-        const uint64_t p1_lo = p1 & 0xFFFFFFFF;
-        const uint64_t p1_hi = p1 >> 32;
-        const uint64_t p2_lo = p2 & 0xFFFFFFFF;
-        const uint64_t p2_hi = p2 >> 32;
-
-        uint64_t Q = p0_hi + p1_lo + p2_lo;
-
-        // The full product might now be computed as
-        //
-        // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)
-        // p_lo = p0_lo + (Q << 32)
-        //
-        // But in this particular case here, the full p_lo is not required.
-        // Effectively we only need to add the highest bit in p_lo to p_hi (and
-        // Q_hi + 1 does not overflow).
-
-        Q += uint64_t{1} << (64 - 32 - 1); // round, ties up
-
-        const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32);
-
-        return {h, x.e + y.e + 64};
-    }
-
-    /*!
-    @brief normalize x such that the significand is >= 2^(q-1)
-    @pre x.f != 0
-    */
-    static diyfp normalize(diyfp x) noexcept
-    {
-        assert(x.f != 0);
-
-        while ((x.f >> 63) == 0)
-        {
-            x.f <<= 1;
-            x.e--;
-        }
-
-        return x;
-    }
-
-    /*!
-    @brief normalize x such that the result has the exponent E
-    @pre e >= x.e and the upper e - x.e bits of x.f must be zero.
-    */
-    static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept
-    {
-        const int delta = x.e - target_exponent;
-
-        assert(delta >= 0);
-        assert(((x.f << delta) >> delta) == x.f);
-
-        return {x.f << delta, target_exponent};
-    }
-};
-
-struct boundaries
-{
-    diyfp w;
-    diyfp minus;
-    diyfp plus;
-};
-
-/*!
-Compute the (normalized) diyfp representing the input number 'value' and its
-boundaries.
-
-@pre value must be finite and positive
-*/
-template <typename FloatType>
-boundaries compute_boundaries(FloatType value)
-{
-    assert(std::isfinite(value));
-    assert(value > 0);
-
-    // Convert the IEEE representation into a diyfp.
-    //
-    // If v is denormal:
-    //      value = 0.F * 2^(1 - bias) = (          F) * 2^(1 - bias - (p-1))
-    // If v is normalized:
-    //      value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))
-
-    static_assert(std::numeric_limits<FloatType>::is_iec559,
-                  "internal error: dtoa_short requires an IEEE-754 floating-point implementation");
-
-    constexpr int      kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)
-    constexpr int      kBias      = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);
-    constexpr int      kMinExp    = 1 - kBias;
-    constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1)
-
-    using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type;
-
-    const uint64_t bits = reinterpret_bits<bits_type>(value);
-    const uint64_t E = bits >> (kPrecision - 1);
-    const uint64_t F = bits & (kHiddenBit - 1);
-
-    const bool is_denormal = (E == 0);
-    const diyfp v = is_denormal
-                    ? diyfp(F, kMinExp)
-                    : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);
-
-    // Compute the boundaries m- and m+ of the floating-point value
-    // v = f * 2^e.
-    //
-    // Determine v- and v+, the floating-point predecessor and successor if v,
-    // respectively.
-    //
-    //      v- = v - 2^e        if f != 2^(p-1) or e == e_min                (A)
-    //         = v - 2^(e-1)    if f == 2^(p-1) and e > e_min                (B)
-    //
-    //      v+ = v + 2^e
-    //
-    // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_
-    // between m- and m+ round to v, regardless of how the input rounding
-    // algorithm breaks ties.
-    //
-    //      ---+-------------+-------------+-------------+-------------+---  (A)
-    //         v-            m-            v             m+            v+
-    //
-    //      -----------------+------+------+-------------+-------------+---  (B)
-    //                       v-     m-     v             m+            v+
-
-    const bool lower_boundary_is_closer = (F == 0 and E > 1);
-    const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);
-    const diyfp m_minus = lower_boundary_is_closer
-                          ? diyfp(4 * v.f - 1, v.e - 2)  // (B)
-                          : diyfp(2 * v.f - 1, v.e - 1); // (A)
-
-    // Determine the normalized w+ = m+.
-    const diyfp w_plus = diyfp::normalize(m_plus);
-
-    // Determine w- = m- such that e_(w-) = e_(w+).
-    const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);
-
-    return {diyfp::normalize(v), w_minus, w_plus};
-}
-
-// Given normalized diyfp w, Grisu needs to find a (normalized) cached
-// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies
-// within a certain range [alpha, gamma] (Definition 3.2 from [1])
-//
-//      alpha <= e = e_c + e_w + q <= gamma
-//
-// or
-//
-//      f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q
-//                          <= f_c * f_w * 2^gamma
-//
-// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies
-//
-//      2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma
-//
-// or
-//
-//      2^(q - 2 + alpha) <= c * w < 2^(q + gamma)
-//
-// The choice of (alpha,gamma) determines the size of the table and the form of
-// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well
-// in practice:
-//
-// The idea is to cut the number c * w = f * 2^e into two parts, which can be
-// processed independently: An integral part p1, and a fractional part p2:
-//
-//      f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e
-//              = (f div 2^-e) + (f mod 2^-e) * 2^e
-//              = p1 + p2 * 2^e
-//
-// The conversion of p1 into decimal form requires a series of divisions and
-// modulos by (a power of) 10. These operations are faster for 32-bit than for
-// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be
-// achieved by choosing
-//
-//      -e >= 32   or   e <= -32 := gamma
-//
-// In order to convert the fractional part
-//
-//      p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...
-//
-// into decimal form, the fraction is repeatedly multiplied by 10 and the digits
-// d[-i] are extracted in order:
-//
-//      (10 * p2) div 2^-e = d[-1]
-//      (10 * p2) mod 2^-e = d[-2] / 10^1 + ...
-//
-// The multiplication by 10 must not overflow. It is sufficient to choose
-//
-//      10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.
-//
-// Since p2 = f mod 2^-e < 2^-e,
-//
-//      -e <= 60   or   e >= -60 := alpha
-
-constexpr int kAlpha = -60;
-constexpr int kGamma = -32;
-
-struct cached_power // c = f * 2^e ~= 10^k
-{
-    uint64_t f;
-    int e;
-    int k;
-};
-
-/*!
-For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached
-power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c
-satisfies (Definition 3.2 from [1])
-
-     alpha <= e_c + e + q <= gamma.
-*/
-inline cached_power get_cached_power_for_binary_exponent(int e)
-{
-    // Now
-    //
-    //      alpha <= e_c + e + q <= gamma                                    (1)
-    //      ==> f_c * 2^alpha <= c * 2^e * 2^q
-    //
-    // and since the c's are normalized, 2^(q-1) <= f_c,
-    //
-    //      ==> 2^(q - 1 + alpha) <= c * 2^(e + q)
-    //      ==> 2^(alpha - e - 1) <= c
-    //
-    // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as
-    //
-    //      k = ceil( log_10( 2^(alpha - e - 1) ) )
-    //        = ceil( (alpha - e - 1) * log_10(2) )
-    //
-    // From the paper:
-    // "In theory the result of the procedure could be wrong since c is rounded,
-    //  and the computation itself is approximated [...]. In practice, however,
-    //  this simple function is sufficient."
-    //
-    // For IEEE double precision floating-point numbers converted into
-    // normalized diyfp's w = f * 2^e, with q = 64,
-    //
-    //      e >= -1022      (min IEEE exponent)
-    //           -52        (p - 1)
-    //           -52        (p - 1, possibly normalize denormal IEEE numbers)
-    //           -11        (normalize the diyfp)
-    //         = -1137
-    //
-    // and
-    //
-    //      e <= +1023      (max IEEE exponent)
-    //           -52        (p - 1)
-    //           -11        (normalize the diyfp)
-    //         = 960
-    //
-    // This binary exponent range [-1137,960] results in a decimal exponent
-    // range [-307,324]. One does not need to store a cached power for each
-    // k in this range. For each such k it suffices to find a cached power
-    // such that the exponent of the product lies in [alpha,gamma].
-    // This implies that the difference of the decimal exponents of adjacent
-    // table entries must be less than or equal to
-    //
-    //      floor( (gamma - alpha) * log_10(2) ) = 8.
-    //
-    // (A smaller distance gamma-alpha would require a larger table.)
-
-    // NB:
-    // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.
-
-    constexpr int kCachedPowersSize = 79;
-    constexpr int kCachedPowersMinDecExp = -300;
-    constexpr int kCachedPowersDecStep = 8;
-
-    static constexpr cached_power kCachedPowers[] =
-    {
-        { 0xAB70FE17C79AC6CA, -1060, -300 },
-        { 0xFF77B1FCBEBCDC4F, -1034, -292 },
-        { 0xBE5691EF416BD60C, -1007, -284 },
-        { 0x8DD01FAD907FFC3C,  -980, -276 },
-        { 0xD3515C2831559A83,  -954, -268 },
-        { 0x9D71AC8FADA6C9B5,  -927, -260 },
-        { 0xEA9C227723EE8BCB,  -901, -252 },
-        { 0xAECC49914078536D,  -874, -244 },
-        { 0x823C12795DB6CE57,  -847, -236 },
-        { 0xC21094364DFB5637,  -821, -228 },
-        { 0x9096EA6F3848984F,  -794, -220 },
-        { 0xD77485CB25823AC7,  -768, -212 },
-        { 0xA086CFCD97BF97F4,  -741, -204 },
-        { 0xEF340A98172AACE5,  -715, -196 },
-        { 0xB23867FB2A35B28E,  -688, -188 },
-        { 0x84C8D4DFD2C63F3B,  -661, -180 },
-        { 0xC5DD44271AD3CDBA,  -635, -172 },
-        { 0x936B9FCEBB25C996,  -608, -164 },
-        { 0xDBAC6C247D62A584,  -582, -156 },
-        { 0xA3AB66580D5FDAF6,  -555, -148 },
-        { 0xF3E2F893DEC3F126,  -529, -140 },
-        { 0xB5B5ADA8AAFF80B8,  -502, -132 },
-        { 0x87625F056C7C4A8B,  -475, -124 },
-        { 0xC9BCFF6034C13053,  -449, -116 },
-        { 0x964E858C91BA2655,  -422, -108 },
-        { 0xDFF9772470297EBD,  -396, -100 },
-        { 0xA6DFBD9FB8E5B88F,  -369,  -92 },
-        { 0xF8A95FCF88747D94,  -343,  -84 },
-        { 0xB94470938FA89BCF,  -316,  -76 },
-        { 0x8A08F0F8BF0F156B,  -289,  -68 },
-        { 0xCDB02555653131B6,  -263,  -60 },
-        { 0x993FE2C6D07B7FAC,  -236,  -52 },
-        { 0xE45C10C42A2B3B06,  -210,  -44 },
-        { 0xAA242499697392D3,  -183,  -36 },
-        { 0xFD87B5F28300CA0E,  -157,  -28 },
-        { 0xBCE5086492111AEB,  -130,  -20 },
-        { 0x8CBCCC096F5088CC,  -103,  -12 },
-        { 0xD1B71758E219652C,   -77,   -4 },
-        { 0x9C40000000000000,   -50,    4 },
-        { 0xE8D4A51000000000,   -24,   12 },
-        { 0xAD78EBC5AC620000,     3,   20 },
-        { 0x813F3978F8940984,    30,   28 },
-        { 0xC097CE7BC90715B3,    56,   36 },
-        { 0x8F7E32CE7BEA5C70,    83,   44 },
-        { 0xD5D238A4ABE98068,   109,   52 },
-        { 0x9F4F2726179A2245,   136,   60 },
-        { 0xED63A231D4C4FB27,   162,   68 },
-        { 0xB0DE65388CC8ADA8,   189,   76 },
-        { 0x83C7088E1AAB65DB,   216,   84 },
-        { 0xC45D1DF942711D9A,   242,   92 },
-        { 0x924D692CA61BE758,   269,  100 },
-        { 0xDA01EE641A708DEA,   295,  108 },
-        { 0xA26DA3999AEF774A,   322,  116 },
-        { 0xF209787BB47D6B85,   348,  124 },
-        { 0xB454E4A179DD1877,   375,  132 },
-        { 0x865B86925B9BC5C2,   402,  140 },
-        { 0xC83553C5C8965D3D,   428,  148 },
-        { 0x952AB45CFA97A0B3,   455,  156 },
-        { 0xDE469FBD99A05FE3,   481,  164 },
-        { 0xA59BC234DB398C25,   508,  172 },
-        { 0xF6C69A72A3989F5C,   534,  180 },
-        { 0xB7DCBF5354E9BECE,   561,  188 },
-        { 0x88FCF317F22241E2,   588,  196 },
-        { 0xCC20CE9BD35C78A5,   614,  204 },
-        { 0x98165AF37B2153DF,   641,  212 },
-        { 0xE2A0B5DC971F303A,   667,  220 },
-        { 0xA8D9D1535CE3B396,   694,  228 },
-        { 0xFB9B7CD9A4A7443C,   720,  236 },
-        { 0xBB764C4CA7A44410,   747,  244 },
-        { 0x8BAB8EEFB6409C1A,   774,  252 },
-        { 0xD01FEF10A657842C,   800,  260 },
-        { 0x9B10A4E5E9913129,   827,  268 },
-        { 0xE7109BFBA19C0C9D,   853,  276 },
-        { 0xAC2820D9623BF429,   880,  284 },
-        { 0x80444B5E7AA7CF85,   907,  292 },
-        { 0xBF21E44003ACDD2D,   933,  300 },
-        { 0x8E679C2F5E44FF8F,   960,  308 },
-        { 0xD433179D9C8CB841,   986,  316 },
-        { 0x9E19DB92B4E31BA9,  1013,  324 },
-    };
-
-    // This computation gives exactly the same results for k as
-    //      k = ceil((kAlpha - e - 1) * 0.30102999566398114)
-    // for |e| <= 1500, but doesn't require floating-point operations.
-    // NB: log_10(2) ~= 78913 / 2^18
-    assert(e >= -1500);
-    assert(e <=  1500);
-    const int f = kAlpha - e - 1;
-    const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);
-
-    const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;
-    assert(index >= 0);
-    assert(index < kCachedPowersSize);
-    static_cast<void>(kCachedPowersSize); // Fix warning.
-
-    const cached_power cached = kCachedPowers[index];
-    assert(kAlpha <= cached.e + e + 64);
-    assert(kGamma >= cached.e + e + 64);
-
-    return cached;
-}
-
-/*!
-For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.
-For n == 0, returns 1 and sets pow10 := 1.
-*/
-inline int find_largest_pow10(const uint32_t n, uint32_t& pow10)
-{
-    // LCOV_EXCL_START
-    if (n >= 1000000000)
-    {
-        pow10 = 1000000000;
-        return 10;
-    }
-    // LCOV_EXCL_STOP
-    else if (n >= 100000000)
-    {
-        pow10 = 100000000;
-        return  9;
-    }
-    else if (n >= 10000000)
-    {
-        pow10 = 10000000;
-        return  8;
-    }
-    else if (n >= 1000000)
-    {
-        pow10 = 1000000;
-        return  7;
-    }
-    else if (n >= 100000)
-    {
-        pow10 = 100000;
-        return  6;
-    }
-    else if (n >= 10000)
-    {
-        pow10 = 10000;
-        return  5;
-    }
-    else if (n >= 1000)
-    {
-        pow10 = 1000;
-        return  4;
-    }
-    else if (n >= 100)
-    {
-        pow10 = 100;
-        return  3;
-    }
-    else if (n >= 10)
-    {
-        pow10 = 10;
-        return  2;
-    }
-    else
-    {
-        pow10 = 1;
-        return 1;
-    }
-}
-
-inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta,
-                         uint64_t rest, uint64_t ten_k)
-{
-    assert(len >= 1);
-    assert(dist <= delta);
-    assert(rest <= delta);
-    assert(ten_k > 0);
-
-    //               <--------------------------- delta ---->
-    //                                  <---- dist --------->
-    // --------------[------------------+-------------------]--------------
-    //               M-                 w                   M+
-    //
-    //                                  ten_k
-    //                                <------>
-    //                                       <---- rest ---->
-    // --------------[------------------+----+--------------]--------------
-    //                                  w    V
-    //                                       = buf * 10^k
-    //
-    // ten_k represents a unit-in-the-last-place in the decimal representation
-    // stored in buf.
-    // Decrement buf by ten_k while this takes buf closer to w.
-
-    // The tests are written in this order to avoid overflow in unsigned
-    // integer arithmetic.
-
-    while (rest < dist
-            and delta - rest >= ten_k
-            and (rest + ten_k < dist or dist - rest > rest + ten_k - dist))
-    {
-        assert(buf[len - 1] != '0');
-        buf[len - 1]--;
-        rest += ten_k;
-    }
-}
-
-/*!
-Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.
-M- and M+ must be normalized and share the same exponent -60 <= e <= -32.
-*/
-inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,
-                             diyfp M_minus, diyfp w, diyfp M_plus)
-{
-    static_assert(kAlpha >= -60, "internal error");
-    static_assert(kGamma <= -32, "internal error");
-
-    // Generates the digits (and the exponent) of a decimal floating-point
-    // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's
-    // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.
-    //
-    //               <--------------------------- delta ---->
-    //                                  <---- dist --------->
-    // --------------[------------------+-------------------]--------------
-    //               M-                 w                   M+
-    //
-    // Grisu2 generates the digits of M+ from left to right and stops as soon as
-    // V is in [M-,M+].
-
-    assert(M_plus.e >= kAlpha);
-    assert(M_plus.e <= kGamma);
-
-    uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)
-    uint64_t dist  = diyfp::sub(M_plus, w      ).f; // (significand of (M+ - w ), implicit exponent is e)
-
-    // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):
-    //
-    //      M+ = f * 2^e
-    //         = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e
-    //         = ((p1        ) * 2^-e + (p2        )) * 2^e
-    //         = p1 + p2 * 2^e
-
-    const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e);
-
-    auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)
-    uint64_t p2 = M_plus.f & (one.f - 1);                    // p2 = f mod 2^-e
-
-    // 1)
-    //
-    // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]
-
-    assert(p1 > 0);
-
-    uint32_t pow10;
-    const int k = find_largest_pow10(p1, pow10);
-
-    //      10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)
-    //
-    //      p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))
-    //         = (d[k-1]         ) * 10^(k-1) + (p1 mod 10^(k-1))
-    //
-    //      M+ = p1                                             + p2 * 2^e
-    //         = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1))          + p2 * 2^e
-    //         = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e
-    //         = d[k-1] * 10^(k-1) + (                         rest) * 2^e
-    //
-    // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)
-    //
-    //      p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]
-    //
-    // but stop as soon as
-    //
-    //      rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e
-
-    int n = k;
-    while (n > 0)
-    {
-        // Invariants:
-        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)    (buffer = 0 for n = k)
-        //      pow10 = 10^(n-1) <= p1 < 10^n
-        //
-        const uint32_t d = p1 / pow10;  // d = p1 div 10^(n-1)
-        const uint32_t r = p1 % pow10;  // r = p1 mod 10^(n-1)
-        //
-        //      M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e
-        //         = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)
-        //
-        assert(d <= 9);
-        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
-        //
-        //      M+ = buffer * 10^(n-1) + (r + p2 * 2^e)
-        //
-        p1 = r;
-        n--;
-        //
-        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)
-        //      pow10 = 10^n
-        //
-
-        // Now check if enough digits have been generated.
-        // Compute
-        //
-        //      p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e
-        //
-        // Note:
-        // Since rest and delta share the same exponent e, it suffices to
-        // compare the significands.
-        const uint64_t rest = (uint64_t{p1} << -one.e) + p2;
-        if (rest <= delta)
-        {
-            // V = buffer * 10^n, with M- <= V <= M+.
-
-            decimal_exponent += n;
-
-            // We may now just stop. But instead look if the buffer could be
-            // decremented to bring V closer to w.
-            //
-            // pow10 = 10^n is now 1 ulp in the decimal representation V.
-            // The rounding procedure works with diyfp's with an implicit
-            // exponent of e.
-            //
-            //      10^n = (10^n * 2^-e) * 2^e = ulp * 2^e
-            //
-            const uint64_t ten_n = uint64_t{pow10} << -one.e;
-            grisu2_round(buffer, length, dist, delta, rest, ten_n);
-
-            return;
-        }
-
-        pow10 /= 10;
-        //
-        //      pow10 = 10^(n-1) <= p1 < 10^n
-        // Invariants restored.
-    }
-
-    // 2)
-    //
-    // The digits of the integral part have been generated:
-    //
-    //      M+ = d[k-1]...d[1]d[0] + p2 * 2^e
-    //         = buffer            + p2 * 2^e
-    //
-    // Now generate the digits of the fractional part p2 * 2^e.
-    //
-    // Note:
-    // No decimal point is generated: the exponent is adjusted instead.
-    //
-    // p2 actually represents the fraction
-    //
-    //      p2 * 2^e
-    //          = p2 / 2^-e
-    //          = d[-1] / 10^1 + d[-2] / 10^2 + ...
-    //
-    // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)
-    //
-    //      p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m
-    //                      + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)
-    //
-    // using
-    //
-    //      10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)
-    //                = (                   d) * 2^-e + (                   r)
-    //
-    // or
-    //      10^m * p2 * 2^e = d + r * 2^e
-    //
-    // i.e.
-    //
-    //      M+ = buffer + p2 * 2^e
-    //         = buffer + 10^-m * (d + r * 2^e)
-    //         = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e
-    //
-    // and stop as soon as 10^-m * r * 2^e <= delta * 2^e
-
-    assert(p2 > delta);
-
-    int m = 0;
-    for (;;)
-    {
-        // Invariant:
-        //      M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e
-        //         = buffer * 10^-m + 10^-m * (p2                                 ) * 2^e
-        //         = buffer * 10^-m + 10^-m * (1/10 * (10 * p2)                   ) * 2^e
-        //         = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e
-        //
-        assert(p2 <= UINT64_MAX / 10);
-        p2 *= 10;
-        const uint64_t d = p2 >> -one.e;     // d = (10 * p2) div 2^-e
-        const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e
-        //
-        //      M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e
-        //         = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))
-        //         = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e
-        //
-        assert(d <= 9);
-        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d
-        //
-        //      M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e
-        //
-        p2 = r;
-        m++;
-        //
-        //      M+ = buffer * 10^-m + 10^-m * p2 * 2^e
-        // Invariant restored.
-
-        // Check if enough digits have been generated.
-        //
-        //      10^-m * p2 * 2^e <= delta * 2^e
-        //              p2 * 2^e <= 10^m * delta * 2^e
-        //                    p2 <= 10^m * delta
-        delta *= 10;
-        dist  *= 10;
-        if (p2 <= delta)
-        {
-            break;
-        }
-    }
-
-    // V = buffer * 10^-m, with M- <= V <= M+.
-
-    decimal_exponent -= m;
-
-    // 1 ulp in the decimal representation is now 10^-m.
-    // Since delta and dist are now scaled by 10^m, we need to do the
-    // same with ulp in order to keep the units in sync.
-    //
-    //      10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e
-    //
-    const uint64_t ten_m = one.f;
-    grisu2_round(buffer, length, dist, delta, p2, ten_m);
-
-    // By construction this algorithm generates the shortest possible decimal
-    // number (Loitsch, Theorem 6.2) which rounds back to w.
-    // For an input number of precision p, at least
-    //
-    //      N = 1 + ceil(p * log_10(2))
-    //
-    // decimal digits are sufficient to identify all binary floating-point
-    // numbers (Matula, "In-and-Out conversions").
-    // This implies that the algorithm does not produce more than N decimal
-    // digits.
-    //
-    //      N = 17 for p = 53 (IEEE double precision)
-    //      N = 9  for p = 24 (IEEE single precision)
-}
-
-/*!
-v = buf * 10^decimal_exponent
-len is the length of the buffer (number of decimal digits)
-The buffer must be large enough, i.e. >= max_digits10.
-*/
-inline void grisu2(char* buf, int& len, int& decimal_exponent,
-                   diyfp m_minus, diyfp v, diyfp m_plus)
-{
-    assert(m_plus.e == m_minus.e);
-    assert(m_plus.e == v.e);
-
-    //  --------(-----------------------+-----------------------)--------    (A)
-    //          m-                      v                       m+
-    //
-    //  --------------------(-----------+-----------------------)--------    (B)
-    //                      m-          v                       m+
-    //
-    // First scale v (and m- and m+) such that the exponent is in the range
-    // [alpha, gamma].
-
-    const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);
-
-    const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k
-
-    // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]
-    const diyfp w       = diyfp::mul(v,       c_minus_k);
-    const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);
-    const diyfp w_plus  = diyfp::mul(m_plus,  c_minus_k);
-
-    //  ----(---+---)---------------(---+---)---------------(---+---)----
-    //          w-                      w                       w+
-    //          = c*m-                  = c*v                   = c*m+
-    //
-    // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and
-    // w+ are now off by a small amount.
-    // In fact:
-    //
-    //      w - v * 10^k < 1 ulp
-    //
-    // To account for this inaccuracy, add resp. subtract 1 ulp.
-    //
-    //  --------+---[---------------(---+---)---------------]---+--------
-    //          w-  M-                  w                   M+  w+
-    //
-    // Now any number in [M-, M+] (bounds included) will round to w when input,
-    // regardless of how the input rounding algorithm breaks ties.
-    //
-    // And digit_gen generates the shortest possible such number in [M-, M+].
-    // Note that this does not mean that Grisu2 always generates the shortest
-    // possible number in the interval (m-, m+).
-    const diyfp M_minus(w_minus.f + 1, w_minus.e);
-    const diyfp M_plus (w_plus.f  - 1, w_plus.e );
-
-    decimal_exponent = -cached.k; // = -(-k) = k
-
-    grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);
-}
-
-/*!
-v = buf * 10^decimal_exponent
-len is the length of the buffer (number of decimal digits)
-The buffer must be large enough, i.e. >= max_digits10.
-*/
-template <typename FloatType>
-void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
-{
-    static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,
-                  "internal error: not enough precision");
-
-    assert(std::isfinite(value));
-    assert(value > 0);
-
-    // If the neighbors (and boundaries) of 'value' are always computed for double-precision
-    // numbers, all float's can be recovered using strtod (and strtof). However, the resulting
-    // decimal representations are not exactly "short".
-    //
-    // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)
-    // says "value is converted to a string as if by std::sprintf in the default ("C") locale"
-    // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars'
-    // does.
-    // On the other hand, the documentation for 'std::to_chars' requires that "parsing the
-    // representation using the corresponding std::from_chars function recovers value exactly". That
-    // indicates that single precision floating-point numbers should be recovered using
-    // 'std::strtof'.
-    //
-    // NB: If the neighbors are computed for single-precision numbers, there is a single float
-    //     (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision
-    //     value is off by 1 ulp.
-#if 0
-    const boundaries w = compute_boundaries(static_cast<double>(value));
-#else
-    const boundaries w = compute_boundaries(value);
-#endif
-
-    grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
-}
-
-/*!
-@brief appends a decimal representation of e to buf
-@return a pointer to the element following the exponent.
-@pre -1000 < e < 1000
-*/
-inline char* append_exponent(char* buf, int e)
-{
-    assert(e > -1000);
-    assert(e <  1000);
-
-    if (e < 0)
-    {
-        e = -e;
-        *buf++ = '-';
-    }
-    else
-    {
-        *buf++ = '+';
-    }
-
-    auto k = static_cast<uint32_t>(e);
-    if (k < 10)
-    {
-        // Always print at least two digits in the exponent.
-        // This is for compatibility with printf("%g").
-        *buf++ = '0';
-        *buf++ = static_cast<char>('0' + k);
-    }
-    else if (k < 100)
-    {
-        *buf++ = static_cast<char>('0' + k / 10);
-        k %= 10;
-        *buf++ = static_cast<char>('0' + k);
-    }
-    else
-    {
-        *buf++ = static_cast<char>('0' + k / 100);
-        k %= 100;
-        *buf++ = static_cast<char>('0' + k / 10);
-        k %= 10;
-        *buf++ = static_cast<char>('0' + k);
-    }
-
-    return buf;
-}
-
-/*!
-@brief prettify v = buf * 10^decimal_exponent
-
-If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point
-notation. Otherwise it will be printed in exponential notation.
-
-@pre min_exp < 0
-@pre max_exp > 0
-*/
-inline char* format_buffer(char* buf, int len, int decimal_exponent,
-                           int min_exp, int max_exp)
-{
-    assert(min_exp < 0);
-    assert(max_exp > 0);
-
-    const int k = len;
-    const int n = len + decimal_exponent;
-
-    // v = buf * 10^(n-k)
-    // k is the length of the buffer (number of decimal digits)
-    // n is the position of the decimal point relative to the start of the buffer.
-
-    if (k <= n and n <= max_exp)
-    {
-        // digits[000]
-        // len <= max_exp + 2
-
-        std::memset(buf + k, '0', static_cast<size_t>(n - k));
-        // Make it look like a floating-point number (#362, #378)
-        buf[n + 0] = '.';
-        buf[n + 1] = '0';
-        return buf + (n + 2);
-    }
-
-    if (0 < n and n <= max_exp)
-    {
-        // dig.its
-        // len <= max_digits10 + 1
-
-        assert(k > n);
-
-        std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n));
-        buf[n] = '.';
-        return buf + (k + 1);
-    }
-
-    if (min_exp < n and n <= 0)
-    {
-        // 0.[000]digits
-        // len <= 2 + (-min_exp - 1) + max_digits10
-
-        std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k));
-        buf[0] = '0';
-        buf[1] = '.';
-        std::memset(buf + 2, '0', static_cast<size_t>(-n));
-        return buf + (2 + (-n) + k);
-    }
-
-    if (k == 1)
-    {
-        // dE+123
-        // len <= 1 + 5
-
-        buf += 1;
-    }
-    else
-    {
-        // d.igitsE+123
-        // len <= max_digits10 + 1 + 5
-
-        std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1));
-        buf[1] = '.';
-        buf += 1 + k;
-    }
-
-    *buf++ = 'e';
-    return append_exponent(buf, n - 1);
-}
-
-} // namespace dtoa_impl
-
-/*!
-@brief generates a decimal representation of the floating-point number value in [first, last).
-
-The format of the resulting decimal representation is similar to printf's %g
-format. Returns an iterator pointing past-the-end of the decimal representation.
-
-@note The input number must be finite, i.e. NaN's and Inf's are not supported.
-@note The buffer must be large enough.
-@note The result is NOT null-terminated.
-*/
-template <typename FloatType>
-char* to_chars(char* first, const char* last, FloatType value)
-{
-    static_cast<void>(last); // maybe unused - fix warning
-    assert(std::isfinite(value));
-
-    // Use signbit(value) instead of (value < 0) since signbit works for -0.
-    if (std::signbit(value))
-    {
-        value = -value;
-        *first++ = '-';
-    }
-
-    if (value == 0) // +-0
-    {
-        *first++ = '0';
-        // Make it look like a floating-point number (#362, #378)
-        *first++ = '.';
-        *first++ = '0';
-        return first;
-    }
-
-    assert(last - first >= std::numeric_limits<FloatType>::max_digits10);
-
-    // Compute v = buffer * 10^decimal_exponent.
-    // The decimal digits are stored in the buffer, which needs to be interpreted
-    // as an unsigned decimal integer.
-    // len is the length of the buffer, i.e. the number of decimal digits.
-    int len = 0;
-    int decimal_exponent = 0;
-    dtoa_impl::grisu2(first, len, decimal_exponent, value);
-
-    assert(len <= std::numeric_limits<FloatType>::max_digits10);
-
-    // Format the buffer like printf("%.*g", prec, value)
-    constexpr int kMinExp = -4;
-    // Use digits10 here to increase compatibility with version 2.
-    constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;
-
-    assert(last - first >= kMaxExp + 2);
-    assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);
-    assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);
-
-    return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
-}
-
-} // namespace detail
-} // namespace nlohmann
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/meta/cpp_future.hpp>
-
-// #include <nlohmann/detail/output/binary_writer.hpp>
-
-// #include <nlohmann/detail/output/output_adapters.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-///////////////////
-// serialization //
-///////////////////
-
-/// how to treat decoding errors
-enum class error_handler_t
-{
-    strict,  ///< throw a type_error exception in case of invalid UTF-8
-    replace, ///< replace invalid UTF-8 sequences with U+FFFD
-    ignore   ///< ignore invalid UTF-8 sequences
-};
-
-template<typename BasicJsonType>
-class serializer
-{
-    using string_t = typename BasicJsonType::string_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    static constexpr uint8_t UTF8_ACCEPT = 0;
-    static constexpr uint8_t UTF8_REJECT = 1;
-
-  public:
-    /*!
-    @param[in] s  output stream to serialize to
-    @param[in] ichar  indentation character to use
-    @param[in] error_handler_  how to react on decoding errors
-    */
-    serializer(output_adapter_t<char> s, const char ichar,
-               error_handler_t error_handler_ = error_handler_t::strict)
-        : o(std::move(s))
-        , loc(std::localeconv())
-        , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep))
-        , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point))
-        , indent_char(ichar)
-        , indent_string(512, indent_char)
-        , error_handler(error_handler_)
-    {}
-
-    // delete because of pointer members
-    serializer(const serializer&) = delete;
-    serializer& operator=(const serializer&) = delete;
-    serializer(serializer&&) = delete;
-    serializer& operator=(serializer&&) = delete;
-    ~serializer() = default;
-
-    /*!
-    @brief internal implementation of the serialization function
-
-    This function is called by the public member function dump and organizes
-    the serialization internally. The indentation level is propagated as
-    additional parameter. In case of arrays and objects, the function is
-    called recursively.
-
-    - strings and object keys are escaped using `escape_string()`
-    - integer numbers are converted implicitly via `operator<<`
-    - floating-point numbers are converted to a string using `"%g"` format
-
-    @param[in] val             value to serialize
-    @param[in] pretty_print    whether the output shall be pretty-printed
-    @param[in] indent_step     the indent level
-    @param[in] current_indent  the current indent level (only used internally)
-    */
-    void dump(const BasicJsonType& val, const bool pretty_print,
-              const bool ensure_ascii,
-              const unsigned int indent_step,
-              const unsigned int current_indent = 0)
-    {
-        switch (val.m_type)
-        {
-            case value_t::object:
-            {
-                if (val.m_value.object->empty())
-                {
-                    o->write_characters("{}", 2);
-                    return;
-                }
-
-                if (pretty_print)
-                {
-                    o->write_characters("{\n", 2);
-
-                    // variable to hold indentation for recursive calls
-                    const auto new_indent = current_indent + indent_step;
-                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
-                    {
-                        indent_string.resize(indent_string.size() * 2, ' ');
-                    }
-
-                    // first n-1 elements
-                    auto i = val.m_value.object->cbegin();
-                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
-                    {
-                        o->write_characters(indent_string.c_str(), new_indent);
-                        o->write_character('\"');
-                        dump_escaped(i->first, ensure_ascii);
-                        o->write_characters("\": ", 3);
-                        dump(i->second, true, ensure_ascii, indent_step, new_indent);
-                        o->write_characters(",\n", 2);
-                    }
-
-                    // last element
-                    assert(i != val.m_value.object->cend());
-                    assert(std::next(i) == val.m_value.object->cend());
-                    o->write_characters(indent_string.c_str(), new_indent);
-                    o->write_character('\"');
-                    dump_escaped(i->first, ensure_ascii);
-                    o->write_characters("\": ", 3);
-                    dump(i->second, true, ensure_ascii, indent_step, new_indent);
-
-                    o->write_character('\n');
-                    o->write_characters(indent_string.c_str(), current_indent);
-                    o->write_character('}');
-                }
-                else
-                {
-                    o->write_character('{');
-
-                    // first n-1 elements
-                    auto i = val.m_value.object->cbegin();
-                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
-                    {
-                        o->write_character('\"');
-                        dump_escaped(i->first, ensure_ascii);
-                        o->write_characters("\":", 2);
-                        dump(i->second, false, ensure_ascii, indent_step, current_indent);
-                        o->write_character(',');
-                    }
-
-                    // last element
-                    assert(i != val.m_value.object->cend());
-                    assert(std::next(i) == val.m_value.object->cend());
-                    o->write_character('\"');
-                    dump_escaped(i->first, ensure_ascii);
-                    o->write_characters("\":", 2);
-                    dump(i->second, false, ensure_ascii, indent_step, current_indent);
-
-                    o->write_character('}');
-                }
-
-                return;
-            }
-
-            case value_t::array:
-            {
-                if (val.m_value.array->empty())
-                {
-                    o->write_characters("[]", 2);
-                    return;
-                }
-
-                if (pretty_print)
-                {
-                    o->write_characters("[\n", 2);
-
-                    // variable to hold indentation for recursive calls
-                    const auto new_indent = current_indent + indent_step;
-                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
-                    {
-                        indent_string.resize(indent_string.size() * 2, ' ');
-                    }
-
-                    // first n-1 elements
-                    for (auto i = val.m_value.array->cbegin();
-                            i != val.m_value.array->cend() - 1; ++i)
-                    {
-                        o->write_characters(indent_string.c_str(), new_indent);
-                        dump(*i, true, ensure_ascii, indent_step, new_indent);
-                        o->write_characters(",\n", 2);
-                    }
-
-                    // last element
-                    assert(not val.m_value.array->empty());
-                    o->write_characters(indent_string.c_str(), new_indent);
-                    dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
-
-                    o->write_character('\n');
-                    o->write_characters(indent_string.c_str(), current_indent);
-                    o->write_character(']');
-                }
-                else
-                {
-                    o->write_character('[');
-
-                    // first n-1 elements
-                    for (auto i = val.m_value.array->cbegin();
-                            i != val.m_value.array->cend() - 1; ++i)
-                    {
-                        dump(*i, false, ensure_ascii, indent_step, current_indent);
-                        o->write_character(',');
-                    }
-
-                    // last element
-                    assert(not val.m_value.array->empty());
-                    dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
-
-                    o->write_character(']');
-                }
-
-                return;
-            }
-
-            case value_t::string:
-            {
-                o->write_character('\"');
-                dump_escaped(*val.m_value.string, ensure_ascii);
-                o->write_character('\"');
-                return;
-            }
-
-            case value_t::boolean:
-            {
-                if (val.m_value.boolean)
-                {
-                    o->write_characters("true", 4);
-                }
-                else
-                {
-                    o->write_characters("false", 5);
-                }
-                return;
-            }
-
-            case value_t::number_integer:
-            {
-                dump_integer(val.m_value.number_integer);
-                return;
-            }
-
-            case value_t::number_unsigned:
-            {
-                dump_integer(val.m_value.number_unsigned);
-                return;
-            }
-
-            case value_t::number_float:
-            {
-                dump_float(val.m_value.number_float);
-                return;
-            }
-
-            case value_t::discarded:
-            {
-                o->write_characters("<discarded>", 11);
-                return;
-            }
-
-            case value_t::null:
-            {
-                o->write_characters("null", 4);
-                return;
-            }
-        }
-    }
-
-  private:
-    /*!
-    @brief dump escaped string
-
-    Escape a string by replacing certain special characters by a sequence of an
-    escape character (backslash) and another character and other control
-    characters by a sequence of "\u" followed by a four-digit hex
-    representation. The escaped string is written to output stream @a o.
-
-    @param[in] s  the string to escape
-    @param[in] ensure_ascii  whether to escape non-ASCII characters with
-                             \uXXXX sequences
-
-    @complexity Linear in the length of string @a s.
-    */
-    void dump_escaped(const string_t& s, const bool ensure_ascii)
-    {
-        uint32_t codepoint;
-        uint8_t state = UTF8_ACCEPT;
-        std::size_t bytes = 0;  // number of bytes written to string_buffer
-
-        // number of bytes written at the point of the last valid byte
-        std::size_t bytes_after_last_accept = 0;
-        std::size_t undumped_chars = 0;
-
-        for (std::size_t i = 0; i < s.size(); ++i)
-        {
-            const auto byte = static_cast<uint8_t>(s[i]);
-
-            switch (decode(state, codepoint, byte))
-            {
-                case UTF8_ACCEPT:  // decode found a new code point
-                {
-                    switch (codepoint)
-                    {
-                        case 0x08: // backspace
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'b';
-                            break;
-                        }
-
-                        case 0x09: // horizontal tab
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 't';
-                            break;
-                        }
-
-                        case 0x0A: // newline
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'n';
-                            break;
-                        }
-
-                        case 0x0C: // formfeed
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'f';
-                            break;
-                        }
-
-                        case 0x0D: // carriage return
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'r';
-                            break;
-                        }
-
-                        case 0x22: // quotation mark
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = '\"';
-                            break;
-                        }
-
-                        case 0x5C: // reverse solidus
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = '\\';
-                            break;
-                        }
-
-                        default:
-                        {
-                            // escape control characters (0x00..0x1F) or, if
-                            // ensure_ascii parameter is used, non-ASCII characters
-                            if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F)))
-                            {
-                                if (codepoint <= 0xFFFF)
-                                {
-                                    (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
-                                                    static_cast<uint16_t>(codepoint));
-                                    bytes += 6;
-                                }
-                                else
-                                {
-                                    (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
-                                                    static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)),
-                                                    static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF)));
-                                    bytes += 12;
-                                }
-                            }
-                            else
-                            {
-                                // copy byte to buffer (all previous bytes
-                                // been copied have in default case above)
-                                string_buffer[bytes++] = s[i];
-                            }
-                            break;
-                        }
-                    }
-
-                    // write buffer and reset index; there must be 13 bytes
-                    // left, as this is the maximal number of bytes to be
-                    // written ("\uxxxx\uxxxx\0") for one code point
-                    if (string_buffer.size() - bytes < 13)
-                    {
-                        o->write_characters(string_buffer.data(), bytes);
-                        bytes = 0;
-                    }
-
-                    // remember the byte position of this accept
-                    bytes_after_last_accept = bytes;
-                    undumped_chars = 0;
-                    break;
-                }
-
-                case UTF8_REJECT:  // decode found invalid UTF-8 byte
-                {
-                    switch (error_handler)
-                    {
-                        case error_handler_t::strict:
-                        {
-                            std::string sn(3, '\0');
-                            (std::snprintf)(&sn[0], sn.size(), "%.2X", byte);
-                            JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn));
-                        }
-
-                        case error_handler_t::ignore:
-                        case error_handler_t::replace:
-                        {
-                            // in case we saw this character the first time, we
-                            // would like to read it again, because the byte
-                            // may be OK for itself, but just not OK for the
-                            // previous sequence
-                            if (undumped_chars > 0)
-                            {
-                                --i;
-                            }
-
-                            // reset length buffer to the last accepted index;
-                            // thus removing/ignoring the invalid characters
-                            bytes = bytes_after_last_accept;
-
-                            if (error_handler == error_handler_t::replace)
-                            {
-                                // add a replacement character
-                                if (ensure_ascii)
-                                {
-                                    string_buffer[bytes++] = '\\';
-                                    string_buffer[bytes++] = 'u';
-                                    string_buffer[bytes++] = 'f';
-                                    string_buffer[bytes++] = 'f';
-                                    string_buffer[bytes++] = 'f';
-                                    string_buffer[bytes++] = 'd';
-                                }
-                                else
-                                {
-                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF');
-                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
-                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
-                                }
-                                bytes_after_last_accept = bytes;
-                            }
-
-                            undumped_chars = 0;
-
-                            // continue processing the string
-                            state = UTF8_ACCEPT;
-                            break;
-                        }
-                    }
-                    break;
-                }
-
-                default:  // decode found yet incomplete multi-byte code point
-                {
-                    if (not ensure_ascii)
-                    {
-                        // code point will not be escaped - copy byte to buffer
-                        string_buffer[bytes++] = s[i];
-                    }
-                    ++undumped_chars;
-                    break;
-                }
-            }
-        }
-
-        // we finished processing the string
-        if (JSON_LIKELY(state == UTF8_ACCEPT))
-        {
-            // write buffer
-            if (bytes > 0)
-            {
-                o->write_characters(string_buffer.data(), bytes);
-            }
-        }
-        else
-        {
-            // we finish reading, but do not accept: string was incomplete
-            switch (error_handler)
-            {
-                case error_handler_t::strict:
-                {
-                    std::string sn(3, '\0');
-                    (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back()));
-                    JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn));
-                }
-
-                case error_handler_t::ignore:
-                {
-                    // write all accepted bytes
-                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
-                    break;
-                }
-
-                case error_handler_t::replace:
-                {
-                    // write all accepted bytes
-                    o->write_characters(string_buffer.data(), bytes_after_last_accept);
-                    // add a replacement character
-                    if (ensure_ascii)
-                    {
-                        o->write_characters("\\ufffd", 6);
-                    }
-                    else
-                    {
-                        o->write_characters("\xEF\xBF\xBD", 3);
-                    }
-                    break;
-                }
-            }
-        }
-    }
-
-    /*!
-    @brief dump an integer
-
-    Dump a given integer to output stream @a o. Works internally with
-    @a number_buffer.
-
-    @param[in] x  integer number (signed or unsigned) to dump
-    @tparam NumberType either @a number_integer_t or @a number_unsigned_t
-    */
-    template<typename NumberType, detail::enable_if_t<
-                 std::is_same<NumberType, number_unsigned_t>::value or
-                 std::is_same<NumberType, number_integer_t>::value,
-                 int> = 0>
-    void dump_integer(NumberType x)
-    {
-        // special case for "0"
-        if (x == 0)
-        {
-            o->write_character('0');
-            return;
-        }
-
-        const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0);  // see issue #755
-        std::size_t i = 0;
-
-        while (x != 0)
-        {
-            // spare 1 byte for '\0'
-            assert(i < number_buffer.size() - 1);
-
-            const auto digit = std::labs(static_cast<long>(x % 10));
-            number_buffer[i++] = static_cast<char>('0' + digit);
-            x /= 10;
-        }
-
-        if (is_negative)
-        {
-            // make sure there is capacity for the '-'
-            assert(i < number_buffer.size() - 2);
-            number_buffer[i++] = '-';
-        }
-
-        std::reverse(number_buffer.begin(), number_buffer.begin() + i);
-        o->write_characters(number_buffer.data(), i);
-    }
-
-    /*!
-    @brief dump a floating-point number
-
-    Dump a given floating-point number to output stream @a o. Works internally
-    with @a number_buffer.
-
-    @param[in] x  floating-point number to dump
-    */
-    void dump_float(number_float_t x)
-    {
-        // NaN / inf
-        if (not std::isfinite(x))
-        {
-            o->write_characters("null", 4);
-            return;
-        }
-
-        // If number_float_t is an IEEE-754 single or double precision number,
-        // use the Grisu2 algorithm to produce short numbers which are
-        // guaranteed to round-trip, using strtof and strtod, resp.
-        //
-        // NB: The test below works if <long double> == <double>.
-        static constexpr bool is_ieee_single_or_double
-            = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or
-              (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024);
-
-        dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());
-    }
-
-    void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)
-    {
-        char* begin = number_buffer.data();
-        char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);
-
-        o->write_characters(begin, static_cast<size_t>(end - begin));
-    }
-
-    void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)
-    {
-        // get number of digits for a float -> text -> float round-trip
-        static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;
-
-        // the actual conversion
-        std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
-
-        // negative value indicates an error
-        assert(len > 0);
-        // check if buffer was large enough
-        assert(static_cast<std::size_t>(len) < number_buffer.size());
-
-        // erase thousands separator
-        if (thousands_sep != '\0')
-        {
-            const auto end = std::remove(number_buffer.begin(),
-                                         number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            assert((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
-
-        // convert decimal point to '.'
-        if (decimal_point != '\0' and decimal_point != '.')
-        {
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
-            if (dec_pos != number_buffer.end())
-            {
-                *dec_pos = '.';
-            }
-        }
-
-        o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));
-
-        // determine if need to append ".0"
-        const bool value_is_int_like =
-            std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,
-                         [](char c)
-        {
-            return (c == '.' or c == 'e');
-        });
-
-        if (value_is_int_like)
-        {
-            o->write_characters(".0", 2);
-        }
-    }
-
-    /*!
-    @brief check whether a string is UTF-8 encoded
-
-    The function checks each byte of a string whether it is UTF-8 encoded. The
-    result of the check is stored in the @a state parameter. The function must
-    be called initially with state 0 (accept). State 1 means the string must
-    be rejected, because the current byte is not allowed. If the string is
-    completely processed, but the state is non-zero, the string ended
-    prematurely; that is, the last byte indicated more bytes should have
-    followed.
-
-    @param[in,out] state  the state of the decoding
-    @param[in,out] codep  codepoint (valid only if resulting state is UTF8_ACCEPT)
-    @param[in] byte       next byte to decode
-    @return               new state
-
-    @note The function has been edited: a std::array is used.
-
-    @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
-    @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
-    */
-    static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept
-    {
-        static const std::array<uint8_t, 400> utf8d =
-        {
-            {
-                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
-                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
-                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
-                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
-                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
-                7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
-                8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
-                0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
-                0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
-                0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
-                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
-                1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
-                1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
-                1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
-            }
-        };
-
-        const uint8_t type = utf8d[byte];
-
-        codep = (state != UTF8_ACCEPT)
-                ? (byte & 0x3fu) | (codep << 6)
-                : static_cast<uint32_t>(0xff >> type) & (byte);
-
-        state = utf8d[256u + state * 16u + type];
-        return state;
-    }
-
-  private:
-    /// the output of the serializer
-    output_adapter_t<char> o = nullptr;
-
-    /// a (hopefully) large enough character buffer
-    std::array<char, 64> number_buffer{{}};
-
-    /// the locale
-    const std::lconv* loc = nullptr;
-    /// the locale's thousand separator character
-    const char thousands_sep = '\0';
-    /// the locale's decimal point character
-    const char decimal_point = '\0';
-
-    /// string buffer
-    std::array<char, 512> string_buffer{{}};
-
-    /// the indentation character
-    const char indent_char;
-    /// the indentation string
-    string_t indent_string;
-
-    /// error_handler how to react on decoding errors
-    const error_handler_t error_handler;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/json_ref.hpp>
-
-
-#include <initializer_list>
-#include <utility>
-
-// #include <nlohmann/detail/meta/type_traits.hpp>
-
-
-namespace nlohmann
-{
-namespace detail
-{
-template<typename BasicJsonType>
-class json_ref
-{
-  public:
-    using value_type = BasicJsonType;
-
-    json_ref(value_type&& value)
-        : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true)
-    {}
-
-    json_ref(const value_type& value)
-        : value_ref(const_cast<value_type*>(&value)), is_rvalue(false)
-    {}
-
-    json_ref(std::initializer_list<json_ref> init)
-        : owned_value(init), value_ref(&owned_value), is_rvalue(true)
-    {}
-
-    template <
-        class... Args,
-        enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >
-    json_ref(Args && ... args)
-        : owned_value(std::forward<Args>(args)...), value_ref(&owned_value),
-          is_rvalue(true) {}
-
-    // class should be movable only
-    json_ref(json_ref&&) = default;
-    json_ref(const json_ref&) = delete;
-    json_ref& operator=(const json_ref&) = delete;
-    json_ref& operator=(json_ref&&) = delete;
-    ~json_ref() = default;
-
-    value_type moved_or_copied() const
-    {
-        if (is_rvalue)
-        {
-            return std::move(*value_ref);
-        }
-        return *value_ref;
-    }
-
-    value_type const& operator*() const
-    {
-        return *static_cast<value_type const*>(value_ref);
-    }
-
-    value_type const* operator->() const
-    {
-        return static_cast<value_type const*>(value_ref);
-    }
-
-  private:
-    mutable value_type owned_value = nullptr;
-    value_type* value_ref = nullptr;
-    const bool is_rvalue;
-};
-}  // namespace detail
-}  // namespace nlohmann
-
-// #include <nlohmann/detail/json_pointer.hpp>
-
-
-#include <cassert> // assert
-#include <numeric> // accumulate
-#include <string> // string
-#include <vector> // vector
-
-// #include <nlohmann/detail/macro_scope.hpp>
-
-// #include <nlohmann/detail/exceptions.hpp>
-
-// #include <nlohmann/detail/value_t.hpp>
-
-
-namespace nlohmann
-{
-template<typename BasicJsonType>
-class json_pointer
-{
-    // allow basic_json to access private members
-    NLOHMANN_BASIC_JSON_TPL_DECLARATION
-    friend class basic_json;
-
-  public:
-    /*!
-    @brief create JSON pointer
-
-    Create a JSON pointer according to the syntax described in
-    [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3).
-
-    @param[in] s  string representing the JSON pointer; if omitted, the empty
-                  string is assumed which references the whole JSON value
-
-    @throw parse_error.107 if the given JSON pointer @a s is nonempty and does
-                           not begin with a slash (`/`); see example below
-
-    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is
-    not followed by `0` (representing `~`) or `1` (representing `/`); see
-    example below
-
-    @liveexample{The example shows the construction several valid JSON pointers
-    as well as the exceptional behavior.,json_pointer}
-
-    @since version 2.0.0
-    */
-    explicit json_pointer(const std::string& s = "")
-        : reference_tokens(split(s))
-    {}
-
-    /*!
-    @brief return a string representation of the JSON pointer
-
-    @invariant For each JSON pointer `ptr`, it holds:
-    @code {.cpp}
-    ptr == json_pointer(ptr.to_string());
-    @endcode
-
-    @return a string representation of the JSON pointer
-
-    @liveexample{The example shows the result of `to_string`.,
-    json_pointer__to_string}
-
-    @since version 2.0.0
-    */
-    std::string to_string() const
-    {
-        return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
-                               std::string{},
-                               [](const std::string & a, const std::string & b)
-        {
-            return a + "/" + escape(b);
-        });
-    }
-
-    /// @copydoc to_string()
-    operator std::string() const
-    {
-        return to_string();
-    }
-
-    /*!
-    @param[in] s  reference token to be converted into an array index
-
-    @return integer representation of @a s
-
-    @throw out_of_range.404 if string @a s could not be converted to an integer
-    */
-    static int array_index(const std::string& s)
-    {
-        std::size_t processed_chars = 0;
-        const int res = std::stoi(s, &processed_chars);
-
-        // check if the string was completely read
-        if (JSON_UNLIKELY(processed_chars != s.size()))
-        {
-            JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
-        }
-
-        return res;
-    }
-
-  private:
-    /*!
-    @brief remove and return last reference pointer
-    @throw out_of_range.405 if JSON pointer has no parent
-    */
-    std::string pop_back()
-    {
-        if (JSON_UNLIKELY(is_root()))
-        {
-            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
-        }
-
-        auto last = reference_tokens.back();
-        reference_tokens.pop_back();
-        return last;
-    }
-
-    /// return whether pointer points to the root document
-    bool is_root() const noexcept
-    {
-        return reference_tokens.empty();
-    }
-
-    json_pointer top() const
-    {
-        if (JSON_UNLIKELY(is_root()))
-        {
-            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
-        }
-
-        json_pointer result = *this;
-        result.reference_tokens = {reference_tokens[0]};
-        return result;
-    }
-
-    /*!
-    @brief create and return a reference to the pointed to value
-
-    @complexity Linear in the number of reference tokens.
-
-    @throw parse_error.109 if array index is not a number
-    @throw type_error.313 if value cannot be unflattened
-    */
-    BasicJsonType& get_and_create(BasicJsonType& j) const
-    {
-        using size_type = typename BasicJsonType::size_type;
-        auto result = &j;
-
-        // in case no reference tokens exist, return a reference to the JSON value
-        // j which will be overwritten by a primitive value
-        for (const auto& reference_token : reference_tokens)
-        {
-            switch (result->m_type)
-            {
-                case detail::value_t::null:
-                {
-                    if (reference_token == "0")
-                    {
-                        // start a new array if reference token is 0
-                        result = &result->operator[](0);
-                    }
-                    else
-                    {
-                        // start a new object otherwise
-                        result = &result->operator[](reference_token);
-                    }
-                    break;
-                }
-
-                case detail::value_t::object:
-                {
-                    // create an entry in the object
-                    result = &result->operator[](reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    // create an entry in the array
-                    JSON_TRY
-                    {
-                        result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                    break;
-                }
-
-                /*
-                The following code is only reached if there exists a reference
-                token _and_ the current value is primitive. In this case, we have
-                an error situation, because primitive values may only occur as
-                single value; that is, with an empty list of reference tokens.
-                */
-                default:
-                    JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
-            }
-        }
-
-        return *result;
-    }
-
-    /*!
-    @brief return a reference to the pointed to value
-
-    @note This version does not throw if a value is not present, but tries to
-          create nested values instead. For instance, calling this function
-          with pointer `"/this/that"` on a null value is equivalent to calling
-          `operator[]("this").operator[]("that")` on that value, effectively
-          changing the null value to an object.
-
-    @param[in] ptr  a JSON value
-
-    @return reference to the JSON value pointed to by the JSON pointer
-
-    @complexity Linear in the length of the JSON pointer.
-
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-    */
-    BasicJsonType& get_unchecked(BasicJsonType* ptr) const
-    {
-        using size_type = typename BasicJsonType::size_type;
-        for (const auto& reference_token : reference_tokens)
-        {
-            // convert null values to arrays or objects before continuing
-            if (ptr->m_type == detail::value_t::null)
-            {
-                // check if reference token is a number
-                const bool nums =
-                    std::all_of(reference_token.begin(), reference_token.end(),
-                                [](const char x)
-                {
-                    return (x >= '0' and x <= '9');
-                });
-
-                // change value to array for numbers or "-" or to object otherwise
-                *ptr = (nums or reference_token == "-")
-                       ? detail::value_t::array
-                       : detail::value_t::object;
-            }
-
-            switch (ptr->m_type)
-            {
-                case detail::value_t::object:
-                {
-                    // use unchecked object access
-                    ptr = &ptr->operator[](reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    // error condition (cf. RFC 6901, Sect. 4)
-                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                    {
-                        JSON_THROW(detail::parse_error::create(106, 0,
-                                                               "array index '" + reference_token +
-                                                               "' must not begin with '0'"));
-                    }
-
-                    if (reference_token == "-")
-                    {
-                        // explicitly treat "-" as index beyond the end
-                        ptr = &ptr->operator[](ptr->m_value.array->size());
-                    }
-                    else
-                    {
-                        // convert array index to number; unchecked access
-                        JSON_TRY
-                        {
-                            ptr = &ptr->operator[](
-                                static_cast<size_type>(array_index(reference_token)));
-                        }
-                        JSON_CATCH(std::invalid_argument&)
-                        {
-                            JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                        }
-                    }
-                    break;
-                }
-
-                default:
-                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-            }
-        }
-
-        return *ptr;
-    }
-
-    /*!
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.402  if the array index '-' is used
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-    */
-    BasicJsonType& get_checked(BasicJsonType* ptr) const
-    {
-        using size_type = typename BasicJsonType::size_type;
-        for (const auto& reference_token : reference_tokens)
-        {
-            switch (ptr->m_type)
-            {
-                case detail::value_t::object:
-                {
-                    // note: at performs range check
-                    ptr = &ptr->at(reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    if (JSON_UNLIKELY(reference_token == "-"))
-                    {
-                        // "-" always fails the range check
-                        JSON_THROW(detail::out_of_range::create(402,
-                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                                ") is out of range"));
-                    }
-
-                    // error condition (cf. RFC 6901, Sect. 4)
-                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                    {
-                        JSON_THROW(detail::parse_error::create(106, 0,
-                                                               "array index '" + reference_token +
-                                                               "' must not begin with '0'"));
-                    }
-
-                    // note: at performs range check
-                    JSON_TRY
-                    {
-                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                    break;
-                }
-
-                default:
-                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-            }
-        }
-
-        return *ptr;
-    }
-
-    /*!
-    @brief return a const reference to the pointed to value
-
-    @param[in] ptr  a JSON value
-
-    @return const reference to the JSON value pointed to by the JSON
-    pointer
-
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.402  if the array index '-' is used
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-    */
-    const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const
-    {
-        using size_type = typename BasicJsonType::size_type;
-        for (const auto& reference_token : reference_tokens)
-        {
-            switch (ptr->m_type)
-            {
-                case detail::value_t::object:
-                {
-                    // use unchecked object access
-                    ptr = &ptr->operator[](reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    if (JSON_UNLIKELY(reference_token == "-"))
-                    {
-                        // "-" cannot be used for const access
-                        JSON_THROW(detail::out_of_range::create(402,
-                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                                ") is out of range"));
-                    }
-
-                    // error condition (cf. RFC 6901, Sect. 4)
-                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                    {
-                        JSON_THROW(detail::parse_error::create(106, 0,
-                                                               "array index '" + reference_token +
-                                                               "' must not begin with '0'"));
-                    }
-
-                    // use unchecked array access
-                    JSON_TRY
-                    {
-                        ptr = &ptr->operator[](
-                            static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                    break;
-                }
-
-                default:
-                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-            }
-        }
-
-        return *ptr;
-    }
-
-    /*!
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.402  if the array index '-' is used
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-    */
-    const BasicJsonType& get_checked(const BasicJsonType* ptr) const
-    {
-        using size_type = typename BasicJsonType::size_type;
-        for (const auto& reference_token : reference_tokens)
-        {
-            switch (ptr->m_type)
-            {
-                case detail::value_t::object:
-                {
-                    // note: at performs range check
-                    ptr = &ptr->at(reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    if (JSON_UNLIKELY(reference_token == "-"))
-                    {
-                        // "-" always fails the range check
-                        JSON_THROW(detail::out_of_range::create(402,
-                                                                "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
-                                                                ") is out of range"));
-                    }
-
-                    // error condition (cf. RFC 6901, Sect. 4)
-                    if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
-                    {
-                        JSON_THROW(detail::parse_error::create(106, 0,
-                                                               "array index '" + reference_token +
-                                                               "' must not begin with '0'"));
-                    }
-
-                    // note: at performs range check
-                    JSON_TRY
-                    {
-                        ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
-                    }
-                    JSON_CATCH(std::invalid_argument&)
-                    {
-                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                    }
-                    break;
-                }
-
-                default:
-                    JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-            }
-        }
-
-        return *ptr;
-    }
-
-    /*!
-    @brief split the string input to reference tokens
-
-    @note This function is only called by the json_pointer constructor.
-          All exceptions below are documented there.
-
-    @throw parse_error.107  if the pointer is not empty or begins with '/'
-    @throw parse_error.108  if character '~' is not followed by '0' or '1'
-    */
-    static std::vector<std::string> split(const std::string& reference_string)
-    {
-        std::vector<std::string> result;
-
-        // special case: empty reference string -> no reference tokens
-        if (reference_string.empty())
-        {
-            return result;
-        }
-
-        // check if nonempty reference string begins with slash
-        if (JSON_UNLIKELY(reference_string[0] != '/'))
-        {
-            JSON_THROW(detail::parse_error::create(107, 1,
-                                                   "JSON pointer must be empty or begin with '/' - was: '" +
-                                                   reference_string + "'"));
-        }
-
-        // extract the reference tokens:
-        // - slash: position of the last read slash (or end of string)
-        // - start: position after the previous slash
-        for (
-            // search for the first slash after the first character
-            std::size_t slash = reference_string.find_first_of('/', 1),
-            // set the beginning of the first reference token
-            start = 1;
-            // we can stop if start == 0 (if slash == std::string::npos)
-            start != 0;
-            // set the beginning of the next reference token
-            // (will eventually be 0 if slash == std::string::npos)
-            start = (slash == std::string::npos) ? 0 : slash + 1,
-            // find next slash
-            slash = reference_string.find_first_of('/', start))
-        {
-            // use the text between the beginning of the reference token
-            // (start) and the last slash (slash).
-            auto reference_token = reference_string.substr(start, slash - start);
-
-            // check reference tokens are properly escaped
-            for (std::size_t pos = reference_token.find_first_of('~');
-                    pos != std::string::npos;
-                    pos = reference_token.find_first_of('~', pos + 1))
-            {
-                assert(reference_token[pos] == '~');
-
-                // ~ must be followed by 0 or 1
-                if (JSON_UNLIKELY(pos == reference_token.size() - 1 or
-                                  (reference_token[pos + 1] != '0' and
-                                   reference_token[pos + 1] != '1')))
-                {
-                    JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
-                }
-            }
-
-            // finally, store the reference token
-            unescape(reference_token);
-            result.push_back(reference_token);
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief replace all occurrences of a substring by another string
-
-    @param[in,out] s  the string to manipulate; changed so that all
-                   occurrences of @a f are replaced with @a t
-    @param[in]     f  the substring to replace with @a t
-    @param[in]     t  the string to replace @a f
-
-    @pre The search string @a f must not be empty. **This precondition is
-    enforced with an assertion.**
-
-    @since version 2.0.0
-    */
-    static void replace_substring(std::string& s, const std::string& f,
-                                  const std::string& t)
-    {
-        assert(not f.empty());
-        for (auto pos = s.find(f);                // find first occurrence of f
-                pos != std::string::npos;         // make sure f was found
-                s.replace(pos, f.size(), t),      // replace with t, and
-                pos = s.find(f, pos + t.size()))  // find next occurrence of f
-        {}
-    }
-
-    /// escape "~" to "~0" and "/" to "~1"
-    static std::string escape(std::string s)
-    {
-        replace_substring(s, "~", "~0");
-        replace_substring(s, "/", "~1");
-        return s;
-    }
-
-    /// unescape "~1" to tilde and "~0" to slash (order is important!)
-    static void unescape(std::string& s)
-    {
-        replace_substring(s, "~1", "/");
-        replace_substring(s, "~0", "~");
-    }
-
-    /*!
-    @param[in] reference_string  the reference string to the current value
-    @param[in] value             the value to consider
-    @param[in,out] result        the result object to insert values to
-
-    @note Empty objects or arrays are flattened to `null`.
-    */
-    static void flatten(const std::string& reference_string,
-                        const BasicJsonType& value,
-                        BasicJsonType& result)
-    {
-        switch (value.m_type)
-        {
-            case detail::value_t::array:
-            {
-                if (value.m_value.array->empty())
-                {
-                    // flatten empty array as null
-                    result[reference_string] = nullptr;
-                }
-                else
-                {
-                    // iterate array and use index as reference string
-                    for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
-                    {
-                        flatten(reference_string + "/" + std::to_string(i),
-                                value.m_value.array->operator[](i), result);
-                    }
-                }
-                break;
-            }
-
-            case detail::value_t::object:
-            {
-                if (value.m_value.object->empty())
-                {
-                    // flatten empty object as null
-                    result[reference_string] = nullptr;
-                }
-                else
-                {
-                    // iterate object and use keys as reference string
-                    for (const auto& element : *value.m_value.object)
-                    {
-                        flatten(reference_string + "/" + escape(element.first), element.second, result);
-                    }
-                }
-                break;
-            }
-
-            default:
-            {
-                // add primitive value with its reference string
-                result[reference_string] = value;
-                break;
-            }
-        }
-    }
-
-    /*!
-    @param[in] value  flattened JSON
-
-    @return unflattened JSON
-
-    @throw parse_error.109 if array index is not a number
-    @throw type_error.314  if value is not an object
-    @throw type_error.315  if object values are not primitive
-    @throw type_error.313  if value cannot be unflattened
-    */
-    static BasicJsonType
-    unflatten(const BasicJsonType& value)
-    {
-        if (JSON_UNLIKELY(not value.is_object()))
-        {
-            JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
-        }
-
-        BasicJsonType result;
-
-        // iterate the JSON object values
-        for (const auto& element : *value.m_value.object)
-        {
-            if (JSON_UNLIKELY(not element.second.is_primitive()))
-            {
-                JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
-            }
-
-            // assign value to reference pointed to by JSON pointer; Note that if
-            // the JSON pointer is "" (i.e., points to the whole value), function
-            // get_and_create returns a reference to result itself. An assignment
-            // will then create a primitive value.
-            json_pointer(element.first).get_and_create(result) = element.second;
-        }
-
-        return result;
-    }
-
-    friend bool operator==(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept
-    {
-        return (lhs.reference_tokens == rhs.reference_tokens);
-    }
-
-    friend bool operator!=(json_pointer const& lhs,
-                           json_pointer const& rhs) noexcept
-    {
-        return not (lhs == rhs);
-    }
-
-    /// the reference tokens
-    std::vector<std::string> reference_tokens;
-};
-}  // namespace nlohmann
-
-// #include <nlohmann/adl_serializer.hpp>
-
-
-#include <utility>
-
-// #include <nlohmann/detail/conversions/from_json.hpp>
-
-// #include <nlohmann/detail/conversions/to_json.hpp>
-
-
-namespace nlohmann
-{
-
-template<typename, typename>
-struct adl_serializer
-{
-    /*!
-    @brief convert a JSON value to any value type
-
-    This function is usually called by the `get()` function of the
-    @ref basic_json class (either explicit or via conversion operators).
-
-    @param[in] j        JSON value to read from
-    @param[in,out] val  value to write to
-    */
-    template<typename BasicJsonType, typename ValueType>
-    static auto from_json(BasicJsonType&& j, ValueType& val) noexcept(
-        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
-    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())
-    {
-        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
-    }
-
-    /*!
-    @brief convert any value type to a JSON value
-
-    This function is usually called by the constructors of the @ref basic_json
-    class.
-
-    @param[in,out] j  JSON value to write to
-    @param[in] val    value to read from
-    */
-    template <typename BasicJsonType, typename ValueType>
-    static auto to_json(BasicJsonType& j, ValueType&& val) noexcept(
-        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
-    -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void())
-    {
-        ::nlohmann::to_json(j, std::forward<ValueType>(val));
-    }
-};
-
-}  // namespace nlohmann
-
-
-/*!
-@brief namespace for Niels Lohmann
-@see https://github.com/nlohmann
-@since version 1.0.0
-*/
-namespace nlohmann
-{
-
-/*!
-@brief a class to store JSON values
-
-@tparam ObjectType type for JSON objects (`std::map` by default; will be used
-in @ref object_t)
-@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used
-in @ref array_t)
-@tparam StringType type for JSON strings and object keys (`std::string` by
-default; will be used in @ref string_t)
-@tparam BooleanType type for JSON booleans (`bool` by default; will be used
-in @ref boolean_t)
-@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by
-default; will be used in @ref number_integer_t)
-@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c
-`uint64_t` by default; will be used in @ref number_unsigned_t)
-@tparam NumberFloatType type for JSON floating-point numbers (`double` by
-default; will be used in @ref number_float_t)
-@tparam AllocatorType type of the allocator to use (`std::allocator` by
-default)
-@tparam JSONSerializer the serializer to resolve internal calls to `to_json()`
-and `from_json()` (@ref adl_serializer by default)
-
-@requirement The class satisfies the following concept requirements:
-- Basic
- - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible):
-   JSON values can be default constructed. The result will be a JSON null
-   value.
- - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible):
-   A JSON value can be constructed from an rvalue argument.
- - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible):
-   A JSON value can be copy-constructed from an lvalue expression.
- - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable):
-   A JSON value van be assigned from an rvalue argument.
- - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable):
-   A JSON value can be copy-assigned from an lvalue expression.
- - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible):
-   JSON values can be destructed.
-- Layout
- - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType):
-   JSON values have
-   [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
-   All non-static data members are private and standard layout types, the
-   class has no virtual functions or (virtual) base classes.
-- Library-wide
- - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable):
-   JSON values can be compared with `==`, see @ref
-   operator==(const_reference,const_reference).
- - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable):
-   JSON values can be compared with `<`, see @ref
-   operator<(const_reference,const_reference).
- - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable):
-   Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of
-   other compatible types, using unqualified function call @ref swap().
- - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer):
-   JSON values can be compared against `std::nullptr_t` objects which are used
-   to model the `null` value.
-- Container
- - [Container](https://en.cppreference.com/w/cpp/named_req/Container):
-   JSON values can be used like STL containers and provide iterator access.
- - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer);
-   JSON values can be used like STL containers and provide reverse iterator
-   access.
-
-@invariant The member variables @a m_value and @a m_type have the following
-relationship:
-- If `m_type == value_t::object`, then `m_value.object != nullptr`.
-- If `m_type == value_t::array`, then `m_value.array != nullptr`.
-- If `m_type == value_t::string`, then `m_value.string != nullptr`.
-The invariants are checked by member function assert_invariant().
-
-@internal
-@note ObjectType trick from http://stackoverflow.com/a/9860911
-@endinternal
-
-@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange
-Format](http://rfc7159.net/rfc7159)
-
-@since version 1.0.0
-
-@nosubgrouping
-*/
-NLOHMANN_BASIC_JSON_TPL_DECLARATION
-class basic_json
-{
-  private:
-    template<detail::value_t> friend struct detail::external_constructor;
-    friend ::nlohmann::json_pointer<basic_json>;
-    friend ::nlohmann::detail::parser<basic_json>;
-    friend ::nlohmann::detail::serializer<basic_json>;
-    template<typename BasicJsonType>
-    friend class ::nlohmann::detail::iter_impl;
-    template<typename BasicJsonType, typename CharType>
-    friend class ::nlohmann::detail::binary_writer;
-    template<typename BasicJsonType, typename SAX>
-    friend class ::nlohmann::detail::binary_reader;
-    template<typename BasicJsonType>
-    friend class ::nlohmann::detail::json_sax_dom_parser;
-    template<typename BasicJsonType>
-    friend class ::nlohmann::detail::json_sax_dom_callback_parser;
-
-    /// workaround type for MSVC
-    using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
-
-    // convenience aliases for types residing in namespace detail;
-    using lexer = ::nlohmann::detail::lexer<basic_json>;
-    using parser = ::nlohmann::detail::parser<basic_json>;
-
-    using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;
-    template<typename BasicJsonType>
-    using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;
-    template<typename BasicJsonType>
-    using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;
-    template<typename Iterator>
-    using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;
-    template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;
-
-    template<typename CharType>
-    using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;
-
-    using binary_reader = ::nlohmann::detail::binary_reader<basic_json>;
-    template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;
-
-    using serializer = ::nlohmann::detail::serializer<basic_json>;
-
-  public:
-    using value_t = detail::value_t;
-    /// JSON Pointer, see @ref nlohmann::json_pointer
-    using json_pointer = ::nlohmann::json_pointer<basic_json>;
-    template<typename T, typename SFINAE>
-    using json_serializer = JSONSerializer<T, SFINAE>;
-    /// how to treat decoding errors
-    using error_handler_t = detail::error_handler_t;
-    /// helper type for initializer lists of basic_json values
-    using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
-
-    using input_format_t = detail::input_format_t;
-    /// SAX interface type, see @ref nlohmann::json_sax
-    using json_sax_t = json_sax<basic_json>;
-
-    ////////////////
-    // exceptions //
-    ////////////////
-
-    /// @name exceptions
-    /// Classes to implement user-defined exceptions.
-    /// @{
-
-    /// @copydoc detail::exception
-    using exception = detail::exception;
-    /// @copydoc detail::parse_error
-    using parse_error = detail::parse_error;
-    /// @copydoc detail::invalid_iterator
-    using invalid_iterator = detail::invalid_iterator;
-    /// @copydoc detail::type_error
-    using type_error = detail::type_error;
-    /// @copydoc detail::out_of_range
-    using out_of_range = detail::out_of_range;
-    /// @copydoc detail::other_error
-    using other_error = detail::other_error;
-
-    /// @}
-
-
-    /////////////////////
-    // container types //
-    /////////////////////
-
-    /// @name container types
-    /// The canonic container types to use @ref basic_json like any other STL
-    /// container.
-    /// @{
-
-    /// the type of elements in a basic_json container
-    using value_type = basic_json;
-
-    /// the type of an element reference
-    using reference = value_type&;
-    /// the type of an element const reference
-    using const_reference = const value_type&;
-
-    /// a type to represent differences between iterators
-    using difference_type = std::ptrdiff_t;
-    /// a type to represent container sizes
-    using size_type = std::size_t;
-
-    /// the allocator type
-    using allocator_type = AllocatorType<basic_json>;
-
-    /// the type of an element pointer
-    using pointer = typename std::allocator_traits<allocator_type>::pointer;
-    /// the type of an element const pointer
-    using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
-
-    /// an iterator for a basic_json container
-    using iterator = iter_impl<basic_json>;
-    /// a const iterator for a basic_json container
-    using const_iterator = iter_impl<const basic_json>;
-    /// a reverse iterator for a basic_json container
-    using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>;
-    /// a const reverse iterator for a basic_json container
-    using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>;
-
-    /// @}
-
-
-    /*!
-    @brief returns the allocator associated with the container
-    */
-    static allocator_type get_allocator()
-    {
-        return allocator_type();
-    }
-
-    /*!
-    @brief returns version information on the library
-
-    This function returns a JSON object with information about the library,
-    including the version number and information on the platform and compiler.
-
-    @return JSON object holding version information
-    key         | description
-    ----------- | ---------------
-    `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version).
-    `copyright` | The copyright line for the library as string.
-    `name`      | The name of the library as string.
-    `platform`  | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`.
-    `url`       | The URL of the project as string.
-    `version`   | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string).
-
-    @liveexample{The following code shows an example output of the `meta()`
-    function.,meta}
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @complexity Constant.
-
-    @since 2.1.0
-    */
-    static basic_json meta()
-    {
-        basic_json result;
-
-        result["copyright"] = "(C) 2013-2017 Niels Lohmann";
-        result["name"] = "JSON for Modern C++";
-        result["url"] = "https://github.com/nlohmann/json";
-        result["version"]["string"] =
-            std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." +
-            std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." +
-            std::to_string(NLOHMANN_JSON_VERSION_PATCH);
-        result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR;
-        result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR;
-        result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH;
-
-#ifdef _WIN32
-        result["platform"] = "win32";
-#elif defined __linux__
-        result["platform"] = "linux";
-#elif defined __APPLE__
-        result["platform"] = "apple";
-#elif defined __unix__
-        result["platform"] = "unix";
-#else
-        result["platform"] = "unknown";
-#endif
-
-#if defined(__ICC) || defined(__INTEL_COMPILER)
-        result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
-#elif defined(__clang__)
-        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
-#elif defined(__GNUC__) || defined(__GNUG__)
-        result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}};
-#elif defined(__HP_cc) || defined(__HP_aCC)
-        result["compiler"] = "hp"
-#elif defined(__IBMCPP__)
-        result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}};
-#elif defined(_MSC_VER)
-        result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}};
-#elif defined(__PGI)
-        result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}};
-#elif defined(__SUNPRO_CC)
-        result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}};
-#else
-        result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};
-#endif
-
-#ifdef __cplusplus
-        result["compiler"]["c++"] = std::to_string(__cplusplus);
-#else
-        result["compiler"]["c++"] = "unknown";
-#endif
-        return result;
-    }
-
-
-    ///////////////////////////
-    // JSON value data types //
-    ///////////////////////////
-
-    /// @name JSON value data types
-    /// The data types to store a JSON value. These types are derived from
-    /// the template arguments passed to class @ref basic_json.
-    /// @{
-
-#if defined(JSON_HAS_CPP_14)
-    // Use transparent comparator if possible, combined with perfect forwarding
-    // on find() and count() calls prevents unnecessary string construction.
-    using object_comparator_t = std::less<>;
-#else
-    using object_comparator_t = std::less<StringType>;
-#endif
-
-    /*!
-    @brief a type for an object
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows:
-    > An object is an unordered collection of zero or more name/value pairs,
-    > where a name is a string and a value is a string, number, boolean, null,
-    > object, or array.
-
-    To store objects in C++, a type is defined by the template parameters
-    described below.
-
-    @tparam ObjectType  the container to store objects (e.g., `std::map` or
-    `std::unordered_map`)
-    @tparam StringType the type of the keys or names (e.g., `std::string`).
-    The comparison function `std::less<StringType>` is used to order elements
-    inside the container.
-    @tparam AllocatorType the allocator to use for objects (e.g.,
-    `std::allocator`)
-
-    #### Default type
-
-    With the default values for @a ObjectType (`std::map`), @a StringType
-    (`std::string`), and @a AllocatorType (`std::allocator`), the default
-    value for @a object_t is:
-
-    @code {.cpp}
-    std::map<
-      std::string, // key_type
-      basic_json, // value_type
-      std::less<std::string>, // key_compare
-      std::allocator<std::pair<const std::string, basic_json>> // allocator_type
-    >
-    @endcode
-
-    #### Behavior
-
-    The choice of @a object_t influences the behavior of the JSON class. With
-    the default type, objects have the following behavior:
-
-    - When all names are unique, objects will be interoperable in the sense
-      that all software implementations receiving that object will agree on
-      the name-value mappings.
-    - When the names within an object are not unique, it is unspecified which
-      one of the values for a given key will be chosen. For instance,
-      `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or
-      `{"key": 2}`.
-    - Internally, name/value pairs are stored in lexicographical order of the
-      names. Objects will also be serialized (see @ref dump) in this order.
-      For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored
-      and serialized as `{"a": 2, "b": 1}`.
-    - When comparing objects, the order of the name/value pairs is irrelevant.
-      This makes objects interoperable in the sense that they will not be
-      affected by these differences. For instance, `{"b": 1, "a": 2}` and
-      `{"a": 2, "b": 1}` will be treated as equal.
-
-    #### Limits
-
-    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
-    > An implementation may set limits on the maximum depth of nesting.
-
-    In this class, the object's limit of nesting is not explicitly constrained.
-    However, a maximum depth of nesting may be introduced by the compiler or
-    runtime environment. A theoretical limit can be queried by calling the
-    @ref max_size function of a JSON object.
-
-    #### Storage
-
-    Objects are stored as pointers in a @ref basic_json type. That is, for any
-    access to object values, a pointer of type `object_t*` must be
-    dereferenced.
-
-    @sa @ref array_t -- type for an array value
-
-    @since version 1.0.0
-
-    @note The order name/value pairs are added to the object is *not*
-    preserved by the library. Therefore, iterating an object may return
-    name/value pairs in a different order than they were originally stored. In
-    fact, keys will be traversed in alphabetical order as `std::map` with
-    `std::less` is used by default. Please note this behavior conforms to [RFC
-    7159](http://rfc7159.net/rfc7159), because any order implements the
-    specified "unordered" nature of JSON objects.
-    */
-    using object_t = ObjectType<StringType,
-          basic_json,
-          object_comparator_t,
-          AllocatorType<std::pair<const StringType,
-          basic_json>>>;
-
-    /*!
-    @brief a type for an array
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows:
-    > An array is an ordered sequence of zero or more values.
-
-    To store objects in C++, a type is defined by the template parameters
-    explained below.
-
-    @tparam ArrayType  container type to store arrays (e.g., `std::vector` or
-    `std::list`)
-    @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`)
-
-    #### Default type
-
-    With the default values for @a ArrayType (`std::vector`) and @a
-    AllocatorType (`std::allocator`), the default value for @a array_t is:
-
-    @code {.cpp}
-    std::vector<
-      basic_json, // value_type
-      std::allocator<basic_json> // allocator_type
-    >
-    @endcode
-
-    #### Limits
-
-    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
-    > An implementation may set limits on the maximum depth of nesting.
-
-    In this class, the array's limit of nesting is not explicitly constrained.
-    However, a maximum depth of nesting may be introduced by the compiler or
-    runtime environment. A theoretical limit can be queried by calling the
-    @ref max_size function of a JSON array.
-
-    #### Storage
-
-    Arrays are stored as pointers in a @ref basic_json type. That is, for any
-    access to array values, a pointer of type `array_t*` must be dereferenced.
-
-    @sa @ref object_t -- type for an object value
-
-    @since version 1.0.0
-    */
-    using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
-
-    /*!
-    @brief a type for a string
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows:
-    > A string is a sequence of zero or more Unicode characters.
-
-    To store objects in C++, a type is defined by the template parameter
-    described below. Unicode values are split by the JSON class into
-    byte-sized characters during deserialization.
-
-    @tparam StringType  the container to store strings (e.g., `std::string`).
-    Note this container is used for keys/names in objects, see @ref object_t.
-
-    #### Default type
-
-    With the default values for @a StringType (`std::string`), the default
-    value for @a string_t is:
-
-    @code {.cpp}
-    std::string
-    @endcode
-
-    #### Encoding
-
-    Strings are stored in UTF-8 encoding. Therefore, functions like
-    `std::string::size()` or `std::string::length()` return the number of
-    bytes in the string rather than the number of characters or glyphs.
-
-    #### String comparison
-
-    [RFC 7159](http://rfc7159.net/rfc7159) states:
-    > Software implementations are typically required to test names of object
-    > members for equality. Implementations that transform the textual
-    > representation into sequences of Unicode code units and then perform the
-    > comparison numerically, code unit by code unit, are interoperable in the
-    > sense that implementations will agree in all cases on equality or
-    > inequality of two strings. For example, implementations that compare
-    > strings with escaped characters unconverted may incorrectly find that
-    > `"a\\b"` and `"a\u005Cb"` are not equal.
-
-    This implementation is interoperable as it does compare strings code unit
-    by code unit.
-
-    #### Storage
-
-    String values are stored as pointers in a @ref basic_json type. That is,
-    for any access to string values, a pointer of type `string_t*` must be
-    dereferenced.
-
-    @since version 1.0.0
-    */
-    using string_t = StringType;
-
-    /*!
-    @brief a type for a boolean
-
-    [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a
-    type which differentiates the two literals `true` and `false`.
-
-    To store objects in C++, a type is defined by the template parameter @a
-    BooleanType which chooses the type to use.
-
-    #### Default type
-
-    With the default values for @a BooleanType (`bool`), the default value for
-    @a boolean_t is:
-
-    @code {.cpp}
-    bool
-    @endcode
-
-    #### Storage
-
-    Boolean values are stored directly inside a @ref basic_json type.
-
-    @since version 1.0.0
-    */
-    using boolean_t = BooleanType;
-
-    /*!
-    @brief a type for a number (integer)
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
-    > The representation of numbers is similar to that used in most
-    > programming languages. A number is represented in base 10 using decimal
-    > digits. It contains an integer component that may be prefixed with an
-    > optional minus sign, which may be followed by a fraction part and/or an
-    > exponent part. Leading zeros are not allowed. (...) Numeric values that
-    > cannot be represented in the grammar below (such as Infinity and NaN)
-    > are not permitted.
-
-    This description includes both integer and floating-point numbers.
-    However, C++ allows more precise storage if it is known whether the number
-    is a signed integer, an unsigned integer or a floating-point number.
-    Therefore, three different types, @ref number_integer_t, @ref
-    number_unsigned_t and @ref number_float_t are used.
-
-    To store integer numbers in C++, a type is defined by the template
-    parameter @a NumberIntegerType which chooses the type to use.
-
-    #### Default type
-
-    With the default values for @a NumberIntegerType (`int64_t`), the default
-    value for @a number_integer_t is:
-
-    @code {.cpp}
-    int64_t
-    @endcode
-
-    #### Default behavior
-
-    - The restrictions about leading zeros is not enforced in C++. Instead,
-      leading zeros in integer literals lead to an interpretation as octal
-      number. Internally, the value will be stored as decimal number. For
-      instance, the C++ integer literal `010` will be serialized to `8`.
-      During deserialization, leading zeros yield an error.
-    - Not-a-number (NaN) values will be serialized to `null`.
-
-    #### Limits
-
-    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
-    > An implementation may set limits on the range and precision of numbers.
-
-    When the default type is used, the maximal integer number that can be
-    stored is `9223372036854775807` (INT64_MAX) and the minimal integer number
-    that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers
-    that are out of range will yield over/underflow when used in a
-    constructor. During deserialization, too large or small integer numbers
-    will be automatically be stored as @ref number_unsigned_t or @ref
-    number_float_t.
-
-    [RFC 7159](http://rfc7159.net/rfc7159) further states:
-    > Note that when such software is used, numbers that are integers and are
-    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
-    > that implementations will agree exactly on their numeric values.
-
-    As this range is a subrange of the exactly supported range [INT64_MIN,
-    INT64_MAX], this class's integer type is interoperable.
-
-    #### Storage
-
-    Integer number values are stored directly inside a @ref basic_json type.
-
-    @sa @ref number_float_t -- type for number values (floating-point)
-
-    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
-
-    @since version 1.0.0
-    */
-    using number_integer_t = NumberIntegerType;
-
-    /*!
-    @brief a type for a number (unsigned)
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
-    > The representation of numbers is similar to that used in most
-    > programming languages. A number is represented in base 10 using decimal
-    > digits. It contains an integer component that may be prefixed with an
-    > optional minus sign, which may be followed by a fraction part and/or an
-    > exponent part. Leading zeros are not allowed. (...) Numeric values that
-    > cannot be represented in the grammar below (such as Infinity and NaN)
-    > are not permitted.
-
-    This description includes both integer and floating-point numbers.
-    However, C++ allows more precise storage if it is known whether the number
-    is a signed integer, an unsigned integer or a floating-point number.
-    Therefore, three different types, @ref number_integer_t, @ref
-    number_unsigned_t and @ref number_float_t are used.
-
-    To store unsigned integer numbers in C++, a type is defined by the
-    template parameter @a NumberUnsignedType which chooses the type to use.
-
-    #### Default type
-
-    With the default values for @a NumberUnsignedType (`uint64_t`), the
-    default value for @a number_unsigned_t is:
-
-    @code {.cpp}
-    uint64_t
-    @endcode
-
-    #### Default behavior
-
-    - The restrictions about leading zeros is not enforced in C++. Instead,
-      leading zeros in integer literals lead to an interpretation as octal
-      number. Internally, the value will be stored as decimal number. For
-      instance, the C++ integer literal `010` will be serialized to `8`.
-      During deserialization, leading zeros yield an error.
-    - Not-a-number (NaN) values will be serialized to `null`.
-
-    #### Limits
-
-    [RFC 7159](http://rfc7159.net/rfc7159) specifies:
-    > An implementation may set limits on the range and precision of numbers.
-
-    When the default type is used, the maximal integer number that can be
-    stored is `18446744073709551615` (UINT64_MAX) and the minimal integer
-    number that can be stored is `0`. Integer numbers that are out of range
-    will yield over/underflow when used in a constructor. During
-    deserialization, too large or small integer numbers will be automatically
-    be stored as @ref number_integer_t or @ref number_float_t.
-
-    [RFC 7159](http://rfc7159.net/rfc7159) further states:
-    > Note that when such software is used, numbers that are integers and are
-    > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense
-    > that implementations will agree exactly on their numeric values.
-
-    As this range is a subrange (when considered in conjunction with the
-    number_integer_t type) of the exactly supported range [0, UINT64_MAX],
-    this class's integer type is interoperable.
-
-    #### Storage
-
-    Integer number values are stored directly inside a @ref basic_json type.
-
-    @sa @ref number_float_t -- type for number values (floating-point)
-    @sa @ref number_integer_t -- type for number values (integer)
-
-    @since version 2.0.0
-    */
-    using number_unsigned_t = NumberUnsignedType;
-
-    /*!
-    @brief a type for a number (floating-point)
-
-    [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows:
-    > The representation of numbers is similar to that used in most
-    > programming languages. A number is represented in base 10 using decimal
-    > digits. It contains an integer component that may be prefixed with an
-    > optional minus sign, which may be followed by a fraction part and/or an
-    > exponent part. Leading zeros are not allowed. (...) Numeric values that
-    > cannot be represented in the grammar below (such as Infinity and NaN)
-    > are not permitted.
-
-    This description includes both integer and floating-point numbers.
-    However, C++ allows more precise storage if it is known whether the number
-    is a signed integer, an unsigned integer or a floating-point number.
-    Therefore, three different types, @ref number_integer_t, @ref
-    number_unsigned_t and @ref number_float_t are used.
-
-    To store floating-point numbers in C++, a type is defined by the template
-    parameter @a NumberFloatType which chooses the type to use.
-
-    #### Default type
-
-    With the default values for @a NumberFloatType (`double`), the default
-    value for @a number_float_t is:
-
-    @code {.cpp}
-    double
-    @endcode
-
-    #### Default behavior
-
-    - The restrictions about leading zeros is not enforced in C++. Instead,
-      leading zeros in floating-point literals will be ignored. Internally,
-      the value will be stored as decimal number. For instance, the C++
-      floating-point literal `01.2` will be serialized to `1.2`. During
-      deserialization, leading zeros yield an error.
-    - Not-a-number (NaN) values will be serialized to `null`.
-
-    #### Limits
-
-    [RFC 7159](http://rfc7159.net/rfc7159) states:
-    > This specification allows implementations to set limits on the range and
-    > precision of numbers accepted. Since software that implements IEEE
-    > 754-2008 binary64 (double precision) numbers is generally available and
-    > widely used, good interoperability can be achieved by implementations
-    > that expect no more precision or range than these provide, in the sense
-    > that implementations will approximate JSON numbers within the expected
-    > precision.
-
-    This implementation does exactly follow this approach, as it uses double
-    precision floating-point numbers. Note values smaller than
-    `-1.79769313486232e+308` and values greater than `1.79769313486232e+308`
-    will be stored as NaN internally and be serialized to `null`.
-
-    #### Storage
-
-    Floating-point number values are stored directly inside a @ref basic_json
-    type.
-
-    @sa @ref number_integer_t -- type for number values (integer)
-
-    @sa @ref number_unsigned_t -- type for number values (unsigned integer)
-
-    @since version 1.0.0
-    */
-    using number_float_t = NumberFloatType;
-
-    /// @}
-
-  private:
-
-    /// helper for exception-safe object creation
-    template<typename T, typename... Args>
-    static T* create(Args&& ... args)
-    {
-        AllocatorType<T> alloc;
-        using AllocatorTraits = std::allocator_traits<AllocatorType<T>>;
-
-        auto deleter = [&](T * object)
-        {
-            AllocatorTraits::deallocate(alloc, object, 1);
-        };
-        std::unique_ptr<T, decltype(deleter)> object(AllocatorTraits::allocate(alloc, 1), deleter);
-        AllocatorTraits::construct(alloc, object.get(), std::forward<Args>(args)...);
-        assert(object != nullptr);
-        return object.release();
-    }
-
-    ////////////////////////
-    // JSON value storage //
-    ////////////////////////
-
-    /*!
-    @brief a JSON value
-
-    The actual storage for a JSON value of the @ref basic_json class. This
-    union combines the different storage types for the JSON value types
-    defined in @ref value_t.
-
-    JSON type | value_t type    | used type
-    --------- | --------------- | ------------------------
-    object    | object          | pointer to @ref object_t
-    array     | array           | pointer to @ref array_t
-    string    | string          | pointer to @ref string_t
-    boolean   | boolean         | @ref boolean_t
-    number    | number_integer  | @ref number_integer_t
-    number    | number_unsigned | @ref number_unsigned_t
-    number    | number_float    | @ref number_float_t
-    null      | null            | *no value is stored*
-
-    @note Variable-length types (objects, arrays, and strings) are stored as
-    pointers. The size of the union should not exceed 64 bits if the default
-    value types are used.
-
-    @since version 1.0.0
-    */
-    union json_value
-    {
-        /// object (stored with pointer to save storage)
-        object_t* object;
-        /// array (stored with pointer to save storage)
-        array_t* array;
-        /// string (stored with pointer to save storage)
-        string_t* string;
-        /// boolean
-        boolean_t boolean;
-        /// number (integer)
-        number_integer_t number_integer;
-        /// number (unsigned integer)
-        number_unsigned_t number_unsigned;
-        /// number (floating-point)
-        number_float_t number_float;
-
-        /// default constructor (for null values)
-        json_value() = default;
-        /// constructor for booleans
-        json_value(boolean_t v) noexcept : boolean(v) {}
-        /// constructor for numbers (integer)
-        json_value(number_integer_t v) noexcept : number_integer(v) {}
-        /// constructor for numbers (unsigned)
-        json_value(number_unsigned_t v) noexcept : number_unsigned(v) {}
-        /// constructor for numbers (floating-point)
-        json_value(number_float_t v) noexcept : number_float(v) {}
-        /// constructor for empty values of a given type
-        json_value(value_t t)
-        {
-            switch (t)
-            {
-                case value_t::object:
-                {
-                    object = create<object_t>();
-                    break;
-                }
-
-                case value_t::array:
-                {
-                    array = create<array_t>();
-                    break;
-                }
-
-                case value_t::string:
-                {
-                    string = create<string_t>("");
-                    break;
-                }
-
-                case value_t::boolean:
-                {
-                    boolean = boolean_t(false);
-                    break;
-                }
-
-                case value_t::number_integer:
-                {
-                    number_integer = number_integer_t(0);
-                    break;
-                }
-
-                case value_t::number_unsigned:
-                {
-                    number_unsigned = number_unsigned_t(0);
-                    break;
-                }
-
-                case value_t::number_float:
-                {
-                    number_float = number_float_t(0.0);
-                    break;
-                }
-
-                case value_t::null:
-                {
-                    object = nullptr;  // silence warning, see #821
-                    break;
-                }
-
-                default:
-                {
-                    object = nullptr;  // silence warning, see #821
-                    if (JSON_UNLIKELY(t == value_t::null))
-                    {
-                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE
-                    }
-                    break;
-                }
-            }
-        }
-
-        /// constructor for strings
-        json_value(const string_t& value)
-        {
-            string = create<string_t>(value);
-        }
-
-        /// constructor for rvalue strings
-        json_value(string_t&& value)
-        {
-            string = create<string_t>(std::move(value));
-        }
-
-        /// constructor for objects
-        json_value(const object_t& value)
-        {
-            object = create<object_t>(value);
-        }
-
-        /// constructor for rvalue objects
-        json_value(object_t&& value)
-        {
-            object = create<object_t>(std::move(value));
-        }
-
-        /// constructor for arrays
-        json_value(const array_t& value)
-        {
-            array = create<array_t>(value);
-        }
-
-        /// constructor for rvalue arrays
-        json_value(array_t&& value)
-        {
-            array = create<array_t>(std::move(value));
-        }
-
-        void destroy(value_t t) noexcept
-        {
-            switch (t)
-            {
-                case value_t::object:
-                {
-                    AllocatorType<object_t> alloc;
-                    std::allocator_traits<decltype(alloc)>::destroy(alloc, object);
-                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1);
-                    break;
-                }
-
-                case value_t::array:
-                {
-                    AllocatorType<array_t> alloc;
-                    std::allocator_traits<decltype(alloc)>::destroy(alloc, array);
-                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1);
-                    break;
-                }
-
-                case value_t::string:
-                {
-                    AllocatorType<string_t> alloc;
-                    std::allocator_traits<decltype(alloc)>::destroy(alloc, string);
-                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1);
-                    break;
-                }
-
-                default:
-                {
-                    break;
-                }
-            }
-        }
-    };
-
-    /*!
-    @brief checks the class invariants
-
-    This function asserts the class invariants. It needs to be called at the
-    end of every constructor to make sure that created objects respect the
-    invariant. Furthermore, it has to be called each time the type of a JSON
-    value is changed, because the invariant expresses a relationship between
-    @a m_type and @a m_value.
-    */
-    void assert_invariant() const noexcept
-    {
-        assert(m_type != value_t::object or m_value.object != nullptr);
-        assert(m_type != value_t::array or m_value.array != nullptr);
-        assert(m_type != value_t::string or m_value.string != nullptr);
-    }
-
-  public:
-    //////////////////////////
-    // JSON parser callback //
-    //////////////////////////
-
-    /*!
-    @brief parser event types
-
-    The parser callback distinguishes the following events:
-    - `object_start`: the parser read `{` and started to process a JSON object
-    - `key`: the parser read a key of a value in an object
-    - `object_end`: the parser read `}` and finished processing a JSON object
-    - `array_start`: the parser read `[` and started to process a JSON array
-    - `array_end`: the parser read `]` and finished processing a JSON array
-    - `value`: the parser finished reading a JSON value
-
-    @image html callback_events.png "Example when certain parse events are triggered"
-
-    @sa @ref parser_callback_t for more information and examples
-    */
-    using parse_event_t = typename parser::parse_event_t;
-
-    /*!
-    @brief per-element parser callback type
-
-    With a parser callback function, the result of parsing a JSON text can be
-    influenced. When passed to @ref parse, it is called on certain events
-    (passed as @ref parse_event_t via parameter @a event) with a set recursion
-    depth @a depth and context JSON value @a parsed. The return value of the
-    callback function is a boolean indicating whether the element that emitted
-    the callback shall be kept or not.
-
-    We distinguish six scenarios (determined by the event type) in which the
-    callback function can be called. The following table describes the values
-    of the parameters @a depth, @a event, and @a parsed.
-
-    parameter @a event | description | parameter @a depth | parameter @a parsed
-    ------------------ | ----------- | ------------------ | -------------------
-    parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded
-    parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key
-    parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object
-    parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded
-    parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array
-    parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value
-
-    @image html callback_events.png "Example when certain parse events are triggered"
-
-    Discarding a value (i.e., returning `false`) has different effects
-    depending on the context in which function was called:
-
-    - Discarded values in structured types are skipped. That is, the parser
-      will behave as if the discarded value was never read.
-    - In case a value outside a structured type is skipped, it is replaced
-      with `null`. This case happens if the top-level element is skipped.
-
-    @param[in] depth  the depth of the recursion during parsing
-
-    @param[in] event  an event of type parse_event_t indicating the context in
-    the callback function has been called
-
-    @param[in,out] parsed  the current intermediate parse result; note that
-    writing to this value has no effect for parse_event_t::key events
-
-    @return Whether the JSON value which called the function during parsing
-    should be kept (`true`) or not (`false`). In the latter case, it is either
-    skipped completely or replaced by an empty discarded object.
-
-    @sa @ref parse for examples
-
-    @since version 1.0.0
-    */
-    using parser_callback_t = typename parser::parser_callback_t;
-
-    //////////////////
-    // constructors //
-    //////////////////
-
-    /// @name constructors and destructors
-    /// Constructors of class @ref basic_json, copy/move constructor, copy
-    /// assignment, static functions creating objects, and the destructor.
-    /// @{
-
-    /*!
-    @brief create an empty value with a given type
-
-    Create an empty JSON value with a given type. The value will be default
-    initialized with an empty value which depends on the type:
-
-    Value type  | initial value
-    ----------- | -------------
-    null        | `null`
-    boolean     | `false`
-    string      | `""`
-    number      | `0`
-    object      | `{}`
-    array       | `[]`
-
-    @param[in] v  the type of the value to create
-
-    @complexity Constant.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The following code shows the constructor for different @ref
-    value_t values,basic_json__value_t}
-
-    @sa @ref clear() -- restores the postcondition of this constructor
-
-    @since version 1.0.0
-    */
-    basic_json(const value_t v)
-        : m_type(v), m_value(v)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a null object
-
-    Create a `null` JSON value. It either takes a null pointer as parameter
-    (explicitly creating `null`) or no parameter (implicitly creating `null`).
-    The passed null pointer itself is not read -- it is only used to choose
-    the right constructor.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this constructor never throws
-    exceptions.
-
-    @liveexample{The following code shows the constructor with and without a
-    null pointer parameter.,basic_json__nullptr_t}
-
-    @since version 1.0.0
-    */
-    basic_json(std::nullptr_t = nullptr) noexcept
-        : basic_json(value_t::null)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a JSON value
-
-    This is a "catch all" constructor for all compatible JSON types; that is,
-    types for which a `to_json()` method exists. The constructor forwards the
-    parameter @a val to that method (to `json_serializer<U>::to_json` method
-    with `U = uncvref_t<CompatibleType>`, to be exact).
-
-    Template type @a CompatibleType includes, but is not limited to, the
-    following types:
-    - **arrays**: @ref array_t and all kinds of compatible containers such as
-      `std::vector`, `std::deque`, `std::list`, `std::forward_list`,
-      `std::array`, `std::valarray`, `std::set`, `std::unordered_set`,
-      `std::multiset`, and `std::unordered_multiset` with a `value_type` from
-      which a @ref basic_json value can be constructed.
-    - **objects**: @ref object_t and all kinds of compatible associative
-      containers such as `std::map`, `std::unordered_map`, `std::multimap`,
-      and `std::unordered_multimap` with a `key_type` compatible to
-      @ref string_t and a `value_type` from which a @ref basic_json value can
-      be constructed.
-    - **strings**: @ref string_t, string literals, and all compatible string
-      containers can be used.
-    - **numbers**: @ref number_integer_t, @ref number_unsigned_t,
-      @ref number_float_t, and all convertible number types such as `int`,
-      `size_t`, `int64_t`, `float` or `double` can be used.
-    - **boolean**: @ref boolean_t / `bool` can be used.
-
-    See the examples below.
-
-    @tparam CompatibleType a type such that:
-    - @a CompatibleType is not derived from `std::istream`,
-    - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
-         constructors),
-    - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments)
-    - @a CompatibleType is not a @ref basic_json nested type (e.g.,
-         @ref json_pointer, @ref iterator, etc ...)
-    - @ref @ref json_serializer<U> has a
-         `to_json(basic_json_t&, CompatibleType&&)` method
-
-    @tparam U = `uncvref_t<CompatibleType>`
-
-    @param[in] val the value to be forwarded to the respective constructor
-
-    @complexity Usually linear in the size of the passed @a val, also
-                depending on the implementation of the called `to_json()`
-                method.
-
-    @exceptionsafety Depends on the called constructor. For types directly
-    supported by the library (i.e., all types for which no `to_json()` function
-    was provided), strong guarantee holds: if an exception is thrown, there are
-    no changes to any JSON value.
-
-    @liveexample{The following code shows the constructor with several
-    compatible types.,basic_json__CompatibleType}
-
-    @since version 2.1.0
-    */
-    template <typename CompatibleType,
-              typename U = detail::uncvref_t<CompatibleType>,
-              detail::enable_if_t<
-                  not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0>
-    basic_json(CompatibleType && val) noexcept(noexcept(
-                JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
-                                           std::forward<CompatibleType>(val))))
-    {
-        JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a JSON value from an existing one
-
-    This is a constructor for existing @ref basic_json types.
-    It does not hijack copy/move constructors, since the parameter has different
-    template arguments than the current ones.
-
-    The constructor tries to convert the internal @ref m_value of the parameter.
-
-    @tparam BasicJsonType a type such that:
-    - @a BasicJsonType is a @ref basic_json type.
-    - @a BasicJsonType has different template arguments than @ref basic_json_t.
-
-    @param[in] val the @ref basic_json value to be converted.
-
-    @complexity Usually linear in the size of the passed @a val, also
-                depending on the implementation of the called `to_json()`
-                method.
-
-    @exceptionsafety Depends on the called constructor. For types directly
-    supported by the library (i.e., all types for which no `to_json()` function
-    was provided), strong guarantee holds: if an exception is thrown, there are
-    no changes to any JSON value.
-
-    @since version 3.2.0
-    */
-    template <typename BasicJsonType,
-              detail::enable_if_t<
-                  detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0>
-    basic_json(const BasicJsonType& val)
-    {
-        using other_boolean_t = typename BasicJsonType::boolean_t;
-        using other_number_float_t = typename BasicJsonType::number_float_t;
-        using other_number_integer_t = typename BasicJsonType::number_integer_t;
-        using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-        using other_string_t = typename BasicJsonType::string_t;
-        using other_object_t = typename BasicJsonType::object_t;
-        using other_array_t = typename BasicJsonType::array_t;
-
-        switch (val.type())
-        {
-            case value_t::boolean:
-                JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());
-                break;
-            case value_t::number_float:
-                JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());
-                break;
-            case value_t::number_integer:
-                JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());
-                break;
-            case value_t::number_unsigned:
-                JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());
-                break;
-            case value_t::string:
-                JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());
-                break;
-            case value_t::object:
-                JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());
-                break;
-            case value_t::array:
-                JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());
-                break;
-            case value_t::null:
-                *this = nullptr;
-                break;
-            case value_t::discarded:
-                m_type = value_t::discarded;
-                break;
-        }
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a container (array or object) from an initializer list
-
-    Creates a JSON value of type array or object from the passed initializer
-    list @a init. In case @a type_deduction is `true` (default), the type of
-    the JSON value to be created is deducted from the initializer list @a init
-    according to the following rules:
-
-    1. If the list is empty, an empty JSON object value `{}` is created.
-    2. If the list consists of pairs whose first element is a string, a JSON
-       object value is created where the first elements of the pairs are
-       treated as keys and the second elements are as values.
-    3. In all other cases, an array is created.
-
-    The rules aim to create the best fit between a C++ initializer list and
-    JSON values. The rationale is as follows:
-
-    1. The empty initializer list is written as `{}` which is exactly an empty
-       JSON object.
-    2. C++ has no way of describing mapped types other than to list a list of
-       pairs. As JSON requires that keys must be of type string, rule 2 is the
-       weakest constraint one can pose on initializer lists to interpret them
-       as an object.
-    3. In all other cases, the initializer list could not be interpreted as
-       JSON object type, so interpreting it as JSON array type is safe.
-
-    With the rules described above, the following JSON values cannot be
-    expressed by an initializer list:
-
-    - the empty array (`[]`): use @ref array(initializer_list_t)
-      with an empty initializer list in this case
-    - arrays whose elements satisfy rule 2: use @ref
-      array(initializer_list_t) with the same initializer list
-      in this case
-
-    @note When used without parentheses around an empty initializer list, @ref
-    basic_json() is called instead of this function, yielding the JSON null
-    value.
-
-    @param[in] init  initializer list with JSON values
-
-    @param[in] type_deduction internal parameter; when set to `true`, the type
-    of the JSON value is deducted from the initializer list @a init; when set
-    to `false`, the type provided via @a manual_type is forced. This mode is
-    used by the functions @ref array(initializer_list_t) and
-    @ref object(initializer_list_t).
-
-    @param[in] manual_type internal parameter; when @a type_deduction is set
-    to `false`, the created JSON value will use the provided type (only @ref
-    value_t::array and @ref value_t::object are valid); when @a type_deduction
-    is set to `true`, this parameter has no effect
-
-    @throw type_error.301 if @a type_deduction is `false`, @a manual_type is
-    `value_t::object`, but @a init contains an element which is not a pair
-    whose first element is a string. In this case, the constructor could not
-    create an object. If @a type_deduction would have be `true`, an array
-    would have been created. See @ref object(initializer_list_t)
-    for an example.
-
-    @complexity Linear in the size of the initializer list @a init.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The example below shows how JSON values are created from
-    initializer lists.,basic_json__list_init_t}
-
-    @sa @ref array(initializer_list_t) -- create a JSON array
-    value from an initializer list
-    @sa @ref object(initializer_list_t) -- create a JSON object
-    value from an initializer list
-
-    @since version 1.0.0
-    */
-    basic_json(initializer_list_t init,
-               bool type_deduction = true,
-               value_t manual_type = value_t::array)
-    {
-        // check if each element is an array with two elements whose first
-        // element is a string
-        bool is_an_object = std::all_of(init.begin(), init.end(),
-                                        [](const detail::json_ref<basic_json>& element_ref)
-        {
-            return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string());
-        });
-
-        // adjust type if type deduction is not wanted
-        if (not type_deduction)
-        {
-            // if array is wanted, do not create an object though possible
-            if (manual_type == value_t::array)
-            {
-                is_an_object = false;
-            }
-
-            // if object is wanted but impossible, throw an exception
-            if (JSON_UNLIKELY(manual_type == value_t::object and not is_an_object))
-            {
-                JSON_THROW(type_error::create(301, "cannot create object from initializer list"));
-            }
-        }
-
-        if (is_an_object)
-        {
-            // the initializer list is a list of pairs -> create object
-            m_type = value_t::object;
-            m_value = value_t::object;
-
-            std::for_each(init.begin(), init.end(), [this](const detail::json_ref<basic_json>& element_ref)
-            {
-                auto element = element_ref.moved_or_copied();
-                m_value.object->emplace(
-                    std::move(*((*element.m_value.array)[0].m_value.string)),
-                    std::move((*element.m_value.array)[1]));
-            });
-        }
-        else
-        {
-            // the initializer list describes an array -> create array
-            m_type = value_t::array;
-            m_value.array = create<array_t>(init.begin(), init.end());
-        }
-
-        assert_invariant();
-    }
-
-    /*!
-    @brief explicitly create an array from an initializer list
-
-    Creates a JSON array value from a given initializer list. That is, given a
-    list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the
-    initializer list is empty, the empty array `[]` is created.
-
-    @note This function is only needed to express two edge cases that cannot
-    be realized with the initializer list constructor (@ref
-    basic_json(initializer_list_t, bool, value_t)). These cases
-    are:
-    1. creating an array whose elements are all pairs whose first element is a
-    string -- in this case, the initializer list constructor would create an
-    object, taking the first elements as keys
-    2. creating an empty array -- passing the empty initializer list to the
-    initializer list constructor yields an empty object
-
-    @param[in] init  initializer list with JSON values to create an array from
-    (optional)
-
-    @return JSON array value
-
-    @complexity Linear in the size of @a init.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The following code shows an example for the `array`
-    function.,array}
-
-    @sa @ref basic_json(initializer_list_t, bool, value_t) --
-    create a JSON value from an initializer list
-    @sa @ref object(initializer_list_t) -- create a JSON object
-    value from an initializer list
-
-    @since version 1.0.0
-    */
-    static basic_json array(initializer_list_t init = {})
-    {
-        return basic_json(init, false, value_t::array);
-    }
-
-    /*!
-    @brief explicitly create an object from an initializer list
-
-    Creates a JSON object value from a given initializer list. The initializer
-    lists elements must be pairs, and their first elements must be strings. If
-    the initializer list is empty, the empty object `{}` is created.
-
-    @note This function is only added for symmetry reasons. In contrast to the
-    related function @ref array(initializer_list_t), there are
-    no cases which can only be expressed by this function. That is, any
-    initializer list @a init can also be passed to the initializer list
-    constructor @ref basic_json(initializer_list_t, bool, value_t).
-
-    @param[in] init  initializer list to create an object from (optional)
-
-    @return JSON object value
-
-    @throw type_error.301 if @a init is not a list of pairs whose first
-    elements are strings. In this case, no object can be created. When such a
-    value is passed to @ref basic_json(initializer_list_t, bool, value_t),
-    an array would have been created from the passed initializer list @a init.
-    See example below.
-
-    @complexity Linear in the size of @a init.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The following code shows an example for the `object`
-    function.,object}
-
-    @sa @ref basic_json(initializer_list_t, bool, value_t) --
-    create a JSON value from an initializer list
-    @sa @ref array(initializer_list_t) -- create a JSON array
-    value from an initializer list
-
-    @since version 1.0.0
-    */
-    static basic_json object(initializer_list_t init = {})
-    {
-        return basic_json(init, false, value_t::object);
-    }
-
-    /*!
-    @brief construct an array with count copies of given value
-
-    Constructs a JSON array value by creating @a cnt copies of a passed value.
-    In case @a cnt is `0`, an empty array is created.
-
-    @param[in] cnt  the number of JSON copies of @a val to create
-    @param[in] val  the JSON value to copy
-
-    @post `std::distance(begin(),end()) == cnt` holds.
-
-    @complexity Linear in @a cnt.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The following code shows examples for the @ref
-    basic_json(size_type\, const basic_json&)
-    constructor.,basic_json__size_type_basic_json}
-
-    @since version 1.0.0
-    */
-    basic_json(size_type cnt, const basic_json& val)
-        : m_type(value_t::array)
-    {
-        m_value.array = create<array_t>(cnt, val);
-        assert_invariant();
-    }
-
-    /*!
-    @brief construct a JSON container given an iterator range
-
-    Constructs the JSON value with the contents of the range `[first, last)`.
-    The semantics depends on the different types a JSON value can have:
-    - In case of a null type, invalid_iterator.206 is thrown.
-    - In case of other primitive types (number, boolean, or string), @a first
-      must be `begin()` and @a last must be `end()`. In this case, the value is
-      copied. Otherwise, invalid_iterator.204 is thrown.
-    - In case of structured types (array, object), the constructor behaves as
-      similar versions for `std::vector` or `std::map`; that is, a JSON array
-      or object is constructed from the values in the range.
-
-    @tparam InputIT an input iterator type (@ref iterator or @ref
-    const_iterator)
-
-    @param[in] first begin of the range to copy from (included)
-    @param[in] last end of the range to copy from (excluded)
-
-    @pre Iterators @a first and @a last must be initialized. **This
-         precondition is enforced with an assertion (see warning).** If
-         assertions are switched off, a violation of this precondition yields
-         undefined behavior.
-
-    @pre Range `[first, last)` is valid. Usually, this precondition cannot be
-         checked efficiently. Only certain edge cases are detected; see the
-         description of the exceptions below. A violation of this precondition
-         yields undefined behavior.
-
-    @warning A precondition is enforced with a runtime assertion that will
-             result in calling `std::abort` if this precondition is not met.
-             Assertions can be disabled by defining `NDEBUG` at compile time.
-             See https://en.cppreference.com/w/cpp/error/assert for more
-             information.
-
-    @throw invalid_iterator.201 if iterators @a first and @a last are not
-    compatible (i.e., do not belong to the same JSON value). In this case,
-    the range `[first, last)` is undefined.
-    @throw invalid_iterator.204 if iterators @a first and @a last belong to a
-    primitive type (number, boolean, or string), but @a first does not point
-    to the first element any more. In this case, the range `[first, last)` is
-    undefined. See example code below.
-    @throw invalid_iterator.206 if iterators @a first and @a last belong to a
-    null value. In this case, the range `[first, last)` is undefined.
-
-    @complexity Linear in distance between @a first and @a last.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @liveexample{The example below shows several ways to create JSON values by
-    specifying a subrange with iterators.,basic_json__InputIt_InputIt}
-
-    @since version 1.0.0
-    */
-    template<class InputIT, typename std::enable_if<
-                 std::is_same<InputIT, typename basic_json_t::iterator>::value or
-                 std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int>::type = 0>
-    basic_json(InputIT first, InputIT last)
-    {
-        assert(first.m_object != nullptr);
-        assert(last.m_object != nullptr);
-
-        // make sure iterator fits the current value
-        if (JSON_UNLIKELY(first.m_object != last.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(201, "iterators are not compatible"));
-        }
-
-        // copy type from first iterator
-        m_type = first.m_object->m_type;
-
-        // check if iterator range is complete for primitive values
-        switch (m_type)
-        {
-            case value_t::boolean:
-            case value_t::number_float:
-            case value_t::number_integer:
-            case value_t::number_unsigned:
-            case value_t::string:
-            {
-                if (JSON_UNLIKELY(not first.m_it.primitive_iterator.is_begin()
-                                  or not last.m_it.primitive_iterator.is_end()))
-                {
-                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
-                }
-                break;
-            }
-
-            default:
-                break;
-        }
-
-        switch (m_type)
-        {
-            case value_t::number_integer:
-            {
-                m_value.number_integer = first.m_object->m_value.number_integer;
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                m_value.number_unsigned = first.m_object->m_value.number_unsigned;
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                m_value.number_float = first.m_object->m_value.number_float;
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                m_value.boolean = first.m_object->m_value.boolean;
-                break;
-            }
-
-            case value_t::string:
-            {
-                m_value = *first.m_object->m_value.string;
-                break;
-            }
-
-            case value_t::object:
-            {
-                m_value.object = create<object_t>(first.m_it.object_iterator,
-                                                  last.m_it.object_iterator);
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_value.array = create<array_t>(first.m_it.array_iterator,
-                                                last.m_it.array_iterator);
-                break;
-            }
-
-            default:
-                JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " +
-                                                    std::string(first.m_object->type_name())));
-        }
-
-        assert_invariant();
-    }
-
-
-    ///////////////////////////////////////
-    // other constructors and destructor //
-    ///////////////////////////////////////
-
-    /// @private
-    basic_json(const detail::json_ref<basic_json>& ref)
-        : basic_json(ref.moved_or_copied())
-    {}
-
-    /*!
-    @brief copy constructor
-
-    Creates a copy of a given JSON value.
-
-    @param[in] other  the JSON value to copy
-
-    @post `*this == other`
-
-    @complexity Linear in the size of @a other.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes to any JSON value.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is linear.
-    - As postcondition, it holds: `other == basic_json(other)`.
-
-    @liveexample{The following code shows an example for the copy
-    constructor.,basic_json__basic_json}
-
-    @since version 1.0.0
-    */
-    basic_json(const basic_json& other)
-        : m_type(other.m_type)
-    {
-        // check of passed value is valid
-        other.assert_invariant();
-
-        switch (m_type)
-        {
-            case value_t::object:
-            {
-                m_value = *other.m_value.object;
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_value = *other.m_value.array;
-                break;
-            }
-
-            case value_t::string:
-            {
-                m_value = *other.m_value.string;
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                m_value = other.m_value.boolean;
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                m_value = other.m_value.number_integer;
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                m_value = other.m_value.number_unsigned;
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                m_value = other.m_value.number_float;
-                break;
-            }
-
-            default:
-                break;
-        }
-
-        assert_invariant();
-    }
-
-    /*!
-    @brief move constructor
-
-    Move constructor. Constructs a JSON value with the contents of the given
-    value @a other using move semantics. It "steals" the resources from @a
-    other and leaves it as JSON null value.
-
-    @param[in,out] other  value to move to this object
-
-    @post `*this` has the same value as @a other before the call.
-    @post @a other is a JSON null value.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this constructor never throws
-    exceptions.
-
-    @requirement This function helps `basic_json` satisfying the
-    [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible)
-    requirements.
-
-    @liveexample{The code below shows the move constructor explicitly called
-    via std::move.,basic_json__moveconstructor}
-
-    @since version 1.0.0
-    */
-    basic_json(basic_json&& other) noexcept
-        : m_type(std::move(other.m_type)),
-          m_value(std::move(other.m_value))
-    {
-        // check that passed value is valid
-        other.assert_invariant();
-
-        // invalidate payload
-        other.m_type = value_t::null;
-        other.m_value = {};
-
-        assert_invariant();
-    }
-
-    /*!
-    @brief copy assignment
-
-    Copy assignment operator. Copies a JSON value via the "copy and swap"
-    strategy: It is expressed in terms of the copy constructor, destructor,
-    and the `swap()` member function.
-
-    @param[in] other  value to copy from
-
-    @complexity Linear.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is linear.
-
-    @liveexample{The code below shows and example for the copy assignment. It
-    creates a copy of value `a` which is then swapped with `b`. Finally\, the
-    copy of `a` (which is the null value after the swap) is
-    destroyed.,basic_json__copyassignment}
-
-    @since version 1.0.0
-    */
-    basic_json& operator=(basic_json other) noexcept (
-        std::is_nothrow_move_constructible<value_t>::value and
-        std::is_nothrow_move_assignable<value_t>::value and
-        std::is_nothrow_move_constructible<json_value>::value and
-        std::is_nothrow_move_assignable<json_value>::value
-    )
-    {
-        // check that passed value is valid
-        other.assert_invariant();
-
-        using std::swap;
-        swap(m_type, other.m_type);
-        swap(m_value, other.m_value);
-
-        assert_invariant();
-        return *this;
-    }
-
-    /*!
-    @brief destructor
-
-    Destroys the JSON value and frees all allocated memory.
-
-    @complexity Linear.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is linear.
-    - All stored elements are destroyed and all memory is freed.
-
-    @since version 1.0.0
-    */
-    ~basic_json() noexcept
-    {
-        assert_invariant();
-        m_value.destroy(m_type);
-    }
-
-    /// @}
-
-  public:
-    ///////////////////////
-    // object inspection //
-    ///////////////////////
-
-    /// @name object inspection
-    /// Functions to inspect the type of a JSON value.
-    /// @{
-
-    /*!
-    @brief serialization
-
-    Serialization function for JSON values. The function tries to mimic
-    Python's `json.dumps()` function, and currently supports its @a indent
-    and @a ensure_ascii parameters.
-
-    @param[in] indent If indent is nonnegative, then array elements and object
-    members will be pretty-printed with that indent level. An indent level of
-    `0` will only insert newlines. `-1` (the default) selects the most compact
-    representation.
-    @param[in] indent_char The character to use for indentation if @a indent is
-    greater than `0`. The default is ` ` (space).
-    @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
-    in the output are escaped with `\uXXXX` sequences, and the result consists
-    of ASCII characters only.
-    @param[in] error_handler  how to react on decoding errors; there are three
-    possible values: `strict` (throws and exception in case a decoding error
-    occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD),
-    and `ignore` (ignore invalid UTF-8 sequences during serialization).
-
-    @return string containing the serialization of the JSON value
-
-    @throw type_error.316 if a string stored inside the JSON value is not
-                          UTF-8 encoded
-
-    @complexity Linear.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @liveexample{The following example shows the effect of different @a indent\,
-    @a indent_char\, and @a ensure_ascii parameters to the result of the
-    serialization.,dump}
-
-    @see https://docs.python.org/2/library/json.html#json.dump
-
-    @since version 1.0.0; indentation character @a indent_char, option
-           @a ensure_ascii and exceptions added in version 3.0.0; error
-           handlers added in version 3.4.0.
-    */
-    string_t dump(const int indent = -1,
-                  const char indent_char = ' ',
-                  const bool ensure_ascii = false,
-                  const error_handler_t error_handler = error_handler_t::strict) const
-    {
-        string_t result;
-        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
-
-        if (indent >= 0)
-        {
-            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
-        }
-        else
-        {
-            s.dump(*this, false, ensure_ascii, 0);
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief return the type of the JSON value (explicit)
-
-    Return the type of the JSON value as a value from the @ref value_t
-    enumeration.
-
-    @return the type of the JSON value
-            Value type                | return value
-            ------------------------- | -------------------------
-            null                      | value_t::null
-            boolean                   | value_t::boolean
-            string                    | value_t::string
-            number (integer)          | value_t::number_integer
-            number (unsigned integer) | value_t::number_unsigned
-            number (floating-point)   | value_t::number_float
-            object                    | value_t::object
-            array                     | value_t::array
-            discarded                 | value_t::discarded
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `type()` for all JSON
-    types.,type}
-
-    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
-    @sa @ref type_name() -- return the type as string
-
-    @since version 1.0.0
-    */
-    constexpr value_t type() const noexcept
-    {
-        return m_type;
-    }
-
-    /*!
-    @brief return whether type is primitive
-
-    This function returns true if and only if the JSON type is primitive
-    (string, number, boolean, or null).
-
-    @return `true` if type is primitive (string, number, boolean, or null),
-    `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_primitive()` for all JSON
-    types.,is_primitive}
-
-    @sa @ref is_structured() -- returns whether JSON value is structured
-    @sa @ref is_null() -- returns whether JSON value is `null`
-    @sa @ref is_string() -- returns whether JSON value is a string
-    @sa @ref is_boolean() -- returns whether JSON value is a boolean
-    @sa @ref is_number() -- returns whether JSON value is a number
-
-    @since version 1.0.0
-    */
-    constexpr bool is_primitive() const noexcept
-    {
-        return is_null() or is_string() or is_boolean() or is_number();
-    }
-
-    /*!
-    @brief return whether type is structured
-
-    This function returns true if and only if the JSON type is structured
-    (array or object).
-
-    @return `true` if type is structured (array or object), `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_structured()` for all JSON
-    types.,is_structured}
-
-    @sa @ref is_primitive() -- returns whether value is primitive
-    @sa @ref is_array() -- returns whether value is an array
-    @sa @ref is_object() -- returns whether value is an object
-
-    @since version 1.0.0
-    */
-    constexpr bool is_structured() const noexcept
-    {
-        return is_array() or is_object();
-    }
-
-    /*!
-    @brief return whether value is null
-
-    This function returns true if and only if the JSON value is null.
-
-    @return `true` if type is null, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_null()` for all JSON
-    types.,is_null}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_null() const noexcept
-    {
-        return (m_type == value_t::null);
-    }
-
-    /*!
-    @brief return whether value is a boolean
-
-    This function returns true if and only if the JSON value is a boolean.
-
-    @return `true` if type is boolean, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_boolean()` for all JSON
-    types.,is_boolean}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_boolean() const noexcept
-    {
-        return (m_type == value_t::boolean);
-    }
-
-    /*!
-    @brief return whether value is a number
-
-    This function returns true if and only if the JSON value is a number. This
-    includes both integer (signed and unsigned) and floating-point values.
-
-    @return `true` if type is number (regardless whether integer, unsigned
-    integer or floating-type), `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_number()` for all JSON
-    types.,is_number}
-
-    @sa @ref is_number_integer() -- check if value is an integer or unsigned
-    integer number
-    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
-    number
-    @sa @ref is_number_float() -- check if value is a floating-point number
-
-    @since version 1.0.0
-    */
-    constexpr bool is_number() const noexcept
-    {
-        return is_number_integer() or is_number_float();
-    }
-
-    /*!
-    @brief return whether value is an integer number
-
-    This function returns true if and only if the JSON value is a signed or
-    unsigned integer number. This excludes floating-point values.
-
-    @return `true` if type is an integer or unsigned integer number, `false`
-    otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_number_integer()` for all
-    JSON types.,is_number_integer}
-
-    @sa @ref is_number() -- check if value is a number
-    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
-    number
-    @sa @ref is_number_float() -- check if value is a floating-point number
-
-    @since version 1.0.0
-    */
-    constexpr bool is_number_integer() const noexcept
-    {
-        return (m_type == value_t::number_integer or m_type == value_t::number_unsigned);
-    }
-
-    /*!
-    @brief return whether value is an unsigned integer number
-
-    This function returns true if and only if the JSON value is an unsigned
-    integer number. This excludes floating-point and signed integer values.
-
-    @return `true` if type is an unsigned integer number, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_number_unsigned()` for all
-    JSON types.,is_number_unsigned}
-
-    @sa @ref is_number() -- check if value is a number
-    @sa @ref is_number_integer() -- check if value is an integer or unsigned
-    integer number
-    @sa @ref is_number_float() -- check if value is a floating-point number
-
-    @since version 2.0.0
-    */
-    constexpr bool is_number_unsigned() const noexcept
-    {
-        return (m_type == value_t::number_unsigned);
-    }
-
-    /*!
-    @brief return whether value is a floating-point number
-
-    This function returns true if and only if the JSON value is a
-    floating-point number. This excludes signed and unsigned integer values.
-
-    @return `true` if type is a floating-point number, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_number_float()` for all
-    JSON types.,is_number_float}
-
-    @sa @ref is_number() -- check if value is number
-    @sa @ref is_number_integer() -- check if value is an integer number
-    @sa @ref is_number_unsigned() -- check if value is an unsigned integer
-    number
-
-    @since version 1.0.0
-    */
-    constexpr bool is_number_float() const noexcept
-    {
-        return (m_type == value_t::number_float);
-    }
-
-    /*!
-    @brief return whether value is an object
-
-    This function returns true if and only if the JSON value is an object.
-
-    @return `true` if type is object, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_object()` for all JSON
-    types.,is_object}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_object() const noexcept
-    {
-        return (m_type == value_t::object);
-    }
-
-    /*!
-    @brief return whether value is an array
-
-    This function returns true if and only if the JSON value is an array.
-
-    @return `true` if type is array, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_array()` for all JSON
-    types.,is_array}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_array() const noexcept
-    {
-        return (m_type == value_t::array);
-    }
-
-    /*!
-    @brief return whether value is a string
-
-    This function returns true if and only if the JSON value is a string.
-
-    @return `true` if type is string, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_string()` for all JSON
-    types.,is_string}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_string() const noexcept
-    {
-        return (m_type == value_t::string);
-    }
-
-    /*!
-    @brief return whether value is discarded
-
-    This function returns true if and only if the JSON value was discarded
-    during parsing with a callback function (see @ref parser_callback_t).
-
-    @note This function will always be `false` for JSON values after parsing.
-    That is, discarded values can only occur during parsing, but will be
-    removed when inside a structured value or replaced by null in other cases.
-
-    @return `true` if type is discarded, `false` otherwise.
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies `is_discarded()` for all JSON
-    types.,is_discarded}
-
-    @since version 1.0.0
-    */
-    constexpr bool is_discarded() const noexcept
-    {
-        return (m_type == value_t::discarded);
-    }
-
-    /*!
-    @brief return the type of the JSON value (implicit)
-
-    Implicitly return the type of the JSON value as a value from the @ref
-    value_t enumeration.
-
-    @return the type of the JSON value
-
-    @complexity Constant.
-
-    @exceptionsafety No-throw guarantee: this member function never throws
-    exceptions.
-
-    @liveexample{The following code exemplifies the @ref value_t operator for
-    all JSON types.,operator__value_t}
-
-    @sa @ref type() -- return the type of the JSON value (explicit)
-    @sa @ref type_name() -- return the type as string
-
-    @since version 1.0.0
-    */
-    constexpr operator value_t() const noexcept
-    {
-        return m_type;
-    }
-
-    /// @}
-
-  private:
-    //////////////////
-    // value access //
-    //////////////////
-
-    /// get a boolean (explicit)
-    boolean_t get_impl(boolean_t* /*unused*/) const
-    {
-        if (JSON_LIKELY(is_boolean()))
-        {
-            return m_value.boolean;
-        }
-
-        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name())));
-    }
-
-    /// get a pointer to the value (object)
-    object_t* get_impl_ptr(object_t* /*unused*/) noexcept
-    {
-        return is_object() ? m_value.object : nullptr;
-    }
-
-    /// get a pointer to the value (object)
-    constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept
-    {
-        return is_object() ? m_value.object : nullptr;
-    }
-
-    /// get a pointer to the value (array)
-    array_t* get_impl_ptr(array_t* /*unused*/) noexcept
-    {
-        return is_array() ? m_value.array : nullptr;
-    }
-
-    /// get a pointer to the value (array)
-    constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept
-    {
-        return is_array() ? m_value.array : nullptr;
-    }
-
-    /// get a pointer to the value (string)
-    string_t* get_impl_ptr(string_t* /*unused*/) noexcept
-    {
-        return is_string() ? m_value.string : nullptr;
-    }
-
-    /// get a pointer to the value (string)
-    constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept
-    {
-        return is_string() ? m_value.string : nullptr;
-    }
-
-    /// get a pointer to the value (boolean)
-    boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept
-    {
-        return is_boolean() ? &m_value.boolean : nullptr;
-    }
-
-    /// get a pointer to the value (boolean)
-    constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept
-    {
-        return is_boolean() ? &m_value.boolean : nullptr;
-    }
-
-    /// get a pointer to the value (integer number)
-    number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept
-    {
-        return is_number_integer() ? &m_value.number_integer : nullptr;
-    }
-
-    /// get a pointer to the value (integer number)
-    constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept
-    {
-        return is_number_integer() ? &m_value.number_integer : nullptr;
-    }
-
-    /// get a pointer to the value (unsigned number)
-    number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept
-    {
-        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
-    }
-
-    /// get a pointer to the value (unsigned number)
-    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept
-    {
-        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
-    }
-
-    /// get a pointer to the value (floating-point number)
-    number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept
-    {
-        return is_number_float() ? &m_value.number_float : nullptr;
-    }
-
-    /// get a pointer to the value (floating-point number)
-    constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept
-    {
-        return is_number_float() ? &m_value.number_float : nullptr;
-    }
-
-    /*!
-    @brief helper function to implement get_ref()
-
-    This function helps to implement get_ref() without code duplication for
-    const and non-const overloads
-
-    @tparam ThisType will be deduced as `basic_json` or `const basic_json`
-
-    @throw type_error.303 if ReferenceType does not match underlying value
-    type of the current JSON
-    */
-    template<typename ReferenceType, typename ThisType>
-    static ReferenceType get_ref_impl(ThisType& obj)
-    {
-        // delegate the call to get_ptr<>()
-        auto ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();
-
-        if (JSON_LIKELY(ptr != nullptr))
-        {
-            return *ptr;
-        }
-
-        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name())));
-    }
-
-  public:
-    /// @name value access
-    /// Direct access to the stored value of a JSON value.
-    /// @{
-
-    /*!
-    @brief get special-case overload
-
-    This overloads avoids a lot of template boilerplate, it can be seen as the
-    identity method
-
-    @tparam BasicJsonType == @ref basic_json
-
-    @return a copy of *this
-
-    @complexity Constant.
-
-    @since version 2.1.0
-    */
-    template<typename BasicJsonType, detail::enable_if_t<
-                 std::is_same<typename std::remove_const<BasicJsonType>::type, basic_json_t>::value,
-                 int> = 0>
-    basic_json get() const
-    {
-        return *this;
-    }
-
-    /*!
-    @brief get special-case overload
-
-    This overloads converts the current @ref basic_json in a different
-    @ref basic_json type
-
-    @tparam BasicJsonType == @ref basic_json
-
-    @return a copy of *this, converted into @tparam BasicJsonType
-
-    @complexity Depending on the implementation of the called `from_json()`
-                method.
-
-    @since version 3.2.0
-    */
-    template<typename BasicJsonType, detail::enable_if_t<
-                 not std::is_same<BasicJsonType, basic_json>::value and
-                 detail::is_basic_json<BasicJsonType>::value, int> = 0>
-    BasicJsonType get() const
-    {
-        return *this;
-    }
-
-    /*!
-    @brief get a value (explicit)
-
-    Explicit type conversion between the JSON value and a compatible value
-    which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
-    and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
-    The value is converted by calling the @ref json_serializer<ValueType>
-    `from_json()` method.
-
-    The function is equivalent to executing
-    @code {.cpp}
-    ValueType ret;
-    JSONSerializer<ValueType>::from_json(*this, ret);
-    return ret;
-    @endcode
-
-    This overloads is chosen if:
-    - @a ValueType is not @ref basic_json,
-    - @ref json_serializer<ValueType> has a `from_json()` method of the form
-      `void from_json(const basic_json&, ValueType&)`, and
-    - @ref json_serializer<ValueType> does not have a `from_json()` method of
-      the form `ValueType from_json(const basic_json&)`
-
-    @tparam ValueTypeCV the provided value type
-    @tparam ValueType the returned value type
-
-    @return copy of the JSON value, converted to @a ValueType
-
-    @throw what @ref json_serializer<ValueType> `from_json()` method throws
-
-    @liveexample{The example below shows several conversions from JSON values
-    to other types. There a few things to note: (1) Floating-point numbers can
-    be converted to integers\, (2) A JSON array can be converted to a standard
-    `std::vector<short>`\, (3) A JSON object can be converted to C++
-    associative containers such as `std::unordered_map<std::string\,
-    json>`.,get__ValueType_const}
-
-    @since version 2.1.0
-    */
-    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
-             detail::enable_if_t <
-                 not detail::is_basic_json<ValueType>::value and
-                 detail::has_from_json<basic_json_t, ValueType>::value and
-                 not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
-                 int> = 0>
-    ValueType get() const noexcept(noexcept(
-                                       JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
-    {
-        // we cannot static_assert on ValueTypeCV being non-const, because
-        // there is support for get<const basic_json_t>(), which is why we
-        // still need the uncvref
-        static_assert(not std::is_reference<ValueTypeCV>::value,
-                      "get() cannot be used with reference types, you might want to use get_ref()");
-        static_assert(std::is_default_constructible<ValueType>::value,
-                      "types must be DefaultConstructible when used with get()");
-
-        ValueType ret;
-        JSONSerializer<ValueType>::from_json(*this, ret);
-        return ret;
-    }
-
-    /*!
-    @brief get a value (explicit); special case
-
-    Explicit type conversion between the JSON value and a compatible value
-    which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)
-    and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).
-    The value is converted by calling the @ref json_serializer<ValueType>
-    `from_json()` method.
-
-    The function is equivalent to executing
-    @code {.cpp}
-    return JSONSerializer<ValueTypeCV>::from_json(*this);
-    @endcode
-
-    This overloads is chosen if:
-    - @a ValueType is not @ref basic_json and
-    - @ref json_serializer<ValueType> has a `from_json()` method of the form
-      `ValueType from_json(const basic_json&)`
-
-    @note If @ref json_serializer<ValueType> has both overloads of
-    `from_json()`, this one is chosen.
-
-    @tparam ValueTypeCV the provided value type
-    @tparam ValueType the returned value type
-
-    @return copy of the JSON value, converted to @a ValueType
-
-    @throw what @ref json_serializer<ValueType> `from_json()` method throws
-
-    @since version 2.1.0
-    */
-    template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>,
-             detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
-                                 detail::has_non_default_from_json<basic_json_t, ValueType>::value,
-                                 int> = 0>
-    ValueType get() const noexcept(noexcept(
-                                       JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>())))
-    {
-        static_assert(not std::is_reference<ValueTypeCV>::value,
-                      "get() cannot be used with reference types, you might want to use get_ref()");
-        return JSONSerializer<ValueTypeCV>::from_json(*this);
-    }
-
-    /*!
-    @brief get a value (explicit)
-
-    Explicit type conversion between the JSON value and a compatible value.
-    The value is filled into the input parameter by calling the @ref json_serializer<ValueType>
-    `from_json()` method.
-
-    The function is equivalent to executing
-    @code {.cpp}
-    ValueType v;
-    JSONSerializer<ValueType>::from_json(*this, v);
-    @endcode
-
-    This overloads is chosen if:
-    - @a ValueType is not @ref basic_json,
-    - @ref json_serializer<ValueType> has a `from_json()` method of the form
-      `void from_json(const basic_json&, ValueType&)`, and
-
-    @tparam ValueType the input parameter type.
-
-    @return the input parameter, allowing chaining calls.
-
-    @throw what @ref json_serializer<ValueType> `from_json()` method throws
-
-    @liveexample{The example below shows several conversions from JSON values
-    to other types. There a few things to note: (1) Floating-point numbers can
-    be converted to integers\, (2) A JSON array can be converted to a standard
-    `std::vector<short>`\, (3) A JSON object can be converted to C++
-    associative containers such as `std::unordered_map<std::string\,
-    json>`.,get_to}
-
-    @since version 3.3.0
-    */
-    template<typename ValueType,
-             detail::enable_if_t <
-                 not detail::is_basic_json<ValueType>::value and
-                 detail::has_from_json<basic_json_t, ValueType>::value,
-                 int> = 0>
-    ValueType & get_to(ValueType& v) const noexcept(noexcept(
-                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))
-    {
-        JSONSerializer<ValueType>::from_json(*this, v);
-        return v;
-    }
-
-
-    /*!
-    @brief get a pointer value (implicit)
-
-    Implicit pointer access to the internally stored JSON value. No copies are
-    made.
-
-    @warning Writing data to the pointee of the result yields an undefined
-    state.
-
-    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
-    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t. Enforced by a static
-    assertion.
-
-    @return pointer to the internally stored JSON value if the requested
-    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how pointers to internal values of a
-    JSON value can be requested. Note that no type conversions are made and a
-    `nullptr` is returned if the value and the requested pointer type does not
-    match.,get_ptr}
-
-    @since version 1.0.0
-    */
-    template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value, int>::type = 0>
-    auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
-    {
-        // delegate the call to get_impl_ptr<>()
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
-    }
-
-    /*!
-    @brief get a pointer value (implicit)
-    @copydoc get_ptr()
-    */
-    template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value and
-                 std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0>
-    constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))
-    {
-        // delegate the call to get_impl_ptr<>() const
-        return get_impl_ptr(static_cast<PointerType>(nullptr));
-    }
-
-    /*!
-    @brief get a pointer value (explicit)
-
-    Explicit pointer access to the internally stored JSON value. No copies are
-    made.
-
-    @warning The pointer becomes invalid if the underlying JSON object
-    changes.
-
-    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref
-    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,
-    @ref number_unsigned_t, or @ref number_float_t.
-
-    @return pointer to the internally stored JSON value if the requested
-    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how pointers to internal values of a
-    JSON value can be requested. Note that no type conversions are made and a
-    `nullptr` is returned if the value and the requested pointer type does not
-    match.,get__PointerType}
-
-    @sa @ref get_ptr() for explicit pointer-member access
-
-    @since version 1.0.0
-    */
-    template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value, int>::type = 0>
-    auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())
-    {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
-    }
-
-    /*!
-    @brief get a pointer value (explicit)
-    @copydoc get()
-    */
-    template<typename PointerType, typename std::enable_if<
-                 std::is_pointer<PointerType>::value, int>::type = 0>
-    constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())
-    {
-        // delegate the call to get_ptr
-        return get_ptr<PointerType>();
-    }
-
-    /*!
-    @brief get a reference value (implicit)
-
-    Implicit reference access to the internally stored JSON value. No copies
-    are made.
-
-    @warning Writing data to the referee of the result yields an undefined
-    state.
-
-    @tparam ReferenceType reference type; must be a reference to @ref array_t,
-    @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or
-    @ref number_float_t. Enforced by static assertion.
-
-    @return reference to the internally stored JSON value if the requested
-    reference type @a ReferenceType fits to the JSON value; throws
-    type_error.303 otherwise
-
-    @throw type_error.303 in case passed type @a ReferenceType is incompatible
-    with the stored JSON value; see example below
-
-    @complexity Constant.
-
-    @liveexample{The example shows several calls to `get_ref()`.,get_ref}
-
-    @since version 1.1.0
-    */
-    template<typename ReferenceType, typename std::enable_if<
-                 std::is_reference<ReferenceType>::value, int>::type = 0>
-    ReferenceType get_ref()
-    {
-        // delegate call to get_ref_impl
-        return get_ref_impl<ReferenceType>(*this);
-    }
-
-    /*!
-    @brief get a reference value (implicit)
-    @copydoc get_ref()
-    */
-    template<typename ReferenceType, typename std::enable_if<
-                 std::is_reference<ReferenceType>::value and
-                 std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int>::type = 0>
-    ReferenceType get_ref() const
-    {
-        // delegate call to get_ref_impl
-        return get_ref_impl<ReferenceType>(*this);
-    }
-
-    /*!
-    @brief get a value (implicit)
-
-    Implicit type conversion between the JSON value and a compatible value.
-    The call is realized by calling @ref get() const.
-
-    @tparam ValueType non-pointer type compatible to the JSON value, for
-    instance `int` for JSON integer numbers, `bool` for JSON booleans, or
-    `std::vector` types for JSON arrays. The character type of @ref string_t
-    as well as an initializer list of this type is excluded to avoid
-    ambiguities as these types implicitly convert to `std::string`.
-
-    @return copy of the JSON value, converted to type @a ValueType
-
-    @throw type_error.302 in case passed type @a ValueType is incompatible
-    to the JSON value type (e.g., the JSON value is of type boolean, but a
-    string is requested); see example below
-
-    @complexity Linear in the size of the JSON value.
-
-    @liveexample{The example below shows several conversions from JSON values
-    to other types. There a few things to note: (1) Floating-point numbers can
-    be converted to integers\, (2) A JSON array can be converted to a standard
-    `std::vector<short>`\, (3) A JSON object can be converted to C++
-    associative containers such as `std::unordered_map<std::string\,
-    json>`.,operator__ValueType}
-
-    @since version 1.0.0
-    */
-    template < typename ValueType, typename std::enable_if <
-                   not std::is_pointer<ValueType>::value and
-                   not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
-                   not std::is_same<ValueType, typename string_t::value_type>::value and
-                   not detail::is_basic_json<ValueType>::value
-
-#ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
-                   and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
-#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914
-                   and not std::is_same<ValueType, typename std::string_view>::value
-#endif
-#endif
-                   and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value
-                   , int >::type = 0 >
-    operator ValueType() const
-    {
-        // delegate the call to get<>() const
-        return get<ValueType>();
-    }
-
-    /// @}
-
-
-    ////////////////////
-    // element access //
-    ////////////////////
-
-    /// @name element access
-    /// Access to the JSON value.
-    /// @{
-
-    /*!
-    @brief access specified array element with bounds checking
-
-    Returns a reference to the element at specified location @a idx, with
-    bounds checking.
-
-    @param[in] idx  index of the element to access
-
-    @return reference to the element at index @a idx
-
-    @throw type_error.304 if the JSON value is not an array; in this case,
-    calling `at` with an index makes no sense. See example below.
-    @throw out_of_range.401 if the index @a idx is out of range of the array;
-    that is, `idx >= size()`. See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @since version 1.0.0
-
-    @liveexample{The example below shows how array elements can be read and
-    written using `at()`. It also demonstrates the different exceptions that
-    can be thrown.,at__size_type}
-    */
-    reference at(size_type idx)
-    {
-        // at only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            JSON_TRY
-            {
-                return m_value.array->at(idx);
-            }
-            JSON_CATCH (std::out_of_range&)
-            {
-                // create better exception explanation
-                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
-            }
-        }
-        else
-        {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief access specified array element with bounds checking
-
-    Returns a const reference to the element at specified location @a idx,
-    with bounds checking.
-
-    @param[in] idx  index of the element to access
-
-    @return const reference to the element at index @a idx
-
-    @throw type_error.304 if the JSON value is not an array; in this case,
-    calling `at` with an index makes no sense. See example below.
-    @throw out_of_range.401 if the index @a idx is out of range of the array;
-    that is, `idx >= size()`. See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @since version 1.0.0
-
-    @liveexample{The example below shows how array elements can be read using
-    `at()`. It also demonstrates the different exceptions that can be thrown.,
-    at__size_type_const}
-    */
-    const_reference at(size_type idx) const
-    {
-        // at only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            JSON_TRY
-            {
-                return m_value.array->at(idx);
-            }
-            JSON_CATCH (std::out_of_range&)
-            {
-                // create better exception explanation
-                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
-            }
-        }
-        else
-        {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief access specified object element with bounds checking
-
-    Returns a reference to the element at with specified key @a key, with
-    bounds checking.
-
-    @param[in] key  key of the element to access
-
-    @return reference to the element at key @a key
-
-    @throw type_error.304 if the JSON value is not an object; in this case,
-    calling `at` with a key makes no sense. See example below.
-    @throw out_of_range.403 if the key @a key is is not stored in the object;
-    that is, `find(key) == end()`. See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Logarithmic in the size of the container.
-
-    @sa @ref operator[](const typename object_t::key_type&) for unchecked
-    access by reference
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-
-    @liveexample{The example below shows how object elements can be read and
-    written using `at()`. It also demonstrates the different exceptions that
-    can be thrown.,at__object_t_key_type}
-    */
-    reference at(const typename object_t::key_type& key)
-    {
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            JSON_TRY
-            {
-                return m_value.object->at(key);
-            }
-            JSON_CATCH (std::out_of_range&)
-            {
-                // create better exception explanation
-                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
-            }
-        }
-        else
-        {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief access specified object element with bounds checking
-
-    Returns a const reference to the element at with specified key @a key,
-    with bounds checking.
-
-    @param[in] key  key of the element to access
-
-    @return const reference to the element at key @a key
-
-    @throw type_error.304 if the JSON value is not an object; in this case,
-    calling `at` with a key makes no sense. See example below.
-    @throw out_of_range.403 if the key @a key is is not stored in the object;
-    that is, `find(key) == end()`. See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Logarithmic in the size of the container.
-
-    @sa @ref operator[](const typename object_t::key_type&) for unchecked
-    access by reference
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-
-    @liveexample{The example below shows how object elements can be read using
-    `at()`. It also demonstrates the different exceptions that can be thrown.,
-    at__object_t_key_type_const}
-    */
-    const_reference at(const typename object_t::key_type& key) const
-    {
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            JSON_TRY
-            {
-                return m_value.object->at(key);
-            }
-            JSON_CATCH (std::out_of_range&)
-            {
-                // create better exception explanation
-                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
-            }
-        }
-        else
-        {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief access specified array element
-
-    Returns a reference to the element at specified location @a idx.
-
-    @note If @a idx is beyond the range of the array (i.e., `idx >= size()`),
-    then the array is silently filled up with `null` values to make `idx` a
-    valid reference to the last stored element.
-
-    @param[in] idx  index of the element to access
-
-    @return reference to the element at index @a idx
-
-    @throw type_error.305 if the JSON value is not an array or null; in that
-    cases, using the [] operator with an index makes no sense.
-
-    @complexity Constant if @a idx is in the range of the array. Otherwise
-    linear in `idx - size()`.
-
-    @liveexample{The example below shows how array elements can be read and
-    written using `[]` operator. Note the addition of `null`
-    values.,operatorarray__size_type}
-
-    @since version 1.0.0
-    */
-    reference operator[](size_type idx)
-    {
-        // implicitly convert null value to an empty array
-        if (is_null())
-        {
-            m_type = value_t::array;
-            m_value.array = create<array_t>();
-            assert_invariant();
-        }
-
-        // operator[] only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            // fill up array with null values if given idx is outside range
-            if (idx >= m_value.array->size())
-            {
-                m_value.array->insert(m_value.array->end(),
-                                      idx - m_value.array->size() + 1,
-                                      basic_json());
-            }
-
-            return m_value.array->operator[](idx);
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief access specified array element
-
-    Returns a const reference to the element at specified location @a idx.
-
-    @param[in] idx  index of the element to access
-
-    @return const reference to the element at index @a idx
-
-    @throw type_error.305 if the JSON value is not an array; in that case,
-    using the [] operator with an index makes no sense.
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how array elements can be read using
-    the `[]` operator.,operatorarray__size_type_const}
-
-    @since version 1.0.0
-    */
-    const_reference operator[](size_type idx) const
-    {
-        // const operator[] only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            return m_value.array->operator[](idx);
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief access specified object element
-
-    Returns a reference to the element at with specified key @a key.
-
-    @note If @a key is not found in the object, then it is silently added to
-    the object and filled with a `null` value to make `key` a valid reference.
-    In case the value was `null` before, it is converted to an object.
-
-    @param[in] key  key of the element to access
-
-    @return reference to the element at key @a key
-
-    @throw type_error.305 if the JSON value is not an object or null; in that
-    cases, using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read and
-    written using the `[]` operator.,operatorarray__key_type}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-    */
-    reference operator[](const typename object_t::key_type& key)
-    {
-        // implicitly convert null value to an empty object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value.object = create<object_t>();
-            assert_invariant();
-        }
-
-        // operator[] only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            return m_value.object->operator[](key);
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief read-only access specified object element
-
-    Returns a const reference to the element at with specified key @a key. No
-    bounds checking is performed.
-
-    @warning If the element with key @a key does not exist, the behavior is
-    undefined.
-
-    @param[in] key  key of the element to access
-
-    @return const reference to the element at key @a key
-
-    @pre The element with key @a key must exist. **This precondition is
-         enforced with an assertion.**
-
-    @throw type_error.305 if the JSON value is not an object; in that case,
-    using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read using
-    the `[]` operator.,operatorarray__key_type_const}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-    */
-    const_reference operator[](const typename object_t::key_type& key) const
-    {
-        // const operator[] only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            assert(m_value.object->find(key) != m_value.object->end());
-            return m_value.object->find(key)->second;
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief access specified object element
-
-    Returns a reference to the element at with specified key @a key.
-
-    @note If @a key is not found in the object, then it is silently added to
-    the object and filled with a `null` value to make `key` a valid reference.
-    In case the value was `null` before, it is converted to an object.
-
-    @param[in] key  key of the element to access
-
-    @return reference to the element at key @a key
-
-    @throw type_error.305 if the JSON value is not an object or null; in that
-    cases, using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read and
-    written using the `[]` operator.,operatorarray__key_type}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.1.0
-    */
-    template<typename T>
-    reference operator[](T* key)
-    {
-        // implicitly convert null to object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value = value_t::object;
-            assert_invariant();
-        }
-
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            return m_value.object->operator[](key);
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief read-only access specified object element
-
-    Returns a const reference to the element at with specified key @a key. No
-    bounds checking is performed.
-
-    @warning If the element with key @a key does not exist, the behavior is
-    undefined.
-
-    @param[in] key  key of the element to access
-
-    @return const reference to the element at key @a key
-
-    @pre The element with key @a key must exist. **This precondition is
-         enforced with an assertion.**
-
-    @throw type_error.305 if the JSON value is not an object; in that case,
-    using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read using
-    the `[]` operator.,operatorarray__key_type_const}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.1.0
-    */
-    template<typename T>
-    const_reference operator[](T* key) const
-    {
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            assert(m_value.object->find(key) != m_value.object->end());
-            return m_value.object->find(key)->second;
-        }
-
-        JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief access specified object element with default value
-
-    Returns either a copy of an object's element at the specified key @a key
-    or a given default value if no element with key @a key exists.
-
-    The function is basically equivalent to executing
-    @code {.cpp}
-    try {
-        return at(key);
-    } catch(out_of_range) {
-        return default_value;
-    }
-    @endcode
-
-    @note Unlike @ref at(const typename object_t::key_type&), this function
-    does not throw if the given key @a key was not found.
-
-    @note Unlike @ref operator[](const typename object_t::key_type& key), this
-    function does not implicitly add an element to the position defined by @a
-    key. This function is furthermore also applicable to const objects.
-
-    @param[in] key  key of the element to access
-    @param[in] default_value  the value to return if @a key is not found
-
-    @tparam ValueType type compatible to JSON values, for instance `int` for
-    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
-    JSON arrays. Note the type of the expected value at @a key and the default
-    value @a default_value must be compatible.
-
-    @return copy of the element at key @a key or @a default_value if @a key
-    is not found
-
-    @throw type_error.306 if the JSON value is not an object; in that case,
-    using `value()` with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be queried
-    with a default value.,basic_json__value}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref operator[](const typename object_t::key_type&) for unchecked
-    access by reference
-
-    @since version 1.0.0
-    */
-    template<class ValueType, typename std::enable_if<
-                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
-    ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const
-    {
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            // if key is found, return value and given default value otherwise
-            const auto it = find(key);
-            if (it != end())
-            {
-                return *it;
-            }
-
-            return default_value;
-        }
-
-        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief overload for a default value of type const char*
-    @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const
-    */
-    string_t value(const typename object_t::key_type& key, const char* default_value) const
-    {
-        return value(key, string_t(default_value));
-    }
-
-    /*!
-    @brief access specified object element via JSON Pointer with default value
-
-    Returns either a copy of an object's element at the specified key @a key
-    or a given default value if no element with key @a key exists.
-
-    The function is basically equivalent to executing
-    @code {.cpp}
-    try {
-        return at(ptr);
-    } catch(out_of_range) {
-        return default_value;
-    }
-    @endcode
-
-    @note Unlike @ref at(const json_pointer&), this function does not throw
-    if the given key @a key was not found.
-
-    @param[in] ptr  a JSON pointer to the element to access
-    @param[in] default_value  the value to return if @a ptr found no value
-
-    @tparam ValueType type compatible to JSON values, for instance `int` for
-    JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for
-    JSON arrays. Note the type of the expected value at @a key and the default
-    value @a default_value must be compatible.
-
-    @return copy of the element at key @a key or @a default_value if @a key
-    is not found
-
-    @throw type_error.306 if the JSON value is not an object; in that case,
-    using `value()` with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be queried
-    with a default value.,basic_json__value_ptr}
-
-    @sa @ref operator[](const json_pointer&) for unchecked access by reference
-
-    @since version 2.0.2
-    */
-    template<class ValueType, typename std::enable_if<
-                 std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
-    ValueType value(const json_pointer& ptr, const ValueType& default_value) const
-    {
-        // at only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            // if pointer resolves a value, return it or use default value
-            JSON_TRY
-            {
-                return ptr.get_checked(this);
-            }
-            JSON_INTERNAL_CATCH (out_of_range&)
-            {
-                return default_value;
-            }
-        }
-
-        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief overload for a default value of type const char*
-    @copydoc basic_json::value(const json_pointer&, ValueType) const
-    */
-    string_t value(const json_pointer& ptr, const char* default_value) const
-    {
-        return value(ptr, string_t(default_value));
-    }
-
-    /*!
-    @brief access the first element
-
-    Returns a reference to the first element in the container. For a JSON
-    container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
-
-    @return In case of a structured type (array or object), a reference to the
-    first element is returned. In case of number, string, or boolean values, a
-    reference to the value is returned.
-
-    @complexity Constant.
-
-    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
-    or an empty array or object (undefined behavior, **guarded by
-    assertions**).
-    @post The JSON value remains unchanged.
-
-    @throw invalid_iterator.214 when called on `null` value
-
-    @liveexample{The following code shows an example for `front()`.,front}
-
-    @sa @ref back() -- access the last element
-
-    @since version 1.0.0
-    */
-    reference front()
-    {
-        return *begin();
-    }
-
-    /*!
-    @copydoc basic_json::front()
-    */
-    const_reference front() const
-    {
-        return *cbegin();
-    }
-
-    /*!
-    @brief access the last element
-
-    Returns a reference to the last element in the container. For a JSON
-    container `c`, the expression `c.back()` is equivalent to
-    @code {.cpp}
-    auto tmp = c.end();
-    --tmp;
-    return *tmp;
-    @endcode
-
-    @return In case of a structured type (array or object), a reference to the
-    last element is returned. In case of number, string, or boolean values, a
-    reference to the value is returned.
-
-    @complexity Constant.
-
-    @pre The JSON value must not be `null` (would throw `std::out_of_range`)
-    or an empty array or object (undefined behavior, **guarded by
-    assertions**).
-    @post The JSON value remains unchanged.
-
-    @throw invalid_iterator.214 when called on a `null` value. See example
-    below.
-
-    @liveexample{The following code shows an example for `back()`.,back}
-
-    @sa @ref front() -- access the first element
-
-    @since version 1.0.0
-    */
-    reference back()
-    {
-        auto tmp = end();
-        --tmp;
-        return *tmp;
-    }
-
-    /*!
-    @copydoc basic_json::back()
-    */
-    const_reference back() const
-    {
-        auto tmp = cend();
-        --tmp;
-        return *tmp;
-    }
-
-    /*!
-    @brief remove element given an iterator
-
-    Removes the element specified by iterator @a pos. The iterator @a pos must
-    be valid and dereferenceable. Thus the `end()` iterator (which is valid,
-    but is not dereferenceable) cannot be used as a value for @a pos.
-
-    If called on a primitive type other than `null`, the resulting JSON value
-    will be `null`.
-
-    @param[in] pos iterator to the element to remove
-    @return Iterator following the last removed element. If the iterator @a
-    pos refers to the last element, the `end()` iterator is returned.
-
-    @tparam IteratorType an @ref iterator or @ref const_iterator
-
-    @post Invalidates iterators and references at or after the point of the
-    erase, including the `end()` iterator.
-
-    @throw type_error.307 if called on a `null` value; example: `"cannot use
-    erase() with null"`
-    @throw invalid_iterator.202 if called on an iterator which does not belong
-    to the current JSON value; example: `"iterator does not fit current
-    value"`
-    @throw invalid_iterator.205 if called on a primitive type with invalid
-    iterator (i.e., any iterator which is not `begin()`); example: `"iterator
-    out of range"`
-
-    @complexity The complexity depends on the type:
-    - objects: amortized constant
-    - arrays: linear in distance between @a pos and the end of the container
-    - strings: linear in the length of the string
-    - other types: constant
-
-    @liveexample{The example shows the result of `erase()` for different JSON
-    types.,erase__IteratorType}
-
-    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
-    the given range
-    @sa @ref erase(const typename object_t::key_type&) -- removes the element
-    from an object at the given key
-    @sa @ref erase(const size_type) -- removes the element from an array at
-    the given index
-
-    @since version 1.0.0
-    */
-    template<class IteratorType, typename std::enable_if<
-                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
-                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
-             = 0>
-    IteratorType erase(IteratorType pos)
-    {
-        // make sure iterator fits the current value
-        if (JSON_UNLIKELY(this != pos.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
-        }
-
-        IteratorType result = end();
-
-        switch (m_type)
-        {
-            case value_t::boolean:
-            case value_t::number_float:
-            case value_t::number_integer:
-            case value_t::number_unsigned:
-            case value_t::string:
-            {
-                if (JSON_UNLIKELY(not pos.m_it.primitive_iterator.is_begin()))
-                {
-                    JSON_THROW(invalid_iterator::create(205, "iterator out of range"));
-                }
-
-                if (is_string())
-                {
-                    AllocatorType<string_t> alloc;
-                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
-                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
-                    m_value.string = nullptr;
-                }
-
-                m_type = value_t::null;
-                assert_invariant();
-                break;
-            }
-
-            case value_t::object:
-            {
-                result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator);
-                break;
-            }
-
-            case value_t::array:
-            {
-                result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator);
-                break;
-            }
-
-            default:
-                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief remove elements given an iterator range
-
-    Removes the element specified by the range `[first; last)`. The iterator
-    @a first does not need to be dereferenceable if `first == last`: erasing
-    an empty range is a no-op.
-
-    If called on a primitive type other than `null`, the resulting JSON value
-    will be `null`.
-
-    @param[in] first iterator to the beginning of the range to remove
-    @param[in] last iterator past the end of the range to remove
-    @return Iterator following the last removed element. If the iterator @a
-    second refers to the last element, the `end()` iterator is returned.
-
-    @tparam IteratorType an @ref iterator or @ref const_iterator
-
-    @post Invalidates iterators and references at or after the point of the
-    erase, including the `end()` iterator.
-
-    @throw type_error.307 if called on a `null` value; example: `"cannot use
-    erase() with null"`
-    @throw invalid_iterator.203 if called on iterators which does not belong
-    to the current JSON value; example: `"iterators do not fit current value"`
-    @throw invalid_iterator.204 if called on a primitive type with invalid
-    iterators (i.e., if `first != begin()` and `last != end()`); example:
-    `"iterators out of range"`
-
-    @complexity The complexity depends on the type:
-    - objects: `log(size()) + std::distance(first, last)`
-    - arrays: linear in the distance between @a first and @a last, plus linear
-      in the distance between @a last and end of the container
-    - strings: linear in the length of the string
-    - other types: constant
-
-    @liveexample{The example shows the result of `erase()` for different JSON
-    types.,erase__IteratorType_IteratorType}
-
-    @sa @ref erase(IteratorType) -- removes the element at a given position
-    @sa @ref erase(const typename object_t::key_type&) -- removes the element
-    from an object at the given key
-    @sa @ref erase(const size_type) -- removes the element from an array at
-    the given index
-
-    @since version 1.0.0
-    */
-    template<class IteratorType, typename std::enable_if<
-                 std::is_same<IteratorType, typename basic_json_t::iterator>::value or
-                 std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type
-             = 0>
-    IteratorType erase(IteratorType first, IteratorType last)
-    {
-        // make sure iterator fits the current value
-        if (JSON_UNLIKELY(this != first.m_object or this != last.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value"));
-        }
-
-        IteratorType result = end();
-
-        switch (m_type)
-        {
-            case value_t::boolean:
-            case value_t::number_float:
-            case value_t::number_integer:
-            case value_t::number_unsigned:
-            case value_t::string:
-            {
-                if (JSON_LIKELY(not first.m_it.primitive_iterator.is_begin()
-                                or not last.m_it.primitive_iterator.is_end()))
-                {
-                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
-                }
-
-                if (is_string())
-                {
-                    AllocatorType<string_t> alloc;
-                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);
-                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);
-                    m_value.string = nullptr;
-                }
-
-                m_type = value_t::null;
-                assert_invariant();
-                break;
-            }
-
-            case value_t::object:
-            {
-                result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator,
-                                              last.m_it.object_iterator);
-                break;
-            }
-
-            case value_t::array:
-            {
-                result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator,
-                                             last.m_it.array_iterator);
-                break;
-            }
-
-            default:
-                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief remove element from a JSON object given a key
-
-    Removes elements from a JSON object with the key value @a key.
-
-    @param[in] key value of the elements to remove
-
-    @return Number of elements removed. If @a ObjectType is the default
-    `std::map` type, the return value will always be `0` (@a key was not
-    found) or `1` (@a key was found).
-
-    @post References and iterators to the erased elements are invalidated.
-    Other references and iterators are not affected.
-
-    @throw type_error.307 when called on a type other than JSON object;
-    example: `"cannot use erase() with null"`
-
-    @complexity `log(size()) + count(key)`
-
-    @liveexample{The example shows the effect of `erase()`.,erase__key_type}
-
-    @sa @ref erase(IteratorType) -- removes the element at a given position
-    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
-    the given range
-    @sa @ref erase(const size_type) -- removes the element from an array at
-    the given index
-
-    @since version 1.0.0
-    */
-    size_type erase(const typename object_t::key_type& key)
-    {
-        // this erase only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            return m_value.object->erase(key);
-        }
-
-        JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief remove element from a JSON array given an index
-
-    Removes element from a JSON array at the index @a idx.
-
-    @param[in] idx index of the element to remove
-
-    @throw type_error.307 when called on a type other than JSON object;
-    example: `"cannot use erase() with null"`
-    @throw out_of_range.401 when `idx >= size()`; example: `"array index 17
-    is out of range"`
-
-    @complexity Linear in distance between @a idx and the end of the container.
-
-    @liveexample{The example shows the effect of `erase()`.,erase__size_type}
-
-    @sa @ref erase(IteratorType) -- removes the element at a given position
-    @sa @ref erase(IteratorType, IteratorType) -- removes the elements in
-    the given range
-    @sa @ref erase(const typename object_t::key_type&) -- removes the element
-    from an object at the given key
-
-    @since version 1.0.0
-    */
-    void erase(const size_type idx)
-    {
-        // this erase only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            if (JSON_UNLIKELY(idx >= size()))
-            {
-                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
-            }
-
-            m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));
-        }
-        else
-        {
-            JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
-        }
-    }
-
-    /// @}
-
-
-    ////////////
-    // lookup //
-    ////////////
-
-    /// @name lookup
-    /// @{
-
-    /*!
-    @brief find an element in a JSON object
-
-    Finds an element in a JSON object with key equivalent to @a key. If the
-    element is not found or the JSON value is not an object, end() is
-    returned.
-
-    @note This method always returns @ref end() when executed on a JSON type
-          that is not an object.
-
-    @param[in] key key value of the element to search for.
-
-    @return Iterator to an element with key equivalent to @a key. If no such
-    element is found or the JSON value is not an object, past-the-end (see
-    @ref end()) iterator is returned.
-
-    @complexity Logarithmic in the size of the JSON object.
-
-    @liveexample{The example shows how `find()` is used.,find__key_type}
-
-    @since version 1.0.0
-    */
-    template<typename KeyT>
-    iterator find(KeyT&& key)
-    {
-        auto result = end();
-
-        if (is_object())
-        {
-            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief find an element in a JSON object
-    @copydoc find(KeyT&&)
-    */
-    template<typename KeyT>
-    const_iterator find(KeyT&& key) const
-    {
-        auto result = cend();
-
-        if (is_object())
-        {
-            result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key));
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief returns the number of occurrences of a key in a JSON object
-
-    Returns the number of elements with key @a key. If ObjectType is the
-    default `std::map` type, the return value will always be `0` (@a key was
-    not found) or `1` (@a key was found).
-
-    @note This method always returns `0` when executed on a JSON type that is
-          not an object.
-
-    @param[in] key key value of the element to count
-
-    @return Number of elements with key @a key. If the JSON value is not an
-    object, the return value will be `0`.
-
-    @complexity Logarithmic in the size of the JSON object.
-
-    @liveexample{The example shows how `count()` is used.,count}
-
-    @since version 1.0.0
-    */
-    template<typename KeyT>
-    size_type count(KeyT&& key) const
-    {
-        // return 0 for all nonobject types
-        return is_object() ? m_value.object->count(std::forward<KeyT>(key)) : 0;
-    }
-
-    /// @}
-
-
-    ///////////////
-    // iterators //
-    ///////////////
-
-    /// @name iterators
-    /// @{
-
-    /*!
-    @brief returns an iterator to the first element
-
-    Returns an iterator to the first element.
-
-    @image html range-begin-end.svg "Illustration from cppreference.com"
-
-    @return iterator to the first element
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-
-    @liveexample{The following code shows an example for `begin()`.,begin}
-
-    @sa @ref cbegin() -- returns a const iterator to the beginning
-    @sa @ref end() -- returns an iterator to the end
-    @sa @ref cend() -- returns a const iterator to the end
-
-    @since version 1.0.0
-    */
-    iterator begin() noexcept
-    {
-        iterator result(this);
-        result.set_begin();
-        return result;
-    }
-
-    /*!
-    @copydoc basic_json::cbegin()
-    */
-    const_iterator begin() const noexcept
-    {
-        return cbegin();
-    }
-
-    /*!
-    @brief returns a const iterator to the first element
-
-    Returns a const iterator to the first element.
-
-    @image html range-begin-end.svg "Illustration from cppreference.com"
-
-    @return const iterator to the first element
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `const_cast<const basic_json&>(*this).begin()`.
-
-    @liveexample{The following code shows an example for `cbegin()`.,cbegin}
-
-    @sa @ref begin() -- returns an iterator to the beginning
-    @sa @ref end() -- returns an iterator to the end
-    @sa @ref cend() -- returns a const iterator to the end
-
-    @since version 1.0.0
-    */
-    const_iterator cbegin() const noexcept
-    {
-        const_iterator result(this);
-        result.set_begin();
-        return result;
-    }
-
-    /*!
-    @brief returns an iterator to one past the last element
-
-    Returns an iterator to one past the last element.
-
-    @image html range-begin-end.svg "Illustration from cppreference.com"
-
-    @return iterator one past the last element
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-
-    @liveexample{The following code shows an example for `end()`.,end}
-
-    @sa @ref cend() -- returns a const iterator to the end
-    @sa @ref begin() -- returns an iterator to the beginning
-    @sa @ref cbegin() -- returns a const iterator to the beginning
-
-    @since version 1.0.0
-    */
-    iterator end() noexcept
-    {
-        iterator result(this);
-        result.set_end();
-        return result;
-    }
-
-    /*!
-    @copydoc basic_json::cend()
-    */
-    const_iterator end() const noexcept
-    {
-        return cend();
-    }
-
-    /*!
-    @brief returns a const iterator to one past the last element
-
-    Returns a const iterator to one past the last element.
-
-    @image html range-begin-end.svg "Illustration from cppreference.com"
-
-    @return const iterator one past the last element
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `const_cast<const basic_json&>(*this).end()`.
-
-    @liveexample{The following code shows an example for `cend()`.,cend}
-
-    @sa @ref end() -- returns an iterator to the end
-    @sa @ref begin() -- returns an iterator to the beginning
-    @sa @ref cbegin() -- returns a const iterator to the beginning
-
-    @since version 1.0.0
-    */
-    const_iterator cend() const noexcept
-    {
-        const_iterator result(this);
-        result.set_end();
-        return result;
-    }
-
-    /*!
-    @brief returns an iterator to the reverse-beginning
-
-    Returns an iterator to the reverse-beginning; that is, the last element.
-
-    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `reverse_iterator(end())`.
-
-    @liveexample{The following code shows an example for `rbegin()`.,rbegin}
-
-    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
-    @sa @ref rend() -- returns a reverse iterator to the end
-    @sa @ref crend() -- returns a const reverse iterator to the end
-
-    @since version 1.0.0
-    */
-    reverse_iterator rbegin() noexcept
-    {
-        return reverse_iterator(end());
-    }
-
-    /*!
-    @copydoc basic_json::crbegin()
-    */
-    const_reverse_iterator rbegin() const noexcept
-    {
-        return crbegin();
-    }
-
-    /*!
-    @brief returns an iterator to the reverse-end
-
-    Returns an iterator to the reverse-end; that is, one before the first
-    element.
-
-    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `reverse_iterator(begin())`.
-
-    @liveexample{The following code shows an example for `rend()`.,rend}
-
-    @sa @ref crend() -- returns a const reverse iterator to the end
-    @sa @ref rbegin() -- returns a reverse iterator to the beginning
-    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
-
-    @since version 1.0.0
-    */
-    reverse_iterator rend() noexcept
-    {
-        return reverse_iterator(begin());
-    }
-
-    /*!
-    @copydoc basic_json::crend()
-    */
-    const_reverse_iterator rend() const noexcept
-    {
-        return crend();
-    }
-
-    /*!
-    @brief returns a const reverse iterator to the last element
-
-    Returns a const iterator to the reverse-beginning; that is, the last
-    element.
-
-    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`.
-
-    @liveexample{The following code shows an example for `crbegin()`.,crbegin}
-
-    @sa @ref rbegin() -- returns a reverse iterator to the beginning
-    @sa @ref rend() -- returns a reverse iterator to the end
-    @sa @ref crend() -- returns a const reverse iterator to the end
-
-    @since version 1.0.0
-    */
-    const_reverse_iterator crbegin() const noexcept
-    {
-        return const_reverse_iterator(cend());
-    }
-
-    /*!
-    @brief returns a const reverse iterator to one before the first
-
-    Returns a const reverse iterator to the reverse-end; that is, one before
-    the first element.
-
-    @image html range-rbegin-rend.svg "Illustration from cppreference.com"
-
-    @complexity Constant.
-
-    @requirement This function helps `basic_json` satisfying the
-    [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `const_cast<const basic_json&>(*this).rend()`.
-
-    @liveexample{The following code shows an example for `crend()`.,crend}
-
-    @sa @ref rend() -- returns a reverse iterator to the end
-    @sa @ref rbegin() -- returns a reverse iterator to the beginning
-    @sa @ref crbegin() -- returns a const reverse iterator to the beginning
-
-    @since version 1.0.0
-    */
-    const_reverse_iterator crend() const noexcept
-    {
-        return const_reverse_iterator(cbegin());
-    }
-
-  public:
-    /*!
-    @brief wrapper to access iterator member functions in range-based for
-
-    This function allows to access @ref iterator::key() and @ref
-    iterator::value() during range-based for loops. In these loops, a
-    reference to the JSON values is returned, so there is no access to the
-    underlying iterator.
-
-    For loop without iterator_wrapper:
-
-    @code{cpp}
-    for (auto it = j_object.begin(); it != j_object.end(); ++it)
-    {
-        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
-    }
-    @endcode
-
-    Range-based for loop without iterator proxy:
-
-    @code{cpp}
-    for (auto it : j_object)
-    {
-        // "it" is of type json::reference and has no key() member
-        std::cout << "value: " << it << '\n';
-    }
-    @endcode
-
-    Range-based for loop with iterator proxy:
-
-    @code{cpp}
-    for (auto it : json::iterator_wrapper(j_object))
-    {
-        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
-    }
-    @endcode
-
-    @note When iterating over an array, `key()` will return the index of the
-          element as string (see example).
-
-    @param[in] ref  reference to a JSON value
-    @return iteration proxy object wrapping @a ref with an interface to use in
-            range-based for loops
-
-    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @note The name of this function is not yet final and may change in the
-    future.
-
-    @deprecated This stream operator is deprecated and will be removed in
-                future 4.0.0 of the library. Please use @ref items() instead;
-                that is, replace `json::iterator_wrapper(j)` with `j.items()`.
-    */
-    JSON_DEPRECATED
-    static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept
-    {
-        return ref.items();
-    }
-
-    /*!
-    @copydoc iterator_wrapper(reference)
-    */
-    JSON_DEPRECATED
-    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept
-    {
-        return ref.items();
-    }
-
-    /*!
-    @brief helper to access iterator member functions in range-based for
-
-    This function allows to access @ref iterator::key() and @ref
-    iterator::value() during range-based for loops. In these loops, a
-    reference to the JSON values is returned, so there is no access to the
-    underlying iterator.
-
-    For loop without `items()` function:
-
-    @code{cpp}
-    for (auto it = j_object.begin(); it != j_object.end(); ++it)
-    {
-        std::cout << "key: " << it.key() << ", value:" << it.value() << '\n';
-    }
-    @endcode
-
-    Range-based for loop without `items()` function:
-
-    @code{cpp}
-    for (auto it : j_object)
-    {
-        // "it" is of type json::reference and has no key() member
-        std::cout << "value: " << it << '\n';
-    }
-    @endcode
-
-    Range-based for loop with `items()` function:
-
-    @code{cpp}
-    for (auto& el : j_object.items())
-    {
-        std::cout << "key: " << el.key() << ", value:" << el.value() << '\n';
-    }
-    @endcode
-
-    The `items()` function also allows to use
-    [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding)
-    (C++17):
-
-    @code{cpp}
-    for (auto& [key, val] : j_object.items())
-    {
-        std::cout << "key: " << key << ", value:" << val << '\n';
-    }
-    @endcode
-
-    @note When iterating over an array, `key()` will return the index of the
-          element as string (see example). For primitive types (e.g., numbers),
-          `key()` returns an empty string.
-
-    @return iteration proxy object wrapping @a ref with an interface to use in
-            range-based for loops
-
-    @liveexample{The following code shows how the function is used.,items}
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @since version 3.1.0, structured bindings support since 3.5.0.
-    */
-    iteration_proxy<iterator> items() noexcept
-    {
-        return iteration_proxy<iterator>(*this);
-    }
-
-    /*!
-    @copydoc items()
-    */
-    iteration_proxy<const_iterator> items() const noexcept
-    {
-        return iteration_proxy<const_iterator>(*this);
-    }
-
-    /// @}
-
-
-    //////////////
-    // capacity //
-    //////////////
-
-    /// @name capacity
-    /// @{
-
-    /*!
-    @brief checks whether the container is empty.
-
-    Checks if a JSON value has no elements (i.e. whether its @ref size is `0`).
-
-    @return The return value depends on the different types and is
-            defined as follows:
-            Value type  | return value
-            ----------- | -------------
-            null        | `true`
-            boolean     | `false`
-            string      | `false`
-            number      | `false`
-            object      | result of function `object_t::empty()`
-            array       | result of function `array_t::empty()`
-
-    @liveexample{The following code uses `empty()` to check if a JSON
-    object contains any elements.,empty}
-
-    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
-    the Container concept; that is, their `empty()` functions have constant
-    complexity.
-
-    @iterators No changes.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @note This function does not return whether a string stored as JSON value
-    is empty - it returns whether the JSON container itself is empty which is
-    false in the case of a string.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `begin() == end()`.
-
-    @sa @ref size() -- returns the number of elements
-
-    @since version 1.0.0
-    */
-    bool empty() const noexcept
-    {
-        switch (m_type)
-        {
-            case value_t::null:
-            {
-                // null values are empty
-                return true;
-            }
-
-            case value_t::array:
-            {
-                // delegate call to array_t::empty()
-                return m_value.array->empty();
-            }
-
-            case value_t::object:
-            {
-                // delegate call to object_t::empty()
-                return m_value.object->empty();
-            }
-
-            default:
-            {
-                // all other types are nonempty
-                return false;
-            }
-        }
-    }
-
-    /*!
-    @brief returns the number of elements
-
-    Returns the number of elements in a JSON value.
-
-    @return The return value depends on the different types and is
-            defined as follows:
-            Value type  | return value
-            ----------- | -------------
-            null        | `0`
-            boolean     | `1`
-            string      | `1`
-            number      | `1`
-            object      | result of function object_t::size()
-            array       | result of function array_t::size()
-
-    @liveexample{The following code calls `size()` on the different value
-    types.,size}
-
-    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
-    the Container concept; that is, their size() functions have constant
-    complexity.
-
-    @iterators No changes.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @note This function does not return the length of a string stored as JSON
-    value - it returns the number of elements in the JSON value which is 1 in
-    the case of a string.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of `std::distance(begin(), end())`.
-
-    @sa @ref empty() -- checks whether the container is empty
-    @sa @ref max_size() -- returns the maximal number of elements
-
-    @since version 1.0.0
-    */
-    size_type size() const noexcept
-    {
-        switch (m_type)
-        {
-            case value_t::null:
-            {
-                // null values are empty
-                return 0;
-            }
-
-            case value_t::array:
-            {
-                // delegate call to array_t::size()
-                return m_value.array->size();
-            }
-
-            case value_t::object:
-            {
-                // delegate call to object_t::size()
-                return m_value.object->size();
-            }
-
-            default:
-            {
-                // all other types have size 1
-                return 1;
-            }
-        }
-    }
-
-    /*!
-    @brief returns the maximum possible number of elements
-
-    Returns the maximum number of elements a JSON value is able to hold due to
-    system or library implementation limitations, i.e. `std::distance(begin(),
-    end())` for the JSON value.
-
-    @return The return value depends on the different types and is
-            defined as follows:
-            Value type  | return value
-            ----------- | -------------
-            null        | `0` (same as `size()`)
-            boolean     | `1` (same as `size()`)
-            string      | `1` (same as `size()`)
-            number      | `1` (same as `size()`)
-            object      | result of function `object_t::max_size()`
-            array       | result of function `array_t::max_size()`
-
-    @liveexample{The following code calls `max_size()` on the different value
-    types. Note the output is implementation specific.,max_size}
-
-    @complexity Constant, as long as @ref array_t and @ref object_t satisfy
-    the Container concept; that is, their `max_size()` functions have constant
-    complexity.
-
-    @iterators No changes.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @requirement This function helps `basic_json` satisfying the
-    [Container](https://en.cppreference.com/w/cpp/named_req/Container)
-    requirements:
-    - The complexity is constant.
-    - Has the semantics of returning `b.size()` where `b` is the largest
-      possible JSON value.
-
-    @sa @ref size() -- returns the number of elements
-
-    @since version 1.0.0
-    */
-    size_type max_size() const noexcept
-    {
-        switch (m_type)
-        {
-            case value_t::array:
-            {
-                // delegate call to array_t::max_size()
-                return m_value.array->max_size();
-            }
-
-            case value_t::object:
-            {
-                // delegate call to object_t::max_size()
-                return m_value.object->max_size();
-            }
-
-            default:
-            {
-                // all other types have max_size() == size()
-                return size();
-            }
-        }
-    }
-
-    /// @}
-
-
-    ///////////////
-    // modifiers //
-    ///////////////
-
-    /// @name modifiers
-    /// @{
-
-    /*!
-    @brief clears the contents
-
-    Clears the content of a JSON value and resets it to the default value as
-    if @ref basic_json(value_t) would have been called with the current value
-    type from @ref type():
-
-    Value type  | initial value
-    ----------- | -------------
-    null        | `null`
-    boolean     | `false`
-    string      | `""`
-    number      | `0`
-    object      | `{}`
-    array       | `[]`
-
-    @post Has the same effect as calling
-    @code {.cpp}
-    *this = basic_json(type());
-    @endcode
-
-    @liveexample{The example below shows the effect of `clear()` to different
-    JSON types.,clear}
-
-    @complexity Linear in the size of the JSON value.
-
-    @iterators All iterators, pointers and references related to this container
-               are invalidated.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @sa @ref basic_json(value_t) -- constructor that creates an object with the
-        same value than calling `clear()`
-
-    @since version 1.0.0
-    */
-    void clear() noexcept
-    {
-        switch (m_type)
-        {
-            case value_t::number_integer:
-            {
-                m_value.number_integer = 0;
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                m_value.number_unsigned = 0;
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                m_value.number_float = 0.0;
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                m_value.boolean = false;
-                break;
-            }
-
-            case value_t::string:
-            {
-                m_value.string->clear();
-                break;
-            }
-
-            case value_t::array:
-            {
-                m_value.array->clear();
-                break;
-            }
-
-            case value_t::object:
-            {
-                m_value.object->clear();
-                break;
-            }
-
-            default:
-                break;
-        }
-    }
-
-    /*!
-    @brief add an object to an array
-
-    Appends the given element @a val to the end of the JSON value. If the
-    function is called on a JSON null value, an empty array is created before
-    appending @a val.
-
-    @param[in] val the value to add to the JSON array
-
-    @throw type_error.308 when called on a type other than JSON array or
-    null; example: `"cannot use push_back() with number"`
-
-    @complexity Amortized constant.
-
-    @liveexample{The example shows how `push_back()` and `+=` can be used to
-    add elements to a JSON array. Note how the `null` value was silently
-    converted to a JSON array.,push_back}
-
-    @since version 1.0.0
-    */
-    void push_back(basic_json&& val)
-    {
-        // push_back only works for null objects or arrays
-        if (JSON_UNLIKELY(not(is_null() or is_array())))
-        {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
-        }
-
-        // transform null object into an array
-        if (is_null())
-        {
-            m_type = value_t::array;
-            m_value = value_t::array;
-            assert_invariant();
-        }
-
-        // add element to array (move semantics)
-        m_value.array->push_back(std::move(val));
-        // invalidate object
-        val.m_type = value_t::null;
-    }
-
-    /*!
-    @brief add an object to an array
-    @copydoc push_back(basic_json&&)
-    */
-    reference operator+=(basic_json&& val)
-    {
-        push_back(std::move(val));
-        return *this;
-    }
-
-    /*!
-    @brief add an object to an array
-    @copydoc push_back(basic_json&&)
-    */
-    void push_back(const basic_json& val)
-    {
-        // push_back only works for null objects or arrays
-        if (JSON_UNLIKELY(not(is_null() or is_array())))
-        {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
-        }
-
-        // transform null object into an array
-        if (is_null())
-        {
-            m_type = value_t::array;
-            m_value = value_t::array;
-            assert_invariant();
-        }
-
-        // add element to array
-        m_value.array->push_back(val);
-    }
-
-    /*!
-    @brief add an object to an array
-    @copydoc push_back(basic_json&&)
-    */
-    reference operator+=(const basic_json& val)
-    {
-        push_back(val);
-        return *this;
-    }
-
-    /*!
-    @brief add an object to an object
-
-    Inserts the given element @a val to the JSON object. If the function is
-    called on a JSON null value, an empty object is created before inserting
-    @a val.
-
-    @param[in] val the value to add to the JSON object
-
-    @throw type_error.308 when called on a type other than JSON object or
-    null; example: `"cannot use push_back() with number"`
-
-    @complexity Logarithmic in the size of the container, O(log(`size()`)).
-
-    @liveexample{The example shows how `push_back()` and `+=` can be used to
-    add elements to a JSON object. Note how the `null` value was silently
-    converted to a JSON object.,push_back__object_t__value}
-
-    @since version 1.0.0
-    */
-    void push_back(const typename object_t::value_type& val)
-    {
-        // push_back only works for null objects or objects
-        if (JSON_UNLIKELY(not(is_null() or is_object())))
-        {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
-        }
-
-        // transform null object into an object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value = value_t::object;
-            assert_invariant();
-        }
-
-        // add element to array
-        m_value.object->insert(val);
-    }
-
-    /*!
-    @brief add an object to an object
-    @copydoc push_back(const typename object_t::value_type&)
-    */
-    reference operator+=(const typename object_t::value_type& val)
-    {
-        push_back(val);
-        return *this;
-    }
-
-    /*!
-    @brief add an object to an object
-
-    This function allows to use `push_back` with an initializer list. In case
-
-    1. the current value is an object,
-    2. the initializer list @a init contains only two elements, and
-    3. the first element of @a init is a string,
-
-    @a init is converted into an object element and added using
-    @ref push_back(const typename object_t::value_type&). Otherwise, @a init
-    is converted to a JSON value and added using @ref push_back(basic_json&&).
-
-    @param[in] init  an initializer list
-
-    @complexity Linear in the size of the initializer list @a init.
-
-    @note This function is required to resolve an ambiguous overload error,
-          because pairs like `{"key", "value"}` can be both interpreted as
-          `object_t::value_type` or `std::initializer_list<basic_json>`, see
-          https://github.com/nlohmann/json/issues/235 for more information.
-
-    @liveexample{The example shows how initializer lists are treated as
-    objects when possible.,push_back__initializer_list}
-    */
-    void push_back(initializer_list_t init)
-    {
-        if (is_object() and init.size() == 2 and (*init.begin())->is_string())
-        {
-            basic_json&& key = init.begin()->moved_or_copied();
-            push_back(typename object_t::value_type(
-                          std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));
-        }
-        else
-        {
-            push_back(basic_json(init));
-        }
-    }
-
-    /*!
-    @brief add an object to an object
-    @copydoc push_back(initializer_list_t)
-    */
-    reference operator+=(initializer_list_t init)
-    {
-        push_back(init);
-        return *this;
-    }
-
-    /*!
-    @brief add an object to an array
-
-    Creates a JSON value from the passed parameters @a args to the end of the
-    JSON value. If the function is called on a JSON null value, an empty array
-    is created before appending the value created from @a args.
-
-    @param[in] args arguments to forward to a constructor of @ref basic_json
-    @tparam Args compatible types to create a @ref basic_json object
-
-    @throw type_error.311 when called on a type other than JSON array or
-    null; example: `"cannot use emplace_back() with number"`
-
-    @complexity Amortized constant.
-
-    @liveexample{The example shows how `push_back()` can be used to add
-    elements to a JSON array. Note how the `null` value was silently converted
-    to a JSON array.,emplace_back}
-
-    @since version 2.0.8
-    */
-    template<class... Args>
-    void emplace_back(Args&& ... args)
-    {
-        // emplace_back only works for null objects or arrays
-        if (JSON_UNLIKELY(not(is_null() or is_array())))
-        {
-            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name())));
-        }
-
-        // transform null object into an array
-        if (is_null())
-        {
-            m_type = value_t::array;
-            m_value = value_t::array;
-            assert_invariant();
-        }
-
-        // add element to array (perfect forwarding)
-        m_value.array->emplace_back(std::forward<Args>(args)...);
-    }
-
-    /*!
-    @brief add an object to an object if key does not exist
-
-    Inserts a new element into a JSON object constructed in-place with the
-    given @a args if there is no element with the key in the container. If the
-    function is called on a JSON null value, an empty object is created before
-    appending the value created from @a args.
-
-    @param[in] args arguments to forward to a constructor of @ref basic_json
-    @tparam Args compatible types to create a @ref basic_json object
-
-    @return a pair consisting of an iterator to the inserted element, or the
-            already-existing element if no insertion happened, and a bool
-            denoting whether the insertion took place.
-
-    @throw type_error.311 when called on a type other than JSON object or
-    null; example: `"cannot use emplace() with number"`
-
-    @complexity Logarithmic in the size of the container, O(log(`size()`)).
-
-    @liveexample{The example shows how `emplace()` can be used to add elements
-    to a JSON object. Note how the `null` value was silently converted to a
-    JSON object. Further note how no value is added if there was already one
-    value stored with the same key.,emplace}
-
-    @since version 2.0.8
-    */
-    template<class... Args>
-    std::pair<iterator, bool> emplace(Args&& ... args)
-    {
-        // emplace only works for null objects or arrays
-        if (JSON_UNLIKELY(not(is_null() or is_object())))
-        {
-            JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name())));
-        }
-
-        // transform null object into an object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value = value_t::object;
-            assert_invariant();
-        }
-
-        // add element to array (perfect forwarding)
-        auto res = m_value.object->emplace(std::forward<Args>(args)...);
-        // create result iterator and set iterator to the result of emplace
-        auto it = begin();
-        it.m_it.object_iterator = res.first;
-
-        // return pair of iterator and boolean
-        return {it, res.second};
-    }
-
-    /// Helper for insertion of an iterator
-    /// @note: This uses std::distance to support GCC 4.8,
-    ///        see https://github.com/nlohmann/json/pull/1257
-    template<typename... Args>
-    iterator insert_iterator(const_iterator pos, Args&& ... args)
-    {
-        iterator result(this);
-        assert(m_value.array != nullptr);
-
-        auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);
-        m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);
-        result.m_it.array_iterator = m_value.array->begin() + insert_pos;
-
-        // This could have been written as:
-        // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
-        // but the return value of insert is missing in GCC 4.8, so it is written this way instead.
-
-        return result;
-    }
-
-    /*!
-    @brief inserts element
-
-    Inserts element @a val before iterator @a pos.
-
-    @param[in] pos iterator before which the content will be inserted; may be
-    the end() iterator
-    @param[in] val element to insert
-    @return iterator pointing to the inserted @a val.
-
-    @throw type_error.309 if called on JSON values other than arrays;
-    example: `"cannot use insert() with string"`
-    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
-    example: `"iterator does not fit current value"`
-
-    @complexity Constant plus linear in the distance between @a pos and end of
-    the container.
-
-    @liveexample{The example shows how `insert()` is used.,insert}
-
-    @since version 1.0.0
-    */
-    iterator insert(const_iterator pos, const basic_json& val)
-    {
-        // insert only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            // check if iterator pos fits to this JSON value
-            if (JSON_UNLIKELY(pos.m_object != this))
-            {
-                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
-            }
-
-            // insert to array and return iterator
-            return insert_iterator(pos, val);
-        }
-
-        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief inserts element
-    @copydoc insert(const_iterator, const basic_json&)
-    */
-    iterator insert(const_iterator pos, basic_json&& val)
-    {
-        return insert(pos, val);
-    }
-
-    /*!
-    @brief inserts elements
-
-    Inserts @a cnt copies of @a val before iterator @a pos.
-
-    @param[in] pos iterator before which the content will be inserted; may be
-    the end() iterator
-    @param[in] cnt number of copies of @a val to insert
-    @param[in] val element to insert
-    @return iterator pointing to the first element inserted, or @a pos if
-    `cnt==0`
-
-    @throw type_error.309 if called on JSON values other than arrays; example:
-    `"cannot use insert() with string"`
-    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
-    example: `"iterator does not fit current value"`
-
-    @complexity Linear in @a cnt plus linear in the distance between @a pos
-    and end of the container.
-
-    @liveexample{The example shows how `insert()` is used.,insert__count}
-
-    @since version 1.0.0
-    */
-    iterator insert(const_iterator pos, size_type cnt, const basic_json& val)
-    {
-        // insert only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            // check if iterator pos fits to this JSON value
-            if (JSON_UNLIKELY(pos.m_object != this))
-            {
-                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
-            }
-
-            // insert to array and return iterator
-            return insert_iterator(pos, cnt, val);
-        }
-
-        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
-    }
-
-    /*!
-    @brief inserts elements
-
-    Inserts elements from range `[first, last)` before iterator @a pos.
-
-    @param[in] pos iterator before which the content will be inserted; may be
-    the end() iterator
-    @param[in] first begin of the range of elements to insert
-    @param[in] last end of the range of elements to insert
-
-    @throw type_error.309 if called on JSON values other than arrays; example:
-    `"cannot use insert() with string"`
-    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
-    example: `"iterator does not fit current value"`
-    @throw invalid_iterator.210 if @a first and @a last do not belong to the
-    same JSON value; example: `"iterators do not fit"`
-    @throw invalid_iterator.211 if @a first or @a last are iterators into
-    container for which insert is called; example: `"passed iterators may not
-    belong to container"`
-
-    @return iterator pointing to the first element inserted, or @a pos if
-    `first==last`
-
-    @complexity Linear in `std::distance(first, last)` plus linear in the
-    distance between @a pos and end of the container.
-
-    @liveexample{The example shows how `insert()` is used.,insert__range}
-
-    @since version 1.0.0
-    */
-    iterator insert(const_iterator pos, const_iterator first, const_iterator last)
-    {
-        // insert only works for arrays
-        if (JSON_UNLIKELY(not is_array()))
-        {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
-        }
-
-        // check if iterator pos fits to this JSON value
-        if (JSON_UNLIKELY(pos.m_object != this))
-        {
-            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
-        }
-
-        // check if range iterators belong to the same JSON object
-        if (JSON_UNLIKELY(first.m_object != last.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
-        }
-
-        if (JSON_UNLIKELY(first.m_object == this))
-        {
-            JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container"));
-        }
-
-        // insert to array and return iterator
-        return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
-    }
-
-    /*!
-    @brief inserts elements
-
-    Inserts elements from initializer list @a ilist before iterator @a pos.
-
-    @param[in] pos iterator before which the content will be inserted; may be
-    the end() iterator
-    @param[in] ilist initializer list to insert the values from
-
-    @throw type_error.309 if called on JSON values other than arrays; example:
-    `"cannot use insert() with string"`
-    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
-    example: `"iterator does not fit current value"`
-
-    @return iterator pointing to the first element inserted, or @a pos if
-    `ilist` is empty
-
-    @complexity Linear in `ilist.size()` plus linear in the distance between
-    @a pos and end of the container.
-
-    @liveexample{The example shows how `insert()` is used.,insert__ilist}
-
-    @since version 1.0.0
-    */
-    iterator insert(const_iterator pos, initializer_list_t ilist)
-    {
-        // insert only works for arrays
-        if (JSON_UNLIKELY(not is_array()))
-        {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
-        }
-
-        // check if iterator pos fits to this JSON value
-        if (JSON_UNLIKELY(pos.m_object != this))
-        {
-            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
-        }
-
-        // insert to array and return iterator
-        return insert_iterator(pos, ilist.begin(), ilist.end());
-    }
-
-    /*!
-    @brief inserts elements
-
-    Inserts elements from range `[first, last)`.
-
-    @param[in] first begin of the range of elements to insert
-    @param[in] last end of the range of elements to insert
-
-    @throw type_error.309 if called on JSON values other than objects; example:
-    `"cannot use insert() with string"`
-    @throw invalid_iterator.202 if iterator @a first or @a last does does not
-    point to an object; example: `"iterators first and last must point to
-    objects"`
-    @throw invalid_iterator.210 if @a first and @a last do not belong to the
-    same JSON value; example: `"iterators do not fit"`
-
-    @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number
-    of elements to insert.
-
-    @liveexample{The example shows how `insert()` is used.,insert__range_object}
-
-    @since version 3.0.0
-    */
-    void insert(const_iterator first, const_iterator last)
-    {
-        // insert only works for objects
-        if (JSON_UNLIKELY(not is_object()))
-        {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
-        }
-
-        // check if range iterators belong to the same JSON object
-        if (JSON_UNLIKELY(first.m_object != last.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
-        }
-
-        // passed iterators must belong to objects
-        if (JSON_UNLIKELY(not first.m_object->is_object()))
-        {
-            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
-        }
-
-        m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
-    }
-
-    /*!
-    @brief updates a JSON object from another object, overwriting existing keys
-
-    Inserts all values from JSON object @a j and overwrites existing keys.
-
-    @param[in] j  JSON object to read values from
-
-    @throw type_error.312 if called on JSON values other than objects; example:
-    `"cannot use update() with string"`
-
-    @complexity O(N*log(size() + N)), where N is the number of elements to
-                insert.
-
-    @liveexample{The example shows how `update()` is used.,update}
-
-    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
-
-    @since version 3.0.0
-    */
-    void update(const_reference j)
-    {
-        // implicitly convert null value to an empty object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value.object = create<object_t>();
-            assert_invariant();
-        }
-
-        if (JSON_UNLIKELY(not is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
-        }
-        if (JSON_UNLIKELY(not j.is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name())));
-        }
-
-        for (auto it = j.cbegin(); it != j.cend(); ++it)
-        {
-            m_value.object->operator[](it.key()) = it.value();
-        }
-    }
-
-    /*!
-    @brief updates a JSON object from another object, overwriting existing keys
-
-    Inserts all values from from range `[first, last)` and overwrites existing
-    keys.
-
-    @param[in] first begin of the range of elements to insert
-    @param[in] last end of the range of elements to insert
-
-    @throw type_error.312 if called on JSON values other than objects; example:
-    `"cannot use update() with string"`
-    @throw invalid_iterator.202 if iterator @a first or @a last does does not
-    point to an object; example: `"iterators first and last must point to
-    objects"`
-    @throw invalid_iterator.210 if @a first and @a last do not belong to the
-    same JSON value; example: `"iterators do not fit"`
-
-    @complexity O(N*log(size() + N)), where N is the number of elements to
-                insert.
-
-    @liveexample{The example shows how `update()` is used__range.,update}
-
-    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
-
-    @since version 3.0.0
-    */
-    void update(const_iterator first, const_iterator last)
-    {
-        // implicitly convert null value to an empty object
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value.object = create<object_t>();
-            assert_invariant();
-        }
-
-        if (JSON_UNLIKELY(not is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
-        }
-
-        // check if range iterators belong to the same JSON object
-        if (JSON_UNLIKELY(first.m_object != last.m_object))
-        {
-            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
-        }
-
-        // passed iterators must belong to objects
-        if (JSON_UNLIKELY(not first.m_object->is_object()
-                          or not last.m_object->is_object()))
-        {
-            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
-        }
-
-        for (auto it = first; it != last; ++it)
-        {
-            m_value.object->operator[](it.key()) = it.value();
-        }
-    }
-
-    /*!
-    @brief exchanges the values
-
-    Exchanges the contents of the JSON value with those of @a other. Does not
-    invoke any move, copy, or swap operations on individual elements. All
-    iterators and references remain valid. The past-the-end iterator is
-    invalidated.
-
-    @param[in,out] other JSON value to exchange the contents with
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how JSON values can be swapped with
-    `swap()`.,swap__reference}
-
-    @since version 1.0.0
-    */
-    void swap(reference other) noexcept (
-        std::is_nothrow_move_constructible<value_t>::value and
-        std::is_nothrow_move_assignable<value_t>::value and
-        std::is_nothrow_move_constructible<json_value>::value and
-        std::is_nothrow_move_assignable<json_value>::value
-    )
-    {
-        std::swap(m_type, other.m_type);
-        std::swap(m_value, other.m_value);
-        assert_invariant();
-    }
-
-    /*!
-    @brief exchanges the values
-
-    Exchanges the contents of a JSON array with those of @a other. Does not
-    invoke any move, copy, or swap operations on individual elements. All
-    iterators and references remain valid. The past-the-end iterator is
-    invalidated.
-
-    @param[in,out] other array to exchange the contents with
-
-    @throw type_error.310 when JSON value is not an array; example: `"cannot
-    use swap() with string"`
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how arrays can be swapped with
-    `swap()`.,swap__array_t}
-
-    @since version 1.0.0
-    */
-    void swap(array_t& other)
-    {
-        // swap only works for arrays
-        if (JSON_LIKELY(is_array()))
-        {
-            std::swap(*(m_value.array), other);
-        }
-        else
-        {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief exchanges the values
-
-    Exchanges the contents of a JSON object with those of @a other. Does not
-    invoke any move, copy, or swap operations on individual elements. All
-    iterators and references remain valid. The past-the-end iterator is
-    invalidated.
-
-    @param[in,out] other object to exchange the contents with
-
-    @throw type_error.310 when JSON value is not an object; example:
-    `"cannot use swap() with string"`
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how objects can be swapped with
-    `swap()`.,swap__object_t}
-
-    @since version 1.0.0
-    */
-    void swap(object_t& other)
-    {
-        // swap only works for objects
-        if (JSON_LIKELY(is_object()))
-        {
-            std::swap(*(m_value.object), other);
-        }
-        else
-        {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
-        }
-    }
-
-    /*!
-    @brief exchanges the values
-
-    Exchanges the contents of a JSON string with those of @a other. Does not
-    invoke any move, copy, or swap operations on individual elements. All
-    iterators and references remain valid. The past-the-end iterator is
-    invalidated.
-
-    @param[in,out] other string to exchange the contents with
-
-    @throw type_error.310 when JSON value is not a string; example: `"cannot
-    use swap() with boolean"`
-
-    @complexity Constant.
-
-    @liveexample{The example below shows how strings can be swapped with
-    `swap()`.,swap__string_t}
-
-    @since version 1.0.0
-    */
-    void swap(string_t& other)
-    {
-        // swap only works for strings
-        if (JSON_LIKELY(is_string()))
-        {
-            std::swap(*(m_value.string), other);
-        }
-        else
-        {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
-        }
-    }
-
-    /// @}
-
-  public:
-    //////////////////////////////////////////
-    // lexicographical comparison operators //
-    //////////////////////////////////////////
-
-    /// @name lexicographical comparison operators
-    /// @{
-
-    /*!
-    @brief comparison: equal
-
-    Compares two JSON values for equality according to the following rules:
-    - Two JSON values are equal if (1) they are from the same type and (2)
-      their stored values are the same according to their respective
-      `operator==`.
-    - Integer and floating-point numbers are automatically converted before
-      comparison. Note than two NaN values are always treated as unequal.
-    - Two JSON null values are equal.
-
-    @note Floating-point inside JSON values numbers are compared with
-    `json::number_float_t::operator==` which is `double::operator==` by
-    default. To compare floating-point while respecting an epsilon, an alternative
-    [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39)
-    could be used, for instance
-    @code {.cpp}
-    template<typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
-    inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept
-    {
-        return std::abs(a - b) <= epsilon;
-    }
-    @endcode
-
-    @note NaN values never compare equal to themselves or to other NaN values.
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether the values @a lhs and @a rhs are equal
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @complexity Linear.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__equal}
-
-    @since version 1.0.0
-    */
-    friend bool operator==(const_reference lhs, const_reference rhs) noexcept
-    {
-        const auto lhs_type = lhs.type();
-        const auto rhs_type = rhs.type();
-
-        if (lhs_type == rhs_type)
-        {
-            switch (lhs_type)
-            {
-                case value_t::array:
-                    return (*lhs.m_value.array == *rhs.m_value.array);
-
-                case value_t::object:
-                    return (*lhs.m_value.object == *rhs.m_value.object);
-
-                case value_t::null:
-                    return true;
-
-                case value_t::string:
-                    return (*lhs.m_value.string == *rhs.m_value.string);
-
-                case value_t::boolean:
-                    return (lhs.m_value.boolean == rhs.m_value.boolean);
-
-                case value_t::number_integer:
-                    return (lhs.m_value.number_integer == rhs.m_value.number_integer);
-
-                case value_t::number_unsigned:
-                    return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned);
-
-                case value_t::number_float:
-                    return (lhs.m_value.number_float == rhs.m_value.number_float);
-
-                default:
-                    return false;
-            }
-        }
-        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
-        {
-            return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float);
-        }
-        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
-        {
-            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer));
-        }
-        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
-        {
-            return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float);
-        }
-        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
-        {
-            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned));
-        }
-        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
-        {
-            return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer);
-        }
-        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
-        {
-            return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned));
-        }
-
-        return false;
-    }
-
-    /*!
-    @brief comparison: equal
-    @copydoc operator==(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs == basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: equal
-    @copydoc operator==(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) == rhs);
-    }
-
-    /*!
-    @brief comparison: not equal
-
-    Compares two JSON values for inequality by calculating `not (lhs == rhs)`.
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether the values @a lhs and @a rhs are not equal
-
-    @complexity Linear.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__notequal}
-
-    @since version 1.0.0
-    */
-    friend bool operator!=(const_reference lhs, const_reference rhs) noexcept
-    {
-        return not (lhs == rhs);
-    }
-
-    /*!
-    @brief comparison: not equal
-    @copydoc operator!=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs != basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: not equal
-    @copydoc operator!=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) != rhs);
-    }
-
-    /*!
-    @brief comparison: less than
-
-    Compares whether one JSON value @a lhs is less than another JSON value @a
-    rhs according to the following rules:
-    - If @a lhs and @a rhs have the same type, the values are compared using
-      the default `<` operator.
-    - Integer and floating-point numbers are automatically converted before
-      comparison
-    - In case @a lhs and @a rhs have different types, the values are ignored
-      and the order of the types is considered, see
-      @ref operator<(const value_t, const value_t).
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether @a lhs is less than @a rhs
-
-    @complexity Linear.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__less}
-
-    @since version 1.0.0
-    */
-    friend bool operator<(const_reference lhs, const_reference rhs) noexcept
-    {
-        const auto lhs_type = lhs.type();
-        const auto rhs_type = rhs.type();
-
-        if (lhs_type == rhs_type)
-        {
-            switch (lhs_type)
-            {
-                case value_t::array:
-                    return (*lhs.m_value.array) < (*rhs.m_value.array);
-
-                case value_t::object:
-                    return *lhs.m_value.object < *rhs.m_value.object;
-
-                case value_t::null:
-                    return false;
-
-                case value_t::string:
-                    return *lhs.m_value.string < *rhs.m_value.string;
-
-                case value_t::boolean:
-                    return lhs.m_value.boolean < rhs.m_value.boolean;
-
-                case value_t::number_integer:
-                    return lhs.m_value.number_integer < rhs.m_value.number_integer;
-
-                case value_t::number_unsigned:
-                    return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned;
-
-                case value_t::number_float:
-                    return lhs.m_value.number_float < rhs.m_value.number_float;
-
-                default:
-                    return false;
-            }
-        }
-        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
-        {
-            return static_cast<number_float_t>(lhs.m_value.number_integer) < rhs.m_value.number_float;
-        }
-        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
-        {
-            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_integer);
-        }
-        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
-        {
-            return static_cast<number_float_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_float;
-        }
-        else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
-        {
-            return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_unsigned);
-        }
-        else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
-        {
-            return lhs.m_value.number_integer < static_cast<number_integer_t>(rhs.m_value.number_unsigned);
-        }
-        else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
-        {
-            return static_cast<number_integer_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_integer;
-        }
-
-        // We only reach this line if we cannot compare values. In that case,
-        // we compare types. Note we have to call the operator explicitly,
-        // because MSVC has problems otherwise.
-        return operator<(lhs_type, rhs_type);
-    }
-
-    /*!
-    @brief comparison: less than
-    @copydoc operator<(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs < basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: less than
-    @copydoc operator<(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) < rhs);
-    }
-
-    /*!
-    @brief comparison: less than or equal
-
-    Compares whether one JSON value @a lhs is less than or equal to another
-    JSON value by calculating `not (rhs < lhs)`.
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether @a lhs is less than or equal to @a rhs
-
-    @complexity Linear.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__greater}
-
-    @since version 1.0.0
-    */
-    friend bool operator<=(const_reference lhs, const_reference rhs) noexcept
-    {
-        return not (rhs < lhs);
-    }
-
-    /*!
-    @brief comparison: less than or equal
-    @copydoc operator<=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs <= basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: less than or equal
-    @copydoc operator<=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) <= rhs);
-    }
-
-    /*!
-    @brief comparison: greater than
-
-    Compares whether one JSON value @a lhs is greater than another
-    JSON value by calculating `not (lhs <= rhs)`.
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether @a lhs is greater than to @a rhs
-
-    @complexity Linear.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__lessequal}
-
-    @since version 1.0.0
-    */
-    friend bool operator>(const_reference lhs, const_reference rhs) noexcept
-    {
-        return not (lhs <= rhs);
-    }
-
-    /*!
-    @brief comparison: greater than
-    @copydoc operator>(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs > basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: greater than
-    @copydoc operator>(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) > rhs);
-    }
-
-    /*!
-    @brief comparison: greater than or equal
-
-    Compares whether one JSON value @a lhs is greater than or equal to another
-    JSON value by calculating `not (lhs < rhs)`.
-
-    @param[in] lhs  first JSON value to consider
-    @param[in] rhs  second JSON value to consider
-    @return whether @a lhs is greater than or equal to @a rhs
-
-    @complexity Linear.
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @liveexample{The example demonstrates comparing several JSON
-    types.,operator__greaterequal}
-
-    @since version 1.0.0
-    */
-    friend bool operator>=(const_reference lhs, const_reference rhs) noexcept
-    {
-        return not (lhs < rhs);
-    }
-
-    /*!
-    @brief comparison: greater than or equal
-    @copydoc operator>=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept
-    {
-        return (lhs >= basic_json(rhs));
-    }
-
-    /*!
-    @brief comparison: greater than or equal
-    @copydoc operator>=(const_reference, const_reference)
-    */
-    template<typename ScalarType, typename std::enable_if<
-                 std::is_scalar<ScalarType>::value, int>::type = 0>
-    friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept
-    {
-        return (basic_json(lhs) >= rhs);
-    }
-
-    /// @}
-
-    ///////////////////
-    // serialization //
-    ///////////////////
-
-    /// @name serialization
-    /// @{
-
-    /*!
-    @brief serialize to stream
-
-    Serialize the given JSON value @a j to the output stream @a o. The JSON
-    value will be serialized using the @ref dump member function.
-
-    - The indentation of the output can be controlled with the member variable
-      `width` of the output stream @a o. For instance, using the manipulator
-      `std::setw(4)` on @a o sets the indentation level to `4` and the
-      serialization result is the same as calling `dump(4)`.
-
-    - The indentation character can be controlled with the member variable
-      `fill` of the output stream @a o. For instance, the manipulator
-      `std::setfill('\\t')` sets indentation to use a tab character rather than
-      the default space character.
-
-    @param[in,out] o  stream to serialize to
-    @param[in] j  JSON value to serialize
-
-    @return the stream @a o
-
-    @throw type_error.316 if a string stored inside the JSON value is not
-                          UTF-8 encoded
-
-    @complexity Linear.
-
-    @liveexample{The example below shows the serialization with different
-    parameters to `width` to adjust the indentation level.,operator_serialize}
-
-    @since version 1.0.0; indentation character added in version 3.0.0
-    */
-    friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
-    {
-        // read width member and use it as indentation parameter if nonzero
-        const bool pretty_print = (o.width() > 0);
-        const auto indentation = (pretty_print ? o.width() : 0);
-
-        // reset width to 0 for subsequent calls to this stream
-        o.width(0);
-
-        // do the actual serialization
-        serializer s(detail::output_adapter<char>(o), o.fill());
-        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
-        return o;
-    }
-
-    /*!
-    @brief serialize to stream
-    @deprecated This stream operator is deprecated and will be removed in
-                future 4.0.0 of the library. Please use
-                @ref operator<<(std::ostream&, const basic_json&)
-                instead; that is, replace calls like `j >> o;` with `o << j;`.
-    @since version 1.0.0; deprecated since version 3.0.0
-    */
-    JSON_DEPRECATED
-    friend std::ostream& operator>>(const basic_json& j, std::ostream& o)
-    {
-        return o << j;
-    }
-
-    /// @}
-
-
-    /////////////////////
-    // deserialization //
-    /////////////////////
-
-    /// @name deserialization
-    /// @{
-
-    /*!
-    @brief deserialize from a compatible input
-
-    This function reads from a compatible input. Examples are:
-    - an array of 1-byte values
-    - strings with character/literal type with size of 1 byte
-    - input streams
-    - container with contiguous storage of 1-byte values. Compatible container
-      types include `std::vector`, `std::string`, `std::array`,
-      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
-      arrays can be used with `std::begin()`/`std::end()`. User-defined
-      containers can be used as long as they implement random-access iterators
-      and a contiguous storage.
-
-    @pre Each element of the container has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @pre The container storage is contiguous. Violating this precondition
-    yields undefined behavior. **This precondition is enforced with an
-    assertion.**
-    @pre Each element of the container has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @warning There is no way to enforce all preconditions at compile-time. If
-             the function is called with a noncompliant container and with
-             assertions switched off, the behavior is undefined and will most
-             likely yield segmentation violation.
-
-    @param[in] i  input to read from
-    @param[in] cb  a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return result of the deserialization
-
-    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
-    of input; expected string literal""`
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @liveexample{The example below demonstrates the `parse()` function reading
-    from an array.,parse__array__parser_callback_t}
-
-    @liveexample{The example below demonstrates the `parse()` function with
-    and without callback function.,parse__string__parser_callback_t}
-
-    @liveexample{The example below demonstrates the `parse()` function with
-    and without callback function.,parse__istream__parser_callback_t}
-
-    @liveexample{The example below demonstrates the `parse()` function reading
-    from a contiguous container.,parse__contiguouscontainer__parser_callback_t}
-
-    @since version 2.0.3 (contiguous containers)
-    */
-    static basic_json parse(detail::input_adapter&& i,
-                            const parser_callback_t cb = nullptr,
-                            const bool allow_exceptions = true)
-    {
-        basic_json result;
-        parser(i, cb, allow_exceptions).parse(true, result);
-        return result;
-    }
-
-    static bool accept(detail::input_adapter&& i)
-    {
-        return parser(i).accept(true);
-    }
-
-    /*!
-    @brief generate SAX events
-
-    The SAX event lister must follow the interface of @ref json_sax.
-
-    This function reads from a compatible input. Examples are:
-    - an array of 1-byte values
-    - strings with character/literal type with size of 1 byte
-    - input streams
-    - container with contiguous storage of 1-byte values. Compatible container
-      types include `std::vector`, `std::string`, `std::array`,
-      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
-      arrays can be used with `std::begin()`/`std::end()`. User-defined
-      containers can be used as long as they implement random-access iterators
-      and a contiguous storage.
-
-    @pre Each element of the container has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @pre The container storage is contiguous. Violating this precondition
-    yields undefined behavior. **This precondition is enforced with an
-    assertion.**
-    @pre Each element of the container has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @warning There is no way to enforce all preconditions at compile-time. If
-             the function is called with a noncompliant container and with
-             assertions switched off, the behavior is undefined and will most
-             likely yield segmentation violation.
-
-    @param[in] i  input to read from
-    @param[in,out] sax  SAX event listener
-    @param[in] format  the format to parse (JSON, CBOR, MessagePack, or UBJSON)
-    @param[in] strict  whether the input has to be consumed completely
-
-    @return return value of the last processed SAX event
-
-    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
-    of input; expected string literal""`
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the SAX consumer @a sax has
-    a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @liveexample{The example below demonstrates the `sax_parse()` function
-    reading from string and processing the events with a user-defined SAX
-    event consumer.,sax_parse}
-
-    @since version 3.2.0
-    */
-    template <typename SAX>
-    static bool sax_parse(detail::input_adapter&& i, SAX* sax,
-                          input_format_t format = input_format_t::json,
-                          const bool strict = true)
-    {
-        assert(sax);
-        switch (format)
-        {
-            case input_format_t::json:
-                return parser(std::move(i)).sax_parse(sax, strict);
-            default:
-                return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict);
-        }
-    }
-
-    /*!
-    @brief deserialize from an iterator range with contiguous storage
-
-    This function reads from an iterator range of a container with contiguous
-    storage of 1-byte values. Compatible container types include
-    `std::vector`, `std::string`, `std::array`, `std::valarray`, and
-    `std::initializer_list`. Furthermore, C-style arrays can be used with
-    `std::begin()`/`std::end()`. User-defined containers can be used as long
-    as they implement random-access iterators and a contiguous storage.
-
-    @pre The iterator range is contiguous. Violating this precondition yields
-    undefined behavior. **This precondition is enforced with an assertion.**
-    @pre Each element in the range has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @warning There is no way to enforce all preconditions at compile-time. If
-             the function is called with noncompliant iterators and with
-             assertions switched off, the behavior is undefined and will most
-             likely yield segmentation violation.
-
-    @tparam IteratorType iterator of container with contiguous storage
-    @param[in] first  begin of the range to parse (included)
-    @param[in] last  end of the range to parse (excluded)
-    @param[in] cb  a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return result of the deserialization
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @liveexample{The example below demonstrates the `parse()` function reading
-    from an iterator range.,parse__iteratortype__parser_callback_t}
-
-    @since version 2.0.3
-    */
-    template<class IteratorType, typename std::enable_if<
-                 std::is_base_of<
-                     std::random_access_iterator_tag,
-                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
-    static basic_json parse(IteratorType first, IteratorType last,
-                            const parser_callback_t cb = nullptr,
-                            const bool allow_exceptions = true)
-    {
-        basic_json result;
-        parser(detail::input_adapter(first, last), cb, allow_exceptions).parse(true, result);
-        return result;
-    }
-
-    template<class IteratorType, typename std::enable_if<
-                 std::is_base_of<
-                     std::random_access_iterator_tag,
-                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
-    static bool accept(IteratorType first, IteratorType last)
-    {
-        return parser(detail::input_adapter(first, last)).accept(true);
-    }
-
-    template<class IteratorType, class SAX, typename std::enable_if<
-                 std::is_base_of<
-                     std::random_access_iterator_tag,
-                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
-    static bool sax_parse(IteratorType first, IteratorType last, SAX* sax)
-    {
-        return parser(detail::input_adapter(first, last)).sax_parse(sax);
-    }
-
-    /*!
-    @brief deserialize from stream
-    @deprecated This stream operator is deprecated and will be removed in
-                version 4.0.0 of the library. Please use
-                @ref operator>>(std::istream&, basic_json&)
-                instead; that is, replace calls like `j << i;` with `i >> j;`.
-    @since version 1.0.0; deprecated since version 3.0.0
-    */
-    JSON_DEPRECATED
-    friend std::istream& operator<<(basic_json& j, std::istream& i)
-    {
-        return operator>>(i, j);
-    }
-
-    /*!
-    @brief deserialize from stream
-
-    Deserializes an input stream to a JSON value.
-
-    @param[in,out] i  input stream to read a serialized JSON value from
-    @param[in,out] j  JSON value to write the deserialized input to
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @liveexample{The example below shows how a JSON value is constructed by
-    reading a serialization from a stream.,operator_deserialize}
-
-    @sa parse(std::istream&, const parser_callback_t) for a variant with a
-    parser callback function to filter values while parsing
-
-    @since version 1.0.0
-    */
-    friend std::istream& operator>>(std::istream& i, basic_json& j)
-    {
-        parser(detail::input_adapter(i)).parse(false, j);
-        return i;
-    }
-
-    /// @}
-
-    ///////////////////////////
-    // convenience functions //
-    ///////////////////////////
-
-    /*!
-    @brief return the type as string
-
-    Returns the type name as string to be used in error messages - usually to
-    indicate that a function was called on a wrong JSON type.
-
-    @return a string representation of a the @a m_type member:
-            Value type  | return value
-            ----------- | -------------
-            null        | `"null"`
-            boolean     | `"boolean"`
-            string      | `"string"`
-            number      | `"number"` (for all number types)
-            object      | `"object"`
-            array       | `"array"`
-            discarded   | `"discarded"`
-
-    @exceptionsafety No-throw guarantee: this function never throws exceptions.
-
-    @complexity Constant.
-
-    @liveexample{The following code exemplifies `type_name()` for all JSON
-    types.,type_name}
-
-    @sa @ref type() -- return the type of the JSON value
-    @sa @ref operator value_t() -- return the type of the JSON value (implicit)
-
-    @since version 1.0.0, public since 2.1.0, `const char*` and `noexcept`
-    since 3.0.0
-    */
-    const char* type_name() const noexcept
-    {
-        {
-            switch (m_type)
-            {
-                case value_t::null:
-                    return "null";
-                case value_t::object:
-                    return "object";
-                case value_t::array:
-                    return "array";
-                case value_t::string:
-                    return "string";
-                case value_t::boolean:
-                    return "boolean";
-                case value_t::discarded:
-                    return "discarded";
-                default:
-                    return "number";
-            }
-        }
-    }
-
-
-  private:
-    //////////////////////
-    // member variables //
-    //////////////////////
-
-    /// the type of the current element
-    value_t m_type = value_t::null;
-
-    /// the value of the current element
-    json_value m_value = {};
-
-    //////////////////////////////////////////
-    // binary serialization/deserialization //
-    //////////////////////////////////////////
-
-    /// @name binary serialization/deserialization support
-    /// @{
-
-  public:
-    /*!
-    @brief create a CBOR serialization of a given JSON value
-
-    Serializes a given JSON value @a j to a byte vector using the CBOR (Concise
-    Binary Object Representation) serialization format. CBOR is a binary
-    serialization format which aims to be more compact than JSON itself, yet
-    more efficient to parse.
-
-    The library uses the following mapping from JSON values types to
-    CBOR types according to the CBOR specification (RFC 7049):
-
-    JSON value type | value/range                                | CBOR type                          | first byte
-    --------------- | ------------------------------------------ | ---------------------------------- | ---------------
-    null            | `null`                                     | Null                               | 0xF6
-    boolean         | `true`                                     | True                               | 0xF5
-    boolean         | `false`                                    | False                              | 0xF4
-    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3B
-    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3A
-    number_integer  | -32768..-129                               | Negative integer (2 bytes follow)  | 0x39
-    number_integer  | -128..-25                                  | Negative integer (1 byte follow)   | 0x38
-    number_integer  | -24..-1                                    | Negative integer                   | 0x20..0x37
-    number_integer  | 0..23                                      | Integer                            | 0x00..0x17
-    number_integer  | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
-    number_integer  | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
-    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
-    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
-    number_unsigned | 0..23                                      | Integer                            | 0x00..0x17
-    number_unsigned | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
-    number_unsigned | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
-    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1A
-    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1B
-    number_float    | *any value*                                | Double-Precision Float             | 0xFB
-    string          | *length*: 0..23                            | UTF-8 string                       | 0x60..0x77
-    string          | *length*: 23..255                          | UTF-8 string (1 byte follow)       | 0x78
-    string          | *length*: 256..65535                       | UTF-8 string (2 bytes follow)      | 0x79
-    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7A
-    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7B
-    array           | *size*: 0..23                              | array                              | 0x80..0x97
-    array           | *size*: 23..255                            | array (1 byte follow)              | 0x98
-    array           | *size*: 256..65535                         | array (2 bytes follow)             | 0x99
-    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9A
-    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9B
-    object          | *size*: 0..23                              | map                                | 0xA0..0xB7
-    object          | *size*: 23..255                            | map (1 byte follow)                | 0xB8
-    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xB9
-    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xBA
-    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xBB
-
-    @note The mapping is **complete** in the sense that any JSON value type
-          can be converted to a CBOR value.
-
-    @note If NaN or Infinity are stored inside a JSON number, they are
-          serialized properly. This behavior differs from the @ref dump()
-          function which serializes NaN or Infinity to `null`.
-
-    @note The following CBOR types are not used in the conversion:
-          - byte strings (0x40..0x5F)
-          - UTF-8 strings terminated by "break" (0x7F)
-          - arrays terminated by "break" (0x9F)
-          - maps terminated by "break" (0xBF)
-          - date/time (0xC0..0xC1)
-          - bignum (0xC2..0xC3)
-          - decimal fraction (0xC4)
-          - bigfloat (0xC5)
-          - tagged items (0xC6..0xD4, 0xD8..0xDB)
-          - expected conversions (0xD5..0xD7)
-          - simple values (0xE0..0xF3, 0xF8)
-          - undefined (0xF7)
-          - half and single-precision floats (0xF9-0xFA)
-          - break (0xFF)
-
-    @param[in] j  JSON value to serialize
-    @return MessagePack serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in CBOR format.,to_cbor}
-
-    @sa http://cbor.io
-    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
-        analogous deserialization
-    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
-    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
-             related UBJSON format
-
-    @since version 2.0.9
-    */
-    static std::vector<uint8_t> to_cbor(const basic_json& j)
-    {
-        std::vector<uint8_t> result;
-        to_cbor(j, result);
-        return result;
-    }
-
-    static void to_cbor(const basic_json& j, detail::output_adapter<uint8_t> o)
-    {
-        binary_writer<uint8_t>(o).write_cbor(j);
-    }
-
-    static void to_cbor(const basic_json& j, detail::output_adapter<char> o)
-    {
-        binary_writer<char>(o).write_cbor(j);
-    }
-
-    /*!
-    @brief create a MessagePack serialization of a given JSON value
-
-    Serializes a given JSON value @a j to a byte vector using the MessagePack
-    serialization format. MessagePack is a binary serialization format which
-    aims to be more compact than JSON itself, yet more efficient to parse.
-
-    The library uses the following mapping from JSON values types to
-    MessagePack types according to the MessagePack specification:
-
-    JSON value type | value/range                       | MessagePack type | first byte
-    --------------- | --------------------------------- | ---------------- | ----------
-    null            | `null`                            | nil              | 0xC0
-    boolean         | `true`                            | true             | 0xC3
-    boolean         | `false`                           | false            | 0xC2
-    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xD3
-    number_integer  | -2147483648..-32769               | int32            | 0xD2
-    number_integer  | -32768..-129                      | int16            | 0xD1
-    number_integer  | -128..-33                         | int8             | 0xD0
-    number_integer  | -32..-1                           | negative fixint  | 0xE0..0xFF
-    number_integer  | 0..127                            | positive fixint  | 0x00..0x7F
-    number_integer  | 128..255                          | uint 8           | 0xCC
-    number_integer  | 256..65535                        | uint 16          | 0xCD
-    number_integer  | 65536..4294967295                 | uint 32          | 0xCE
-    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xCF
-    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7F
-    number_unsigned | 128..255                          | uint 8           | 0xCC
-    number_unsigned | 256..65535                        | uint 16          | 0xCD
-    number_unsigned | 65536..4294967295                 | uint 32          | 0xCE
-    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xCF
-    number_float    | *any value*                       | float 64         | 0xCB
-    string          | *length*: 0..31                   | fixstr           | 0xA0..0xBF
-    string          | *length*: 32..255                 | str 8            | 0xD9
-    string          | *length*: 256..65535              | str 16           | 0xDA
-    string          | *length*: 65536..4294967295       | str 32           | 0xDB
-    array           | *size*: 0..15                     | fixarray         | 0x90..0x9F
-    array           | *size*: 16..65535                 | array 16         | 0xDC
-    array           | *size*: 65536..4294967295         | array 32         | 0xDD
-    object          | *size*: 0..15                     | fix map          | 0x80..0x8F
-    object          | *size*: 16..65535                 | map 16           | 0xDE
-    object          | *size*: 65536..4294967295         | map 32           | 0xDF
-
-    @note The mapping is **complete** in the sense that any JSON value type
-          can be converted to a MessagePack value.
-
-    @note The following values can **not** be converted to a MessagePack value:
-          - strings with more than 4294967295 bytes
-          - arrays with more than 4294967295 elements
-          - objects with more than 4294967295 elements
-
-    @note The following MessagePack types are not used in the conversion:
-          - bin 8 - bin 32 (0xC4..0xC6)
-          - ext 8 - ext 32 (0xC7..0xC9)
-          - float 32 (0xCA)
-          - fixext 1 - fixext 16 (0xD4..0xD8)
-
-    @note Any MessagePack output created @ref to_msgpack can be successfully
-          parsed by @ref from_msgpack.
-
-    @note If NaN or Infinity are stored inside a JSON number, they are
-          serialized properly. This behavior differs from the @ref dump()
-          function which serializes NaN or Infinity to `null`.
-
-    @param[in] j  JSON value to serialize
-    @return MessagePack serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in MessagePack format.,to_msgpack}
-
-    @sa http://msgpack.org
-    @sa @ref from_msgpack for the analogous deserialization
-    @sa @ref to_cbor(const basic_json& for the related CBOR format
-    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
-             related UBJSON format
-
-    @since version 2.0.9
-    */
-    static std::vector<uint8_t> to_msgpack(const basic_json& j)
-    {
-        std::vector<uint8_t> result;
-        to_msgpack(j, result);
-        return result;
-    }
-
-    static void to_msgpack(const basic_json& j, detail::output_adapter<uint8_t> o)
-    {
-        binary_writer<uint8_t>(o).write_msgpack(j);
-    }
-
-    static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)
-    {
-        binary_writer<char>(o).write_msgpack(j);
-    }
-
-    /*!
-    @brief create a UBJSON serialization of a given JSON value
-
-    Serializes a given JSON value @a j to a byte vector using the UBJSON
-    (Universal Binary JSON) serialization format. UBJSON aims to be more compact
-    than JSON itself, yet more efficient to parse.
-
-    The library uses the following mapping from JSON values types to
-    UBJSON types according to the UBJSON specification:
-
-    JSON value type | value/range                       | UBJSON type | marker
-    --------------- | --------------------------------- | ----------- | ------
-    null            | `null`                            | null        | `Z`
-    boolean         | `true`                            | true        | `T`
-    boolean         | `false`                           | false       | `F`
-    number_integer  | -9223372036854775808..-2147483649 | int64       | `L`
-    number_integer  | -2147483648..-32769               | int32       | `l`
-    number_integer  | -32768..-129                      | int16       | `I`
-    number_integer  | -128..127                         | int8        | `i`
-    number_integer  | 128..255                          | uint8       | `U`
-    number_integer  | 256..32767                        | int16       | `I`
-    number_integer  | 32768..2147483647                 | int32       | `l`
-    number_integer  | 2147483648..9223372036854775807   | int64       | `L`
-    number_unsigned | 0..127                            | int8        | `i`
-    number_unsigned | 128..255                          | uint8       | `U`
-    number_unsigned | 256..32767                        | int16       | `I`
-    number_unsigned | 32768..2147483647                 | int32       | `l`
-    number_unsigned | 2147483648..9223372036854775807   | int64       | `L`
-    number_float    | *any value*                       | float64     | `D`
-    string          | *with shortest length indicator*  | string      | `S`
-    array           | *see notes on optimized format*   | array       | `[`
-    object          | *see notes on optimized format*   | map         | `{`
-
-    @note The mapping is **complete** in the sense that any JSON value type
-          can be converted to a UBJSON value.
-
-    @note The following values can **not** be converted to a UBJSON value:
-          - strings with more than 9223372036854775807 bytes (theoretical)
-          - unsigned integer numbers above 9223372036854775807
-
-    @note The following markers are not used in the conversion:
-          - `Z`: no-op values are not created.
-          - `C`: single-byte strings are serialized with `S` markers.
-
-    @note Any UBJSON output created @ref to_ubjson can be successfully parsed
-          by @ref from_ubjson.
-
-    @note If NaN or Infinity are stored inside a JSON number, they are
-          serialized properly. This behavior differs from the @ref dump()
-          function which serializes NaN or Infinity to `null`.
-
-    @note The optimized formats for containers are supported: Parameter
-          @a use_size adds size information to the beginning of a container and
-          removes the closing marker. Parameter @a use_type further checks
-          whether all elements of a container have the same type and adds the
-          type marker to the beginning of the container. The @a use_type
-          parameter must only be used together with @a use_size = true. Note
-          that @a use_size = true alone may result in larger representations -
-          the benefit of this parameter is that the receiving side is
-          immediately informed on the number of elements of the container.
-
-    @param[in] j  JSON value to serialize
-    @param[in] use_size  whether to add size annotations to container types
-    @param[in] use_type  whether to add type annotations to container types
-                         (must be combined with @a use_size = true)
-    @return UBJSON serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in UBJSON format.,to_ubjson}
-
-    @sa http://ubjson.org
-    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
-        analogous deserialization
-    @sa @ref to_cbor(const basic_json& for the related CBOR format
-    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
-
-    @since version 3.1.0
-    */
-    static std::vector<uint8_t> to_ubjson(const basic_json& j,
-                                          const bool use_size = false,
-                                          const bool use_type = false)
-    {
-        std::vector<uint8_t> result;
-        to_ubjson(j, result, use_size, use_type);
-        return result;
-    }
-
-    static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o,
-                          const bool use_size = false, const bool use_type = false)
-    {
-        binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type);
-    }
-
-    static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
-                          const bool use_size = false, const bool use_type = false)
-    {
-        binary_writer<char>(o).write_ubjson(j, use_size, use_type);
-    }
-
-
-    /*!
-    @brief Serializes the given JSON object `j` to BSON and returns a vector
-           containing the corresponding BSON-representation.
-
-    BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are
-    stored as a single entity (a so-called document).
-
-    The library uses the following mapping from JSON values types to BSON types:
-
-    JSON value type | value/range                       | BSON type   | marker
-    --------------- | --------------------------------- | ----------- | ------
-    null            | `null`                            | null        | 0x0A
-    boolean         | `true`, `false`                   | boolean     | 0x08
-    number_integer  | -9223372036854775808..-2147483649 | int64       | 0x12
-    number_integer  | -2147483648..2147483647           | int32       | 0x10
-    number_integer  | 2147483648..9223372036854775807   | int64       | 0x12
-    number_unsigned | 0..2147483647                     | int32       | 0x10
-    number_unsigned | 2147483648..9223372036854775807   | int64       | 0x12
-    number_unsigned | 9223372036854775808..18446744073709551615| --   | --
-    number_float    | *any value*                       | double      | 0x01
-    string          | *any value*                       | string      | 0x02
-    array           | *any value*                       | document    | 0x04
-    object          | *any value*                       | document    | 0x03
-
-    @warning The mapping is **incomplete**, since only JSON-objects (and things
-    contained therein) can be serialized to BSON.
-    Also, integers larger than 9223372036854775807 cannot be serialized to BSON,
-    and the keys may not contain U+0000, since they are serialized a
-    zero-terminated c-strings.
-
-    @throw out_of_range.407  if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807`
-    @throw out_of_range.409  if a key in `j` contains a NULL (U+0000)
-    @throw type_error.317    if `!j.is_object()`
-
-    @pre The input `j` is required to be an object: `j.is_object() == true`.
-
-    @note Any BSON output created via @ref to_bson can be successfully parsed
-          by @ref from_bson.
-
-    @param[in] j  JSON value to serialize
-    @return BSON serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in BSON format.,to_bson}
-
-    @sa http://bsonspec.org/spec.html
-    @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the
-        analogous deserialization
-    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
-             related UBJSON format
-    @sa @ref to_cbor(const basic_json&) for the related CBOR format
-    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
-    */
-    static std::vector<uint8_t> to_bson(const basic_json& j)
-    {
-        std::vector<uint8_t> result;
-        to_bson(j, result);
-        return result;
-    }
-
-    /*!
-    @brief Serializes the given JSON object `j` to BSON and forwards the
-           corresponding BSON-representation to the given output_adapter `o`.
-    @param j The JSON object to convert to BSON.
-    @param o The output adapter that receives the binary BSON representation.
-    @pre The input `j` shall be an object: `j.is_object() == true`
-    @sa @ref to_bson(const basic_json&)
-    */
-    static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o)
-    {
-        binary_writer<uint8_t>(o).write_bson(j);
-    }
-
-    /*!
-    @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>)
-    */
-    static void to_bson(const basic_json& j, detail::output_adapter<char> o)
-    {
-        binary_writer<char>(o).write_bson(j);
-    }
-
-
-    /*!
-    @brief create a JSON value from an input in CBOR format
-
-    Deserializes a given input @a i to a JSON value using the CBOR (Concise
-    Binary Object Representation) serialization format.
-
-    The library maps CBOR types to JSON value types as follows:
-
-    CBOR type              | JSON value type | first byte
-    ---------------------- | --------------- | ----------
-    Integer                | number_unsigned | 0x00..0x17
-    Unsigned integer       | number_unsigned | 0x18
-    Unsigned integer       | number_unsigned | 0x19
-    Unsigned integer       | number_unsigned | 0x1A
-    Unsigned integer       | number_unsigned | 0x1B
-    Negative integer       | number_integer  | 0x20..0x37
-    Negative integer       | number_integer  | 0x38
-    Negative integer       | number_integer  | 0x39
-    Negative integer       | number_integer  | 0x3A
-    Negative integer       | number_integer  | 0x3B
-    Negative integer       | number_integer  | 0x40..0x57
-    UTF-8 string           | string          | 0x60..0x77
-    UTF-8 string           | string          | 0x78
-    UTF-8 string           | string          | 0x79
-    UTF-8 string           | string          | 0x7A
-    UTF-8 string           | string          | 0x7B
-    UTF-8 string           | string          | 0x7F
-    array                  | array           | 0x80..0x97
-    array                  | array           | 0x98
-    array                  | array           | 0x99
-    array                  | array           | 0x9A
-    array                  | array           | 0x9B
-    array                  | array           | 0x9F
-    map                    | object          | 0xA0..0xB7
-    map                    | object          | 0xB8
-    map                    | object          | 0xB9
-    map                    | object          | 0xBA
-    map                    | object          | 0xBB
-    map                    | object          | 0xBF
-    False                  | `false`         | 0xF4
-    True                   | `true`          | 0xF5
-    Null                   | `null`          | 0xF6
-    Half-Precision Float   | number_float    | 0xF9
-    Single-Precision Float | number_float    | 0xFA
-    Double-Precision Float | number_float    | 0xFB
-
-    @warning The mapping is **incomplete** in the sense that not all CBOR
-             types can be converted to a JSON value. The following CBOR types
-             are not supported and will yield parse errors (parse_error.112):
-             - byte strings (0x40..0x5F)
-             - date/time (0xC0..0xC1)
-             - bignum (0xC2..0xC3)
-             - decimal fraction (0xC4)
-             - bigfloat (0xC5)
-             - tagged items (0xC6..0xD4, 0xD8..0xDB)
-             - expected conversions (0xD5..0xD7)
-             - simple values (0xE0..0xF3, 0xF8)
-             - undefined (0xF7)
-
-    @warning CBOR allows map keys of any type, whereas JSON only allows
-             strings as keys in object values. Therefore, CBOR maps with keys
-             other than UTF-8 strings are rejected (parse_error.113).
-
-    @note Any CBOR output created @ref to_cbor can be successfully parsed by
-          @ref from_cbor.
-
-    @param[in] i  an input in CBOR format convertible to an input adapter
-    @param[in] strict  whether to expect the input to be consumed until EOF
-                       (true by default)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return deserialized JSON value
-
-    @throw parse_error.110 if the given input ends prematurely or the end of
-    file was not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported features from CBOR were
-    used in the given input @a v or if the input is not valid CBOR
-    @throw parse_error.113 if a string was expected as map key, but not found
-
-    @complexity Linear in the size of the input @a i.
-
-    @liveexample{The example shows the deserialization of a byte vector in CBOR
-    format to a JSON value.,from_cbor}
-
-    @sa http://cbor.io
-    @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the
-        related MessagePack format
-    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
-        related UBJSON format
-
-    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
-           consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
-           since 3.2.0
-    */
-    static basic_json from_cbor(detail::input_adapter&& i,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @copydoc from_cbor(detail::input_adapter&&, const bool, const bool)
-    */
-    template<typename A1, typename A2,
-             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_cbor(A1 && a1, A2 && a2,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @brief create a JSON value from an input in MessagePack format
-
-    Deserializes a given input @a i to a JSON value using the MessagePack
-    serialization format.
-
-    The library maps MessagePack types to JSON value types as follows:
-
-    MessagePack type | JSON value type | first byte
-    ---------------- | --------------- | ----------
-    positive fixint  | number_unsigned | 0x00..0x7F
-    fixmap           | object          | 0x80..0x8F
-    fixarray         | array           | 0x90..0x9F
-    fixstr           | string          | 0xA0..0xBF
-    nil              | `null`          | 0xC0
-    false            | `false`         | 0xC2
-    true             | `true`          | 0xC3
-    float 32         | number_float    | 0xCA
-    float 64         | number_float    | 0xCB
-    uint 8           | number_unsigned | 0xCC
-    uint 16          | number_unsigned | 0xCD
-    uint 32          | number_unsigned | 0xCE
-    uint 64          | number_unsigned | 0xCF
-    int 8            | number_integer  | 0xD0
-    int 16           | number_integer  | 0xD1
-    int 32           | number_integer  | 0xD2
-    int 64           | number_integer  | 0xD3
-    str 8            | string          | 0xD9
-    str 16           | string          | 0xDA
-    str 32           | string          | 0xDB
-    array 16         | array           | 0xDC
-    array 32         | array           | 0xDD
-    map 16           | object          | 0xDE
-    map 32           | object          | 0xDF
-    negative fixint  | number_integer  | 0xE0-0xFF
-
-    @warning The mapping is **incomplete** in the sense that not all
-             MessagePack types can be converted to a JSON value. The following
-             MessagePack types are not supported and will yield parse errors:
-              - bin 8 - bin 32 (0xC4..0xC6)
-              - ext 8 - ext 32 (0xC7..0xC9)
-              - fixext 1 - fixext 16 (0xD4..0xD8)
-
-    @note Any MessagePack output created @ref to_msgpack can be successfully
-          parsed by @ref from_msgpack.
-
-    @param[in] i  an input in MessagePack format convertible to an input
-                  adapter
-    @param[in] strict  whether to expect the input to be consumed until EOF
-                       (true by default)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return deserialized JSON value
-
-    @throw parse_error.110 if the given input ends prematurely or the end of
-    file was not reached when @a strict was set to true
-    @throw parse_error.112 if unsupported features from MessagePack were
-    used in the given input @a i or if the input is not valid MessagePack
-    @throw parse_error.113 if a string was expected as map key, but not found
-
-    @complexity Linear in the size of the input @a i.
-
-    @liveexample{The example shows the deserialization of a byte vector in
-    MessagePack format to a JSON value.,from_msgpack}
-
-    @sa http://msgpack.org
-    @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
-        related CBOR format
-    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for
-        the related UBJSON format
-    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
-        the related BSON format
-
-    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
-           consume input adapters, removed start_index parameter, and added
-           @a strict parameter since 3.0.0; added @a allow_exceptions parameter
-           since 3.2.0
-    */
-    static basic_json from_msgpack(detail::input_adapter&& i,
-                                   const bool strict = true,
-                                   const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool)
-    */
-    template<typename A1, typename A2,
-             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_msgpack(A1 && a1, A2 && a2,
-                                   const bool strict = true,
-                                   const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @brief create a JSON value from an input in UBJSON format
-
-    Deserializes a given input @a i to a JSON value using the UBJSON (Universal
-    Binary JSON) serialization format.
-
-    The library maps UBJSON types to JSON value types as follows:
-
-    UBJSON type | JSON value type                         | marker
-    ----------- | --------------------------------------- | ------
-    no-op       | *no value, next value is read*          | `N`
-    null        | `null`                                  | `Z`
-    false       | `false`                                 | `F`
-    true        | `true`                                  | `T`
-    float32     | number_float                            | `d`
-    float64     | number_float                            | `D`
-    uint8       | number_unsigned                         | `U`
-    int8        | number_integer                          | `i`
-    int16       | number_integer                          | `I`
-    int32       | number_integer                          | `l`
-    int64       | number_integer                          | `L`
-    string      | string                                  | `S`
-    char        | string                                  | `C`
-    array       | array (optimized values are supported)  | `[`
-    object      | object (optimized values are supported) | `{`
-
-    @note The mapping is **complete** in the sense that any UBJSON value can
-          be converted to a JSON value.
-
-    @param[in] i  an input in UBJSON format convertible to an input adapter
-    @param[in] strict  whether to expect the input to be consumed until EOF
-                       (true by default)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return deserialized JSON value
-
-    @throw parse_error.110 if the given input ends prematurely or the end of
-    file was not reached when @a strict was set to true
-    @throw parse_error.112 if a parse error occurs
-    @throw parse_error.113 if a string could not be parsed successfully
-
-    @complexity Linear in the size of the input @a i.
-
-    @liveexample{The example shows the deserialization of a byte vector in
-    UBJSON format to a JSON value.,from_ubjson}
-
-    @sa http://ubjson.org
-    @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the
-             analogous serialization
-    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
-        related CBOR format
-    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
-        the related MessagePack format
-    @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for
-        the related BSON format
-
-    @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0
-    */
-    static basic_json from_ubjson(detail::input_adapter&& i,
-                                  const bool strict = true,
-                                  const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool)
-    */
-    template<typename A1, typename A2,
-             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_ubjson(A1 && a1, A2 && a2,
-                                  const bool strict = true,
-                                  const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @brief Create a JSON value from an input in BSON format
-
-    Deserializes a given input @a i to a JSON value using the BSON (Binary JSON)
-    serialization format.
-
-    The library maps BSON record types to JSON value types as follows:
-
-    BSON type       | BSON marker byte | JSON value type
-    --------------- | ---------------- | ---------------------------
-    double          | 0x01             | number_float
-    string          | 0x02             | string
-    document        | 0x03             | object
-    array           | 0x04             | array
-    binary          | 0x05             | still unsupported
-    undefined       | 0x06             | still unsupported
-    ObjectId        | 0x07             | still unsupported
-    boolean         | 0x08             | boolean
-    UTC Date-Time   | 0x09             | still unsupported
-    null            | 0x0A             | null
-    Regular Expr.   | 0x0B             | still unsupported
-    DB Pointer      | 0x0C             | still unsupported
-    JavaScript Code | 0x0D             | still unsupported
-    Symbol          | 0x0E             | still unsupported
-    JavaScript Code | 0x0F             | still unsupported
-    int32           | 0x10             | number_integer
-    Timestamp       | 0x11             | still unsupported
-    128-bit decimal float | 0x13       | still unsupported
-    Max Key         | 0x7F             | still unsupported
-    Min Key         | 0xFF             | still unsupported
-
-    @warning The mapping is **incomplete**. The unsupported mappings
-             are indicated in the table above.
-
-    @param[in] i  an input in BSON format convertible to an input adapter
-    @param[in] strict  whether to expect the input to be consumed until EOF
-                       (true by default)
-    @param[in] allow_exceptions  whether to throw exceptions in case of a
-    parse error (optional, true by default)
-
-    @return deserialized JSON value
-
-    @throw parse_error.114 if an unsupported BSON record type is encountered
-
-    @complexity Linear in the size of the input @a i.
-
-    @liveexample{The example shows the deserialization of a byte vector in
-    BSON format to a JSON value.,from_bson}
-
-    @sa http://bsonspec.org/spec.html
-    @sa @ref to_bson(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the
-        related CBOR format
-    @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for
-        the related MessagePack format
-    @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the
-        related UBJSON format
-    */
-    static basic_json from_bson(detail::input_adapter&& i,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-    /*!
-    @copydoc from_bson(detail::input_adapter&&, const bool, const bool)
-    */
-    template<typename A1, typename A2,
-             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
-    static basic_json from_bson(A1 && a1, A2 && a2,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        basic_json result;
-        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);
-        const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict);
-        return res ? result : basic_json(value_t::discarded);
-    }
-
-
-
-    /// @}
-
-    //////////////////////////
-    // JSON Pointer support //
-    //////////////////////////
-
-    /// @name JSON Pointer functions
-    /// @{
-
-    /*!
-    @brief access specified element via JSON Pointer
-
-    Uses a JSON pointer to retrieve a reference to the respective JSON value.
-    No bound checking is performed. Similar to @ref operator[](const typename
-    object_t::key_type&), `null` values are created in arrays and objects if
-    necessary.
-
-    In particular:
-    - If the JSON pointer points to an object key that does not exist, it
-      is created an filled with a `null` value before a reference to it
-      is returned.
-    - If the JSON pointer points to an array index that does not exist, it
-      is created an filled with a `null` value before a reference to it
-      is returned. All indices between the current maximum and the given
-      index are also filled with `null`.
-    - The special value `-` is treated as a synonym for the index past the
-      end.
-
-    @param[in] ptr  a JSON pointer
-
-    @return reference to the element pointed to by @a ptr
-
-    @complexity Constant.
-
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-
-    @liveexample{The behavior is shown in the example.,operatorjson_pointer}
-
-    @since version 2.0.0
-    */
-    reference operator[](const json_pointer& ptr)
-    {
-        return ptr.get_unchecked(this);
-    }
-
-    /*!
-    @brief access specified element via JSON Pointer
-
-    Uses a JSON pointer to retrieve a reference to the respective JSON value.
-    No bound checking is performed. The function does not change the JSON
-    value; no `null` values are created. In particular, the the special value
-    `-` yields an exception.
-
-    @param[in] ptr  JSON pointer to the desired element
-
-    @return const reference to the element pointed to by @a ptr
-
-    @complexity Constant.
-
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.402  if the array index '-' is used
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-
-    @liveexample{The behavior is shown in the example.,operatorjson_pointer_const}
-
-    @since version 2.0.0
-    */
-    const_reference operator[](const json_pointer& ptr) const
-    {
-        return ptr.get_unchecked(this);
-    }
-
-    /*!
-    @brief access specified element via JSON Pointer
-
-    Returns a reference to the element at with specified JSON pointer @a ptr,
-    with bounds checking.
-
-    @param[in] ptr  JSON pointer to the desired element
-
-    @return reference to the element pointed to by @a ptr
-
-    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
-    begins with '0'. See example below.
-
-    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
-    is not a number. See example below.
-
-    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
-    is out of range. See example below.
-
-    @throw out_of_range.402 if the array index '-' is used in the passed JSON
-    pointer @a ptr. As `at` provides checked access (and no elements are
-    implicitly inserted), the index '-' is always invalid. See example below.
-
-    @throw out_of_range.403 if the JSON pointer describes a key of an object
-    which cannot be found. See example below.
-
-    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
-    See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @since version 2.0.0
-
-    @liveexample{The behavior is shown in the example.,at_json_pointer}
-    */
-    reference at(const json_pointer& ptr)
-    {
-        return ptr.get_checked(this);
-    }
-
-    /*!
-    @brief access specified element via JSON Pointer
-
-    Returns a const reference to the element at with specified JSON pointer @a
-    ptr, with bounds checking.
-
-    @param[in] ptr  JSON pointer to the desired element
-
-    @return reference to the element pointed to by @a ptr
-
-    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
-    begins with '0'. See example below.
-
-    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
-    is not a number. See example below.
-
-    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
-    is out of range. See example below.
-
-    @throw out_of_range.402 if the array index '-' is used in the passed JSON
-    pointer @a ptr. As `at` provides checked access (and no elements are
-    implicitly inserted), the index '-' is always invalid. See example below.
-
-    @throw out_of_range.403 if the JSON pointer describes a key of an object
-    which cannot be found. See example below.
-
-    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
-    See example below.
-
-    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
-    changes in the JSON value.
-
-    @complexity Constant.
-
-    @since version 2.0.0
-
-    @liveexample{The behavior is shown in the example.,at_json_pointer_const}
-    */
-    const_reference at(const json_pointer& ptr) const
-    {
-        return ptr.get_checked(this);
-    }
-
-    /*!
-    @brief return flattened JSON value
-
-    The function creates a JSON object whose keys are JSON pointers (see [RFC
-    6901](https://tools.ietf.org/html/rfc6901)) and whose values are all
-    primitive. The original JSON value can be restored using the @ref
-    unflatten() function.
-
-    @return an object that maps JSON pointers to primitive values
-
-    @note Empty objects and arrays are flattened to `null` and will not be
-          reconstructed correctly by the @ref unflatten() function.
-
-    @complexity Linear in the size the JSON value.
-
-    @liveexample{The following code shows how a JSON object is flattened to an
-    object whose keys consist of JSON pointers.,flatten}
-
-    @sa @ref unflatten() for the reverse function
-
-    @since version 2.0.0
-    */
-    basic_json flatten() const
-    {
-        basic_json result(value_t::object);
-        json_pointer::flatten("", *this, result);
-        return result;
-    }
-
-    /*!
-    @brief unflatten a previously flattened JSON value
-
-    The function restores the arbitrary nesting of a JSON value that has been
-    flattened before using the @ref flatten() function. The JSON value must
-    meet certain constraints:
-    1. The value must be an object.
-    2. The keys must be JSON pointers (see
-       [RFC 6901](https://tools.ietf.org/html/rfc6901))
-    3. The mapped values must be primitive JSON types.
-
-    @return the original JSON from a flattened version
-
-    @note Empty objects and arrays are flattened by @ref flatten() to `null`
-          values and can not unflattened to their original type. Apart from
-          this example, for a JSON value `j`, the following is always true:
-          `j == j.flatten().unflatten()`.
-
-    @complexity Linear in the size the JSON value.
-
-    @throw type_error.314  if value is not an object
-    @throw type_error.315  if object values are not primitive
-
-    @liveexample{The following code shows how a flattened JSON object is
-    unflattened into the original nested JSON object.,unflatten}
-
-    @sa @ref flatten() for the reverse function
-
-    @since version 2.0.0
-    */
-    basic_json unflatten() const
-    {
-        return json_pointer::unflatten(*this);
-    }
-
-    /// @}
-
-    //////////////////////////
-    // JSON Patch functions //
-    //////////////////////////
-
-    /// @name JSON Patch functions
-    /// @{
-
-    /*!
-    @brief applies a JSON patch
-
-    [JSON Patch](http://jsonpatch.com) defines a JSON document structure for
-    expressing a sequence of operations to apply to a JSON) document. With
-    this function, a JSON Patch is applied to the current JSON value by
-    executing all operations from the patch.
-
-    @param[in] json_patch  JSON patch document
-    @return patched document
-
-    @note The application of a patch is atomic: Either all operations succeed
-          and the patched document is returned or an exception is thrown. In
-          any case, the original value is not changed: the patch is applied
-          to a copy of the value.
-
-    @throw parse_error.104 if the JSON patch does not consist of an array of
-    objects
-
-    @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory
-    attributes are missing); example: `"operation add must have member path"`
-
-    @throw out_of_range.401 if an array index is out of range.
-
-    @throw out_of_range.403 if a JSON pointer inside the patch could not be
-    resolved successfully in the current JSON value; example: `"key baz not
-    found"`
-
-    @throw out_of_range.405 if JSON pointer has no parent ("add", "remove",
-    "move")
-
-    @throw other_error.501 if "test" operation was unsuccessful
-
-    @complexity Linear in the size of the JSON value and the length of the
-    JSON patch. As usually only a fraction of the JSON value is affected by
-    the patch, the complexity can usually be neglected.
-
-    @liveexample{The following code shows how a JSON patch is applied to a
-    value.,patch}
-
-    @sa @ref diff -- create a JSON patch by comparing two JSON values
-
-    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
-    @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901)
-
-    @since version 2.0.0
-    */
-    basic_json patch(const basic_json& json_patch) const
-    {
-        // make a working copy to apply the patch to
-        basic_json result = *this;
-
-        // the valid JSON Patch operations
-        enum class patch_operations {add, remove, replace, move, copy, test, invalid};
-
-        const auto get_op = [](const std::string & op)
-        {
-            if (op == "add")
-            {
-                return patch_operations::add;
-            }
-            if (op == "remove")
-            {
-                return patch_operations::remove;
-            }
-            if (op == "replace")
-            {
-                return patch_operations::replace;
-            }
-            if (op == "move")
-            {
-                return patch_operations::move;
-            }
-            if (op == "copy")
-            {
-                return patch_operations::copy;
-            }
-            if (op == "test")
-            {
-                return patch_operations::test;
-            }
-
-            return patch_operations::invalid;
-        };
-
-        // wrapper for "add" operation; add value at ptr
-        const auto operation_add = [&result](json_pointer & ptr, basic_json val)
-        {
-            // adding to the root of the target document means replacing it
-            if (ptr.is_root())
-            {
-                result = val;
-            }
-            else
-            {
-                // make sure the top element of the pointer exists
-                json_pointer top_pointer = ptr.top();
-                if (top_pointer != ptr)
-                {
-                    result.at(top_pointer);
-                }
-
-                // get reference to parent of JSON pointer ptr
-                const auto last_path = ptr.pop_back();
-                basic_json& parent = result[ptr];
-
-                switch (parent.m_type)
-                {
-                    case value_t::null:
-                    case value_t::object:
-                    {
-                        // use operator[] to add value
-                        parent[last_path] = val;
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        if (last_path == "-")
-                        {
-                            // special case: append to back
-                            parent.push_back(val);
-                        }
-                        else
-                        {
-                            const auto idx = json_pointer::array_index(last_path);
-                            if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
-                            {
-                                // avoid undefined behavior
-                                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
-                            }
-
-                            // default case: insert add offset
-                            parent.insert(parent.begin() + static_cast<difference_type>(idx), val);
-                        }
-                        break;
-                    }
-
-                    // LCOV_EXCL_START
-                    default:
-                    {
-                        // if there exists a parent it cannot be primitive
-                        assert(false);
-                    }
-                        // LCOV_EXCL_STOP
-                }
-            }
-        };
-
-        // wrapper for "remove" operation; remove value at ptr
-        const auto operation_remove = [&result](json_pointer & ptr)
-        {
-            // get reference to parent of JSON pointer ptr
-            const auto last_path = ptr.pop_back();
-            basic_json& parent = result.at(ptr);
-
-            // remove child
-            if (parent.is_object())
-            {
-                // perform range check
-                auto it = parent.find(last_path);
-                if (JSON_LIKELY(it != parent.end()))
-                {
-                    parent.erase(it);
-                }
-                else
-                {
-                    JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found"));
-                }
-            }
-            else if (parent.is_array())
-            {
-                // note erase performs range check
-                parent.erase(static_cast<size_type>(json_pointer::array_index(last_path)));
-            }
-        };
-
-        // type check: top level value must be an array
-        if (JSON_UNLIKELY(not json_patch.is_array()))
-        {
-            JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
-        }
-
-        // iterate and apply the operations
-        for (const auto& val : json_patch)
-        {
-            // wrapper to get a value for an operation
-            const auto get_value = [&val](const std::string & op,
-                                          const std::string & member,
-                                          bool string_type) -> basic_json &
-            {
-                // find value
-                auto it = val.m_value.object->find(member);
-
-                // context-sensitive error message
-                const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'";
-
-                // check if desired value is present
-                if (JSON_UNLIKELY(it == val.m_value.object->end()))
-                {
-                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'"));
-                }
-
-                // check if result is of type string
-                if (JSON_UNLIKELY(string_type and not it->second.is_string()))
-                {
-                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'"));
-                }
-
-                // no error: return value
-                return it->second;
-            };
-
-            // type check: every element of the array must be an object
-            if (JSON_UNLIKELY(not val.is_object()))
-            {
-                JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
-            }
-
-            // collect mandatory members
-            const std::string op = get_value("op", "op", true);
-            const std::string path = get_value(op, "path", true);
-            json_pointer ptr(path);
-
-            switch (get_op(op))
-            {
-                case patch_operations::add:
-                {
-                    operation_add(ptr, get_value("add", "value", false));
-                    break;
-                }
-
-                case patch_operations::remove:
-                {
-                    operation_remove(ptr);
-                    break;
-                }
-
-                case patch_operations::replace:
-                {
-                    // the "path" location must exist - use at()
-                    result.at(ptr) = get_value("replace", "value", false);
-                    break;
-                }
-
-                case patch_operations::move:
-                {
-                    const std::string from_path = get_value("move", "from", true);
-                    json_pointer from_ptr(from_path);
-
-                    // the "from" location must exist - use at()
-                    basic_json v = result.at(from_ptr);
-
-                    // The move operation is functionally identical to a
-                    // "remove" operation on the "from" location, followed
-                    // immediately by an "add" operation at the target
-                    // location with the value that was just removed.
-                    operation_remove(from_ptr);
-                    operation_add(ptr, v);
-                    break;
-                }
-
-                case patch_operations::copy:
-                {
-                    const std::string from_path = get_value("copy", "from", true);
-                    const json_pointer from_ptr(from_path);
-
-                    // the "from" location must exist - use at()
-                    basic_json v = result.at(from_ptr);
-
-                    // The copy is functionally identical to an "add"
-                    // operation at the target location using the value
-                    // specified in the "from" member.
-                    operation_add(ptr, v);
-                    break;
-                }
-
-                case patch_operations::test:
-                {
-                    bool success = false;
-                    JSON_TRY
-                    {
-                        // check if "value" matches the one at "path"
-                        // the "path" location must exist - use at()
-                        success = (result.at(ptr) == get_value("test", "value", false));
-                    }
-                    JSON_INTERNAL_CATCH (out_of_range&)
-                    {
-                        // ignore out of range errors: success remains false
-                    }
-
-                    // throw an exception if test fails
-                    if (JSON_UNLIKELY(not success))
-                    {
-                        JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump()));
-                    }
-
-                    break;
-                }
-
-                case patch_operations::invalid:
-                {
-                    // op must be "add", "remove", "replace", "move", "copy", or
-                    // "test"
-                    JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid"));
-                }
-            }
-        }
-
-        return result;
-    }
-
-    /*!
-    @brief creates a diff as a JSON patch
-
-    Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can
-    be changed into the value @a target by calling @ref patch function.
-
-    @invariant For two JSON values @a source and @a target, the following code
-    yields always `true`:
-    @code {.cpp}
-    source.patch(diff(source, target)) == target;
-    @endcode
-
-    @note Currently, only `remove`, `add`, and `replace` operations are
-          generated.
-
-    @param[in] source  JSON value to compare from
-    @param[in] target  JSON value to compare against
-    @param[in] path    helper value to create JSON pointers
-
-    @return a JSON patch to convert the @a source to @a target
-
-    @complexity Linear in the lengths of @a source and @a target.
-
-    @liveexample{The following code shows how a JSON patch is created as a
-    diff for two JSON values.,diff}
-
-    @sa @ref patch -- apply a JSON patch
-    @sa @ref merge_patch -- apply a JSON Merge Patch
-
-    @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902)
-
-    @since version 2.0.0
-    */
-    static basic_json diff(const basic_json& source, const basic_json& target,
-                           const std::string& path = "")
-    {
-        // the patch
-        basic_json result(value_t::array);
-
-        // if the values are the same, return empty patch
-        if (source == target)
-        {
-            return result;
-        }
-
-        if (source.type() != target.type())
-        {
-            // different types: replace value
-            result.push_back(
-            {
-                {"op", "replace"}, {"path", path}, {"value", target}
-            });
-        }
-        else
-        {
-            switch (source.type())
-            {
-                case value_t::array:
-                {
-                    // first pass: traverse common elements
-                    std::size_t i = 0;
-                    while (i < source.size() and i < target.size())
-                    {
-                        // recursive call to compare array values at index i
-                        auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i));
-                        result.insert(result.end(), temp_diff.begin(), temp_diff.end());
-                        ++i;
-                    }
-
-                    // i now reached the end of at least one array
-                    // in a second pass, traverse the remaining elements
-
-                    // remove my remaining elements
-                    const auto end_index = static_cast<difference_type>(result.size());
-                    while (i < source.size())
-                    {
-                        // add operations in reverse order to avoid invalid
-                        // indices
-                        result.insert(result.begin() + end_index, object(
-                        {
-                            {"op", "remove"},
-                            {"path", path + "/" + std::to_string(i)}
-                        }));
-                        ++i;
-                    }
-
-                    // add other remaining elements
-                    while (i < target.size())
-                    {
-                        result.push_back(
-                        {
-                            {"op", "add"},
-                            {"path", path + "/" + std::to_string(i)},
-                            {"value", target[i]}
-                        });
-                        ++i;
-                    }
-
-                    break;
-                }
-
-                case value_t::object:
-                {
-                    // first pass: traverse this object's elements
-                    for (auto it = source.cbegin(); it != source.cend(); ++it)
-                    {
-                        // escape the key name to be used in a JSON patch
-                        const auto key = json_pointer::escape(it.key());
-
-                        if (target.find(it.key()) != target.end())
-                        {
-                            // recursive call to compare object values at key it
-                            auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key);
-                            result.insert(result.end(), temp_diff.begin(), temp_diff.end());
-                        }
-                        else
-                        {
-                            // found a key that is not in o -> remove it
-                            result.push_back(object(
-                            {
-                                {"op", "remove"}, {"path", path + "/" + key}
-                            }));
-                        }
-                    }
-
-                    // second pass: traverse other object's elements
-                    for (auto it = target.cbegin(); it != target.cend(); ++it)
-                    {
-                        if (source.find(it.key()) == source.end())
-                        {
-                            // found a key that is not in this -> add it
-                            const auto key = json_pointer::escape(it.key());
-                            result.push_back(
-                            {
-                                {"op", "add"}, {"path", path + "/" + key},
-                                {"value", it.value()}
-                            });
-                        }
-                    }
-
-                    break;
-                }
-
-                default:
-                {
-                    // both primitive type: replace value
-                    result.push_back(
-                    {
-                        {"op", "replace"}, {"path", path}, {"value", target}
-                    });
-                    break;
-                }
-            }
-        }
-
-        return result;
-    }
-
-    /// @}
-
-    ////////////////////////////////
-    // JSON Merge Patch functions //
-    ////////////////////////////////
-
-    /// @name JSON Merge Patch functions
-    /// @{
-
-    /*!
-    @brief applies a JSON Merge Patch
-
-    The merge patch format is primarily intended for use with the HTTP PATCH
-    method as a means of describing a set of modifications to a target
-    resource's content. This function applies a merge patch to the current
-    JSON value.
-
-    The function implements the following algorithm from Section 2 of
-    [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396):
-
-    ```
-    define MergePatch(Target, Patch):
-      if Patch is an Object:
-        if Target is not an Object:
-          Target = {} // Ignore the contents and set it to an empty Object
-        for each Name/Value pair in Patch:
-          if Value is null:
-            if Name exists in Target:
-              remove the Name/Value pair from Target
-          else:
-            Target[Name] = MergePatch(Target[Name], Value)
-        return Target
-      else:
-        return Patch
-    ```
-
-    Thereby, `Target` is the current object; that is, the patch is applied to
-    the current value.
-
-    @param[in] apply_patch  the patch to apply
-
-    @complexity Linear in the lengths of @a patch.
-
-    @liveexample{The following code shows how a JSON Merge Patch is applied to
-    a JSON document.,merge_patch}
-
-    @sa @ref patch -- apply a JSON patch
-    @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
-
-    @since version 3.0.0
-    */
-    void merge_patch(const basic_json& apply_patch)
-    {
-        if (apply_patch.is_object())
-        {
-            if (not is_object())
-            {
-                *this = object();
-            }
-            for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)
-            {
-                if (it.value().is_null())
-                {
-                    erase(it.key());
-                }
-                else
-                {
-                    operator[](it.key()).merge_patch(it.value());
-                }
-            }
-        }
-        else
-        {
-            *this = apply_patch;
-        }
-    }
-
-    /// @}
-};
-} // namespace nlohmann
-
-///////////////////////
-// nonmember support //
-///////////////////////
-
-// specialization of std::swap, and std::hash
-namespace std
-{
-
-/// hash value for JSON objects
-template<>
-struct hash<nlohmann::json>
-{
-    /*!
-    @brief return a hash value for a JSON object
-
-    @since version 1.0.0
-    */
-    std::size_t operator()(const nlohmann::json& j) const
-    {
-        // a naive hashing via the string representation
-        const auto& h = hash<nlohmann::json::string_t>();
-        return h(j.dump());
-    }
-};
-
-/// specialization for std::less<value_t>
-/// @note: do not remove the space after '<',
-///        see https://github.com/nlohmann/json/pull/679
-template<>
-struct less< ::nlohmann::detail::value_t>
-{
-    /*!
-    @brief compare two value_t enum values
-    @since version 3.0.0
-    */
-    bool operator()(nlohmann::detail::value_t lhs,
-                    nlohmann::detail::value_t rhs) const noexcept
-    {
-        return nlohmann::detail::operator<(lhs, rhs);
-    }
-};
-
-/*!
-@brief exchanges the values of two JSON objects
-
-@since version 1.0.0
-*/
-template<>
-inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept(
-    is_nothrow_move_constructible<nlohmann::json>::value and
-    is_nothrow_move_assignable<nlohmann::json>::value
-)
-{
-    j1.swap(j2);
-}
-
-} // namespace std
-
-/*!
-@brief user-defined string literal for JSON values
-
-This operator implements a user-defined string literal for JSON objects. It
-can be used by adding `"_json"` to a string literal and returns a JSON object
-if no parse error occurred.
-
-@param[in] s  a string representation of a JSON object
-@param[in] n  the length of string @a s
-@return a JSON object
-
-@since version 1.0.0
-*/
-inline nlohmann::json operator "" _json(const char* s, std::size_t n)
-{
-    return nlohmann::json::parse(s, s + n);
-}
-
-/*!
-@brief user-defined string literal for JSON pointer
-
-This operator implements a user-defined string literal for JSON Pointers. It
-can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer
-object if no parse error occurred.
-
-@param[in] s  a string representation of a JSON Pointer
-@param[in] n  the length of string @a s
-@return a JSON pointer object
-
-@since version 2.0.0
-*/
-inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)
-{
-    return nlohmann::json::json_pointer(std::string(s, n));
-}
-
-// #include <nlohmann/detail/macro_unscope.hpp>
-
-
-// restore GCC/clang diagnostic settings
-#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
-    #pragma GCC diagnostic pop
-#endif
-#if defined(__clang__)
-    #pragma GCC diagnostic pop
-#endif
-
-// clean up
-#undef JSON_INTERNAL_CATCH
-#undef JSON_CATCH
-#undef JSON_THROW
-#undef JSON_TRY
-#undef JSON_LIKELY
-#undef JSON_UNLIKELY
-#undef JSON_DEPRECATED
-#undef JSON_HAS_CPP_14
-#undef JSON_HAS_CPP_17
-#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
-#undef NLOHMANN_BASIC_JSON_TPL
-
-
-#endif

From 48f0a76372d5e5be86f6522c760ae1f6c12dbd65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Nov 2019 18:31:16 +0100
Subject: [PATCH 1656/2196] Fix installerScript job

https://hydra.nixos.org/build/105961653
---
 release.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 1af6424dd8d..1d7c2e2d628 100644
--- a/release.nix
+++ b/release.nix
@@ -300,7 +300,7 @@ let
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
+              (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
               [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
             } \
             --replace '@nixVersion@' ${build.x86_64-linux.src.version}

From 0d6774468cf109c9cb6502cb81f1219c6e68ee37 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Nov 2019 15:13:32 +0100
Subject: [PATCH 1657/2196] Move editorFor srom libutil to nix

libutil should not depend on libexpr.
---
 src/libutil/args.cc | 13 -------------
 src/libutil/args.hh |  4 ----
 src/nix/command.cc  | 14 ++++++++++++++
 src/nix/command.hh  |  5 +++++
 4 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index b7baad37536..7af2a1bf731 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -178,19 +178,6 @@ Strings argvToStrings(int argc, char * * argv)
     return args;
 }
 
-Strings editorFor(Pos pos)
-{
-    auto editor = getEnv("EDITOR", "cat");
-    auto args = tokenizeString<Strings>(editor);
-    if (pos.line > 0 && (
-        editor.find("emacs") != std::string::npos ||
-        editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos))
-        args.push_back(fmt("+%d", pos.line));
-    args.push_back(pos.file);
-    return args;
-}
-
 std::string renderLabels(const Strings & labels)
 {
     std::string res;
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 1e29bd4fae1..ad5fcca3941 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -5,7 +5,6 @@
 #include <memory>
 
 #include "util.hh"
-#include "nixexpr.hh"
 
 namespace nix {
 
@@ -191,9 +190,6 @@ public:
 
 Strings argvToStrings(int argc, char * * argv);
 
-/* Helper function to generate args that invoke $EDITOR on filename:lineno */
-Strings editorFor(Pos pos);
-
 /* Helper function for rendering argument labels. */
 std::string renderLabels(const Strings & labels);
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 532f331a757..97c2fcf1c96 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -1,6 +1,7 @@
 #include "command.hh"
 #include "store-api.hh"
 #include "derivations.hh"
+#include "nixexpr.hh"
 
 namespace nix {
 
@@ -153,4 +154,17 @@ void StorePathCommand::run(ref<Store> store)
     run(store, *storePaths.begin());
 }
 
+Strings editorFor(const Pos & pos)
+{
+    auto editor = getEnv("EDITOR", "cat");
+    auto args = tokenizeString<Strings>(editor);
+    if (pos.line > 0 && (
+        editor.find("emacs") != std::string::npos ||
+        editor.find("nano") != std::string::npos ||
+        editor.find("vim") != std::string::npos))
+        args.push_back(fmt("+%d", pos.line));
+    args.push_back(pos.file);
+    return args;
+}
+
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index fad404c7319..26c15e64617 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -10,6 +10,7 @@ extern std::string programPath;
 struct Value;
 class Bindings;
 class EvalState;
+struct Pos;
 
 /* A command is an argument parser that can be executed by calling its
    run() method. */
@@ -216,4 +217,8 @@ PathSet toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
+/* Helper function to generate args that invoke $EDITOR on
+   filename:lineno. */
+Strings editorFor(const Pos & pos);
+
 }

From a08f35392256b1ef23947857e41a9b12b1591245 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Fri, 8 Nov 2019 09:38:54 +0000
Subject: [PATCH 1658/2196] gitignore /precompiled-headers.h.?ch

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index b75c5d48905..fd62dfb3856 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ perl/Makefile.config
 # /
 /aclocal.m4
 /autom4te.cache
+/precompiled-headers.h.gch
+/precompiled-headers.h.pch
 /config.*
 /configure
 /nix.spec

From 6c041e84139fc06110f38d940d6117badcc1fd0a Mon Sep 17 00:00:00 2001
From: Eric Culp <eculperic@gmail.com>
Date: Thu, 31 Oct 2019 17:55:43 -0700
Subject: [PATCH 1659/2196] Replace $TMPDIR with $TEST_ROOT in
 tests/fetchurl.sh

$TMPDIR isn't necessarily set and would cause this test to fail.
---
 tests/fetchurl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 7319ced2b59..2535651b09a 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -35,7 +35,7 @@ clearStore
 hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
 hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
 
-mirror=$TMPDIR/hashed-mirror
+mirror=$TEST_ROOT/hashed-mirror
 rm -rf $mirror
 mkdir -p $mirror/sha512
 ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32

From 2ba9f227154e04aefc4ed47f9f23c97a7aefa269 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sun, 10 Nov 2019 10:02:22 +0100
Subject: [PATCH 1660/2196] De-duplicate struct PrimOp forward declaration

---
 src/libexpr/value.hh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index e1ec87d3b84..6743d7fd801 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -35,7 +35,6 @@ struct Env;
 struct Expr;
 struct ExprLambda;
 struct PrimOp;
-struct PrimOp;
 class Symbol;
 struct Pos;
 class EvalState;

From 96e6e680c1a298613756571e4984c340ce887c2a Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sun, 10 Nov 2019 11:14:26 -0500
Subject: [PATCH 1661/2196] Fix extra ; warnings involving MakeError

---
 src/libexpr/json-to-value.hh |  2 +-
 src/libexpr/nixexpr.hh       | 16 ++++++++--------
 src/libstore/build.cc        |  2 +-
 src/libstore/store-api.hh    | 12 ++++++------
 src/libutil/serialise.hh     |  2 +-
 src/libutil/thread-pool.hh   |  2 +-
 src/libutil/types.hh         |  4 ++--
 src/libutil/util.hh          |  6 +++---
 8 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh
index 33f35b16ce8..3b0fdae115e 100644
--- a/src/libexpr/json-to-value.hh
+++ b/src/libexpr/json-to-value.hh
@@ -6,7 +6,7 @@
 
 namespace nix {
 
-MakeError(JSONParseError, EvalError)
+MakeError(JSONParseError, EvalError);
 
 void parseJSON(EvalState & state, const string & s, Value & v);
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 665a42987dc..f7e9105a401 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -9,14 +9,14 @@
 namespace nix {
 
 
-MakeError(EvalError, Error)
-MakeError(ParseError, Error)
-MakeError(AssertionError, EvalError)
-MakeError(ThrownError, AssertionError)
-MakeError(Abort, EvalError)
-MakeError(TypeError, EvalError)
-MakeError(UndefinedVarError, Error)
-MakeError(RestrictedPathError, Error)
+MakeError(EvalError, Error);
+MakeError(ParseError, Error);
+MakeError(AssertionError, EvalError);
+MakeError(ThrownError, AssertionError);
+MakeError(Abort, EvalError);
+MakeError(TypeError, EvalError);
+MakeError(UndefinedVarError, Error);
+MakeError(RestrictedPathError, Error);
 
 
 /* Position objects. */
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 68c2f2ce361..a1b9c92f80c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1515,7 +1515,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
 }
 
 
-MakeError(NotDeterministic, BuildError)
+MakeError(NotDeterministic, BuildError);
 
 
 void DerivationGoal::buildDone()
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 250e1a4bcc1..b860b48beb4 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -20,12 +20,12 @@
 namespace nix {
 
 
-MakeError(SubstError, Error)
-MakeError(BuildError, Error) /* denotes a permanent build failure */
-MakeError(InvalidPath, Error)
-MakeError(Unsupported, Error)
-MakeError(SubstituteGone, Error)
-MakeError(SubstituterDisabled, Error)
+MakeError(SubstError, Error);
+MakeError(BuildError, Error) /* denotes a permanent build failure */;
+MakeError(InvalidPath, Error);
+MakeError(Unsupported, Error);
+MakeError(SubstituteGone, Error);
+MakeError(SubstituterDisabled, Error);
 
 
 struct BasicDerivation;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 0120aeecb23..128e287f31c 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -274,7 +274,7 @@ Sink & operator << (Sink & sink, const Strings & s);
 Sink & operator << (Sink & sink, const StringSet & s);
 
 
-MakeError(SerialisationError, Error)
+MakeError(SerialisationError, Error);
 
 
 template<typename T>
diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh
index bb16b639a59..b22e0d16225 100644
--- a/src/libutil/thread-pool.hh
+++ b/src/libutil/thread-pool.hh
@@ -11,7 +11,7 @@
 
 namespace nix {
 
-MakeError(ThreadPoolShutDown, Error)
+MakeError(ThreadPoolShutDown, Error);
 
 /* A simple thread pool that executes a queue of work items
    (lambdas). */
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 92bf469b5c6..4bc91828b8a 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -116,9 +116,9 @@ public:
     {                                                   \
     public:                                             \
         using superClass::superClass;                   \
-    };
+    }
 
-MakeError(Error, BaseError)
+MakeError(Error, BaseError);
 
 class SysError : public Error
 {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3493e80b5eb..5f1d1d1817a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -160,7 +160,7 @@ void readFull(int fd, unsigned char * buf, size_t count);
 void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
 void writeFull(int fd, const string & s, bool allowInterrupts = true);
 
-MakeError(EndOfFile, Error)
+MakeError(EndOfFile, Error);
 
 
 /* Read a file descriptor until EOF occurs. */
@@ -333,10 +333,10 @@ void inline checkInterrupt()
         _interrupted();
 }
 
-MakeError(Interrupted, BaseError)
+MakeError(Interrupted, BaseError);
 
 
-MakeError(FormatError, Error)
+MakeError(FormatError, Error);
 
 
 /* String tokenizer. */

From 4c34054673edd67cd1cfd81ab2972f27e518c78a Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sun, 10 Nov 2019 11:23:35 -0500
Subject: [PATCH 1662/2196] Remove unneeded semicolons

---
 src/libexpr/eval.cc |  2 +-
 src/libutil/args.hh | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f60e8d3ab7e..34e527c521f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -432,7 +432,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context)
         !context.empty() && store->isInStore(path)
         ? store->toRealPath(path)
         : path;
-};
+}
 
 
 Value * EvalState::addConstant(const string & name, Value & v)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index ad5fcca3941..f8df39b5c92 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -70,23 +70,23 @@ public:
         Args & args;
         Flag::ptr flag;
         friend class Args;
-        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { };
+        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }
     public:
         ~FlagMaker();
-        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; };
-        FlagMaker & shortName(char s) { flag->shortName = s; return *this; };
-        FlagMaker & description(const std::string & s) { flag->description = s; return *this; };
-        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; };
-        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; };
-        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; };
-        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; };
-        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; };
+        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }
+        FlagMaker & shortName(char s) { flag->shortName = s; return *this; }
+        FlagMaker & description(const std::string & s) { flag->description = s; return *this; }
+        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }
+        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }
+        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }
+        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }
+        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }
         FlagMaker & handler(std::function<void(std::string)> handler) {
             flag->arity = 1;
             flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
             return *this;
-        };
-        FlagMaker & category(const std::string & s) { flag->category = s; return *this; };
+        }
+        FlagMaker & category(const std::string & s) { flag->category = s; return *this; }
 
         template<class T>
         FlagMaker & dest(T * dest)
@@ -94,7 +94,7 @@ public:
             flag->arity = 1;
             flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
             return *this;
-        };
+        }
 
         template<class T>
         FlagMaker & set(T * dest, const T & val)
@@ -102,7 +102,7 @@ public:
             flag->arity = 0;
             flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
             return *this;
-        };
+        }
 
         FlagMaker & mkHashTypeFlag(HashType * ht);
     };

From 07294e988cc54f1c2570c567662cf39eabbd610c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sun, 10 Nov 2019 17:29:13 +0100
Subject: [PATCH 1663/2196] manual: nix-shell: Elaborate on using `-p` with
 expressions.

This documents the outcome of the change in
https://github.com/NixOS/nix/issues/454:

> We can also automatically add parentheses in the generated
> `buildInputs`, so you can type `nix-shell -p "expr"`
> instead of `"(expr").
---
 doc/manual/command-ref/nix-shell.xml  | 24 +++++++++++++++++++-----
 doc/manual/command-ref/opt-common.xml |  9 ++++++++-
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index bb4a4e42012..766482460c8 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -39,7 +39,12 @@
           <arg choice='plain'><option>--packages</option></arg>
           <arg choice='plain'><option>-p</option></arg>
         </group>
-        <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg>
+        <arg choice='plain' rep='repeat'>
+          <group choice='req'>
+            <arg choice="plain"><replaceable>packages</replaceable></arg>
+            <arg choice="plain"><replaceable>expressions</replaceable></arg>
+          </group>
+        </arg>
       </arg>
       <arg><replaceable>path</replaceable></arg>
     </group>
@@ -189,8 +194,8 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 <variablelist>
 
   <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term>
-    
-    <listitem><para>Shell used to start the interactive environment. 
+
+    <listitem><para>Shell used to start the interactive environment.
     Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem>
 
   </varlistentry>
@@ -222,8 +227,9 @@ $ nix-shell '&lt;nixpkgs>' -A pan --pure \
     --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
 </screen>
 
-Nix expressions can also be given on the command line.  For instance,
-the following starts a shell containing the packages
+Nix expressions can also be given on the command line using the
+<command>-E</command> and <command>-p</command> flags.
+For instance, the following starts a shell containing the packages
 <literal>sqlite</literal> and <literal>libX11</literal>:
 
 <screen>
@@ -238,6 +244,14 @@ $ nix-shell -p sqlite xorg.libX11
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
 </screen>
 
+Note that <command>-p</command> accepts multiple full nix expressions that
+are valid in the <literal>buildInputs = [ ... ]</literal> shown above,
+not only package names. So the following is also legal:
+
+<screen>
+$ nix-shell -p sqlite 'git.override { withManual = false; }'
+</screen>
+
 The <command>-p</command> flag looks up Nixpkgs in the Nix search
 path. You can override it by passing <option>-I</option> or setting
 <envar>NIX_PATH</envar>. For example, the following gives you a shell
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 6a8107be91c..0383bfaedf1 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -323,7 +323,14 @@
   Nix expressions to be parsed and evaluated, rather than as a list
   of file names of Nix expressions.
   (<command>nix-instantiate</command>, <command>nix-build</command>
-  and <command>nix-shell</command> only.)</para></listitem>
+  and <command>nix-shell</command> only.)</para>
+
+  <para>For <command>nix-shell</command>, this option is commonly used
+  to give you a shell in which you can build the packages returned
+  by the expression. If you want to get a shell which contain the
+  <emphasis>built</emphasis> packages ready for use, give your
+  expression to the <command>nix-shell -p</command> convenience flag
+  instead.</para></listitem>
 
 </varlistentry>
 

From 8669db1dcc9378bd61438e6c5978aea5d53d546f Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sun, 10 Nov 2019 16:21:59 -0500
Subject: [PATCH 1664/2196] Clean up semicolon and comma

Thanks @bhipple for catching!
---
 src/libstore/store-api.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b860b48beb4..1862fa267e8 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -21,7 +21,7 @@ namespace nix {
 
 
 MakeError(SubstError, Error);
-MakeError(BuildError, Error) /* denotes a permanent build failure */;
+MakeError(BuildError, Error); // denotes a permanent build failure
 MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);

From 0352c1a4f8fdf911a3def97be1ab3d97a1817000 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Nov 2019 17:18:17 +0100
Subject: [PATCH 1665/2196] Typo

---
 src/build-remote/build-remote.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 279ae62f69c..27dd3578458 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -188,7 +188,7 @@ static int _main(int argc, char * * argv)
 
                     Store::Params storeParams;
                     if (hasPrefix(bestMachine->storeUri, "ssh://")) {
-                        storeParams["max-connections"] ="1";
+                        storeParams["max-connections"] = "1";
                         storeParams["log-fd"] = "4";
                         if (bestMachine->sshKey != "")
                             storeParams["ssh-key"] = bestMachine->sshKey;

From 3e2c77d001a31952e62bfbb825bc7e034678ae12 Mon Sep 17 00:00:00 2001
From: Chuck <chuck@intelligence.org>
Date: Fri, 15 Nov 2019 11:55:36 -0800
Subject: [PATCH 1666/2196] Check for and repair bad .links entries

A corrupt entry in .links prevents adding a fixed version of that file
to the store in any path.  The user experience is that corruption
present in the store 'spreads' to new paths added to the store:

(With store optimisation enabled)

1. A file in the store gets corrupted somehow (eg: filesystem bug).
2. The user tries to add a thing to the store which contains a good copy
   of the corrupted file.
3. The file being added to the store is hashed, found to match the bad
   .links entry, and is replaced by a link to the bad .links entry.
   (The .links entry's hash is not verified during add -- this would
   impose a substantial performance burden.)
4. The user observes that the thing in the store that is supposed to be
   a copy of what they were trying to add is not a correct copy -- some
   files have different contents!  Running "nix-store --verify
   --check-contents --repair" does not fix the problem.

This change makes "nix-store --verify --check-contents --repair" fix
this problem.  Bad .links entries are simply removed, allowing future
attempts to insert a good copy of the file to succeed.
---
 src/libstore/local-store.cc | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b5b0f1ba745..2a21f816cde 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1238,7 +1238,35 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     /* Optionally, check the content hashes (slow). */
     if (checkContents) {
-        printInfo("checking hashes...");
+
+        printInfo("checking link hashes...");
+
+        AutoCloseDir dir(opendir(linksDir.c_str()));
+        if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+
+        struct dirent * dirent;
+        while (errno = 0, dirent = readdir(dir.get())) { /* sic */
+            checkInterrupt();
+            if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) continue;
+            Path linkPath = linksDir + "/" + dirent->d_name;
+            string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
+            if (hash != dirent->d_name) {
+                printError(format("link '%1%' was modified! "
+                                  "expected hash '%2%', got '%3%'")
+                                  % linkPath % dirent->d_name % hash);
+                if (repair) {
+                    if (unlink(linkPath.c_str()) == 0)
+                        printError(format("Removed link '%1%'") % linkPath);
+                    else
+                        throw SysError(format("removing corrupt link '%1%'") % linkPath);
+                } else {
+                    errors = true;
+                }
+            }
+        }
+        if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
+
+        printInfo("checking store hashes...");
 
         Hash nullHash(htSHA256);
 

From ba87b08f8529e4d9f8c58d8c625152058ceadb75 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Nov 2019 16:06:44 +0100
Subject: [PATCH 1667/2196] getEnv(): Return std::optional

This allows distinguishing between an empty value and no value.
---
 src/libexpr/eval.cc          | 12 ++++++------
 src/libexpr/primops.cc       |  2 +-
 src/libstore/build.cc        |  4 ++--
 src/libstore/gc.cc           |  6 +++---
 src/libstore/globals.cc      | 22 +++++++++++-----------
 src/libstore/globals.hh      |  2 +-
 src/libstore/local-store.hh  |  2 +-
 src/libstore/ssh.cc          |  2 +-
 src/libutil/util.cc          | 35 +++++++++++++++--------------------
 src/libutil/util.hh          |  2 +-
 src/nix-build/nix-build.cc   | 17 +++++++++--------
 src/nix-daemon/nix-daemon.cc |  5 +++--
 src/nix-env/nix-env.cc       |  2 +-
 src/nix/command.cc           |  2 +-
 src/nix/doctor.cc            |  4 ++--
 src/nix/run.cc               |  2 +-
 src/nix/upgrade-nix.cc       |  2 +-
 17 files changed, 60 insertions(+), 63 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3c3d1a324f7..bb6b23ff15a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -217,7 +217,7 @@ void initGC()
        that GC_expand_hp() causes a lot of virtual, but not physical
        (resident) memory to be allocated.  This might be a problem on
        systems that don't overcommit. */
-    if (!getenv("GC_INITIAL_HEAP_SIZE")) {
+    if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
         size_t size = 32 * 1024 * 1024;
 #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
         size_t maxSize = 384 * 1024 * 1024;
@@ -306,7 +306,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
-    countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
+    countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
     assert(gcInitialised);
 
@@ -314,7 +314,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
+        Strings paths = parseNixPath(getEnv("NIX_PATH").value_or(""));
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }
@@ -1753,7 +1753,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 
 void EvalState::printStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
+    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1769,7 +1769,7 @@ void EvalState::printStats()
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
 #endif
     if (showStats) {
-        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-");
         std::fstream fs;
         if (outPath != "-")
             fs.open(outPath, std::fstream::out);
@@ -1861,7 +1861,7 @@ void EvalState::printStats()
             }
         }
 
-        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+        if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
             auto list = topObj.list("symbols");
             symbols.dump([&](const std::string & s) { list.elem(s); });
         }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index deaffa605fe..2cc03fe61bb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -469,7 +469,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 35a5388c674..67ccfcee0d6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2503,7 +2503,7 @@ void DerivationGoal::initEnv()
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
-            env[i] = getEnv(i);
+            env[i] = getEnv(i).value_or("");
     }
 
     /* Currently structured log messages piggyback on stderr, but we
@@ -3075,7 +3075,7 @@ void DerivationGoal::runChild()
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
+            Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
 
             /* They don't like trailing slashes on subpath directives */
             if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 7d3da1cc850..842980fb7cd 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -870,11 +870,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
 void LocalStore::autoGC(bool sync)
 {
-    static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", "");
+    static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");
 
     auto getAvail = [this]() -> uint64_t {
-        if (!fakeFreeSpaceFile.empty())
-            return std::stoll(readFile(fakeFreeSpaceFile));
+        if (fakeFreeSpaceFile)
+            return std::stoll(readFile(*fakeFreeSpaceFile));
 
         struct statvfs st;
         if (statvfs(realStoreDir.c_str(), &st))
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 249c3667357..aa0823f41be 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -32,20 +32,20 @@ static GlobalConfig::Register r1(&settings);
 
 Settings::Settings()
     : nixPrefix(NIX_PREFIX)
-    , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))))
-    , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)))
-    , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)))
-    , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)))
-    , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
-    , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
-    , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
+    , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR))))
+    , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR)))
+    , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
+    , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
+    , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
+    , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
+    , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
-    lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
+    lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
 
-    caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", ""));
+    caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (caFile == "") {
         for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
             if (pathExists(fn)) {
@@ -56,9 +56,9 @@ Settings::Settings()
 
     /* Backwards compatibility. */
     auto s = getEnv("NIX_REMOTE_SYSTEMS");
-    if (s != "") {
+    if (s) {
         Strings ss;
-        for (auto & p : tokenizeString<Strings>(s, ":"))
+        for (auto & p : tokenizeString<Strings>(*s, ":"))
             ss.push_back("@" + p);
         builders = concatStringsSep(" ", ss);
     }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 1221e4db7ab..3eb2f698cfc 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -66,7 +66,7 @@ public:
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
-    Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store",
+    Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
         "The default Nix store to use."};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 5aa6b051964..1d4b888980e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -107,7 +107,7 @@ private:
 public:
 
     // Hack for build-remote.cc.
-    PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+    PathSet locksHeld;
 
     /* Initialise the local store, upgrading the schema if
        necessary. */
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 5e0e44935cc..ac3ccd63d14 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -16,7 +16,7 @@ SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool
 
 void SSHMaster::addCommonSSHOpts(Strings & args)
 {
-    for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS")))
+    for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS").value_or("")))
         args.push_back(i);
     if (!keyFile.empty())
         args.insert(args.end(), {"-i", keyFile});
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 79716c5a7b8..b4d444a6746 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -58,10 +58,11 @@ std::string SysError::addErrno(const std::string & s)
 }
 
 
-string getEnv(const string & key, const string & def)
+std::optional<std::string> getEnv(const std::string & key)
 {
     char * value = getenv(key.c_str());
-    return value ? string(value) : def;
+    if (!value) return {};
+    return std::string(value);
 }
 
 
@@ -438,7 +439,7 @@ void deletePath(const Path & path, unsigned long long & bytesFreed)
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
     int & counter)
 {
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true);
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
     if (includePid)
         return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
     else
@@ -480,7 +481,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
-    std::string name = pw ? pw->pw_name : getEnv("USER", "");
+    std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
     if (name.empty())
         throw Error("cannot figure out user name");
     return name;
@@ -488,8 +489,8 @@ std::string getUserName()
 
 
 static Lazy<Path> getHome2([]() {
-    Path homeDir = getEnv("HOME");
-    if (homeDir.empty()) {
+    auto homeDir = getEnv("HOME");
+    if (!homeDir) {
         std::vector<char> buf(16384);
         struct passwd pwbuf;
         struct passwd * pw;
@@ -498,7 +499,7 @@ static Lazy<Path> getHome2([]() {
             throw Error("cannot determine user's home directory");
         homeDir = pw->pw_dir;
     }
-    return homeDir;
+    return *homeDir;
 });
 
 Path getHome() { return getHome2(); }
@@ -506,25 +507,21 @@ Path getHome() { return getHome2(); }
 
 Path getCacheDir()
 {
-    Path cacheDir = getEnv("XDG_CACHE_HOME");
-    if (cacheDir.empty())
-        cacheDir = getHome() + "/.cache";
-    return cacheDir;
+    auto cacheDir = getEnv("XDG_CACHE_HOME");
+    return cacheDir ? *cacheDir : getHome() + "/.cache";
 }
 
 
 Path getConfigDir()
 {
-    Path configDir = getEnv("XDG_CONFIG_HOME");
-    if (configDir.empty())
-        configDir = getHome() + "/.config";
-    return configDir;
+    auto configDir = getEnv("XDG_CONFIG_HOME");
+    return configDir ? *configDir : getHome() + "/.config";
 }
 
 std::vector<Path> getConfigDirs()
 {
     Path configHome = getConfigDir();
-    string configDirs = getEnv("XDG_CONFIG_DIRS");
+    string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("");
     std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":");
     result.insert(result.begin(), configHome);
     return result;
@@ -533,10 +530,8 @@ std::vector<Path> getConfigDirs()
 
 Path getDataDir()
 {
-    Path dataDir = getEnv("XDG_DATA_HOME");
-    if (dataDir.empty())
-        dataDir = getHome() + "/.local/share";
-    return dataDir;
+    auto dataDir = getEnv("XDG_DATA_HOME");
+    return dataDir ? *dataDir : getHome() + "/.local/share";
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index e8fed353ce6..b9f9ea8829b 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -35,7 +35,7 @@ extern const std::string nativeSystem;
 
 
 /* Return an environment variable. */
-string getEnv(const string & key, const string & def = "");
+std::optional<std::string> getEnv(const std::string & key);
 
 /* Get the entire environment. */
 std::map<std::string, std::string> getEnv();
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 1c5d7168189..c506f9b0a59 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -344,9 +344,9 @@ static void _main(int argc, char * * argv)
         /* Figure out what bash shell to use. If $NIX_BUILD_SHELL
            is not set, then build bashInteractive from
            <nixpkgs>. */
-        auto shell = getEnv("NIX_BUILD_SHELL", "");
+        auto shell = getEnv("NIX_BUILD_SHELL");
 
-        if (shell == "") {
+        if (!shell) {
 
             try {
                 auto expr = state->parseExprFromString("(import <nixpkgs> {}).bashInteractive", absPath("."));
@@ -382,7 +382,8 @@ static void _main(int argc, char * * argv)
         // Set the environment.
         auto env = getEnv();
 
-        auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp"));
+        auto tmp = getEnv("TMPDIR");
+        if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp");
 
         if (pure) {
             decltype(env) newEnv;
@@ -394,7 +395,7 @@ static void _main(int argc, char * * argv)
             env["__ETC_PROFILE_SOURCED"] = "1";
         }
 
-        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
+        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp;
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 
@@ -439,8 +440,8 @@ static void _main(int argc, char * * argv)
                 (Path) tmpDir,
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
-                dirOf(shell),
-                shell,
+                dirOf(*shell),
+                *shell,
                 (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
                 envCommand));
 
@@ -460,9 +461,9 @@ static void _main(int argc, char * * argv)
 
         restoreSignals();
 
-        execvp(shell.c_str(), argPtrs.data());
+        execvp(shell->c_str(), argPtrs.data());
 
-        throw SysError("executing shell '%s'", shell);
+        throw SysError("executing shell '%s'", *shell);
     }
 
     else {
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 65ae88fafe4..9a4d60726eb 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -161,8 +161,9 @@ static void daemonLoop(char * * argv)
     AutoCloseFD fdSocket;
 
     /* Handle socket-based activation by systemd. */
-    if (getEnv("LISTEN_FDS") != "") {
-        if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1")
+    auto listenFds = getEnv("LISTEN_FDS");
+    if (listenFds) {
+        if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1")
             throw Error("unexpected systemd environment variables");
         fdSocket = SD_LISTEN_FDS_START;
         closeOnExec(fdSocket.get());
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 199dc92aa1c..a9c7439555d 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1425,7 +1425,7 @@ static int _main(int argc, char * * argv)
         globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state);
 
         if (globals.profile == "")
-            globals.profile = getEnv("NIX_PROFILE", "");
+            globals.profile = getEnv("NIX_PROFILE").value_or("");
 
         if (globals.profile == "") {
             Path profileLink = getHome() + "/.nix-profile";
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 97c2fcf1c96..724f03e5d76 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -156,7 +156,7 @@ void StorePathCommand::run(ref<Store> store)
 
 Strings editorFor(const Pos & pos)
 {
-    auto editor = getEnv("EDITOR", "cat");
+    auto editor = getEnv("EDITOR").value_or("cat");
     auto args = tokenizeString<Strings>(editor);
     if (pos.line > 0 && (
         editor.find("emacs") != std::string::npos ||
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 795236c5f50..481c93b45d8 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -68,7 +68,7 @@ struct CmdDoctor : StoreCommand
     {
         PathSet dirs;
 
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH").value_or(""), ":"))
             if (pathExists(dir + "/nix-env"))
                 dirs.insert(dirOf(canonPath(dir + "/nix-env", true)));
 
@@ -87,7 +87,7 @@ struct CmdDoctor : StoreCommand
     {
         PathSet dirs;
 
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) {
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH").value_or(""), ":")) {
             Path profileDir = dirOf(dir);
             try {
                 Path userEnv = canonPath(profileDir, true);
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 90b76d6663e..fd4f9228294 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -128,7 +128,7 @@ struct CmdRun : InstallablesCommand
         std::queue<Path> todo;
         for (auto & path : outPaths) todo.push(path);
 
-        auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
+        auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
 
         while (!todo.empty()) {
             Path path = todo.front();
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 35c44a70cf5..6c9e37d0412 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -106,7 +106,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
     {
         Path where;
 
-        for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":"))
+        for (auto & dir : tokenizeString<Strings>(getEnv("PATH").value_or(""), ":"))
             if (pathExists(dir + "/nix-env")) {
                 where = dir;
                 break;

From 1c3ccba0f5e658fa5d593ddef8aaaf6728f945a5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Nov 2019 16:27:48 +0100
Subject: [PATCH 1668/2196] Remove $NIX_USER_PROFILE_DIR

This is not used anywhere.
---
 scripts/nix-profile-daemon.sh.in | 1 -
 scripts/nix-profile.sh.in        | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index 47655080a67..e0c24954a65 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -2,7 +2,6 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index e15f7cd46b7..0768c805a0c 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -5,7 +5,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_LINK=$HOME/.nix-profile
 
-    NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
 
     # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
     # paths work when the user has fetched the Nixpkgs channel.
@@ -35,5 +34,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     fi
 
     export PATH="$NIX_LINK/bin:$PATH"
-    unset NIX_LINK NIX_USER_PROFILE_DIR
+    unset NIX_LINK
 fi

From ec9dd9a5aeb9b7fca170d86b8906ffac7e5ef057 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Nov 2019 22:08:51 +0100
Subject: [PATCH 1669/2196] Provide a default value for NIX_PATH

---
 scripts/nix-profile-daemon.sh.in |  1 -
 scripts/nix-profile.sh.in        |  5 -----
 src/libexpr/eval.cc              | 14 +++++++++++++-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index e0c24954a65..4bc7c6fc593 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -24,5 +24,4 @@ else
   done
 fi
 
-export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
 export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 0768c805a0c..8cba1c522fa 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -5,11 +5,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_LINK=$HOME/.nix-profile
 
-
-    # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
-    # paths work when the user has fetched the Nixpkgs channel.
-    export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
-
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index bb6b23ff15a..65466a0b4e7 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -275,6 +275,17 @@ static Strings parseNixPath(const string & s)
 }
 
 
+static Strings getDefaultNixPath()
+{
+    Strings res;
+    auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
+    add(getHome() + "/.nix-defexpr/channels");
+    add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
+    add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
+    return res;
+}
+
+
 EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     : sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
@@ -314,7 +325,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        Strings paths = parseNixPath(getEnv("NIX_PATH").value_or(""));
+        auto nixPath = getEnv("NIX_PATH");
+        auto paths = nixPath ? parseNixPath(*nixPath) : getDefaultNixPath();
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : paths) addToSearchPath(i);
     }

From d12d69ea1a871d631d77c8ef5e8468b4a2bff80f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Nov 2019 23:07:35 +0100
Subject: [PATCH 1670/2196] Turn NIX_PATH into a config setting

This allows it to be set in nix.conf.
---
 src/libexpr/eval.cc | 31 +++++++++++++++++--------------
 src/libexpr/eval.hh |  7 +++++++
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 65466a0b4e7..62812f90a88 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -275,17 +275,6 @@ static Strings parseNixPath(const string & s)
 }
 
 
-static Strings getDefaultNixPath()
-{
-    Strings res;
-    auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
-    add(getHome() + "/.nix-defexpr/channels");
-    add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
-    add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
-    return res;
-}
-
-
 EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     : sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
@@ -325,10 +314,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        auto nixPath = getEnv("NIX_PATH");
-        auto paths = nixPath ? parseNixPath(*nixPath) : getDefaultNixPath();
         for (auto & i : _searchPath) addToSearchPath(i);
-        for (auto & i : paths) addToSearchPath(i);
+        for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
     addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
@@ -1986,6 +1973,22 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings::EvalSettings()
+{
+    auto var = getEnv("NIX_PATH");
+    if (var) nixPath = parseNixPath(*var);
+}
+
+Strings EvalSettings::getDefaultNixPath()
+{
+    Strings res;
+    auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
+    add(getHome() + "/.nix-defexpr/channels");
+    add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
+    add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
+    return res;
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register r1(&evalSettings);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 61ee4a73b56..9d075a48a65 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -339,9 +339,16 @@ struct InvalidPathError : EvalError
 
 struct EvalSettings : Config
 {
+    EvalSettings();
+
+    static Strings getDefaultNixPath();
+
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
+    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for <...> file references."};
+
     Setting<bool> restrictEval{this, false, "restrict-eval",
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};

From cd55f91ad2075d5ebb506950dfbc4161cab22db5 Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.com>
Date: Mon, 25 Nov 2019 12:37:14 +0000
Subject: [PATCH 1671/2196] Ensure enough space in attrset bindings when using
 both __overrides and dynamic attributes

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 62812f90a88..3a9b7a3c093 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -875,7 +875,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         if (hasOverrides) {
             Value * vOverrides = (*v.attrs)[overrides->second.displ].value;
             state.forceAttrs(*vOverrides);
-            Bindings * newBnds = state.allocBindings(v.attrs->size() + vOverrides->attrs->size());
+            Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size());
             for (auto & i : *v.attrs)
                 newBnds->push_back(i);
             for (auto & i : *vOverrides->attrs) {

From cdadbf770855e828a0f64d70a47db43e920ad137 Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.com>
Date: Mon, 25 Nov 2019 13:03:54 +0000
Subject: [PATCH 1672/2196] Add testcase for attrset using __overrides and
 dynamic attrs

---
 tests/lang/eval-okay-attrs6.exp | 1 +
 tests/lang/eval-okay-attrs6.nix | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 tests/lang/eval-okay-attrs6.exp
 create mode 100644 tests/lang/eval-okay-attrs6.nix

diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/lang/eval-okay-attrs6.exp
new file mode 100644
index 00000000000..b46938032e7
--- /dev/null
+++ b/tests/lang/eval-okay-attrs6.exp
@@ -0,0 +1 @@
+{ __overrides = { bar = "qux"; }; bar = "qux"; foo = "bar"; }
diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/lang/eval-okay-attrs6.nix
new file mode 100644
index 00000000000..2e5c85483be
--- /dev/null
+++ b/tests/lang/eval-okay-attrs6.nix
@@ -0,0 +1,4 @@
+rec {
+  "${"foo"}" = "bar";
+   __overrides = { bar = "qux"; };
+}

From 4e70652ee33641a6cfddbfb9b7d741515f1bf98a Mon Sep 17 00:00:00 2001
From: Arnout Engelen <arnout@bzzt.net>
Date: Mon, 25 Nov 2019 18:00:05 +0100
Subject: [PATCH 1673/2196] Document builtins.placeholder

---
 doc/manual/expressions/builtins.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index f39b393bf71..394e1fc32c9 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -1120,6 +1120,16 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry xml:id='builtin-placeholder'>
+    <term><function>builtins.placeholder</function>
+    <replaceable>output</replaceable></term>
+
+    <listitem><para>Return a placeholder string for the specified
+    <replaceable>output</replaceable> that will be substituted by the
+    corresponding output path at build time. Typical outputs would be
+    <literal>"out"</literal>, <literal>"bin"</literal> or
+    <literal>"dev"</literal>.</para></listitem>
+  </varlistentry>
 
   <varlistentry xml:id='builtin-readDir'>
     <term><function>builtins.readDir</function>

From 1ec6e6e11e2dbca8803305757d472dabe5878129 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 19:48:34 +0100
Subject: [PATCH 1674/2196] Add feature to disable URL literals

E.g.

  $ nix-build '<nixpkgs>' -A hello --experimental-features no-url-literals
  error: URL literals are disabled, at /nix/store/vsjamkzh15r3c779q2711az826hqgvzr-nixpkgs-20.03pre194957.bef773ed53f/nixpkgs/pkgs/top-level/all-packages.nix:1236:11

Helps with implementing https://github.com/NixOS/rfcs/pull/45.
---
 src/libexpr/parser.y    | 8 +++++++-
 src/libstore/globals.cc | 9 +++++++--
 src/libstore/globals.hh | 2 ++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 93834bec664..10a057062e6 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -20,6 +20,7 @@
 
 #include "nixexpr.hh"
 #include "eval.hh"
+#include "globals.hh"
 
 namespace nix {
 
@@ -401,7 +402,12 @@ expr_simple
               new ExprVar(data->symbols.create("__nixPath"))),
           new ExprString(data->symbols.create(path)));
   }
-  | URI { $$ = new ExprString(data->symbols.create($1)); }
+  | URI {
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      if (noURLLiterals)
+          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+      $$ = new ExprString(data->symbols.create($1));
+  }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index aa0823f41be..6d3b957402a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -105,10 +105,15 @@ StringSet Settings::getDefaultSystemFeatures()
     return features;
 }
 
-void Settings::requireExperimentalFeature(const std::string & name)
+bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 {
     auto & f = experimentalFeatures.get();
-    if (std::find(f.begin(), f.end(), name) == f.end())
+    return std::find(f.begin(), f.end(), name) != f.end();
+}
+
+void Settings::requireExperimentalFeature(const std::string & name)
+{
+    if (!isExperimentalFeatureEnabled(name))
         throw Error("experimental Nix feature '%s' is disabled", name);
 }
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3eb2f698cfc..9f395425ab9 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -357,6 +357,8 @@ public:
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
+    bool isExperimentalFeatureEnabled(const std::string & name);
+
     void requireExperimentalFeature(const std::string & name);
 };
 

From 89db9353d7cc5aaf360c48d17aeb0e9ddd940451 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 20:08:25 +0100
Subject: [PATCH 1675/2196] Doh

---
 src/libstore/local-store.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b5b0f1ba745..e9dd3f96ecd 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -54,6 +54,7 @@ LocalStore::LocalStore(const Params & params)
     , trashDir(realStoreDir + "/trash")
     , tempRootsDir(stateDir + "/temproots")
     , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
+    , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
 {
     auto state(_state.lock());
 

From c13193017fdcb910627698e343c2df4183d91f8f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 20:12:15 +0100
Subject: [PATCH 1676/2196] Disallow empty store path names

Fixes #3239.
---
 src/libstore/local-store.cc | 2 ++
 src/libstore/store-api.cc   | 9 +++++++--
 tests/referrers.sh          | 4 ++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e9dd3f96ecd..7d127840a97 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -585,6 +585,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
+    checkStoreName(storePathToName(info.path));
+
     if (info.ca != "" && !info.isContentAddressed(*this))
         throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path);
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 54430d3baac..0758907e739 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -90,17 +90,22 @@ void checkStoreName(const string & name)
         "Path names are alphanumeric and can include the symbols %1% "
         "and must not begin with a period. "
         "Note: If '%2%' is a source file and you cannot rename it on "
-        "disk, builtins.path { name = ... } can be used to give it an "
+        "disk, 'builtins.path { name = ... }' can be used to give it an "
         "alternative name.") % validChars % name;
 
+    if (name.empty())
+        throw Error(baseError % "it is an empty string");
+
     /* Disallow names starting with a dot for possible security
        reasons (e.g., "." and ".."). */
-    if (string(name, 0, 1) == ".")
+    if (name[0] == '.')
         throw Error(baseError % "it is illegal to start the name with a period");
+
     /* Disallow names longer than 211 characters. ext4’s max is 256,
        but we need extra space for the hash and .chroot extensions. */
     if (name.length() > 211)
         throw Error(baseError % "name must be less than 212 characters");
+
     for (auto & i : name)
         if (!((i >= 'A' && i <= 'Z') ||
               (i >= 'a' && i <= 'z') ||
diff --git a/tests/referrers.sh b/tests/referrers.sh
index 8ab8e5ddfe8..614dd8f5b96 100644
--- a/tests/referrers.sh
+++ b/tests/referrers.sh
@@ -4,9 +4,9 @@ clearStore
 
 max=500
 
-reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla
 touch $reference
-(echo $reference && echo && echo 0) | nix-store --register-validity 
+(echo $reference && echo && echo 0) | nix-store --register-validity
 
 echo "making registration..."
 

From ec5e7b44ff1ce14eb930948df1b89f07abbf16f9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 20:26:22 +0100
Subject: [PATCH 1677/2196] Simplify

---
 src/libstore/local-store.cc | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0d44665b005..60e7bb7af9e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1244,30 +1244,24 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
         printInfo("checking link hashes...");
 
-        AutoCloseDir dir(opendir(linksDir.c_str()));
-        if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
-
-        struct dirent * dirent;
-        while (errno = 0, dirent = readdir(dir.get())) { /* sic */
-            checkInterrupt();
-            if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) continue;
-            Path linkPath = linksDir + "/" + dirent->d_name;
+        for (auto & link : readDirectory(linksDir)) {
+            printMsg(lvlTalkative, "checking contents of '%s'", link.name);
+            Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
-            if (hash != dirent->d_name) {
-                printError(format("link '%1%' was modified! "
-                                  "expected hash '%2%', got '%3%'")
-                                  % linkPath % dirent->d_name % hash);
+            if (hash != link.name) {
+                printError(
+                    "link '%s' was modified! expected hash '%s', got '%s'",
+                    linkPath, link.name, hash);
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
-                        printError(format("Removed link '%1%'") % linkPath);
+                        printError("removed link '%s'", linkPath);
                     else
-                        throw SysError(format("removing corrupt link '%1%'") % linkPath);
+                        throw SysError("removing corrupt link '%s'", linkPath);
                 } else {
                     errors = true;
                 }
             }
         }
-        if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
 
         printInfo("checking store hashes...");
 

From 0be8d7784ffad0131c2e69fe25350fe5d2132e34 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 20:33:46 +0100
Subject: [PATCH 1678/2196] Typo

---
 doc/manual/command-ref/nix-build.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml
index c1b783c87d6..886d2591043 100644
--- a/doc/manual/command-ref/nix-build.xml
+++ b/doc/manual/command-ref/nix-build.xml
@@ -100,7 +100,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
   </varlistentry>
 
   <varlistentry><term><option>--dry-run</option></term>
-   <listitem><para>Show what store paths would be built or downloaded</para></listitem>
+   <listitem><para>Show what store paths would be built or downloaded.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /

From 313106d5491c7c6279f3c962348462ca04cd2e0e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 21:07:44 +0100
Subject: [PATCH 1679/2196] Fix clang warnings

---
 src/libstore/nar-info-disk-cache.hh | 2 ++
 src/libstore/remote-store.hh        | 2 +-
 src/libstore/ssh-store.cc           | 2 +-
 src/libstore/store-api.cc           | 2 +-
 src/nix/command.hh                  | 3 +++
 5 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index 11e6c55ca9c..285873f7ede 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -10,6 +10,8 @@ class NarInfoDiskCache
 public:
     typedef enum { oValid, oInvalid, oUnknown } Outcome;
 
+    virtual ~NarInfoDiskCache() { };
+
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
 
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 84de4c4601c..728df8b003e 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -148,7 +148,7 @@ public:
 
     std::string getUri() override;
 
-    bool sameMachine()
+    bool sameMachine() override
     { return true; }
 
 private:
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 93e11738991..b9d0c41eafc 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -35,7 +35,7 @@ class SSHStore : public RemoteStore
         return uriScheme + host;
     }
 
-    bool sameMachine()
+    bool sameMachine() override
     { return false; }
 
     void narFromPath(const Path & path, Sink & sink) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0758907e739..7e548ecd235 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -216,7 +216,7 @@ static std::string makeType(string && type, const PathSet & references)
         type += ":";
         type += i;
     }
-    return type;
+    return std::move(type);
 }
 
 
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 26c15e64617..2db22dba2f3 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -16,6 +16,7 @@ struct Pos;
    run() method. */
 struct Command : virtual Args
 {
+    virtual ~Command() { }
     virtual std::string name() = 0;
     virtual void prepare() { };
     virtual void run() = 0;
@@ -58,6 +59,8 @@ typedef std::vector<Buildable> Buildables;
 
 struct Installable
 {
+    virtual ~Installable() { }
+
     virtual std::string what() = 0;
 
     virtual Buildables toBuildables()

From abb8ef619ba2fab3ae16fb5b5430215905bac723 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 21:08:56 +0100
Subject: [PATCH 1680/2196] Fix macOS build

https://hydra.nixos.org/build/107457009
---
 src/libmain/shared.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d3dbfbc4459..81fb6b05ad7 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -155,7 +155,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/"))
+    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }

From 11da5b2816e11b081d8ff38db4330addd2014f7e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 14:12:20 +0100
Subject: [PATCH 1681/2196] Add some Rust code

---
 .gitignore          |  2 ++
 Makefile            |  1 +
 nix-rust/Cargo.lock | 59 +++++++++++++++++++++++++++++++++++++++++++
 nix-rust/Cargo.toml | 12 +++++++++
 nix-rust/local.mk   |  7 ++++++
 nix-rust/src/lib.rs | 48 +++++++++++++++++++++++++++++++++++
 release-common.nix  |  1 +
 shell.nix           |  2 +-
 src/nix/local.mk    |  4 +--
 src/nix/test.cc     | 61 +++++++++++++++++++++++++++++++++++++++++++++
 10 files changed, 194 insertions(+), 3 deletions(-)
 create mode 100644 nix-rust/Cargo.lock
 create mode 100644 nix-rust/Cargo.toml
 create mode 100644 nix-rust/local.mk
 create mode 100644 nix-rust/src/lib.rs
 create mode 100644 src/nix/test.cc

diff --git a/.gitignore b/.gitignore
index fd62dfb3856..f58e1f90fc8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -119,3 +119,5 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
+
+nix-rust/target
diff --git a/Makefile b/Makefile
index 866c0961eb6..469070533b5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,7 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \
diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
new file mode 100644
index 00000000000..cfaedc3ed4c
--- /dev/null
+++ b/nix-rust/Cargo.lock
@@ -0,0 +1,59 @@
+[[package]]
+name = "cfg-if"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "filetime"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "nix-rust"
+version = "0.1.0"
+dependencies = [
+ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "tar"
+version = "0.4.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "xattr"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[metadata]
+"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4"
+"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646"
+"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1"
+"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85"
+"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133"
+"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
new file mode 100644
index 00000000000..2d2bf67523d
--- /dev/null
+++ b/nix-rust/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "nix-rust"
+version = "0.1.0"
+authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+
+[lib]
+name = "nixrust"
+crate-type = ["staticlib"]
+
+[dependencies]
+tar = "0.4"
+libc = "0.2"
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
new file mode 100644
index 00000000000..c69e3b9ea2c
--- /dev/null
+++ b/nix-rust/local.mk
@@ -0,0 +1,7 @@
+libnixrust_PATH := $(d)/target/release/libnixrust.a
+libnixrust_INSTALL_PATH := $(libnixrust_PATH)
+libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
+
+$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
new file mode 100644
index 00000000000..abcb8905534
--- /dev/null
+++ b/nix-rust/src/lib.rs
@@ -0,0 +1,48 @@
+extern crate libc;
+extern crate tar;
+
+use std::fs;
+use std::io;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::Path;
+use tar::Archive;
+
+#[no_mangle]
+pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool {
+    // FIXME: handle errors.
+
+    let dest_dir = Path::new(dest_dir);
+
+    let mut tar = Archive::new(data);
+
+    for file in tar.entries().unwrap() {
+        let mut file = file.unwrap();
+
+        let dest_file = dest_dir.join(file.header().path().unwrap());
+
+        fs::create_dir_all(dest_file.parent().unwrap()).unwrap();
+
+        match file.header().entry_type() {
+            tar::EntryType::Directory => {
+                fs::create_dir(dest_file).unwrap();
+            }
+            tar::EntryType::Regular => {
+                let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 {
+                    0o666
+                } else {
+                    0o777
+                };
+                let mut f = fs::OpenOptions::new()
+                    .create(true)
+                    .write(true)
+                    .mode(mode)
+                    .open(dest_file)
+                    .unwrap();
+                io::copy(&mut file, &mut f).unwrap();
+            }
+            t => panic!("Unsupported tar entry type '{:?}'.", t),
+        }
+    }
+
+    true
+}
diff --git a/release-common.nix b/release-common.nix
index 63f39f00548..dd5f939d969 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -51,6 +51,7 @@ rec {
       openssl pkgconfig sqlite boehmgc
       boost
       nlohmann_json
+      rustc cargo
 
       # Tests
       git
diff --git a/shell.nix b/shell.nix
index 9c504f024e9..21ed4712133 100644
--- a/shell.nix
+++ b/shell.nix
@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 
diff --git a/src/nix/local.mk b/src/nix/local.mk
index c09efd1fc89..a145cf9b1d1 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,9 +15,9 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil
+nix_LIBS = libexpr libmain libstore libutil libnixrust
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/src/nix/test.cc b/src/nix/test.cc
new file mode 100644
index 00000000000..70e0a903de4
--- /dev/null
+++ b/src/nix/test.cc
@@ -0,0 +1,61 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "common-args.hh"
+
+using namespace nix;
+
+namespace rust {
+
+// Depending on the internal representation of Rust slices is slightly
+// evil...
+template<typename T> struct Slice
+{
+    const T * ptr;
+    size_t size;
+
+    Slice(const T * ptr, size_t size) : ptr(ptr), size(size)
+    {
+        assert(ptr);
+    }
+};
+
+struct StringSlice : Slice<char>
+{
+    StringSlice(const std::string & s): Slice(s.data(), s.size()) { }
+};
+
+}
+
+extern "C" {
+    bool unpack_tarfile(rust::Slice<uint8_t> data, rust::StringSlice dest_dir);
+}
+
+struct CmdTest : StoreCommand
+{
+    CmdTest()
+    {
+    }
+
+    std::string name() override
+    {
+        return "test";
+    }
+
+    std::string description() override
+    {
+        return "bla bla";
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto data = readFile("./nix-2.2.tar");
+
+        std::string destDir = "./dest";
+
+        deletePath(destDir);
+
+        unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir);
+    }
+};
+
+static RegisterCommand r(make_ref<CmdTest>());

From e60f6bd4ce831ced94fafeb527c429b6f88159ac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 20:45:56 +0100
Subject: [PATCH 1682/2196] Enable Rust code to call C++ Source objects

---
 nix-rust/src/lib.rs | 19 +++++++++++++++++--
 src/nix/test.cc     | 34 ++++++++++++++++++++++++++++------
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index abcb8905534..799d52e313c 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -7,13 +7,28 @@ use std::os::unix::fs::OpenOptionsExt;
 use std::path::Path;
 use tar::Archive;
 
+/// A wrapper around Nix's Source class that provides the Read trait.
+#[repr(C)]
+pub struct Source {
+    fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
+    this: *mut libc::c_void,
+}
+
+impl std::io::Read for Source {
+    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+        let n = (self.fun)(self.this, buf);
+        assert!(n <= buf.len());
+        Ok(n)
+    }
+}
+
 #[no_mangle]
-pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool {
+pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool {
     // FIXME: handle errors.
 
     let dest_dir = Path::new(dest_dir);
 
-    let mut tar = Archive::new(data);
+    let mut tar = Archive::new(source);
 
     for file in tar.entries().unwrap() {
         let mut file = file.unwrap();
diff --git a/src/nix/test.cc b/src/nix/test.cc
index 70e0a903de4..7f1f0d47aab 100644
--- a/src/nix/test.cc
+++ b/src/nix/test.cc
@@ -1,6 +1,7 @@
 #include "command.hh"
 #include "store-api.hh"
 #include "common-args.hh"
+#include "compression.hh"
 
 using namespace nix;
 
@@ -10,10 +11,10 @@ namespace rust {
 // evil...
 template<typename T> struct Slice
 {
-    const T * ptr;
+    T * ptr;
     size_t size;
 
-    Slice(const T * ptr, size_t size) : ptr(ptr), size(size)
+    Slice(T * ptr, size_t size) : ptr(ptr), size(size)
     {
         assert(ptr);
     }
@@ -21,13 +22,30 @@ template<typename T> struct Slice
 
 struct StringSlice : Slice<char>
 {
-    StringSlice(const std::string & s): Slice(s.data(), s.size()) { }
+    StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
+};
+
+struct Source
+{
+    size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
+    nix::Source * _this;
+
+    Source(nix::Source & _this)
+        : fun(sourceWrapper), _this(&_this)
+    {}
+
+    // FIXME: how to propagate exceptions?
+    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
+    {
+        auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
+        return n;
+    }
 };
 
 }
 
 extern "C" {
-    bool unpack_tarfile(rust::Slice<uint8_t> data, rust::StringSlice dest_dir);
+    bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }
 
 struct CmdTest : StoreCommand
@@ -48,13 +66,17 @@ struct CmdTest : StoreCommand
 
     void run(ref<Store> store) override
     {
-        auto data = readFile("./nix-2.2.tar");
+        auto source = sinkToSource([&](Sink & sink) {
+            auto decompressor = makeDecompressionSink("bzip2", sink);
+            readFile("./nix-2.2.tar.bz2", *decompressor);
+            decompressor->finish();
+        });
 
         std::string destDir = "./dest";
 
         deletePath(destDir);
 
-        unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir);
+        unpack_tarfile(*source, destDir);
     }
 };
 

From 045708db4343174f30f3647776971c852f72a9e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 23:40:35 +0100
Subject: [PATCH 1683/2196] Make <nix/unpack-channel.nix> a builtin builder

This was the last function using a shell script, so this allows us to
get rid of tar, coreutils, bash etc.
---
 corepkgs/config.nix.in                  | 10 +-----
 corepkgs/unpack-channel.nix             | 35 +++------------------
 nix-rust/src/lib.rs                     |  5 ++-
 src/libstore/build.cc                   |  2 ++
 src/libstore/builtins.hh                |  1 +
 src/libstore/builtins/unpack-channel.cc | 39 +++++++++++++++++++++++
 src/libstore/local.mk                   |  2 +-
 src/{nix/test.cc => libstore/rust.hh}   | 41 +------------------------
 src/nix/local.mk                        |  4 +--
 9 files changed, 55 insertions(+), 84 deletions(-)
 create mode 100644 src/libstore/builtins/unpack-channel.cc
 rename src/{nix/test.cc => libstore/rust.hh} (53%)

diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in
index 32ce6b399f2..4ea182d8f58 100644
--- a/corepkgs/config.nix.in
+++ b/corepkgs/config.nix.in
@@ -1,3 +1,4 @@
+# FIXME: remove this file?
 let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
@@ -17,13 +18,4 @@ in rec {
   nixLocalstateDir = "@localstatedir@";
   nixSysconfDir = "@sysconfdir@";
   nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-
-  # If Nix is installed in the Nix store, then automatically add it as
-  # a dependency to the core packages. This ensures that they work
-  # properly in a chroot.
-  chrootDeps =
-    if dirOf nixPrefix == builtins.storeDir then
-      [ (builtins.storePath nixPrefix) ]
-    else
-      [ ];
 }
diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix
index d39a2063781..10515bc8b91 100644
--- a/corepkgs/unpack-channel.nix
+++ b/corepkgs/unpack-channel.nix
@@ -1,39 +1,12 @@
-with import <nix/config.nix>;
-
-let
-
-  builder = builtins.toFile "unpack-channel.sh"
-    ''
-      mkdir $out
-      cd $out
-      xzpat="\.xz\$"
-      gzpat="\.gz\$"
-      if [[ "$src" =~ $xzpat ]]; then
-        ${xz} -d < $src | ${tar} xf - ${tarFlags}
-      elif [[ "$src" =~ $gzpat ]]; then
-        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
-      else
-        ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
-      fi
-      if [ * != $channelName ]; then
-        mv * $out/$channelName
-      fi
-    '';
-
-in
-
 { name, channelName, src }:
 
 derivation {
-  system = builtins.currentSystem;
-  builder = shell;
-  args = [ "-e" builder ];
-  inherit name channelName src;
+  builder = "builtin:unpack-channel";
+
+  system = "builtin";
 
-  PATH = "${nixBinDir}:${coreutils}";
+  inherit name channelName src;
 
   # No point in doing this remotely.
   preferLocalBuild = true;
-
-  inherit chrootDeps;
 }
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 799d52e313c..ac6dee543a2 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -33,7 +33,7 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool {
     for file in tar.entries().unwrap() {
         let mut file = file.unwrap();
 
-        let dest_file = dest_dir.join(file.header().path().unwrap());
+        let dest_file = dest_dir.join(file.path().unwrap());
 
         fs::create_dir_all(dest_file.parent().unwrap()).unwrap();
 
@@ -55,6 +55,9 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool {
                     .unwrap();
                 io::copy(&mut file, &mut f).unwrap();
             }
+            tar::EntryType::Symlink => {
+                std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap();
+            }
             t => panic!("Unsupported tar entry type '{:?}'.", t),
         }
     }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 51a9fa35b2b..efbb7fc9f06 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3128,6 +3128,8 @@ void DerivationGoal::runChild()
                     builtinFetchurl(drv2, netrcData);
                 else if (drv->builder == "builtin:buildenv")
                     builtinBuildenv(drv2);
+                else if (drv->builder == "builtin:unpack-channel")
+                    builtinUnpackChannel(drv2);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh
index 0d2da873ece..87d6ce66533 100644
--- a/src/libstore/builtins.hh
+++ b/src/libstore/builtins.hh
@@ -7,5 +7,6 @@ namespace nix {
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
 void builtinBuildenv(const BasicDerivation & drv);
+void builtinUnpackChannel(const BasicDerivation & drv);
 
 }
diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc
new file mode 100644
index 00000000000..88202ec6b7f
--- /dev/null
+++ b/src/libstore/builtins/unpack-channel.cc
@@ -0,0 +1,39 @@
+#include "rust.hh"
+#include "builtins.hh"
+#include "compression.hh"
+
+namespace nix {
+
+void builtinUnpackChannel(const BasicDerivation & drv)
+{
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        return i->second;
+    };
+
+    Path out = getAttr("out");
+    auto channelName = getAttr("channelName");
+    auto src = getAttr("src");
+
+    createDirs(out);
+
+    auto source = sinkToSource([&](Sink & sink) {
+        auto decompressor =
+            hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) :
+            hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) :
+            makeDecompressionSink("none", sink);
+        readFile(src, *decompressor);
+        decompressor->finish();
+    });
+
+    unpack_tarfile(*source, out);
+
+    auto entries = readDirectory(out);
+    if (entries.size() != 1)
+        throw Error("channel tarball '%s' contains more than one file", src);
+    if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
+        throw SysError("renaming channel directory");
+}
+
+}
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d690fea28c2..d3254554dd3 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil
+libstore_LIBS = libutil libnixrust
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
diff --git a/src/nix/test.cc b/src/libstore/rust.hh
similarity index 53%
rename from src/nix/test.cc
rename to src/libstore/rust.hh
index 7f1f0d47aab..7e6c2f54d8a 100644
--- a/src/nix/test.cc
+++ b/src/libstore/rust.hh
@@ -1,9 +1,4 @@
-#include "command.hh"
-#include "store-api.hh"
-#include "common-args.hh"
-#include "compression.hh"
-
-using namespace nix;
+#include "serialise.hh"
 
 namespace rust {
 
@@ -47,37 +42,3 @@ struct Source
 extern "C" {
     bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }
-
-struct CmdTest : StoreCommand
-{
-    CmdTest()
-    {
-    }
-
-    std::string name() override
-    {
-        return "test";
-    }
-
-    std::string description() override
-    {
-        return "bla bla";
-    }
-
-    void run(ref<Store> store) override
-    {
-        auto source = sinkToSource([&](Sink & sink) {
-            auto decompressor = makeDecompressionSink("bzip2", sink);
-            readFile("./nix-2.2.tar.bz2", *decompressor);
-            decompressor->finish();
-        });
-
-        std::string destDir = "./dest";
-
-        deletePath(destDir);
-
-        unpack_tarfile(*source, destDir);
-    }
-};
-
-static RegisterCommand r(make_ref<CmdTest>());
diff --git a/src/nix/local.mk b/src/nix/local.mk
index a145cf9b1d1..c09efd1fc89 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,9 +15,9 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil libnixrust
+nix_LIBS = libexpr libmain libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \

From 6a9c815734de3549a5116796830bd4ebfd006025 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Mar 2019 23:49:14 +0100
Subject: [PATCH 1684/2196] Remove most of <nix/config.nix>

This is no longer needed.
---
 .gitignore                          | 1 +
 corepkgs/config.nix.in              | 8 --------
 tests/{config.nix => config.nix.in} | 6 ++----
 tests/local.mk                      | 2 +-
 4 files changed, 4 insertions(+), 13 deletions(-)
 rename tests/{config.nix => config.nix.in} (86%)

diff --git a/.gitignore b/.gitignore
index f58e1f90fc8..f64031e1220 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,6 +86,7 @@ perl/Makefile.config
 /tests/restricted-innocent
 /tests/shell
 /tests/shell.drv
+/tests/config.nix
 
 # /tests/lang/
 /tests/lang/*.out
diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in
index 4ea182d8f58..cb994594420 100644
--- a/corepkgs/config.nix.in
+++ b/corepkgs/config.nix.in
@@ -4,14 +4,6 @@ let
     let val = builtins.getEnv var; in
     if val != "" then val else def;
 in rec {
-  shell = "@bash@";
-  coreutils = "@coreutils@";
-  bzip2 = "@bzip2@";
-  gzip = "@gzip@";
-  xz = "@xz@";
-  tar = "@tar@";
-  tarFlags = "@tarFlags@";
-  tr = "@tr@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
diff --git a/tests/config.nix b/tests/config.nix.in
similarity index 86%
rename from tests/config.nix
rename to tests/config.nix.in
index 6ba91065b83..51aed539cb5 100644
--- a/tests/config.nix
+++ b/tests/config.nix.in
@@ -1,9 +1,7 @@
-with import <nix/config.nix>;
-
 rec {
-  inherit shell;
+  shell = "@bash@";
 
-  path = coreutils;
+  path = "@coreutils@";
 
   system = builtins.currentSystem;
 
diff --git a/tests/local.mk b/tests/local.mk
index 187f96ea206..f1d215eeca9 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT)
+installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)

From 87b7b25e13206bcfe200f45df822787612f159f7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 28 Mar 2019 00:02:10 +0100
Subject: [PATCH 1685/2196] Clean up the configure script

---
 configure.ac              | 20 +++-----------------
 perl/lib/Nix/Config.pm.in |  4 ----
 2 files changed, 3 insertions(+), 21 deletions(-)

diff --git a/configure.ac b/configure.ac
index 26f16b8460a..8e38f2b8e5f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -117,26 +117,15 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(sed, sed)
-NEED_PROG(tar, tar)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(gzip, gzip)
-NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
-NEED_PROG(cat, cat)
-NEED_PROG(tr, tr)
-AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
-  [path of cat, mkdir, etc.]),
-  coreutils=$withval, coreutils=$(dirname $cat))
-AC_SUBST(coreutils)
+AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LIBS="-latomic $LIBS"
 fi
 
-# Look for OpenSSL, a required dependency.
+# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
+# used by S3BinaryCacheStore.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
@@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 
-
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
-
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
@@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in
index 67a20c3f41d..bc1749e601e 100644
--- a/perl/lib/Nix/Config.pm.in
+++ b/perl/lib/Nix/Config.pm.in
@@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
 $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
 $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
 
-$bzip2 = "@bzip2@";
-$xz = "@xz@";
-$curl = "@curl@";
-
 $useBindings = 1;
 
 %config = ();

From 0dbb249b36f1d058e63982d26746fde48911cce4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Jul 2019 17:06:38 +0200
Subject: [PATCH 1686/2196] Update Rust dependencies

---
 nix-rust/Cargo.lock | 61 +++++++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 19 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index cfaedc3ed4c..ea7ac72db16 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,59 +1,82 @@
 [[package]]
 name = "cfg-if"
-version = "0.1.7"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "filetime"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "libc"
-version = "0.2.50"
+version = "0.2.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "nix-rust"
 version = "0.1.0"
 dependencies = [
- "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
- "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.1.51"
+version = "0.1.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "tar"
-version = "0.4.22"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)",
+ "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "winapi"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "xattr"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [metadata]
-"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4"
-"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646"
-"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1"
-"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85"
-"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133"
+"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"
+"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f"
+"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319"
+"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
+"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"

From d722e2175ef826cd60f05608fb7d58aa82261549 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jul 2019 01:01:18 +0200
Subject: [PATCH 1687/2196] Include cargo dependencies in the Nix tarball

---
 nix-rust/local.mk |  4 +++-
 release.nix       | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index c69e3b9ea2c..ce88fa51b33 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -4,4 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl
 libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
 
 $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
-	$(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a
+
+dist-files += $(d)/vendor
diff --git a/release.nix b/release.nix
index 1d7c2e2d628..64aa35e1134 100644
--- a/release.nix
+++ b/release.nix
@@ -10,6 +10,43 @@ let
 
   jobs = rec {
 
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock, and include it in the Nix tarball. This allows Nix
+    # to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
+
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
+
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
+
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
+
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            mv "$dir" $out/vendor/
+            rmdir $out/vendor/tmp
+          '') files)}
+        '';
+
 
     tarball =
       with pkgs;
@@ -38,6 +75,8 @@ let
 
         distPhase =
           ''
+            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
+
             runHook preDist
             make dist
             mkdir -p $out/tarballs

From afb021893beb598213d4225de5edd746baa81d7a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jul 2019 01:19:30 +0200
Subject: [PATCH 1688/2196] Reduce the size of the vendor directory by removing
 some winapi cruft

---
 release.nix | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/release.nix b/release.nix
index 64aa35e1134..9cf4c74f231 100644
--- a/release.nix
+++ b/release.nix
@@ -42,8 +42,15 @@ let
             # Add just enough metadata to keep Cargo happy.
             printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
 
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
             mv "$dir" $out/vendor/
-            rmdir $out/vendor/tmp
+
+            rm -rf $out/vendor/tmp
           '') files)}
         '';
 

From 343ebcc048c69e103db3f87a9339388bd5dd93cb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jul 2019 01:50:35 +0200
Subject: [PATCH 1689/2196] Only pass '-Z offline' to cargo if we have a vendor
 directory

---
 nix-rust/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index ce88fa51b33..a5e017cef79 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl
 libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
 
 $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
-	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a
 
 dist-files += $(d)/vendor

From 8110b4ebb29174ecd4b22510da0285abf604b8a7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Sep 2019 21:55:32 +0200
Subject: [PATCH 1690/2196] Rust cleanup

---
 nix-rust/Cargo.toml     |  1 +
 nix-rust/src/error.rs   |  5 +++
 nix-rust/src/foreign.rs | 14 +++++++++
 nix-rust/src/lib.rs     | 68 ++++-------------------------------------
 nix-rust/src/tarfile.rs | 47 ++++++++++++++++++++++++++++
 src/libstore/rust.hh    |  2 +-
 6 files changed, 74 insertions(+), 63 deletions(-)
 create mode 100644 nix-rust/src/error.rs
 create mode 100644 nix-rust/src/foreign.rs
 create mode 100644 nix-rust/src/tarfile.rs

diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 2d2bf67523d..8b56dc89cd0 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -2,6 +2,7 @@
 name = "nix-rust"
 version = "0.1.0"
 authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+edition = "2018"
 
 [lib]
 name = "nixrust"
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
new file mode 100644
index 00000000000..28d0abdef99
--- /dev/null
+++ b/nix-rust/src/error.rs
@@ -0,0 +1,5 @@
+#[derive(Debug)]
+pub enum Error {
+    Misc(String),
+    Foreign(libc::c_void), // == std::exception_ptr
+}
diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs
new file mode 100644
index 00000000000..7bce7753c05
--- /dev/null
+++ b/nix-rust/src/foreign.rs
@@ -0,0 +1,14 @@
+/// A wrapper around Nix's Source class that provides the Read trait.
+#[repr(C)]
+pub struct Source {
+    fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
+    this: *mut libc::c_void,
+}
+
+impl std::io::Read for Source {
+    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+        let n = (self.fun)(self.this, buf);
+        assert!(n <= buf.len());
+        Ok(n)
+    }
+}
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index ac6dee543a2..192ca29e4bb 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,66 +1,10 @@
-extern crate libc;
-extern crate tar;
+mod error;
+mod foreign;
+mod tarfile;
 
-use std::fs;
-use std::io;
-use std::os::unix::fs::OpenOptionsExt;
-use std::path::Path;
-use tar::Archive;
-
-/// A wrapper around Nix's Source class that provides the Read trait.
-#[repr(C)]
-pub struct Source {
-    fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
-    this: *mut libc::c_void,
-}
-
-impl std::io::Read for Source {
-    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
-        let n = (self.fun)(self.this, buf);
-        assert!(n <= buf.len());
-        Ok(n)
-    }
-}
+pub use error::Error;
 
 #[no_mangle]
-pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool {
-    // FIXME: handle errors.
-
-    let dest_dir = Path::new(dest_dir);
-
-    let mut tar = Archive::new(source);
-
-    for file in tar.entries().unwrap() {
-        let mut file = file.unwrap();
-
-        let dest_file = dest_dir.join(file.path().unwrap());
-
-        fs::create_dir_all(dest_file.parent().unwrap()).unwrap();
-
-        match file.header().entry_type() {
-            tar::EntryType::Directory => {
-                fs::create_dir(dest_file).unwrap();
-            }
-            tar::EntryType::Regular => {
-                let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 {
-                    0o666
-                } else {
-                    0o777
-                };
-                let mut f = fs::OpenOptions::new()
-                    .create(true)
-                    .write(true)
-                    .mode(mode)
-                    .open(dest_file)
-                    .unwrap();
-                io::copy(&mut file, &mut f).unwrap();
-            }
-            tar::EntryType::Symlink => {
-                std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap();
-            }
-            t => panic!("Unsupported tar entry type '{:?}'.", t),
-        }
-    }
-
-    true
+pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) {
+    tarfile::unpack_tarfile(source, dest_dir).unwrap();
 }
diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs
new file mode 100644
index 00000000000..696118e4d29
--- /dev/null
+++ b/nix-rust/src/tarfile.rs
@@ -0,0 +1,47 @@
+use crate::{foreign::Source, Error};
+use std::fs;
+use std::io;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::Path;
+use tar::Archive;
+
+pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
+    let dest_dir = Path::new(dest_dir);
+
+    let mut tar = Archive::new(source);
+
+    for file in tar.entries().unwrap() {
+        let mut file = file.unwrap();
+
+        let dest_file = dest_dir.join(file.path().unwrap());
+
+        fs::create_dir_all(dest_file.parent().unwrap()).unwrap();
+
+        match file.header().entry_type() {
+            tar::EntryType::Directory => {
+                fs::create_dir(dest_file).unwrap();
+            }
+            tar::EntryType::Regular => {
+                let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 {
+                    0o666
+                } else {
+                    0o777
+                };
+                let mut f = fs::OpenOptions::new()
+                    .create(true)
+                    .write(true)
+                    .mode(mode)
+                    .open(dest_file)
+                    .unwrap();
+                io::copy(&mut file, &mut f).unwrap();
+            }
+            tar::EntryType::Symlink => {
+                std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file)
+                    .unwrap();
+            }
+            t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
+        }
+    }
+
+    Ok(())
+}
diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh
index 7e6c2f54d8a..4366e4723f0 100644
--- a/src/libstore/rust.hh
+++ b/src/libstore/rust.hh
@@ -40,5 +40,5 @@ struct Source
 }
 
 extern "C" {
-    bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+    void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }

From f738cd4d976f4f72159bbcbfa7b451c33f0ea74a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 01:15:20 +0200
Subject: [PATCH 1691/2196] More Rust FFI adventures

We can now convert Rust Errors to C++ exceptions. At the Rust->C++ FFI
boundary, Result<T, Error> will cause Error to be converted to and
thrown as a C++ exception.
---
 nix-rust/src/error.rs                   | 27 ++++++++++-
 nix-rust/src/lib.rs                     | 27 ++++++++++-
 nix-rust/src/tarfile.rs                 | 20 ++++----
 src/libstore/builtins/unpack-channel.cc |  2 +-
 src/libstore/rust.cc                    | 12 +++++
 src/libstore/rust.hh                    | 61 ++++++++++++++++++++++++-
 6 files changed, 132 insertions(+), 17 deletions(-)
 create mode 100644 src/libstore/rust.cc

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 28d0abdef99..a2003be6f70 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -1,5 +1,30 @@
 #[derive(Debug)]
 pub enum Error {
+    IOError(std::io::Error),
     Misc(String),
-    Foreign(libc::c_void), // == std::exception_ptr
+    Foreign(CppException),
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::IOError(err)
+    }
+}
+
+impl From<Error> for CppException {
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Foreign(ex) => ex,
+            Error::Misc(s) => unsafe { make_error(&s) },
+            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct CppException(*const libc::c_void); // == std::exception_ptr*
+
+extern "C" {
+    fn make_error(s: &str) -> CppException;
 }
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 192ca29e4bb..b6b0d746dae 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -4,7 +4,30 @@ mod tarfile;
 
 pub use error::Error;
 
+pub struct CBox<T> {
+    ptr: *mut libc::c_void,
+    phantom: std::marker::PhantomData<T>,
+}
+
+impl<T> CBox<T> {
+    fn new(t: T) -> Self {
+        unsafe {
+            let size = std::mem::size_of::<T>();
+            let ptr = libc::malloc(size);
+            eprintln!("PTR = {:?}, SIZE = {}", ptr, size);
+            *(ptr as *mut T) = t; // FIXME: probably UB
+            Self {
+                ptr,
+                phantom: std::marker::PhantomData,
+            }
+        }
+    }
+}
+
 #[no_mangle]
-pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) {
-    tarfile::unpack_tarfile(source, dest_dir).unwrap();
+pub extern "C" fn unpack_tarfile(
+    source: foreign::Source,
+    dest_dir: &str,
+) -> CBox<Result<(), error::CppException>> {
+    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
 }
diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs
index 696118e4d29..797aa5064e4 100644
--- a/nix-rust/src/tarfile.rs
+++ b/nix-rust/src/tarfile.rs
@@ -10,19 +10,19 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
 
     let mut tar = Archive::new(source);
 
-    for file in tar.entries().unwrap() {
-        let mut file = file.unwrap();
+    for file in tar.entries()? {
+        let mut file = file?;
 
-        let dest_file = dest_dir.join(file.path().unwrap());
+        let dest_file = dest_dir.join(file.path()?);
 
-        fs::create_dir_all(dest_file.parent().unwrap()).unwrap();
+        fs::create_dir_all(dest_file.parent().unwrap())?;
 
         match file.header().entry_type() {
             tar::EntryType::Directory => {
-                fs::create_dir(dest_file).unwrap();
+                fs::create_dir(dest_file)?;
             }
             tar::EntryType::Regular => {
-                let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 {
+                let mode = if file.header().mode()? & libc::S_IXUSR == 0 {
                     0o666
                 } else {
                     0o777
@@ -31,13 +31,11 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
                     .create(true)
                     .write(true)
                     .mode(mode)
-                    .open(dest_file)
-                    .unwrap();
-                io::copy(&mut file, &mut f).unwrap();
+                    .open(dest_file)?;
+                io::copy(&mut file, &mut f)?;
             }
             tar::EntryType::Symlink => {
-                std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file)
-                    .unwrap();
+                std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
             }
             t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
         }
diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc
index 88202ec6b7f..2da26d98e64 100644
--- a/src/libstore/builtins/unpack-channel.cc
+++ b/src/libstore/builtins/unpack-channel.cc
@@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
         decompressor->finish();
     });
 
-    unpack_tarfile(*source, out);
+    unpack_tarfile(*source, out).use()->unwrap();
 
     auto entries = readDirectory(out);
     if (entries.size() != 1)
diff --git a/src/libstore/rust.cc b/src/libstore/rust.cc
new file mode 100644
index 00000000000..a616d83a633
--- /dev/null
+++ b/src/libstore/rust.cc
@@ -0,0 +1,12 @@
+#include "logging.hh"
+#include "rust.hh"
+
+namespace nix {
+
+extern "C" std::exception_ptr * make_error(rust::StringSlice s)
+{
+    // FIXME: leak
+    return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size))));
+}
+
+}
diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh
index 4366e4723f0..4c7720a4427 100644
--- a/src/libstore/rust.hh
+++ b/src/libstore/rust.hh
@@ -4,7 +4,8 @@ namespace rust {
 
 // Depending on the internal representation of Rust slices is slightly
 // evil...
-template<typename T> struct Slice
+template<typename T>
+struct Slice
 {
     T * ptr;
     size_t size;
@@ -37,8 +38,64 @@ struct Source
     }
 };
 
+/* C++ representation of Rust's Result<T, CppException>. */
+template<typename T>
+struct Result
+{
+    unsigned int tag;
+
+    union {
+        T data;
+        std::exception_ptr * exc;
+    };
+
+    /* Rethrow the wrapped exception or return the wrapped value. */
+    T unwrap()
+    {
+        if (tag == 0)
+            return data;
+        else if (tag == 1)
+            std::rethrow_exception(*exc);
+        else
+            abort();
+    }
+};
+
+template<typename T>
+struct CBox
+{
+    T * ptr;
+
+    T * operator ->()
+    {
+        return ptr;
+    }
+
+    CBox(T * ptr) : ptr(ptr) { }
+    CBox(const CBox &) = delete;
+    CBox(CBox &&) = delete;
+
+    ~CBox()
+    {
+        free(ptr);
+    }
+};
+
+// Grrr, this is only needed because 'extern "C"' functions don't
+// support non-POD return types (and CBox has a destructor so it's not
+// POD).
+template<typename T>
+struct CBox2
+{
+    T * ptr;
+    CBox<T> use()
+    {
+        return CBox(ptr);
+    }
+};
+
 }
 
 extern "C" {
-    void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+    rust::CBox2<rust::Result<std::tuple<>>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }

From b7fba16613aed5c2b2a0b4f98e3e0d32b0cddd40 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 12:43:07 +0200
Subject: [PATCH 1692/2196] Move code around

---
 src/libstore/rust.hh                          | 97 +------------------
 src/{libstore/rust.cc => libutil/rust-ffi.cc} |  0
 src/libutil/rust-ffi.hh                       | 97 +++++++++++++++++++
 3 files changed, 98 insertions(+), 96 deletions(-)
 rename src/{libstore/rust.cc => libutil/rust-ffi.cc} (100%)
 create mode 100644 src/libutil/rust-ffi.hh

diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh
index 4c7720a4427..dccc98b4bdd 100644
--- a/src/libstore/rust.hh
+++ b/src/libstore/rust.hh
@@ -1,100 +1,5 @@
 #include "serialise.hh"
-
-namespace rust {
-
-// Depending on the internal representation of Rust slices is slightly
-// evil...
-template<typename T>
-struct Slice
-{
-    T * ptr;
-    size_t size;
-
-    Slice(T * ptr, size_t size) : ptr(ptr), size(size)
-    {
-        assert(ptr);
-    }
-};
-
-struct StringSlice : Slice<char>
-{
-    StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
-};
-
-struct Source
-{
-    size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
-    nix::Source * _this;
-
-    Source(nix::Source & _this)
-        : fun(sourceWrapper), _this(&_this)
-    {}
-
-    // FIXME: how to propagate exceptions?
-    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
-    {
-        auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
-        return n;
-    }
-};
-
-/* C++ representation of Rust's Result<T, CppException>. */
-template<typename T>
-struct Result
-{
-    unsigned int tag;
-
-    union {
-        T data;
-        std::exception_ptr * exc;
-    };
-
-    /* Rethrow the wrapped exception or return the wrapped value. */
-    T unwrap()
-    {
-        if (tag == 0)
-            return data;
-        else if (tag == 1)
-            std::rethrow_exception(*exc);
-        else
-            abort();
-    }
-};
-
-template<typename T>
-struct CBox
-{
-    T * ptr;
-
-    T * operator ->()
-    {
-        return ptr;
-    }
-
-    CBox(T * ptr) : ptr(ptr) { }
-    CBox(const CBox &) = delete;
-    CBox(CBox &&) = delete;
-
-    ~CBox()
-    {
-        free(ptr);
-    }
-};
-
-// Grrr, this is only needed because 'extern "C"' functions don't
-// support non-POD return types (and CBox has a destructor so it's not
-// POD).
-template<typename T>
-struct CBox2
-{
-    T * ptr;
-    CBox<T> use()
-    {
-        return CBox(ptr);
-    }
-};
-
-}
+#include "rust-ffi.hh"
 
 extern "C" {
     rust::CBox2<rust::Result<std::tuple<>>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
diff --git a/src/libstore/rust.cc b/src/libutil/rust-ffi.cc
similarity index 100%
rename from src/libstore/rust.cc
rename to src/libutil/rust-ffi.cc
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
new file mode 100644
index 00000000000..a488b96d69f
--- /dev/null
+++ b/src/libutil/rust-ffi.hh
@@ -0,0 +1,97 @@
+#include "serialise.hh"
+
+namespace rust {
+
+// Depending on the internal representation of Rust slices is slightly
+// evil...
+template<typename T>
+struct Slice
+{
+    T * ptr;
+    size_t size;
+
+    Slice(T * ptr, size_t size) : ptr(ptr), size(size)
+    {
+        assert(ptr);
+    }
+};
+
+struct StringSlice : Slice<char>
+{
+    StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
+};
+
+struct Source
+{
+    size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
+    nix::Source * _this;
+
+    Source(nix::Source & _this)
+        : fun(sourceWrapper), _this(&_this)
+    {}
+
+    // FIXME: how to propagate exceptions?
+    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
+    {
+        auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
+        return n;
+    }
+};
+
+/* C++ representation of Rust's Result<T, CppException>. */
+template<typename T>
+struct Result
+{
+    unsigned int tag;
+
+    union {
+        T data;
+        std::exception_ptr * exc;
+    };
+
+    /* Rethrow the wrapped exception or return the wrapped value. */
+    T unwrap()
+    {
+        if (tag == 0)
+            return data;
+        else if (tag == 1)
+            std::rethrow_exception(*exc);
+        else
+            abort();
+    }
+};
+
+template<typename T>
+struct CBox
+{
+    T * ptr;
+
+    T * operator ->()
+    {
+        return ptr;
+    }
+
+    CBox(T * ptr) : ptr(ptr) { }
+    CBox(const CBox &) = delete;
+    CBox(CBox &&) = delete;
+
+    ~CBox()
+    {
+        free(ptr);
+    }
+};
+
+// Grrr, this is only needed because 'extern "C"' functions don't
+// support non-POD return types (and CBox has a destructor so it's not
+// POD).
+template<typename T>
+struct CBox2
+{
+    T * ptr;
+    CBox<T> use()
+    {
+        return CBox(ptr);
+    }
+};
+
+}

From d14b1c261cd3dfb4be2da943d901a394c3f23205 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 12:44:31 +0200
Subject: [PATCH 1693/2196] Shut up some rust warnings

---
 nix-rust/src/error.rs | 1 +
 nix-rust/src/lib.rs   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index a2003be6f70..519007ea0aa 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -26,5 +26,6 @@ impl From<Error> for CppException {
 pub struct CppException(*const libc::c_void); // == std::exception_ptr*
 
 extern "C" {
+    #[allow(improper_ctypes)] // YOLO
     fn make_error(s: &str) -> CppException;
 }
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index b6b0d746dae..48952d8b9c9 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -5,7 +5,7 @@ mod tarfile;
 pub use error::Error;
 
 pub struct CBox<T> {
-    ptr: *mut libc::c_void,
+    pub ptr: *mut libc::c_void,
     phantom: std::marker::PhantomData<T>,
 }
 

From d33dd6e6c09d87a59989057ad622a6265ddec2e0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 13:10:46 +0200
Subject: [PATCH 1694/2196] Move code around

---
 nix-rust/src/lib.rs                          |  1 -
 src/libstore/builtins/unpack-channel.cc      |  4 ++--
 src/libstore/local.mk                        |  2 +-
 src/libutil/local.mk                         |  2 ++
 src/libutil/rust-ffi.cc                      |  2 +-
 src/{libstore/rust.hh => libutil/tarfile.cc} | 10 +++++++++-
 src/libutil/tarfile.hh                       |  7 +++++++
 7 files changed, 22 insertions(+), 6 deletions(-)
 rename src/{libstore/rust.hh => libutil/tarfile.cc} (53%)
 create mode 100644 src/libutil/tarfile.hh

diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 48952d8b9c9..1b88ac8af88 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -14,7 +14,6 @@ impl<T> CBox<T> {
         unsafe {
             let size = std::mem::size_of::<T>();
             let ptr = libc::malloc(size);
-            eprintln!("PTR = {:?}, SIZE = {}", ptr, size);
             *(ptr as *mut T) = t; // FIXME: probably UB
             Self {
                 ptr,
diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc
index 2da26d98e64..5fc68cd666c 100644
--- a/src/libstore/builtins/unpack-channel.cc
+++ b/src/libstore/builtins/unpack-channel.cc
@@ -1,6 +1,6 @@
-#include "rust.hh"
 #include "builtins.hh"
 #include "compression.hh"
+#include "tarfile.hh"
 
 namespace nix {
 
@@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
         decompressor->finish();
     });
 
-    unpack_tarfile(*source, out).use()->unwrap();
+    unpackTarfile(*source, out);
 
     auto entries = readDirectory(out);
     if (entries.size() != 1)
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d3254554dd3..d690fea28c2 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil libnixrust
+libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index e41a67d1f9e..35c1f6c13dc 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -7,3 +7,5 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
+
+libutil_LIBS = libnixrust
diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc
index a616d83a633..931d29542ee 100644
--- a/src/libutil/rust-ffi.cc
+++ b/src/libutil/rust-ffi.cc
@@ -1,5 +1,5 @@
 #include "logging.hh"
-#include "rust.hh"
+#include "rust-ffi.hh"
 
 namespace nix {
 
diff --git a/src/libstore/rust.hh b/src/libutil/tarfile.cc
similarity index 53%
rename from src/libstore/rust.hh
rename to src/libutil/tarfile.cc
index dccc98b4bdd..ae6d512bd03 100644
--- a/src/libstore/rust.hh
+++ b/src/libutil/tarfile.cc
@@ -1,6 +1,14 @@
-#include "serialise.hh"
 #include "rust-ffi.hh"
 
 extern "C" {
     rust::CBox2<rust::Result<std::tuple<>>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }
+
+namespace nix {
+
+void unpackTarfile(Source & source, Path destDir)
+{
+    unpack_tarfile(source, destDir).use()->unwrap();
+}
+
+}
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
new file mode 100644
index 00000000000..c3e95fb0c92
--- /dev/null
+++ b/src/libutil/tarfile.hh
@@ -0,0 +1,7 @@
+#include "serialise.hh"
+
+namespace nix {
+
+void unpackTarfile(Source & source, Path destDir);
+
+}

From f2bd8470926686361602e545d63a69d4bfc22f90 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 15:03:17 +0200
Subject: [PATCH 1695/2196] Ignore tar header entries

In particular, these are emitted by 'git archive' (in fetchGit).
---
 nix-rust/src/tarfile.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs
index 797aa5064e4..9e32a3f0082 100644
--- a/nix-rust/src/tarfile.rs
+++ b/nix-rust/src/tarfile.rs
@@ -37,6 +37,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
             tar::EntryType::Symlink => {
                 std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
             }
+            tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {}
             t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
         }
     }

From 8918bae09828133259acb36d6aef60ffbfad252c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Sep 2019 15:25:43 +0200
Subject: [PATCH 1696/2196] Drop remaining uses of external "tar"

Also, fetchGit now runs in O(1) memory since we pipe the output of
'git archive' directly into unpackTarball() (rather than first reading
it all into memory).
---
 src/libexpr/primops/fetchGit.cc          | 11 +++++++----
 src/libstore/builtins/unpack-channel.cc  | 12 +-----------
 src/libstore/download.cc                 | 12 ++++++++----
 src/libutil/serialise.hh                 |  1 -
 src/libutil/tarfile.cc                   | 22 +++++++++++++++++++++-
 src/libutil/tarfile.hh                   |  5 ++++-
 src/nix-prefetch-url/nix-prefetch-url.cc |  4 ++--
 7 files changed, 43 insertions(+), 24 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 7ef3b382398..9d0c642911d 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -4,6 +4,7 @@
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "hash.hh"
+#include "tarfile.hh"
 
 #include <sys/time.h>
 
@@ -164,14 +165,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    // FIXME: should pipe this, or find some better way to extract a
-    // revision.
-    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev });
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
-    runProgram("tar", true, { "x", "-C", tmpDir }, tar);
+    unpackTarfile(*source, tmpDir);
 
     gitInfo.storePath = store->addToStore(name, tmpDir);
 
diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc
index 5fc68cd666c..d18e3ddafbc 100644
--- a/src/libstore/builtins/unpack-channel.cc
+++ b/src/libstore/builtins/unpack-channel.cc
@@ -1,5 +1,4 @@
 #include "builtins.hh"
-#include "compression.hh"
 #include "tarfile.hh"
 
 namespace nix {
@@ -18,16 +17,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
 
     createDirs(out);
 
-    auto source = sinkToSource([&](Sink & sink) {
-        auto decompressor =
-            hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) :
-            hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) :
-            makeDecompressionSink("none", sink);
-        readFile(src, *decompressor);
-        decompressor->finish();
-    });
-
-    unpackTarfile(*source, out);
+    unpackTarfile(src, out);
 
     auto entries = readDirectory(out);
     if (entries.size() != 1)
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index e80663dfffe..61e88c5c1ef 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -8,6 +8,7 @@
 #include "compression.hh"
 #include "pathlocks.hh"
 #include "finally.hh"
+#include "tarfile.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached(
                 unpackedStorePath = "";
         }
         if (unpackedStorePath.empty()) {
-            printInfo(format("unpacking '%1%'...") % url);
+            printInfo("unpacking '%s'...", url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
-            // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
-            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
+            unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
+            auto members = readDirectory(tmpDir);
+            if (members.size() != 1)
+                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
+            auto topDir = tmpDir + "/" + members.begin()->name;
+            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
         storePath = unpackedStorePath;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 128e287f31c..5780c93a630 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -77,7 +77,6 @@ struct BufferedSource : Source
 
     size_t read(unsigned char * data, size_t len) override;
 
-
     bool hasData();
 
 protected:
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index ae6d512bd03..f7d3ad4172f 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -1,4 +1,5 @@
 #include "rust-ffi.hh"
+#include "compression.hh"
 
 extern "C" {
     rust::CBox2<rust::Result<std::tuple<>>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
@@ -6,9 +7,28 @@ extern "C" {
 
 namespace nix {
 
-void unpackTarfile(Source & source, Path destDir)
+void unpackTarfile(Source & source, const Path & destDir)
 {
     unpack_tarfile(source, destDir).use()->unwrap();
 }
 
+void unpackTarfile(const Path & tarFile, const Path & destDir,
+    std::optional<std::string> baseName)
+{
+    if (!baseName) baseName = baseNameOf(tarFile);
+
+    auto source = sinkToSource([&](Sink & sink) {
+        // FIXME: look at first few bytes to determine compression type.
+        auto decompressor =
+            // FIXME: add .gz support
+            hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
+            hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
+            makeDecompressionSink("none", sink);
+        readFile(tarFile, *decompressor);
+        decompressor->finish();
+    });
+
+    unpackTarfile(*source, destDir);
+}
+
 }
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index c3e95fb0c92..ce0911e2a32 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -2,6 +2,9 @@
 
 namespace nix {
 
-void unpackTarfile(Source & source, Path destDir);
+void unpackTarfile(Source & source, const Path & destDir);
+
+void unpackTarfile(const Path & tarFile, const Path & destDir,
+    std::optional<std::string> baseName = {});
 
 }
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index f54706a8a01..78c88383377 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -9,6 +9,7 @@
 #include "legacy.hh"
 #include "finally.hh"
 #include "progress-bar.hh"
+#include "tarfile.hh"
 
 #include <iostream>
 
@@ -192,8 +193,7 @@ static int _main(int argc, char * * argv)
                 if (hasSuffix(baseNameOf(uri), ".zip"))
                     runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
                 else
-                    // FIXME: this requires GNU tar for decompression.
-                    runProgram("tar", true, {"xf", tmpFile, "-C", unpacked});
+                    unpackTarfile(tmpFile, unpacked, baseNameOf(uri));
 
                 /* If the archive unpacks to a single file/directory, then use
                    that as the top-level. */

From 88f8063917a4261abfabfe94bc33c63f7405d1d5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 22:45:15 +0100
Subject: [PATCH 1697/2196] -Z offline -> --offline

---
 nix-rust/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index a5e017cef79..fda1c3dee88 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl
 libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
 
 $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
-	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a
 
 dist-files += $(d)/vendor

From e6c1d1b474b1f14753b27f35b2aad4051ffd9e8d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 26 Nov 2019 22:46:36 +0100
Subject: [PATCH 1698/2196] Update Cargo.lock

---
 nix-rust/Cargo.lock | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index ea7ac72db16..0112ed471db 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,29 +1,31 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
 [[package]]
 name = "cfg-if"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "filetime"
-version = "0.2.6"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "libc"
-version = "0.2.58"
+version = "0.2.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "nix-rust"
 version = "0.1.0"
 dependencies = [
- "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
  "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -37,15 +39,15 @@ name = "tar"
 version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "winapi"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -67,16 +69,16 @@ name = "xattr"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [metadata]
-"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33"
-"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f"
-"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319"
+"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
+"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
-"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770"
+"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"

From dbc4f9d478814f3ce4ee23531502247d02c85911 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Nov 2019 00:17:24 +0100
Subject: [PATCH 1699/2196] Fix macOS build

https://hydra.nixos.org/build/107466992
---
 nix-rust/src/tarfile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs
index 9e32a3f0082..379d9098f1c 100644
--- a/nix-rust/src/tarfile.rs
+++ b/nix-rust/src/tarfile.rs
@@ -22,7 +22,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
                 fs::create_dir(dest_file)?;
             }
             tar::EntryType::Regular => {
-                let mode = if file.header().mode()? & libc::S_IXUSR == 0 {
+                let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 {
                     0o666
                 } else {
                     0o777

From 949dc848940b68ffd1327a278df4472e98a455dc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Nov 2019 14:17:15 +0100
Subject: [PATCH 1700/2196] Fix segfault on i686-linux

https://hydra.nixos.org/build/107467517

Seems that on i686-linux, gcc and rustc disagree on how to return
1-word structs: gcc has the caller pass a pointer to the result, while
rustc has the callee return the result in a register. Work around this
by using a bare pointer.
---
 src/libutil/rust-ffi.hh | 13 -------------
 src/libutil/tarfile.cc  |  6 ++++--
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index a488b96d69f..663758bfcb6 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -81,17 +81,4 @@ struct CBox
     }
 };
 
-// Grrr, this is only needed because 'extern "C"' functions don't
-// support non-POD return types (and CBox has a destructor so it's not
-// POD).
-template<typename T>
-struct CBox2
-{
-    T * ptr;
-    CBox<T> use()
-    {
-        return CBox(ptr);
-    }
-};
-
 }
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index f7d3ad4172f..2cc7793fde8 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -2,14 +2,16 @@
 #include "compression.hh"
 
 extern "C" {
-    rust::CBox2<rust::Result<std::tuple<>>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+    rust::Result<std::tuple<>> *
+    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
 }
 
 namespace nix {
 
 void unpackTarfile(Source & source, const Path & destDir)
 {
-    unpack_tarfile(source, destDir).use()->unwrap();
+    rust::Source source2(source);
+    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
 }
 
 void unpackTarfile(const Path & tarFile, const Path & destDir,

From f553a8bdea96afe18b97dee6eba91c0bbc545b79 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Nov 2019 14:18:57 +0100
Subject: [PATCH 1701/2196] When OPTIMIZE=0, build rust code in debug mode

---
 nix-rust/local.mk | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index fda1c3dee88..c147f987407 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -1,9 +1,19 @@
-libnixrust_PATH := $(d)/target/release/libnixrust.a
+ifeq ($(OPTIMIZE), 1)
+  RUST_MODE = --release
+  RUST_DIR = release
+else
+  RUST_MODE =
+  RUST_DIR = debug
+endif
+
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a
 libnixrust_INSTALL_PATH := $(libnixrust_PATH)
-libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
 libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
 
-$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
-	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a
+$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
+	cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.a
 
 dist-files += $(d)/vendor

From 895ce1bb6cc77324c377baa08eb98666768e5c6d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 27 Nov 2019 17:33:59 +0100
Subject: [PATCH 1702/2196] make clean: Delete nix-rust/target

---
 nix-rust/local.mk | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index c147f987407..ed1e640c913 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -17,3 +17,8 @@ $(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
 	&& touch target/$(RUST_DIR)/libnixrust.a
 
 dist-files += $(d)/vendor
+
+clean: clean-rust
+
+clean-rust:
+	$(suppress) rm -rfv nix-rust/target

From 2d6f1ddbb5f2ae2f3153785d7a4e5321c85495fd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 28 Nov 2019 13:52:42 +0100
Subject: [PATCH 1703/2196] Remove builtins.valueSize

Fixes #3246.
---
 src/libexpr/eval.cc    | 91 ------------------------------------------
 src/libexpr/eval.hh    |  1 -
 src/libexpr/primops.cc |  8 ----
 src/libexpr/value.hh   |  9 -----
 4 files changed, 109 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3a9b7a3c093..b89a67b19f8 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -614,13 +614,9 @@ Value * EvalState::allocValue()
 
 Env & EvalState::allocEnv(size_t size)
 {
-    if (size > std::numeric_limits<decltype(Env::size)>::max())
-        throw Error("environment size %d is too big", size);
-
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = (decltype(Env::size)) size;
     env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
@@ -1868,93 +1864,6 @@ void EvalState::printStats()
 }
 
 
-size_t valueSize(Value & v)
-{
-    std::set<const void *> seen;
-
-    auto doString = [&](const char * s) -> size_t {
-        if (!seen.insert(s).second) return 0;
-        return strlen(s) + 1;
-    };
-
-    std::function<size_t(Value & v)> doValue;
-    std::function<size_t(Env & v)> doEnv;
-
-    doValue = [&](Value & v) -> size_t {
-        if (!seen.insert(&v).second) return 0;
-
-        size_t sz = sizeof(Value);
-
-        switch (v.type) {
-        case tString:
-            sz += doString(v.string.s);
-            if (v.string.context)
-                for (const char * * p = v.string.context; *p; ++p)
-                    sz += doString(*p);
-            break;
-        case tPath:
-            sz += doString(v.path);
-            break;
-        case tAttrs:
-            if (seen.insert(v.attrs).second) {
-                sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity();
-                for (auto & i : *v.attrs)
-                    sz += doValue(*i.value);
-            }
-            break;
-        case tList1:
-        case tList2:
-        case tListN:
-            if (seen.insert(v.listElems()).second) {
-                sz += v.listSize() * sizeof(Value *);
-                for (size_t n = 0; n < v.listSize(); ++n)
-                    sz += doValue(*v.listElems()[n]);
-            }
-            break;
-        case tThunk:
-            sz += doEnv(*v.thunk.env);
-            break;
-        case tApp:
-            sz += doValue(*v.app.left);
-            sz += doValue(*v.app.right);
-            break;
-        case tLambda:
-            sz += doEnv(*v.lambda.env);
-            break;
-        case tPrimOpApp:
-            sz += doValue(*v.primOpApp.left);
-            sz += doValue(*v.primOpApp.right);
-            break;
-        case tExternal:
-            if (!seen.insert(v.external).second) break;
-            sz += v.external->valueSize(seen);
-            break;
-        default:
-            ;
-        }
-
-        return sz;
-    };
-
-    doEnv = [&](Env & env) -> size_t {
-        if (!seen.insert(&env).second) return 0;
-
-        size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
-
-        if (env.type != Env::HasWithExpr)
-            for (size_t i = 0; i < env.size; ++i)
-                if (env.values[i])
-                    sz += doValue(*env.values[i]);
-
-        if (env.up) sz += doEnv(*env.up);
-
-        return sz;
-    };
-
-    return doValue(v);
-}
-
-
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError(format("cannot coerce %1% to a string, at %2%") %
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9d075a48a65..419b703fc38 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -37,7 +37,6 @@ struct PrimOp
 struct Env
 {
     Env * up;
-    unsigned short size; // used by ‘valueSize’
     unsigned short prevWith:14; // nr of levels up to next `with' environment
     enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 2cc03fe61bb..d693a3b206f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -506,13 +506,6 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
 }
 
 
-void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    /* We're not forcing the argument on purpose. */
-    mkInt(v, valueSize(*args[0]));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
@@ -2206,7 +2199,6 @@ void EvalState::createBaseEnv()
 
     // Debugging
     addPrimOp("__trace", 2, prim_trace);
-    addPrimOp("__valueSize", 1, prim_valueSize);
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 6743d7fd801..68937387317 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -62,9 +62,6 @@ class ExternalValueBase
     /* Return a string to be used in builtins.typeOf */
     virtual string typeOf() const = 0;
 
-    /* How much space does this value take up */
-    virtual size_t valueSize(std::set<const void *> & seen) const = 0;
-
     /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error
      */
@@ -255,12 +252,6 @@ static inline void mkPathNoCopy(Value & v, const char * s)
 void mkPath(Value & v, const char * s);
 
 
-/* Compute the size in bytes of the given value, including all values
-   and environments reachable from it. Static expressions (Exprs) are
-   not included. */
-size_t valueSize(Value & v);
-
-
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;

From 895ed4cef09729609d9f1c0b0abe1cc86356c086 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 28 Nov 2019 15:10:18 +0100
Subject: [PATCH 1704/2196] Remove RPM spec file

Closes #3225.
Closes #3226.
---
 .gitignore  |   1 -
 local.mk    |   2 +-
 nix.spec.in | 173 ----------------------------------------------------
 3 files changed, 1 insertion(+), 175 deletions(-)
 delete mode 100644 nix.spec.in

diff --git a/.gitignore b/.gitignore
index fd62dfb3856..3f9758b9d85 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,6 @@ perl/Makefile.config
 /precompiled-headers.h.pch
 /config.*
 /configure
-/nix.spec
 /stamp-h1
 /svn-revision
 /libtool
diff --git a/local.mk b/local.mk
index 4b380176f2e..921ea91cd2d 100644
--- a/local.mk
+++ b/local.mk
@@ -2,7 +2,7 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell cat .dist-files)
 endif
 
-dist-files += configure config.h.in nix.spec perl/configure
+dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
diff --git a/nix.spec.in b/nix.spec.in
deleted file mode 100644
index 6b9e3763738..00000000000
--- a/nix.spec.in
+++ /dev/null
@@ -1,173 +0,0 @@
-%undefine _hardened_build
-
-%global nixbld_user "nix-builder-"
-%global nixbld_group "nixbld"
-
-# NOTE: BUILD on EL7 requires
-# - Centos / RHEL7 software collection repository 
-#   yum install centos-release-scl
-#
-# - Recent boost backport
-#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
-#
-
-# Disable documentation generation
-# necessary on some platforms
-%bcond_without docgen
-
-Summary: The Nix software deployment system
-Name: nix
-Version: @PACKAGE_VERSION@
-Release: 2%{?dist}
-License: LGPLv2+
-Group: Applications/System
-URL: http://nixos.org/
-Source0: %{name}-%{version}.tar.bz2
-
-Requires: curl
-Requires: bzip2
-Requires: gzip
-Requires: xz
-BuildRequires: bison
-BuildRequires: boost-devel >= 1.60
-BuildRequires: bzip2-devel
-
-# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
-%if 0%{?rhel}
-BuildRequires: devtoolset-7-gcc
-BuildRequires: devtoolset-7-gcc-c++
-%endif
-
-BuildRequires: flex
-BuildRequires: libcurl-devel
-BuildRequires: libseccomp-devel
-BuildRequires: openssl-devel
-BuildRequires: sqlite-devel
-BuildRequires: xz-devel
-
-%description
-Nix is a purely functional package manager. It allows multiple
-versions of a package to be installed side-by-side, ensures that
-dependency specifications are complete, supports atomic upgrades and
-rollbacks, allows non-root users to install software, and has many
-other features. It is the basis of the NixOS Linux distribution, but
-it can be used equally well under other Unix systems.
-
-%package        devel
-Summary:        Development files for %{name}
-Requires:       %{name}%{?_isa} = %{version}-%{release}
-
-%description   devel
-The %{name}-devel package contains libraries and header files for
-developing applications that use %{name}.
-
-
-%package doc
-Summary:        Documentation files for %{name}
-BuildArch:      noarch
-Requires:       %{name} = %{version}-%{release}
-
-%description   doc
-The %{name}-doc package contains documentation files for %{name}.
-
-%prep
-%setup -q
-
-
-%build
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-extraFlags=
-# - override docdir so large documentation files are owned by the
-#   -doc subpackage
-# - set localstatedir by hand to the preferred nix value
-%configure --localstatedir=/nix/var \
-	   %{!?without_docgen:--disable-doc-gen} \
-           --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
-           $extraFlags
-make V=1 %{?_smp_mflags}
-
-
-%install
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-
-make DESTDIR=$RPM_BUILD_ROOT install
-
-find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
-
-# make the store
-mkdir -p $RPM_BUILD_ROOT/nix/store
-chmod 1775 $RPM_BUILD_ROOT/nix/store
-
-# make per-user directories
-for d in profiles gcroots;
-do
-  mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-  chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-done
-
-# fix permission of nix profile
-# (until this is fixed in the relevant Makefile)
-chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh
-
-# we ship this file in the base package
-rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README
-
-# Get rid of Upstart job.
-rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init
-
-
-%clean
-rm -rf $RPM_BUILD_ROOT
-
-
-%pre
-getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group}
-for i in $(seq 10);
-do
-  getent passwd %{nixbld_user}$i >/dev/null || \
-    useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \
-      -s %{_sbindir}/nologin \
-      -c "Nix build user $i" %{nixbld_user}$i
-done
-
-%post
-chgrp %{nixbld_group} /nix/store
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-# Enable and start Nix worker
-systemctl enable nix-daemon.socket nix-daemon.service
-systemctl start  nix-daemon.socket
-%endif
-
-%files
-%license COPYING
-%{_bindir}/nix*
-%{_libdir}/*.so
-%{_prefix}/libexec/*
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-%{_prefix}/lib/systemd/system/nix-daemon.socket
-%{_prefix}/lib/systemd/system/nix-daemon.service
-%endif
-%{_datadir}/nix
-#%if ! %{without docgen}
-#%{_mandir}/man1/*.1*
-#%{_mandir}/man5/*.5*
-#%{_mandir}/man8/*.8*
-#%endif
-%config(noreplace) %{_sysconfdir}/profile.d/nix.sh
-%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
-/nix
-
-%files devel
-%{_includedir}/nix
-%{_prefix}/lib/pkgconfig/*.pc
-
-
-#%if ! %{without docgen}
-#%files doc
-#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-#%{_defaultdocdir}/%{name}-doc-%{version}
-#%endif

From 39954a958623431acb8642372f881cbdb7bb789d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 29 Nov 2019 18:27:40 +0100
Subject: [PATCH 1705/2196] Make libnixrust a dynamic library

This is a hack to fix the build on macOS, which was failing because
libnixrust.a contains compiler builtins that clash with
libclang_rt.osx.a. There's probably a better solution...

https://hydra.nixos.org/build/107473280
---
 nix-rust/Cargo.toml |  2 +-
 nix-rust/local.mk   | 26 ++++++++++++++++++++------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 8b56dc89cd0..c4f4ceb0949 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2018"
 
 [lib]
 name = "nixrust"
-crate-type = ["staticlib"]
+crate-type = ["cdylib"]
 
 [dependencies]
 tar = "0.4"
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index ed1e640c913..7645d5394ef 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -6,15 +6,29 @@ else
   RUST_DIR = debug
 endif
 
-libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a
-libnixrust_INSTALL_PATH := $(libnixrust_PATH)
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
 libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
-libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE)
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
 
-$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+ifeq ($(OS), Darwin)
+libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
+else
+libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
+libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
+endif
+
+$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
 	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
-	cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
-	&& touch target/$(RUST_DIR)/libnixrust.a
+	$(libnixrust_BUILD_FLAGS) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+
+$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
+	$(target-gen) cp $^ $@
+ifeq ($(OS), Darwin)
+	install_name_tool -id $@ $@
+endif
 
 dist-files += $(d)/vendor
 

From 8737980e75bf14cae278f596ac26577bec94b3f9 Mon Sep 17 00:00:00 2001
From: Brian Wignall <brianwignall@gmail.com>
Date: Sat, 30 Nov 2019 19:04:14 -0500
Subject: [PATCH 1706/2196] Fix typos

---
 src/libstore/globals.hh   | 2 +-
 src/libstore/store-api.hh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 9f395425ab9..41e511c6c49 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -319,7 +319,7 @@ public:
         "A program to run just before a build to set derivation-specific build settings."};
 
     Setting<std::string> postBuildHook{this, "", "post-build-hook",
-        "A program to run just after each succesful build."};
+        "A program to run just after each successful build."};
 
     Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         "Path to the netrc file used to obtain usernames/passwords for downloads."};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ec32e995d8f..8e4b133d575 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -569,7 +569,7 @@ public:
         unsigned long long & downloadSize, unsigned long long & narSize);
 
     /* Sort a set of paths topologically under the references
-       relation.  If p refers to q, then p preceeds q in this list. */
+       relation.  If p refers to q, then p precedes q in this list. */
     Paths topoSortPaths(const PathSet & paths);
 
     /* Export multiple paths in the format expected by ‘nix-store

From 7395e091c535df996f2dde3fb8170f2a211fbae5 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 3 Dec 2019 18:01:45 +0100
Subject: [PATCH 1707/2196] doc/manual: add note to `allowSubstitutes` advanced
 attribute

---
 doc/manual/expressions/advanced-attributes.xml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 07b0d97d3f7..e1fbe629194 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -325,12 +325,23 @@ big = "a very long string";
 
   <varlistentry><term><varname>allowSubstitutes</varname></term>
 
-    <listitem><para>If this attribute is set to
+    <listitem>
+    <para>If this attribute is set to
     <literal>false</literal>, then Nix will always build this
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as
     <function>writeText</function> in Nixpkgs) that are cheaper to
-    build than to substitute from a binary cache.</para></listitem>
+    build than to substitute from a binary cache.</para>
+
+    <note><para>You need to have a builder configured which satisfies
+    the derivation’s <literal>system</literal> attribute, since the
+    derivation cannot be substituted. Thus it is usually a good idea
+    to align <literal>system</literal> with
+    <literal>builtins.currentSystem</literal> when setting
+    <literal>allowSubstitutes</literal> to <literal>false</literal>.
+    For most trivial derivations this should be the case.
+    </para></note>
+    </listitem>
 
   </varlistentry>
 

From 7923e22276f4b7fb65df04f53d5e0c4298b61e6d Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 3 Dec 2019 18:22:27 +0100
Subject: [PATCH 1708/2196] doc/manual: add ids to the advanced attribute
 definitions

This makes it possible to reference single attribute definitions,
for pointing people to their exact definition.
---
 .../expressions/advanced-attributes.xml       | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index e1fbe629194..372d03de705 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -11,7 +11,7 @@ attributes.</para>
 
 <variablelist>
 
-  <varlistentry><term><varname>allowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-allowedReferences"><term><varname>allowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>allowedReferences</varname> specifies a list of legal
@@ -32,7 +32,7 @@ allowedReferences = [];
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-allowedRequisites"><term><varname>allowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>allowedReferences</varname>, but it specifies the legal
@@ -50,7 +50,7 @@ allowedRequisites = [ foobar ];
 
   </varlistentry>
 
-  <varlistentry><term><varname>disallowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedReferences"><term><varname>disallowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>disallowedReferences</varname> specifies a list of illegal
@@ -67,7 +67,7 @@ disallowedReferences = [ foo ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>disallowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedRequisites"><term><varname>disallowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>disallowedReferences</varname>, but it specifies illegal
@@ -85,7 +85,7 @@ disallowedRequisites = [ foobar ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>exportReferencesGraph</varname></term>
+  <varlistentry xml:id="adv-attr-exportReferencesGraph"><term><varname>exportReferencesGraph</varname></term>
 
     <listitem><para>This attribute allows builders access to the
     references graph of their inputs.  The attribute is a list of
@@ -124,7 +124,7 @@ derivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>impureEnvVars</varname></term>
+  <varlistentry xml:id="adv-attr-impureEnvVars"><term><varname>impureEnvVars</varname></term>
 
     <listitem><para>This attribute allows you to specify a list of
     environment variables that should be passed from the environment
@@ -158,9 +158,9 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 
   <varlistentry xml:id="fixed-output-drvs">
-    <term><varname>outputHash</varname></term>
-    <term><varname>outputHashAlgo</varname></term>
-    <term><varname>outputHashMode</varname></term>
+    <term xml:id="adv-attr-outputHash"><varname>outputHash</varname></term>
+    <term xml:id="adv-attr-outputHashAlgo"><varname>outputHashAlgo</varname></term>
+    <term xml:id="adv-attr-outputHashMode"><varname>outputHashMode</varname></term>
 
     <listitem><para>These attributes declare that the derivation is a
     so-called <emphasis>fixed-output derivation</emphasis>, which
@@ -282,7 +282,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>passAsFile</varname></term>
+  <varlistentry xml:id="adv-attr-passAsFile"><term><varname>passAsFile</varname></term>
 
     <listitem><para>A list of names of attributes that should be
     passed via files rather than environment variables.  For example,
@@ -309,7 +309,7 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>preferLocalBuild</varname></term>
+  <varlistentry xml:id="adv-attr-preferLocalBuild"><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
     <literal>true</literal> and <link
@@ -323,7 +323,7 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowSubstitutes</varname></term>
+  <varlistentry xml:id="adv-attr-allowSubstitutes"><term><varname>allowSubstitutes</varname></term>
 
     <listitem>
     <para>If this attribute is set to

From c1d18050b4cfed9eba68d4d21b397c6cce035e37 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 3 Dec 2019 19:18:47 +0100
Subject: [PATCH 1709/2196] Disable recursive Nix test on macOS

https://hydra.nixos.org/build/107724274
---
 tests/recursive.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/recursive.sh b/tests/recursive.sh
index 62896c01590..394ae5ddbb6 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -1,5 +1,8 @@
 source common.sh
 
+# FIXME
+if [[ $(uname) != Linux ]]; then exit; fi
+
 clearStore
 
 export unreachable=$(nix add-to-store ./recursive.sh)

From 334b8f8af1c47970e6cdf3ca6f45dfb8cd8e7938 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 17:39:11 +0100
Subject: [PATCH 1710/2196] fmt(): Pass arguments by reference rather than by
 value

---
 src/libutil/logging.hh | 2 +-
 src/libutil/types.hh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5df03da74e0..3fbd7556248 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -158,7 +158,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
 #define vomit(args...) printMsg(lvlVomit, args)
 
 template<typename... Args>
-inline void warn(const std::string & fs, Args... args)
+inline void warn(const std::string & fs, const Args & ... args)
 {
     boost::format f(fs);
     nop{boost::io::detail::feed(f, args)...};
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 4bc91828b8a..4a6be28a249 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -67,7 +67,7 @@ inline std::string fmt(const FormatOrString & fs)
 }
 
 template<typename... Args>
-inline std::string fmt(const std::string & fs, Args... args)
+inline std::string fmt(const std::string & fs, const Args & ... args)
 {
     boost::format f(fs);
     f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);

From 603b2f583c3be45481d2d0e45e8ee8bfa9cfbfcf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 17:48:16 +0100
Subject: [PATCH 1711/2196] Revert "Make fmt() non-recursive"

This reverts commit 2b761d5f50b7dc17dc55c31980c2253c37b3c920.

Also *really* make fmt() take arguments by reference.
---
 src/libutil/logging.hh |  2 +-
 src/libutil/types.hh   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 3fbd7556248..beb5e6b64e6 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -161,7 +161,7 @@ template<typename... Args>
 inline void warn(const std::string & fs, const Args & ... args)
 {
     boost::format f(fs);
-    nop{boost::io::detail::feed(f, args)...};
+    formatHelper(f, args...);
     logger->warn(f.str());
 }
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 4a6be28a249..5d3e764216f 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -51,6 +51,16 @@ struct FormatOrString
    ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
    takes place). */
 
+inline void formatHelper(boost::format & f)
+{
+}
+
+template<typename T, typename... Args>
+inline void formatHelper(boost::format & f, const T & x, const Args & ... args)
+{
+    formatHelper(f % x, args...);
+}
+
 inline std::string fmt(const std::string & s)
 {
     return s;
@@ -71,7 +81,7 @@ inline std::string fmt(const std::string & fs, const Args & ... args)
 {
     boost::format f(fs);
     f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    nop{boost::io::detail::feed(f, args)...};
+    formatHelper(f, args...);
     return f.str();
 }
 

From 092af3c82692cb346dfac0dcb1bafe46703682ca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 18:23:32 +0100
Subject: [PATCH 1712/2196] Eliminate more pass-by-value in variadic calls

---
 src/libutil/types.hh | 6 +++---
 src/libutil/util.hh  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 5d3e764216f..20b96a85ced 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -97,14 +97,14 @@ public:
     unsigned int status = 1; // exit status
 
     template<typename... Args>
-    BaseError(unsigned int status, Args... args)
+    BaseError(unsigned int status, const Args & ... args)
         : err(fmt(args...))
         , status(status)
     {
     }
 
     template<typename... Args>
-    BaseError(Args... args)
+    BaseError(const Args & ... args)
         : err(fmt(args...))
     {
     }
@@ -136,7 +136,7 @@ public:
     int errNo;
 
     template<typename... Args>
-    SysError(Args... args)
+    SysError(const Args & ... args)
         : Error(addErrno(fmt(args...)))
     { }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index b9f9ea8829b..a1acb49b525 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -299,7 +299,7 @@ public:
     int status;
 
     template<typename... Args>
-    ExecError(int status, Args... args)
+    ExecError(int status, const Args & ... args)
         : Error(args...), status(status)
     { }
 };

From f1b5c76c1a5f69b795bf0b5c1afb1853f81225ef Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Nov 2018 15:59:52 +0100
Subject: [PATCH 1713/2196] MultiCommand: Simplify construction

(cherry picked from commit 15a16e5c05d547ec07170df2392263e5e891447b)
---
 src/nix/command.cc | 8 +++++---
 src/nix/command.hh | 8 ++++----
 src/nix/main.cc    | 7 ++++++-
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 724f03e5d76..a40a113db78 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-Commands * RegisterCommand::commands = 0;
+std::vector<ref<Command>> * RegisterCommand::commands = 0;
 
 void Command::printHelp(const string & programName, std::ostream & out)
 {
@@ -22,9 +22,11 @@ void Command::printHelp(const string & programName, std::ostream & out)
     }
 }
 
-MultiCommand::MultiCommand(const Commands & _commands)
-    : commands(_commands)
+MultiCommand::MultiCommand(const std::vector<ref<Command>> & _commands)
 {
+    for (auto & command : _commands)
+        commands.emplace(command->name(), command);
+
     expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
         assert(!command);
         auto i = commands.find(ss[0]);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 2db22dba2f3..be56f89922b 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -182,7 +182,7 @@ public:
 
     std::shared_ptr<Command> command;
 
-    MultiCommand(const Commands & commands);
+    MultiCommand(const std::vector<ref<Command>> & commands);
 
     void printHelp(const string & programName, std::ostream & out) override;
 
@@ -194,12 +194,12 @@ public:
 /* A helper class for registering commands globally. */
 struct RegisterCommand
 {
-    static Commands * commands;
+    static std::vector<ref<Command>> * commands;
 
     RegisterCommand(ref<Command> command)
     {
-        if (!commands) commands = new Commands;
-        commands->emplace(command->name(), command);
+        if (!commands) commands = new std::vector<ref<Command>>;
+        commands->push_back(command);
     }
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 1c9d909d86b..d5cba7fb9a4 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -104,10 +104,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             "--help-config' for a list of configuration settings.\n";
     }
 
+    void printHelp(const string & programName, std::ostream & out)
+    {
+        MultiCommand::printHelp(programName, out);
+        std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
+    }
+
     void showHelpAndExit()
     {
         printHelp(programName, std::cout);
-        std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
         throw Exit();
     }
 };

From f964f428fe6975e06a273e43c3266928a407454f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 22 Nov 2018 16:03:31 +0100
Subject: [PATCH 1714/2196] Move Command and MultiCommand to libutil

(cherry picked from commit f70434b1fbbdb0e188718f0c55a8156a7aa08744)
---
 src/libutil/args.cc | 69 ++++++++++++++++++++++++++++++++++++++++++
 src/libutil/args.hh | 42 +++++++++++++++++++++++++
 src/nix/command.cc  | 74 ---------------------------------------------
 src/nix/command.hh  | 43 --------------------------
 src/nix/main.cc     |  5 +++
 5 files changed, 116 insertions(+), 117 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 7af2a1bf731..2837dacc9fd 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -200,4 +200,73 @@ void printTable(std::ostream & out, const Table2 & table)
     }
 }
 
+void Command::printHelp(const string & programName, std::ostream & out)
+{
+    Args::printHelp(programName, out);
+
+    auto exs = examples();
+    if (!exs.empty()) {
+        out << "\n";
+        out << "Examples:\n";
+        for (auto & ex : exs)
+            out << "\n"
+                << "  " << ex.description << "\n" // FIXME: wrap
+                << "  $ " << ex.command << "\n";
+    }
+}
+
+MultiCommand::MultiCommand(const std::vector<ref<Command>> & _commands)
+{
+    for (auto & command : _commands)
+        commands.emplace(command->name(), command);
+
+    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
+        assert(!command);
+        auto i = commands.find(ss[0]);
+        if (i == commands.end())
+            throw UsageError("'%s' is not a recognised command", ss[0]);
+        command = i->second;
+    }});
+}
+
+void MultiCommand::printHelp(const string & programName, std::ostream & out)
+{
+    if (command) {
+        command->printHelp(programName + " " + command->name(), out);
+        return;
+    }
+
+    out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n";
+
+    out << "\n";
+    out << "Common flags:\n";
+    printFlags(out);
+
+    out << "\n";
+    out << "Available commands:\n";
+
+    Table2 table;
+    for (auto & command : commands) {
+        auto descr = command.second->description();
+        if (!descr.empty())
+            table.push_back(std::make_pair(command.second->name(), descr));
+    }
+    printTable(out, table);
+}
+
+bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
+{
+    if (Args::processFlag(pos, end)) return true;
+    if (command && command->processFlag(pos, end)) return true;
+    return false;
+}
+
+bool MultiCommand::processArgs(const Strings & args, bool finish)
+{
+    if (command)
+        return command->processArgs(args, finish);
+    else
+        return Args::processArgs(args, finish);
+}
+
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index f8df39b5c92..21f4327c510 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -188,6 +188,48 @@ public:
     friend class MultiCommand;
 };
 
+/* A command is an argument parser that can be executed by calling its
+   run() method. */
+struct Command : virtual Args
+{
+    virtual ~Command() { }
+    virtual std::string name() = 0;
+    virtual void prepare() { };
+    virtual void run() = 0;
+
+    struct Example
+    {
+        std::string description;
+        std::string command;
+    };
+
+    typedef std::list<Example> Examples;
+
+    virtual Examples examples() { return Examples(); }
+
+    void printHelp(const string & programName, std::ostream & out) override;
+};
+
+typedef std::map<std::string, ref<Command>> Commands;
+
+/* An argument parser that supports multiple subcommands,
+   i.e. ‘<command> <subcommand>’. */
+class MultiCommand : virtual Args
+{
+public:
+    Commands commands;
+
+    std::shared_ptr<Command> command;
+
+    MultiCommand(const std::vector<ref<Command>> & commands);
+
+    void printHelp(const string & programName, std::ostream & out) override;
+
+    bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
+
+    bool processArgs(const Strings & args, bool finish) override;
+};
+
 Strings argvToStrings(int argc, char * * argv);
 
 /* Helper function for rendering argument labels. */
diff --git a/src/nix/command.cc b/src/nix/command.cc
index a40a113db78..99848fe73b3 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -7,80 +7,6 @@ namespace nix {
 
 std::vector<ref<Command>> * RegisterCommand::commands = 0;
 
-void Command::printHelp(const string & programName, std::ostream & out)
-{
-    Args::printHelp(programName, out);
-
-    auto exs = examples();
-    if (!exs.empty()) {
-        out << "\n";
-        out << "Examples:\n";
-        for (auto & ex : exs)
-            out << "\n"
-                << "  " << ex.description << "\n" // FIXME: wrap
-                << "  $ " << ex.command << "\n";
-    }
-}
-
-MultiCommand::MultiCommand(const std::vector<ref<Command>> & _commands)
-{
-    for (auto & command : _commands)
-        commands.emplace(command->name(), command);
-
-    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
-        assert(!command);
-        auto i = commands.find(ss[0]);
-        if (i == commands.end())
-            throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = i->second;
-    }});
-}
-
-void MultiCommand::printHelp(const string & programName, std::ostream & out)
-{
-    if (command) {
-        command->printHelp(programName + " " + command->name(), out);
-        return;
-    }
-
-    out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n";
-
-    out << "\n";
-    out << "Common flags:\n";
-    printFlags(out);
-
-    out << "\n";
-    out << "Available commands:\n";
-
-    Table2 table;
-    for (auto & command : commands) {
-        auto descr = command.second->description();
-        if (!descr.empty())
-            table.push_back(std::make_pair(command.second->name(), descr));
-    }
-    printTable(out, table);
-
-#if 0
-    out << "\n";
-    out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
-#endif
-}
-
-bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
-{
-    if (Args::processFlag(pos, end)) return true;
-    if (command && command->processFlag(pos, end)) return true;
-    return false;
-}
-
-bool MultiCommand::processArgs(const Strings & args, bool finish)
-{
-    if (command)
-        return command->processArgs(args, finish);
-    else
-        return Args::processArgs(args, finish);
-}
-
 StoreCommand::StoreCommand()
 {
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index be56f89922b..aa34301d999 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -11,29 +11,6 @@ struct Value;
 class Bindings;
 class EvalState;
 struct Pos;
-
-/* A command is an argument parser that can be executed by calling its
-   run() method. */
-struct Command : virtual Args
-{
-    virtual ~Command() { }
-    virtual std::string name() = 0;
-    virtual void prepare() { };
-    virtual void run() = 0;
-
-    struct Example
-    {
-        std::string description;
-        std::string command;
-    };
-
-    typedef std::list<Example> Examples;
-
-    virtual Examples examples() { return Examples(); }
-
-    void printHelp(const string & programName, std::ostream & out) override;
-};
-
 class Store;
 
 /* A command that requires a Nix store. */
@@ -171,26 +148,6 @@ struct StorePathCommand : public InstallablesCommand
     void run(ref<Store> store) override;
 };
 
-typedef std::map<std::string, ref<Command>> Commands;
-
-/* An argument parser that supports multiple subcommands,
-   i.e. ‘<command> <subcommand>’. */
-class MultiCommand : virtual Args
-{
-public:
-    Commands commands;
-
-    std::shared_ptr<Command> command;
-
-    MultiCommand(const std::vector<ref<Command>> & commands);
-
-    void printHelp(const string & programName, std::ostream & out) override;
-
-    bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
-
-    bool processArgs(const Strings & args, bool finish) override;
-};
-
 /* A helper class for registering commands globally. */
 struct RegisterCommand
 {
diff --git a/src/nix/main.cc b/src/nix/main.cc
index d5cba7fb9a4..dcf351cf475 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -107,6 +107,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     void printHelp(const string & programName, std::ostream & out)
     {
         MultiCommand::printHelp(programName, out);
+
+#if 0
+        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
+#endif
+
         std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
     }
 

From ac676856061677559a21670940ac2fac98add9a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 18 Jun 2019 16:01:35 +0200
Subject: [PATCH 1715/2196] Make subcommand construction in MultiCommand lazy

(cherry picked from commit a0de58f471c9087d8e6cc60a6078f9940a125b15)
---
 src/libutil/args.cc                 | 15 +++++++--------
 src/libutil/args.hh                 | 13 ++++++++++---
 src/nix/add-to-store.cc             |  7 +------
 src/nix/build.cc                    |  7 +------
 src/nix/cat.cc                      | 14 ++------------
 src/nix/command.cc                  |  2 +-
 src/nix/command.hh                  | 15 +++++++++++----
 src/nix/copy.cc                     |  7 +------
 src/nix/doctor.cc                   |  7 +------
 src/nix/dump-path.cc                |  7 +------
 src/nix/edit.cc                     |  7 +------
 src/nix/eval.cc                     |  7 +------
 src/nix/hash.cc                     | 26 ++++++--------------------
 src/nix/log.cc                      | 11 +----------
 src/nix/ls.cc                       | 14 ++------------
 src/nix/make-content-addressable.cc |  7 +------
 src/nix/optimise-store.cc           | 11 +----------
 src/nix/path-info.cc                |  7 +------
 src/nix/ping-store.cc               |  7 +------
 src/nix/repl.cc                     |  4 +---
 src/nix/run.cc                      |  7 +------
 src/nix/search.cc                   |  7 +------
 src/nix/show-config.cc              | 11 +----------
 src/nix/show-derivation.cc          |  7 +------
 src/nix/sigs.cc                     | 14 ++------------
 src/nix/upgrade-nix.cc              |  7 +------
 src/nix/verify.cc                   |  7 +------
 src/nix/why-depends.cc              |  7 +------
 28 files changed, 61 insertions(+), 201 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 2837dacc9fd..217495c262b 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -215,17 +215,15 @@ void Command::printHelp(const string & programName, std::ostream & out)
     }
 }
 
-MultiCommand::MultiCommand(const std::vector<ref<Command>> & _commands)
+MultiCommand::MultiCommand(const Commands & commands)
+    : commands(commands)
 {
-    for (auto & command : _commands)
-        commands.emplace(command->name(), command);
-
     expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
         assert(!command);
         auto i = commands.find(ss[0]);
         if (i == commands.end())
             throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = i->second;
+        command = i->second();
     }});
 }
 
@@ -246,10 +244,11 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out)
     out << "Available commands:\n";
 
     Table2 table;
-    for (auto & command : commands) {
-        auto descr = command.second->description();
+    for (auto & i : commands) {
+        auto command = i.second();
+        auto descr = command->description();
         if (!descr.empty())
-            table.push_back(std::make_pair(command.second->name(), descr));
+            table.push_back(std::make_pair(command->name(), descr));
     }
     printTable(out, table);
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 21f4327c510..54336b17bdd 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -192,8 +192,15 @@ public:
    run() method. */
 struct Command : virtual Args
 {
+private:
+    std::string _name;
+
+public:
+
     virtual ~Command() { }
-    virtual std::string name() = 0;
+
+    std::string name() { return _name; }
+
     virtual void prepare() { };
     virtual void run() = 0;
 
@@ -210,7 +217,7 @@ struct Command : virtual Args
     void printHelp(const string & programName, std::ostream & out) override;
 };
 
-typedef std::map<std::string, ref<Command>> Commands;
+typedef std::map<std::string, std::function<ref<Command>()>> Commands;
 
 /* An argument parser that supports multiple subcommands,
    i.e. ‘<command> <subcommand>’. */
@@ -221,7 +228,7 @@ public:
 
     std::shared_ptr<Command> command;
 
-    MultiCommand(const std::vector<ref<Command>> & commands);
+    MultiCommand(const Commands & commands);
 
     void printHelp(const string & programName, std::ostream & out) override;
 
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index e86b96e3f3f..296b2c7e4cd 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -22,11 +22,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             .dest(&namePart);
     }
 
-    std::string name() override
-    {
-        return "add-to-store";
-    }
-
     std::string description() override
     {
         return "add a path to the Nix store";
@@ -58,4 +53,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdAddToStore>());
+static auto r1 = registerCommand<CmdAddToStore>("add-to-store");
diff --git a/src/nix/build.cc b/src/nix/build.cc
index b329ac38ac2..3c9d2df3954 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -24,11 +24,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand
             .set(&outLink, Path(""));
     }
 
-    std::string name() override
-    {
-        return "build";
-    }
-
     std::string description() override
     {
         return "build a derivation or fetch a store path";
@@ -69,4 +64,4 @@ struct CmdBuild : MixDryRun, InstallablesCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdBuild>());
+static auto r1 = registerCommand<CmdBuild>("build");
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index a35f640d884..851f90abd1b 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -28,11 +28,6 @@ struct CmdCatStore : StoreCommand, MixCat
         expectArg("path", &path);
     }
 
-    std::string name() override
-    {
-        return "cat-store";
-    }
-
     std::string description() override
     {
         return "print the contents of a store file on stdout";
@@ -54,11 +49,6 @@ struct CmdCatNar : StoreCommand, MixCat
         expectArg("path", &path);
     }
 
-    std::string name() override
-    {
-        return "cat-nar";
-    }
-
     std::string description() override
     {
         return "print the contents of a file inside a NAR file";
@@ -70,5 +60,5 @@ struct CmdCatNar : StoreCommand, MixCat
     }
 };
 
-static RegisterCommand r1(make_ref<CmdCatStore>());
-static RegisterCommand r2(make_ref<CmdCatNar>());
+static auto r1 = registerCommand<CmdCatStore>("cat-store");
+static auto r2 = registerCommand<CmdCatNar>("cat-nar");
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 99848fe73b3..9dbc3685afc 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-std::vector<ref<Command>> * RegisterCommand::commands = 0;
+Commands * RegisterCommand::commands = nullptr;
 
 StoreCommand::StoreCommand()
 {
diff --git a/src/nix/command.hh b/src/nix/command.hh
index aa34301d999..7474c5eae26 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -151,15 +151,22 @@ struct StorePathCommand : public InstallablesCommand
 /* A helper class for registering commands globally. */
 struct RegisterCommand
 {
-    static std::vector<ref<Command>> * commands;
+    static Commands * commands;
 
-    RegisterCommand(ref<Command> command)
+    RegisterCommand(const std::string & name,
+        std::function<ref<Command>()> command)
     {
-        if (!commands) commands = new std::vector<ref<Command>>;
-        commands->push_back(command);
+        if (!commands) commands = new Commands;
+        commands->emplace(name, command);
     }
 };
 
+template<class T>
+static RegisterCommand registerCommand(const std::string & name)
+{
+    return RegisterCommand(name, [](){ return make_ref<T>(); });
+}
+
 std::shared_ptr<Installable> parseInstallable(
     SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
     bool useDefaultInstallables);
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 12a9f9cd337..b1aceb15c4e 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -42,11 +42,6 @@ struct CmdCopy : StorePathsCommand
             .set(&substitute, Substitute);
     }
 
-    std::string name() override
-    {
-        return "copy";
-    }
-
     std::string description() override
     {
         return "copy paths between Nix stores";
@@ -97,4 +92,4 @@ struct CmdCopy : StorePathsCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdCopy>());
+static auto r1 = registerCommand<CmdCopy>("copy");
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 481c93b45d8..2f3f9d53f06 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -38,11 +38,6 @@ struct CmdDoctor : StoreCommand
 {
     bool success = true;
 
-    std::string name() override
-    {
-        return "doctor";
-    }
-
     std::string description() override
     {
         return "check your system for potential problems and print a PASS or FAIL for each check.";
@@ -136,4 +131,4 @@ struct CmdDoctor : StoreCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdDoctor>());
+static auto r1 = registerCommand<CmdDoctor>("doctore");
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index f411c0cb7c8..90f1552d92d 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -5,11 +5,6 @@ using namespace nix;
 
 struct CmdDumpPath : StorePathCommand
 {
-    std::string name() override
-    {
-        return "dump-path";
-    }
-
     std::string description() override
     {
         return "dump a store path to stdout (in NAR format)";
@@ -33,4 +28,4 @@ struct CmdDumpPath : StorePathCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdDumpPath>());
+static auto r1 = registerCommand<CmdDumpPath>("dump-path");
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 553765f137a..ca410cd1f80 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -10,11 +10,6 @@ using namespace nix;
 
 struct CmdEdit : InstallableCommand
 {
-    std::string name() override
-    {
-        return "edit";
-    }
-
     std::string description() override
     {
         return "open the Nix expression of a Nix package in $EDITOR";
@@ -50,4 +45,4 @@ struct CmdEdit : InstallableCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdEdit>());
+static auto r1 = registerCommand<CmdEdit>("edit");
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index daefea7578b..276fdf00336 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -18,11 +18,6 @@ struct CmdEval : MixJSON, InstallableCommand
         mkFlag(0, "raw", "print strings unquoted", &raw);
     }
 
-    std::string name() override
-    {
-        return "eval";
-    }
-
     std::string description() override
     {
         return "evaluate a Nix expression";
@@ -74,4 +69,4 @@ struct CmdEval : MixJSON, InstallableCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdEval>());
+static auto r1 = registerCommand<CmdEval>("eval");
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index d7451376c4a..0cc523f507e 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -36,11 +36,6 @@ struct CmdHash : Command
         expectArgs("paths", &paths);
     }
 
-    std::string name() override
-    {
-        return mode == mFile ? "hash-file" : "hash-path";
-    }
-
     std::string description() override
     {
         return mode == mFile
@@ -71,8 +66,8 @@ struct CmdHash : Command
     }
 };
 
-static RegisterCommand r1(make_ref<CmdHash>(CmdHash::mFile));
-static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath));
+static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); });
+static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); });
 
 struct CmdToBase : Command
 {
@@ -88,15 +83,6 @@ struct CmdToBase : Command
         expectArgs("strings", &args);
     }
 
-    std::string name() override
-    {
-        return
-            base == Base16 ? "to-base16" :
-            base == Base32 ? "to-base32" :
-            base == Base64 ? "to-base64" :
-            "to-sri";
-    }
-
     std::string description() override
     {
         return fmt("convert a hash to %s representation",
@@ -113,10 +99,10 @@ struct CmdToBase : Command
     }
 };
 
-static RegisterCommand r3(make_ref<CmdToBase>(Base16));
-static RegisterCommand r4(make_ref<CmdToBase>(Base32));
-static RegisterCommand r5(make_ref<CmdToBase>(Base64));
-static RegisterCommand r6(make_ref<CmdToBase>(SRI));
+static RegisterCommand r3("to-base16", [](){ return make_ref<CmdToBase>(Base16); });
+static RegisterCommand r4("to-base32", [](){ return make_ref<CmdToBase>(Base32); });
+static RegisterCommand r5("to-base64", [](){ return make_ref<CmdToBase>(Base64); });
+static RegisterCommand r6("to-sri", [](){ return make_ref<CmdToBase>(SRI); });
 
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
diff --git a/src/nix/log.cc b/src/nix/log.cc
index f07ec4e93a1..122a3d6905f 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -8,15 +8,6 @@ using namespace nix;
 
 struct CmdLog : InstallableCommand
 {
-    CmdLog()
-    {
-    }
-
-    std::string name() override
-    {
-        return "log";
-    }
-
     std::string description() override
     {
         return "show the build log of the specified packages or paths, if available";
@@ -68,4 +59,4 @@ struct CmdLog : InstallableCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdLog>());
+static auto r1 = registerCommand<CmdLog>("log");
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index d089be42fb2..9408cc9da9d 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -100,11 +100,6 @@ struct CmdLsStore : StoreCommand, MixLs
         };
     }
 
-    std::string name() override
-    {
-        return "ls-store";
-    }
-
     std::string description() override
     {
         return "show information about a store path";
@@ -136,11 +131,6 @@ struct CmdLsNar : Command, MixLs
         };
     }
 
-    std::string name() override
-    {
-        return "ls-nar";
-    }
-
     std::string description() override
     {
         return "show information about the contents of a NAR file";
@@ -152,5 +142,5 @@ struct CmdLsNar : Command, MixLs
     }
 };
 
-static RegisterCommand r1(make_ref<CmdLsStore>());
-static RegisterCommand r2(make_ref<CmdLsNar>());
+static auto r1 = registerCommand<CmdLsStore>("ls-store");
+static auto r2 = registerCommand<CmdLsNar>("ls-nar");
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 16344ee14c5..5b99b5084cb 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -11,11 +11,6 @@ struct CmdMakeContentAddressable : StorePathsCommand
         realiseMode = Build;
     }
 
-    std::string name() override
-    {
-        return "make-content-addressable";
-    }
-
     std::string description() override
     {
         return "rewrite a path or closure to content-addressable form";
@@ -92,4 +87,4 @@ struct CmdMakeContentAddressable : StorePathsCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdMakeContentAddressable>());
+static auto r1 = registerCommand<CmdMakeContentAddressable>("make-content-addressable");
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index 725fb75a19e..fed012b0454 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -8,15 +8,6 @@ using namespace nix;
 
 struct CmdOptimiseStore : StoreCommand
 {
-    CmdOptimiseStore()
-    {
-    }
-
-    std::string name() override
-    {
-        return "optimise-store";
-    }
-
     std::string description() override
     {
         return "replace identical files in the store by hard links";
@@ -38,4 +29,4 @@ struct CmdOptimiseStore : StoreCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdOptimiseStore>());
+static auto r1 = registerCommand<CmdOptimiseStore>("optimise-store");
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index dea5f0557b8..2cb718f123a 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -24,11 +24,6 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         mkFlag(0, "sigs", "show signatures", &showSigs);
     }
 
-    std::string name() override
-    {
-        return "path-info";
-    }
-
     std::string description() override
     {
         return "query information about store paths";
@@ -130,4 +125,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
     }
 };
 
-static RegisterCommand r1(make_ref<CmdPathInfo>());
+static auto r1 = registerCommand<CmdPathInfo>("path-info");
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 310942574a2..3a2e542a38e 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -6,11 +6,6 @@ using namespace nix;
 
 struct CmdPingStore : StoreCommand
 {
-    std::string name() override
-    {
-        return "ping-store";
-    }
-
     std::string description() override
     {
         return "test whether a store can be opened";
@@ -32,4 +27,4 @@ struct CmdPingStore : StoreCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdPingStore>());
+static auto r1 = registerCommand<CmdPingStore>("ping-store");
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 35c7aec6678..c2b65b6eae2 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -801,8 +801,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         expectArgs("files", &files);
     }
 
-    std::string name() override { return "repl"; }
-
     std::string description() override
     {
         return "start an interactive environment for evaluating Nix expressions";
@@ -816,6 +814,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs
     }
 };
 
-static RegisterCommand r1(make_ref<CmdRepl>());
+static auto r1 = registerCommand<CmdRepl>("repl");
 
 }
diff --git a/src/nix/run.cc b/src/nix/run.cc
index fd4f9228294..ece4d38541c 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -61,11 +61,6 @@ struct CmdRun : InstallablesCommand
             .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
     }
 
-    std::string name() override
-    {
-        return "run";
-    }
-
     std::string description() override
     {
         return "run a shell in which the specified packages are available";
@@ -182,7 +177,7 @@ struct CmdRun : InstallablesCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdRun>());
+static auto r1 = registerCommand<CmdRun>("run");
 
 void chrootHelper(int argc, char * * argv)
 {
diff --git a/src/nix/search.cc b/src/nix/search.cc
index eb75493e475..76927454307 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -52,11 +52,6 @@ struct CmdSearch : SourceExprCommand, MixJSON
             .handler([&]() { writeCache = false; useCache = false; });
     }
 
-    std::string name() override
-    {
-        return "search";
-    }
-
     std::string description() override
     {
         return "query available packages";
@@ -277,4 +272,4 @@ struct CmdSearch : SourceExprCommand, MixJSON
     }
 };
 
-static RegisterCommand r1(make_ref<CmdSearch>());
+static auto r1 = registerCommand<CmdSearch>("search");
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 86638b50d2c..87544f93771 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -8,15 +8,6 @@ using namespace nix;
 
 struct CmdShowConfig : Command, MixJSON
 {
-    CmdShowConfig()
-    {
-    }
-
-    std::string name() override
-    {
-        return "show-config";
-    }
-
     std::string description() override
     {
         return "show the Nix configuration";
@@ -37,4 +28,4 @@ struct CmdShowConfig : Command, MixJSON
     }
 };
 
-static RegisterCommand r1(make_ref<CmdShowConfig>());
+static auto r1 = registerCommand<CmdShowConfig>("show-config");
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index ee94fded364..6065adc4de3 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -22,11 +22,6 @@ struct CmdShowDerivation : InstallablesCommand
             .set(&recursive, true);
     }
 
-    std::string name() override
-    {
-        return "show-derivation";
-    }
-
     std::string description() override
     {
         return "show the contents of a store derivation";
@@ -116,4 +111,4 @@ struct CmdShowDerivation : InstallablesCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdShowDerivation>());
+static auto r1 = registerCommand<CmdShowDerivation>("show-derivation");
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index b1825c412c2..23bc83ad02d 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -22,11 +22,6 @@ struct CmdCopySigs : StorePathsCommand
             .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
     }
 
-    std::string name() override
-    {
-        return "copy-sigs";
-    }
-
     std::string description() override
     {
         return "copy path signatures from substituters (like binary caches)";
@@ -93,7 +88,7 @@ struct CmdCopySigs : StorePathsCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdCopySigs>());
+static auto r1 = registerCommand<CmdCopySigs>("copy-sigs");
 
 struct CmdSignPaths : StorePathsCommand
 {
@@ -109,11 +104,6 @@ struct CmdSignPaths : StorePathsCommand
             .dest(&secretKeyFile);
     }
 
-    std::string name() override
-    {
-        return "sign-paths";
-    }
-
     std::string description() override
     {
         return "sign the specified paths";
@@ -146,4 +136,4 @@ struct CmdSignPaths : StorePathsCommand
     }
 };
 
-static RegisterCommand r3(make_ref<CmdSignPaths>());
+static auto r2 = registerCommand<CmdSignPaths>("sign-paths");
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 6c9e37d0412..e6c369a7cf3 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -30,11 +30,6 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
             .dest(&storePathsUrl);
     }
 
-    std::string name() override
-    {
-        return "upgrade-nix";
-    }
-
     std::string description() override
     {
         return "upgrade Nix to the latest stable version";
@@ -157,4 +152,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdUpgradeNix>());
+static auto r1 = registerCommand<CmdUpgradeNix>("upgrade-nix");
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 4b0f80c62b4..fa141419615 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -30,11 +30,6 @@ struct CmdVerify : StorePathsCommand
         mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
     }
 
-    std::string name() override
-    {
-        return "verify";
-    }
-
     std::string description() override
     {
         return "verify the integrity of store paths";
@@ -180,4 +175,4 @@ struct CmdVerify : StorePathsCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdVerify>());
+static auto r1 = registerCommand<CmdVerify>("verify");
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 325a2be0a79..fa2a903a86f 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -44,11 +44,6 @@ struct CmdWhyDepends : SourceExprCommand
             .set(&all, true);
     }
 
-    std::string name() override
-    {
-        return "why-depends";
-    }
-
     std::string description() override
     {
         return "show why a package has another package in its closure";
@@ -264,4 +259,4 @@ struct CmdWhyDepends : SourceExprCommand
     }
 };
 
-static RegisterCommand r1(make_ref<CmdWhyDepends>());
+static auto r1 = registerCommand<CmdWhyDepends>("why-depends");

From 5e449b43ed31a518e13b21363ae3ba1994ce1cd7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Jun 2019 23:37:40 +0200
Subject: [PATCH 1716/2196] Initialize Command::_name

(cherry picked from commit d0a769cb061a13ad880c76e5ea69a76150439853)
---
 src/libutil/args.cc | 2 ++
 src/libutil/args.hh | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 217495c262b..ba15ea57109 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -224,6 +224,7 @@ MultiCommand::MultiCommand(const Commands & commands)
         if (i == commands.end())
             throw UsageError("'%s' is not a recognised command", ss[0]);
         command = i->second();
+        command->_name = ss[0];
     }});
 }
 
@@ -246,6 +247,7 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out)
     Table2 table;
     for (auto & i : commands) {
         auto command = i.second();
+        command->_name = i.first;
         auto descr = command->description();
         if (!descr.empty())
             table.push_back(std::make_pair(command->name(), descr));
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 54336b17bdd..967efbe1c7e 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -195,6 +195,8 @@ struct Command : virtual Args
 private:
     std::string _name;
 
+    friend class MultiCommand;
+
 public:
 
     virtual ~Command() { }

From 50d483a2c1c77bef1bab742eb8708626a61dc00e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 20:26:24 +0100
Subject: [PATCH 1717/2196] Fix precompiled-headers generation

It's now regenerated when util.hh changes, and is ordered after
config.h to fix a race.
---
 local.mk                  | 2 ++
 mk/precompiled-headers.mk | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/local.mk b/local.mk
index 921ea91cd2d..01ca8a295f9 100644
--- a/local.mk
+++ b/local.mk
@@ -10,3 +10,5 @@ GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+
+$(GCH) $(PCH): src/libutil/util.hh config.h
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 779389b262d..1a727ba1b70 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

From 0d118ef0c95d5b2ba306372c97526cd2aaaac679 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Nov 2018 19:57:20 +0100
Subject: [PATCH 1718/2196] Bindings::get(): Add convenience method

This allows writing attribute lookups as

    if (auto name = value.attrs->get(state.sName))
      ...

(cherry picked from commit f216c76c56cdffb5214d074a7d44812843dd174f)
---
 src/libexpr/attr-set.hh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 3119a1848af..6c5fb21ad4c 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -4,6 +4,7 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
+#include <optional>
 
 namespace nix {
 
@@ -63,6 +64,14 @@ public:
         return end();
     }
 
+    std::optional<Attr *> get(const Symbol & name)
+    {
+        Attr key(name, 0);
+        iterator i = std::lower_bound(begin(), end(), key);
+        if (i != end() && i->name == name) return &*i;
+        return {};
+    }
+
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 

From 79142cbbe1d5145a18950f4a955e2f2b1878576b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 May 2019 23:44:42 +0200
Subject: [PATCH 1719/2196] Bindings: Add convenience method for requiring an
 attribute

(cherry picked from commit fb692e5f7b34def8cf590298036ab63e40747062)
---
 src/libexpr/attr-set.hh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 6c5fb21ad4c..d6af999126d 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -72,6 +72,14 @@ public:
         return {};
     }
 
+    Attr & need(const Symbol & name, const Pos & pos = noPos)
+    {
+        auto a = get(name);
+        if (!a)
+            throw Error("attribute '%s' missing, at %s", name, pos);
+        return **a;
+    }
+
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 

From 0678e4d56a839f940af8aa70059ced48b393e817 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Nov 2019 17:38:37 +0100
Subject: [PATCH 1720/2196] Move #include

(cherry picked from commit 8beedd44861d1fe7208609ee8d231ca1c02dedf6)
---
 src/libexpr/eval.cc           | 1 +
 src/libexpr/eval.hh           | 1 -
 src/libexpr/function-trace.hh | 3 ++-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b89a67b19f8..e2070d54626 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -7,6 +7,7 @@
 #include "eval-inline.hh"
 #include "download.hh"
 #include "json.hh"
+#include "function-trace.hh"
 
 #include <algorithm>
 #include <chrono>
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 419b703fc38..85aa1ccfec2 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -6,7 +6,6 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
-#include "function-trace.hh"
 
 #include <map>
 #include <optional>
diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh
index 8b0ec848d5a..2c39b7430fa 100644
--- a/src/libexpr/function-trace.hh
+++ b/src/libexpr/function-trace.hh
@@ -1,7 +1,8 @@
 #pragma once
 
 #include "eval.hh"
-#include <sys/time.h>
+
+#include <chrono>
 
 namespace nix {
 

From 47a937d512007d7d78e242d35dd086d332547913 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 31 May 2019 18:48:28 +0200
Subject: [PATCH 1721/2196] Show hash mismatch warnings in SRI format

(cherry picked from commit 63c5c91cc053cbc1fcb8d3fe71c41142c9f51bfa)
---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e9e1fe4b1ef..31011cd7c3d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3685,7 +3685,7 @@ void DerivationGoal::registerOutputs()
                 worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        dest, h.to_string(), h2.to_string()));
+                        dest, h.to_string(SRI), h2.to_string(SRI)));
 
                 Path actualDest = worker.store.toRealPath(dest);
 

From 80ab95315dcd50183bf981d8eefde82435b65124 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 23 Jun 2019 22:19:14 +0200
Subject: [PATCH 1722/2196] nix doctor: Fix typo

(cherry picked from commit 96c6b08ed7f99be84cb1816515a368392d19dbb5)
---
 src/nix/doctor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 2f3f9d53f06..0aa634d6e0f 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -131,4 +131,4 @@ struct CmdDoctor : StoreCommand
     }
 };
 
-static auto r1 = registerCommand<CmdDoctor>("doctore");
+static auto r1 = registerCommand<CmdDoctor>("doctor");

From 3b9c9d34e5ede028860e919d72cc5ae33dd95443 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 19 Apr 2019 14:41:06 +0200
Subject: [PATCH 1723/2196] Shut up clang warning

(cherry picked from commit 3392f1b77869269580b58e4931b7a79f44799ce0)
---
 src/nix/main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index dcf351cf475..5454f4b342b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -104,7 +104,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             "--help-config' for a list of configuration settings.\n";
     }
 
-    void printHelp(const string & programName, std::ostream & out)
+    void printHelp(const string & programName, std::ostream & out) override
     {
         MultiCommand::printHelp(programName, out);
 

From 9ff5f6492f46b7f3342d47f138b590f09e939865 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sat, 7 Dec 2019 22:35:14 +0700
Subject: [PATCH 1724/2196] libarchive proof of concept

---
 Makefile.config.in                       |   1 +
 configure.ac                             |   2 +
 release-common.nix                       |   1 +
 src/libstore/download.cc                 |   2 +-
 src/libutil/local.mk                     |   2 +-
 src/libutil/tarfile.cc                   | 135 +++++++++++++++++++----
 src/libutil/tarfile.hh                   |   3 +-
 src/nix-prefetch-url/nix-prefetch-url.cc |   5 +-
 8 files changed, 124 insertions(+), 27 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 7e3b35b9819..fe609ce0654 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -18,6 +18,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
diff --git a/configure.ac b/configure.ac
index 9dd0acd86c7..29835195f47 100644
--- a/configure.ac
+++ b/configure.ac
@@ -178,6 +178,8 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+# Checks for libarchive
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.4.0], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
diff --git a/release-common.nix b/release-common.nix
index dd5f939d969..f8c93f76e21 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -49,6 +49,7 @@ rec {
     [ curl
       bzip2 xz brotli editline
       openssl pkgconfig sqlite boehmgc
+      libarchive
       boost
       nlohmann_json
       rustc cargo
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 61e88c5c1ef..c7c1b93ad1f 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -907,7 +907,7 @@ CachedDownloadResult Downloader::downloadCached(
             printInfo("unpacking '%s'...", url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
+            unpackTarfile(store->toRealPath(storePath), tmpDir);
             auto members = readDirectory(tmpDir);
             if (members.size() != 1)
                 throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 35c1f6c13dc..16c1fa03ff8 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,6 +6,6 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 libutil_LIBS = libnixrust
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 2cc7793fde8..ab30002ddb9 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -1,5 +1,8 @@
 #include "rust-ffi.hh"
 #include "compression.hh"
+#include <archive.h>
+#include <archive_entry.h>
+#include "finally.hh"
 
 extern "C" {
     rust::Result<std::tuple<>> *
@@ -8,29 +11,123 @@ extern "C" {
 
 namespace nix {
 
-void unpackTarfile(Source & source, const Path & destDir)
-{
-    rust::Source source2(source);
-    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
+std::shared_ptr<struct archive> archive_read_ptr() {
+    return std::shared_ptr<struct archive>(archive_read_new(),
+        [](auto p) {
+            archive_read_close(p);
+            archive_read_free(p);
+        });
 }
-
-void unpackTarfile(const Path & tarFile, const Path & destDir,
-    std::optional<std::string> baseName)
+void archive_read_open_source(std::shared_ptr<struct archive> a, Source& s, unsigned int bufsize = 1024) {
+    std::shared_ptr<unsigned char> buffer((unsigned char*)malloc(bufsize), [](auto p) { free(p); });
+    typedef struct {
+        decltype(buffer) buf;
+        Source& src;
+        unsigned int bs;
+    } St;
+    St* state = new St({buffer, s, bufsize});
+    if (archive_read_open(a.get(), state,
+            NULL /* open */,
+            ([] (struct archive*, void* sptr, const void** buf) -> long int {
+                 St& s = *(static_cast<St*>(sptr));
+                 *buf = s.buf.get();
+                 try {
+                     return s.src.read(s.buf.get(), s.bs);
+                 } catch (EndOfFile &) {
+                     return 0;
+                 }
+                 /* TODO: I don't know what happens if anything else is thrown here */
+             }), [] (struct archive*, void* sptr) {
+                     delete static_cast<St*>(sptr);
+                     return ARCHIVE_OK;
+                 })) {
+        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
+    }
+}
+std::shared_ptr<struct archive> archive_write_ptr() {
+    return std::shared_ptr<struct archive>(archive_write_disk_new(),
+        [](auto p) {
+            archive_write_close(p);
+            archive_write_free(p);
+        });
+}
+static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct archive> aw)
 {
-    if (!baseName) baseName = baseNameOf(tarFile);
+  int r;
+  const void *buff;
+  size_t size;
+  la_int64_t offset;
 
-    auto source = sinkToSource([&](Sink & sink) {
-        // FIXME: look at first few bytes to determine compression type.
-        auto decompressor =
-            // FIXME: add .gz support
-            hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
-            hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
-            makeDecompressionSink("none", sink);
-        readFile(tarFile, *decompressor);
-        decompressor->finish();
-    });
+  for (;;) {
+      r = archive_read_data_block(ar.get(), &buff, &size, &offset);
+      if (r == ARCHIVE_EOF) return;
+      if (r < ARCHIVE_OK) {
+          throw Error("archive is corrupt (%s)", archive_error_string(ar.get()));
+      }
+      r = archive_write_data_block(aw.get(), buff, size, offset);
+      if (r < ARCHIVE_OK) {
+          throw Error("could not write archive output (%s)", archive_error_string(aw.get()));
+      }
+  }
+}
 
-    unpackTarfile(*source, destDir);
+static void extract_archive(std::shared_ptr<struct archive> a, const Path & destDir) {
+    char * cwd = getcwd(0, 0);
+    if (!cwd) throw SysError("getting current directory");
+    Finally freeCwd([&]() { free(cwd); });
+    int r = chdir(destDir.c_str());
+    if (r != 0) throw SysError("setting directory to tar output path");
+    struct archive_entry *entry;
+    r = archive_read_next_header(a.get(), &entry);
+    if (r != ARCHIVE_OK) {
+        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
+    }
+    int flags = 0;
+    auto ext = archive_write_ptr();
+    flags |= ARCHIVE_EXTRACT_PERM;
+    flags |= ARCHIVE_EXTRACT_FFLAGS;
+    archive_write_disk_set_options(ext.get(), flags);
+    archive_write_disk_set_standard_lookup(ext.get());
+    for(;;) {
+        r = archive_read_next_header(a.get(), &entry);
+        if (r == ARCHIVE_EOF) break;
+        if (r == ARCHIVE_WARN) {
+            std::cerr << "warning: " << archive_error_string(a.get());
+        } else if (r < ARCHIVE_WARN) {
+            throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
+        }
+        r = archive_write_header(ext.get(), entry);
+        if (r != ARCHIVE_OK) {
+            throw Error("could not write archive output (%s)", archive_error_string(ext.get()));
+        }
+        if (archive_entry_size(entry) > 0) {
+            copy_data(a, ext);
+        }
+        archive_write_finish_entry(ext.get());
+    }
+    r = chdir(cwd);
+    if (r != 0) throw SysError("resetting directory after archive extraction");
+}
+void unpackTarfile(Source & source, const Path & destDir)
+{
+    auto a = archive_read_ptr();
+    archive_read_support_filter_all(a.get());
+    archive_read_support_format_all(a.get());
+    archive_read_open_source(a, source);
+    createDirs(destDir);
+    extract_archive(a, destDir);
+}
+void unpackTarfile(const Path & tarFile, const Path & destDir)
+{
+    auto a = archive_read_ptr();
+    archive_read_support_filter_all(a.get());
+    archive_read_support_format_all(a.get());
+    int r = archive_read_open_filename(a.get(), tarFile.c_str(), 16384);
+    if (r != ARCHIVE_OK) {
+        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
+    }
+    createDirs(destDir);
+    extract_archive(a, destDir);
 }
 
 }
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index ce0911e2a32..89a024f1d10 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -4,7 +4,6 @@ namespace nix {
 
 void unpackTarfile(Source & source, const Path & destDir);
 
-void unpackTarfile(const Path & tarFile, const Path & destDir,
-    std::optional<std::string> baseName = {});
+void unpackTarfile(const Path & tarFile, const Path & destDir);
 
 }
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 78c88383377..48714446bbe 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -190,10 +190,7 @@ static int _main(int argc, char * * argv)
                 printInfo("unpacking...");
                 Path unpacked = (Path) tmpDir + "/unpacked";
                 createDirs(unpacked);
-                if (hasSuffix(baseNameOf(uri), ".zip"))
-                    runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
-                else
-                    unpackTarfile(tmpFile, unpacked, baseNameOf(uri));
+                unpackTarfile(tmpFile, unpacked);
 
                 /* If the archive unpacks to a single file/directory, then use
                    that as the top-level. */

From 232b39076688c0720f0b5d567cd2a2ac1061c676 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sat, 7 Dec 2019 23:00:37 +0700
Subject: [PATCH 1725/2196] fixup! libarchive proof of concept

---
 src/libutil/tarfile.cc | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index ab30002ddb9..6c3fb46348d 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -61,11 +61,11 @@ static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct
   for (;;) {
       r = archive_read_data_block(ar.get(), &buff, &size, &offset);
       if (r == ARCHIVE_EOF) return;
-      if (r < ARCHIVE_OK) {
+      if (r != ARCHIVE_OK) {
           throw Error("archive is corrupt (%s)", archive_error_string(ar.get()));
       }
       r = archive_write_data_block(aw.get(), buff, size, offset);
-      if (r < ARCHIVE_OK) {
+      if (r != ARCHIVE_OK) {
           throw Error("could not write archive output (%s)", archive_error_string(aw.get()));
       }
   }
@@ -74,14 +74,10 @@ static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct
 static void extract_archive(std::shared_ptr<struct archive> a, const Path & destDir) {
     char * cwd = getcwd(0, 0);
     if (!cwd) throw SysError("getting current directory");
-    Finally freeCwd([&]() { free(cwd); });
+    Finally freeCwd([&]() { chdir(cwd); free(cwd); });
     int r = chdir(destDir.c_str());
     if (r != 0) throw SysError("setting directory to tar output path");
     struct archive_entry *entry;
-    r = archive_read_next_header(a.get(), &entry);
-    if (r != ARCHIVE_OK) {
-        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
-    }
     int flags = 0;
     auto ext = archive_write_ptr();
     flags |= ARCHIVE_EXTRACT_PERM;
@@ -100,13 +96,10 @@ static void extract_archive(std::shared_ptr<struct archive> a, const Path & dest
         if (r != ARCHIVE_OK) {
             throw Error("could not write archive output (%s)", archive_error_string(ext.get()));
         }
-        if (archive_entry_size(entry) > 0) {
-            copy_data(a, ext);
-        }
+        copy_data(a, ext);
         archive_write_finish_entry(ext.get());
     }
-    r = chdir(cwd);
-    if (r != 0) throw SysError("resetting directory after archive extraction");
+    //if (r != 0) throw SysError("resetting directory after archive extraction");
 }
 void unpackTarfile(Source & source, const Path & destDir)
 {

From f54c168031a2a9b63c0cbf61fb06fc8d4e40841b Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sat, 7 Dec 2019 23:10:27 +0700
Subject: [PATCH 1726/2196] add wrapper function around libarchive to c++
 errors

---
 src/libutil/tarfile.cc | 48 ++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 6c3fb46348d..8e2f7c8cbd5 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -18,6 +18,14 @@ std::shared_ptr<struct archive> archive_read_ptr() {
             archive_read_free(p);
         });
 }
+static void ac(std::shared_ptr<struct archive> a, int r, const char* str = "archive is corrupt (%s)") {
+    if (r == ARCHIVE_EOF) {
+        throw EndOfFile("tar ended");
+    }
+    if (r != ARCHIVE_OK) {
+        throw Error(str, archive_error_string(a.get()));
+    }
+}
 void archive_read_open_source(std::shared_ptr<struct archive> a, Source& s, unsigned int bufsize = 1024) {
     std::shared_ptr<unsigned char> buffer((unsigned char*)malloc(bufsize), [](auto p) { free(p); });
     typedef struct {
@@ -26,7 +34,7 @@ void archive_read_open_source(std::shared_ptr<struct archive> a, Source& s, unsi
         unsigned int bs;
     } St;
     St* state = new St({buffer, s, bufsize});
-    if (archive_read_open(a.get(), state,
+    ac(a, archive_read_open(a.get(), state,
             NULL /* open */,
             ([] (struct archive*, void* sptr, const void** buf) -> long int {
                  St& s = *(static_cast<St*>(sptr));
@@ -40,9 +48,7 @@ void archive_read_open_source(std::shared_ptr<struct archive> a, Source& s, unsi
              }), [] (struct archive*, void* sptr) {
                      delete static_cast<St*>(sptr);
                      return ARCHIVE_OK;
-                 })) {
-        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
-    }
+                 }));
 }
 std::shared_ptr<struct archive> archive_write_ptr() {
     return std::shared_ptr<struct archive>(archive_write_disk_new(),
@@ -53,28 +59,27 @@ std::shared_ptr<struct archive> archive_write_ptr() {
 }
 static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct archive> aw)
 {
-  int r;
   const void *buff;
   size_t size;
   la_int64_t offset;
 
   for (;;) {
-      r = archive_read_data_block(ar.get(), &buff, &size, &offset);
-      if (r == ARCHIVE_EOF) return;
-      if (r != ARCHIVE_OK) {
-          throw Error("archive is corrupt (%s)", archive_error_string(ar.get()));
-      }
-      r = archive_write_data_block(aw.get(), buff, size, offset);
-      if (r != ARCHIVE_OK) {
-          throw Error("could not write archive output (%s)", archive_error_string(aw.get()));
+      try {
+          ac(ar, archive_read_data_block(ar.get(), &buff, &size, &offset));
+      } catch (EndOfFile &) {
+          return;
       }
+      ac(aw, archive_write_data_block(aw.get(), buff, size, offset), "could not write archive output (%s)");
   }
 }
-
 static void extract_archive(std::shared_ptr<struct archive> a, const Path & destDir) {
     char * cwd = getcwd(0, 0);
     if (!cwd) throw SysError("getting current directory");
-    Finally freeCwd([&]() { chdir(cwd); free(cwd); });
+    Finally backCwd([&]() {
+                        int r = chdir(cwd);
+                        free(cwd);
+                        if (r != 0) throw SysError("resetting directory after archive extraction");
+                    });
     int r = chdir(destDir.c_str());
     if (r != 0) throw SysError("setting directory to tar output path");
     struct archive_entry *entry;
@@ -90,16 +95,12 @@ static void extract_archive(std::shared_ptr<struct archive> a, const Path & dest
         if (r == ARCHIVE_WARN) {
             std::cerr << "warning: " << archive_error_string(a.get());
         } else if (r < ARCHIVE_WARN) {
-            throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
-        }
-        r = archive_write_header(ext.get(), entry);
-        if (r != ARCHIVE_OK) {
-            throw Error("could not write archive output (%s)", archive_error_string(ext.get()));
+            ac(a, r);
         }
+        ac(ext, archive_write_header(ext.get(), entry), "could not write archive output (%s)");
         copy_data(a, ext);
         archive_write_finish_entry(ext.get());
     }
-    //if (r != 0) throw SysError("resetting directory after archive extraction");
 }
 void unpackTarfile(Source & source, const Path & destDir)
 {
@@ -115,10 +116,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir)
     auto a = archive_read_ptr();
     archive_read_support_filter_all(a.get());
     archive_read_support_format_all(a.get());
-    int r = archive_read_open_filename(a.get(), tarFile.c_str(), 16384);
-    if (r != ARCHIVE_OK) {
-        throw Error("archive is corrupt (%s)", archive_error_string(a.get()));
-    }
+    ac(a, archive_read_open_filename(a.get(), tarFile.c_str(), 16384));
     createDirs(destDir);
     extract_archive(a, destDir);
 }

From 1355554d1207267ccfb272532917e8085eeb9fb9 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sat, 7 Dec 2019 23:23:11 +0700
Subject: [PATCH 1727/2196] code 'cleanup'

---
 src/libutil/tarfile.cc | 42 ++++++++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 8e2f7c8cbd5..b44ebea647a 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -20,7 +20,7 @@ std::shared_ptr<struct archive> archive_read_ptr() {
 }
 static void ac(std::shared_ptr<struct archive> a, int r, const char* str = "archive is corrupt (%s)") {
     if (r == ARCHIVE_EOF) {
-        throw EndOfFile("tar ended");
+        throw EndOfFile("reached end of archive");
     }
     if (r != ARCHIVE_OK) {
         throw Error(str, archive_error_string(a.get()));
@@ -72,16 +72,25 @@ static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct
       ac(aw, archive_write_data_block(aw.get(), buff, size, offset), "could not write archive output (%s)");
   }
 }
+struct PushD {
+    char * oldDir;
+    PushD(std::string newDir)  {
+        oldDir = getcwd(0, 0);
+        if (!oldDir) throw SysError("getcwd");
+        int r = chdir(newDir.c_str());
+        if (r != 0) throw SysError("changing directory to tar output path");
+    }
+    ~PushD() {
+        int r = chdir(oldDir);
+        free(oldDir);
+        if (r != 0)
+            std::cerr << "warning: popd failed to chdir";
+        /* can't throw out of a destructor */
+    }
+};
 static void extract_archive(std::shared_ptr<struct archive> a, const Path & destDir) {
-    char * cwd = getcwd(0, 0);
-    if (!cwd) throw SysError("getting current directory");
-    Finally backCwd([&]() {
-                        int r = chdir(cwd);
-                        free(cwd);
-                        if (r != 0) throw SysError("resetting directory after archive extraction");
-                    });
-    int r = chdir(destDir.c_str());
-    if (r != 0) throw SysError("setting directory to tar output path");
+    // need to chdir back *after* archive closing
+    PushD newDir(destDir);
     struct archive_entry *entry;
     int flags = 0;
     auto ext = archive_write_ptr();
@@ -90,7 +99,7 @@ static void extract_archive(std::shared_ptr<struct archive> a, const Path & dest
     archive_write_disk_set_options(ext.get(), flags);
     archive_write_disk_set_standard_lookup(ext.get());
     for(;;) {
-        r = archive_read_next_header(a.get(), &entry);
+        int r = archive_read_next_header(a.get(), &entry);
         if (r == ARCHIVE_EOF) break;
         if (r == ARCHIVE_WARN) {
             std::cerr << "warning: " << archive_error_string(a.get());
@@ -101,22 +110,27 @@ static void extract_archive(std::shared_ptr<struct archive> a, const Path & dest
         copy_data(a, ext);
         archive_write_finish_entry(ext.get());
     }
+    // done in dtor, but this error can be 'failed to set permissions'
+    // so it's important
+    ac(ext, archive_write_close(ext.get()), "finishing archive extraction");
 }
 void unpackTarfile(Source & source, const Path & destDir)
 {
-    auto a = archive_read_ptr();
+    auto a = nix::archive_read_ptr();
     archive_read_support_filter_all(a.get());
     archive_read_support_format_all(a.get());
-    archive_read_open_source(a, source);
+    nix::archive_read_open_source(a, source);
+
     createDirs(destDir);
     extract_archive(a, destDir);
 }
 void unpackTarfile(const Path & tarFile, const Path & destDir)
 {
-    auto a = archive_read_ptr();
+    auto a = nix::archive_read_ptr();
     archive_read_support_filter_all(a.get());
     archive_read_support_format_all(a.get());
     ac(a, archive_read_open_filename(a.get(), tarFile.c_str(), 16384));
+
     createDirs(destDir);
     extract_archive(a, destDir);
 }

From fe7ec70e6bf8458a95b216b5bb72e078a0025486 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Sat, 7 Dec 2019 23:28:31 +0700
Subject: [PATCH 1728/2196] remove rust unpack_tarfile ffi

---
 src/libutil/tarfile.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index b44ebea647a..4da59628183 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -4,11 +4,6 @@
 #include <archive_entry.h>
 #include "finally.hh"
 
-extern "C" {
-    rust::Result<std::tuple<>> *
-    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
-}
-
 namespace nix {
 
 std::shared_ptr<struct archive> archive_read_ptr() {

From 28ee687adff95f35a52cbadf8052a21130b3cdfc Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.com>
Date: Sat, 7 Dec 2019 18:08:33 +0000
Subject: [PATCH 1729/2196] Clean up libarchive support

---
 src/libutil/tarfile.cc | 171 ++++++++++++++++++++---------------------
 1 file changed, 82 insertions(+), 89 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 4da59628183..0dc62b21a2b 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -6,75 +6,78 @@
 
 namespace nix {
 
-std::shared_ptr<struct archive> archive_read_ptr() {
-    return std::shared_ptr<struct archive>(archive_read_new(),
-        [](auto p) {
-            archive_read_close(p);
-            archive_read_free(p);
-        });
-}
-static void ac(std::shared_ptr<struct archive> a, int r, const char* str = "archive is corrupt (%s)") {
-    if (r == ARCHIVE_EOF) {
-        throw EndOfFile("reached end of archive");
+struct TarArchive {
+    struct archive *archive;
+    Source *source;
+    unsigned char buffer[4096];
+
+    void check(int err, const char *reason = "Failed to extract archive (%s)") {
+        if (err == ARCHIVE_EOF)
+            throw EndOfFile("reached end of archive");
+        else if (err != ARCHIVE_OK)
+            throw Error(reason, archive_error_string(this->archive));
     }
-    if (r != ARCHIVE_OK) {
-        throw Error(str, archive_error_string(a.get()));
+
+    TarArchive(Source& source) {
+        this->archive = archive_read_new();
+        this->source = &source;
+
+        archive_read_support_filter_all(archive);
+        archive_read_support_format_all(archive);
+        check(archive_read_open(archive, (void *)this, TarArchive::callback_open, TarArchive::callback_read, TarArchive::callback_close), "Failed to open archive (%s)");
     }
-}
-void archive_read_open_source(std::shared_ptr<struct archive> a, Source& s, unsigned int bufsize = 1024) {
-    std::shared_ptr<unsigned char> buffer((unsigned char*)malloc(bufsize), [](auto p) { free(p); });
-    typedef struct {
-        decltype(buffer) buf;
-        Source& src;
-        unsigned int bs;
-    } St;
-    St* state = new St({buffer, s, bufsize});
-    ac(a, archive_read_open(a.get(), state,
-            NULL /* open */,
-            ([] (struct archive*, void* sptr, const void** buf) -> long int {
-                 St& s = *(static_cast<St*>(sptr));
-                 *buf = s.buf.get();
-                 try {
-                     return s.src.read(s.buf.get(), s.bs);
-                 } catch (EndOfFile &) {
-                     return 0;
-                 }
-                 /* TODO: I don't know what happens if anything else is thrown here */
-             }), [] (struct archive*, void* sptr) {
-                     delete static_cast<St*>(sptr);
-                     return ARCHIVE_OK;
-                 }));
-}
-std::shared_ptr<struct archive> archive_write_ptr() {
-    return std::shared_ptr<struct archive>(archive_write_disk_new(),
-        [](auto p) {
-            archive_write_close(p);
-            archive_write_free(p);
-        });
-}
-static void copy_data(std::shared_ptr<struct archive> ar, std::shared_ptr<struct archive> aw)
-{
-  const void *buff;
-  size_t size;
-  la_int64_t offset;
-
-  for (;;) {
-      try {
-          ac(ar, archive_read_data_block(ar.get(), &buff, &size, &offset));
-      } catch (EndOfFile &) {
-          return;
-      }
-      ac(aw, archive_write_data_block(aw.get(), buff, size, offset), "could not write archive output (%s)");
-  }
-}
+
+    TarArchive(const Path &path) {
+        this->archive = archive_read_new();
+
+        archive_read_support_filter_all(archive);
+        archive_read_support_format_all(archive);
+        check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)");
+    }
+
+    void close() {
+        check(archive_read_close(archive), "Failed to close archive (%s)");
+    }
+
+    ~TarArchive() {
+        if (this->archive) archive_read_free(this->archive);
+    }
+
+private:
+    static int callback_open(struct archive *, void *self) {
+        return ARCHIVE_OK;
+    }
+    
+    static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) {
+        TarArchive *self = (TarArchive *)_self; 
+        *buffer = self->buffer;
+
+        try {
+            return self->source->read(self->buffer, 4096);
+        } catch (EndOfFile &) {
+            return 0;
+        } catch (std::exception &err) {
+            archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+
+            return -1;
+        }
+    }
+
+    static int callback_close(struct archive *, void *self) {
+        return ARCHIVE_OK;
+    }
+};
+
 struct PushD {
     char * oldDir;
-    PushD(std::string newDir)  {
+
+    PushD(const std::string &newDir)  {
         oldDir = getcwd(0, 0);
         if (!oldDir) throw SysError("getcwd");
         int r = chdir(newDir.c_str());
         if (r != 0) throw SysError("changing directory to tar output path");
     }
+
     ~PushD() {
         int r = chdir(oldDir);
         free(oldDir);
@@ -83,51 +86,41 @@ struct PushD {
         /* can't throw out of a destructor */
     }
 };
-static void extract_archive(std::shared_ptr<struct archive> a, const Path & destDir) {
+
+static void extract_archive(TarArchive &archive, const Path & destDir) {
     // need to chdir back *after* archive closing
     PushD newDir(destDir);
     struct archive_entry *entry;
-    int flags = 0;
-    auto ext = archive_write_ptr();
-    flags |= ARCHIVE_EXTRACT_PERM;
-    flags |= ARCHIVE_EXTRACT_FFLAGS;
-    archive_write_disk_set_options(ext.get(), flags);
-    archive_write_disk_set_standard_lookup(ext.get());
+    int flags = ARCHIVE_EXTRACT_FFLAGS | ARCHIVE_EXTRACT_PERM;
+
     for(;;) {
-        int r = archive_read_next_header(a.get(), &entry);
+        int r = archive_read_next_header(archive.archive, &entry);
         if (r == ARCHIVE_EOF) break;
-        if (r == ARCHIVE_WARN) {
-            std::cerr << "warning: " << archive_error_string(a.get());
-        } else if (r < ARCHIVE_WARN) {
-            ac(a, r);
-        }
-        ac(ext, archive_write_header(ext.get(), entry), "could not write archive output (%s)");
-        copy_data(a, ext);
-        archive_write_finish_entry(ext.get());
+        else if (r == ARCHIVE_WARN)
+            std::cerr << "warning: " << archive_error_string(archive.archive) << std::endl;
+        else
+            archive.check(r);
+
+        archive.check(archive_read_extract(archive.archive, entry, flags));
     }
-    // done in dtor, but this error can be 'failed to set permissions'
-    // so it's important
-    ac(ext, archive_write_close(ext.get()), "finishing archive extraction");
+
+    archive.close();
 }
+
 void unpackTarfile(Source & source, const Path & destDir)
 {
-    auto a = nix::archive_read_ptr();
-    archive_read_support_filter_all(a.get());
-    archive_read_support_format_all(a.get());
-    nix::archive_read_open_source(a, source);
+    auto archive = TarArchive(source);
 
     createDirs(destDir);
-    extract_archive(a, destDir);
+    extract_archive(archive, destDir);
 }
+
 void unpackTarfile(const Path & tarFile, const Path & destDir)
 {
-    auto a = nix::archive_read_ptr();
-    archive_read_support_filter_all(a.get());
-    archive_read_support_format_all(a.get());
-    ac(a, archive_read_open_filename(a.get(), tarFile.c_str(), 16384));
+    auto archive = TarArchive(tarFile);
 
     createDirs(destDir);
-    extract_archive(a, destDir);
+    extract_archive(archive, destDir);
 }
 
 }

From eba82b7c88b36445feaccfd396756f9701766bc6 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 9 Dec 2019 17:21:46 +0700
Subject: [PATCH 1730/2196] further clean up libarchive code

---
 src/libutil/tarfile.cc | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 0dc62b21a2b..68e918891d9 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -1,15 +1,14 @@
-#include "rust-ffi.hh"
-#include "compression.hh"
 #include <archive.h>
 #include <archive_entry.h>
-#include "finally.hh"
+
+#include "serialise.hh"
 
 namespace nix {
 
 struct TarArchive {
     struct archive *archive;
     Source *source;
-    unsigned char buffer[4096];
+    std::vector<unsigned char> buffer;
 
     void check(int err, const char *reason = "Failed to extract archive (%s)") {
         if (err == ARCHIVE_EOF)
@@ -18,7 +17,7 @@ struct TarArchive {
             throw Error(reason, archive_error_string(this->archive));
     }
 
-    TarArchive(Source& source) {
+    TarArchive(Source& source) : buffer(4096) {
         this->archive = archive_read_new();
         this->source = &source;
 
@@ -35,6 +34,9 @@ struct TarArchive {
         check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)");
     }
 
+    // disable copy constructor
+    TarArchive(const TarArchive&) = delete;
+
     void close() {
         check(archive_read_close(archive), "Failed to close archive (%s)");
     }
@@ -47,13 +49,13 @@ struct TarArchive {
     static int callback_open(struct archive *, void *self) {
         return ARCHIVE_OK;
     }
-    
+
     static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) {
-        TarArchive *self = (TarArchive *)_self; 
-        *buffer = self->buffer;
+        TarArchive *self = (TarArchive *)_self;
+        *buffer = self->buffer.data();
 
         try {
-            return self->source->read(self->buffer, 4096);
+            return self->source->read(self->buffer.data(), 4096);
         } catch (EndOfFile &) {
             return 0;
         } catch (std::exception &err) {
@@ -82,7 +84,7 @@ struct PushD {
         int r = chdir(oldDir);
         free(oldDir);
         if (r != 0)
-            std::cerr << "warning: popd failed to chdir";
+            std::cerr << "warning: failed to change directory back after tar extraction";
         /* can't throw out of a destructor */
     }
 };
@@ -91,7 +93,11 @@ static void extract_archive(TarArchive &archive, const Path & destDir) {
     // need to chdir back *after* archive closing
     PushD newDir(destDir);
     struct archive_entry *entry;
-    int flags = ARCHIVE_EXTRACT_FFLAGS | ARCHIVE_EXTRACT_PERM;
+    int flags = ARCHIVE_EXTRACT_FFLAGS
+        | ARCHIVE_EXTRACT_PERM
+        | ARCHIVE_EXTRACT_SECURE_SYMLINKS
+        | ARCHIVE_EXTRACT_SECURE_NODOTDOT
+        | ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS;
 
     for(;;) {
         int r = archive_read_next_header(archive.archive, &entry);

From b232eea40ae0dd08293b5ef6542366a0018a1587 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 9 Dec 2019 17:28:15 +0700
Subject: [PATCH 1731/2196] nix-rust: remove unused tar file code

---
 nix-rust/Cargo.lock     | 74 ++---------------------------------------
 nix-rust/Cargo.toml     |  1 -
 nix-rust/src/lib.rs     |  8 -----
 nix-rust/src/tarfile.rs | 46 -------------------------
 4 files changed, 3 insertions(+), 126 deletions(-)
 delete mode 100644 nix-rust/src/tarfile.rs

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 0112ed471db..53e84282698 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,84 +1,16 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "filetime"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "libc"
-version = "0.2.65"
+version = "0.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "nix-rust"
 version = "0.1.0"
 dependencies = [
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
- "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.1.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "tar"
-version = "0.4.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "xattr"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [metadata]
-"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
-"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
-"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
-"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
-"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
-"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
+"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index c4f4ceb0949..40b6b7d50fa 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -9,5 +9,4 @@ name = "nixrust"
 crate-type = ["cdylib"]
 
 [dependencies]
-tar = "0.4"
 libc = "0.2"
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 1b88ac8af88..a25a791e113 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,6 +1,5 @@
 mod error;
 mod foreign;
-mod tarfile;
 
 pub use error::Error;
 
@@ -23,10 +22,3 @@ impl<T> CBox<T> {
     }
 }
 
-#[no_mangle]
-pub extern "C" fn unpack_tarfile(
-    source: foreign::Source,
-    dest_dir: &str,
-) -> CBox<Result<(), error::CppException>> {
-    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
-}
diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs
deleted file mode 100644
index 379d9098f1c..00000000000
--- a/nix-rust/src/tarfile.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use crate::{foreign::Source, Error};
-use std::fs;
-use std::io;
-use std::os::unix::fs::OpenOptionsExt;
-use std::path::Path;
-use tar::Archive;
-
-pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
-    let dest_dir = Path::new(dest_dir);
-
-    let mut tar = Archive::new(source);
-
-    for file in tar.entries()? {
-        let mut file = file?;
-
-        let dest_file = dest_dir.join(file.path()?);
-
-        fs::create_dir_all(dest_file.parent().unwrap())?;
-
-        match file.header().entry_type() {
-            tar::EntryType::Directory => {
-                fs::create_dir(dest_file)?;
-            }
-            tar::EntryType::Regular => {
-                let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 {
-                    0o666
-                } else {
-                    0o777
-                };
-                let mut f = fs::OpenOptions::new()
-                    .create(true)
-                    .write(true)
-                    .mode(mode)
-                    .open(dest_file)?;
-                io::copy(&mut file, &mut f)?;
-            }
-            tar::EntryType::Symlink => {
-                std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
-            }
-            tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {}
-            t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
-        }
-    }
-
-    Ok(())
-}

From 3663a8a7e92bd66f21fa653ba4d825a357ca6168 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 9 Dec 2019 17:31:05 +0700
Subject: [PATCH 1732/2196] release.nix: add libarchive to rpm and deb
 dependencies

---
 release.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release.nix b/release.nix
index 9cf4c74f231..5d742958ce1 100644
--- a/release.nix
+++ b/release.nix
@@ -394,10 +394,10 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libarchive-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
+    memSize = 2047;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;
@@ -417,7 +417,7 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libarchive-dev" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
             ++ extraPackages; };
       memSize = 2047;
       meta.schedulingPriority = 50;

From f765e441237cb6679c33cb44372a5b30168c42c8 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 9 Dec 2019 18:39:37 +0700
Subject: [PATCH 1733/2196] downgrade required libarchive version (ubuntu
 16.04)

---
 configure.ac | 2 +-
 release.nix  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 29835195f47..28a7b68fbc7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -179,7 +179,7 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 # Checks for libarchive
-PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.4.0], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
diff --git a/release.nix b/release.nix
index 5d742958ce1..65c94b17c72 100644
--- a/release.nix
+++ b/release.nix
@@ -424,7 +424,7 @@ let
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libarchive" "libssl1.0.0" "liblzma5" "libseccomp2" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
       doInstallCheck = true;

From 61cc9f34d208f838db3a8935e224aade6038ffdc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 9 Dec 2019 23:57:33 +0100
Subject: [PATCH 1734/2196] Remove UserLock self-lock check

This is no longer needed since we're not using POSIX locks anymore.
---
 src/libstore/build.cc | 75 +++++++++++++------------------------------
 1 file changed, 22 insertions(+), 53 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 31011cd7c3d..69649162e1c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -499,12 +499,6 @@ void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Pa
 class UserLock
 {
 private:
-    /* POSIX locks suck.  If we have a lock on a file, and we open and
-       close that file again (without closing the original file
-       descriptor), we lose the lock.  So we have to be *very* careful
-       not to open a lock file on which we are holding a lock. */
-    static Sync<PathSet> lockedPaths_;
-
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
@@ -515,7 +509,6 @@ class UserLock
 
 public:
     UserLock();
-    ~UserLock();
 
     void kill();
 
@@ -529,9 +522,6 @@ class UserLock
 };
 
 
-Sync<PathSet> UserLock::lockedPaths_;
-
-
 UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
@@ -568,47 +558,34 @@ UserLock::UserLock()
 
         fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
 
-        {
-            auto lockedPaths(lockedPaths_.lock());
-            if (!lockedPaths->insert(fnUserLock).second)
-                /* We already have a lock on this one. */
-                continue;
-        }
+        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+        if (!fd)
+            throw SysError(format("opening user lock '%1%'") % fnUserLock);
 
-        try {
-
-            AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-            if (!fd)
-                throw SysError(format("opening user lock '%1%'") % fnUserLock);
+        if (lockFile(fd.get(), ltWrite, false)) {
+            fdUserLock = std::move(fd);
+            user = i;
+            uid = pw->pw_uid;
 
-            if (lockFile(fd.get(), ltWrite, false)) {
-                fdUserLock = std::move(fd);
-                user = i;
-                uid = pw->pw_uid;
-
-                /* Sanity check... */
-                if (uid == getuid() || uid == geteuid())
-                    throw Error(format("the Nix user should not be a member of '%1%'")
-                        % settings.buildUsersGroup);
+            /* Sanity check... */
+            if (uid == getuid() || uid == geteuid())
+                throw Error(format("the Nix user should not be a member of '%1%'")
+                    % settings.buildUsersGroup);
 
 #if __linux__
-                /* Get the list of supplementary groups of this build user.  This
-                   is usually either empty or contains a group such as "kvm".  */
-                supplementaryGIDs.resize(10);
-                int ngroups = supplementaryGIDs.size();
-                int err = getgrouplist(pw->pw_name, pw->pw_gid,
-                    supplementaryGIDs.data(), &ngroups);
-                if (err == -1)
-                    throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name);
-
-                supplementaryGIDs.resize(ngroups);
+            /* Get the list of supplementary groups of this build user.  This
+               is usually either empty or contains a group such as "kvm".  */
+            supplementaryGIDs.resize(10);
+            int ngroups = supplementaryGIDs.size();
+            int err = getgrouplist(pw->pw_name, pw->pw_gid,
+                supplementaryGIDs.data(), &ngroups);
+            if (err == -1)
+                throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name);
+
+            supplementaryGIDs.resize(ngroups);
 #endif
 
-                return;
-            }
-
-        } catch (...) {
-            lockedPaths_.lock()->erase(fnUserLock);
+            return;
         }
     }
 
@@ -618,14 +595,6 @@ UserLock::UserLock()
 }
 
 
-UserLock::~UserLock()
-{
-    auto lockedPaths(lockedPaths_.lock());
-    auto erased = lockedPaths->erase(fnUserLock);
-    assert(erased);
-}
-
-
 void UserLock::kill()
 {
     killUser(uid);

From 98ef11677c43db9aa669768d9f0cbec704e8831c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Dec 2019 13:32:30 +0100
Subject: [PATCH 1735/2196] EvalState::callFunction(): Make FunctionCallTrace
 use less stack space

The FunctionCallTrace object consumes a few hundred bytes of stack
space, even when tracing is disabled. This was causing stack overflows:

  $ nix-instantiate '<nixpkgs> -A texlive.combined.scheme-full --dry-run
  error: stack overflow (possible infinite recursion)

This is with the default stack size of 8 MiB.

Putting the object on the heap reduces stack usage to < 5 MiB.
---
 src/libexpr/eval.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e2070d54626..4be4cfeea45 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1091,10 +1091,9 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
-    std::optional<FunctionCallTrace> trace;
-    if (evalSettings.traceFunctionCalls) {
-        trace.emplace(pos);
-    }
+    std::unique_ptr<FunctionCallTrace> trace;
+    if (evalSettings.traceFunctionCalls)
+        trace = std::make_unique<FunctionCallTrace>(pos);
 
     forceValue(fun, pos);
 

From dd5d76e2ed2907eec5dce1c0eaea9df31caef27c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 12 Sep 2019 17:39:25 +0200
Subject: [PATCH 1736/2196] Basic BinaryCacheStore implementation using async
 Rust

---
 nix-rust/Cargo.lock                | 1617 +++++++++++++++++++++++++++-
 nix-rust/Cargo.toml                |    5 +
 nix-rust/local.mk                  |    6 +-
 nix-rust/src/binary_cache_store.rs |   51 +
 nix-rust/src/error.rs              |   18 +-
 nix-rust/src/lib.rs                |   38 +
 nix-rust/src/path_info.rs          |   70 ++
 nix-rust/src/store.rs              |  101 ++
 8 files changed, 1886 insertions(+), 20 deletions(-)
 create mode 100644 nix-rust/src/binary_cache_store.rs
 create mode 100644 nix-rust/src/path_info.rs
 create mode 100644 nix-rust/src/store.rs

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 0112ed471db..5a5b1fbd7f3 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,38 +1,1136 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "adler32"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "backtrace"
+version = "0.3.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "backtrace-sys"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "base64"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "byteorder"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "bytes"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "c2-chacha"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "cc"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
-name = "filetime"
-version = "0.2.8"
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "cookie"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "cookie_store"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "crc32fast"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-queue"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "dtoa"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "either"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "error-chain"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "failure"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "filetime"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "flate2"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fuchsia-zircon"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "fuchsia-zircon-sys"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "futures"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "futures-channel-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "futures-core-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "futures-cpupool"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "futures-executor-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "futures-io-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "futures-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-executor-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "futures-sink-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "futures-util-preview"
+version = "0.3.0-alpha.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "h2"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "http"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "http-body"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "httparse"
+version = "1.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "hyper"
+version = "0.12.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "idna"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "idna"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "iovec"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "kernel32-sys"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "libc"
+version = "0.2.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "lock_api"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "log"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "matches"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "maybe-uninit"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "memchr"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "memoffset"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "mime"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "mime_guess"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "mio"
+version = "0.6.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "mio-uds"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "miow"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)",
+ "schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "security-framework 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "net2"
+version = "0.2.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "nix-rust"
+version = "0.1.0"
+dependencies = [
+ "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "openssl"
+version = "0.10.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "percent-encoding"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0-alpha.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "publicsuffix"
+version = "1.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "regex"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "remove_dir_all"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "libc"
-version = "0.2.65"
+name = "reqwest"
+version = "0.9.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cookie_store 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
-name = "nix-rust"
-version = "0.1.0"
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
- "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "redox_syscall"
-version = "0.1.56"
+name = "ryu"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "schannel"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "security-framework"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "serde"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "smallvec"
+version = "0.6.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "string"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "synstructure"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "tar"
@@ -40,11 +1138,318 @@ version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "time"
+version = "0.1.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-buf"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-codec"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-current-thread"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-executor"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-fs"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-io"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-reactor"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-sync"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-tcp"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-threadpool"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-timer"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-udp"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-uds"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "try_from"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "unicase"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "url"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "url"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "uuid"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "vcpkg"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "version_check"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "version_check"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "want"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "winapi"
 version = "0.3.8"
@@ -54,6 +1459,11 @@ dependencies = [
  "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "winapi-build"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
@@ -64,21 +1474,196 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "winreg"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ws2_32-sys"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "xattr"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [metadata]
+"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
+"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
+"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
+"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
+"checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
+"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
+"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+"checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
+"checksum cookie_store 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46750b3f362965f197996c4448e4a0935e791bf7d6631bfce9ee0af3d24c919c"
+"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d"
+"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
+"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
+"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
+"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
+"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
+"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
+"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
+"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
+"checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
+"checksum error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ab49e9dcb602294bc42f9a7dfc9bc6e936fca4418ea300dbfb84fe16de0b7d9"
+"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
+"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
 "checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
-"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
+"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
+"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
+"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
+"checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef"
+"checksum futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "f477fd0292c4a4ae77044454e7f2b413207942ad405f759bb0b4698b7ace5b12"
+"checksum futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2f26f774b81b3847dcda0c81bd4b6313acfb4f69e5a0390c7cb12c058953e9"
+"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
+"checksum futures-executor-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "80705612926df8a1bc05f0057e77460e29318801f988bf7d803a734cf54e7528"
+"checksum futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "ee7de0c1c9ed23f9457b0437fec7663ce64d9cc3c906597e714e529377b5ddd1"
+"checksum futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "efa8f90c4fb2328e381f8adfd4255b4a2b696f77d1c63a3dee6700b564c4e4b5"
+"checksum futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b65a2481863d1b78e094a07e9c0eed458cc7dc6e72b22b7138b8a67d924859"
+"checksum futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "7df53daff1e98cc024bf2720f3ceb0414d96fbb0a94f3cad3a5c3bf3be1d261c"
+"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
+"checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
+"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
+"checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
+"checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
+"checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
+"checksum hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)" = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6"
+"checksum hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
+"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
+"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
+"checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
+"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
+"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
+"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
+"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
+"checksum lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e57b3997725d2b60dbec1297f6c2e2957cc383db1cebd6be812163f969c7d586"
+"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
+"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
+"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
+"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
+"checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
+"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
+"checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625"
+"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
+"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
+"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
+"checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
+"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
+"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
+"checksum openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3cc5799d98e1088141b8e01ff760112bbd9f19d850c124500566ca6901a585"
+"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
+"checksum openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)" = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f"
+"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
+"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
+"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
+"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
+"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
+"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
+"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27"
+"checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b"
+"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
+"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
+"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
+"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
+"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
+"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
+"checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650"
+"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
+"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
+"checksum schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "87f550b06b6cba9c8b8be3ee73f391990116bf527450d2556e9b9ce263b9a021"
+"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
+"checksum security-framework 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8ef2429d7cefe5fd28bd1d2ed41c944547d4ff84776f5935b456da44593a16df"
+"checksum security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e31493fc37615debb8c5090a7aeb4a9730bc61e77ab10b9af59f1a202284f895"
+"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+"checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702"
+"checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0"
+"checksum serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7406489f3d6d6633ea32abb80dede59df6e3d1a6fdd6e867db84bbaff3c7d079"
+"checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a"
+"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
+"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
+"checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
+"checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d"
+"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
+"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
+"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
+"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
+"checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
+"checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
+"checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f"
+"checksum tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443"
+"checksum tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ca6df436c42b0c3330a82d855d2ef017cd793090ad550a6bc2184f4b933532ab"
+"checksum tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af"
+"checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926"
+"checksum tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6732fe6b53c8d11178dcb77ac6d9682af27fc6d4cb87789449152e5377377146"
+"checksum tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d06554cce1ae4a50f42fba8023918afa931413aded705b560e29600ccf7c6d76"
+"checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119"
+"checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c"
+"checksum tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1739638e364e558128461fc1ad84d997702c8e31c2e6b18fb99842268199e827"
+"checksum tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f02298505547f73e60f568359ef0d016d5acd6e830ab9bc7c4a5b3403440121b"
+"checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445"
+"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
+"checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b"
+"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
+"checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
+"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+"checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a"
+"checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61"
+"checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a"
+"checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
+"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
+"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
+"checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
+"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
+"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+"checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
+"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index c4f4ceb0949..8d882f98a0b 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -11,3 +11,8 @@ crate-type = ["cdylib"]
 [dependencies]
 tar = "0.4"
 libc = "0.2"
+futures-preview = { version = "=0.3.0-alpha.18", features = ["compat"] }
+#hyper = "0.12"
+reqwest = "0.9"
+http = "0.1"
+tokio = "0.1"
\ No newline at end of file
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index 7645d5394ef..caaac31c959 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -8,8 +8,8 @@ endif
 
 libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
 libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
-libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
-libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -lssl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl -lssl
 
 ifeq ($(OS), Darwin)
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
@@ -21,7 +21,7 @@ endif
 $(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
 	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
 	$(libnixrust_BUILD_FLAGS) \
-	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	  RUSTC_BOOTSTRAP=1 cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
 	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
 
 $(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
diff --git a/nix-rust/src/binary_cache_store.rs b/nix-rust/src/binary_cache_store.rs
new file mode 100644
index 00000000000..b7e61eb2df4
--- /dev/null
+++ b/nix-rust/src/binary_cache_store.rs
@@ -0,0 +1,51 @@
+use crate::store::{Store, StorePath};
+use crate::path_info::PathInfo;
+use crate::Error;
+use futures::compat::Future01CompatExt;
+
+pub struct BinaryCacheStore {
+    base_uri: String,
+    client: reqwest::r#async::Client,
+}
+
+impl BinaryCacheStore {
+    pub fn new(base_uri: String) -> Self {
+        Self {
+            base_uri,
+            client: reqwest::r#async::Client::new(),
+        }
+    }
+}
+
+impl Store for BinaryCacheStore {
+    fn query_path_info(
+        &self,
+        path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>> {
+        let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash);
+        let path = path.clone();
+        let client = self.client.clone();
+        let store_dir = self.store_dir().to_string();
+
+        Box::pin(async move {
+            let response = client
+                .get(&uri)
+                .send()
+                .compat()
+                .await?;
+
+            if response.status() == reqwest::StatusCode::NOT_FOUND || response.status() == reqwest::StatusCode::FORBIDDEN {
+                return Err(Error::InvalidPath(path));
+            }
+
+            let mut response = response.error_for_status()?;
+
+            let body = response
+                .text()
+                .compat()
+                .await?;
+
+            PathInfo::parse_nar_info(&body, &store_dir)
+        })
+    }
+}
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 519007ea0aa..08b9bcac20c 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -1,6 +1,10 @@
 #[derive(Debug)]
 pub enum Error {
+    InvalidPath(crate::store::StorePath),
+    BadStorePath(std::path::PathBuf),
+    BadNarInfo,
     IOError(std::io::Error),
+    HttpError(reqwest::Error),
     Misc(String),
     Foreign(CppException),
 }
@@ -11,12 +15,24 @@ impl From<std::io::Error> for Error {
     }
 }
 
+impl From<reqwest::Error> for Error {
+    fn from(err: reqwest::Error) -> Self {
+        Error::HttpError(err)
+    }
+}
+
 impl From<Error> for CppException {
     fn from(err: Error) -> Self {
         match err {
+            Error::InvalidPath(_) => unsafe { make_error("invalid path") }, // FIXME
+            Error::BadNarInfo => unsafe { make_error(".narinfo file is corrupt") }, // FIXME
+            Error::BadStorePath(path) => unsafe {
+                make_error(&format!("path '{}' is not a store path", path.display()))
+            }, // FIXME
+            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
+            Error::HttpError(err) => unsafe { make_error(&err.to_string()) },
             Error::Foreign(ex) => ex,
             Error::Misc(s) => unsafe { make_error(&s) },
-            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
         }
     }
 }
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 1b88ac8af88..7445e005c5c 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,6 +1,11 @@
+#![feature(await_macro, async_await)]
+
+mod binary_cache_store;
 mod error;
 mod foreign;
+mod store;
 mod tarfile;
+mod path_info;
 
 pub use error::Error;
 
@@ -30,3 +35,36 @@ pub extern "C" fn unpack_tarfile(
 ) -> CBox<Result<(), error::CppException>> {
     CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
 }
+
+#[no_mangle]
+pub extern "C" fn rust_test() {
+    use crate::store::Store;
+    use futures::future::{FutureExt, TryFutureExt};
+    use std::path::Path;
+
+    let fut = async move {
+        let store: Box<dyn Store> = Box::new(binary_cache_store::BinaryCacheStore::new(
+            "https://cache.nixos.org".to_string(),
+        ));
+
+        let path = store
+            .parse_store_path(&Path::new(
+                "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3",
+            ))
+            .unwrap();
+
+        /*
+        let info = store.query_path_info(&path).await.unwrap();
+
+        eprintln!("INFO = {:?}", info);
+         */
+
+        let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap();
+
+        eprintln!("CLOSURE = {:?}", closure.len());
+
+        Ok(())
+    };
+
+    tokio::run(fut.boxed().compat());
+}
diff --git a/nix-rust/src/path_info.rs b/nix-rust/src/path_info.rs
new file mode 100644
index 00000000000..2759e03d4d5
--- /dev/null
+++ b/nix-rust/src/path_info.rs
@@ -0,0 +1,70 @@
+use crate::store::StorePath;
+use crate::Error;
+use std::collections::BTreeSet;
+
+#[derive(Clone, Debug)]
+pub struct PathInfo {
+    pub path: StorePath,
+    pub references: BTreeSet<StorePath>,
+    pub nar_size: u64,
+    pub deriver: Option<StorePath>,
+
+    // Additional binary cache info.
+    pub url: Option<String>,
+    pub compression: Option<String>,
+    pub file_size: Option<u64>,
+}
+
+impl PathInfo {
+    pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result<Self, Error> {
+        let mut path = None;
+        let mut references = BTreeSet::new();
+        let mut nar_size = None;
+        let mut deriver = None;
+        let mut url = None;
+        let mut compression = None;
+        let mut file_size = None;
+
+        for line in nar_info.lines() {
+            let colon = line.find(':').ok_or(Error::BadNarInfo)?;
+
+            let (name, value) = line.split_at(colon);
+
+            if !value.starts_with(": ") {
+                return Err(Error::BadNarInfo);
+            }
+
+            let value = &value[2..];
+
+            if name == "StorePath" {
+                path = Some(StorePath::new(std::path::Path::new(value), store_dir)?);
+            } else if name == "NarSize" {
+                nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            } else if name == "References" {
+                if !value.is_empty() {
+                    for r in value.split(' ') {
+                        references.insert(StorePath::new_short(r)?);
+                    }
+                }
+            } else if name == "Deriver" {
+                deriver = Some(StorePath::new_short(value)?);
+            } else if name == "URL" {
+                url = Some(value.into());
+            } else if name == "Compression" {
+                compression = Some(value.into());
+            } else if name == "FileSize" {
+                file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            }
+        }
+
+        Ok(PathInfo {
+            path: path.ok_or(Error::BadNarInfo)?,
+            references,
+            nar_size: nar_size.ok_or(Error::BadNarInfo)?,
+            deriver,
+            url: Some(url.ok_or(Error::BadNarInfo)?),
+            compression,
+            file_size,
+        })
+    }
+}
diff --git a/nix-rust/src/store.rs b/nix-rust/src/store.rs
new file mode 100644
index 00000000000..256dcf0f3a5
--- /dev/null
+++ b/nix-rust/src/store.rs
@@ -0,0 +1,101 @@
+use crate::path_info::PathInfo;
+use crate::Error;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePath {
+    pub hash: String,
+    pub name: String,
+}
+
+pub const STORE_PATH_HASH_CHARS: usize = 32;
+
+impl StorePath {
+    pub fn new(path: &Path, store_dir: &str) -> Result<Self, Error> {
+        // FIXME: check store_dir
+        Self::new_short(
+            path.file_name()
+                .ok_or(Error::BadStorePath(path.into()))?
+                .to_str()
+                .ok_or(Error::BadStorePath(path.into()))?,
+        )
+    }
+
+    pub fn new_short(base_name: &str) -> Result<Self, Error> {
+        if base_name.len() < STORE_PATH_HASH_CHARS + 2
+            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
+        {
+            return Err(Error::BadStorePath(base_name.into()));
+        }
+
+        // FIXME: validate name
+
+        Ok(StorePath {
+            hash: base_name[0..STORE_PATH_HASH_CHARS].to_string(),
+            name: base_name[STORE_PATH_HASH_CHARS + 1..].to_string(),
+        })
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePathHash {
+    bytes: [u8; 20],
+}
+
+/*
+impl StorePathHash {
+    pub fn to_base32(&self) -> String {
+        "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz".to_string()
+    }
+}
+*/
+
+pub trait Store: Send + Sync {
+    fn store_dir(&self) -> &str {
+        "/nix/store"
+    }
+
+    fn query_path_info(
+        &self,
+        store_path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>>;
+}
+
+impl dyn Store {
+    pub fn parse_store_path(&self, path: &Path) -> Result<StorePath, Error> {
+        StorePath::new(path, self.store_dir())
+    }
+
+    pub async fn compute_path_closure(
+        &self,
+        roots: BTreeSet<StorePath>,
+    ) -> Result<BTreeMap<StorePath, PathInfo>, Error> {
+        let mut done = BTreeSet::new();
+        let mut result = BTreeMap::new();
+        let mut pending = vec![];
+
+        for root in roots {
+            pending.push(self.query_path_info(&root));
+            done.insert(root);
+        }
+
+        while !pending.is_empty() {
+            let (info, _, remaining) = futures::future::select_all(pending).await;
+            pending = remaining;
+
+            let info = info?;
+
+            for path in &info.references {
+                if !done.contains(path) {
+                    pending.push(self.query_path_info(&path));
+                    done.insert(path.clone());
+                }
+            }
+
+            result.insert(info.path.clone(), info);
+        }
+
+        Ok(result)
+    }
+}

From d832a355ea40930a6d61e454a162ef286b81891f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 12 Sep 2019 18:22:48 +0200
Subject: [PATCH 1737/2196] Use rustls

In particular, this enables HTTP/2 support in reqwest, which is a lot
more efficient.
---
 nix-rust/Cargo.lock | 453 +++++++++++++++++++++++---------------------
 nix-rust/Cargo.toml |   2 +-
 nix-rust/local.mk   |   4 +-
 3 files changed, 240 insertions(+), 219 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 5a5b1fbd7f3..193a4522b5a 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -5,6 +5,11 @@ name = "adler32"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "anyhow"
+version = "1.0.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "autocfg"
 version = "0.1.7"
@@ -43,6 +48,11 @@ name = "bitflags"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "bumpalo"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "byteorder"
 version = "1.3.2"
@@ -58,14 +68,6 @@ dependencies = [
  "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "c2-chacha"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "cc"
 version = "1.0.47"
@@ -110,20 +112,6 @@ dependencies = [
  "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "core-foundation"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "crc32fast"
 version = "1.2.0"
@@ -181,6 +169,14 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ct-logs"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "dtoa"
 version = "0.4.4"
@@ -254,19 +250,6 @@ name = "fnv"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@@ -365,16 +348,6 @@ dependencies = [
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "getrandom"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "h2"
 version = "0.1.26"
@@ -392,6 +365,14 @@ dependencies = [
  "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "heck"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.3"
@@ -456,15 +437,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "hyper-tls"
-version = "0.3.2"
+name = "hyper-rustls"
+version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -508,6 +493,14 @@ name = "itoa"
 version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "js-sys"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"
@@ -627,23 +620,6 @@ dependencies = [
  "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "native-tls"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)",
- "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)",
- "schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "security-framework 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "net2"
 version = "0.2.33"
@@ -667,42 +643,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "num_cpus"
-version = "1.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "openssl"
-version = "0.10.26"
+name = "nom"
+version = "4.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "openssl-probe"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.53"
+name = "num_cpus"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
- "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -744,16 +699,6 @@ name = "pin-utils"
 version = "0.1.0-alpha.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "pkg-config"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.6"
@@ -800,18 +745,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rand"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rand_chacha"
 version = "0.1.1"
@@ -821,15 +754,6 @@ dependencies = [
  "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rand_chacha"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@@ -843,14 +767,6 @@ name = "rand_core"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "rand_core"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rand_hc"
 version = "0.1.0"
@@ -859,14 +775,6 @@ dependencies = [
  "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rand_isaac"
 version = "0.1.1"
@@ -941,14 +849,6 @@ name = "regex-syntax"
 version = "0.6.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "remove_dir_all"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.9.22"
@@ -963,11 +863,11 @@ dependencies = [
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -975,13 +875,29 @@ dependencies = [
  "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ring"
+version = "0.16.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.16"
@@ -996,41 +912,34 @@ dependencies = [
 ]
 
 [[package]]
-name = "ryu"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "schannel"
-version = "0.1.16"
+name = "rustls"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "scopeguard"
-version = "1.0.0"
+name = "ryu"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
-name = "security-framework"
-version = "0.3.4"
+name = "scopeguard"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
-name = "security-framework-sys"
-version = "0.3.3"
+name = "sct"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1103,6 +1012,16 @@ name = "smallvec"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "sourcefile"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "string"
 version = "0.2.1"
@@ -1143,19 +1062,6 @@ dependencies = [
  "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "tempfile"
-version = "3.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "time"
 version = "0.1.42"
@@ -1265,6 +1171,19 @@ dependencies = [
  "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "tokio-sync"
 version = "0.1.7"
@@ -1382,11 +1301,21 @@ dependencies = [
  "smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "untrusted"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "url"
 version = "1.7.2"
@@ -1415,11 +1344,6 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "vcpkg"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "version_check"
 version = "0.1.5"
@@ -1441,9 +1365,105 @@ dependencies = [
 ]
 
 [[package]]
-name = "wasi"
-version = "0.7.0"
+name = "wasm-bindgen"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-macro-support 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "wasm-bindgen-webidl"
+version = "0.2.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+ "weedle 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "webpki"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "weedle"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "winapi"
@@ -1501,27 +1521,27 @@ dependencies = [
 
 [metadata]
 "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
+"checksum anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9267dff192e68f3399525901e709a48c1d3982c9c072fa32f2127a0cb0babf14"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
 "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
 "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+"checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
-"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
 "checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 "checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
 "checksum cookie_store 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46750b3f362965f197996c4448e4a0935e791bf7d6631bfce9ee0af3d24c919c"
-"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d"
-"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
 "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
 "checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
 "checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
 "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
 "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
+"checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113"
 "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 "checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
@@ -1531,8 +1551,6 @@ dependencies = [
 "checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
 "checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
-"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
@@ -1545,19 +1563,20 @@ dependencies = [
 "checksum futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "efa8f90c4fb2328e381f8adfd4255b4a2b696f77d1c63a3dee6700b564c4e4b5"
 "checksum futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b65a2481863d1b78e094a07e9c0eed458cc7dc6e72b22b7138b8a67d924859"
 "checksum futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "7df53daff1e98cc024bf2720f3ceb0414d96fbb0a94f3cad3a5c3bf3be1d261c"
-"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
 "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
+"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
 "checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
 "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
 "checksum hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)" = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6"
-"checksum hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
+"checksum hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)" = "719d85c7df4a7f309a77d145340a063ea929dcb2e025bae46a80345cffec2952"
 "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
 "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
 "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
+"checksum js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "1c840fdb2167497b0bd0db43d6dfe61e91637fa72f9d061f8bd17ddc44ba6414"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
@@ -1573,31 +1592,22 @@ dependencies = [
 "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
 "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
-"checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
+"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
 "checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
-"checksum openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3cc5799d98e1088141b8e01ff760112bbd9f19d850c124500566ca6901a585"
-"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
-"checksum openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)" = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f"
 "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
 "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
-"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
-"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
 "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27"
 "checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b"
 "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
 "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
-"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
 "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
-"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
 "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
 "checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
-"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
 "checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
-"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
 "checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
 "checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
 "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
@@ -1607,15 +1617,14 @@ dependencies = [
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
-"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
 "checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650"
+"checksum ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6747f8da1f2b1fabbee1aaa4eb8a11abf9adef0bf58a41cee45db5d59cecdfac"
 "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+"checksum rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e"
 "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
-"checksum schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "87f550b06b6cba9c8b8be3ee73f391990116bf527450d2556e9b9ce263b9a021"
 "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
-"checksum security-framework 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8ef2429d7cefe5fd28bd1d2ed41c944547d4ff84776f5935b456da44593a16df"
-"checksum security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e31493fc37615debb8c5090a7aeb4a9730bc61e77ab10b9af59f1a202284f895"
+"checksum sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 "checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702"
@@ -1625,11 +1634,12 @@ dependencies = [
 "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
 "checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
+"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"
+"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d"
 "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
 "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
-"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
 "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
@@ -1639,6 +1649,7 @@ dependencies = [
 "checksum tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af"
 "checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926"
 "checksum tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6732fe6b53c8d11178dcb77ac6d9682af27fc6d4cb87789449152e5377377146"
+"checksum tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1df2fa53ac211c136832f530ccb081af9af891af22d685a9493e232c7a359bc2"
 "checksum tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d06554cce1ae4a50f42fba8023918afa931413aded705b560e29600ccf7c6d76"
 "checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119"
 "checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c"
@@ -1650,15 +1661,25 @@ dependencies = [
 "checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
 "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
 "checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
+"checksum unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
 "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+"checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece"
 "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a"
 "checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61"
 "checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a"
-"checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
 "checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
 "checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
-"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+"checksum wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "29ae32af33bacd663a9a28241abecf01f2be64e6a185c6139b04f18b6385c5f2"
+"checksum wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "1845584bd3593442dc0de6e6d9f84454a59a057722f36f005e44665d6ab19d85"
+"checksum wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "87fcc747e6b73c93d22c947a6334644d22cfec5abd8b66238484dc2b0aeb9fe4"
+"checksum wasm-bindgen-macro-support 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3dc4b3f2c4078c8c4a5f363b92fcf62604c5913cbd16c6ff5aaf0f74ec03f570"
+"checksum wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "ca0b78d6d3be8589b95d1d49cdc0794728ca734adf36d7c9f07e6459508bb53d"
+"checksum wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3126356474ceb717c8fb5549ae387c9fbf4872818454f4d87708bee997214bb5"
+"checksum web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "98405c0a2e722ed3db341b4c5b70eb9fe0021621f7350bab76df93b09b649bbf"
+"checksum webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7e664e770ac0110e2384769bcc59ed19e329d81f555916a6e072714957b81b4"
+"checksum webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a262ae37dd9d60f60dd473d1158f9fbebf110ba7b6a5051c8160460f6043718b"
+"checksum weedle 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 8d882f98a0b..107fd976fd8 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -13,6 +13,6 @@ tar = "0.4"
 libc = "0.2"
 futures-preview = { version = "=0.3.0-alpha.18", features = ["compat"] }
 #hyper = "0.12"
-reqwest = "0.9"
+reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"] }
 http = "0.1"
 tokio = "0.1"
\ No newline at end of file
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index caaac31c959..cf1a5c5c526 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -8,8 +8,8 @@ endif
 
 libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
 libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
-libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -lssl
-libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl -lssl
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
 
 ifeq ($(OS), Darwin)
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"

From ce3c41aef0797954412719a8c4597efe8276dcd2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 12 Sep 2019 18:30:34 +0200
Subject: [PATCH 1738/2196] Drop some dependencies

---
 nix-rust/Cargo.lock | 71 ---------------------------------------------
 nix-rust/Cargo.toml |  2 +-
 2 files changed, 1 insertion(+), 72 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 193a4522b5a..a372d299e91 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -599,16 +599,6 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "mio-uds"
-version = "0.6.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "miow"
 version = "0.2.1"
@@ -1081,18 +1071,13 @@ dependencies = [
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1105,16 +1090,6 @@ dependencies = [
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "tokio-codec"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "tokio-current-thread"
 version = "0.1.6"
@@ -1133,16 +1108,6 @@ dependencies = [
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "tokio-fs"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "tokio-io"
 version = "0.1.12"
@@ -1233,37 +1198,6 @@ dependencies = [
  "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "tokio-udp"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-uds"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "try-lock"
 version = "0.2.2"
@@ -1590,7 +1524,6 @@ dependencies = [
 "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
 "checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625"
 "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
-"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
 "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
@@ -1643,10 +1576,8 @@ dependencies = [
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
 "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
-"checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f"
 "checksum tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443"
 "checksum tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ca6df436c42b0c3330a82d855d2ef017cd793090ad550a6bc2184f4b933532ab"
-"checksum tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe6dc22b08d6993916647d108a1a7d15b9cd29c4f4496c62b92c45b5041b7af"
 "checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926"
 "checksum tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6732fe6b53c8d11178dcb77ac6d9682af27fc6d4cb87789449152e5377377146"
 "checksum tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1df2fa53ac211c136832f530ccb081af9af891af22d685a9493e232c7a359bc2"
@@ -1654,8 +1585,6 @@ dependencies = [
 "checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119"
 "checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c"
 "checksum tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1739638e364e558128461fc1ad84d997702c8e31c2e6b18fb99842268199e827"
-"checksum tokio-udp 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f02298505547f73e60f568359ef0d016d5acd6e830ab9bc7c4a5b3403440121b"
-"checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
 "checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b"
 "checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 107fd976fd8..a3807a9365c 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -15,4 +15,4 @@ futures-preview = { version = "=0.3.0-alpha.18", features = ["compat"] }
 #hyper = "0.12"
 reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"] }
 http = "0.1"
-tokio = "0.1"
\ No newline at end of file
+tokio = { version = "0.1", default-features = false }

From a1ff43045b3f1d075c67c262ff6f623b2b5d569e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 15 Sep 2019 23:09:30 +0200
Subject: [PATCH 1739/2196] Move stuff around

---
 nix-rust/local.mk                             |  2 +-
 nix-rust/src/c.rs                             | 42 ++++++++++++
 nix-rust/src/foreign.rs                       | 19 ++++++
 nix-rust/src/lib.rs                           | 65 +------------------
 .../src/{ => store}/binary_cache_store.rs     |  3 +-
 nix-rust/src/store/mod.rs                     |  7 ++
 nix-rust/src/{ => store}/path_info.rs         |  0
 nix-rust/src/{ => store}/store.rs             |  2 +-
 nix-rust/src/util/mod.rs                      |  1 +
 nix-rust/src/{ => util}/tarfile.rs            |  0
 10 files changed, 74 insertions(+), 67 deletions(-)
 create mode 100644 nix-rust/src/c.rs
 rename nix-rust/src/{ => store}/binary_cache_store.rs (95%)
 create mode 100644 nix-rust/src/store/mod.rs
 rename nix-rust/src/{ => store}/path_info.rs (100%)
 rename nix-rust/src/{ => store}/store.rs (98%)
 create mode 100644 nix-rust/src/util/mod.rs
 rename nix-rust/src/{ => util}/tarfile.rs (100%)

diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index cf1a5c5c526..1157f789492 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -18,7 +18,7 @@ libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
 libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
 endif
 
-$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
 	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
 	$(libnixrust_BUILD_FLAGS) \
 	  RUSTC_BOOTSTRAP=1 cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
new file mode 100644
index 00000000000..1abd3419837
--- /dev/null
+++ b/nix-rust/src/c.rs
@@ -0,0 +1,42 @@
+use super::{foreign::{self, CBox}, error, util, store};
+
+#[no_mangle]
+pub extern "C" fn unpack_tarfile(
+    source: foreign::Source,
+    dest_dir: &str,
+) -> CBox<Result<(), error::CppException>> {
+    CBox::new(util::tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
+}
+
+#[no_mangle]
+pub extern "C" fn rust_test() {
+    use crate::store::Store;
+    use futures::future::{FutureExt, TryFutureExt};
+    use std::path::Path;
+
+    let fut = async move {
+        let store: Box<dyn Store> = Box::new(store::BinaryCacheStore::new(
+            "https://cache.nixos.org".to_string(),
+        ));
+
+        let path = store
+            .parse_store_path(&Path::new(
+                "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3",
+            ))
+            .unwrap();
+
+        /*
+        let info = store.query_path_info(&path).await.unwrap();
+
+        eprintln!("INFO = {:?}", info);
+         */
+
+        let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap();
+
+        eprintln!("CLOSURE = {:?}", closure.len());
+
+        Ok(())
+    };
+
+    tokio::run(fut.boxed().compat());
+}
diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs
index 7bce7753c05..8e04280f3e6 100644
--- a/nix-rust/src/foreign.rs
+++ b/nix-rust/src/foreign.rs
@@ -12,3 +12,22 @@ impl std::io::Read for Source {
         Ok(n)
     }
 }
+
+pub struct CBox<T> {
+    pub ptr: *mut libc::c_void,
+    phantom: std::marker::PhantomData<T>,
+}
+
+impl<T> CBox<T> {
+    pub fn new(t: T) -> Self {
+        unsafe {
+            let size = std::mem::size_of::<T>();
+            let ptr = libc::malloc(size);
+            *(ptr as *mut T) = t; // FIXME: probably UB
+            Self {
+                ptr,
+                phantom: std::marker::PhantomData,
+            }
+        }
+    }
+}
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 7445e005c5c..2dbe6a21743 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,70 +1,9 @@
 #![feature(await_macro, async_await)]
 
-mod binary_cache_store;
+mod c;
 mod error;
 mod foreign;
 mod store;
-mod tarfile;
-mod path_info;
+mod util;
 
 pub use error::Error;
-
-pub struct CBox<T> {
-    pub ptr: *mut libc::c_void,
-    phantom: std::marker::PhantomData<T>,
-}
-
-impl<T> CBox<T> {
-    fn new(t: T) -> Self {
-        unsafe {
-            let size = std::mem::size_of::<T>();
-            let ptr = libc::malloc(size);
-            *(ptr as *mut T) = t; // FIXME: probably UB
-            Self {
-                ptr,
-                phantom: std::marker::PhantomData,
-            }
-        }
-    }
-}
-
-#[no_mangle]
-pub extern "C" fn unpack_tarfile(
-    source: foreign::Source,
-    dest_dir: &str,
-) -> CBox<Result<(), error::CppException>> {
-    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
-}
-
-#[no_mangle]
-pub extern "C" fn rust_test() {
-    use crate::store::Store;
-    use futures::future::{FutureExt, TryFutureExt};
-    use std::path::Path;
-
-    let fut = async move {
-        let store: Box<dyn Store> = Box::new(binary_cache_store::BinaryCacheStore::new(
-            "https://cache.nixos.org".to_string(),
-        ));
-
-        let path = store
-            .parse_store_path(&Path::new(
-                "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3",
-            ))
-            .unwrap();
-
-        /*
-        let info = store.query_path_info(&path).await.unwrap();
-
-        eprintln!("INFO = {:?}", info);
-         */
-
-        let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap();
-
-        eprintln!("CLOSURE = {:?}", closure.len());
-
-        Ok(())
-    };
-
-    tokio::run(fut.boxed().compat());
-}
diff --git a/nix-rust/src/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs
similarity index 95%
rename from nix-rust/src/binary_cache_store.rs
rename to nix-rust/src/store/binary_cache_store.rs
index b7e61eb2df4..f3c7b39bbaa 100644
--- a/nix-rust/src/binary_cache_store.rs
+++ b/nix-rust/src/store/binary_cache_store.rs
@@ -1,5 +1,4 @@
-use crate::store::{Store, StorePath};
-use crate::path_info::PathInfo;
+use super::{Store, StorePath, PathInfo};
 use crate::Error;
 use futures::compat::Future01CompatExt;
 
diff --git a/nix-rust/src/store/mod.rs b/nix-rust/src/store/mod.rs
new file mode 100644
index 00000000000..a0d1c72abe4
--- /dev/null
+++ b/nix-rust/src/store/mod.rs
@@ -0,0 +1,7 @@
+mod binary_cache_store;
+mod path_info;
+mod store;
+
+pub use binary_cache_store::BinaryCacheStore;
+pub use path_info::PathInfo;
+pub use store::{Store, StorePath};
diff --git a/nix-rust/src/path_info.rs b/nix-rust/src/store/path_info.rs
similarity index 100%
rename from nix-rust/src/path_info.rs
rename to nix-rust/src/store/path_info.rs
diff --git a/nix-rust/src/store.rs b/nix-rust/src/store/store.rs
similarity index 98%
rename from nix-rust/src/store.rs
rename to nix-rust/src/store/store.rs
index 256dcf0f3a5..ceaed6648ea 100644
--- a/nix-rust/src/store.rs
+++ b/nix-rust/src/store/store.rs
@@ -1,4 +1,4 @@
-use crate::path_info::PathInfo;
+use super::PathInfo;
 use crate::Error;
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
diff --git a/nix-rust/src/util/mod.rs b/nix-rust/src/util/mod.rs
new file mode 100644
index 00000000000..209627893e2
--- /dev/null
+++ b/nix-rust/src/util/mod.rs
@@ -0,0 +1 @@
+pub mod tarfile;
diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/util/tarfile.rs
similarity index 100%
rename from nix-rust/src/tarfile.rs
rename to nix-rust/src/util/tarfile.rs

From cce218f95050e82587f376d51e5f732d1f68eab3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Sep 2019 00:18:17 +0200
Subject: [PATCH 1740/2196] Add base32 encoder/decoder

---
 nix-rust/Cargo.lock         | 183 ++++++++++++++++++++++++++++++++++++
 nix-rust/Cargo.toml         |   6 ++
 nix-rust/local.mk           |   5 +
 nix-rust/src/error.rs       |   2 +
 nix-rust/src/lib.rs         |  11 +++
 nix-rust/src/store/store.rs |   2 +-
 nix-rust/src/util/base32.rs | 140 +++++++++++++++++++++++++++
 nix-rust/src/util/mod.rs    |   1 +
 tests/local.mk              |   3 -
 9 files changed, 349 insertions(+), 4 deletions(-)
 create mode 100644 nix-rust/src/util/base32.rs

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index a372d299e91..81074d60bbd 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -10,6 +10,11 @@ name = "anyhow"
 version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "assert_matches"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "autocfg"
 version = "0.1.7"
@@ -43,6 +48,19 @@ dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "bit-set"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -68,6 +86,14 @@ dependencies = [
  "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "c2-chacha"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.47"
@@ -348,6 +374,16 @@ dependencies = [
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "h2"
 version = "0.1.26"
@@ -381,6 +417,11 @@ dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hex"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "http"
 version = "0.1.21"
@@ -624,9 +665,13 @@ dependencies = [
 name = "nix-rust"
 version = "0.1.0"
 dependencies = [
+ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -641,6 +686,14 @@ dependencies = [
  "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "num-traits"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.11.1"
@@ -689,6 +742,11 @@ name = "pin-utils"
 version = "0.1.0-alpha.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.6"
@@ -697,6 +755,25 @@ dependencies = [
  "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "proptest"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "publicsuffix"
 version = "1.5.4"
@@ -709,6 +786,11 @@ dependencies = [
  "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "quick-error"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "quote"
 version = "1.0.2"
@@ -735,6 +817,18 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rand"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.1.1"
@@ -744,6 +838,15 @@ dependencies = [
  "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.3.1"
@@ -757,6 +860,14 @@ name = "rand_core"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.1.0"
@@ -765,6 +876,14 @@ dependencies = [
  "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand_isaac"
 version = "0.1.1"
@@ -839,6 +958,14 @@ name = "regex-syntax"
 version = "0.6.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.9.22"
@@ -913,6 +1040,17 @@ dependencies = [
  "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rusty-fork"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.2"
@@ -1052,6 +1190,19 @@ dependencies = [
  "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "time"
 version = "0.1.42"
@@ -1288,6 +1439,14 @@ name = "version_check"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "want"
 version = "0.2.0"
@@ -1298,6 +1457,11 @@ dependencies = [
  "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.55"
@@ -1456,14 +1620,18 @@ dependencies = [
 [metadata]
 "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
 "checksum anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9267dff192e68f3399525901e709a48c1d3982c9c072fa32f2127a0cb0babf14"
+"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
 "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
 "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
+"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
+"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
 "checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
@@ -1497,9 +1665,11 @@ dependencies = [
 "checksum futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "efa8f90c4fb2328e381f8adfd4255b4a2b696f77d1c63a3dee6700b564c4e4b5"
 "checksum futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b65a2481863d1b78e094a07e9c0eed458cc7dc6e72b22b7138b8a67d924859"
 "checksum futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "7df53daff1e98cc024bf2720f3ceb0414d96fbb0a94f3cad3a5c3bf3be1d261c"
+"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
 "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
+"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
 "checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
 "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
@@ -1527,20 +1697,28 @@ dependencies = [
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
 "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
+"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
 "checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
 "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
 "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
+"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
 "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27"
+"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
 "checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b"
+"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
 "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
 "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
 "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
 "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
 "checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
 "checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
 "checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
 "checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
 "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
@@ -1550,11 +1728,13 @@ dependencies = [
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
+"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
 "checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650"
 "checksum ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6747f8da1f2b1fabbee1aaa4eb8a11abf9adef0bf58a41cee45db5d59cecdfac"
 "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e"
+"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
 "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
 "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
 "checksum sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
@@ -1573,6 +1753,7 @@ dependencies = [
 "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
 "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
+"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
 "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
@@ -1598,7 +1779,9 @@ dependencies = [
 "checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
 "checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
+"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
 "checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
+"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
 "checksum wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "29ae32af33bacd663a9a28241abecf01f2be64e6a185c6139b04f18b6385c5f2"
 "checksum wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "1845584bd3593442dc0de6e6d9f84454a59a057722f36f005e44665d6ab19d85"
 "checksum wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "87fcc747e6b73c93d22c947a6334644d22cfec5abd8b66238484dc2b0aeb9fe4"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index a3807a9365c..3cfd2a48b41 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -16,3 +16,9 @@ futures-preview = { version = "=0.3.0-alpha.18", features = ["compat"] }
 reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"] }
 http = "0.1"
 tokio = { version = "0.1", default-features = false }
+lazy_static = "1.4"
+
+[dev-dependencies]
+hex = "0.3"
+assert_matches = "1.3"
+proptest = "0.9"
\ No newline at end of file
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index 1157f789492..eaa56fe4862 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -36,3 +36,8 @@ clean: clean-rust
 
 clean-rust:
 	$(suppress) rm -rfv nix-rust/target
+
+check: rust-tests
+
+rust-tests:
+	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) RUSTC_BOOTSTRAP=1 cargo test --release $$(if [[ -d vendor ]]; then echo -Z offline; fi)
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 08b9bcac20c..9d6fa4c517d 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -3,6 +3,7 @@ pub enum Error {
     InvalidPath(crate::store::StorePath),
     BadStorePath(std::path::PathBuf),
     BadNarInfo,
+    BadBase32,
     IOError(std::io::Error),
     HttpError(reqwest::Error),
     Misc(String),
@@ -29,6 +30,7 @@ impl From<Error> for CppException {
             Error::BadStorePath(path) => unsafe {
                 make_error(&format!("path '{}' is not a store path", path.display()))
             }, // FIXME
+            Error::BadBase32 => unsafe { make_error("invalid base32 string") }, // FIXME
             Error::IOError(err) => unsafe { make_error(&err.to_string()) },
             Error::HttpError(err) => unsafe { make_error(&err.to_string()) },
             Error::Foreign(ex) => ex,
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 2dbe6a21743..635c00affb9 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,5 +1,16 @@
 #![feature(await_macro, async_await)]
 
+#[macro_use]
+extern crate lazy_static;
+
+#[cfg(test)]
+#[macro_use]
+extern crate assert_matches;
+
+#[cfg(test)]
+#[macro_use]
+extern crate proptest;
+
 mod c;
 mod error;
 mod foreign;
diff --git a/nix-rust/src/store/store.rs b/nix-rust/src/store/store.rs
index ceaed6648ea..32949a0782b 100644
--- a/nix-rust/src/store/store.rs
+++ b/nix-rust/src/store/store.rs
@@ -12,7 +12,7 @@ pub struct StorePath {
 pub const STORE_PATH_HASH_CHARS: usize = 32;
 
 impl StorePath {
-    pub fn new(path: &Path, store_dir: &str) -> Result<Self, Error> {
+    pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
         // FIXME: check store_dir
         Self::new_short(
             path.file_name()
diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs
new file mode 100644
index 00000000000..ee42a0362d8
--- /dev/null
+++ b/nix-rust/src/util/base32.rs
@@ -0,0 +1,140 @@
+use crate::error::Error;
+use std::collections::HashMap;
+
+pub fn encoded_len(input_len: usize) -> usize {
+    if input_len == 0 {
+        0
+    } else {
+        (input_len * 8 - 1) / 5 + 1
+    }
+}
+
+static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
+
+lazy_static! {
+    static ref BASE32_CHARS_REVERSE: HashMap<char, u8> = BASE32_CHARS
+        .iter()
+        .enumerate()
+        .map(|(n, c)| (*c as char, n as u8))
+        .collect();
+}
+
+pub fn encode(input: &[u8]) -> String {
+    let mut res = String::new();
+    res.reserve(encoded_len(input.len()));
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+
+    for b in input {
+        bits_left |= (*b as u16) << nr_bits_left;
+        nr_bits_left += 8;
+        while nr_bits_left > 5 {
+            res.push(BASE32_CHARS[(bits_left & 0x1f) as usize] as char);
+            bits_left >>= 5;
+            nr_bits_left -= 5;
+        }
+    }
+
+    if nr_bits_left > 0 {
+        res.push(BASE32_CHARS[(bits_left & 0x1f) as usize] as char);
+    }
+
+    res.chars().rev().collect()
+}
+
+pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
+    let mut res = Vec::new();
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+
+    for c in input.chars().rev() {
+        let x = BASE32_CHARS_REVERSE.get(&c).ok_or(Error::BadBase32)?;
+        bits_left |= (*x as u16) << nr_bits_left;
+        nr_bits_left += 5;
+        if nr_bits_left >= 8 {
+            res.push((bits_left & 0xff) as u8);
+            bits_left >>= 8;
+            nr_bits_left -= 8;
+        }
+    }
+
+    if nr_bits_left > 0 && bits_left != 0 {
+        return Err(Error::BadBase32);
+    }
+
+    Ok(res)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use hex;
+
+    #[test]
+    fn test_encode() {
+        assert_eq!(encode(&[]), "");
+
+        assert_eq!(
+            encode(&hex::decode("0839703786356bca59b0f4a32987eb2e6de43ae8").unwrap()),
+            "x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+                    .unwrap()
+            ),
+            "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
+                    .unwrap()
+            ),
+            "2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx"
+        );
+    }
+
+    #[test]
+    fn test_decode() {
+        assert_eq!(hex::encode(decode("").unwrap()), "");
+
+        assert_eq!(
+            hex::encode(decode("x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88").unwrap()),
+            "0839703786356bca59b0f4a32987eb2e6de43ae8"
+        );
+
+        assert_eq!(
+            hex::encode(decode("1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s").unwrap()),
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+        );
+
+        assert_eq!(
+            hex::encode(decode("2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx").unwrap()),
+            "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
+        );
+
+        assert_matches!(
+            decode("xoxf8v9fxf3jk8zln1cwlsrmhqvp0f88"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(
+            decode("2b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(decode("2"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs8"), Err(Error::BadBase32));
+    }
+
+    proptest! {
+
+        #[test]
+        fn roundtrip(s: Vec<u8>) {
+            assert_eq!(s, decode(&encode(&s)).unwrap());
+        }
+    }
+}
diff --git a/nix-rust/src/util/mod.rs b/nix-rust/src/util/mod.rs
index 209627893e2..cd852c55fba 100644
--- a/nix-rust/src/util/mod.rs
+++ b/nix-rust/src/util/mod.rs
@@ -1 +1,2 @@
+pub mod base32;
 pub mod tarfile;
diff --git a/tests/local.mk b/tests/local.mk
index 8c7c673d7c9..dab3a23b6b9 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -1,6 +1,3 @@
-check:
-	@echo "Warning: Nix has no 'make check'. Please install Nix and run 'make installcheck' instead."
-
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   gc.sh \

From 6317f0f7a0663aed9d877031f7815fa292bd6b09 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Sep 2019 22:26:49 +0200
Subject: [PATCH 1741/2196] StorePath improvements

---
 nix-rust/src/error.rs           |  33 ++++++++---
 nix-rust/src/store/mod.rs       |   4 +-
 nix-rust/src/store/path.rs      | 100 ++++++++++++++++++++++++++++++++
 nix-rust/src/store/path_info.rs |   4 +-
 nix-rust/src/store/store.rs     |  50 +---------------
 5 files changed, 130 insertions(+), 61 deletions(-)
 create mode 100644 nix-rust/src/store/path.rs

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 9d6fa4c517d..5717a7a4707 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -1,9 +1,13 @@
+use std::fmt;
+
 #[derive(Debug)]
 pub enum Error {
     InvalidPath(crate::store::StorePath),
     BadStorePath(std::path::PathBuf),
     BadNarInfo,
     BadBase32,
+    StorePathNameTooLong,
+    BadStorePathName,
     IOError(std::io::Error),
     HttpError(reqwest::Error),
     Misc(String),
@@ -22,19 +26,30 @@ impl From<reqwest::Error> for Error {
     }
 }
 
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::InvalidPath(_) => write!(f, "invalid path"),
+            Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
+            Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
+            Error::BadBase32 => write!(f, "invalid base32 string"),
+            Error::StorePathNameTooLong => {
+                write!(f, "store path name is longer than 211 characters")
+            }
+            Error::BadStorePathName => write!(f, "store path name contains forbidden character"),
+            Error::IOError(err) => write!(f, "I/O error: {}", err),
+            Error::HttpError(err) => write!(f, "HTTP error: {}", err),
+            Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
+            Error::Misc(s) => write!(f, "{}", s),
+        }
+    }
+}
+
 impl From<Error> for CppException {
     fn from(err: Error) -> Self {
         match err {
-            Error::InvalidPath(_) => unsafe { make_error("invalid path") }, // FIXME
-            Error::BadNarInfo => unsafe { make_error(".narinfo file is corrupt") }, // FIXME
-            Error::BadStorePath(path) => unsafe {
-                make_error(&format!("path '{}' is not a store path", path.display()))
-            }, // FIXME
-            Error::BadBase32 => unsafe { make_error("invalid base32 string") }, // FIXME
-            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
-            Error::HttpError(err) => unsafe { make_error(&err.to_string()) },
             Error::Foreign(ex) => ex,
-            Error::Misc(s) => unsafe { make_error(&s) },
+            _ => unsafe { make_error(&err.to_string()) },
         }
     }
 }
diff --git a/nix-rust/src/store/mod.rs b/nix-rust/src/store/mod.rs
index a0d1c72abe4..85355b594c7 100644
--- a/nix-rust/src/store/mod.rs
+++ b/nix-rust/src/store/mod.rs
@@ -1,7 +1,9 @@
 mod binary_cache_store;
+mod path;
 mod path_info;
 mod store;
 
 pub use binary_cache_store::BinaryCacheStore;
+pub use path::{StorePath, StorePathHash, StorePathName};
 pub use path_info::PathInfo;
-pub use store::{Store, StorePath};
+pub use store::Store;
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
new file mode 100644
index 00000000000..4b866f9b473
--- /dev/null
+++ b/nix-rust/src/store/path.rs
@@ -0,0 +1,100 @@
+use crate::error::Error;
+use crate::util::base32;
+use std::fmt;
+use std::path::Path;
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePath {
+    pub hash: StorePathHash,
+    pub name: StorePathName,
+}
+
+pub const STORE_PATH_HASH_BYTES: usize = 20;
+pub const STORE_PATH_HASH_CHARS: usize = 32;
+
+impl StorePath {
+    pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
+        // FIXME: check store_dir
+        Self::new_from_base_name(
+            path.file_name()
+                .ok_or(Error::BadStorePath(path.into()))?
+                .to_str()
+                .ok_or(Error::BadStorePath(path.into()))?,
+        )
+    }
+
+    pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
+        if base_name.len() < STORE_PATH_HASH_CHARS + 2
+            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
+        {
+            return Err(Error::BadStorePath(base_name.into()));
+        }
+
+        Ok(StorePath {
+            hash: StorePathHash::new(&base_name[0..STORE_PATH_HASH_CHARS])?,
+            name: StorePathName::new(&base_name[STORE_PATH_HASH_CHARS + 1..])?,
+        })
+    }
+}
+
+impl fmt::Display for StorePath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-{}", self.hash, self.name)
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]);
+
+impl StorePathHash {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        assert_eq!(s.len(), STORE_PATH_HASH_CHARS);
+        let v = base32::decode(s)?;
+        assert_eq!(v.len(), STORE_PATH_HASH_BYTES);
+        let mut bytes: [u8; 20] = Default::default();
+        bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]);
+        Ok(Self(bytes))
+    }
+}
+
+impl fmt::Display for StorePathHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&base32::encode(&self.0))
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePathName(String);
+
+impl StorePathName {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        if s.len() > 211 {
+            return Err(Error::StorePathNameTooLong);
+        }
+
+        if s.starts_with('.')
+            || !s.chars().all(|c| {
+                c.is_ascii_alphabetic()
+                    || c.is_ascii_digit()
+                    || c == '+'
+                    || c == '-'
+                    || c == '.'
+                    || c == '_'
+                    || c == '?'
+                    || c == '='
+            })
+        {
+            return Err(Error::BadStorePathName);
+        }
+
+        Ok(Self(s.to_string()))
+    }
+}
+
+impl fmt::Display for StorePathName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+// FIXME: add tests
diff --git a/nix-rust/src/store/path_info.rs b/nix-rust/src/store/path_info.rs
index 2759e03d4d5..c2903ed29dd 100644
--- a/nix-rust/src/store/path_info.rs
+++ b/nix-rust/src/store/path_info.rs
@@ -43,11 +43,11 @@ impl PathInfo {
             } else if name == "References" {
                 if !value.is_empty() {
                     for r in value.split(' ') {
-                        references.insert(StorePath::new_short(r)?);
+                        references.insert(StorePath::new_from_base_name(r)?);
                     }
                 }
             } else if name == "Deriver" {
-                deriver = Some(StorePath::new_short(value)?);
+                deriver = Some(StorePath::new_from_base_name(value)?);
             } else if name == "URL" {
                 url = Some(value.into());
             } else if name == "Compression" {
diff --git a/nix-rust/src/store/store.rs b/nix-rust/src/store/store.rs
index 32949a0782b..c33dc4a90f1 100644
--- a/nix-rust/src/store/store.rs
+++ b/nix-rust/src/store/store.rs
@@ -1,56 +1,8 @@
-use super::PathInfo;
+use super::{PathInfo, StorePath};
 use crate::Error;
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct StorePath {
-    pub hash: String,
-    pub name: String,
-}
-
-pub const STORE_PATH_HASH_CHARS: usize = 32;
-
-impl StorePath {
-    pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
-        // FIXME: check store_dir
-        Self::new_short(
-            path.file_name()
-                .ok_or(Error::BadStorePath(path.into()))?
-                .to_str()
-                .ok_or(Error::BadStorePath(path.into()))?,
-        )
-    }
-
-    pub fn new_short(base_name: &str) -> Result<Self, Error> {
-        if base_name.len() < STORE_PATH_HASH_CHARS + 2
-            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
-        {
-            return Err(Error::BadStorePath(base_name.into()));
-        }
-
-        // FIXME: validate name
-
-        Ok(StorePath {
-            hash: base_name[0..STORE_PATH_HASH_CHARS].to_string(),
-            name: base_name[STORE_PATH_HASH_CHARS + 1..].to_string(),
-        })
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct StorePathHash {
-    bytes: [u8; 20],
-}
-
-/*
-impl StorePathHash {
-    pub fn to_base32(&self) -> String {
-        "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz".to_string()
-    }
-}
-*/
-
 pub trait Store: Send + Sync {
     fn store_dir(&self) -> &str {
         "/nix/store"

From 7f08975050e19b3e3ed002743ed19022789c7ccc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Sep 2019 00:31:51 +0200
Subject: [PATCH 1742/2196] Add NAR parser

---
 nix-rust/Cargo.lock   |   1 +
 nix-rust/Cargo.toml   |   3 +-
 nix-rust/src/c.rs     |  14 ++++-
 nix-rust/src/error.rs |  18 ++++++
 nix-rust/src/lib.rs   |   1 +
 nix-rust/src/nar.rs   | 126 ++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 160 insertions(+), 3 deletions(-)
 create mode 100644 nix-rust/src/nar.rs

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 81074d60bbd..e4b61c44421 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -666,6 +666,7 @@ name = "nix-rust"
 version = "0.1.0"
 dependencies = [
  "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 3cfd2a48b41..20d1fdaa81d 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -17,8 +17,9 @@ reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"]
 http = "0.1"
 tokio = { version = "0.1", default-features = false }
 lazy_static = "1.4"
+byteorder = "1.3"
 
 [dev-dependencies]
 hex = "0.3"
 assert_matches = "1.3"
-proptest = "0.9"
\ No newline at end of file
+proptest = "0.9"
diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 1abd3419837..542df93d2fe 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -1,4 +1,8 @@
-use super::{foreign::{self, CBox}, error, util, store};
+use super::{
+    error,
+    foreign::{self, CBox},
+    util,
+};
 
 #[no_mangle]
 pub extern "C" fn unpack_tarfile(
@@ -10,7 +14,8 @@ pub extern "C" fn unpack_tarfile(
 
 #[no_mangle]
 pub extern "C" fn rust_test() {
-    use crate::store::Store;
+    /*
+    use crate::store::{self, Store};
     use futures::future::{FutureExt, TryFutureExt};
     use std::path::Path;
 
@@ -39,4 +44,9 @@ pub extern "C" fn rust_test() {
     };
 
     tokio::run(fut.boxed().compat());
+     */
+
+    let file = std::fs::File::open("test.nar").unwrap();
+
+    crate::nar::parse(&mut std::io::BufReader::new(file)).unwrap();
 }
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 5717a7a4707..4defc646d19 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -8,6 +8,15 @@ pub enum Error {
     BadBase32,
     StorePathNameTooLong,
     BadStorePathName,
+    NarSizeFieldTooBig,
+    BadNarString,
+    BadNarPadding,
+    BadNarVersionMagic,
+    MissingNarOpenTag,
+    MissingNarCloseTag,
+    MissingNarField,
+    BadNarField(String),
+    BadExecutableField,
     IOError(std::io::Error),
     HttpError(reqwest::Error),
     Misc(String),
@@ -37,6 +46,15 @@ impl fmt::Display for Error {
                 write!(f, "store path name is longer than 211 characters")
             }
             Error::BadStorePathName => write!(f, "store path name contains forbidden character"),
+            Error::NarSizeFieldTooBig => write!(f, "size field in NAR is too big"),
+            Error::BadNarString => write!(f, "NAR string is not valid UTF-8"),
+            Error::BadNarPadding => write!(f, "NAR padding is not zero"),
+            Error::BadNarVersionMagic => write!(f, "unsupported NAR version"),
+            Error::MissingNarOpenTag => write!(f, "NAR open tag is missing"),
+            Error::MissingNarCloseTag => write!(f, "NAR close tag is missing"),
+            Error::MissingNarField => write!(f, "expected NAR field is missing"),
+            Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s),
+            Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"),
             Error::IOError(err) => write!(f, "I/O error: {}", err),
             Error::HttpError(err) => write!(f, "HTTP error: {}", err),
             Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 635c00affb9..cda63d2c717 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -14,6 +14,7 @@ extern crate proptest;
 mod c;
 mod error;
 mod foreign;
+mod nar;
 mod store;
 mod util;
 
diff --git a/nix-rust/src/nar.rs b/nix-rust/src/nar.rs
new file mode 100644
index 00000000000..aa05d815b2e
--- /dev/null
+++ b/nix-rust/src/nar.rs
@@ -0,0 +1,126 @@
+use crate::Error;
+use byteorder::{LittleEndian, ReadBytesExt};
+use std::convert::TryFrom;
+use std::io::Read;
+
+pub fn parse<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != NAR_VERSION_MAGIC {
+        return Err(Error::BadNarVersionMagic);
+    }
+
+    parse_file(input)
+}
+
+const NAR_VERSION_MAGIC: &str = "nix-archive-1";
+
+fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != "(" {
+        return Err(Error::MissingNarOpenTag);
+    }
+
+    if String::read(input)? != "type" {
+        return Err(Error::MissingNarField);
+    }
+
+    match String::read(input)?.as_ref() {
+        "regular" => {
+            let mut executable = false;
+            let mut tag = String::read(input)?;
+            if tag == "executable" {
+                executable = true;
+                if String::read(input)? != "" {
+                    return Err(Error::BadExecutableField);
+                }
+                tag = String::read(input)?;
+            }
+            if tag != "contents" {
+                return Err(Error::MissingNarField);
+            }
+            let contents = Vec::<u8>::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        "directory" => loop {
+            match String::read(input)?.as_ref() {
+                "entry" => {
+                    if String::read(input)? != "(" {
+                        return Err(Error::MissingNarOpenTag);
+                    }
+                    if String::read(input)? != "name" {
+                        return Err(Error::MissingNarField);
+                    }
+                    let name = String::read(input)?;
+                    if String::read(input)? != "node" {
+                        return Err(Error::MissingNarField);
+                    }
+                    parse_file(input)?;
+                    let tag = String::read(input)?;
+                    if tag != ")" {
+                        return Err(Error::MissingNarCloseTag);
+                    }
+                }
+                ")" => break,
+                s => return Err(Error::BadNarField(s.into())),
+            }
+        },
+        "symlink" => {
+            if String::read(input)? != "target" {
+                return Err(Error::MissingNarField);
+            }
+            let target = String::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        s => return Err(Error::BadNarField(s.into())),
+    }
+
+    Ok(())
+}
+
+trait Deserialize: Sized {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error>;
+}
+
+impl Deserialize for String {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let buf = Deserialize::read(input)?;
+        Ok(String::from_utf8(buf).map_err(|_| Error::BadNarString)?)
+    }
+}
+
+impl Deserialize for Vec<u8> {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: usize = Deserialize::read(input)?;
+        let mut buf = vec![0; n];
+        input.read_exact(&mut buf)?;
+        skip_padding(input, n)?;
+        Ok(buf)
+    }
+}
+
+fn skip_padding<R: Read>(input: &mut R, len: usize) -> Result<(), Error> {
+    if len % 8 != 0 {
+        let mut buf = [0; 8];
+        let buf = &mut buf[0..8 - (len % 8)];
+        input.read_exact(buf)?;
+        if !buf.iter().all(|b| *b == 0) {
+            return Err(Error::BadNarPadding);
+        }
+    }
+    Ok(())
+}
+
+impl Deserialize for u64 {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        Ok(input.read_u64::<LittleEndian>()?)
+    }
+}
+
+impl Deserialize for usize {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: u64 = Deserialize::read(input)?;
+        Ok(usize::try_from(n).map_err(|_| Error::NarSizeFieldTooBig)?)
+    }
+}

From a6f0bef0a77ccc2c03d85a8e5841e830e52f5642 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Dec 2019 15:32:28 +0100
Subject: [PATCH 1743/2196] Update to async/await-enabled tokio

---
 nix-rust/Cargo.lock                      | 734 ++++++++---------------
 nix-rust/Cargo.toml                      |   6 +-
 nix-rust/local.mk                        |   2 +-
 nix-rust/src/c.rs                        |  17 +-
 nix-rust/src/lib.rs                      |   2 -
 nix-rust/src/store/binary_cache_store.rs |  20 +-
 shell.nix                                |   2 +-
 7 files changed, 262 insertions(+), 521 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index e4b61c44421..6af10d8336d 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,10 +1,5 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-[[package]]
-name = "adler32"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "anyhow"
 version = "1.0.25"
@@ -20,33 +15,10 @@ name = "autocfg"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "backtrace"
-version = "0.3.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "backtrace-sys"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "base64"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "bit-set"
@@ -82,7 +54,6 @@ version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -94,11 +65,6 @@ dependencies = [
  "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "cc"
-version = "1.0.47"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -113,37 +79,11 @@ dependencies = [
 ]
 
 [[package]]
-name = "cookie"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "cookie_store"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crc32fast"
-version = "1.2.0"
+name = "crossbeam-channel"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -195,24 +135,11 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "ct-logs"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "dtoa"
 version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "either"
-version = "1.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.20"
@@ -221,34 +148,6 @@ dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "error-chain"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "failure"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "failure_derive"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "filetime"
 version = "0.2.8"
@@ -260,17 +159,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "flate2"
-version = "1.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "fnv"
 version = "1.0.6"
@@ -302,73 +190,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "futures-channel-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "futures-core-preview"
-version = "0.3.0-alpha.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "futures-cpupool"
-version = "0.1.8"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "futures-executor-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "futures-io-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "futures-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-executor-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-executor-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "futures-sink-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "futures-util-preview"
-version = "0.3.0-alpha.18"
+version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -386,19 +262,22 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.1.26"
+version = "0.2.0-alpha.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -434,13 +313,11 @@ dependencies = [
 
 [[package]]
 name = "http-body"
-version = "0.1.0"
+version = "0.2.0-alpha.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -450,57 +327,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "hyper"
-version = "0.12.35"
+version = "0.13.0-alpha.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "h2 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.17.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "idna"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tower-make 0.3.0-alpha.2a (registry+https://github.com/rust-lang/crates.io-index)",
+ "tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -555,6 +406,9 @@ dependencies = [
 name = "lazy_static"
 version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "libc"
@@ -614,14 +468,6 @@ dependencies = [
  "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "miniz_oxide"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "mio"
 version = "0.6.21"
@@ -640,6 +486,16 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "mio-uds"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "miow"
 version = "0.2.1"
@@ -667,15 +523,15 @@ version = "0.1.0"
 dependencies = [
  "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "reqwest 0.10.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -730,13 +586,26 @@ dependencies = [
 
 [[package]]
 name = "percent-encoding"
-version = "1.0.1"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
-name = "percent-encoding"
-version = "2.1.0"
+name = "pin-project"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "pin-utils"
@@ -775,18 +644,6 @@ dependencies = [
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "publicsuffix"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "quick-error"
 version = "1.2.2"
@@ -946,14 +803,6 @@ name = "redox_syscall"
 version = "0.1.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "regex"
-version = "1.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "regex-syntax"
 version = "0.6.12"
@@ -969,58 +818,35 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.9.22"
+version = "0.10.0-alpha.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "cookie_store 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
- "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "ring"
-version = "0.16.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen-futures 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rustc-demangle"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -1029,18 +855,6 @@ dependencies = [
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rustls"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rusty-fork"
 version = "0.2.2"
@@ -1062,15 +876,6 @@ name = "scopeguard"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "sct"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -1088,19 +893,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "serde"
 version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.103"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "serde_json"
@@ -1114,13 +906,13 @@ dependencies = [
 
 [[package]]
 name = "serde_urlencoded"
-version = "0.5.5"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1169,17 +961,6 @@ dependencies = [
  "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "synstructure"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "tar"
 version = "0.4.26"
@@ -1216,152 +997,170 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "0.1.22"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-macros 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-buf"
-version = "0.1.1"
+name = "tokio-codec"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-current-thread"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "tokio-executor"
-version = "0.1.9"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
+ "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "tokio-io"
-version = "0.1.12"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-reactor"
-version = "0.1.11"
+name = "tokio-macros"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tokio-net"
+version = "0.2.0-alpha.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-rustls"
-version = "0.10.2"
+name = "tokio-sync"
+version = "0.2.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-sync"
-version = "0.1.7"
+name = "tokio-timer"
+version = "0.3.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-tcp"
-version = "0.1.3"
+name = "tower-make"
+version = "0.3.0-alpha.2a"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-threadpool"
-version = "0.1.16"
+name = "tower-service"
+version = "0.3.0-alpha.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "tracing"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "tokio-timer"
-version = "0.2.12"
+name = "tracing-attributes"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
-name = "try-lock"
-version = "0.2.2"
+name = "tracing-core"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
-name = "try_from"
-version = "0.3.2"
+name = "try-lock"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "unicase"
@@ -1397,21 +1196,6 @@ name = "unicode-xid"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "untrusted"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "url"
-version = "1.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "url"
 version = "2.1.0"
@@ -1422,14 +1206,6 @@ dependencies = [
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "uuid"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "version_check"
 version = "0.1.5"
@@ -1450,10 +1226,9 @@ dependencies = [
 
 [[package]]
 name = "want"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -1469,6 +1244,8 @@ version = "0.2.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
  "wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1486,6 +1263,17 @@ dependencies = [
  "wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
+ "web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.55"
@@ -1539,23 +1327,6 @@ dependencies = [
  "wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "webpki"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "weedle"
 version = "0.10.0"
@@ -1619,13 +1390,10 @@ dependencies = [
 ]
 
 [metadata]
-"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
 "checksum anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9267dff192e68f3399525901e709a48c1d3982c9c072fa32f2127a0cb0babf14"
 "checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
-"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
-"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
-"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
 "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
 "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
@@ -1633,50 +1401,38 @@ dependencies = [
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
-"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
-"checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
-"checksum cookie_store 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46750b3f362965f197996c4448e4a0935e791bf7d6631bfce9ee0af3d24c919c"
-"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
 "checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
 "checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
 "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
 "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
-"checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113"
 "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
-"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 "checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
-"checksum error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ab49e9dcb602294bc42f9a7dfc9bc6e936fca4418ea300dbfb84fe16de0b7d9"
-"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
-"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
 "checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
-"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 "checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef"
-"checksum futures-channel-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "f477fd0292c4a4ae77044454e7f2b413207942ad405f759bb0b4698b7ace5b12"
-"checksum futures-core-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2f26f774b81b3847dcda0c81bd4b6313acfb4f69e5a0390c7cb12c058953e9"
-"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
-"checksum futures-executor-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "80705612926df8a1bc05f0057e77460e29318801f988bf7d803a734cf54e7528"
-"checksum futures-io-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "ee7de0c1c9ed23f9457b0437fec7663ce64d9cc3c906597e714e529377b5ddd1"
-"checksum futures-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "efa8f90c4fb2328e381f8adfd4255b4a2b696f77d1c63a3dee6700b564c4e4b5"
-"checksum futures-sink-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b65a2481863d1b78e094a07e9c0eed458cc7dc6e72b22b7138b8a67d924859"
-"checksum futures-util-preview 0.3.0-alpha.18 (registry+https://github.com/rust-lang/crates.io-index)" = "7df53daff1e98cc024bf2720f3ceb0414d96fbb0a94f3cad3a5c3bf3be1d261c"
+"checksum futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e5f4df964fa9c1c2f8bddeb5c3611631cacd93baf810fc8bb2fb4b495c263a"
+"checksum futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "b35b6263fb1ef523c3056565fa67b1d16f0a8604ff12b11b08c25f28a734c60a"
+"checksum futures-executor-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "75236e88bd9fe88e5e8bfcd175b665d0528fe03ca4c5207fabc028c8f9d93e98"
+"checksum futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "f4914ae450db1921a56c91bde97a27846287d062087d4a652efc09bb3a01ebda"
+"checksum futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "3b1dce2a0267ada5c6ff75a8ba864b4e679a9e2aa44262af7a3b5516d530d76e"
+"checksum futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "86f148ef6b69f75bb610d4f9a2336d4fc88c4b5b67129d1a340dd0fd362efeec"
+"checksum futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "5ce968633c17e5f97936bd2797b6e38fb56cf16a7422319f7ec2e30d3c470e8d"
 "checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
-"checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
+"checksum h2 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f107db1419ef8271686187b1a5d47c6431af4a7f4d98b495e7b7fc249bb0a78"
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
 "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
 "checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
-"checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
+"checksum http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1f3aef6f3de2bd8585f5b366f3f550b5774500b4764d00cf00f903c95749eec3"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
-"checksum hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)" = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6"
-"checksum hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)" = "719d85c7df4a7f309a77d145340a063ea929dcb2e025bae46a80345cffec2952"
-"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
+"checksum hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d05aa523087ac0b9d8b93dd80d5d482a697308ed3b0dca7b0667511a7fa7cdc"
 "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
@@ -1693,8 +1449,8 @@ dependencies = [
 "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
 "checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
 "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
-"checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625"
 "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
+"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
 "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
@@ -1702,13 +1458,13 @@ dependencies = [
 "checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
 "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
-"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+"checksum pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "94b90146c7216e4cb534069fb91366de4ea0ea353105ee45ed297e2d1619e469"
+"checksum pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "44ca92f893f0656d3cba8158dd0f2b99b94de256a4a54e870bd6922fcc6c8355"
 "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
 "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
 "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27"
 "checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
-"checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b"
 "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
 "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
 "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
@@ -1727,24 +1483,18 @@ dependencies = [
 "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
 "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
-"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
 "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
-"checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650"
-"checksum ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6747f8da1f2b1fabbee1aaa4eb8a11abf9adef0bf58a41cee45db5d59cecdfac"
-"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
+"checksum reqwest 0.10.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e83b47defcad97ddbe592fd5fe49e16661f754b0ba5847cf41bcd870a2d338d7"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-"checksum rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e"
 "checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
 "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
 "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
-"checksum sct 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 "checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702"
-"checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0"
 "checksum serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7406489f3d6d6633ea32abb80dede59df6e3d1a6fdd6e867db84bbaff3c7d079"
-"checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a"
+"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
 "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
 "checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
@@ -1752,46 +1502,42 @@ dependencies = [
 "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d"
 "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
-"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
-"checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
-"checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
-"checksum tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443"
-"checksum tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ca6df436c42b0c3330a82d855d2ef017cd793090ad550a6bc2184f4b933532ab"
-"checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926"
-"checksum tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6732fe6b53c8d11178dcb77ac6d9682af27fc6d4cb87789449152e5377377146"
-"checksum tokio-rustls 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1df2fa53ac211c136832f530ccb081af9af891af22d685a9493e232c7a359bc2"
-"checksum tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d06554cce1ae4a50f42fba8023918afa931413aded705b560e29600ccf7c6d76"
-"checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119"
-"checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c"
-"checksum tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1739638e364e558128461fc1ad84d997702c8e31c2e6b18fb99842268199e827"
+"checksum tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1f17f5d6ab0f35c1506678b28fb1798bdf74fcb737e9843c7b17b73e426eba38"
+"checksum tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9f5d22fd1e84bd4045d28813491cb7d7caae34d45c80517c2213f09a85e8787a"
+"checksum tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9ee9ceecf69145923834ea73f32ba40c790fd877b74a7817dd0b089f1eb9c7c8"
+"checksum tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "112784d5543df30660b04a72ca423bfbd90e8bb32f94dcf610f15401218b22c5"
+"checksum tokio-macros 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "86b616374bcdadd95974e1f0dfca07dc913f1163c53840c0d664aca35114964e"
+"checksum tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a441682cd32f3559383112c4a7f372f5c9fa1950c5cf8c8dd05274a2ce8c2654"
+"checksum tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4f1aaeb685540f7407ea0e27f1c9757d258c7c6bf4e3eb19da6fc59b747239d2"
+"checksum tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b97c1587fe71018eb245a4a9daa13a5a3b681bbc1f7fdadfe24720e141472c13"
+"checksum tower-make 0.3.0-alpha.2a (registry+https://github.com/rust-lang/crates.io-index)" = "316d47dd40cde4ac5d88110eaf9a10a4e2a68612d9c056cd2aa24e37dcb484cd"
+"checksum tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)" = "63ff37396cd966ce43bea418bfa339f802857495f797dafa00bea5b7221ebdfa"
+"checksum tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ff4e4f59e752cb3beb5b61c6d5e11191c7946231ba84faec2902c9efdd8691c5"
+"checksum tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a4263b12c3d3c403274493eb805966093b53214124796552d674ca1dd5d27c2b"
+"checksum tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bc913647c520c959b6d21e35ed8fa6984971deca9f0a2fcb8c51207e0c56af1d"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
-"checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b"
 "checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
 "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
 "checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
 "checksum unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
 "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
-"checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece"
-"checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a"
 "checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61"
-"checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
 "checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
 "checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
-"checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
+"checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
 "checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
 "checksum wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "29ae32af33bacd663a9a28241abecf01f2be64e6a185c6139b04f18b6385c5f2"
 "checksum wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "1845584bd3593442dc0de6e6d9f84454a59a057722f36f005e44665d6ab19d85"
+"checksum wasm-bindgen-futures 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1458706aa1b8fe6898d19433c9f110d93a05d1f22ae6adf55810409a94df34b4"
 "checksum wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "87fcc747e6b73c93d22c947a6334644d22cfec5abd8b66238484dc2b0aeb9fe4"
 "checksum wasm-bindgen-macro-support 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3dc4b3f2c4078c8c4a5f363b92fcf62604c5913cbd16c6ff5aaf0f74ec03f570"
 "checksum wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "ca0b78d6d3be8589b95d1d49cdc0794728ca734adf36d7c9f07e6459508bb53d"
 "checksum wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3126356474ceb717c8fb5549ae387c9fbf4872818454f4d87708bee997214bb5"
 "checksum web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "98405c0a2e722ed3db341b4c5b70eb9fe0021621f7350bab76df93b09b649bbf"
-"checksum webpki 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7e664e770ac0110e2384769bcc59ed19e329d81f555916a6e072714957b81b4"
-"checksum webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a262ae37dd9d60f60dd473d1158f9fbebf110ba7b6a5051c8160460f6043718b"
 "checksum weedle 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 20d1fdaa81d..1d199699ca8 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -11,11 +11,11 @@ crate-type = ["cdylib"]
 [dependencies]
 tar = "0.4"
 libc = "0.2"
-futures-preview = { version = "=0.3.0-alpha.18", features = ["compat"] }
+futures-preview = { version = "=0.3.0-alpha.19", features = ["compat"] }
 #hyper = "0.12"
-reqwest = { version = "0.9", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "=0.10.0-alpha.2", default-features = false }
 http = "0.1"
-tokio = { version = "0.1", default-features = false }
+tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
 lazy_static = "1.4"
 byteorder = "1.3"
 
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index eaa56fe4862..51116b0b351 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -21,7 +21,7 @@ endif
 $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
 	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
 	$(libnixrust_BUILD_FLAGS) \
-	  RUSTC_BOOTSTRAP=1 cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
 	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
 
 $(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 542df93d2fe..e8aa9939246 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -14,10 +14,9 @@ pub extern "C" fn unpack_tarfile(
 
 #[no_mangle]
 pub extern "C" fn rust_test() {
-    /*
     use crate::store::{self, Store};
-    use futures::future::{FutureExt, TryFutureExt};
     use std::path::Path;
+    use tokio::runtime::Runtime;
 
     let fut = async move {
         let store: Box<dyn Store> = Box::new(store::BinaryCacheStore::new(
@@ -36,17 +35,21 @@ pub extern "C" fn rust_test() {
         eprintln!("INFO = {:?}", info);
          */
 
-        let closure = store.compute_path_closure(vec![path].into_iter().collect()).await.unwrap();
+        let closure = store
+            .compute_path_closure(vec![path].into_iter().collect())
+            .await
+            .unwrap();
 
         eprintln!("CLOSURE = {:?}", closure.len());
-
-        Ok(())
     };
 
-    tokio::run(fut.boxed().compat());
-     */
+    let rt = Runtime::new().unwrap();
 
+    rt.block_on(fut);
+
+    /*
     let file = std::fs::File::open("test.nar").unwrap();
 
     crate::nar::parse(&mut std::io::BufReader::new(file)).unwrap();
+    */
 }
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index cda63d2c717..4654d4cc343 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,5 +1,3 @@
-#![feature(await_macro, async_await)]
-
 #[macro_use]
 extern crate lazy_static;
 
diff --git a/nix-rust/src/store/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs
index f3c7b39bbaa..8241dc8f5c0 100644
--- a/nix-rust/src/store/binary_cache_store.rs
+++ b/nix-rust/src/store/binary_cache_store.rs
@@ -1,6 +1,5 @@
-use super::{Store, StorePath, PathInfo};
+use super::{PathInfo, Store, StorePath};
 use crate::Error;
-use futures::compat::Future01CompatExt;
 
 pub struct BinaryCacheStore {
     base_uri: String,
@@ -27,22 +26,17 @@ impl Store for BinaryCacheStore {
         let store_dir = self.store_dir().to_string();
 
         Box::pin(async move {
-            let response = client
-                .get(&uri)
-                .send()
-                .compat()
-                .await?;
+            let response = client.get(&uri).send().await?;
 
-            if response.status() == reqwest::StatusCode::NOT_FOUND || response.status() == reqwest::StatusCode::FORBIDDEN {
+            if response.status() == reqwest::StatusCode::NOT_FOUND
+                || response.status() == reqwest::StatusCode::FORBIDDEN
+            {
                 return Err(Error::InvalidPath(path));
             }
 
-            let mut response = response.error_for_status()?;
+            let response = response.error_for_status()?;
 
-            let body = response
-                .text()
-                .compat()
-                .await?;
+            let body = response.text().await?;
 
             PathInfo::parse_nar_info(&body, &store_dir)
         })
diff --git a/shell.nix b/shell.nix
index 21ed4712133..c709f6a5fab 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/bb1013511e1e5edcf314df8321acf2f3c536df0d.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From 14aa0c3259a7997d7273f3b3ce1a41af03293832 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Dec 2019 21:01:16 +0100
Subject: [PATCH 1744/2196] Use hyper directly instead of reqwest

---
 nix-rust/Cargo.lock                      | 360 +----------------------
 nix-rust/Cargo.toml                      |   5 +-
 nix-rust/src/c.rs                        |   2 +-
 nix-rust/src/error.rs                    |   6 +-
 nix-rust/src/store/binary_cache_store.rs |  21 +-
 5 files changed, 21 insertions(+), 373 deletions(-)

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index 6af10d8336d..a2bfb03f283 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -1,10 +1,5 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-[[package]]
-name = "anyhow"
-version = "1.0.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "assert_matches"
 version = "1.3.0"
@@ -15,11 +10,6 @@ name = "autocfg"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "base64"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "bit-set"
 version = "0.5.1"
@@ -38,11 +28,6 @@ name = "bitflags"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "bumpalo"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "byteorder"
 version = "1.3.2"
@@ -135,19 +120,6 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "dtoa"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "encoding_rs"
-version = "0.8.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "filetime"
 version = "0.2.8"
@@ -183,11 +155,6 @@ name = "fuchsia-zircon-sys"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "futures"
-version = "0.1.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "futures-channel-preview"
 version = "0.3.0-alpha.19"
@@ -240,7 +207,6 @@ name = "futures-util-preview"
 version = "0.3.0-alpha.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -280,14 +246,6 @@ dependencies = [
  "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "heck"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.3"
@@ -344,6 +302,7 @@ dependencies = [
  "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -354,16 +313,6 @@ dependencies = [
  "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "idna"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.3.0"
@@ -385,14 +334,6 @@ name = "itoa"
 version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "js-sys"
-version = "0.3.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"
@@ -431,11 +372,6 @@ dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "matches"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "maybe-uninit"
 version = "2.0.0"
@@ -454,20 +390,6 @@ dependencies = [
  "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "mime"
-version = "0.3.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "mime_guess"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "mio"
 version = "0.6.21"
@@ -526,23 +448,14 @@ dependencies = [
  "futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "reqwest 0.10.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "nom"
-version = "4.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.10"
@@ -584,11 +497,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "percent-encoding"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "pin-project"
 version = "0.4.6"
@@ -816,37 +724,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "reqwest"
-version = "0.10.0-alpha.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-futures 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -866,11 +743,6 @@ dependencies = [
  "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "ryu"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "scopeguard"
 version = "1.0.0"
@@ -889,32 +761,6 @@ name = "semver-parser"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "serde"
-version = "1.0.103"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "serde_json"
-version = "1.0.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "serde_urlencoded"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.2"
@@ -928,16 +774,6 @@ dependencies = [
  "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "smallvec"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "sourcefile"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -1162,60 +998,11 @@ name = "try-lock"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "unicase"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "unicode-segmentation"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "url"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "version_check"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "version_check"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "wait-timeout"
 version = "0.2.0"
@@ -1238,103 +1025,6 @@ name = "wasi"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasm-bindgen-futures"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
- "web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-macro-support 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "wasm-bindgen-webidl"
-version = "0.2.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
- "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
- "weedle 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "web-sys"
-version = "0.3.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
- "js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)",
- "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "weedle"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "winapi"
 version = "0.2.8"
@@ -1364,14 +1054,6 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "winreg"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "ws2_32-sys"
 version = "0.2.1"
@@ -1390,14 +1072,11 @@ dependencies = [
 ]
 
 [metadata]
-"checksum anyhow 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9267dff192e68f3399525901e709a48c1d3982c9c072fa32f2127a0cb0babf14"
 "checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
-"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
 "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
 "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
-"checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
@@ -1409,14 +1088,11 @@ dependencies = [
 "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
 "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
-"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
-"checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
 "checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
-"checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef"
 "checksum futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e5f4df964fa9c1c2f8bddeb5c3611631cacd93baf810fc8bb2fb4b495c263a"
 "checksum futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "b35b6263fb1ef523c3056565fa67b1d16f0a8604ff12b11b08c25f28a734c60a"
 "checksum futures-executor-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "75236e88bd9fe88e5e8bfcd175b665d0528fe03ca4c5207fabc028c8f9d93e98"
@@ -1426,39 +1102,31 @@ dependencies = [
 "checksum futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "5ce968633c17e5f97936bd2797b6e38fb56cf16a7422319f7ec2e30d3c470e8d"
 "checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
 "checksum h2 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f107db1419ef8271686187b1a5d47c6431af4a7f4d98b495e7b7fc249bb0a78"
-"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
 "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
 "checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
 "checksum http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1f3aef6f3de2bd8585f5b366f3f550b5774500b4764d00cf00f903c95749eec3"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
 "checksum hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d05aa523087ac0b9d8b93dd80d5d482a697308ed3b0dca7b0667511a7fa7cdc"
-"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
 "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
-"checksum js-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "1c840fdb2167497b0bd0db43d6dfe61e91637fa72f9d061f8bd17ddc44ba6414"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
 "checksum lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e57b3997725d2b60dbec1297f6c2e2957cc383db1cebd6be812163f969c7d586"
 "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
-"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
 "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
 "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
-"checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
-"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
 "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
 "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
-"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
 "checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
 "checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
 "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
-"checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 "checksum pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "94b90146c7216e4cb534069fb91366de4ea0ea353105ee45ed297e2d1619e469"
 "checksum pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "44ca92f893f0656d3cba8158dd0f2b99b94de256a4a54e870bd6922fcc6c8355"
 "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
@@ -1485,20 +1153,13 @@ dependencies = [
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
 "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
-"checksum reqwest 0.10.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e83b47defcad97ddbe592fd5fe49e16661f754b0ba5847cf41bcd870a2d338d7"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
-"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
 "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-"checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702"
-"checksum serde_json 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7406489f3d6d6633ea32abb80dede59df6e3d1a6fdd6e867db84bbaff3c7d079"
-"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
 "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
-"checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
-"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"
 "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d"
 "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
@@ -1519,31 +1180,14 @@ dependencies = [
 "checksum tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a4263b12c3d3c403274493eb805966093b53214124796552d674ca1dd5d27c2b"
 "checksum tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bc913647c520c959b6d21e35ed8fa6984971deca9f0a2fcb8c51207e0c56af1d"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
-"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
-"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
-"checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
-"checksum unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
 "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
-"checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61"
-"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
-"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
 "checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
 "checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
 "checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
-"checksum wasm-bindgen 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "29ae32af33bacd663a9a28241abecf01f2be64e6a185c6139b04f18b6385c5f2"
-"checksum wasm-bindgen-backend 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "1845584bd3593442dc0de6e6d9f84454a59a057722f36f005e44665d6ab19d85"
-"checksum wasm-bindgen-futures 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1458706aa1b8fe6898d19433c9f110d93a05d1f22ae6adf55810409a94df34b4"
-"checksum wasm-bindgen-macro 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "87fcc747e6b73c93d22c947a6334644d22cfec5abd8b66238484dc2b0aeb9fe4"
-"checksum wasm-bindgen-macro-support 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3dc4b3f2c4078c8c4a5f363b92fcf62604c5913cbd16c6ff5aaf0f74ec03f570"
-"checksum wasm-bindgen-shared 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "ca0b78d6d3be8589b95d1d49cdc0794728ca734adf36d7c9f07e6459508bb53d"
-"checksum wasm-bindgen-webidl 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "3126356474ceb717c8fb5549ae387c9fbf4872818454f4d87708bee997214bb5"
-"checksum web-sys 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "98405c0a2e722ed3db341b4c5b70eb9fe0021621f7350bab76df93b09b649bbf"
-"checksum weedle 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-"checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index 1d199699ca8..cd5c545ccc1 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -11,9 +11,8 @@ crate-type = ["cdylib"]
 [dependencies]
 tar = "0.4"
 libc = "0.2"
-futures-preview = { version = "=0.3.0-alpha.19", features = ["compat"] }
-#hyper = "0.12"
-reqwest = { version = "=0.10.0-alpha.2", default-features = false }
+futures-preview = { version = "=0.3.0-alpha.19" }
+hyper = "0.13.0-alpha.4"
 http = "0.1"
 tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
 lazy_static = "1.4"
diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index e8aa9939246..42ecb9866c9 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -20,7 +20,7 @@ pub extern "C" fn rust_test() {
 
     let fut = async move {
         let store: Box<dyn Store> = Box::new(store::BinaryCacheStore::new(
-            "https://cache.nixos.org".to_string(),
+            "http://cache.nixos.org".to_string(),
         ));
 
         let path = store
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 4defc646d19..1066a266eb0 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -18,7 +18,7 @@ pub enum Error {
     BadNarField(String),
     BadExecutableField,
     IOError(std::io::Error),
-    HttpError(reqwest::Error),
+    HttpError(hyper::error::Error),
     Misc(String),
     Foreign(CppException),
 }
@@ -29,8 +29,8 @@ impl From<std::io::Error> for Error {
     }
 }
 
-impl From<reqwest::Error> for Error {
-    fn from(err: reqwest::Error) -> Self {
+impl From<hyper::error::Error> for Error {
+    fn from(err: hyper::error::Error) -> Self {
         Error::HttpError(err)
     }
 }
diff --git a/nix-rust/src/store/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs
index 8241dc8f5c0..09be4a7fd47 100644
--- a/nix-rust/src/store/binary_cache_store.rs
+++ b/nix-rust/src/store/binary_cache_store.rs
@@ -1,16 +1,18 @@
 use super::{PathInfo, Store, StorePath};
 use crate::Error;
+use hyper::client::Client;
+use futures::stream::Concat;
 
 pub struct BinaryCacheStore {
     base_uri: String,
-    client: reqwest::r#async::Client,
+    client: Client<hyper::client::HttpConnector, hyper::Body>,
 }
 
 impl BinaryCacheStore {
     pub fn new(base_uri: String) -> Self {
         Self {
             base_uri,
-            client: reqwest::r#async::Client::new(),
+            client: Client::new(),
         }
     }
 }
@@ -26,19 +28,22 @@ impl Store for BinaryCacheStore {
         let store_dir = self.store_dir().to_string();
 
         Box::pin(async move {
-            let response = client.get(&uri).send().await?;
+            let response = client.get(uri.parse::<hyper::Uri>().unwrap()).await?;
 
-            if response.status() == reqwest::StatusCode::NOT_FOUND
-                || response.status() == reqwest::StatusCode::FORBIDDEN
+            if response.status() == hyper::StatusCode::NOT_FOUND
+                || response.status() == hyper::StatusCode::FORBIDDEN
             {
                 return Err(Error::InvalidPath(path));
             }
 
-            let response = response.error_for_status()?;
+            let mut body = response.into_body();
 
-            let body = response.text().await?;
+            let mut bytes = Vec::new();
+            while let Some(next) = body.next().await {
+                bytes.extend(next?);
+            }
 
-            PathInfo::parse_nar_info(&body, &store_dir)
+            PathInfo::parse_nar_info(std::str::from_utf8(&bytes).unwrap(), &store_dir)
         })
     }
 }

From 9e565781c664f20df24a295fcfcfd20fb432c94b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 14:24:46 +0100
Subject: [PATCH 1745/2196] Shut up warnings

---
 nix-rust/src/nar.rs                      | 10 +++++-----
 nix-rust/src/store/binary_cache_store.rs |  1 -
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/nix-rust/src/nar.rs b/nix-rust/src/nar.rs
index aa05d815b2e..cb520935e63 100644
--- a/nix-rust/src/nar.rs
+++ b/nix-rust/src/nar.rs
@@ -24,10 +24,10 @@ fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
 
     match String::read(input)?.as_ref() {
         "regular" => {
-            let mut executable = false;
+            let mut _executable = false;
             let mut tag = String::read(input)?;
             if tag == "executable" {
-                executable = true;
+                _executable = true;
                 if String::read(input)? != "" {
                     return Err(Error::BadExecutableField);
                 }
@@ -36,7 +36,7 @@ fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
             if tag != "contents" {
                 return Err(Error::MissingNarField);
             }
-            let contents = Vec::<u8>::read(input)?;
+            let _contents = Vec::<u8>::read(input)?;
             if String::read(input)? != ")" {
                 return Err(Error::MissingNarCloseTag);
             }
@@ -50,7 +50,7 @@ fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
                     if String::read(input)? != "name" {
                         return Err(Error::MissingNarField);
                     }
-                    let name = String::read(input)?;
+                    let _name = String::read(input)?;
                     if String::read(input)? != "node" {
                         return Err(Error::MissingNarField);
                     }
@@ -68,7 +68,7 @@ fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
             if String::read(input)? != "target" {
                 return Err(Error::MissingNarField);
             }
-            let target = String::read(input)?;
+            let _target = String::read(input)?;
             if String::read(input)? != ")" {
                 return Err(Error::MissingNarCloseTag);
             }
diff --git a/nix-rust/src/store/binary_cache_store.rs b/nix-rust/src/store/binary_cache_store.rs
index 09be4a7fd47..9e1e88b7c85 100644
--- a/nix-rust/src/store/binary_cache_store.rs
+++ b/nix-rust/src/store/binary_cache_store.rs
@@ -1,7 +1,6 @@
 use super::{PathInfo, Store, StorePath};
 use crate::Error;
 use hyper::client::Client;
-use futures::stream::Concat;
 
 pub struct BinaryCacheStore {
     base_uri: String,

From bca0afb943da714c93b4c8621c0ed727ae86d0e2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 6 Dec 2019 15:57:23 +0100
Subject: [PATCH 1746/2196] Shut up about deprecated functions

---
 local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/local.mk b/local.mk
index 01ca8a295f9..55f85a04406 100644
--- a/local.mk
+++ b/local.mk
@@ -6,7 +6,7 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

From ebd89999c28e25e71048d523f9c4f8b856dee9a9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 23:10:16 +0100
Subject: [PATCH 1747/2196] Add StorePath tests

---
 nix-rust/src/error.rs      |  2 +
 nix-rust/src/store/path.rs | 94 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 94 insertions(+), 2 deletions(-)

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 1066a266eb0..28b5c43afc2 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -6,6 +6,7 @@ pub enum Error {
     BadStorePath(std::path::PathBuf),
     BadNarInfo,
     BadBase32,
+    StorePathNameEmpty,
     StorePathNameTooLong,
     BadStorePathName,
     NarSizeFieldTooBig,
@@ -42,6 +43,7 @@ impl fmt::Display for Error {
             Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
             Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
             Error::BadBase32 => write!(f, "invalid base32 string"),
+            Error::StorePathNameEmpty => write!(f, "store path name is empty"),
             Error::StorePathNameTooLong => {
                 write!(f, "store path name is longer than 211 characters")
             }
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 4b866f9b473..03307ca7cfa 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -24,7 +24,7 @@ impl StorePath {
     }
 
     pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
-        if base_name.len() < STORE_PATH_HASH_CHARS + 2
+        if base_name.len() < STORE_PATH_HASH_CHARS + 1
             || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
         {
             return Err(Error::BadStorePath(base_name.into()));
@@ -68,6 +68,10 @@ pub struct StorePathName(String);
 
 impl StorePathName {
     pub fn new(s: &str) -> Result<Self, Error> {
+        if s.len() == 0 {
+            return Err(Error::StorePathNameEmpty);
+        }
+
         if s.len() > 211 {
             return Err(Error::StorePathNameTooLong);
         }
@@ -97,4 +101,90 @@ impl fmt::Display for StorePathName {
     }
 }
 
-// FIXME: add tests
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        let p = StorePath::new_from_base_name(&s).unwrap();
+        assert_eq!(p.name.0, "konsole-18.12.3");
+        assert_eq!(
+            p.hash.0,
+            [
+                0x9f, 0x76, 0x49, 0x20, 0xf6, 0x5d, 0xe9, 0x71, 0xc4, 0xca, 0x46, 0x21, 0xab, 0xff,
+                0x9b, 0x44, 0xef, 0x87, 0x0f, 0x3c
+            ]
+        );
+    }
+
+    #[test]
+    fn test_no_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameEmpty)
+        );
+    }
+
+    #[test]
+    fn test_no_dash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_short_hash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxl-konsole-18.12.3";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_invalid_hash() {
+        let s = "7h7qgvs4kgzsn8e6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_matches!(StorePath::new_from_base_name(&s), Err(Error::BadBase32));
+    }
+
+    #[test]
+    fn test_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(StorePath::new_from_base_name(&s), Ok(_));
+    }
+
+    #[test]
+    fn test_too_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameTooLong)
+        );
+    }
+
+    #[test]
+    fn test_bad_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-foo bar";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-kónsole";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+    }
+
+    #[test]
+    fn test_roundtrip() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_eq!(StorePath::new_from_base_name(&s).unwrap().to_string(), s);
+    }
+}

From bbe97dff8b3054d96e758f486f9ce3fa09e64de3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 5 Dec 2019 19:11:09 +0100
Subject: [PATCH 1748/2196] Make the Store API more type-safe

Most functions now take a StorePath argument rather than a Path (which
is just an alias for std::string). The StorePath constructor ensures
that the path is syntactically correct (i.e. it looks like
<store-dir>/<base32-hash>-<name>). Similarly, functions like
buildPaths() now take a StorePathWithOutputs, rather than abusing Path
by adding a '!<outputs>' suffix.

Note that the StorePath type is implemented in Rust. This involves
some hackery to allow Rust values to be used directly in C++, via a
helper type whose destructor calls the Rust type's drop()
function. The main issue is the dynamic nature of C++ move semantics:
after we have moved a Rust value, we should not call the drop function
on the original value. So when we move a value, we set the original
value to bitwise zero, and the destructor only calls drop() if the
value is not bitwise zero. This should be sufficient for most types.

Also lots of minor cleanups to the C++ API to make it more modern
(e.g. using std::optional and std::string_view in some places).
---
 nix-rust/Cargo.lock                           | 760 ------------------
 nix-rust/Cargo.toml                           |  10 +-
 nix-rust/local.mk                             |   4 +-
 nix-rust/src/c.rs                             |  88 +-
 nix-rust/src/error.rs                         |   3 +
 nix-rust/src/lib.rs                           |   1 +
 nix-rust/src/store/mod.rs                     |  12 +-
 nix-rust/src/store/path.rs                    |  32 +-
 perl/lib/Nix/Store.xs                         |  85 +-
 src/build-remote/build-remote.cc              |  21 +-
 src/libexpr/eval.cc                           |  42 +-
 src/libexpr/eval.hh                           |   5 +-
 src/libexpr/get-drvs.cc                       |  18 +-
 src/libexpr/local.mk                          |   2 +-
 src/libexpr/names.cc                          |   8 +-
 src/libexpr/names.hh                          |   2 +-
 src/libexpr/primops.cc                        | 133 ++-
 src/libexpr/primops/context.cc                |   2 +-
 src/libexpr/primops/fetchGit.cc               |   6 +-
 src/libexpr/primops/fetchMercurial.cc         |   6 +-
 src/libexpr/symbol-table.hh                   |   7 +-
 src/libmain/shared.cc                         |  18 +-
 src/libmain/shared.hh                         |  11 +-
 src/libstore/binary-cache-store.cc            |  72 +-
 src/libstore/binary-cache-store.hh            |  24 +-
 src/libstore/build.cc                         | 725 +++++++++--------
 src/libstore/builtins/fetchurl.cc             |   2 +-
 src/libstore/daemon.cc                        | 149 ++--
 src/libstore/derivations.cc                   | 182 +++--
 src/libstore/derivations.hh                   |  64 +-
 src/libstore/download.cc                      |  69 +-
 src/libstore/export-import.cc                 |  37 +-
 src/libstore/gc.cc                            | 184 +++--
 src/libstore/legacy-ssh-store.cc              |  80 +-
 src/libstore/local-binary-cache-store.cc      |   6 +-
 src/libstore/local-fs-store.cc                |  24 +-
 src/libstore/local-store.cc                   | 293 ++++---
 src/libstore/local-store.hh                   |  62 +-
 src/libstore/local.mk                         |   2 +-
 src/libstore/misc.cc                          | 144 ++--
 src/libstore/nar-info-disk-cache.cc           |  15 +-
 src/libstore/nar-info.cc                      |  31 +-
 src/libstore/nar-info.hh                      |   5 +-
 src/libstore/optimise-store.cc                |   6 +-
 src/libstore/parsed-derivations.cc            |  14 +-
 src/libstore/parsed-derivations.hh            |   4 +-
 src/libstore/path.cc                          |  99 +++
 src/libstore/path.hh                          |  81 ++
 src/libstore/profiles.cc                      |   4 +-
 src/libstore/references.cc                    |   2 +-
 src/libstore/remote-fs-accessor.cc            |   4 +-
 src/libstore/remote-store.cc                  | 226 +++---
 src/libstore/remote-store.hh                  |  42 +-
 src/libstore/s3-binary-cache-store.cc         |   8 +-
 src/libstore/ssh-store.cc                     |   6 +-
 src/libstore/store-api.cc                     | 310 ++++---
 src/libstore/store-api.hh                     | 203 +++--
 src/libstore/worker-protocol.hh               |   5 +-
 src/libutil/rust-ffi.cc                       |  12 +-
 src/libutil/rust-ffi.hh                       | 133 ++-
 src/libutil/tarfile.cc                        |   2 +-
 src/libutil/util.cc                           |  13 +-
 src/libutil/util.hh                           |   8 +-
 src/nix-build/nix-build.cc                    |  25 +-
 src/nix-channel/nix-channel.cc                |   7 +-
 src/nix-copy-closure/nix-copy-closure.cc      |   4 +-
 src/nix-daemon/nix-daemon.cc                  |   2 +-
 src/nix-env/nix-env.cc                        |  57 +-
 src/nix-env/user-env.cc                       |  26 +-
 src/nix-instantiate/nix-instantiate.cc        |   6 +-
 src/nix-prefetch-url/nix-prefetch-url.cc      |  18 +-
 src/nix-store/dotgraph.cc                     |  30 +-
 src/nix-store/dotgraph.hh                     |   6 +-
 src/nix-store/graphml.cc                      |  36 +-
 src/nix-store/graphml.hh                      |   6 +-
 src/nix-store/nix-store.cc                    | 310 +++----
 src/nix/add-to-store.cc                       |   9 +-
 src/nix/command.cc                            |  17 +-
 src/nix/command.hh                            |  15 +-
 src/nix/copy.cc                               |   4 +-
 src/nix/dump-path.cc                          |   2 +-
 src/nix/installables.cc                       |  68 +-
 src/nix/local.mk                              |   2 +-
 src/nix/log.cc                                |   2 +-
 src/nix/ls.cc                                 |   2 +-
 src/nix/main.cc                               |   2 +-
 src/nix/make-content-addressable.cc           |  40 +-
 src/nix/path-info.cc                          |  16 +-
 src/nix/progress-bar.cc                       |  11 +-
 src/nix/repl.cc                               |   6 +-
 src/nix/run.cc                                |  16 +-
 src/nix/show-derivation.cc                    |  18 +-
 src/nix/sigs.cc                               |  18 +-
 src/nix/upgrade-nix.cc                        |  27 +-
 src/nix/verify.cc                             |  20 +-
 src/nix/why-depends.cc                        |  30 +-
 src/resolve-system-dependencies/local.mk      |   2 +-
 .../resolve-system-dependencies.cc            |   4 +-
 98 files changed, 2625 insertions(+), 2867 deletions(-)
 create mode 100644 src/libstore/path.cc
 create mode 100644 src/libstore/path.hh

diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock
index a2bfb03f283..6e5ecf45ddf 100644
--- a/nix-rust/Cargo.lock
+++ b/nix-rust/Cargo.lock
@@ -33,15 +33,6 @@ name = "byteorder"
 version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "bytes"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "c2-chacha"
 version = "0.2.3"
@@ -63,63 +54,6 @@ dependencies = [
  "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "crossbeam-channel"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-queue"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "filetime"
 version = "0.2.8"
@@ -141,81 +75,6 @@ name = "fuchsia-cprng"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "fuchsia-zircon"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "fuchsia-zircon-sys"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "futures-channel-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "futures-core-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "futures-executor-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "futures-io-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "futures-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-executor-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "futures-sink-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "futures-util-preview"
-version = "0.3.0-alpha.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.1.13"
@@ -226,234 +85,31 @@ dependencies = [
  "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "h2"
-version = "0.2.0-alpha.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "hex"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "http"
-version = "0.1.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "http-body"
-version = "0.2.0-alpha.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "httparse"
-version = "1.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "hyper"
-version = "0.13.0-alpha.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "h2 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tower-make 0.3.0-alpha.2a (registry+https://github.com/rust-lang/crates.io-index)",
- "tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "iovec"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "itoa"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "kernel32-sys"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "libc"
 version = "0.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "lock_api"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "log"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "maybe-uninit"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "memchr"
-version = "2.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "memoffset"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "mio"
-version = "0.6.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "mio-uds"
-version = "0.6.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "miow"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "net2"
-version = "0.2.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "nix-rust"
 version = "0.1.0"
 dependencies = [
  "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -464,75 +120,11 @@ dependencies = [
  "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "num_cpus"
-version = "1.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "parking_lot"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "pin-project"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "pin-utils"
-version = "0.1.0-alpha.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "proc-macro2"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "proptest"
 version = "0.9.4"
@@ -557,14 +149,6 @@ name = "quick-error"
 version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "quote"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rand"
 version = "0.6.5"
@@ -724,14 +308,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "rusty-fork"
 version = "0.2.2"
@@ -743,60 +319,6 @@ dependencies = [
  "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "scopeguard"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "slab"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "smallvec"
-version = "0.6.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "spin"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "string"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "tar"
 version = "0.4.26"
@@ -821,188 +343,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "time"
-version = "0.1.42"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-macros 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-codec"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-executor"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-io"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-net"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
- "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-sync"
-version = "0.2.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tokio-timer"
-version = "0.3.0-alpha.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tower-make"
-version = "0.3.0-alpha.2a"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tower-service"
-version = "0.3.0-alpha.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "tracing"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tracing-attributes"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "try-lock"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "wait-timeout"
 version = "0.2.0"
@@ -1011,25 +351,11 @@ dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "want"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "wasi"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "winapi"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "winapi"
 version = "0.3.8"
@@ -1039,11 +365,6 @@ dependencies = [
  "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "winapi-build"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
@@ -1054,15 +375,6 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "ws2_32-sys"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "xattr"
 version = "0.2.2"
@@ -1078,63 +390,20 @@ dependencies = [
 "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
-"checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
-"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
-"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
-"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
-"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
-"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
-"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
 "checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
-"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
-"checksum futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e5f4df964fa9c1c2f8bddeb5c3611631cacd93baf810fc8bb2fb4b495c263a"
-"checksum futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "b35b6263fb1ef523c3056565fa67b1d16f0a8604ff12b11b08c25f28a734c60a"
-"checksum futures-executor-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "75236e88bd9fe88e5e8bfcd175b665d0528fe03ca4c5207fabc028c8f9d93e98"
-"checksum futures-io-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "f4914ae450db1921a56c91bde97a27846287d062087d4a652efc09bb3a01ebda"
-"checksum futures-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "3b1dce2a0267ada5c6ff75a8ba864b4e679a9e2aa44262af7a3b5516d530d76e"
-"checksum futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "86f148ef6b69f75bb610d4f9a2336d4fc88c4b5b67129d1a340dd0fd362efeec"
-"checksum futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "5ce968633c17e5f97936bd2797b6e38fb56cf16a7422319f7ec2e30d3c470e8d"
 "checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
-"checksum h2 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f107db1419ef8271686187b1a5d47c6431af4a7f4d98b495e7b7fc249bb0a78"
-"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
 "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
-"checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
-"checksum http-body 0.2.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1f3aef6f3de2bd8585f5b366f3f550b5774500b4764d00cf00f903c95749eec3"
-"checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
-"checksum hyper 0.13.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d05aa523087ac0b9d8b93dd80d5d482a697308ed3b0dca7b0667511a7fa7cdc"
-"checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
-"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
-"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
-"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
-"checksum lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e57b3997725d2b60dbec1297f6c2e2957cc383db1cebd6be812163f969c7d586"
-"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
-"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
-"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
-"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
-"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
-"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
-"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
-"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
 "checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
-"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
-"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
-"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
-"checksum pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "94b90146c7216e4cb534069fb91366de4ea0ea353105ee45ed297e2d1619e469"
-"checksum pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "44ca92f893f0656d3cba8158dd0f2b99b94de256a4a54e870bd6922fcc6c8355"
-"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
 "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
-"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27"
 "checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
 "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
-"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
 "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
 "checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
 "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
@@ -1153,41 +422,12 @@ dependencies = [
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
 "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
-"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
-"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d"
-"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
-"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
-"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
-"checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d"
-"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
 "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
-"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
-"checksum tokio 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1f17f5d6ab0f35c1506678b28fb1798bdf74fcb737e9843c7b17b73e426eba38"
-"checksum tokio-codec 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9f5d22fd1e84bd4045d28813491cb7d7caae34d45c80517c2213f09a85e8787a"
-"checksum tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9ee9ceecf69145923834ea73f32ba40c790fd877b74a7817dd0b089f1eb9c7c8"
-"checksum tokio-io 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "112784d5543df30660b04a72ca423bfbd90e8bb32f94dcf610f15401218b22c5"
-"checksum tokio-macros 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "86b616374bcdadd95974e1f0dfca07dc913f1163c53840c0d664aca35114964e"
-"checksum tokio-net 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a441682cd32f3559383112c4a7f372f5c9fa1950c5cf8c8dd05274a2ce8c2654"
-"checksum tokio-sync 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4f1aaeb685540f7407ea0e27f1c9757d258c7c6bf4e3eb19da6fc59b747239d2"
-"checksum tokio-timer 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b97c1587fe71018eb245a4a9daa13a5a3b681bbc1f7fdadfe24720e141472c13"
-"checksum tower-make 0.3.0-alpha.2a (registry+https://github.com/rust-lang/crates.io-index)" = "316d47dd40cde4ac5d88110eaf9a10a4e2a68612d9c056cd2aa24e37dcb484cd"
-"checksum tower-service 0.3.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)" = "63ff37396cd966ce43bea418bfa339f802857495f797dafa00bea5b7221ebdfa"
-"checksum tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ff4e4f59e752cb3beb5b61c6d5e11191c7946231ba84faec2902c9efdd8691c5"
-"checksum tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a4263b12c3d3c403274493eb805966093b53214124796552d674ca1dd5d27c2b"
-"checksum tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bc913647c520c959b6d21e35ed8fa6984971deca9f0a2fcb8c51207e0c56af1d"
-"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
-"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
 "checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
-"checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
 "checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
-"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
-"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml
index cd5c545ccc1..cd388bed6d4 100644
--- a/nix-rust/Cargo.toml
+++ b/nix-rust/Cargo.toml
@@ -11,12 +11,12 @@ crate-type = ["cdylib"]
 [dependencies]
 tar = "0.4"
 libc = "0.2"
-futures-preview = { version = "=0.3.0-alpha.19" }
-hyper = "0.13.0-alpha.4"
-http = "0.1"
-tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
+#futures-preview = { version = "=0.3.0-alpha.19" }
+#hyper = "0.13.0-alpha.4"
+#http = "0.1"
+#tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
 lazy_static = "1.4"
-byteorder = "1.3"
+#byteorder = "1.3"
 
 [dev-dependencies]
 hex = "0.3"
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index 51116b0b351..1e006e500c8 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -37,7 +37,9 @@ clean: clean-rust
 clean-rust:
 	$(suppress) rm -rfv nix-rust/target
 
+ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) RUSTC_BOOTSTRAP=1 cargo test --release $$(if [[ -d vendor ]]; then echo -Z offline; fi)
+	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+endif
diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 42ecb9866c9..4c1724eb3e3 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -1,6 +1,7 @@
 use super::{
     error,
     foreign::{self, CBox},
+    store::StorePath,
     util,
 };
 
@@ -13,43 +14,72 @@ pub extern "C" fn unpack_tarfile(
 }
 
 #[no_mangle]
-pub extern "C" fn rust_test() {
-    use crate::store::{self, Store};
-    use std::path::Path;
-    use tokio::runtime::Runtime;
+pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) {
+    // FIXME: check whether 's' is valid UTF-8?
+    out.write(s.to_string())
+}
 
-    let fut = async move {
-        let store: Box<dyn Store> = Box::new(store::BinaryCacheStore::new(
-            "http://cache.nixos.org".to_string(),
-        ));
+#[no_mangle]
+pub unsafe extern "C" fn ffi_String_drop(self_: *mut String) {
+    std::ptr::drop_in_place(self_);
+}
 
-        let path = store
-            .parse_store_path(&Path::new(
-                "/nix/store/7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3",
-            ))
-            .unwrap();
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new(
+    path: &str,
+    store_dir: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new(std::path::Path::new(path), store_dir).map_err(|err| err.into())
+}
 
-        /*
-        let info = store.query_path_info(&path).await.unwrap();
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new2(
+    hash: &[u8; crate::store::path::STORE_PATH_HASH_BYTES],
+    name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::from_parts(*hash, name).map_err(|err| err.into())
+}
 
-        eprintln!("INFO = {:?}", info);
-         */
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_fromBaseName(
+    base_name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new_from_base_name(base_name).map_err(|err| err.into())
+}
 
-        let closure = store
-            .compute_path_closure(vec![path].into_iter().collect())
-            .await
-            .unwrap();
+#[no_mangle]
+pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) {
+    std::ptr::drop_in_place(self_);
+}
 
-        eprintln!("CLOSURE = {:?}", closure.len());
-    };
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> String {
+    format!("{}", self_)
+}
 
-    let rt = Runtime::new().unwrap();
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_less_than(a: &StorePath, b: &StorePath) -> bool {
+    a < b
+}
 
-    rt.block_on(fut);
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_eq(a: &StorePath, b: &StorePath) -> bool {
+    a == b
+}
 
-    /*
-    let file = std::fs::File::open("test.nar").unwrap();
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_clone(self_: &StorePath) -> StorePath {
+    self_.clone()
+}
 
-    crate::nar::parse(&mut std::io::BufReader::new(file)).unwrap();
-    */
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_name(self_: &StorePath) -> &str {
+    self_.name.name()
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_hash_data(
+    self_: &StorePath,
+) -> &[u8; crate::store::path::STORE_PATH_HASH_BYTES] {
+    self_.hash.hash()
 }
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 28b5c43afc2..c2ba5d5048c 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -19,6 +19,7 @@ pub enum Error {
     BadNarField(String),
     BadExecutableField,
     IOError(std::io::Error),
+    #[cfg(unused)]
     HttpError(hyper::error::Error),
     Misc(String),
     Foreign(CppException),
@@ -30,6 +31,7 @@ impl From<std::io::Error> for Error {
     }
 }
 
+#[cfg(unused)]
 impl From<hyper::error::Error> for Error {
     fn from(err: hyper::error::Error) -> Self {
         Error::HttpError(err)
@@ -58,6 +60,7 @@ impl fmt::Display for Error {
             Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s),
             Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"),
             Error::IOError(err) => write!(f, "I/O error: {}", err),
+            #[cfg(unused)]
             Error::HttpError(err) => write!(f, "HTTP error: {}", err),
             Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
             Error::Misc(s) => write!(f, "{}", s),
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 4654d4cc343..0c3cf5276fd 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -12,6 +12,7 @@ extern crate proptest;
 mod c;
 mod error;
 mod foreign;
+#[cfg(unused)]
 mod nar;
 mod store;
 mod util;
diff --git a/nix-rust/src/store/mod.rs b/nix-rust/src/store/mod.rs
index 85355b594c7..da972482c61 100644
--- a/nix-rust/src/store/mod.rs
+++ b/nix-rust/src/store/mod.rs
@@ -1,9 +1,17 @@
+pub mod path;
+
+#[cfg(unused)]
 mod binary_cache_store;
-mod path;
+#[cfg(unused)]
 mod path_info;
+#[cfg(unused)]
 mod store;
 
-pub use binary_cache_store::BinaryCacheStore;
 pub use path::{StorePath, StorePathHash, StorePathName};
+
+#[cfg(unused)]
+pub use binary_cache_store::BinaryCacheStore;
+#[cfg(unused)]
 pub use path_info::PathInfo;
+#[cfg(unused)]
 pub use store::Store;
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 03307ca7cfa..3ded5c428e9 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -23,6 +23,13 @@ impl StorePath {
         )
     }
 
+    pub fn from_parts(hash: [u8; STORE_PATH_HASH_BYTES], name: &str) -> Result<Self, Error> {
+        Ok(StorePath {
+            hash: StorePathHash(hash),
+            name: StorePathName::new(name)?,
+        })
+    }
+
     pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
         if base_name.len() < STORE_PATH_HASH_CHARS + 1
             || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
@@ -43,7 +50,7 @@ impl fmt::Display for StorePath {
     }
 }
 
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+#[derive(Clone, PartialEq, Eq, Debug)]
 pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]);
 
 impl StorePathHash {
@@ -55,6 +62,10 @@ impl StorePathHash {
         bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]);
         Ok(Self(bytes))
     }
+
+    pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] {
+        &self.0
+    }
 }
 
 impl fmt::Display for StorePathHash {
@@ -63,6 +74,21 @@ impl fmt::Display for StorePathHash {
     }
 }
 
+impl Ord for StorePathHash {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        // Historically we've sorted store paths by their base32
+        // serialization, but our base32 encodes bytes in reverse
+        // order. So compare them in reverse order as well.
+        self.0.iter().rev().cmp(other.0.iter().rev())
+    }
+}
+
+impl PartialOrd for StorePathHash {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
 pub struct StorePathName(String);
 
@@ -93,6 +119,10 @@ impl StorePathName {
 
         Ok(Self(s.to_string()))
     }
+
+    pub fn name<'a>(&'a self) -> &'a str {
+        &self.0
+    }
 }
 
 impl fmt::Display for StorePathName {
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index ce553bb53eb..fefcd5ae6e8 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -59,7 +59,7 @@ void setVerbosity(int level)
 int isValidPath(char * path)
     CODE:
         try {
-            RETVAL = store()->isValidPath(path);
+            RETVAL = store()->isValidPath(store()->parseStorePath(path));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -70,9 +70,8 @@ int isValidPath(char * path)
 SV * queryReferences(char * path)
     PPCODE:
         try {
-            PathSet paths = store()->queryPathInfo(path)->references;
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -81,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(path)->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -91,9 +90,9 @@ SV * queryPathHash(char * path)
 SV * queryDeriver(char * path)
     PPCODE:
         try {
-            auto deriver = store()->queryPathInfo(path)->deriver;
-            if (deriver == "") XSRETURN_UNDEF;
-            XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0)));
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (!info->deriver) XSRETURN_UNDEF;
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -102,18 +101,18 @@ SV * queryDeriver(char * path)
 SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
-            auto info = store()->queryPathInfo(path);
-            if (info->deriver == "")
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
-                XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0)));
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
             auto s = info->narHash.to_string(base32 ? Base32 : Base16);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * arr = newAV();
-            for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i)
-                av_push(arr, newSVpv(i->c_str(), 0));
+            for (auto & i : info->references)
+                av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) arr)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -123,8 +122,8 @@ SV * queryPathInfo(char * path, int base32)
 SV * queryPathFromHashPart(char * hashPart)
     PPCODE:
         try {
-            Path path = store()->queryPathFromHashPart(hashPart);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->queryPathFromHashPart(hashPart);
+            XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -133,11 +132,11 @@ SV * queryPathFromHashPart(char * hashPart)
 SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
     PPCODE:
         try {
-            PathSet paths;
+            StorePathSet paths;
             for (int n = 2; n < items; ++n)
-                store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs);
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+                store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs);
+            for (auto & i : paths)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -146,11 +145,11 @@ SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
 SV * topoSortPaths(...)
     PPCODE:
         try {
-            PathSet paths;
-            for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n)));
-            Paths sorted = store()->topoSortPaths(paths);
-            for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            StorePathSet paths;
+            for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
+            auto sorted = store()->topoSortPaths(paths);
+            for (auto & i : sorted)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -159,7 +158,7 @@ SV * topoSortPaths(...)
 SV * followLinksToStorePath(char * path)
     CODE:
         try {
-            RETVAL = newSVpv(store()->followLinksToStorePath(path).c_str(), 0);
+            RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -170,8 +169,8 @@ SV * followLinksToStorePath(char * path)
 void exportPaths(int fd, ...)
     PPCODE:
         try {
-            Paths paths;
-            for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
+            StorePathSet paths;
+            for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
             FdSink sink(fd);
             store()->exportPaths(paths, sink);
         } catch (Error & e) {
@@ -275,8 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo));
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -286,8 +285,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            Path path = store()->makeFixedOutputPath(recursive, h, name);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -298,35 +297,35 @@ SV * derivationFromPath(char * drvPath)
         HV *hash;
     CODE:
         try {
-            Derivation drv = store()->derivationFromPath(drvPath);
+            Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath));
             hash = newHV();
 
             HV * outputs = newHV();
-            for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
-                hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0);
+            for (auto & i : drv.outputs)
+                hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();
-            for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
-                av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second
+            for (auto & i : drv.inputDrvs)
+                av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
 
             AV * inputSrcs = newAV();
-            for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
-                av_push(inputSrcs, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.inputSrcs)
+                av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0));
             hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs));
 
             hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0));
             hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0));
 
             AV * args = newAV();
-            for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i)
-                av_push(args, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.args)
+                av_push(args, newSVpv(i.c_str(), 0));
             hv_stores(hash, "args", newRV((SV *) args));
 
             HV * env = newHV();
-            for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i)
-                hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0);
+            for (auto & i : drv.env)
+                hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0);
             hv_stores(hash, "env", newRV((SV *) env));
 
             RETVAL = newRV_noinc((SV *)hash);
@@ -340,7 +339,7 @@ SV * derivationFromPath(char * drvPath)
 void addTempRoot(char * storePath)
     PPCODE:
         try {
-            store()->addTempRoot(storePath);
+            store()->addTempRoot(store()->parseStorePath(storePath));
         } catch (Error & e) {
             croak("%s", e.what());
         }
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 27dd3578458..69d1c6f7e46 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -88,7 +88,7 @@ static int _main(int argc, char * * argv)
             return 0;
         }
 
-        string drvPath;
+        std::optional<StorePath> drvPath;
         string storeUri;
 
         while (true) {
@@ -100,7 +100,7 @@ static int _main(int argc, char * * argv)
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
-            source >> drvPath;
+            drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
              auto canBuildLocally = amWilling
@@ -236,26 +236,27 @@ static int _main(int argc, char * * argv)
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
+            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
-        drv.inputSrcs = inputs;
+        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        drv.inputSrcs = store->parseStorePathSet(inputs);
 
-        auto result = sshStore->buildDerivation(drvPath, drv);
+        auto result = sshStore->buildDerivation(*drvPath, drv);
 
         if (!result.success())
-            throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
+            throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        PathSet missing;
+        StorePathSet missing;
         for (auto & path : outputs)
-            if (!store->isValidPath(path)) missing.insert(path);
+            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
+            for (auto & i : missing)
+                store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 4be4cfeea45..6539a346aed 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -43,6 +43,19 @@ static char * dupString(const char * s)
 }
 
 
+static char * dupStringWithLen(const char * s, size_t size)
+{
+    char * t;
+#if HAVE_BOEHMGC
+    t = GC_STRNDUP(s, size);
+#else
+    t = strndup(s, size);
+#endif
+    if (!t) throw std::bad_alloc();
+    return t;
+}
+
+
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
@@ -330,10 +343,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
             auto path = r.second;
 
             if (store->isInStore(r.second)) {
-                PathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second), closure);
+                StorePathSet closure;
+                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
                 for (auto & path : closure)
-                    allowedPaths->insert(path);
+                    allowedPaths->insert(store->printStorePath(path));
             } else
                 allowedPaths->insert(r.second);
         }
@@ -550,9 +563,11 @@ void mkString(Value & v, const char * s)
 }
 
 
-Value & mkString(Value & v, const string & s, const PathSet & context)
+Value & mkString(Value & v, std::string_view s, const PathSet & context)
 {
-    mkString(v, s.c_str());
+    v.type = tString;
+    v.string.s = dupStringWithLen(s.data(), s.size());
+    v.string.context = 0;
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
@@ -1639,15 +1654,16 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         throwEvalError("file names are not allowed to end in '%1%'", drvExtension);
 
     Path dstPath;
-    if (srcToStore[path] != "")
-        dstPath = srcToStore[path];
+    auto i = srcToStore.find(path);
+    if (i != srcToStore.end())
+        dstPath = store->printStorePath(i->second);
     else {
-        dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
-            : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
-        srcToStore[path] = dstPath;
-        printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")
-            % path % dstPath);
+        auto p = settings.readOnlyMode
+            ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+        dstPath = store->printStorePath(p);
+        srcToStore.insert_or_assign(path, std::move(p));
+        printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
     }
 
     context.insert(dstPath);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 85aa1ccfec2..cabc92d15c4 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -17,6 +17,7 @@ namespace nix {
 
 class Store;
 class EvalState;
+struct StorePath;
 enum RepairFlag : bool;
 
 
@@ -42,14 +43,14 @@ struct Env
 };
 
 
-Value & mkString(Value & v, const string & s, const PathSet & context = PathSet());
+Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
 
 void copyContext(const Value & v, PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
    paths. */
-typedef std::map<Path, Path> SrcToStore;
+typedef std::map<Path, StorePath> SrcToStore;
 
 
 std::ostream & operator << (std::ostream & str, const Value & v);
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 4e22a1d776f..a3412eab946 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -19,27 +19,27 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = store->parseDrvPathWithOutputs(drvPathWithOutputs);
 
-    drvPath = spec.first;
+    this->drvPath = store->printStorePath(drvPath);
 
     auto drv = store->derivationFromPath(drvPath);
 
-    name = storePathToName(drvPath);
+    name = drvPath.name();
 
-    if (spec.second.size() > 1)
+    if (selectedOutputs.size() > 1)
         throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
 
     outputName =
-        spec.second.empty()
-        ? get(drv.env, "outputName", "out")
-        : *spec.second.begin();
+        selectedOutputs.empty()
+        ? get(drv.env, "outputName").value_or("out")
+        : *selectedOutputs.begin();
 
     auto i = drv.outputs.find(outputName);
     if (i == drv.outputs.end())
-        throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
+        throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
 
-    outPath = i->second.path;
+    outPath = store->printStorePath(i->second.path);
 }
 
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ccd5293e4e5..26b9f14baa8 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,7 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore
+libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc
index 382088c7887..d1c8a6101f8 100644
--- a/src/libexpr/names.cc
+++ b/src/libexpr/names.cc
@@ -16,14 +16,14 @@ DrvName::DrvName()
    a letter.  The `version' part is the rest (excluding the separating
    dash).  E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
    '2.0.48'). */
-DrvName::DrvName(const string & s) : hits(0)
+DrvName::DrvName(std::string_view s) : hits(0)
 {
-    name = fullName = s;
+    name = fullName = std::string(s);
     for (unsigned int i = 0; i < s.size(); ++i) {
         /* !!! isalpha/isdigit are affected by the locale. */
         if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) {
-            name = string(s, 0, i);
-            version = string(s, i + 1);
+            name = s.substr(0, i);
+            version = s.substr(i + 1);
             break;
         }
     }
diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh
index 13c3093e77b..00e14b8c797 100644
--- a/src/libexpr/names.hh
+++ b/src/libexpr/names.hh
@@ -15,7 +15,7 @@ struct DrvName
     unsigned int hits;
 
     DrvName();
-    DrvName(const string & s);
+    DrvName(std::string_view s);
     bool matches(DrvName & n);
 
 private:
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d693a3b206f..3ef827ebcc6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -44,29 +44,28 @@ std::pair<string, string> decodeContext(const string & s)
 
 
 InvalidPathError::InvalidPathError(const Path & path) :
-    EvalError(format("path '%1%' is not valid") % path), path(path) {}
+    EvalError("path '%s' is not valid", path), path(path) {}
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    PathSet drvs;
+    std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
         std::pair<string, string> decoded = decodeContext(i);
-        Path ctx = decoded.first;
-        assert(store->isStorePath(ctx));
+        auto ctx = store->parseStorePath(decoded.first);
         if (!store->isValidPath(ctx))
-            throw InvalidPathError(ctx);
-        if (!decoded.second.empty() && nix::isDerivation(ctx)) {
-            drvs.insert(decoded.first + "!" + decoded.second);
+            throw InvalidPathError(store->printStorePath(ctx));
+        if (!decoded.second.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(decoded.first);
+                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
                 DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
                 if (i == drv.outputs.end())
                     throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
-                allowedPaths->insert(i->second.path);
+                allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
     }
@@ -74,10 +73,11 @@ void EvalState::realiseContext(const PathSet & context)
     if (drvs.empty()) return;
 
     if (!evalSettings.enableImportFromDerivation)
-        throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+        throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
+            store->printStorePath(drvs.begin()->path));
 
     /* For performance, prefetch all substitute info. */
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
     store->buildPaths(drvs);
@@ -100,8 +100,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
 
-    if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) {
-        Derivation drv = readDerivation(realPath);
+    // FIXME
+    if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
+        Derivation drv = readDerivation(*state.store, realPath);
         Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
         Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -115,7 +116,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
         for (const auto & o : drv.outputs) {
             v2 = state.allocAttr(w, state.symbols.create(o.first));
-            mkString(*v2, o.second.path, {"!" + o.first + "!" + path});
+            mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
@@ -676,24 +677,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            runs. */
         if (path.at(0) == '=') {
             /* !!! This doesn't work if readOnlyMode is set. */
-            PathSet refs;
-            state.store->computeFSClosure(string(path, 1), refs);
+            StorePathSet refs;
+            state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
             for (auto & j : refs) {
-                drv.inputSrcs.insert(j);
-                if (isDerivation(j))
-                    drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j);
+                drv.inputSrcs.insert(j.clone());
+                if (j.isDerivation())
+                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
             }
         }
 
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
-            drv.inputDrvs[ctx.first].insert(ctx.second);
+            drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
         }
 
         /* Otherwise it's a source file. */
         else
-            drv.inputSrcs.insert(path);
+            drv.inputSrcs.insert(state.store->parseStorePath(path));
     }
 
     /* Do we have all required attributes? */
@@ -703,10 +704,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
 
     /* Check whether the derivation name is valid. */
-    checkStoreName(drvName);
     if (isDerivation(drvName))
-        throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%")
-            % drvExtension % posDrvName);
+        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
@@ -716,52 +715,51 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
 
-        Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
-        if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath,
-            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-            h.to_string(Base16, false));
+        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
+        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
+                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+                h.to_string(Base16, false)));
     }
 
     else {
-        /* Construct the "masked" store derivation, which is the final
-           one except that in the list of outputs, the output paths
-           are empty, and the corresponding environment variables have
-           an empty value.  This ensures that changes in the set of
-           output names do get reflected in the hash. */
+        /* Compute a hash over the "masked" store derivation, which is
+           the final one except that in the list of outputs, the
+           output paths are empty strings, and the corresponding
+           environment variables have an empty value.  This ensures
+           that changes in the set of output names do get reflected in
+           the hash. */
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
-            drv.outputs[i] = DerivationOutput("", "", "");
         }
 
-        /* Use the masked derivation expression to compute the output
-           path. */
-        Hash h = hashDerivationModulo(*state.store, drv);
+        Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
 
-        for (auto & i : drv.outputs)
-            if (i.second.path == "") {
-                Path outPath = state.store->makeOutputPath(i.first, h, drvName);
-                if (!jsonObject) drv.env[i.first] = outPath;
-                i.second.path = outPath;
-            }
+        for (auto & i : outputs) {
+            auto outPath = state.store->makeOutputPath(i, h, drvName);
+            if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(std::move(outPath), "", ""));
+        }
     }
 
     /* Write the resulting term into the Nix store directory. */
-    Path drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPathS = state.store->printStorePath(drvPath);
 
-    printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'")
-        % drvName % drvPath);
+    printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
        read them later. */
-    drvHashes[drvPath] = hashDerivationModulo(*state.store, drv);
+    drvHashes.insert_or_assign(drvPath.clone(),
+        hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
-    mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath});
+    mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
     for (auto & i : drv.outputs) {
         mkString(*state.allocAttr(v, state.symbols.create(i.first)),
-            i.second.path, {"!" + i.first + "!" + drvPath});
+            state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
     }
     v.attrs->sort();
 }
@@ -814,7 +812,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
         throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
-        state.store->ensurePath(path2);
+        state.store->ensurePath(state.store->parseStorePath(path2));
     context.insert(path2);
     mkString(v, path, context);
 }
@@ -1010,17 +1008,17 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     string name = state.forceStringNoCtx(*args[0], pos);
     string contents = state.forceString(*args[1], context, pos);
 
-    PathSet refs;
+    StorePathSet refs;
 
     for (auto path : context) {
         if (path.at(0) != '/')
             throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-        refs.insert(path);
+        refs.insert(state.store->parseStorePath(path));
     }
 
-    Path storePath = settings.readOnlyMode
+    auto storePath = state.store->printStorePath(settings.readOnlyMode
         ? state.store->computeStorePathForText(name, contents, refs)
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+        : state.store->addTextToStore(name, contents, refs, state.repair));
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
@@ -1060,21 +1058,18 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
         return state.forceBool(res, pos);
     }) : defaultPathFilter;
 
-    Path expectedStorePath;
-    if (expectedHash) {
-        expectedStorePath =
-            state.store->makeFixedOutputPath(recursive, expectedHash, name);
-    }
+    std::optional<StorePath> expectedStorePath;
+    if (expectedHash)
+        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
     Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
-        dstPath = settings.readOnlyMode
+    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+        dstPath = state.store->printStorePath(settings.readOnlyMode
             ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
-        if (expectedHash && expectedStorePath != dstPath) {
-            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
-        }
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
-        dstPath = expectedStorePath;
+        dstPath = state.store->printStorePath(*expectedStorePath);
 
     mkString(v, dstPath, {dstPath});
 }
@@ -1091,7 +1086,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
-    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -2151,7 +2146,7 @@ void EvalState::createBaseEnv()
     }
 
     if (!evalSettings.pureEval) {
-        mkString(v, settings.thisSystem);
+        mkString(v, settings.thisSystem.get());
         addConstant("__currentSystem", v);
     }
 
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 2d79739ea04..94fa0158c80 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
         if (!settings.readOnlyMode)
-            state.store->ensurePath(i.name);
+            state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
         auto iter = i.value->attrs->find(sPath);
         if (iter != i.value->attrs->end()) {
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 9d0c642911d..4aee1073eeb 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -69,7 +69,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return gitInfo;
         }
@@ -156,7 +156,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         gitInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(gitInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
             gitInfo.revCount = json["revCount"];
             return gitInfo;
         }
@@ -176,7 +176,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     unpackTarfile(*source, tmpDir);
 
-    gitInfo.storePath = store->addToStore(name, tmpDir);
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
 
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a907d0e1cd8..db274fa4f70 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -63,7 +63,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return hgInfo;
         }
@@ -134,7 +134,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
         hgInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(hgInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
             printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
             return hgInfo;
         }
@@ -150,7 +150,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
     deletePath(tmpDir + "/.hg_archival.txt");
 
-    hgInfo.storePath = store->addToStore(name, tmpDir);
+    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     nlohmann::json json;
     json["storePath"] = hgInfo.storePath;
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index 91faea122ce..7ba5e1c14f1 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -38,7 +38,12 @@ public:
         return s < s2.s;
     }
 
-    operator const string & () const
+    operator const std::string & () const
+    {
+        return *s;
+    }
+
+    operator const std::string_view () const
     {
         return *s;
     }
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 81fb6b05ad7..d41e772e9a7 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -33,25 +33,25 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
 {
     unsigned long long downloadSize, narSize;
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
     printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
 }
 
 
-void printMissing(ref<Store> store, const PathSet & willBuild,
-    const PathSet & willSubstitute, const PathSet & unknown,
+void printMissing(ref<Store> store, const StorePathSet & willBuild,
+    const StorePathSet & willSubstitute, const StorePathSet & unknown,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
         printMsg(lvl, "these derivations will be built:");
-        Paths sorted = store->topoSortPaths(willBuild);
+        auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 
     if (!willSubstitute.empty()) {
@@ -59,14 +59,14 @@ void printMissing(ref<Store> store, const PathSet & willBuild,
                 downloadSize / (1024.0 * 1024.0),
                 narSize / (1024.0 * 1024.0)));
         for (auto & i : willSubstitute)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 
     if (!unknown.empty()) {
         printMsg(lvl, fmt("don't know how to build these paths%s:",
                 (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
         for (auto & i : unknown)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 }
 
@@ -237,7 +237,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
 void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
 {
-    parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
+    parseCmdLine(std::string(baseNameOf(argv[0])), argvToStrings(argc, argv), parseArg);
 }
 
 
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index 8e4861232db..b4957465227 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -3,6 +3,7 @@
 #include "util.hh"
 #include "args.hh"
 #include "common-args.hh"
+#include "path.hh"
 
 #include <signal.h>
 
@@ -37,11 +38,15 @@ void printVersion(const string & programName);
 void printGCWarning();
 
 class Store;
+struct StorePathWithOutputs;
 
-void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo);
+void printMissing(
+    ref<Store> store,
+    const std::vector<StorePathWithOutputs> & paths,
+    Verbosity lvl = lvlInfo);
 
-void printMissing(ref<Store> store, const PathSet & willBuild,
-    const PathSet & willSubstitute, const PathSet & unknown,
+void printMissing(ref<Store> store, const StorePathSet & willBuild,
+    const StorePathSet & willSubstitute, const StorePathSet & unknown,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
 
 string getArg(const string & opt,
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 7e3e5ff8847..aaacf828152 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -91,19 +91,18 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
     return sink.s;
 }
 
-Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
+std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    assertStorePath(storePath);
-    return storePathToHash(storePath) + ".narinfo";
+    return storePathToHash(printStorePath(storePath)) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 {
     auto narInfoFile = narInfoFileFor(narInfo->path);
 
-    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
+    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(narInfo->path);
+    auto hashPart = storePathToHash(printStorePath(narInfo->path));
 
     {
         auto state_(state.lock());
@@ -126,8 +125,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             if (ref != info.path)
                 queryPathInfo(ref);
         } catch (InvalidPath &) {
-            throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
-                % info.path % ref);
+            throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
+                printStorePath(info.path), printStorePath(ref));
         }
 
     assert(nar->compare(0, narMagic.size(), narMagic) == 0);
@@ -138,14 +137,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
+        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
 
     auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
 
     auto narAccessor = makeNarAccessor(nar);
 
     if (accessor_)
-        accessor_->addToCache(info.path, *nar, narAccessor);
+        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
 
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
@@ -162,7 +161,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             }
         }
 
-        upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
     }
 
     /* Compress the NAR. */
@@ -174,10 +173,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->fileSize = narCompressed->size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
-        % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
-        % duration);
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), narInfo->narSize,
+        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
+        duration);
 
     narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
@@ -254,14 +253,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*secretKey);
+    if (secretKey) narInfo->sign(*this, *secretKey);
 
     writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
 }
 
-bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
+bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
 {
     // FIXME: this only checks whether a .narinfo with a matching hash
     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
@@ -269,7 +268,7 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
     return fileExists(narInfoFileFor(storePath));
 }
 
-void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
+void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
@@ -295,12 +294,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
     stats.narReadBytes += narSize;
 }
 
-void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
+void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     auto uri = getUri();
+    auto storePathS = printStorePath(storePath);
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
-        fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
+        fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri});
     PushActivity pact(act->id);
 
     auto narInfoFile = narInfoFileFor(storePath);
@@ -326,7 +326,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
         }});
 }
 
-Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
+StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
@@ -345,20 +345,18 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info;
-    info.path = makeFixedOutputPath(recursive, h, name);
+    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
 
     addToStore(info, sink.s, repair, CheckSigs, nullptr);
 
-    return info.path;
+    return std::move(info.path);
 }
 
-Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, RepairFlag repair)
+StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
+    const StorePathSet & references, RepairFlag repair)
 {
-    ValidPathInfo info;
-    info.path = computeStorePathForText(name, s, references);
-    info.references = references;
+    ValidPathInfo info(computeStorePathForText(name, s, references));
+    info.references = cloneStorePathSet(references);
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
@@ -366,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
         addToStore(info, sink.s, repair, CheckSigs, nullptr);
     }
 
-    return info.path;
+    return std::move(info.path);
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
@@ -374,7 +372,7 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
 }
 
-void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
+void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     /* Note: this is inherently racy since there is no locking on
        binary caches. In particular, with S3 this unreliable, even
@@ -390,24 +388,22 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
     writeNarInfo(narInfo);
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
+std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    Path drvPath;
+    auto drvPath = path.clone();
 
-    if (isDerivation(path))
-        drvPath = path;
-    else {
+    if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
-            if (info->deriver == "") return nullptr;
-            drvPath = info->deriver;
+            if (!info->deriver) return nullptr;
+            drvPath = info->deriver->clone();
         } catch (InvalidPath &) {
             return nullptr;
         }
     }
 
-    auto logPath = "log/" + baseNameOf(drvPath);
+    auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
 
     debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 2d7cd1947bf..fa2200ad8b6 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -65,18 +65,18 @@ private:
 
     std::string narMagic;
 
-    std::string narInfoFileFor(const Path & storePath);
+    std::string narInfoFileFor(const StorePath & storePath);
 
     void writeNarInfo(ref<NarInfo> narInfo);
 
 public:
 
-    bool isValidPathUncached(const Path & path) override;
+    bool isValidPathUncached(const StorePath & path) override;
 
-    void queryPathInfoUncached(const Path & path,
+    void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
-    Path queryPathFromHashPart(const string & hashPart) override
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
     bool wantMassQuery() override { return wantMassQuery_; }
@@ -85,27 +85,27 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair) override;
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override;
 
-    void narFromPath(const Path & path, Sink & sink) override;
+    void narFromPath(const StorePath & path, Sink & sink) override;
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     { unsupported("buildDerivation"); }
 
-    void ensurePath(const Path & path) override
+    void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
     ref<FSAccessor> getFSAccessor() override;
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override;
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    std::shared_ptr<std::string> getBuildLog(const Path & path) override;
+    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
 
     int getPriority() override { return priority; }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 69649162e1c..de0ce9ae163 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -14,6 +14,7 @@
 #include "parsed-derivations.hh"
 #include "machines.hh"
 #include "daemon.hh"
+#include "worker-protocol.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -98,7 +99,7 @@ typedef set<GoalPtr, CompareGoalPtrs> Goals;
 typedef list<WeakGoalPtr> WeakGoals;
 
 /* A map of paths to goals (and the other way around). */
-typedef map<Path, WeakGoalPtr> WeakGoalMap;
+typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
 
 
@@ -253,7 +254,7 @@ class Worker
     steady_time_point lastWokenUp;
 
     /* Cache for pathContentsGood(). */
-    std::map<Path, bool> pathContentsGoodCache;
+    std::map<StorePath, bool> pathContentsGoodCache;
 
 public:
 
@@ -300,10 +301,10 @@ class Worker
     ~Worker();
 
     /* Make a goal (with caching). */
-    GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath,
+    GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath,
         const BasicDerivation & drv, BuildMode buildMode = bmNormal);
-    GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair);
+    GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
 
     /* Remove a dead goal. */
     void removeGoal(GoalPtr goal);
@@ -351,9 +352,9 @@ class Worker
 
     /* Check whether the given valid path exists and has the right
        contents. */
-    bool pathContentsGood(const Path & path);
+    bool pathContentsGood(const StorePath & path);
 
-    void markContentsGood(const Path & path);
+    void markContentsGood(StorePath && path);
 
     void updateProgress()
     {
@@ -470,7 +471,10 @@ static void commonChildInit(Pipe & logPipe)
     close(fdDevNull);
 }
 
-void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir)
+void handleDiffHook(
+    uid_t uid, uid_t gid,
+    const Path & tryA, const Path & tryB,
+    const Path & drvPath, const Path & tmpDir)
 {
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
@@ -662,7 +666,7 @@ HookInstance::HookInstance()
             throw SysError("dupping builder's stdout/stderr");
 
         Strings args = {
-            baseNameOf(settings.buildHook),
+            std::string(baseNameOf(settings.buildHook.get())),
             std::to_string(verbosity),
         };
 
@@ -709,7 +713,7 @@ class DerivationGoal : public Goal
     bool useDerivation;
 
     /* The path of the derivation. */
-    Path drvPath;
+    StorePath drvPath;
 
     /* The specific outputs that we need to build.  Empty means all of
        them. */
@@ -734,14 +738,14 @@ class DerivationGoal : public Goal
 
     /* All input paths (that is, the union of FS closures of the
        immediate input paths). */
-    PathSet inputPaths;
+    StorePathSet inputPaths;
 
     /* Outputs that are already valid.  If we're repairing, these are
        the outputs that are valid *and* not corrupt. */
-    PathSet validPaths;
+    StorePathSet validPaths;
 
     /* Outputs that are corrupt or not valid. */
-    PathSet missingPaths;
+    StorePathSet missingPaths;
 
     /* User selected for running the builder. */
     std::unique_ptr<UserLock> buildUser;
@@ -821,7 +825,7 @@ class DerivationGoal : public Goal
 
     /* Hash rewriting. */
     StringMap inputRewrites, outputRewrites;
-    typedef map<Path, Path> RedirectedOutputs;
+    typedef map<StorePath, StorePath> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
     BuildMode buildMode;
@@ -829,7 +833,7 @@ class DerivationGoal : public Goal
     /* If we're repairing without a chroot, there may be outputs that
        are valid but corrupt.  So we redirect these outputs to
        temporary paths. */
-    PathSet redirectedBadOutputs;
+    StorePathSet redirectedBadOutputs;
 
     BuildResult result;
 
@@ -868,23 +872,23 @@ class DerivationGoal : public Goal
     std::vector<std::thread> daemonWorkerThreads;
 
     /* Paths that were added via recursive Nix calls. */
-    PathSet addedPaths;
+    StorePathSet addedPaths;
 
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
        /nix/store/<bla> is some arbitrary path in a binary cache). */
-    bool isAllowed(const Path & path)
+    bool isAllowed(const StorePath & path)
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
 
-    friend class RestrictedStore;
+    friend struct RestrictedStore;
 
 public:
-    DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
+    DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs,
         Worker & worker, BuildMode buildMode = bmNormal);
-    DerivationGoal(const Path & drvPath, const BasicDerivation & drv,
+    DerivationGoal(StorePath && drvPath, const BasicDerivation & drv,
         Worker & worker, BuildMode buildMode = bmNormal);
     ~DerivationGoal();
 
@@ -899,14 +903,14 @@ class DerivationGoal : public Goal
            i.e. a derivation named "aardvark" always comes before
            "baboon". And substitution goals always happen before
            derivation goals (due to "b$"). */
-        return "b$" + storePathToName(drvPath) + "$" + drvPath;
+        return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
     }
 
     void work() override;
 
-    Path getDrvPath()
+    StorePath getDrvPath()
     {
-        return drvPath;
+        return drvPath.clone();
     }
 
     /* Add wanted outputs to an already existing derivation goal. */
@@ -946,7 +950,7 @@ class DerivationGoal : public Goal
 
     /* Add 'path' to the set of paths that may be referenced by the
        outputs, and make it appear in the sandbox. */
-    void addDependency(const Path & path);
+    void addDependency(const StorePath & path);
 
     /* Make a file owned by the builder. */
     void chownToBuilder(const Path & path);
@@ -980,15 +984,12 @@ class DerivationGoal : public Goal
     void flushLine();
 
     /* Return the set of (in)valid paths. */
-    PathSet checkPathValidity(bool returnValid, bool checkHash);
-
-    /* Abort the goal if `path' failed to build. */
-    bool pathFailed(const Path & path);
+    StorePathSet checkPathValidity(bool returnValid, bool checkHash);
 
     /* Forcibly kill the child process, if any. */
     void killChild();
 
-    Path addHashRewrite(const Path & path);
+    void addHashRewrite(const StorePath & path);
 
     void repairClosure();
 
@@ -999,23 +1000,23 @@ class DerivationGoal : public Goal
 
     void done(BuildResult::Status status, const string & msg = "");
 
-    PathSet exportReferences(PathSet storePaths);
+    StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 
 
 const Path DerivationGoal::homeDir = "/homeless-shelter";
 
 
-DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs,
+DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs,
     Worker & worker, BuildMode buildMode)
     : Goal(worker)
     , useDerivation(true)
-    , drvPath(drvPath)
+    , drvPath(std::move(drvPath))
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
     state = &DerivationGoal::getDerivation;
-    name = (format("building of '%1%'") % drvPath).str();
+    name = fmt("building of '%s'", worker.store.printStorePath(drvPath));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1023,16 +1024,16 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
 }
 
 
-DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv,
+DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv,
     Worker & worker, BuildMode buildMode)
     : Goal(worker)
     , useDerivation(false)
-    , drvPath(drvPath)
+    , drvPath(std::move(drvPath))
     , buildMode(buildMode)
 {
-    this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv));
+    this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
     state = &DerivationGoal::haveDerivation;
-    name = (format("building of %1%") % showPaths(drv.outputPaths())).str();
+    name = fmt("building of %s", worker.store.showPaths(drv.outputPaths()));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1142,7 +1143,7 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        printError(format("cannot build missing derivation '%1%'") % drvPath);
+        printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath));
         done(BuildResult::MiscFailure);
         return;
     }
@@ -1171,7 +1172,7 @@ void DerivationGoal::haveDerivation()
         worker.store.addTempRoot(i.second.path);
 
     /* Check what outputs paths are not already valid. */
-    PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
+    auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
 
     /* If they are all valid, then we're done. */
     if (invalidOutputs.size() == 0 && buildMode == bmNormal) {
@@ -1179,7 +1180,7 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv);
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
@@ -1200,7 +1201,9 @@ void DerivationGoal::outputsSubstituted()
     trace("all outputs substituted (maybe)");
 
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
-        done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str());
+        done(BuildResult::TransientFailure,
+            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
+                worker.store.printStorePath(drvPath)));
         return;
     }
 
@@ -1227,14 +1230,15 @@ void DerivationGoal::outputsSubstituted()
         return;
     }
     if (buildMode == bmCheck && nrInvalid > 0)
-        throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath);
+        throw Error("some outputs of '%s' are not valid, so checking is not possible",
+            worker.store.printStorePath(drvPath));
 
     /* Otherwise, at least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
 
     /* Make sure checkPathValidity() from now on checks all
        outputs. */
-    wantedOutputs = PathSet();
+    wantedOutputs.clear();
 
     /* The inputs must be built before we can build this goal. */
     if (useDerivation)
@@ -1244,8 +1248,8 @@ void DerivationGoal::outputsSubstituted()
     for (auto & i : drv->inputSrcs) {
         if (worker.store.isValidPath(i)) continue;
         if (!settings.useSubstitutes)
-            throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled")
-                % i % drvPath);
+            throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
+                worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         addWaitee(worker.makeSubstitutionGoal(i));
     }
 
@@ -1264,7 +1268,7 @@ void DerivationGoal::repairClosure()
        that produced those outputs. */
 
     /* Get the output closure. */
-    PathSet outputClosure;
+    StorePathSet outputClosure;
     for (auto & i : drv->outputs) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
         worker.store.computeFSClosure(i.second.path, outputClosure);
@@ -1277,26 +1281,26 @@ void DerivationGoal::repairClosure()
     /* Get all dependencies of this derivation so that we know which
        derivation is responsible for which path in the output
        closure. */
-    PathSet inputClosure;
+    StorePathSet inputClosure;
     if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<Path, Path> outputsToDrv;
+    std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
-        if (isDerivation(i)) {
+        if (i.isDerivation()) {
             Derivation drv = worker.store.derivationFromPath(i);
             for (auto & j : drv.outputs)
-                outputsToDrv[j.second.path] = i;
+                outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone());
         }
 
     /* Check each path (slow!). */
-    PathSet broken;
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
-        printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath);
-        Path drvPath2 = outputsToDrv[i];
-        if (drvPath2 == "")
+        printError("found corrupted or missing path '%s' in the output closure of '%s'",
+            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
+        auto drvPath2 = outputsToDrv.find(i);
+        if (drvPath2 == outputsToDrv.end())
             addWaitee(worker.makeSubstitutionGoal(i, Repair));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair));
+            addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
     }
 
     if (waitees.empty()) {
@@ -1312,7 +1316,8 @@ void DerivationGoal::closureRepaired()
 {
     trace("closure repaired");
     if (nrFailed > 0)
-        throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath);
+        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
+            worker.store.printStorePath(drvPath));
     done(BuildResult::AlreadyValid);
 }
 
@@ -1323,10 +1328,9 @@ void DerivationGoal::inputsRealised()
 
     if (nrFailed != 0) {
         if (!useDerivation)
-            throw Error(format("some dependencies of '%1%' are missing") % drvPath);
-        printError(
-            format("cannot build derivation '%1%': %2% dependencies couldn't be built")
-            % drvPath % nrFailed);
+            throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
+        printError("cannot build derivation '%s': %s dependencies couldn't be built",
+            worker.store.printStorePath(drvPath), nrFailed);
         done(BuildResult::DependencyFailed);
         return;
     }
@@ -1349,19 +1353,21 @@ void DerivationGoal::inputsRealised()
                that are specified as inputs. */
             assert(worker.store.isValidPath(i.first));
             Derivation inDrv = worker.store.derivationFromPath(i.first);
-            for (auto & j : i.second)
-                if (inDrv.outputs.find(j) != inDrv.outputs.end())
-                    worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths);
+            for (auto & j : i.second) {
+                auto k = inDrv.outputs.find(j);
+                if (k != inDrv.outputs.end())
+                    worker.store.computeFSClosure(k->second.path, inputPaths);
                 else
                     throw Error(
-                        format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'")
-                        % drvPath % j % i.first);
+                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
+                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first));
+            }
         }
 
     /* Second, the input sources. */
     worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
 
-    debug(format("added input paths %1%") % showPaths(inputPaths));
+    debug("added input paths %s", worker.store.showPaths(inputPaths));
 
     /* Is this a fixed-output derivation? */
     fixedOutput = drv->isFixedOutput();
@@ -1391,7 +1397,7 @@ void DerivationGoal::tryToBuild()
        few seconds and then retry this goal. */
     PathSet lockFiles;
     for (auto & outPath : drv->outputPaths())
-        lockFiles.insert(worker.store.toRealPath(outPath));
+        lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath)));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         worker.waitForAWhile(shared_from_this());
@@ -1407,23 +1413,22 @@ void DerivationGoal::tryToBuild()
        build this derivation, so no further checks are necessary. */
     validPaths = checkPathValidity(true, buildMode == bmRepair);
     if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) {
-        debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath);
+        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
         outputLocks.setDeletion(true);
         done(BuildResult::AlreadyValid);
         return;
     }
 
-    missingPaths = drv->outputPaths();
+    missingPaths = cloneStorePathSet(drv->outputPaths());
     if (buildMode != bmCheck)
         for (auto & i : validPaths) missingPaths.erase(i);
 
     /* If any of the outputs already exist but are not valid, delete
        them. */
     for (auto & i : drv->outputs) {
-        Path path = i.second.path;
-        if (worker.store.isValidPath(path)) continue;
-        debug(format("removing invalid path '%1%'") % path);
-        deletePath(worker.store.toRealPath(path));
+        if (worker.store.isValidPath(i.second.path)) continue;
+        debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
+        deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path)));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1436,11 +1441,11 @@ void DerivationGoal::tryToBuild()
             buildMode == bmRepair ? "repairing outputs of '%s'" :
             buildMode == bmCheck ? "checking outputs of '%s'" :
             nrRounds > 1 ? "building '%s' (round %d/%d)" :
-            "building '%s'", drvPath, curRound, nrRounds);
-        fmt("building '%s'", drvPath);
+            "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+        fmt("building '%s'", worker.store.printStorePath(drvPath));
         if (hook) msg += fmt(" on '%s'", machineName);
         act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-            Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds});
+            Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
         mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
         worker.updateProgress();
     };
@@ -1509,7 +1514,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
     if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
-        throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath);
+        throw SysError("moving '%s' to '%s'", tmpPath, storePath);
     deletePath(oldPath);
 }
 
@@ -1534,7 +1539,7 @@ void DerivationGoal::buildDone()
        kill it. */
     int status = hook ? hook->pid.kill() : pid.kill();
 
-    debug(format("builder process for '%1%' finished") % drvPath);
+    debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
 
     result.timesBuilt++;
     result.stopTime = time(0);
@@ -1590,12 +1595,15 @@ void DerivationGoal::buildDone()
             /* Move paths out of the chroot for easier debugging of
                build failures. */
             if (useChroot && buildMode == bmNormal)
-                for (auto & i : missingPaths)
-                    if (pathExists(chrootRootDir + i))
-                        rename((chrootRootDir + i).c_str(), i.c_str());
+                for (auto & i : missingPaths) {
+                    auto p = worker.store.printStorePath(i);
+                    if (pathExists(chrootRootDir + p))
+                        rename((chrootRootDir + p).c_str(), p.c_str());
+                }
 
-            std::string msg = (format("builder for '%1%' %2%")
-                % drvPath % statusToString(status)).str();
+            auto msg = fmt("builder for '%s' %s",
+                worker.store.printStorePath(drvPath),
+                statusToString(status));
 
             if (!settings.verboseBuild && !logTail.empty()) {
                 msg += (format("; last %d log lines:") % logTail.size()).str();
@@ -1616,13 +1624,13 @@ void DerivationGoal::buildDone()
         if (settings.postBuildHook != "") {
             Activity act(*logger, lvlInfo, actPostBuildHook,
                 fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{drvPath});
+                Logger::Fields{worker.store.printStorePath(drvPath)});
             PushActivity pact(act.id);
             auto outputPaths = drv->outputPaths();
             std::map<std::string, std::string> hookEnvironment = getEnv();
 
-            hookEnvironment.emplace("DRV_PATH", drvPath);
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
+            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
 
             RunOptions opts(settings.postBuildHook, {});
             opts.environment = hookEnvironment;
@@ -1675,7 +1683,7 @@ void DerivationGoal::buildDone()
 
         /* Delete unused redirected outputs (when doing hash rewriting). */
         for (auto & i : redirectedOutputs)
-            deletePath(i.second);
+            deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second)));
 
         /* Delete the chroot (if we were using one). */
         autoDelChroot.reset(); /* this runs the destructor */
@@ -1740,7 +1748,7 @@ HookReply DerivationGoal::tryBuildHook()
             << "try"
             << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
             << drv->platform
-            << drvPath
+            << worker.store.printStorePath(drvPath)
             << parsedDrv->getRequiredSystemFeatures();
         worker.hook->sink.flush();
 
@@ -1773,7 +1781,7 @@ HookReply DerivationGoal::tryBuildHook()
         else if (reply == "postpone")
             return rpPostpone;
         else if (reply != "accept")
-            throw Error(format("bad hook reply '%1%'") % reply);
+            throw Error("bad hook reply '%s'", reply);
 
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
@@ -1791,11 +1799,11 @@ HookReply DerivationGoal::tryBuildHook()
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    hook->sink << inputPaths;
+    writeStorePaths(worker.store, hook->sink, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
-    hook->sink << missingPaths;
+    writeStorePaths(worker.store, hook->sink, missingPaths);
 
     hook->sink = FdSink();
     hook->toHook.writeSide = -1;
@@ -1812,10 +1820,10 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-void chmod_(const Path & path, mode_t mode)
+static void chmod_(const Path & path, mode_t mode)
 {
     if (chmod(path.c_str(), mode) == -1)
-        throw SysError(format("setting permissions on '%1%'") % path);
+        throw SysError("setting permissions on '%s'", path);
 }
 
 
@@ -1826,33 +1834,25 @@ int childEntry(void * arg)
 }
 
 
-PathSet DerivationGoal::exportReferences(PathSet storePaths)
+StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
 {
-    PathSet paths;
-
-    for (auto storePath : storePaths) {
-
-        /* Check that the store path is valid. */
-        if (!worker.store.isInStore(storePath))
-            throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'")
-                % storePath);
-
-        storePath = worker.store.toStorePath(storePath);
+    StorePathSet paths;
 
+    for (auto & storePath : storePaths) {
         if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath);
+            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
 
-        worker.store.computeFSClosure(storePath, paths);
+        worker.store.computeFSClosure(singleton(storePath), paths);
     }
 
     /* If there are derivations in the graph, then include their
        outputs as well.  This is useful if you want to do things
        like passing all build-time dependencies of some path to a
        derivation that builds a NixOS DVD image. */
-    PathSet paths2(paths);
+    auto paths2 = cloneStorePathSet(paths);
 
     for (auto & j : paths2) {
-        if (isDerivation(j)) {
+        if (j.isDerivation()) {
             Derivation drv = worker.store.derivationFromPath(j);
             for (auto & k : drv.outputs)
                 worker.store.computeFSClosure(k.second.path, paths);
@@ -1900,7 +1900,7 @@ void DerivationGoal::startBuilder()
         throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
             drv->platform,
             concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            drvPath,
+            worker.store.printStorePath(drvPath),
             settings.thisSystem,
             concatStringsSep(", ", settings.systemFeatures));
 
@@ -1916,12 +1916,12 @@ void DerivationGoal::startBuilder()
         auto noChroot = parsedDrv->getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
             if (noChroot)
-                throw Error(format("derivation '%1%' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'") % drvPath);
+                throw Error("derivation '%s' has '__noChroot' set, "
+                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
 #if __APPLE__
             if (additionalSandboxProfile != "")
-                throw Error(format("derivation '%1%' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath);
+                throw Error("derivation '%s' specifies a sandbox profile, "
+                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
 #endif
             useChroot = true;
         }
@@ -1957,14 +1957,13 @@ void DerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    auto drvName = storePathToName(drvPath);
-    tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700);
+    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
 
     chownToBuilder(tmpDir);
 
     /* Substitute output placeholders with the actual output paths. */
     for (auto & output : drv->outputs)
-        inputRewrites[hashPlaceholder(output.first)] = output.second.path;
+        inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path);
 
     /* Construct the environment passed to the builder. */
     initEnv();
@@ -1980,19 +1979,22 @@ void DerivationGoal::startBuilder()
            temporary build directory.  The text files have the format used
            by `nix-store --register-validity'.  However, the deriver
            fields are left empty. */
-        string s = get(drv->env, "exportReferencesGraph");
+        string s = get(drv->env, "exportReferencesGraph").value_or("");
         Strings ss = tokenizeString<Strings>(s);
         if (ss.size() % 2 != 0)
             throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
         for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
             string fileName = *i++;
-            checkStoreName(fileName); /* !!! abuse of this function */
-            Path storePath = *i++;
+            static std::regex regex("[A-Za-z_][A-Za-z0-9_.]*");
+            if (!std::regex_match(fileName, regex))
+                throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
+
+            auto storePath = worker.store.parseStorePath(*i++);
 
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
+                    exportReferences(singleton(storePath)), false, false));
         }
     }
 
@@ -2022,17 +2024,19 @@ void DerivationGoal::startBuilder()
         dirsInChroot[tmpDirInSandbox] = tmpDir;
 
         /* Add the closure of store paths to the chroot. */
-        PathSet closure;
+        StorePathSet closure;
         for (auto & i : dirsInChroot)
             try {
                 if (worker.store.isInStore(i.second.source))
-                    worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure);
+                    worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
                 throw Error(format("while processing 'sandbox-paths': %s") % e.what());
             }
-        for (auto & i : closure)
-            dirsInChroot[i] = i;
+        for (auto & i : closure) {
+            auto p = worker.store.printStorePath(i);
+            dirsInChroot.insert_or_assign(p, p);
+        }
 
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
@@ -2054,7 +2058,8 @@ void DerivationGoal::startBuilder()
                 }
             }
             if (!found)
-                throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i);
+                throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
+                    worker.store.printStorePath(drvPath), i);
 
             dirsInChroot[i] = i;
         }
@@ -2064,7 +2069,7 @@ void DerivationGoal::startBuilder()
            environment using bind-mounts.  We put it in the Nix store
            to ensure that we can create hard-links to non-directory
            inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot";
+        chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot";
         deletePath(chrootRootDir);
 
         /* Clean up the chroot directory automatically. */
@@ -2123,14 +2128,15 @@ void DerivationGoal::startBuilder()
             throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir);
 
         for (auto & i : inputPaths) {
-            Path r = worker.store.toRealPath(i);
+            auto p = worker.store.printStorePath(i);
+            Path r = worker.store.toRealPath(p);
             struct stat st;
             if (lstat(r.c_str(), &st))
-                throw SysError(format("getting attributes of path '%1%'") % i);
+                throw SysError("getting attributes of path '%s'", p);
             if (S_ISDIR(st.st_mode))
-                dirsInChroot[i] = r;
+                dirsInChroot.insert_or_assign(p, r);
             else
-                linkOrCopy(r, chrootRootDir + i);
+                linkOrCopy(r, chrootRootDir + p);
         }
 
         /* If we're repairing, checking or rebuilding part of a
@@ -2139,7 +2145,7 @@ void DerivationGoal::startBuilder()
            (typically the dependencies of /bin/sh).  Throw them
            out. */
         for (auto & i : drv->outputs)
-            dirsInChroot.erase(i.second.path);
+            dirsInChroot.erase(worker.store.printStorePath(i.second.path));
 
 #elif __APPLE__
         /* We don't really have any parent prep work to do (yet?)
@@ -2171,17 +2177,17 @@ void DerivationGoal::startBuilder()
            corrupt outputs in advance.  So rewrite them as well. */
         if (buildMode == bmRepair)
             for (auto & i : missingPaths)
-                if (worker.store.isValidPath(i) && pathExists(i)) {
+                if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) {
                     addHashRewrite(i);
-                    redirectedBadOutputs.insert(i);
+                    redirectedBadOutputs.insert(i.clone());
                 }
     }
 
     if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
         printMsg(lvlChatty, format("executing pre-build hook '%1%'")
             % settings.preBuildHook);
-        auto args = useChroot ? Strings({drvPath, chrootRootDir}) :
-            Strings({ drvPath });
+        auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
+            Strings({ worker.store.printStorePath(drvPath) });
         enum BuildHookState {
             stBegin,
             stExtraChrootDirs
@@ -2448,7 +2454,7 @@ void DerivationGoal::initTmpDir() {
        there is no size constraint). */
     if (!parsedDrv->getStructuredAttrs()) {
 
-        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile"));
+        StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
         int fileNr = 0;
         for (auto & i : drv->env) {
             if (passAsFile.find(i.first) == passAsFile.end()) {
@@ -2478,6 +2484,7 @@ void DerivationGoal::initTmpDir() {
     env["PWD"] = tmpDirInSandbox;
 }
 
+
 void DerivationGoal::initEnv()
 {
     env.clear();
@@ -2549,7 +2556,7 @@ void DerivationGoal::writeStructuredAttrs()
     /* Add an "outputs" object containing the output paths. */
     nlohmann::json outputs;
     for (auto & i : drv->outputs)
-        outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
+        outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites);
     json["outputs"] = outputs;
 
     /* Handle exportReferencesGraph. */
@@ -2559,9 +2566,9 @@ void DerivationGoal::writeStructuredAttrs()
             std::ostringstream str;
             {
                 JSONPlaceholder jsonRoot(str, true);
-                PathSet storePaths;
+                StorePathSet storePaths;
                 for (auto & p : *i)
-                    storePaths.insert(p.get<std::string>());
+                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
                 worker.store.pathInfoToJSON(jsonRoot,
                     exportReferences(storePaths), false, true);
             }
@@ -2660,22 +2667,22 @@ struct RestrictedStore : public LocalFSStore
     std::string getUri() override
     { return next->getUri(); }
 
-    PathSet queryAllValidPaths() override
+    StorePathSet queryAllValidPaths() override
     {
-        PathSet paths;
-        for (auto & p : goal.inputPaths) paths.insert(p);
-        for (auto & p : goal.addedPaths) paths.insert(p);
+        StorePathSet paths;
+        for (auto & p : goal.inputPaths) paths.insert(p.clone());
+        for (auto & p : goal.addedPaths) paths.insert(p.clone());
         return paths;
     }
 
-    void queryPathInfoUncached(const Path & path,
+    void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
     {
         if (goal.isAllowed(path)) {
             try {
                 /* Censor impure information. */
                 auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
-                info->deriver.clear();
+                info->deriver.reset();
                 info->registrationTime = 0;
                 info->ultimate = false;
                 info->sigs.clear();
@@ -2687,19 +2694,19 @@ struct RestrictedStore : public LocalFSStore
             callback(nullptr);
     };
 
-    void queryReferrers(const Path & path, PathSet & referrers) override
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    PathSet queryDerivationOutputs(const Path & path) override
+    StorePathSet queryDerivationOutputs(const StorePath & path) override
     { throw Error("queryDerivationOutputs"); }
 
-    StringSet queryDerivationOutputNames(const Path & path) override
+    StringSet queryDerivationOutputNames(const StorePath & path) override
     { throw Error("queryDerivationOutputNames"); }
 
-    Path queryPathFromHashPart(const string & hashPart) override
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
     { throw Error("addToStore"); }
@@ -2712,7 +2719,7 @@ struct RestrictedStore : public LocalFSStore
         goal.addDependency(info.path);
     }
 
-    Path addToStoreFromDump(const string & dump, const string & name,
+    StorePath addToStoreFromDump(const string & dump, const string & name,
         bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
     {
         auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
@@ -2720,89 +2727,87 @@ struct RestrictedStore : public LocalFSStore
         return path;
     }
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair = NoRepair) override
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair = NoRepair) override
     {
         auto path = next->addTextToStore(name, s, references, repair);
         goal.addDependency(path);
         return path;
     }
 
-    void narFromPath(const Path & path, Sink & sink) override 
+    void narFromPath(const StorePath & path, Sink & sink) override
     {
         if (!goal.isAllowed(path))
-            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path);
+            throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
         LocalFSStore::narFromPath(path, sink);
     }
 
-    void ensurePath(const Path & path) override
+    void ensurePath(const StorePath & path) override
     {
         if (!goal.isAllowed(path))
-            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path);
+            throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
         /* Nothing to be done; 'path' must already be valid. */
     }
 
-    void buildPaths(const PathSet & paths, BuildMode buildMode) override
+    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
-        PathSet newPaths;
+        StorePathSet newPaths;
 
         for (auto & path : paths) {
-            DrvPathWithOutputs i = parseDrvPathWithOutputs(path);
-            if (isDerivation(i.first)) {
-                if (!goal.isAllowed(i.first))
-                    throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first);
-                auto drv = derivationFromPath(i.first);
+            if (path.path.isDerivation()) {
+                if (!goal.isAllowed(path.path))
+                    throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
+                auto drv = derivationFromPath(path.path);
                 for (auto & output : drv.outputs)
-                    if (wantOutput(output.first, i.second))
-                        newPaths.insert(output.second.path);
-            } else if (!goal.isAllowed(path))
-                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path);
+                    if (wantOutput(output.first, path.outputs))
+                        newPaths.insert(output.second.path.clone());
+            } else if (!goal.isAllowed(path.path))
+                throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
         }
 
         next->buildPaths(paths, buildMode);
 
-        PathSet closure;
+        StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
     }
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode = bmNormal) override
     { unsupported("buildDerivation"); }
 
-    void addTempRoot(const Path & path)
+    void addTempRoot(const StorePath & path) override
     { }
 
-    void addIndirectRoot(const Path & path)
+    void addIndirectRoot(const Path & path) override
     { }
 
-    Roots findRoots()
+    Roots findRoots(bool censor) override
     { return Roots(); }
 
-    void collectGarbage(const GCOptions & options, GCResults & results)
+    void collectGarbage(const GCOptions & options, GCResults & results) override
     { }
 
-    void addSignatures(const Path & storePath, const StringSet & sigs)
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override
     { unsupported("addSignatures"); }
 
-    void queryMissing(const PathSet & targets,
-        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
-        unsigned long long & downloadSize, unsigned long long & narSize)
+    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
+        unsigned long long & downloadSize, unsigned long long & narSize) override
     {
         /* This is slightly impure since it leaks information to the
            client about what paths will be built/substituted or are
            already present. Probably not a big deal. */
 
-        PathSet allowed;
+        std::vector<StorePathWithOutputs> allowed;
         for (auto & path : targets) {
-            DrvPathWithOutputs i = parseDrvPathWithOutputs(path);
-            if (goal.isAllowed(i.first))
-                allowed.insert(i.first);
+            if (goal.isAllowed(path.path))
+                allowed.emplace_back(path);
             else
-                unknown.insert(i.first);
+                unknown.insert(path.path.clone());
         }
 
         next->queryMissing(allowed, willBuild, willSubstitute,
@@ -2855,7 +2860,7 @@ void DerivationGoal::startDaemon()
 
             debug("received daemon connection");
 
-            auto workerThread = std::thread([this, store, remote{std::move(remote)}]() {
+            auto workerThread = std::thread([store, remote{std::move(remote)}]() {
                 FdSource from(remote.get());
                 FdSink to(remote.get());
                 try {
@@ -2893,28 +2898,26 @@ void DerivationGoal::stopDaemon()
 }
 
 
-void DerivationGoal::addDependency(const Path & path)
+void DerivationGoal::addDependency(const StorePath & path)
 {
-    worker.store.assertStorePath(path);
-
     if (isAllowed(path)) return;
 
-    addedPaths.insert(path);
+    addedPaths.insert(path.clone());
 
     /* If we're doing a sandbox build, then we have to make the path
        appear in the sandbox. */
     if (useChroot) {
 
-        debug("materialising '%s' in the sandbox", path);
+        debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
 
         #if __linux__
 
-            Path source = worker.store.toRealPath(path);
-            Path target = chrootRootDir + path;
+            Path source = worker.store.toRealPath(worker.store.printStorePath(path));
+            Path target = chrootRootDir + worker.store.printStorePath(path);
             debug("bind-mounting %s -> %s", target, source);
 
             if (pathExists(target))
-                throw Error("store path '%s' already exists in the sandbox", path);
+                throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
 
             struct stat st;
             if (lstat(source.c_str(), &st))
@@ -2941,13 +2944,14 @@ void DerivationGoal::addDependency(const Path & path)
 
                 int status = child.wait();
                 if (status != 0)
-                    throw Error("could not add path '%s' to sandbox", path);
+                    throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
 
             } else
                 linkOrCopy(source, target);
 
         #else
-            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path);
+            throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
+                worker.store.printStorePath(path));
         #endif
 
     }
@@ -3363,8 +3367,10 @@ void DerivationGoal::runChild()
                 }
 
                 /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths)
-                    dirsInChroot[i] = i;
+                for (auto & i : inputPaths) {
+                    auto p = worker.store.printStorePath(i);
+                    dirsInChroot[p] = p;
+                }
 
                 /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
                 if (settings.darwinLogSandboxViolations) {
@@ -3380,13 +3386,13 @@ void DerivationGoal::runChild()
 
                 /* Our rwx outputs */
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : missingPaths) {
-                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
-                }
+                for (auto & i : missingPaths)
+                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i));
+
                 /* Also add redirected outputs to the chroot */
-                for (auto & i : redirectedOutputs) {
-                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
-                }
+                for (auto & i : redirectedOutputs)
+                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second));
+
                 sandboxProfile += ")\n";
 
                 /* Our inputs (transitive dependencies and any impurities computed above)
@@ -3405,19 +3411,19 @@ void DerivationGoal::runChild()
                     if (lstat(path.c_str(), &st)) {
                         if (i.second.optional && errno == ENOENT)
                             continue;
-                        throw SysError(format("getting attributes of path '%1%'") % path);
+                        throw SysError("getting attributes of path '%s", path);
                     }
                     if (S_ISDIR(st.st_mode))
-                        sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
+                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
                     else
-                        sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str();
+                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
                 }
                 sandboxProfile += ")\n";
 
                 /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
                 sandboxProfile += "(allow file-read*\n";
                 for (auto & i : ancestry) {
-                    sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str();
+                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
                 }
                 sandboxProfile += ")\n";
 
@@ -3458,8 +3464,7 @@ void DerivationGoal::runChild()
 #endif
         else {
             builder = drv->builder.c_str();
-            string builderBasename = baseNameOf(drv->builder);
-            args.push_back(builderBasename);
+            args.push_back(std::string(baseNameOf(drv->builder)));
         }
 
         for (auto & i : drv->args)
@@ -3473,7 +3478,7 @@ void DerivationGoal::runChild()
             try {
                 logger = makeJSONLogger(*logger);
 
-                BasicDerivation drv2(*drv);
+                BasicDerivation & drv2(*drv);
                 for (auto & e : drv2.env)
                     e.second = rewriteStrings(e.second, inputRewrites);
 
@@ -3506,16 +3511,15 @@ void DerivationGoal::runChild()
 /* Parse a list of reference specifiers.  Each element must either be
    a store path, or the symbolic name of the output of the derivation
    (such as `out'). */
-PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
+StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths)
 {
-    PathSet result;
+    StorePathSet result;
     for (auto & i : paths) {
         if (store.isStorePath(i))
-            result.insert(i);
-        else if (drv.outputs.find(i) != drv.outputs.end())
-            result.insert(drv.outputs.find(i)->second.path);
-        else throw BuildError(
-            format("derivation contains an illegal reference specifier '%1%'") % i);
+            result.insert(store.parseStorePath(i));
+        else if (drv.outputs.count(i))
+            result.insert(drv.outputs.find(i)->second.path.clone());
+        else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
     }
     return result;
 }
@@ -3548,19 +3552,17 @@ void DerivationGoal::registerOutputs()
     /* The paths that can be referenced are the input closures, the
        output paths, and any paths that have been built via recursive
        Nix calls. */
-    PathSet referenceablePaths;
-    for (auto & p : inputPaths) referenceablePaths.insert(p);
-    for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
-    for (auto & p : addedPaths) referenceablePaths.insert(p);
+    StorePathSet referenceablePaths;
+    for (auto & p : inputPaths) referenceablePaths.insert(p.clone());
+    for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone());
+    for (auto & p : addedPaths) referenceablePaths.insert(p.clone());
 
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
     for (auto & i : drv->outputs) {
-        Path path = i.second.path;
-        if (missingPaths.find(path) == missingPaths.end()) continue;
-
-        ValidPathInfo info;
+        auto path = worker.store.printStorePath(i.second.path);
+        if (!missingPaths.count(i.second.path)) continue;
 
         Path actualPath = path;
         if (useChroot) {
@@ -3577,22 +3579,25 @@ void DerivationGoal::registerOutputs()
         }
 
         if (needsHashRewrite()) {
-            Path redirected = redirectedOutputs[path];
-            if (buildMode == bmRepair
-                && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()
-                && pathExists(redirected))
-                replaceValidPath(path, redirected);
-            if (buildMode == bmCheck && redirected != "")
-                actualPath = redirected;
+            auto r = redirectedOutputs.find(i.second.path);
+            if (r != redirectedOutputs.end()) {
+                auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second));
+                if (buildMode == bmRepair
+                    && redirectedBadOutputs.count(i.second.path)
+                    && pathExists(redirected))
+                    replaceValidPath(path, redirected);
+                if (buildMode == bmCheck)
+                    actualPath = redirected;
+            }
         }
 
         struct stat st;
         if (lstat(actualPath.c_str(), &st) == -1) {
             if (errno == ENOENT)
                 throw BuildError(
-                    format("builder for '%1%' failed to produce output path '%2%'")
-                    % drvPath % path);
-            throw SysError(format("getting attributes of path '%1%'") % actualPath);
+                    "builder for '%s' failed to produce output path '%s'",
+                    worker.store.printStorePath(drvPath), path);
+            throw SysError("getting attributes of path '%s'", actualPath);
         }
 
 #ifndef __CYGWIN__
@@ -3629,6 +3634,8 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
+        std::string ca;
+
         if (fixedOutput) {
 
             bool recursive; Hash h;
@@ -3645,7 +3652,7 @@ void DerivationGoal::registerOutputs()
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
 
-            Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path));
+            auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
 
             if (h != h2) {
 
@@ -3654,9 +3661,9 @@ void DerivationGoal::registerOutputs()
                 worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        dest, h.to_string(SRI), h2.to_string(SRI)));
+                        worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
 
-                Path actualDest = worker.store.toRealPath(dest);
+                Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest));
 
                 if (worker.store.isValidPath(dest))
                     std::rethrow_exception(delayedException);
@@ -3665,16 +3672,16 @@ void DerivationGoal::registerOutputs()
                     PathLocks outputLocks({actualDest});
                     deletePath(actualDest);
                     if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
-                        throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest);
+                        throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest));
                 }
 
-                path = dest;
+                path = worker.store.printStorePath(dest);
                 actualPath = actualDest;
             }
             else
-                assert(path == dest);
+                assert(worker.store.parseStorePath(path) == dest);
 
-            info.ca = makeFixedOutputCA(recursive, h2);
+            ca = makeFixedOutputCA(recursive, h2);
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -3688,11 +3695,11 @@ void DerivationGoal::registerOutputs()
            verify later on whether nobody has messed with the store. */
         debug("scanning for references inside '%1%'", path);
         HashResult hash;
-        PathSet references = scanForReferences(actualPath, referenceablePaths, hash);
+        auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
 
         if (buildMode == bmCheck) {
-            if (!worker.store.isValidPath(path)) continue;
-            auto info = *worker.store.queryPathInfo(path);
+            if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
+            ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path)));
             if (hash.first != info.narHash) {
                 worker.checkMismatch = true;
                 if (settings.runDiffHook || settings.keepFailed) {
@@ -3704,20 +3711,22 @@ void DerivationGoal::registerOutputs()
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
                         buildUser ? buildUser->getGID() : getgid(),
-                        path, dst, drvPath, tmpDir);
+                        path, dst, worker.store.printStorePath(drvPath), tmpDir);
 
-                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'")
-                        % drvPath % path % dst);
+                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
+                        worker.store.printStorePath(drvPath), path, dst);
                 } else
-                    throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs")
-                        % drvPath % path);
+                    throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
+                        worker.store.printStorePath(drvPath), path);
             }
 
             /* Since we verified the build, it's now ultimately trusted. */
             if (!info.ultimate) {
                 info.ultimate = true;
                 worker.store.signPathInfo(info);
-                worker.store.registerValidPaths({info});
+                ValidPathInfos infos;
+                infos.push_back(std::move(info));
+                worker.store.registerValidPaths(infos);
             }
 
             continue;
@@ -3725,29 +3734,30 @@ void DerivationGoal::registerOutputs()
 
         /* For debugging, print out the referenced and unreferenced paths. */
         for (auto & i : inputPaths) {
-            PathSet::iterator j = references.find(i);
+            auto j = references.find(i);
             if (j == references.end())
-                debug(format("unreferenced input: '%1%'") % i);
+                debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
             else
-                debug(format("referenced input: '%1%'") % i);
+                debug("referenced input: '%1%'", worker.store.printStorePath(i));
         }
 
         if (curRound == nrRounds) {
             worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(path);
+            worker.markContentsGood(worker.store.parseStorePath(path));
         }
 
-        info.path = path;
+        ValidPathInfo info(worker.store.parseStorePath(path));
         info.narHash = hash.first;
         info.narSize = hash.second;
-        info.references = references;
-        info.deriver = drvPath;
+        info.references = std::move(references);
+        info.deriver = drvPath.clone();
         info.ultimate = true;
+        info.ca = ca;
         worker.store.signPathInfo(info);
 
         if (!info.references.empty()) info.ca.clear();
 
-        infos[i.first] = info;
+        infos.emplace(i.first, std::move(info));
     }
 
     if (buildMode == bmCheck) return;
@@ -3762,16 +3772,19 @@ void DerivationGoal::registerOutputs()
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
                 result.isNonDeterministic = true;
-                Path prev = i->second.path + checkSuffix;
+                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
                 auto msg = prevExists
-                    ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev)
-                    : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath);
+                    ? fmt("output '%s' of '%s' differs from '%s' from previous round",
+                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
+                    : fmt("output '%s' of '%s' differs from previous round",
+                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
 
                 handleDiffHook(
                     buildUser ? buildUser->getUID() : getuid(),
                     buildUser ? buildUser->getGID() : getgid(),
-                    prev, i->second.path, drvPath, tmpDir);
+                    prev, worker.store.printStorePath(i->second.path),
+                    worker.store.printStorePath(drvPath), tmpDir);
 
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(msg);
@@ -3784,16 +3797,17 @@ void DerivationGoal::registerOutputs()
     /* If this is the first round of several, then move the output out of the way. */
     if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
         for (auto & i : drv->outputs) {
-            Path prev = i.second.path + checkSuffix;
+            auto path = worker.store.printStorePath(i.second.path);
+            Path prev = path + checkSuffix;
             deletePath(prev);
-            Path dst = i.second.path + checkSuffix;
-            if (rename(i.second.path.c_str(), dst.c_str()))
-                throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst);
+            Path dst = path + checkSuffix;
+            if (rename(path.c_str(), dst.c_str()))
+                throw SysError("renaming '%s' to '%s'", path, dst);
         }
     }
 
     if (curRound < nrRounds) {
-        prevInfos = infos;
+        prevInfos = std::move(infos);
         return;
     }
 
@@ -3801,7 +3815,7 @@ void DerivationGoal::registerOutputs()
        if the result was not determistic? */
     if (curRound == nrRounds) {
         for (auto & i : drv->outputs) {
-            Path prev = i.second.path + checkSuffix;
+            Path prev = worker.store.printStorePath(i.second.path) + checkSuffix;
             deletePath(prev);
         }
     }
@@ -3826,7 +3840,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 {
     std::map<Path, const ValidPathInfo &> outputsByPath;
     for (auto & output : outputs)
-        outputsByPath.emplace(output.second.path, output.second);
+        outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
 
     for (auto & output : outputs) {
         auto & outputName = output.first;
@@ -3842,76 +3856,77 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
         /* Compute the closure and closure size of some output. This
            is slightly tricky because some of its references (namely
            other outputs) may not be valid yet. */
-        auto getClosure = [&](const Path & path)
+        auto getClosure = [&](const StorePath & path)
         {
             uint64_t closureSize = 0;
-            PathSet pathsDone;
-            std::queue<Path> pathsLeft;
-            pathsLeft.push(path);
+            StorePathSet pathsDone;
+            std::queue<StorePath> pathsLeft;
+            pathsLeft.push(path.clone());
 
             while (!pathsLeft.empty()) {
-                auto path = pathsLeft.front();
+                auto path = pathsLeft.front().clone();
                 pathsLeft.pop();
-                if (!pathsDone.insert(path).second) continue;
+                if (!pathsDone.insert(path.clone()).second) continue;
 
-                auto i = outputsByPath.find(path);
+                auto i = outputsByPath.find(worker.store.printStorePath(path));
                 if (i != outputsByPath.end()) {
                     closureSize += i->second.narSize;
                     for (auto & ref : i->second.references)
-                        pathsLeft.push(ref);
+                        pathsLeft.push(ref.clone());
                 } else {
                     auto info = worker.store.queryPathInfo(path);
                     closureSize += info->narSize;
                     for (auto & ref : info->references)
-                        pathsLeft.push(ref);
+                        pathsLeft.push(ref.clone());
                 }
             }
 
-            return std::make_pair(pathsDone, closureSize);
+            return std::make_pair(std::move(pathsDone), closureSize);
         };
 
         auto applyChecks = [&](const Checks & checks)
         {
             if (checks.maxSize && info.narSize > *checks.maxSize)
                 throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
-                    info.path, info.narSize, *checks.maxSize);
+                    worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
 
             if (checks.maxClosureSize) {
                 uint64_t closureSize = getClosure(info.path).second;
                 if (closureSize > *checks.maxClosureSize)
                     throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
-                        info.path, closureSize, *checks.maxClosureSize);
+                        worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
             }
 
             auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
             {
                 if (!value) return;
 
-                PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value);
+                auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
 
-                PathSet used = recursive ? getClosure(info.path).first : info.references;
+                auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
 
                 if (recursive && checks.ignoreSelfRefs)
                     used.erase(info.path);
 
-                PathSet badPaths;
+                StorePathSet badPaths;
 
                 for (auto & i : used)
                     if (allowed) {
                         if (!spec.count(i))
-                            badPaths.insert(i);
+                            badPaths.insert(i.clone());
                     } else {
                         if (spec.count(i))
-                            badPaths.insert(i);
+                            badPaths.insert(i.clone());
                     }
 
                 if (!badPaths.empty()) {
                     string badPathsStr;
                     for (auto & i : badPaths) {
                         badPathsStr += "\n  ";
-                        badPathsStr += i;
+                        badPathsStr += worker.store.printStorePath(i);
                     }
-                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr);
+                    throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
+                        worker.store.printStorePath(info.path), badPathsStr);
                 }
             };
 
@@ -3943,7 +3958,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
                             Strings res;
                             for (auto j = i->begin(); j != i->end(); ++j) {
                                 if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
                                 res.push_back(j->get<std::string>());
                             }
                             checks.disallowedRequisites = res;
@@ -3980,7 +3995,7 @@ Path DerivationGoal::openLogFile()
 
     if (!settings.keepLog) return "";
 
-    string baseName = baseNameOf(drvPath);
+    auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
 
     /* Create a log file. */
     Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
@@ -4019,9 +4034,7 @@ void DerivationGoal::deleteTmpDir(bool force)
         /* Don't keep temporary directories for builtins because they
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {
-            printError(
-                format("note: keeping build directory '%2%'")
-                % drvPath % tmpDir);
+            printError("note: keeping build directory '%s'", tmpDir);
             chmod(tmpDir.c_str(), 0755);
         }
         else
@@ -4100,31 +4113,31 @@ void DerivationGoal::flushLine()
 }
 
 
-PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
+StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
 {
-    PathSet result;
+    StorePathSet result;
     for (auto & i : drv->outputs) {
         if (!wantOutput(i.first, wantedOutputs)) continue;
         bool good =
             worker.store.isValidPath(i.second.path) &&
             (!checkHash || worker.pathContentsGood(i.second.path));
-        if (good == returnValid) result.insert(i.second.path);
+        if (good == returnValid) result.insert(i.second.path.clone());
     }
     return result;
 }
 
 
-Path DerivationGoal::addHashRewrite(const Path & path)
+void DerivationGoal::addHashRewrite(const StorePath & path)
 {
-    string h1 = string(path, worker.store.storeDir.size() + 1, 32);
-    string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32);
-    Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33);
-    deletePath(p);
-    assert(path.size() == p.size());
+    auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32));
+    auto p = worker.store.makeStorePath(
+        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
+        Hash(htSHA256), path.name());
+    auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32));
+    deletePath(worker.store.printStorePath(p));
     inputRewrites[h1] = h2;
     outputRewrites[h2] = h1;
-    redirectedOutputs[path] = p;
-    return p;
+    redirectedOutputs.insert_or_assign(path.clone(), std::move(p));
 }
 
 
@@ -4162,7 +4175,7 @@ class SubstitutionGoal : public Goal
 
 private:
     /* The store path that should be realised through a substitute. */
-    Path storePath;
+    StorePath storePath;
 
     /* The remaining substituters. */
     std::list<ref<Store>> subs;
@@ -4198,7 +4211,7 @@ class SubstitutionGoal : public Goal
     GoalState state;
 
 public:
-    SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair);
+    SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair);
     ~SubstitutionGoal();
 
     void timedOut() override { abort(); };
@@ -4207,7 +4220,7 @@ class SubstitutionGoal : public Goal
     {
         /* "a$" ensures substitution goals happen before derivation
            goals. */
-        return "a$" + storePathToName(storePath) + "$" + storePath;
+        return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
     }
 
     void work() override;
@@ -4224,7 +4237,7 @@ class SubstitutionGoal : public Goal
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
 
-    Path getStorePath() { return storePath; }
+    StorePath getStorePath() { return storePath.clone(); }
 
     void amDone(ExitCode result) override
     {
@@ -4233,13 +4246,13 @@ class SubstitutionGoal : public Goal
 };
 
 
-SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair)
     : Goal(worker)
+    , storePath(std::move(storePath))
     , repair(repair)
 {
-    this->storePath = storePath;
     state = &SubstitutionGoal::init;
-    name = (format("substitution of '%1%'") % storePath).str();
+    name = fmt("substitution of '%s'", worker.store.printStorePath(storePath));
     trace("created");
     maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
 }
@@ -4278,7 +4291,7 @@ void SubstitutionGoal::init()
     }
 
     if (settings.readOnlyMode)
-        throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath);
+        throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
 
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
 
@@ -4293,7 +4306,7 @@ void SubstitutionGoal::tryNext()
     if (subs.size() == 0) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
-        debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath);
+        debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
 
         /* Hack: don't indicate failure if there were no substituters.
            In that case the calling derivation should just do a
@@ -4357,7 +4370,7 @@ void SubstitutionGoal::tryNext()
         && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
     {
         printError("warning: substituter '%s' does not have a valid signature for path '%s'",
-            sub->getUri(), storePath);
+            sub->getUri(), worker.store.printStorePath(storePath));
         tryNext();
         return;
     }
@@ -4380,7 +4393,7 @@ void SubstitutionGoal::referencesValid()
     trace("all references realised");
 
     if (nrFailed > 0) {
-        debug(format("some references of path '%1%' could not be realised") % storePath);
+        debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
         amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
         return;
     }
@@ -4419,7 +4432,7 @@ void SubstitutionGoal::tryToRun()
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide = -1; });
 
-            Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()});
+            Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
             PushActivity pact(act.id);
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
@@ -4466,10 +4479,9 @@ void SubstitutionGoal::finished()
         return;
     }
 
-    worker.markContentsGood(storePath);
+    worker.markContentsGood(storePath.clone());
 
-    printMsg(lvlChatty,
-        format("substitution of path '%1%' succeeded") % storePath);
+    printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
 
     maintainRunningSubstitutions.reset();
 
@@ -4535,13 +4547,13 @@ Worker::~Worker()
 }
 
 
-GoalPtr Worker::makeDerivationGoal(const Path & path,
+GoalPtr Worker::makeDerivationGoal(const StorePath & path,
     const StringSet & wantedOutputs, BuildMode buildMode)
 {
-    GoalPtr goal = derivationGoals[path].lock();
+    GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME
     if (!goal) {
-        goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode);
-        derivationGoals[path] = goal;
+        goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode);
+        derivationGoals.insert_or_assign(path.clone(), goal);
         wakeUp(goal);
     } else
         (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
@@ -4549,21 +4561,21 @@ GoalPtr Worker::makeDerivationGoal(const Path & path,
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath,
+std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath,
     const BasicDerivation & drv, BuildMode buildMode)
 {
-    auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode);
+    auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode);
     wakeUp(goal);
     return goal;
 }
 
 
-GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
 {
-    GoalPtr goal = substitutionGoals[path].lock();
+    GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME
     if (!goal) {
-        goal = std::make_shared<SubstitutionGoal>(path, *this, repair);
-        substitutionGoals[path] = goal;
+        goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair);
+        substitutionGoals.insert_or_assign(path.clone(), goal);
         wakeUp(goal);
     }
     return goal;
@@ -4902,38 +4914,38 @@ unsigned int Worker::exitStatus()
 }
 
 
-bool Worker::pathContentsGood(const Path & path)
+bool Worker::pathContentsGood(const StorePath & path)
 {
-    std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
+    auto i = pathContentsGoodCache.find(path);
     if (i != pathContentsGoodCache.end()) return i->second;
-    printInfo(format("checking path '%1%'...") % path);
+    printInfo("checking path '%s'...", store.printStorePath(path));
     auto info = store.queryPathInfo(path);
     bool res;
-    if (!pathExists(path))
+    if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.type, path);
+        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
         Hash nullHash(htSHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }
-    pathContentsGoodCache[path] = res;
-    if (!res) printError(format("path '%1%' is corrupted or missing!") % path);
+    pathContentsGoodCache.insert_or_assign(path.clone(), res);
+    if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path));
     return res;
 }
 
 
-void Worker::markContentsGood(const Path & path)
+void Worker::markContentsGood(StorePath && path)
 {
-    pathContentsGoodCache[path] = true;
+    pathContentsGoodCache.insert_or_assign(std::move(path), true);
 }
 
 
 //////////////////////////////////////////////////////////////////////
 
 
-static void primeCache(Store & store, const PathSet & paths)
+static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
 {
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
@@ -4944,24 +4956,23 @@ static void primeCache(Store & store, const PathSet & paths)
 }
 
 
-void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
+void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
 {
     Worker worker(*this);
 
     primeCache(*this, drvPaths);
 
     Goals goals;
-    for (auto & i : drvPaths) {
-        DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i);
-        if (isDerivation(i2.first))
-            goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode));
+    for (auto & path : drvPaths) {
+        if (path.path.isDerivation())
+            goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
         else
-            goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+            goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
     }
 
     worker.run(goals);
 
-    PathSet failed;
+    StorePathSet failed;
     for (auto & i : goals) {
         if (i->getExitCode() != Goal::ecSuccess) {
             DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
@@ -4975,11 +4986,11 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 }
 
 
-BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
     Worker worker(*this);
-    auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode);
+    auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode);
 
     BuildResult result;
 
@@ -4995,12 +5006,12 @@ BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivat
 }
 
 
-void LocalStore::ensurePath(const Path & path)
+void LocalStore::ensurePath(const StorePath & path)
 {
     /* If the path is already valid, we're done. */
     if (isValidPath(path)) return;
 
-    primeCache(*this, {path});
+    primeCache(*this, {StorePathWithOutputs(path)});
 
     Worker worker(*this);
     GoalPtr goal = worker.makeSubstitutionGoal(path);
@@ -5009,11 +5020,11 @@ void LocalStore::ensurePath(const Path & path)
     worker.run(goals);
 
     if (goal->getExitCode() != Goal::ecSuccess)
-        throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path);
+        throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
 }
 
 
-void LocalStore::repairPath(const Path & path)
+void LocalStore::repairPath(const StorePath & path)
 {
     Worker worker(*this);
     GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
@@ -5024,13 +5035,13 @@ void LocalStore::repairPath(const Path & path)
     if (goal->getExitCode() != Goal::ecSuccess) {
         /* Since substituting the path didn't work, if we have a valid
            deriver, then rebuild the deriver. */
-        auto deriver = queryPathInfo(path)->deriver;
-        if (deriver != "" && isValidPath(deriver)) {
+        auto info = queryPathInfo(path);
+        if (info->deriver && isValidPath(*info->deriver)) {
             goals.clear();
-            goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair));
+            goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
             worker.run(goals);
         } else
-            throw Error(worker.exitStatus(), "cannot repair path '%s'", path);
+            throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
     }
 }
 
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index b1af3b4fc31..f6ae5d2e622 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     Path storePath = getAttr("out");
     auto mainUrl = getAttr("url");
-    bool unpack = get(drv.env, "unpack", "") == "1";
+    bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
     /* Note: have to use a fresh downloader here because we're in
        a forked process. */
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index d3c17e7721b..c53dfdd56fe 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -260,14 +260,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     switch (op) {
 
     case wopIsValidPath: {
-        /* 'readStorePath' could raise an error leading to the connection
-           being closed.  To be able to recover from an invalid path error,
-           call 'startWork' early, and do 'assertStorePath' afterwards so
-           that the 'Error' exception handler doesn't close the
-           connection.  */
-        Path path = readString(from);
-        logger->startWork();
-        store->assertStorePath(path);
+        auto path = store->parseStorePath(readString(from));
+        logger->startWork();
         bool result = store->isValidPath(path);
         logger->stopWork();
         to << result;
@@ -275,34 +269,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
+        auto paths = readStorePaths<StorePathSet>(*store, from);
         logger->startWork();
-        PathSet res = store->queryValidPaths(paths);
+        auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        to << res;
+        writeStorePaths(*store, to, res);
         break;
     }
 
     case wopHasSubstitutes: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        PathSet res = store->querySubstitutablePaths({path});
+        StorePathSet paths; // FIXME
+        paths.insert(path.clone());
+        auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        to << (res.find(path) != res.end());
+        to << (res.count(path) != 0);
         break;
     }
 
     case wopQuerySubstitutablePaths: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
+        auto paths = readStorePaths<StorePathSet>(*store, from);
         logger->startWork();
-        PathSet res = store->querySubstitutablePaths(paths);
+        auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        to << res;
+        writeStorePaths(*store, to, res);
         break;
     }
 
     case wopQueryPathHash: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         auto hash = store->queryPathInfo(path)->narHash;
         logger->stopWork();
@@ -314,23 +310,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryReferrers:
     case wopQueryValidDerivers:
     case wopQueryDerivationOutputs: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        PathSet paths;
+        StorePathSet paths;
         if (op == wopQueryReferences)
-            paths = store->queryPathInfo(path)->references;
+            for (auto & i : store->queryPathInfo(path)->references)
+                paths.insert(i.clone());
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
         else if (op == wopQueryValidDerivers)
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        to << paths;
+        writeStorePaths(*store, to, paths);
         break;
     }
 
     case wopQueryDerivationOutputNames: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         StringSet names;
         names = store->queryDerivationOutputNames(path);
@@ -340,20 +337,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryDeriver: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        auto deriver = store->queryPathInfo(path)->deriver;
+        auto info = store->queryPathInfo(path);
         logger->stopWork();
-        to << deriver;
+        to << (info->deriver ? store->printStorePath(*info->deriver) : "");
         break;
     }
 
     case wopQueryPathFromHashPart: {
-        string hashPart = readString(from);
+        auto hashPart = readString(from);
         logger->startWork();
-        Path path = store->queryPathFromHashPart(hashPart);
+        auto path = store->queryPathFromHashPart(hashPart);
         logger->stopWork();
-        to << path;
+        to << (path ? store->printStorePath(*path) : "");
         break;
     }
 
@@ -383,26 +380,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
         logger->stopWork();
 
-        to << path;
+        to << store->printStorePath(path);
         break;
     }
 
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        PathSet refs = readStorePaths<PathSet>(*store, from);
+        auto refs = readStorePaths<StorePathSet>(*store, from);
         logger->startWork();
-        Path path = store->addTextToStore(suffix, s, refs, NoRepair);
+        auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
-        to << path;
+        to << store->printStorePath(path);
         break;
     }
 
     case wopExportPath: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         readInt(from); // obsolete
         logger->startWork();
         TunnelSink sink(to);
@@ -415,15 +412,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        Paths paths = store->importPaths(source, nullptr,
+        auto paths = store->importPaths(source, nullptr,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
-        to << paths;
+        Strings paths2;
+        for (auto & i : paths) paths2.push_back(store->printStorePath(i));
+        to << paths2;
         break;
     }
 
     case wopBuildPaths: {
-        PathSet drvs = readStorePaths<PathSet>(*store, from);
+        std::vector<StorePathWithOutputs> drvs;
+        for (auto & s : readStrings<Strings>(from))
+            drvs.push_back(store->parseDrvPathWithOutputs(s));
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -441,7 +442,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopBuildDerivation: {
-        Path drvPath = readStorePath(*store, from);
+        auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
         readDerivation(from, *store, drv);
         BuildMode buildMode = (BuildMode) readInt(from);
@@ -455,7 +456,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopEnsurePath: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         store->ensurePath(path);
         logger->stopWork();
@@ -464,7 +465,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddTempRoot: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         store->addTempRoot(path);
         logger->stopWork();
@@ -502,7 +503,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         for (auto & [target, links] : roots)
             for (auto & link : links)
-                to << link << target;
+                to << link << store->printStorePath(target);
 
         break;
     }
@@ -510,7 +511,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = readStorePaths<PathSet>(*store, from);
+        options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -568,44 +569,52 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePathInfo: {
-        Path path = absPath(readString(from));
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos({path}, infos);
+        StorePathSet paths;
+        paths.insert(path.clone()); // FIXME
+        store->querySubstitutablePathInfos(paths, infos);
         logger->stopWork();
-        SubstitutablePathInfos::iterator i = infos.find(path);
+        auto i = infos.find(path);
         if (i == infos.end())
             to << 0;
         else {
-            to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize;
+            to << 1
+               << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
+            writeStorePaths(*store, to, i->second.references);
+            to << i->second.downloadSize
+               << i->second.narSize;
         }
         break;
     }
 
     case wopQuerySubstitutablePathInfos: {
-        PathSet paths = readStorePaths<PathSet>(*store, from);
+        auto paths = readStorePaths<StorePathSet>(*store, from);
         logger->startWork();
         SubstitutablePathInfos infos;
         store->querySubstitutablePathInfos(paths, infos);
         logger->stopWork();
         to << infos.size();
         for (auto & i : infos) {
-            to << i.first << i.second.deriver << i.second.references
-               << i.second.downloadSize << i.second.narSize;
+            to << store->printStorePath(i.first)
+               << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
+            writeStorePaths(*store, to, i.second.references);
+            to << i.second.downloadSize << i.second.narSize;
         }
         break;
     }
 
     case wopQueryAllValidPaths: {
         logger->startWork();
-        PathSet paths = store->queryAllValidPaths();
+        auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        to << paths;
+        writeStorePaths(*store, to, paths);
         break;
     }
 
     case wopQueryPathInfo: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         std::shared_ptr<const ValidPathInfo> info;
         logger->startWork();
         try {
@@ -617,8 +626,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (info) {
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
-            to << info->deriver << info->narHash.to_string(Base16, false) << info->references
-               << info->registrationTime << info->narSize;
+            to << (info->deriver ? store->printStorePath(*info->deriver) : "")
+               << info->narHash.to_string(Base16, false);
+            writeStorePaths(*store, to, info->references);
+            to << info->registrationTime << info->narSize;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
                    << info->sigs
@@ -651,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddSignatures: {
-        Path path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         StringSet sigs = readStrings<StringSet>(from);
         logger->startWork();
         if (!trusted)
@@ -663,22 +674,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopNarFromPath: {
-        auto path = readStorePath(*store, from);
+        auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(path, to);
+        dumpPath(store->printStorePath(path), to);
         break;
     }
 
     case wopAddToStoreNar: {
         bool repair, dontCheckSigs;
-        ValidPathInfo info;
-        info.path = readStorePath(*store, from);
-        from >> info.deriver;
-        if (!info.deriver.empty())
-            store->assertStorePath(info.deriver);
+        ValidPathInfo info(store->parseStorePath(readString(from)));
+        auto deriver = readString(from);
+        if (deriver != "")
+            info.deriver = store->parseStorePath(deriver);
         info.narHash = Hash(readString(from), htSHA256);
-        info.references = readStorePaths<PathSet>(*store, from);
+        info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         from >> info.ca >> repair >> dontCheckSigs;
@@ -709,13 +719,18 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryMissing: {
-        PathSet targets = readStorePaths<PathSet>(*store, from);
+        std::vector<StorePathWithOutputs> targets;
+        for (auto & s : readStrings<Strings>(from))
+            targets.push_back(store->parseDrvPathWithOutputs(s));
         logger->startWork();
-        PathSet willBuild, willSubstitute, unknown;
+        StorePathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        to << willBuild << willSubstitute << unknown << downloadSize << narSize;
+        writeStorePaths(*store, to, willBuild);
+        writeStorePaths(*store, to, willSubstitute);
+        writeStorePaths(*store, to, unknown);
+        to << downloadSize << narSize;
         break;
     }
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 23fcfb281a7..726e34479f8 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -21,17 +21,39 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
 
     HashType hashType = parseHashType(algo);
     if (hashType == htUnknown)
-        throw Error(format("unknown hash algorithm '%1%'") % algo);
+        throw Error("unknown hash algorithm '%s'", algo);
 
     hash = Hash(this->hash, hashType);
 }
 
 
-Path BasicDerivation::findOutput(const string & id) const
+BasicDerivation::BasicDerivation(const BasicDerivation & other)
+    : platform(other.platform)
+    , builder(other.builder)
+    , args(other.args)
+    , env(other.env)
+{
+    for (auto & i : other.outputs)
+        outputs.insert_or_assign(i.first,
+            DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
+    for (auto & i : other.inputSrcs)
+        inputSrcs.insert(i.clone());
+}
+
+
+Derivation::Derivation(const Derivation & other)
+    : BasicDerivation(other)
+{
+    for (auto & i : other.inputDrvs)
+        inputDrvs.insert_or_assign(i.first.clone(), i.second);
+}
+
+
+const StorePath & BasicDerivation::findOutput(const string & id) const
 {
     auto i = outputs.find(id);
     if (i == outputs.end())
-        throw Error(format("derivation has no output '%1%'") % id);
+        throw Error("derivation has no output '%s'", id);
     return i->second.path;
 }
 
@@ -42,18 +64,17 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-Path writeDerivation(ref<Store> store,
+StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, const string & name, RepairFlag repair)
 {
-    PathSet references;
-    references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
+    auto references = cloneStorePathSet(drv.inputSrcs);
     for (auto & i : drv.inputDrvs)
-        references.insert(i.first);
+        references.insert(i.first.clone());
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
     string suffix = name + drvExtension;
-    string contents = drv.unparse();
+    string contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references, repair);
@@ -121,7 +142,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static Derivation parseDerivation(const string & s)
+static Derivation parseDerivation(const Store & store, const string & s)
 {
     Derivation drv;
     istringstream_nocopy str(s);
@@ -129,13 +150,12 @@ static Derivation parseDerivation(const string & s)
 
     /* Parse the list of outputs. */
     while (!endOfList(str)) {
-        DerivationOutput out;
-        expect(str, "("); string id = parseString(str);
-        expect(str, ","); out.path = parsePath(str);
-        expect(str, ","); out.hashAlgo = parseString(str);
-        expect(str, ","); out.hash = parseString(str);
+        expect(str, "("); std::string id = parseString(str);
+        expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+        expect(str, ","); auto hashAlgo = parseString(str);
+        expect(str, ","); auto hash = parseString(str);
         expect(str, ")");
-        drv.outputs[id] = out;
+        drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
     }
 
     /* Parse the list of input derivations. */
@@ -144,11 +164,11 @@ static Derivation parseDerivation(const string & s)
         expect(str, "(");
         Path drvPath = parsePath(str);
         expect(str, ",[");
-        drv.inputDrvs[drvPath] = parseStrings(str, false);
+        drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
         expect(str, ")");
     }
 
-    expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
+    expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
     expect(str, ","); drv.platform = parseString(str);
     expect(str, ","); drv.builder = parseString(str);
 
@@ -171,25 +191,24 @@ static Derivation parseDerivation(const string & s)
 }
 
 
-Derivation readDerivation(const Path & drvPath)
+Derivation readDerivation(const Store & store, const Path & drvPath)
 {
     try {
-        return parseDerivation(readFile(drvPath));
+        return parseDerivation(store, readFile(drvPath));
     } catch (FormatError & e) {
         throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
     }
 }
 
 
-Derivation Store::derivationFromPath(const Path & drvPath)
+Derivation Store::derivationFromPath(const StorePath & drvPath)
 {
-    assertStorePath(drvPath);
     ensurePath(drvPath);
     auto accessor = getFSAccessor();
     try {
-        return parseDerivation(accessor->readFile(drvPath));
+        return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
+        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
     }
 }
 
@@ -220,33 +239,56 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
 }
 
 
-string Derivation::unparse() const
+string Derivation::unparse(const Store & store, bool maskOutputs,
+    std::map<std::string, StringSet> * actualInputs) const
 {
     string s;
     s.reserve(65536);
     s += "Derive([";
 
-    bool first = true;
-    for (auto & i : outputs) {
-        if (first) first = false; else s += ',';
-        s += '('; printString(s, i.first);
-        s += ','; printString(s, i.second.path);
-        s += ','; printString(s, i.second.hashAlgo);
-        s += ','; printString(s, i.second.hash);
-        s += ')';
+    StringSet maskedOutputs;
+
+    if (maskOutputs) {
+        bool first = true;
+        maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " ");
+        for (auto & i : maskedOutputs) {
+            if (first) first = false; else s += ',';
+            s += '('; printString(s, i);
+            s += ",\"\",\"\",\"\")";
+        }
+    } else {
+        bool first = true;
+        for (auto & i : outputs) {
+            if (first) first = false; else s += ',';
+            s += '('; printString(s, i.first);
+            s += ','; printString(s, store.printStorePath(i.second.path));
+            s += ','; printString(s, i.second.hashAlgo);
+            s += ','; printString(s, i.second.hash);
+            s += ')';
+        }
     }
 
     s += "],[";
-    first = true;
-    for (auto & i : inputDrvs) {
-        if (first) first = false; else s += ',';
-        s += '('; printString(s, i.first);
-        s += ','; printStrings(s, i.second.begin(), i.second.end());
-        s += ')';
+    bool first = true;
+    if (actualInputs) {
+        for (auto & i : *actualInputs) {
+            if (first) first = false; else s += ',';
+            s += '('; printString(s, i.first);
+            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += ')';
+        }
+    } else {
+        for (auto & i : inputDrvs) {
+            if (first) first = false; else s += ',';
+            s += '('; printString(s, store.printStorePath(i.first));
+            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += ')';
+        }
     }
 
     s += "],";
-    printStrings(s, inputSrcs.begin(), inputSrcs.end());
+    auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
+    printStrings(s, paths.begin(), paths.end());
 
     s += ','; printString(s, platform);
     s += ','; printString(s, builder);
@@ -257,7 +299,7 @@ string Derivation::unparse() const
     for (auto & i : env) {
         if (first) first = false; else s += ',';
         s += '('; printString(s, i.first);
-        s += ','; printString(s, i.second);
+        s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second);
         s += ')';
     }
 
@@ -267,6 +309,7 @@ string Derivation::unparse() const
 }
 
 
+// FIXME: remove
 bool isDerivation(const string & fileName)
 {
     return hasSuffix(fileName, drvExtension);
@@ -304,7 +347,7 @@ DrvHashes drvHashes;
    paths have been replaced by the result of a recursive call to this
    function, and that for fixed-output derivations we return a hash of
    its output path. */
-Hash hashDerivationModulo(Store & store, Derivation drv)
+Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
     /* Return a fixed hash for fixed-output derivations. */
     if (drv.isFixedOutput()) {
@@ -312,42 +355,41 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
         return hashString(htSHA256, "fixed:out:"
             + i->second.hashAlgo + ":"
             + i->second.hash + ":"
-            + i->second.path);
+            + store.printStorePath(i->second.path));
     }
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function.*/
-    DerivationInputs inputs2;
+    std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
-        Hash h = drvHashes[i.first];
-        if (!h) {
+        auto h = drvHashes.find(i.first);
+        if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
-            Derivation drv2 = readDerivation(store.toRealPath(i.first));
-            h = hashDerivationModulo(store, drv2);
-            drvHashes[i.first] = h;
+            h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
+                readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
         }
-        inputs2[h.to_string(Base16, false)] = i.second;
+        inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
-    drv.inputDrvs = inputs2;
 
-    return hashString(htSHA256, drv.unparse());
+    return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
 }
 
 
-DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
+StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s)
 {
     size_t n = s.find("!");
     return n == s.npos
-        ? DrvPathWithOutputs(s, std::set<string>())
-        : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
+        ? StorePathWithOutputs{parseStorePath(s), std::set<string>()}
+        : StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)),
+            tokenizeString<std::set<string>>(string(s, n + 1), ",")};
 }
 
 
-Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
+std::string StorePathWithOutputs::to_string(const Store & store) const
 {
     return outputs.empty()
-        ? drvPath
-        : drvPath + "!" + concatStringsSep(",", outputs);
+        ? store.printStorePath(path)
+        : store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
 }
 
 
@@ -357,28 +399,28 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
-PathSet BasicDerivation::outputPaths() const
+StorePathSet BasicDerivation::outputPaths() const
 {
-    PathSet paths;
+    StorePathSet paths;
     for (auto & i : outputs)
-        paths.insert(i.second.path);
+        paths.insert(i.second.path.clone());
     return paths;
 }
 
 
-Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 {
     drv.outputs.clear();
     auto nr = readNum<size_t>(in);
     for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
-        DerivationOutput o;
-        in >> o.path >> o.hashAlgo >> o.hash;
-        store.assertStorePath(o.path);
-        drv.outputs[name] = o;
+        auto path = store.parseStorePath(readString(in));
+        auto hashAlgo = readString(in);
+        auto hash = readString(in);
+        drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
     }
 
-    drv.inputSrcs = readStorePaths<PathSet>(store, in);
+    drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -393,16 +435,16 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
 }
 
 
-Sink & operator << (Sink & out, const BasicDerivation & drv)
+void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs)
-        out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
-    out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
+        out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
+    writeStorePaths(store, out, drv.inputSrcs);
+    out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
         out << i.first << i.second;
-    return out;
 }
 
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 8e02c9bc57c..c2df66229ad 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -10,26 +10,18 @@
 namespace nix {
 
 
-/* Extension of derivations in the Nix store. */
-const string drvExtension = ".drv";
-
-
 /* Abstract syntax of derivations. */
 
 struct DerivationOutput
 {
-    Path path;
-    string hashAlgo; /* hash used for expected hash computation */
-    string hash; /* expected hash, may be null */
-    DerivationOutput()
-    {
-    }
-    DerivationOutput(Path path, string hashAlgo, string hash)
-    {
-        this->path = path;
-        this->hashAlgo = hashAlgo;
-        this->hash = hash;
-    }
+    StorePath path;
+    std::string hashAlgo; /* hash used for expected hash computation */
+    std::string hash; /* expected hash, may be null */
+    DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
+        : path(std::move(path))
+        , hashAlgo(std::move(hashAlgo))
+        , hash(std::move(hash))
+    { }
     void parseHashInfo(bool & recursive, Hash & hash) const;
 };
 
@@ -37,24 +29,26 @@ typedef std::map<string, DerivationOutput> DerivationOutputs;
 
 /* For inputs that are sub-derivations, we specify exactly which
    output IDs we are interested in. */
-typedef std::map<Path, StringSet> DerivationInputs;
+typedef std::map<StorePath, StringSet> DerivationInputs;
 
 typedef std::map<string, string> StringPairs;
 
 struct BasicDerivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
-    PathSet inputSrcs; /* inputs that are sources */
+    StorePathSet inputSrcs; /* inputs that are sources */
     string platform;
     Path builder;
     Strings args;
     StringPairs env;
 
+    BasicDerivation() { }
+    explicit BasicDerivation(const BasicDerivation & other);
     virtual ~BasicDerivation() { };
 
     /* Return the path corresponding to the output identifier `id' in
        the given derivation. */
-    Path findOutput(const string & id) const;
+    const StorePath & findOutput(const std::string & id) const;
 
     bool isBuiltin() const;
 
@@ -62,7 +56,7 @@ struct BasicDerivation
     bool isFixedOutput() const;
 
     /* Return the output paths of a derivation. */
-    PathSet outputPaths() const;
+    StorePathSet outputPaths() const;
 
 };
 
@@ -71,7 +65,12 @@ struct Derivation : BasicDerivation
     DerivationInputs inputDrvs; /* inputs that are sub-derivations */
 
     /* Print a derivation. */
-    std::string unparse() const;
+    std::string unparse(const Store & store, bool maskOutputs,
+        std::map<std::string, StringSet> * actualInputs = nullptr) const;
+
+    Derivation() { }
+    Derivation(Derivation && other) = default;
+    explicit Derivation(const Derivation & other);
 };
 
 
@@ -79,38 +78,29 @@ class Store;
 
 
 /* Write a derivation to the Nix store, and return its path. */
-Path writeDerivation(ref<Store> store,
+StorePath writeDerivation(ref<Store> store,
     const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
-Derivation readDerivation(const Path & drvPath);
+Derivation readDerivation(const Store & store, const Path & drvPath);
 
-/* Check whether a file name ends with the extension for
-   derivations. */
+// FIXME: remove
 bool isDerivation(const string & fileName);
 
-Hash hashDerivationModulo(Store & store, Derivation drv);
+Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
 
 /* Memoisation of hashDerivationModulo(). */
-typedef std::map<Path, Hash> DrvHashes;
+typedef std::map<StorePath, Hash> DrvHashes;
 
 extern DrvHashes drvHashes; // FIXME: global, not thread-safe
 
-/* Split a string specifying a derivation and a set of outputs
-   (/nix/store/hash-foo!out1,out2,...) into the derivation path and
-   the outputs. */
-typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
-DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
-
-Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
-
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
 struct Source;
 struct Sink;
 
-Source & readDerivation(Source & in, Store & store, BasicDerivation & drv);
-Sink & operator << (Sink & out, const BasicDerivation & drv);
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
+void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
 
 std::string hashPlaceholder(const std::string & outputName);
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 61e88c5c1ef..af058ba420a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -650,10 +650,10 @@ struct CurlDownloader : public Downloader
 #ifdef ENABLE_S3
                 auto [bucketName, key, params] = parseS3Uri(request.uri);
 
-                std::string profile = get(params, "profile", "");
-                std::string region = get(params, "region", Aws::Region::US_EAST_1);
-                std::string scheme = get(params, "scheme", "");
-                std::string endpoint = get(params, "endpoint", "");
+                std::string profile = get(params, "profile").value_or("");
+                std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1);
+                std::string scheme = get(params, "scheme").value_or("");
+                std::string endpoint = get(params, "endpoint").value_or("");
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
@@ -811,13 +811,13 @@ CachedDownloadResult Downloader::downloadCached(
         if (p != string::npos) name = string(url, p + 1);
     }
 
-    Path expectedStorePath;
+    std::optional<StorePath> expectedStorePath;
     if (request.expectedHash) {
         expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
-        if (store->isValidPath(expectedStorePath)) {
+        if (store->isValidPath(*expectedStorePath)) {
             CachedDownloadResult result;
-            result.storePath = expectedStorePath;
-            result.path = store->toRealPath(expectedStorePath);
+            result.storePath = store->printStorePath(*expectedStorePath);
+            result.path = store->toRealPath(result.storePath);
             return result;
         }
     }
@@ -832,7 +832,7 @@ CachedDownloadResult Downloader::downloadCached(
 
     PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
 
-    Path storePath;
+    std::optional<StorePath> storePath;
 
     string expectedETag;
 
@@ -841,9 +841,10 @@ CachedDownloadResult Downloader::downloadCached(
     CachedDownloadResult result;
 
     if (pathExists(fileLink) && pathExists(dataFile)) {
-        storePath = readLink(fileLink);
-        store->addTempRoot(storePath);
-        if (store->isValidPath(storePath)) {
+        storePath = store->parseStorePath(readLink(fileLink));
+        // FIXME
+        store->addTempRoot(*storePath);
+        if (store->isValidPath(*storePath)) {
             auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
             if (ss.size() >= 3 && ss[0] == url) {
                 time_t lastChecked;
@@ -857,7 +858,7 @@ CachedDownloadResult Downloader::downloadCached(
                 }
             }
         } else
-            storePath = "";
+            storePath.reset();
     }
 
     if (!skip) {
@@ -870,65 +871,65 @@ CachedDownloadResult Downloader::downloadCached(
             result.etag = res.etag;
 
             if (!res.cached) {
-                ValidPathInfo info;
                 StringSink sink;
                 dumpString(*res.data, sink);
                 Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
-                info.path = store->makeFixedOutputPath(false, hash, name);
+                ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
                 info.narHash = hashString(htSHA256, *sink.s);
                 info.narSize = sink.s->size();
                 info.ca = makeFixedOutputCA(false, hash);
                 store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
-                storePath = info.path;
+                storePath = info.path.clone();
             }
 
-            assert(!storePath.empty());
-            replaceSymlink(storePath, fileLink);
+            assert(storePath);
+            replaceSymlink(store->printStorePath(*storePath), fileLink);
 
             writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
         } catch (DownloadError & e) {
-            if (storePath.empty()) throw;
+            if (!storePath) throw;
             warn("warning: %s; using cached result", e.msg());
             result.etag = expectedETag;
         }
     }
 
     if (request.unpack) {
-        Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
+        Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
         PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
-        Path unpackedStorePath;
+        std::optional<StorePath> unpackedStorePath;
         if (pathExists(unpackedLink)) {
-            unpackedStorePath = readLink(unpackedLink);
-            store->addTempRoot(unpackedStorePath);
-            if (!store->isValidPath(unpackedStorePath))
-                unpackedStorePath = "";
+            unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
+            // FIXME
+            store->addTempRoot(*unpackedStorePath);
+            if (!store->isValidPath(*unpackedStorePath))
+                unpackedStorePath.reset();
         }
-        if (unpackedStorePath.empty()) {
+        if (!unpackedStorePath) {
             printInfo("unpacking '%s'...", url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
+            unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url)));
             auto members = readDirectory(tmpDir);
             if (members.size() != 1)
                 throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
             auto topDir = tmpDir + "/" + members.begin()->name;
             unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
-        replaceSymlink(unpackedStorePath, unpackedLink);
-        storePath = unpackedStorePath;
+        replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
+        storePath = std::move(*unpackedStorePath);
     }
 
-    if (expectedStorePath != "" && storePath != expectedStorePath) {
+    if (expectedStorePath && *storePath != *expectedStorePath) {
         unsigned int statusCode = 102;
         Hash gotHash = request.unpack
-            ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
-            : hashFile(request.expectedHash.type, store->toRealPath(storePath));
+            ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
+            : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
         throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
             url, request.expectedHash.to_string(), gotHash.to_string());
     }
 
-    result.storePath = storePath;
-    result.path = store->toRealPath(storePath);
+    result.storePath = store->printStorePath(*storePath);
+    result.path = store->toRealPath(result.storePath);
     return result;
 }
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index ef3fea7c838..4692d1a7be3 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -24,9 +24,9 @@ struct HashAndWriteSink : Sink
     }
 };
 
-void Store::exportPaths(const Paths & paths, Sink & sink)
+void Store::exportPaths(const StorePathSet & paths, Sink & sink)
 {
-    Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
+    auto sorted = topoSortPaths(paths);
     std::reverse(sorted.begin(), sorted.end());
 
     std::string doneLabel("paths exported");
@@ -42,7 +42,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
     sink << 0;
 }
 
-void Store::exportPath(const Path & path, Sink & sink)
+void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
@@ -55,15 +55,21 @@ void Store::exportPath(const Path & path, Sink & sink)
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
-        throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path
-            % info->narHash.to_string() % hash.to_string());
-
-    hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
+        throw Error("hash of path '%s' has changed from '%s' to '%s'!",
+            printStorePath(path), info->narHash.to_string(), hash.to_string());
+
+    hashAndWriteSink
+        << exportMagic
+        << printStorePath(path);
+    writeStorePaths(*this, hashAndWriteSink, info->references);
+    hashAndWriteSink
+        << (info->deriver ? printStorePath(*info->deriver) : "")
+        << 0;
 }
 
-Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
 {
-    Paths res;
+    StorePaths res;
     while (true) {
         auto n = readNum<uint64_t>(source);
         if (n == 0) break;
@@ -77,16 +83,15 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         if (magic != exportMagic)
             throw Error("Nix archive cannot be imported; wrong format");
 
-        ValidPathInfo info;
-
-        info.path = readStorePath(*this, source);
+        ValidPathInfo info(parseStorePath(readString(source)));
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        info.references = readStorePaths<PathSet>(*this, source);
+        info.references = readStorePaths<StorePathSet>(*this, source);
 
-        info.deriver = readString(source);
-        if (info.deriver != "") assertStorePath(info.deriver);
+        auto deriver = readString(source);
+        if (deriver != "")
+            info.deriver = parseStorePath(deriver);
 
         info.narHash = hashString(htSHA256, *tee.source.data);
         info.narSize = tee.source.data->size();
@@ -97,7 +102,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 
         addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
 
-        res.push_back(info.path);
+        res.push_back(info.path.clone());
     }
 
     return res;
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 842980fb7cd..ed81186af16 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -85,12 +85,10 @@ void LocalStore::addIndirectRoot(const Path & path)
 }
 
 
-Path LocalFSStore::addPermRoot(const Path & _storePath,
+Path LocalFSStore::addPermRoot(const StorePath & storePath,
     const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
 {
-    Path storePath(canonPath(_storePath));
     Path gcRoot(canonPath(_gcRoot));
-    assertStorePath(storePath);
 
     if (isInStore(gcRoot))
         throw Error(format(
@@ -102,7 +100,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
             throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
-        makeSymlink(gcRoot, storePath);
+        makeSymlink(gcRoot, printStorePath(storePath));
         addIndirectRoot(gcRoot);
     }
 
@@ -117,10 +115,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
                     % gcRoot % rootsDir);
         }
 
-        if (baseNameOf(gcRoot) == baseNameOf(storePath))
+        if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
             writeFile(gcRoot, "");
         else
-            makeSymlink(gcRoot, storePath);
+            makeSymlink(gcRoot, printStorePath(storePath));
     }
 
     /* Check that the root can be found by the garbage collector.
@@ -129,13 +127,12 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
        check if the root is in a directory in or linked from the
        gcroots directory. */
     if (settings.checkRootReachability) {
-        Roots roots = findRoots(false);
-        if (roots[storePath].count(gcRoot) == 0)
+        auto roots = findRoots(false);
+        if (roots[storePath.clone()].count(gcRoot) == 0)
             printError(
-                format(
-                    "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                    "therefore, '%2%' might be removed by the garbage collector")
-                % gcRoot % storePath);
+                "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+                "therefore, '%2%' might be removed by the garbage collector",
+                gcRoot, printStorePath(storePath));
     }
 
     /* Grab the global GC root, causing us to block while a GC is in
@@ -147,7 +144,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
 }
 
 
-void LocalStore::addTempRoot(const Path & path)
+void LocalStore::addTempRoot(const StorePath & path)
 {
     auto state(_state.lock());
 
@@ -188,7 +185,7 @@ void LocalStore::addTempRoot(const Path & path)
     debug(format("acquiring write lock on '%1%'") % fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltWrite, true);
 
-    string s = path + '\0';
+    string s = printStorePath(path) + '\0';
     writeFull(state->fdTempRoots.get(), s);
 
     /* Downgrade to a read lock. */
@@ -246,8 +243,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         while ((end = contents.find((char) 0, pos)) != string::npos) {
             Path root(contents, pos, end - pos);
             debug("got temporary root '%s'", root);
-            assertStorePath(root);
-            tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
+            tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
 
@@ -260,10 +256,11 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
         Path storePath = toStorePath(target);
-        if (isStorePath(storePath) && isValidPath(storePath))
-            roots[storePath].emplace(path);
+        // FIXME
+        if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
+            roots[parseStorePath(storePath)].emplace(path);
         else
-            printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
+            printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath);
     };
 
     try {
@@ -299,9 +296,10 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         }
 
         else if (type == DT_REG) {
-            Path storePath = storeDir + "/" + baseNameOf(path);
-            if (isStorePath(storePath) && isValidPath(storePath))
-                roots[storePath].emplace(path);
+            Path storePath = storeDir + "/" + std::string(baseNameOf(path));
+            // FIXME
+            if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
+                roots[parseStorePath(storePath)].emplace(path);
         }
 
     }
@@ -309,7 +307,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
     catch (SysError & e) {
         /* We only ignore permanent failures. */
         if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
-            printInfo(format("cannot read potential root '%1%'") % path);
+            printInfo("cannot read potential root '%1%'", path);
         else
             throw;
     }
@@ -340,7 +338,9 @@ Roots LocalStore::findRoots(bool censor)
     return roots;
 }
 
-static void readProcLink(const string & file, Roots & roots)
+typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
+
+static void readProcLink(const string & file, UncheckedRoots & roots)
 {
     /* 64 is the starting buffer size gnu readlink uses... */
     auto bufsiz = ssize_t{64};
@@ -369,7 +369,7 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
-static void readFileRoots(const char * path, Roots & roots)
+static void readFileRoots(const char * path, UncheckedRoots & roots)
 {
     try {
         roots[readFile(path)].emplace(path);
@@ -381,7 +381,7 @@ static void readFileRoots(const char * path, Roots & roots)
 
 void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
-    Roots unchecked;
+    UncheckedRoots unchecked;
 
     auto procDir = AutoCloseDir{opendir("/proc")};
     if (procDir) {
@@ -466,16 +466,16 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 #endif
 
     for (auto & [target, links] : unchecked) {
-        if (isInStore(target)) {
-            Path path = toStorePath(target);
-            if (isStorePath(path) && isValidPath(path)) {
-                debug(format("got additional root '%1%'") % path);
-                if (censor)
-                    roots[path].insert(censored);
-                else
-                    roots[path].insert(links.begin(), links.end());
-            }
-        }
+        if (!isInStore(target)) continue;
+        Path pathS = toStorePath(target);
+        if (!isStorePath(pathS)) continue;
+        auto path = parseStorePath(pathS);
+        if (!isValidPath(path)) continue;
+        debug("got additional root '%1%'", pathS);
+        if (censor)
+            roots[path.clone()].insert(censored);
+        else
+            roots[path.clone()].insert(links.begin(), links.end());
     }
 }
 
@@ -485,18 +485,19 @@ struct GCLimitReached { };
 
 struct LocalStore::GCState
 {
-    GCOptions options;
+    const GCOptions & options;
     GCResults & results;
-    PathSet roots;
-    PathSet tempRoots;
-    PathSet dead;
-    PathSet alive;
+    StorePathSet roots;
+    StorePathSet tempRoots;
+    StorePathSet dead;
+    StorePathSet alive;
     bool gcKeepOutputs;
     bool gcKeepDerivations;
     unsigned long long bytesInvalidated;
     bool moveToTrash = true;
     bool shouldDelete;
-    GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
+    GCState(const GCOptions & options, GCResults & results)
+        : options(options), results(results), bytesInvalidated(0) { }
 };
 
 
@@ -504,7 +505,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
     const Path & path, const string & suffix)
 {
     return hasSuffix(path, suffix)
-        && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
+        && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
 }
 
 
@@ -522,16 +523,17 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 
     unsigned long long size = 0;
 
-    if (isStorePath(path) && isValidPath(path)) {
-        PathSet referrers;
-        queryReferrers(path, referrers);
+    // FIXME
+    if (isStorePath(path) && isValidPath(parseStorePath(path))) {
+        StorePathSet referrers;
+        queryReferrers(parseStorePath(path), referrers);
         for (auto & i : referrers)
-            if (i != path) deletePathRecursive(state, i);
-        size = queryPathInfo(path)->narSize;
-        invalidatePathChecked(path);
+            if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
+        size = queryPathInfo(parseStorePath(path))->narSize;
+        invalidatePathChecked(parseStorePath(path));
     }
 
-    Path realPath = realStoreDir + "/" + baseNameOf(path);
+    Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
 
     struct stat st;
     if (lstat(realPath.c_str(), &st)) {
@@ -555,7 +557,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         try {
             if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
                 throw SysError(format("making '%1%' writable") % realPath);
-            Path tmp = trashDir + "/" + baseNameOf(path);
+            Path tmp = trashDir + "/" + std::string(baseNameOf(path));
             if (rename(realPath.c_str(), tmp.c_str()))
                 throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
             state.bytesInvalidated += size;
@@ -575,7 +577,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 }
 
 
-bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
+bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
 {
     if (visited.count(path)) return false;
 
@@ -584,41 +586,41 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
     if (state.dead.count(path)) return false;
 
     if (state.roots.count(path)) {
-        debug(format("cannot delete '%1%' because it's a root") % path);
-        state.alive.insert(path);
+        debug("cannot delete '%1%' because it's a root", printStorePath(path));
+        state.alive.insert(path.clone());
         return true;
     }
 
-    visited.insert(path);
+    visited.insert(path.clone());
 
-    if (!isStorePath(path) || !isValidPath(path)) return false;
+    //FIXME
+    if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false;
 
-    PathSet incoming;
+    StorePathSet incoming;
 
     /* Don't delete this path if any of its referrers are alive. */
     queryReferrers(path, incoming);
 
     /* If keep-derivations is set and this is a derivation, then
        don't delete the derivation if any of the outputs are alive. */
-    if (state.gcKeepDerivations && isDerivation(path)) {
-        PathSet outputs = queryDerivationOutputs(path);
-        for (auto & i : outputs)
+    if (state.gcKeepDerivations && path.isDerivation()) {
+        for (auto & i : queryDerivationOutputs(path))
             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i);
+                incoming.insert(i.clone());
     }
 
     /* If keep-outputs is set, then don't delete this path if there
        are derivers of this path that are not garbage. */
     if (state.gcKeepOutputs) {
-        PathSet derivers = queryValidDerivers(path);
+        auto derivers = queryValidDerivers(path);
         for (auto & i : derivers)
-            incoming.insert(i);
+            incoming.insert(i.clone());
     }
 
     for (auto & i : incoming)
         if (i != path)
             if (canReachRoot(state, visited, i)) {
-                state.alive.insert(path);
+                state.alive.insert(path.clone());
                 return true;
             }
 
@@ -630,12 +632,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 {
     checkInterrupt();
 
-    auto realPath = realStoreDir + "/" + baseNameOf(path);
+    auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
     if (realPath == linksDir || realPath == trashDir) return;
 
     //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
 
-    if (!isStorePath(path) || !isValidPath(path)) {
+    // FIXME
+    if (!isStorePath(path) || !isValidPath(parseStorePath(path))) {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
         if (isActiveTempFile(state, path, ".lock")) return;
@@ -650,16 +653,17 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
         if (isActiveTempFile(state, path, ".check")) return;
     }
 
-    PathSet visited;
+    StorePathSet visited;
 
-    if (canReachRoot(state, visited, path)) {
-        debug(format("cannot delete '%1%' because it's still reachable") % path);
+    if (canReachRoot(state, visited, parseStorePath(path))) {
+        debug("cannot delete '%s' because it's still reachable", path);
     } else {
         /* No path we visited was a root, so everything is garbage.
            But we only delete ‘path’ and its referrers here so that
            ‘nix-store --delete’ doesn't have the unexpected effect of
            recursing into derivations and outputs. */
-        state.dead.insert(visited.begin(), visited.end());
+        for (auto & i : visited)
+            state.dead.insert(i.clone());
         if (state.shouldDelete)
             deletePathRecursive(state, path);
     }
@@ -715,8 +719,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
 
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    GCState state(results);
-    state.options = options;
+    GCState state(options, results);
     state.gcKeepOutputs = settings.gcKeepOutputs;
     state.gcKeepDerivations = settings.gcKeepDerivations;
 
@@ -741,12 +744,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    printError(format("finding garbage collector roots..."));
+    printError("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.first);
+    for (auto & i : rootMap) state.roots.insert(i.first.clone());
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
@@ -754,9 +757,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     FDs fds;
     Roots tempRoots;
     findTempRoots(fds, tempRoots, true);
-    for (auto & root : tempRoots)
-        state.tempRoots.insert(root.first);
-    state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
+    for (auto & root : tempRoots) {
+        state.tempRoots.insert(root.first.clone());
+        state.roots.insert(root.first.clone());
+    }
 
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
@@ -768,7 +772,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             createDirs(trashDir);
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
-                printInfo(format("note: can't create trash directory: %1%") % e.msg());
+                printInfo("note: can't create trash directory: %s", e.msg());
                 state.moveToTrash = false;
             }
         }
@@ -780,22 +784,21 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     if (options.action == GCOptions::gcDeleteSpecific) {
 
         for (auto & i : options.pathsToDelete) {
-            assertStorePath(i);
-            tryToDelete(state, i);
+            tryToDelete(state, printStorePath(i));
             if (state.dead.find(i) == state.dead.end())
-                throw Error(format(
+                throw Error(
                     "cannot delete path '%1%' since it is still alive. "
                     "To find out why use: "
-                    "nix-store --query --roots"
-                    ) % i);
+                    "nix-store --query --roots",
+                    printStorePath(i));
         }
 
     } else if (options.maxFreed > 0) {
 
         if (state.shouldDelete)
-            printError(format("deleting garbage..."));
+            printError("deleting garbage...");
         else
-            printError(format("determining live/dead paths..."));
+            printError("determining live/dead paths...");
 
         try {
 
@@ -815,7 +818,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
                 Path path = storeDir + "/" + name;
-                if (isStorePath(path) && isValidPath(path))
+                // FIXME
+                if (isStorePath(path) && isValidPath(parseStorePath(path)))
                     entries.push_back(path);
                 else
                     tryToDelete(state, path);
@@ -840,12 +844,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     }
 
     if (state.options.action == GCOptions::gcReturnLive) {
-        state.results.paths = state.alive;
+        for (auto & i : state.alive)
+            state.results.paths.insert(printStorePath(i));
         return;
     }
 
     if (state.options.action == GCOptions::gcReturnDead) {
-        state.results.paths = state.dead;
+        for (auto & i : state.dead)
+            state.results.paths.insert(printStorePath(i));
         return;
     }
 
@@ -859,7 +865,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
-        printError(format("deleting unused links..."));
+        printError("deleting unused links...");
         removeUnusedLinks(state);
     }
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 1b8b5908cfa..458266be00f 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -87,25 +87,27 @@ struct LegacySSHStore : public Store
         return uriScheme + host;
     }
 
-    void queryPathInfoUncached(const Path & path,
+    void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
     {
         try {
             auto conn(connections->get());
 
-            debug("querying remote host '%s' for info on '%s'", host, path);
+            debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
 
-            conn->to << cmdQueryPathInfos << PathSet{path};
+            conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)};
             conn->to.flush();
 
-            auto info = std::make_shared<ValidPathInfo>();
-            conn->from >> info->path;
-            if (info->path.empty()) return callback(nullptr);
+            auto p = readString(conn->from);
+            if (p.empty()) return callback(nullptr);
+            auto info = std::make_shared<ValidPathInfo>(parseStorePath(p));
             assert(path == info->path);
 
             PathSet references;
-            conn->from >> info->deriver;
-            info->references = readStorePaths<PathSet>(*this, conn->from);
+            auto deriver = readString(conn->from);
+            if (deriver != "")
+                info->deriver = parseStorePath(deriver);
+            info->references = readStorePaths<StorePathSet>(*this, conn->from);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -127,7 +129,7 @@ struct LegacySSHStore : public Store
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override
     {
-        debug("adding path '%s' to remote host '%s'", info.path, host);
+        debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
 
         auto conn(connections->get());
 
@@ -135,10 +137,11 @@ struct LegacySSHStore : public Store
 
             conn->to
                 << cmdAddToStoreNar
-                << info.path
-                << info.deriver
-                << info.narHash.to_string(Base16, false)
-                << info.references
+                << printStorePath(info.path)
+                << (info.deriver ? printStorePath(*info.deriver) : "")
+                << info.narHash.to_string(Base16, false);
+            writeStorePaths(*this, conn->to, info.references);
+            conn->to
                 << info.registrationTime
                 << info.narSize
                 << info.ultimate
@@ -165,9 +168,10 @@ struct LegacySSHStore : public Store
             }
             conn->to
                 << exportMagic
-                << info.path
-                << info.references
-                << info.deriver
+                << printStorePath(info.path);
+            writeStorePaths(*this, conn->to, info.references);
+            conn->to
+                << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
                 << 0;
             conn->to.flush();
@@ -175,39 +179,40 @@ struct LegacySSHStore : public Store
         }
 
         if (readInt(conn->from) != 1)
-            throw Error("failed to add path '%s' to remote host '%s', info.path, host");
+            throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
     }
 
-    void narFromPath(const Path & path, Sink & sink) override
+    void narFromPath(const StorePath & path, Sink & sink) override
     {
         auto conn(connections->get());
 
-        conn->to << cmdDumpStorePath << path;
+        conn->to << cmdDumpStorePath << printStorePath(path);
         conn->to.flush();
         copyNAR(conn->from, sink);
     }
 
-    Path queryPathFromHashPart(const string & hashPart) override
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
     { unsupported("addToStore"); }
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair) override
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override
     { unsupported("addTextToStore"); }
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     {
         auto conn(connections->get());
 
         conn->to
             << cmdBuildDerivation
-            << drvPath
-            << drv
+            << printStorePath(drvPath);
+        writeDerivation(conn->to, *this, drv);
+        conn->to
             << settings.maxSilentTime
             << settings.buildTimeout;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
@@ -230,11 +235,11 @@ struct LegacySSHStore : public Store
         return status;
     }
 
-    void ensurePath(const Path & path) override
+    void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
-    void computeFSClosure(const PathSet & paths,
-        PathSet & out, bool flipDirection = false,
+    void computeFSClosure(const StorePathSet & paths,
+        StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false) override
     {
         if (flipDirection || includeDerivers) {
@@ -246,16 +251,15 @@ struct LegacySSHStore : public Store
 
         conn->to
             << cmdQueryClosure
-            << includeOutputs
-            << paths;
+            << includeOutputs;
+        writeStorePaths(*this, conn->to, paths);
         conn->to.flush();
 
-        auto res = readStorePaths<PathSet>(*this, conn->from);
-
-        out.insert(res.begin(), res.end());
+        for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+            out.insert(i.clone());
     }
 
-    PathSet queryValidPaths(const PathSet & paths,
+    StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute) override
     {
         auto conn(connections->get());
@@ -263,11 +267,11 @@ struct LegacySSHStore : public Store
         conn->to
             << cmdQueryValidPaths
             << false // lock
-            << maybeSubstitute
-            << paths;
+            << maybeSubstitute;
+        writeStorePaths(*this, conn->to, paths);
         conn->to.flush();
 
-        return readStorePaths<PathSet>(*this, conn->from);
+        return readStorePaths<StorePathSet>(*this, conn->from);
     }
 
     void connect() override
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 9f99ee76d84..363be144334 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -44,15 +44,15 @@ class LocalBinaryCacheStore : public BinaryCacheStore
         }
     }
 
-    PathSet queryAllValidPaths() override
+    StorePathSet queryAllValidPaths() override
     {
-        PathSet paths;
+        StorePathSet paths;
 
         for (auto & entry : readDirectory(binaryCacheDir)) {
             if (entry.name.size() != 40 ||
                 !hasSuffix(entry.name, ".narinfo"))
                 continue;
-            paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8));
+            paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
         }
 
         return paths;
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index 642e4070d45..aa5abd835d7 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -21,7 +21,7 @@ struct LocalStoreAccessor : public FSAccessor
     Path toRealPath(const Path & path)
     {
         Path storePath = store->toStorePath(path);
-        if (!store->isValidPath(storePath))
+        if (!store->isValidPath(store->parseStorePath(storePath)))
             throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
@@ -77,34 +77,32 @@ ref<FSAccessor> LocalFSStore::getFSAccessor()
             std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
 }
 
-void LocalFSStore::narFromPath(const Path & path, Sink & sink)
+void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
 {
     if (!isValidPath(path))
-        throw Error(format("path '%s' is not valid") % path);
-    dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
+        throw Error("path '%s' is not valid", printStorePath(path));
+    dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
 }
 
 const string LocalFSStore::drvsLogDir = "drvs";
 
 
 
-std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
+std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
 {
-    auto path(path_);
+    auto path = path_.clone();
 
-    assertStorePath(path);
-
-
-    if (!isDerivation(path)) {
+    if (!path.isDerivation()) {
         try {
-            path = queryPathInfo(path)->deriver;
+            auto info = queryPathInfo(path);
+            if (!info->deriver) return nullptr;
+            path = info->deriver->clone();
         } catch (InvalidPath &) {
             return nullptr;
         }
-        if (path == "") return nullptr;
     }
 
-    string baseName = baseNameOf(path);
+    auto baseName = std::string(baseNameOf(printStorePath(path)));
 
     for (int j = 0; j < 2; j++) {
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 60e7bb7af9e..c1380ef0977 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -541,42 +541,36 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid)
 }
 
 
-void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
+void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv)
 {
-    string drvName = storePathToName(drvPath);
-    assert(isDerivation(drvName));
+    assert(drvPath.isDerivation());
+    std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator out = drv.outputs.find("out");
         if (out == drv.outputs.end())
-            throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath);
+            throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
         bool recursive; Hash h;
         out->second.parseHashInfo(recursive, h);
-        Path outPath = makeFixedOutputPath(recursive, h, drvName);
+        auto outPath = makeFixedOutputPath(recursive, h, drvName);
 
         StringPairs::const_iterator j = drv.env.find("out");
-        if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
-            throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
-                % drvPath % out->second.path % outPath);
+        if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
+            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath));
     }
 
     else {
-        Derivation drvCopy(drv);
-        for (auto & i : drvCopy.outputs) {
-            i.second.path = "";
-            drvCopy.env[i.first] = "";
-        }
-
-        Hash h = hashDerivationModulo(*this, drvCopy);
+        Hash h = hashDerivationModulo(*this, drv, true);
 
         for (auto & i : drv.outputs) {
-            Path outPath = makeOutputPath(i.first, h, drvName);
+            auto outPath = makeOutputPath(i.first, h, drvName);
             StringPairs::const_iterator j = drv.env.find(i.first);
-            if (i.second.path != outPath || j == drv.env.end() || j->second != outPath)
-                throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
-                    % drvPath % i.second.path % outPath);
+            if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
+                throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                    printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath));
         }
     }
 }
@@ -585,16 +579,15 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    checkStoreName(storePathToName(info.path));
-
     if (info.ca != "" && !info.isContentAddressed(*this))
-        throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path);
+        throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
+            printStorePath(info.path));
 
     state.stmtRegisterValidPath.use()
-        (info.path)
+        (printStorePath(info.path))
         (info.narHash.to_string(Base16))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
-        (info.deriver, info.deriver != "")
+        (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
         (info.narSize, info.narSize != 0)
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
@@ -606,8 +599,8 @@ uint64_t LocalStore::addValidPath(State & state,
        the database.  This is useful for the garbage collector: it can
        efficiently query whether a path is an output of some
        derivation. */
-    if (isDerivation(info.path)) {
-        Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path));
+    if (info.path.isDerivation()) {
+        auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -620,34 +613,31 @@ uint64_t LocalStore::addValidPath(State & state,
             state.stmtAddDerivationOutput.use()
                 (id)
                 (i.first)
-                (i.second.path)
+                (printStorePath(i.second.path))
                 .exec();
         }
     }
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info));
+        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info));
     }
 
     return id;
 }
 
 
-void LocalStore::queryPathInfoUncached(const Path & path,
+void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto info = std::make_shared<ValidPathInfo>();
-        info->path = path;
-
-        assertStorePath(path);
+        auto info = std::make_shared<ValidPathInfo>(path.clone());
 
         callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
 
             /* Get the path info. */
-            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path));
+            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path)));
 
             if (!useQueryPathInfo.next())
                 return std::shared_ptr<ValidPathInfo>();
@@ -657,13 +647,13 @@ void LocalStore::queryPathInfoUncached(const Path & path,
             try {
                 info->narHash = Hash(useQueryPathInfo.getStr(1));
             } catch (BadHash & e) {
-                throw Error("in valid-path entry for '%s': %s", path, e.what());
+                throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
             }
 
             info->registrationTime = useQueryPathInfo.getInt(2);
 
             auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
-            if (s) info->deriver = s;
+            if (s) info->deriver = parseStorePath(s);
 
             /* Note that narSize = NULL yields 0. */
             info->narSize = useQueryPathInfo.getInt(4);
@@ -680,7 +670,7 @@ void LocalStore::queryPathInfoUncached(const Path & path,
             auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
 
             while (useQueryReferences.next())
-                info->references.insert(useQueryReferences.getStr(0));
+                info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
 
             return info;
         }));
@@ -698,27 +688,27 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (info.ca, !info.ca.empty())
-        (info.path)
+        (printStorePath(info.path))
         .exec();
 }
 
 
-uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
+uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 {
-    auto use(state.stmtQueryPathInfo.use()(path));
+    auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
     if (!use.next())
-        throw Error(format("path '%1%' is not valid") % path);
+        throw Error("path '%s' is not valid", printStorePath(path));
     return use.getInt(0);
 }
 
 
-bool LocalStore::isValidPath_(State & state, const Path & path)
+bool LocalStore::isValidPath_(State & state, const StorePath & path)
 {
-    return state.stmtQueryPathInfo.use()(path).next();
+    return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
 }
 
 
-bool LocalStore::isValidPathUncached(const Path & path)
+bool LocalStore::isValidPathUncached(const StorePath & path)
 {
     return retrySQLite<bool>([&]() {
         auto state(_state.lock());
@@ -727,39 +717,38 @@ bool LocalStore::isValidPathUncached(const Path & path)
 }
 
 
-PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
+StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
-    PathSet res;
+    StorePathSet res;
     for (auto & i : paths)
-        if (isValidPath(i)) res.insert(i);
+        if (isValidPath(i)) res.insert(i.clone());
     return res;
 }
 
 
-PathSet LocalStore::queryAllValidPaths()
+StorePathSet LocalStore::queryAllValidPaths()
 {
-    return retrySQLite<PathSet>([&]() {
+    return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
         auto use(state->stmtQueryValidPaths.use());
-        PathSet res;
-        while (use.next()) res.insert(use.getStr(0));
+        StorePathSet res;
+        while (use.next()) res.insert(parseStorePath(use.getStr(0)));
         return res;
     });
 }
 
 
-void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers)
+void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
 {
-    auto useQueryReferrers(state.stmtQueryReferrers.use()(path));
+    auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
 
     while (useQueryReferrers.next())
-        referrers.insert(useQueryReferrers.getStr(0));
+        referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
 }
 
 
-void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
+void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
 {
-    assertStorePath(path);
     return retrySQLite<void>([&]() {
         auto state(_state.lock());
         queryReferrers(*state, path, referrers);
@@ -767,42 +756,40 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 }
 
 
-PathSet LocalStore::queryValidDerivers(const Path & path)
+StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 {
-    assertStorePath(path);
-
-    return retrySQLite<PathSet>([&]() {
+    return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
 
-        auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path));
+        auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
 
-        PathSet derivers;
+        StorePathSet derivers;
         while (useQueryValidDerivers.next())
-            derivers.insert(useQueryValidDerivers.getStr(1));
+            derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1)));
 
         return derivers;
     });
 }
 
 
-PathSet LocalStore::queryDerivationOutputs(const Path & path)
+StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
 {
-    return retrySQLite<PathSet>([&]() {
+    return retrySQLite<StorePathSet>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        PathSet outputs;
+        StorePathSet outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.insert(useQueryDerivationOutputs.getStr(1));
+            outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
 
         return outputs;
     });
 }
 
 
-StringSet LocalStore::queryDerivationOutputNames(const Path & path)
+StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
 {
     return retrySQLite<StringSet>([&]() {
         auto state(_state.lock());
@@ -819,31 +806,36 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 }
 
 
-Path LocalStore::queryPathFromHashPart(const string & hashPart)
+std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
     if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
 
     Path prefix = storeDir + "/" + hashPart;
 
-    return retrySQLite<Path>([&]() -> std::string {
+    return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
         auto state(_state.lock());
 
         auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
 
-        if (!useQueryPathFromHashPart.next()) return "";
+        if (!useQueryPathFromHashPart.next()) return {};
 
         const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
-        return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
+        if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
+            return parseStorePath(s);
+        return {};
     });
 }
 
 
-PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
+StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 {
-    if (!settings.useSubstitutes) return PathSet();
+    if (!settings.useSubstitutes) return StorePathSet();
 
-    auto remaining = paths;
-    PathSet res;
+    StorePathSet remaining;
+    for (auto & i : paths)
+        remaining.insert(i.clone());
+
+    StorePathSet res;
 
     for (auto & sub : getDefaultSubstituters()) {
         if (remaining.empty()) break;
@@ -852,12 +844,12 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 
         auto valid = sub->queryValidPaths(remaining);
 
-        PathSet remaining2;
+        StorePathSet remaining2;
         for (auto & path : remaining)
             if (valid.count(path))
-                res.insert(path);
+                res.insert(path.clone());
             else
-                remaining2.insert(path);
+                remaining2.insert(path.clone());
 
         std::swap(remaining, remaining2);
     }
@@ -866,7 +858,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 }
 
 
-void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
     SubstitutablePathInfos & infos)
 {
     if (!settings.useSubstitutes) return;
@@ -874,17 +866,16 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
         if (sub->storeDir != storeDir) continue;
         for (auto & path : paths) {
             if (infos.count(path)) continue;
-            debug(format("checking substituter '%s' for path '%s'")
-                % sub->getUri() % path);
+            debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
             try {
                 auto info = sub->queryPathInfo(path);
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
-                infos[path] = SubstitutablePathInfo{
-                    info->deriver,
-                    info->references,
+                infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
+                    info->deriver ? info->deriver->clone() : std::optional<StorePath>(),
+                    cloneStorePathSet(info->references),
                     narInfo ? narInfo->fileSize : 0,
-                    info->narSize};
+                    info->narSize});
             } catch (InvalidPath &) {
             } catch (SubstituterDisabled &) {
             } catch (Error & e) {
@@ -918,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         auto state(_state.lock());
 
         SQLiteTxn txn(state->db);
-        PathSet paths;
+        StorePathSet paths;
 
         for (auto & i : infos) {
             assert(i.narHash.type == htSHA256);
@@ -926,7 +917,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
                 updatePathInfo(*state, i);
             else
                 addValidPath(*state, i, false);
-            paths.insert(i.path);
+            paths.insert(i.path.clone());
         }
 
         for (auto & i : infos) {
@@ -939,10 +930,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
            this in addValidPath() above, because the references might
            not be valid yet. */
         for (auto & i : infos)
-            if (isDerivation(i.path)) {
+            if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path));
-                checkDerivationOutputs(i.path, drv);
+                checkDerivationOutputs(i.path,
+                    readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
             }
 
         /* Do a topological sort of the paths.  This will throw an
@@ -958,18 +949,18 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
 
 /* Invalidate a path.  The caller is responsible for checking that
    there are no referrers. */
-void LocalStore::invalidatePath(State & state, const Path & path)
+void LocalStore::invalidatePath(State & state, const StorePath & path)
 {
-    debug(format("invalidating path '%1%'") % path);
+    debug("invalidating path '%s'", printStorePath(path));
 
-    state.stmtInvalidatePath.use()(path).exec();
+    state.stmtInvalidatePath.use()(printStorePath(path)).exec();
 
     /* Note that the foreign key constraints on the Refs table take
        care of deleting the references entries for `path'. */
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(storePathToHash(path));
+        state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
     }
 }
 
@@ -987,10 +978,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
     if (!info.narHash)
-        throw Error("cannot add path '%s' because it lacks a hash", info.path);
+        throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
 
     if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
-        throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
+        throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
 
     addTempRoot(info.path);
 
@@ -998,12 +989,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + baseNameOf(info.path);
+        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        if (!locksHeld.count(info.path))
+        if (!locksHeld.count(printStorePath(info.path)))
             outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {
@@ -1018,7 +1009,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             else {
                 if (!info.references.empty())
                     settings.requireExperimentalFeature("ca-references");
-                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
             }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
@@ -1033,11 +1024,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
-                    info.path, info.narHash.to_string(), hashResult.first.to_string());
+                    printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
-                    info.path, info.narSize, hashResult.second);
+                    printStorePath(info.path), info.narSize, hashResult.second);
 
             autoGC();
 
@@ -1053,12 +1044,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
+StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
     bool recursive, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
-    Path dstPath = makeFixedOutputPath(recursive, h, name);
+    auto dstPath = makeFixedOutputPath(recursive, h, name);
 
     addTempRoot(dstPath);
 
@@ -1067,7 +1058,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
+        Path realPath = realStoreDir + "/";
+        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 
@@ -1098,8 +1090,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 
             optimisePath(realPath); // FIXME: combine with hashPath()
 
-            ValidPathInfo info;
-            info.path = dstPath;
+            ValidPathInfo info(dstPath.clone());
             info.narHash = hash.first;
             info.narSize = hash.second;
             info.ca = makeFixedOutputCA(recursive, h);
@@ -1113,7 +1104,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 }
 
 
-Path LocalStore::addToStore(const string & name, const Path & _srcPath,
+StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
@@ -1131,8 +1122,8 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath,
 }
 
 
-Path LocalStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, RepairFlag repair)
+StorePath LocalStore::addTextToStore(const string & name, const string & s,
+    const StorePathSet & references, RepairFlag repair)
 {
     auto hash = hashString(htSHA256, s);
     auto dstPath = makeTextPath(name, hash, references);
@@ -1141,7 +1132,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
+        Path realPath = realStoreDir + "/";
+        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 
@@ -1161,11 +1153,10 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             optimisePath(realPath);
 
-            ValidPathInfo info;
-            info.path = dstPath;
+            ValidPathInfo info(dstPath.clone());
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.references = references;
+            info.references = cloneStorePathSet(references);
             info.ca = "text:" + hash.to_string();
             registerValidPath(info);
         }
@@ -1187,27 +1178,25 @@ Path LocalStore::createTempDirInStore()
            the GC between createTempDir() and addTempRoot(), so repeat
            until `tmpDir' exists. */
         tmpDir = createTempDir(realStoreDir);
-        addTempRoot(tmpDir);
+        addTempRoot(parseStorePath(tmpDir));
     } while (!pathExists(tmpDir));
     return tmpDir;
 }
 
 
-void LocalStore::invalidatePathChecked(const Path & path)
+void LocalStore::invalidatePathChecked(const StorePath & path)
 {
-    assertStorePath(path);
-
     retrySQLite<void>([&]() {
         auto state(_state.lock());
 
         SQLiteTxn txn(state->db);
 
         if (isValidPath_(*state, path)) {
-            PathSet referrers; queryReferrers(*state, path, referrers);
+            StorePathSet referrers; queryReferrers(*state, path, referrers);
             referrers.erase(path); /* ignore self-references */
             if (!referrers.empty())
-                throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%")
-                    % path % showPaths(referrers));
+                throw PathInUse("cannot delete path '%s' because it is in use by %s",
+                    printStorePath(path), showPaths(referrers));
             invalidatePath(*state, path);
         }
 
@@ -1226,18 +1215,19 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
        existing and valid paths. */
     AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
-    PathSet store;
+    StringSet store;
     for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
 
     /* Check whether all valid paths actually exist. */
     printInfo("checking path existence...");
 
-    PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
+    StorePathSet validPaths;
+    PathSet done;
 
     fdGCLock = -1;
 
-    for (auto & i : validPaths2)
-        verifyPath(i, store, done, validPaths, repair, errors);
+    for (auto & i : queryAllValidPaths())
+        verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
 
     /* Optionally, check the content hashes (slow). */
     if (checkContents) {
@@ -1272,21 +1262,20 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
 
                 /* Check the content hash (optionally - slow). */
-                printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
+                printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
                 if (info->ca == "")
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
 
-                dumpPath(toRealPath(i), *hashSink);
+                dumpPath(toRealPath(printStorePath(i)), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    printError(format("path '%1%' was modified! "
-                            "expected hash '%2%', got '%3%'")
-                        % i % info->narHash.to_string() % current.first.to_string());
+                    printError("path '%s' was modified! expected hash '%s', got '%s'",
+                        printStorePath(i), info->narHash.to_string(), current.first.to_string());
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
@@ -1294,14 +1283,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                     /* Fill in missing hashes. */
                     if (info->narHash == nullHash) {
-                        printError(format("fixing missing hash on '%1%'") % i);
+                        printError("fixing missing hash on '%s'", printStorePath(i));
                         info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
                     if (info->narSize == 0) {
-                        printError(format("updating size field on '%1%' to %2%") % i % current.second);
+                        printError("updating size field on '%s' to %s", printStorePath(i), current.second);
                         info->narSize = current.second;
                         update = true;
                     }
@@ -1317,9 +1306,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* It's possible that the path got GC'ed, so ignore
                    errors on invalid paths. */
                 if (isValidPath(i))
-                    printError(format("error: %1%") % e.msg());
+                    printError("error: %s", e.msg());
                 else
-                    printError(format("warning: %1%") % e.msg());
+                    warn(e.msg());
                 errors = true;
             }
         }
@@ -1329,43 +1318,43 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 }
 
 
-void LocalStore::verifyPath(const Path & path, const PathSet & store,
-    PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors)
+void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
+    PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors)
 {
     checkInterrupt();
 
-    if (!done.insert(path).second) return;
+    if (!done.insert(pathS).second) return;
 
-    if (!isStorePath(path)) {
-        printError(format("path '%1%' is not in the Nix store") % path);
-        auto state(_state.lock());
-        invalidatePath(*state, path);
+    if (!isStorePath(pathS)) {
+        printError("path '%s' is not in the Nix store", pathS);
         return;
     }
 
-    if (store.find(baseNameOf(path)) == store.end()) {
+    auto path = parseStorePath(pathS);
+
+    if (!store.count(std::string(path.to_string()))) {
         /* Check any referrers first.  If we can invalidate them
            first, then we can invalidate this path as well. */
         bool canInvalidate = true;
-        PathSet referrers; queryReferrers(path, referrers);
+        StorePathSet referrers; queryReferrers(path, referrers);
         for (auto & i : referrers)
             if (i != path) {
-                verifyPath(i, store, done, validPaths, repair, errors);
-                if (validPaths.find(i) != validPaths.end())
+                verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
+                if (validPaths.count(i))
                     canInvalidate = false;
             }
 
         if (canInvalidate) {
-            printError(format("path '%1%' disappeared, removing from database...") % path);
+            printError("path '%s' disappeared, removing from database...", pathS);
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            printError(format("path '%1%' disappeared, but it still has valid referrers!") % path);
+            printError("path '%s' disappeared, but it still has valid referrers!", pathS);
             if (repair)
                 try {
                     repairPath(path);
                 } catch (Error & e) {
-                    printError(format("warning: %1%") % e.msg());
+                    warn(e.msg());
                     errors = true;
                 }
             else errors = true;
@@ -1374,7 +1363,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
         return;
     }
 
-    validPaths.insert(path);
+    validPaths.insert(std::move(path));
 }
 
 
@@ -1443,7 +1432,7 @@ void LocalStore::vacuumDB()
 }
 
 
-void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs)
+void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     retrySQLite<void>([&]() {
         auto state(_state.lock());
@@ -1469,7 +1458,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        info.sign(secretKey);
+        info.sign(*this, secretKey);
     }
 }
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 1d4b888980e..16aeab0ad9e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -119,36 +119,36 @@ public:
 
     std::string getUri() override;
 
-    bool isValidPathUncached(const Path & path) override;
+    bool isValidPathUncached(const StorePath & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths,
+    StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
-    PathSet queryAllValidPaths() override;
+    StorePathSet queryAllValidPaths() override;
 
-    void queryPathInfoUncached(const Path & path,
+    void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
-    void queryReferrers(const Path & path, PathSet & referrers) override;
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
 
-    PathSet queryValidDerivers(const Path & path) override;
+    StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    PathSet queryDerivationOutputs(const Path & path) override;
+    StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const Path & path) override;
+    StringSet queryDerivationOutputNames(const StorePath & path) override;
 
-    Path queryPathFromHashPart(const string & hashPart) override;
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
-    PathSet querySubstitutablePaths(const PathSet & paths) override;
+    StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const PathSet & paths,
+    void querySubstitutablePathInfos(const StorePathSet & paths,
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
@@ -156,20 +156,22 @@ public:
        in `dump', which is either a NAR serialisation (if recursive ==
        true) or simply the contents of a regular file (if recursive ==
        false). */
-    Path addToStoreFromDump(const string & dump, const string & name,
+    StorePath addToStoreFromDump(const string & dump, const string & name,
         bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair) override;
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override;
 
-    void buildPaths(const PathSet & paths, BuildMode buildMode) override;
+    void buildPaths(
+        const std::vector<StorePathWithOutputs> & paths,
+        BuildMode buildMode) override;
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
 
-    void ensurePath(const Path & path) override;
+    void ensurePath(const StorePath & path) override;
 
-    void addTempRoot(const Path & path) override;
+    void addTempRoot(const StorePath & path) override;
 
     void addIndirectRoot(const Path & path) override;
 
@@ -215,9 +217,9 @@ public:
 
     /* Repair the contents of the given path by redownloading it using
        a substituter (if available). */
-    void repairPath(const Path & path);
+    void repairPath(const StorePath & path);
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override;
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
     /* If free disk space in /nix/store if below minFree, delete
        garbage until it exceeds maxFree. */
@@ -231,17 +233,17 @@ private:
 
     void makeStoreWritable();
 
-    uint64_t queryValidPathId(State & state, const Path & path);
+    uint64_t queryValidPathId(State & state, const StorePath & path);
 
     uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
 
-    void invalidatePath(State & state, const Path & path);
+    void invalidatePath(State & state, const StorePath & path);
 
     /* Delete a path from the Nix store. */
-    void invalidatePathChecked(const Path & path);
+    void invalidatePathChecked(const StorePath & path);
 
-    void verifyPath(const Path & path, const PathSet & store,
-        PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors);
+    void verifyPath(const Path & path, const StringSet & store,
+        PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
 
     void updatePathInfo(State & state, const ValidPathInfo & info);
 
@@ -256,7 +258,7 @@ private:
 
     void tryToDelete(GCState & state, const Path & path);
 
-    bool canReachRoot(GCState & state, PathSet & visited, const Path & path);
+    bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
 
     void deletePathRecursive(GCState & state, const Path & path);
 
@@ -275,7 +277,7 @@ private:
 
     Path createTempDirInStore();
 
-    void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
+    void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
 
     typedef std::unordered_set<ino_t> InodeHash;
 
@@ -284,8 +286,8 @@ private:
     void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
 
     // Internal versions that are not wrapped in retry_sqlite.
-    bool isValidPath_(State & state, const Path & path);
-    void queryReferrers(State & state, const Path & path, PathSet & referrers);
+    bool isValidPath_(State & state, const StorePath & path);
+    void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
 
     /* Add signatures to a ValidPathInfo using the secret keys
        specified by the ‘secret-key-files’ option. */
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d690fea28c2..d3254554dd3 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
 
-libstore_LIBS = libutil
+libstore_LIBS = libutil libnixrust
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifneq ($(OS), FreeBSD)
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 2fceb9b9ac3..9c47fe52454 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -9,13 +9,13 @@
 namespace nix {
 
 
-void Store::computeFSClosure(const PathSet & startPaths,
-    PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
+void Store::computeFSClosure(const StorePathSet & startPaths,
+    StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
     struct State
     {
         size_t pending;
-        PathSet & paths;
+        StorePathSet & paths;
         std::exception_ptr exc;
     };
 
@@ -29,45 +29,47 @@ void Store::computeFSClosure(const PathSet & startPaths,
         {
             auto state(state_.lock());
             if (state->exc) return;
-            if (!state->paths.insert(path).second) return;
+            if (!state->paths.insert(parseStorePath(path)).second) return;
             state->pending++;
         }
 
-        queryPathInfo(path, {[&, path](std::future<ref<const ValidPathInfo>> fut) {
+        queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
             // FIXME: calls to isValidPath() should be async
 
             try {
                 auto info = fut.get();
 
+                auto path = parseStorePath(pathS);
+
                 if (flipDirection) {
 
-                    PathSet referrers;
+                    StorePathSet referrers;
                     queryReferrers(path, referrers);
                     for (auto & ref : referrers)
                         if (ref != path)
-                            enqueue(ref);
+                            enqueue(printStorePath(ref));
 
                     if (includeOutputs)
                         for (auto & i : queryValidDerivers(path))
-                            enqueue(i);
+                            enqueue(printStorePath(i));
 
-                    if (includeDerivers && isDerivation(path))
+                    if (includeDerivers && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
                             if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                                enqueue(i);
+                                enqueue(printStorePath(i));
 
                 } else {
 
                     for (auto & ref : info->references)
                         if (ref != path)
-                            enqueue(ref);
+                            enqueue(printStorePath(ref));
 
-                    if (includeOutputs && isDerivation(path))
+                    if (includeOutputs && path.isDerivation())
                         for (auto & i : queryDerivationOutputs(path))
-                            if (isValidPath(i)) enqueue(i);
+                            if (isValidPath(i)) enqueue(printStorePath(i));
 
-                    if (includeDerivers && isValidPath(info->deriver))
-                        enqueue(info->deriver);
+                    if (includeDerivers && info->deriver && isValidPath(*info->deriver))
+                        enqueue(printStorePath(*info->deriver));
 
                 }
 
@@ -87,7 +89,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
     };
 
     for (auto & startPath : startPaths)
-        enqueue(startPath);
+        enqueue(printStorePath(startPath));
 
     {
         auto state(state_.lock());
@@ -97,15 +99,17 @@ void Store::computeFSClosure(const PathSet & startPaths,
 }
 
 
-void Store::computeFSClosure(const Path & startPath,
-    PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
+void Store::computeFSClosure(const StorePath & startPath,
+    StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
-    computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
+    StorePathSet paths;
+    paths.insert(startPath.clone());
+    computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers);
 }
 
 
-void Store::queryMissing(const PathSet & targets,
-    PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
+void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
     unsigned long long & downloadSize_, unsigned long long & narSize_)
 {
     Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@@ -116,8 +120,8 @@ void Store::queryMissing(const PathSet & targets,
 
     struct State
     {
-        PathSet done;
-        PathSet & unknown, & willSubstitute, & willBuild;
+        std::unordered_set<std::string> done;
+        StorePathSet & unknown, & willSubstitute, & willBuild;
         unsigned long long & downloadSize;
         unsigned long long & narSize;
     };
@@ -126,31 +130,36 @@ void Store::queryMissing(const PathSet & targets,
     {
         size_t left;
         bool done = false;
-        PathSet outPaths;
+        StorePathSet outPaths;
         DrvState(size_t left) : left(left) { }
     };
 
-    Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
+    Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
 
-    std::function<void(Path)> doPath;
+    std::function<void(StorePathWithOutputs)> doPath;
 
-    auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) {
+    auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
         {
             auto state(state_.lock());
-            state->willBuild.insert(drvPath);
+            state->willBuild.insert(drvPath.clone());
         }
 
         for (auto & i : drv.inputDrvs)
-            pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
+            pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second)));
     };
 
     auto checkOutput = [&](
-        const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_)
+        const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
     {
         if (drvState_->lock()->done) return;
 
+        auto drvPath = parseStorePath(drvPathS);
+        auto outPath = parseStorePath(outPathS);
+
         SubstitutablePathInfos infos;
-        querySubstitutablePathInfos({outPath}, infos);
+        StorePathSet paths; // FIXME
+        paths.insert(outPath.clone());
+        querySubstitutablePathInfos(paths, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;
@@ -161,74 +170,74 @@ void Store::queryMissing(const PathSet & targets,
                 if (drvState->done) return;
                 assert(drvState->left);
                 drvState->left--;
-                drvState->outPaths.insert(outPath);
+                drvState->outPaths.insert(outPath.clone());
                 if (!drvState->left) {
                     for (auto & path : drvState->outPaths)
-                        pool.enqueue(std::bind(doPath, path));
+                        pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone())));
                 }
             }
         }
     };
 
-    doPath = [&](const Path & path) {
+    doPath = [&](const StorePathWithOutputs & path) {
 
         {
             auto state(state_.lock());
-            if (!state->done.insert(path).second) return;
+            if (!state->done.insert(path.to_string(*this)).second) return;
         }
 
-        DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path);
-
-        if (isDerivation(i2.first)) {
-            if (!isValidPath(i2.first)) {
+        if (path.path.isDerivation()) {
+            if (!isValidPath(path.path)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
-                state->unknown.insert(path);
+                state->unknown.insert(path.path.clone());
                 return;
             }
 
-            Derivation drv = derivationFromPath(i2.first);
-            ParsedDerivation parsedDrv(i2.first, drv);
+            auto drv = make_ref<Derivation>(derivationFromPath(path.path));
+            ParsedDerivation parsedDrv(path.path.clone(), *drv);
 
             PathSet invalid;
-            for (auto & j : drv.outputs)
-                if (wantOutput(j.first, i2.second)
+            for (auto & j : drv->outputs)
+                if (wantOutput(j.first, path.outputs)
                     && !isValidPath(j.second.path))
-                    invalid.insert(j.second.path);
+                    invalid.insert(printStorePath(j.second.path));
             if (invalid.empty()) return;
 
             if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
-                    pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
+                    pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
             } else
-                mustBuildDrv(i2.first, drv);
+                mustBuildDrv(path.path, *drv);
 
         } else {
 
-            if (isValidPath(path)) return;
+            if (isValidPath(path.path)) return;
 
             SubstitutablePathInfos infos;
-            querySubstitutablePathInfos({path}, infos);
+            StorePathSet paths; // FIXME
+            paths.insert(path.path.clone());
+            querySubstitutablePathInfos(paths, infos);
 
             if (infos.empty()) {
                 auto state(state_.lock());
-                state->unknown.insert(path);
+                state->unknown.insert(path.path.clone());
                 return;
             }
 
-            auto info = infos.find(path);
+            auto info = infos.find(path.path);
             assert(info != infos.end());
 
             {
                 auto state(state_.lock());
-                state->willSubstitute.insert(path);
+                state->willSubstitute.insert(path.path.clone());
                 state->downloadSize += info->second.downloadSize;
                 state->narSize += info->second.narSize;
             }
 
             for (auto & ref : info->second.references)
-                pool.enqueue(std::bind(doPath, ref));
+                pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref)));
         }
     };
 
@@ -239,39 +248,42 @@ void Store::queryMissing(const PathSet & targets,
 }
 
 
-Paths Store::topoSortPaths(const PathSet & paths)
+StorePaths Store::topoSortPaths(const StorePathSet & paths)
 {
-    Paths sorted;
-    PathSet visited, parents;
+    StorePaths sorted;
+    StorePathSet visited, parents;
 
-    std::function<void(const Path & path, const Path * parent)> dfsVisit;
+    std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
 
-    dfsVisit = [&](const Path & path, const Path * parent) {
-        if (parents.find(path) != parents.end())
-            throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
+    dfsVisit = [&](const StorePath & path, const StorePath * parent) {
+        if (parents.count(path))
+            throw BuildError("cycle detected in the references of '%s' from '%s'",
+                printStorePath(path), printStorePath(*parent));
 
-        if (!visited.insert(path).second) return;
-        parents.insert(path);
+        if (!visited.insert(path.clone()).second) return;
+        parents.insert(path.clone());
 
-        PathSet references;
+        StorePathSet references;
         try {
-            references = queryPathInfo(path)->references;
+            references = cloneStorePathSet(queryPathInfo(path)->references);
         } catch (InvalidPath &) {
         }
 
         for (auto & i : references)
             /* Don't traverse into paths that don't exist.  That can
                happen due to substitutes for non-existent paths. */
-            if (i != path && paths.find(i) != paths.end())
+            if (i != path && paths.count(i))
                 dfsVisit(i, &path);
 
-        sorted.push_front(path);
+        sorted.push_back(path.clone());
         parents.erase(path);
     };
 
     for (auto & i : paths)
         dfsVisit(i, nullptr);
 
+    std::reverse(sorted.begin(), sorted.end());
+
     return sorted;
 }
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 5bf98219585..3a2cde55f20 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -193,11 +193,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             if (!queryNAR.getInt(0))
                 return {oInvalid, 0};
 
-            auto narInfo = make_ref<NarInfo>();
-
             auto namePart = queryNAR.getStr(1);
-            narInfo->path = cache.storeDir + "/" +
-                hashPart + (namePart.empty() ? "" : "-" + namePart);
+            auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart));
             narInfo->url = queryNAR.getStr(2);
             narInfo->compression = queryNAR.getStr(3);
             if (!queryNAR.isNull(4))
@@ -206,9 +203,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             narInfo->narHash = Hash(queryNAR.getStr(6));
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
-                narInfo->references.insert(cache.storeDir + "/" + r);
+                narInfo->references.insert(StorePath::fromBaseName(r));
             if (!queryNAR.isNull(9))
-                narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
+                narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9));
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
             narInfo->ca = queryNAR.getStr(11);
@@ -230,12 +227,12 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
-                assert(hashPart == storePathToHash(info->path));
+                //assert(hashPart == storePathToHash(info->path));
 
                 state->insertNAR.use()
                     (cache.id)
                     (hashPart)
-                    (storePathToName(info->path))
+                    (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
                     (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
@@ -243,7 +240,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                     (info->narHash.to_string())
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
-                    (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
+                    (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
                     (concatStringsSep(" ", info->sigs))
                     (info->ca)
                     (time(0)).exec();
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index cb568ccdc82..fb02cf3fd5a 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -4,6 +4,7 @@
 namespace nix {
 
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
+    : ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack
 {
     auto corrupt = [&]() {
         throw Error(format("NAR info file '%1%' is corrupt") % whence);
@@ -18,6 +19,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         }
     };
 
+    bool havePath = false;
+
     size_t pos = 0;
     while (pos < s.size()) {
 
@@ -32,8 +35,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         std::string value(s, colon + 2, eol - colon - 2);
 
         if (name == "StorePath") {
-            if (!store.isStorePath(value)) corrupt();
-            path = value;
+            path = store.parseStorePath(value);
+            havePath = true;
         }
         else if (name == "URL")
             url = value;
@@ -52,18 +55,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "References") {
             auto refs = tokenizeString<Strings>(value, " ");
             if (!references.empty()) corrupt();
-            for (auto & r : refs) {
-                auto r2 = store.storeDir + "/" + r;
-                if (!store.isStorePath(r2)) corrupt();
-                references.insert(r2);
-            }
+            for (auto & r : refs)
+                references.insert(StorePath::fromBaseName(r));
         }
         else if (name == "Deriver") {
-            if (value != "unknown-deriver") {
-                auto p = store.storeDir + "/" + value;
-                if (!store.isStorePath(p)) corrupt();
-                deriver = p;
-            }
+            if (value != "unknown-deriver")
+                deriver = StorePath::fromBaseName(value);
         }
         else if (name == "System")
             system = value;
@@ -79,13 +76,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
 
     if (compression == "") compression = "bzip2";
 
-    if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
+    if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
 }
 
-std::string NarInfo::to_string() const
+std::string NarInfo::to_string(const Store & store) const
 {
     std::string res;
-    res += "StorePath: " + path + "\n";
+    res += "StorePath: " + store.printStorePath(path) + "\n";
     res += "URL: " + url + "\n";
     assert(compression != "");
     res += "Compression: " + compression + "\n";
@@ -98,8 +95,8 @@ std::string NarInfo::to_string() const
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
 
-    if (!deriver.empty())
-        res += "Deriver: " + baseNameOf(deriver) + "\n";
+    if (deriver)
+        res += "Deriver: " + std::string(deriver->to_string()) + "\n";
 
     if (!system.empty())
         res += "System: " + system + "\n";
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index 4995061fbb6..373c3342767 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -14,11 +14,12 @@ struct NarInfo : ValidPathInfo
     uint64_t fileSize = 0;
     std::string system;
 
-    NarInfo() { }
+    NarInfo() = delete;
+    NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { }
     NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
     NarInfo(const Store & store, const std::string & s, const std::string & whence);
 
-    std::string to_string() const;
+    std::string to_string(const Store & store) const;
 };
 
 }
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 991512f2179..8ac382e9dbf 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -254,7 +254,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
 {
     Activity act(*logger, actOptimiseStore);
 
-    PathSet paths = queryAllValidPaths();
+    auto paths = queryAllValidPaths();
     InodeHash inodeHash = loadInodeHash();
 
     act.progress(0, paths.size());
@@ -265,8 +265,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
         {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i));
-            optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
+            optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash);
         }
         done++;
         act.progress(done, paths.size());
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 87be8a24ead..d0f289a0f57 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -2,8 +2,8 @@
 
 namespace nix {
 
-ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
-    : drvPath(drvPath), drv(drv)
+ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
+    : drvPath(std::move(drvPath)), drv(drv)
 {
     /* Parse the __json attribute, if any. */
     auto jsonAttr = drv.env.find("__json");
@@ -11,7 +11,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
         try {
             structuredAttrs = nlohmann::json::parse(jsonAttr->second);
         } catch (std::exception & e) {
-            throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
+            throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
         }
     }
 }
@@ -24,7 +24,7 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n
             return {};
         else {
             if (!i->is_string())
-                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
+                throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string());
             return i->get<std::string>();
         }
     } else {
@@ -44,7 +44,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
             return def;
         else {
             if (!i->is_boolean())
-                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
+                throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string());
             return i->get<bool>();
         }
     } else {
@@ -64,11 +64,11 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
             return {};
         else {
             if (!i->is_array())
-                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string());
             Strings res;
             for (auto j = i->begin(); j != i->end(); ++j) {
                 if (!j->is_string())
-                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
+                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string());
                 res.push_back(j->get<std::string>());
             }
             return res;
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index 9bde4b4dcfc..cec868754d9 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -6,13 +6,13 @@ namespace nix {
 
 class ParsedDerivation
 {
-    Path drvPath;
+    StorePath drvPath;
     BasicDerivation & drv;
     std::optional<nlohmann::json> structuredAttrs;
 
 public:
 
-    ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
+    ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
 
     const std::optional<nlohmann::json> & getStructuredAttrs() const
     {
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
new file mode 100644
index 00000000000..81ae495a1a0
--- /dev/null
+++ b/src/libstore/path.cc
@@ -0,0 +1,99 @@
+#include "store-api.hh"
+
+namespace nix {
+
+extern "C" {
+    rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir);
+    rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir);
+    rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName);
+    rust::String ffi_StorePath_to_string(const StorePath & _this);
+    StorePath ffi_StorePath_clone(const StorePath & _this);
+    rust::StringSlice ffi_StorePath_name(const StorePath & _this);
+}
+
+StorePath StorePath::make(std::string_view path, std::string_view storeDir)
+{
+    return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap();
+}
+
+StorePath StorePath::make(unsigned char hash[20], std::string_view name)
+{
+    return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap();
+}
+
+StorePath StorePath::fromBaseName(std::string_view baseName)
+{
+    return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap();
+}
+
+rust::String StorePath::to_string() const
+{
+    return ffi_StorePath_to_string(*this);
+}
+
+StorePath StorePath::clone() const
+{
+    return ffi_StorePath_clone(*this);
+}
+
+bool StorePath::isDerivation() const
+{
+    return hasSuffix(name(), drvExtension);
+}
+
+std::string_view StorePath::name() const
+{
+    return ffi_StorePath_name(*this);
+}
+
+StorePath Store::parseStorePath(std::string_view path) const
+{
+    return StorePath::make(path, storeDir);
+}
+
+
+StorePathSet Store::parseStorePathSet(const PathSet & paths) const
+{
+    StorePathSet res;
+    for (auto & i : paths) res.insert(parseStorePath(i));
+    return res;
+}
+
+std::string Store::printStorePath(const StorePath & path) const
+{
+    auto s = storeDir + "/";
+    s += (std::string_view) path.to_string();
+    return s;
+}
+
+PathSet Store::printStorePathSet(const StorePathSet & paths) const
+{
+    PathSet res;
+    for (auto & i : paths) res.insert(printStorePath(i));
+    return res;
+}
+
+StorePathSet cloneStorePathSet(const StorePathSet & paths)
+{
+    StorePathSet res;
+    for (auto & p : paths)
+        res.insert(p.clone());
+    return res;
+}
+
+StorePathSet storePathsToSet(const StorePaths & paths)
+{
+    StorePathSet res;
+    for (auto & p : paths)
+        res.insert(p.clone());
+    return res;
+}
+
+StorePathSet singleton(const StorePath & path)
+{
+    StorePathSet res;
+    res.insert(path.clone());
+    return res;
+}
+
+}
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
new file mode 100644
index 00000000000..273808f02e4
--- /dev/null
+++ b/src/libstore/path.hh
@@ -0,0 +1,81 @@
+#pragma once
+
+#include "rust-ffi.hh"
+
+namespace nix {
+
+/* See path.rs. */
+struct StorePath;
+
+extern "C" {
+    void ffi_StorePath_drop(void *);
+    bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
+    bool ffi_StorePath_eq(const StorePath & a, const StorePath & b);
+    unsigned char * ffi_StorePath_hash_data(const StorePath & p);
+}
+
+struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
+{
+    static StorePath make(std::string_view path, std::string_view storeDir);
+
+    static StorePath make(unsigned char hash[20], std::string_view name);
+
+    static StorePath fromBaseName(std::string_view baseName);
+
+    rust::String to_string() const;
+
+    bool operator < (const StorePath & other) const
+    {
+        return ffi_StorePath_less_than(*this, other);
+    }
+
+    bool operator == (const StorePath & other) const
+    {
+        return ffi_StorePath_eq(*this, other);
+    }
+
+    bool operator != (const StorePath & other) const
+    {
+        return !(*this == other);
+    }
+
+    StorePath clone() const;
+
+    /* Check whether a file name ends with the extension for
+       derivations. */
+    bool isDerivation() const;
+
+    std::string_view name() const;
+
+    unsigned char * hashData() const
+    {
+        return ffi_StorePath_hash_data(*this);
+    }
+};
+
+typedef std::set<StorePath> StorePathSet;
+typedef std::vector<StorePath> StorePaths;
+
+StorePathSet cloneStorePathSet(const StorePathSet & paths);
+StorePathSet storePathsToSet(const StorePaths & paths);
+
+StorePathSet singleton(const StorePath & path);
+
+/* Size of the hash part of store paths, in base-32 characters. */
+const size_t storePathHashLen = 32; // i.e. 160 bits
+
+/* Extension of derivations in the Nix store. */
+const std::string drvExtension = ".drv";
+
+}
+
+namespace std {
+
+template<> struct hash<nix::StorePath> {
+    std::size_t operator()(const nix::StorePath & path) const noexcept
+    {
+        return * (std::size_t *) path.hashData();
+    }
+};
+
+}
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 4c6af567ae6..dae3f2d3251 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -40,7 +40,7 @@ Generations findGenerations(Path profile, int & curGen)
     Generations gens;
 
     Path profileDir = dirOf(profile);
-    string profileName = baseNameOf(profile);
+    auto profileName = std::string(baseNameOf(profile));
 
     for (auto & i : readDirectory(profileDir)) {
         int n;
@@ -108,7 +108,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(outPath, generation, false, true);
+    store->addPermRoot(store->parseStorePath(outPath), generation, false, true);
 
     return generation;
 }
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 605ca98154f..102e1592180 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -89,7 +89,7 @@ PathSet scanForReferences(const string & path,
        hash part of the file name.  (This assumes that all references
        have the form `HASH-bla'). */
     for (auto & i : refs) {
-        string baseName = baseNameOf(i);
+        auto baseName = std::string(baseNameOf(i));
         string::size_type pos = baseName.find('-');
         if (pos == string::npos)
             throw Error(format("bad reference '%1%'") % i);
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 5233fb2c239..5a2d103b973 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -50,7 +50,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     auto storePath = store->toStorePath(path);
     std::string restPath = std::string(path, storePath.size());
 
-    if (!store->isValidPath(storePath))
+    if (!store->isValidPath(store->parseStorePath(storePath)))
         throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
 
     auto i = nars.find(storePath);
@@ -96,7 +96,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
         } catch (SysError &) { }
     }
 
-    store->narFromPath(storePath, sink);
+    store->narFromPath(store->parseStorePath(storePath), sink);
     auto narAccessor = makeNarAccessor(sink.s);
     addToCache(storePath, *sink.s, narAccessor);
     return {narAccessor, restPath};
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 2a89b7c98dc..8c55da2686e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -22,23 +22,22 @@
 namespace nix {
 
 
-Path readStorePath(Store & store, Source & from)
+template<> StorePathSet readStorePaths(const Store & store, Source & from)
 {
-    Path path = readString(from);
-    store.assertStorePath(path);
-    return path;
+    StorePathSet paths;
+    for (auto & i : readStrings<Strings>(from))
+        paths.insert(store.parseStorePath(i));
+    return paths;
 }
 
 
-template<class T> T readStorePaths(Store & store, Source & from)
+void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
 {
-    T paths = readStrings<T>(from);
-    for (auto & i : paths) store.assertStorePath(i);
-    return paths;
+    out << paths.size();
+    for (auto & i : paths)
+        out << store.printStorePath(i);
 }
 
-template PathSet readStorePaths(Store & store, Source & from);
-template Paths readStorePaths(Store & store, Source & from);
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -254,60 +253,62 @@ ConnectionHandle RemoteStore::getConnection()
 }
 
 
-bool RemoteStore::isValidPathUncached(const Path & path)
+bool RemoteStore::isValidPathUncached(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopIsValidPath << path;
+    conn->to << wopIsValidPath << printStorePath(path);
     conn.processStderr();
     return readInt(conn->from);
 }
 
 
-PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
+StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
     auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
-        PathSet res;
+        StorePathSet res;
         for (auto & i : paths)
-            if (isValidPath(i)) res.insert(i);
+            if (isValidPath(i)) res.insert(i.clone());
         return res;
     } else {
-        conn->to << wopQueryValidPaths << paths;
+        conn->to << wopQueryValidPaths;
+        writeStorePaths(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<PathSet>(*this, conn->from);
+        return readStorePaths<StorePathSet>(*this, conn->from);
     }
 }
 
 
-PathSet RemoteStore::queryAllValidPaths()
+StorePathSet RemoteStore::queryAllValidPaths()
 {
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return readStorePaths<PathSet>(*this, conn->from);
+    return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
-PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
+StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 {
     auto conn(getConnection());
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
-        PathSet res;
+        StorePathSet res;
         for (auto & i : paths) {
-            conn->to << wopHasSubstitutes << i;
+            conn->to << wopHasSubstitutes << printStorePath(i);
             conn.processStderr();
-            if (readInt(conn->from)) res.insert(i);
+            if (readInt(conn->from)) res.insert(i.clone());
         }
         return res;
     } else {
-        conn->to << wopQuerySubstitutablePaths << paths;
+        conn->to << wopQuerySubstitutablePaths;
+        writeStorePaths(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<PathSet>(*this, conn->from);
+        return readStorePaths<StorePathSet>(*this, conn->from);
     }
 }
 
 
-void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
     SubstitutablePathInfos & infos)
 {
     if (paths.empty()) return;
@@ -318,29 +319,31 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
         for (auto & i : paths) {
             SubstitutablePathInfo info;
-            conn->to << wopQuerySubstitutablePathInfo << i;
+            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
             conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
-            info.deriver = readString(conn->from);
-            if (info.deriver != "") assertStorePath(info.deriver);
-            info.references = readStorePaths<PathSet>(*this, conn->from);
+            auto deriver = readString(conn->from);
+            if (deriver != "")
+                info.deriver = parseStorePath(deriver);
+            info.references = readStorePaths<StorePathSet>(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
-            infos[i] = info;
+            infos.insert_or_assign(i.clone(), std::move(info));
         }
 
     } else {
 
-        conn->to << wopQuerySubstitutablePathInfos << paths;
+        conn->to << wopQuerySubstitutablePathInfos;
+        writeStorePaths(*this, conn->to, paths);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
-            Path path = readStorePath(*this, conn->from);
-            SubstitutablePathInfo & info(infos[path]);
-            info.deriver = readString(conn->from);
-            if (info.deriver != "") assertStorePath(info.deriver);
-            info.references = readStorePaths<PathSet>(*this, conn->from);
+            SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
+            auto deriver = readString(conn->from);
+            if (deriver != "")
+                info.deriver = parseStorePath(deriver);
+            info.references = readStorePaths<StorePathSet>(*this, conn->from);
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -349,14 +352,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 }
 
 
-void RemoteStore::queryPathInfoUncached(const Path & path,
+void RemoteStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
         std::shared_ptr<ValidPathInfo> info;
         {
             auto conn(getConnection());
-            conn->to << wopQueryPathInfo << path;
+            conn->to << wopQueryPathInfo << printStorePath(path);
             try {
                 conn.processStderr();
             } catch (Error & e) {
@@ -367,14 +370,13 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
             }
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
                 bool valid; conn->from >> valid;
-                if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
+                if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
             }
-            info = std::make_shared<ValidPathInfo>();
-            info->path = path;
-            info->deriver = readString(conn->from);
-            if (info->deriver != "") assertStorePath(info->deriver);
+            info = std::make_shared<ValidPathInfo>(path.clone());
+            auto deriver = readString(conn->from);
+            if (deriver != "") info->deriver = parseStorePath(deriver);
             info->narHash = Hash(readString(conn->from), htSHA256);
-            info->references = readStorePaths<PathSet>(*this, conn->from);
+            info->references = readStorePaths<StorePathSet>(*this, conn->from);
             conn->from >> info->registrationTime >> info->narSize;
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
                 conn->from >> info->ultimate;
@@ -387,52 +389,52 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
 }
 
 
-void RemoteStore::queryReferrers(const Path & path,
-    PathSet & referrers)
+void RemoteStore::queryReferrers(const StorePath & path,
+    StorePathSet & referrers)
 {
     auto conn(getConnection());
-    conn->to << wopQueryReferrers << path;
+    conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from);
-    referrers.insert(referrers2.begin(), referrers2.end());
+    for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+        referrers.insert(i.clone());
 }
 
 
-PathSet RemoteStore::queryValidDerivers(const Path & path)
+StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopQueryValidDerivers << path;
+    conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<PathSet>(*this, conn->from);
+    return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
-PathSet RemoteStore::queryDerivationOutputs(const Path & path)
+StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputs << path;
+    conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<PathSet>(*this, conn->from);
+    return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
-PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
+PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputNames << path;
+    conn->to << wopQueryDerivationOutputNames << printStorePath(path);
     conn.processStderr();
     return readStrings<PathSet>(conn->from);
 }
 
 
-Path RemoteStore::queryPathFromHashPart(const string & hashPart)
+std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
     conn->to << wopQueryPathFromHashPart << hashPart;
     conn.processStderr();
     Path path = readString(conn->from);
-    if (!path.empty()) assertStorePath(path);
-    return path;
+    if (path.empty()) return {};
+    return parseStorePath(path);
 }
 
 
@@ -450,9 +452,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             copyNAR(source, sink);
             sink
                 << exportMagic
-                << info.path
-                << info.references
-                << info.deriver
+                << printStorePath(info.path);
+            writeStorePaths(*this, sink, info.references);
+            sink
+                << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
                 << 0 // == no path follows
                 ;
@@ -460,14 +463,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
+        auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
         assert(importedPaths.size() <= 1);
     }
 
     else {
         conn->to << wopAddToStoreNar
-                 << info.path << info.deriver << info.narHash.to_string(Base16, false)
-                 << info.references << info.registrationTime << info.narSize
+                 << printStorePath(info.path)
+                 << (info.deriver ? printStorePath(*info.deriver) : "")
+                 << info.narHash.to_string(Base16, false);
+        writeStorePaths(*this, conn->to, info.references);
+        conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << info.ca
                  << repair << !checkSigs;
         bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
@@ -477,7 +483,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
+StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@@ -511,54 +517,52 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
         throw;
     }
 
-    return readStorePath(*this, conn->from);
+    return parseStorePath(readString(conn->from));
 }
 
 
-Path RemoteStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, RepairFlag repair)
+StorePath RemoteStore::addTextToStore(const string & name, const string & s,
+    const StorePathSet & references, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
     auto conn(getConnection());
-    conn->to << wopAddTextToStore << name << s << references;
+    conn->to << wopAddTextToStore << name << s;
+    writeStorePaths(*this, conn->to, references);
 
     conn.processStderr();
-    return readStorePath(*this, conn->from);
+    return parseStorePath(readString(conn->from));
 }
 
 
-void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
+void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
 {
     auto conn(getConnection());
     conn->to << wopBuildPaths;
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
-        conn->to << drvPaths;
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
-            conn->to << buildMode;
-        else
-            /* Old daemons did not take a 'buildMode' parameter, so we
-               need to validate it here on the client side.  */
-            if (buildMode != bmNormal)
-                throw Error("repairing or checking is not supported when building through the Nix daemon");
-    } else {
-        /* For backwards compatibility with old daemons, strip output
-           identifiers. */
-        PathSet drvPaths2;
-        for (auto & i : drvPaths)
-            drvPaths2.insert(string(i, 0, i.find('!')));
-        conn->to << drvPaths2;
-    }
+    assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
+    Strings ss;
+    for (auto & p : drvPaths)
+        ss.push_back(p.to_string(*this));
+    conn->to << ss;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
+        conn->to << buildMode;
+    else
+        /* Old daemons did not take a 'buildMode' parameter, so we
+           need to validate it here on the client side.  */
+        if (buildMode != bmNormal)
+            throw Error("repairing or checking is not supported when building through the Nix daemon");
     conn.processStderr();
     readInt(conn->from);
 }
 
 
-BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
     auto conn(getConnection());
-    conn->to << wopBuildDerivation << drvPath << drv << buildMode;
+    conn->to << wopBuildDerivation << printStorePath(drvPath);
+    writeDerivation(conn->to, *this, drv);
+    conn->to << buildMode;
     conn.processStderr();
     BuildResult res;
     unsigned int status;
@@ -568,19 +572,19 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
 }
 
 
-void RemoteStore::ensurePath(const Path & path)
+void RemoteStore::ensurePath(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopEnsurePath << path;
+    conn->to << wopEnsurePath << printStorePath(path);
     conn.processStderr();
     readInt(conn->from);
 }
 
 
-void RemoteStore::addTempRoot(const Path & path)
+void RemoteStore::addTempRoot(const StorePath & path)
 {
     auto conn(getConnection());
-    conn->to << wopAddTempRoot << path;
+    conn->to << wopAddTempRoot << printStorePath(path);
     conn.processStderr();
     readInt(conn->from);
 }
@@ -613,8 +617,8 @@ Roots RemoteStore::findRoots(bool censor)
     Roots result;
     while (count--) {
         Path link = readString(conn->from);
-        Path target = readStorePath(*this, conn->from);
-        result[target].emplace(link);
+        auto target = parseStorePath(readString(conn->from));
+        result[std::move(target)].emplace(link);
     }
     return result;
 }
@@ -625,7 +629,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     auto conn(getConnection());
 
     conn->to
-        << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
+        << wopCollectGarbage << options.action;
+    writeStorePaths(*this, conn->to, options.pathsToDelete);
+    conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
         << 0 << 0 << 0;
@@ -661,17 +667,17 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
 }
 
 
-void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
+void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     auto conn(getConnection());
-    conn->to << wopAddSignatures << storePath << sigs;
+    conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
     conn.processStderr();
     readInt(conn->from);
 }
 
 
-void RemoteStore::queryMissing(const PathSet & targets,
-    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+    StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
     unsigned long long & downloadSize, unsigned long long & narSize)
 {
     {
@@ -680,11 +686,15 @@ void RemoteStore::queryMissing(const PathSet & targets,
             // Don't hold the connection handle in the fallback case
             // to prevent a deadlock.
             goto fallback;
-        conn->to << wopQueryMissing << targets;
+        conn->to << wopQueryMissing;
+        Strings ss;
+        for (auto & p : targets)
+            ss.push_back(p.to_string(*this));
+        conn->to << ss;
         conn.processStderr();
-        willBuild = readStorePaths<PathSet>(*this, conn->from);
-        willSubstitute = readStorePaths<PathSet>(*this, conn->from);
-        unknown = readStorePaths<PathSet>(*this, conn->from);
+        willBuild = readStorePaths<StorePathSet>(*this, conn->from);
+        willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
+        unknown = readStorePaths<StorePathSet>(*this, conn->from);
         conn->from >> downloadSize >> narSize;
         return;
     }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 728df8b003e..f301a97d8c4 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -35,50 +35,50 @@ public:
 
     /* Implementations of abstract store API methods. */
 
-    bool isValidPathUncached(const Path & path) override;
+    bool isValidPathUncached(const StorePath & path) override;
 
-    PathSet queryValidPaths(const PathSet & paths,
+    StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
-    PathSet queryAllValidPaths() override;
+    StorePathSet queryAllValidPaths() override;
 
-    void queryPathInfoUncached(const Path & path,
+    void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
-    void queryReferrers(const Path & path, PathSet & referrers) override;
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
 
-    PathSet queryValidDerivers(const Path & path) override;
+    StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    PathSet queryDerivationOutputs(const Path & path) override;
+    StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const Path & path) override;
+    StringSet queryDerivationOutputNames(const StorePath & path) override;
 
-    Path queryPathFromHashPart(const string & hashPart) override;
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
-    PathSet querySubstitutablePaths(const PathSet & paths) override;
+    StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const PathSet & paths,
+    void querySubstitutablePathInfos(const StorePathSet & paths,
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair) override;
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override;
 
-    void buildPaths(const PathSet & paths, BuildMode buildMode) override;
+    void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
 
-    void ensurePath(const Path & path) override;
+    void ensurePath(const StorePath & path) override;
 
-    void addTempRoot(const Path & path) override;
+    void addTempRoot(const StorePath & path) override;
 
     void addIndirectRoot(const Path & path) override;
 
@@ -92,10 +92,10 @@ public:
 
     bool verifyStore(bool checkContents, RepairFlag repair) override;
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override;
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    void queryMissing(const PathSet & targets,
-        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         unsigned long long & downloadSize, unsigned long long & narSize) override;
 
     void connect() override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index cd547a96485..58966a5b796 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -222,7 +222,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
        fetches the .narinfo file, rather than first checking for its
        existence via a HEAD request. Since .narinfos are small, doing
        a GET is unlikely to be slower than HEAD. */
-    bool isValidPathUncached(const Path & storePath) override
+    bool isValidPathUncached(const StorePath & storePath) override
     {
         try {
             queryPathInfo(storePath);
@@ -382,9 +382,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
     }
 
-    PathSet queryAllValidPaths() override
+    StorePathSet queryAllValidPaths() override
     {
-        PathSet paths;
+        StorePathSet paths;
         std::string marker;
 
         do {
@@ -405,7 +405,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             for (auto object : contents) {
                 auto & key = object.GetKey();
                 if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
-                paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
+                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
             }
 
             marker = res.GetNextMarker();
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index b9d0c41eafc..42ee0650197 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -38,7 +38,7 @@ class SSHStore : public RemoteStore
     bool sameMachine() override
     { return false; }
 
-    void narFromPath(const Path & path, Sink & sink) override;
+    void narFromPath(const StorePath & path, Sink & sink) override;
 
     ref<FSAccessor> getFSAccessor() override;
 
@@ -66,10 +66,10 @@ class SSHStore : public RemoteStore
     };
 };
 
-void SSHStore::narFromPath(const Path & path, Sink & sink)
+void SSHStore::narFromPath(const StorePath & path, Sink & sink)
 {
     auto conn(connections->get());
-    conn->to << wopNarFromPath << path;
+    conn->to << wopNarFromPath << printStorePath(path);
     conn->processStderr();
     copyNAR(conn->from, sink);
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7e548ecd235..c98931885df 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -27,13 +27,6 @@ bool Store::isStorePath(const Path & path) const
 }
 
 
-void Store::assertStorePath(const Path & path) const
-{
-    if (!isStorePath(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
-}
-
-
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
@@ -60,17 +53,9 @@ Path Store::followLinksToStore(const Path & _path) const
 }
 
 
-Path Store::followLinksToStorePath(const Path & path) const
+StorePath Store::followLinksToStorePath(const Path & path) const
 {
-    return toStorePath(followLinksToStore(path));
-}
-
-
-string storePathToName(const Path & path)
-{
-    auto base = baseNameOf(path);
-    assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
-    return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
+    return parseStorePath(toStorePath(followLinksToStore(path)));
 }
 
 
@@ -82,41 +67,6 @@ string storePathToHash(const Path & path)
 }
 
 
-void checkStoreName(const string & name)
-{
-    string validChars = "+-._?=";
-
-    auto baseError = format("The path name '%2%' is invalid: %3%. "
-        "Path names are alphanumeric and can include the symbols %1% "
-        "and must not begin with a period. "
-        "Note: If '%2%' is a source file and you cannot rename it on "
-        "disk, 'builtins.path { name = ... }' can be used to give it an "
-        "alternative name.") % validChars % name;
-
-    if (name.empty())
-        throw Error(baseError % "it is an empty string");
-
-    /* Disallow names starting with a dot for possible security
-       reasons (e.g., "." and ".."). */
-    if (name[0] == '.')
-        throw Error(baseError % "it is illegal to start the name with a period");
-
-    /* Disallow names longer than 211 characters. ext4’s max is 256,
-       but we need extra space for the hash and .chroot extensions. */
-    if (name.length() > 211)
-        throw Error(baseError % "name must be less than 212 characters");
-
-    for (auto & i : name)
-        if (!((i >= 'A' && i <= 'Z') ||
-              (i >= 'a' && i <= 'z') ||
-              (i >= '0' && i <= '9') ||
-              validChars.find(i) != string::npos))
-        {
-            throw Error(baseError % (format("the '%1%' character is invalid") % i));
-        }
-}
-
-
 /* Store paths have the following form:
 
    <store>/<h>-<name>
@@ -188,43 +138,48 @@ void checkStoreName(const string & name)
 */
 
 
-Path Store::makeStorePath(const string & type,
-    const Hash & hash, const string & name) const
+StorePath Store::makeStorePath(const string & type,
+    const Hash & hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
-
-    checkStoreName(name);
-
-    return storeDir + "/"
-        + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false)
-        + "-" + name;
+    string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name);
+    auto h = compressHash(hashString(htSHA256, s), 20);
+    return StorePath::make(h.hash, name);
 }
 
 
-Path Store::makeOutputPath(const string & id,
-    const Hash & hash, const string & name) const
+StorePath Store::makeOutputPath(const string & id,
+    const Hash & hash, std::string_view name) const
 {
     return makeStorePath("output:" + id, hash,
-        name + (id == "out" ? "" : "-" + id));
+        std::string(name) + (id == "out" ? "" : "-" + id));
 }
 
 
-static std::string makeType(string && type, const PathSet & references)
+static std::string makeType(
+    const Store & store,
+    string && type,
+    const StorePathSet & references,
+    bool hasSelfReference = false)
 {
     for (auto & i : references) {
         type += ":";
-        type += i;
+        type += store.printStorePath(i);
     }
+    if (hasSelfReference) type += ":self";
     return std::move(type);
 }
 
 
-Path Store::makeFixedOutputPath(bool recursive,
-    const Hash & hash, const string & name, const PathSet & references) const
+StorePath Store::makeFixedOutputPath(
+    bool recursive,
+    const Hash & hash,
+    std::string_view name,
+    const StorePathSet & references,
+    bool hasSelfReference) const
 {
     if (hash.type == htSHA256 && recursive) {
-        return makeStorePath(makeType("source", references), hash, name);
+        return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
         return makeStorePath("output:out", hashString(htSHA256,
@@ -234,28 +189,27 @@ Path Store::makeFixedOutputPath(bool recursive,
 }
 
 
-Path Store::makeTextPath(const string & name, const Hash & hash,
-    const PathSet & references) const
+StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
+    const StorePathSet & references) const
 {
     assert(hash.type == htSHA256);
     /* Stuff the references (if any) into the type.  This is a bit
        hacky, but we can't put them in `s' since that would be
        ambiguous. */
-    return makeStorePath(makeType("text", references), hash, name);
+    return makeStorePath(makeType(*this, "text", references), hash, name);
 }
 
 
-std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
+std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
     const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
 {
     Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    Path dstPath = makeFixedOutputPath(recursive, h, name);
-    return std::pair<Path, Hash>(dstPath, h);
+    return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
 }
 
 
-Path Store::computeStorePathForText(const string & name, const string & s,
-    const PathSet & references) const
+StorePath Store::computeStorePathForText(const string & name, const string & s,
+    const StorePathSet & references) const
 {
     return makeTextPath(name, hashString(htSHA256, s), references);
 }
@@ -274,11 +228,9 @@ std::string Store::getUri()
 }
 
 
-bool Store::isValidPath(const Path & storePath)
+bool Store::isValidPath(const StorePath & storePath)
 {
-    assertStorePath(storePath);
-
-    auto hashPart = storePathToHash(storePath);
+    auto hashPart = storePathToHash(printStorePath(storePath));
 
     {
         auto state_(state.lock());
@@ -312,7 +264,7 @@ bool Store::isValidPath(const Path & storePath)
 
 /* Default implementation for stores that only implement
    queryPathInfoUncached(). */
-bool Store::isValidPathUncached(const Path & path)
+bool Store::isValidPathUncached(const StorePath & path)
 {
     try {
         queryPathInfo(path);
@@ -323,7 +275,7 @@ bool Store::isValidPathUncached(const Path & path)
 }
 
 
-ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
+ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
 {
     std::promise<ref<const ValidPathInfo>> promise;
 
@@ -340,22 +292,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 }
 
 
-void Store::queryPathInfo(const Path & storePath,
+void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
 {
     std::string hashPart;
 
     try {
-        assertStorePath(storePath);
-
-        hashPart = storePathToHash(storePath);
+        hashPart = storePathToHash(printStorePath(storePath));
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
             if (res) {
                 stats.narInfoReadAverted++;
                 if (!*res)
-                    throw InvalidPath(format("path '%s' is not valid") % storePath);
+                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
                 return callback(ref<const ValidPathInfo>(*res));
             }
         }
@@ -369,8 +319,8 @@ void Store::queryPathInfo(const Path & storePath,
                     state_->pathInfoCache.upsert(hashPart,
                         res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
                     if (res.first == NarInfoDiskCache::oInvalid ||
-                        (res.second->path != storePath && storePathToName(storePath) != ""))
-                        throw InvalidPath(format("path '%s' is not valid") % storePath);
+                        res.second->path != storePath)
+                        throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
                 }
                 return callback(ref<const ValidPathInfo>(res.second));
             }
@@ -381,7 +331,7 @@ void Store::queryPathInfo(const Path & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     queryPathInfoUncached(storePath,
-        {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+        {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -394,9 +344,7 @@ void Store::queryPathInfo(const Path & storePath,
                     state_->pathInfoCache.upsert(hashPart, info);
                 }
 
-                if (!info
-                    || (info->path != storePath && storePathToName(storePath) != ""))
-                {
+                if (!info || info->path != parseStorePath(storePath)) {
                     stats.narInfoMissing++;
                     throw InvalidPath("path '%s' is not valid", storePath);
                 }
@@ -407,27 +355,27 @@ void Store::queryPathInfo(const Path & storePath,
 }
 
 
-PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
+StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
     struct State
     {
         size_t left;
-        PathSet valid;
+        StorePathSet valid;
         std::exception_ptr exc;
     };
 
-    Sync<State> state_(State{paths.size(), PathSet()});
+    Sync<State> state_(State{paths.size(), StorePathSet()});
 
     std::condition_variable wakeup;
     ThreadPool pool;
 
-    auto doQuery = [&](const Path & path ) {
+    auto doQuery = [&](const Path & path) {
         checkInterrupt();
-        queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
+        queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
             auto state(state_.lock());
             try {
                 auto info = fut.get();
-                state->valid.insert(path);
+                state->valid.insert(parseStorePath(path));
             } catch (InvalidPath &) {
             } catch (...) {
                 state->exc = std::current_exception();
@@ -439,7 +387,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
     };
 
     for (auto & path : paths)
-        pool.enqueue(std::bind(doQuery, path));
+        pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME
 
     pool.process();
 
@@ -447,7 +395,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
         auto state(state_.lock());
         if (!state->left) {
             if (state->exc) std::rethrow_exception(state->exc);
-            return state->valid;
+            return std::move(state->valid);
         }
         state.wait(wakeup);
     }
@@ -457,13 +405,13 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
 /* Return a string accepted by decodeValidPathInfo() that
    registers the specified paths as valid.  Note: it's the
    responsibility of the caller to provide a closure. */
-string Store::makeValidityRegistration(const PathSet & paths,
+string Store::makeValidityRegistration(const StorePathSet & paths,
     bool showDerivers, bool showHash)
 {
     string s = "";
 
     for (auto & i : paths) {
-        s += i + "\n";
+        s += printStorePath(i) + "\n";
 
         auto info = queryPathInfo(i);
 
@@ -472,31 +420,30 @@ string Store::makeValidityRegistration(const PathSet & paths,
             s += (format("%1%\n") % info->narSize).str();
         }
 
-        Path deriver = showDerivers ? info->deriver : "";
+        auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : "";
         s += deriver + "\n";
 
         s += (format("%1%\n") % info->references.size()).str();
 
         for (auto & j : info->references)
-            s += j + "\n";
+            s += printStorePath(j) + "\n";
     }
 
     return s;
 }
 
 
-void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
+void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
     bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
 {
     auto jsonList = jsonOut.list();
 
-    for (auto storePath : storePaths) {
+    for (auto & storePath : storePaths) {
         auto jsonPath = jsonList.object();
-        jsonPath.attr("path", storePath);
+        jsonPath.attr("path", printStorePath(storePath));
 
         try {
             auto info = queryPathInfo(storePath);
-            storePath = info->path;
 
             jsonPath
                 .attr("narHash", info->narHash.to_string())
@@ -505,7 +452,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
             {
                 auto jsonRefs = jsonPath.list("references");
                 for (auto & ref : info->references)
-                    jsonRefs.elem(ref);
+                    jsonRefs.elem(printStorePath(ref));
             }
 
             if (info->ca != "")
@@ -514,14 +461,14 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
             std::pair<uint64_t, uint64_t> closureSizes;
 
             if (showClosureSize) {
-                closureSizes = getClosureSize(storePath);
+                closureSizes = getClosureSize(info->path);
                 jsonPath.attr("closureSize", closureSizes.first);
             }
 
             if (includeImpureInfo) {
 
-                if (info->deriver != "")
-                    jsonPath.attr("deriver", info->deriver);
+                if (info->deriver)
+                    jsonPath.attr("deriver", printStorePath(*info->deriver));
 
                 if (info->registrationTime)
                     jsonPath.attr("registrationTime", info->registrationTime);
@@ -557,10 +504,10 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
 }
 
 
-std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
+std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath)
 {
     uint64_t totalNarSize = 0, totalDownloadSize = 0;
-    PathSet closure;
+    StorePathSet closure;
     computeFSClosure(storePath, closure, false, false);
     for (auto & p : closure) {
         auto info = queryPathInfo(p);
@@ -584,30 +531,34 @@ const Store::Stats & Store::getStats()
 }
 
 
-void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
+void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode)
 {
-    for (auto & path : paths)
-        if (isDerivation(path))
+    StorePathSet paths2;
+
+    for (auto & path : paths) {
+        if (path.path.isDerivation())
             unsupported("buildPaths");
+        paths2.insert(path.path.clone());
+    }
 
-    if (queryValidPaths(paths).size() != paths.size())
+    if (queryValidPaths(paths2).size() != paths2.size())
         unsupported("buildPaths");
 }
 
 
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
-    const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
+    const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
 {
     auto srcUri = srcStore->getUri();
     auto dstUri = dstStore->getUri();
 
     Activity act(*logger, lvlInfo, actCopyPath,
         srcUri == "local" || srcUri == "daemon"
-          ? fmt("copying path '%s' to '%s'", storePath, dstUri)
+        ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri)
           : dstUri == "local" || dstUri == "daemon"
-            ? fmt("copying path '%s' from '%s'", storePath, srcUri)
-            : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
-        {storePath, srcUri, dstUri});
+        ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri)
+          : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri),
+        {srcStore->printStorePath(storePath), srcUri, dstUri});
     PushActivity pact(act.id);
 
     auto info = srcStore->queryPathInfo(storePath);
@@ -640,23 +591,23 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
             total += len;
             act.progress(total, info->narSize);
         });
-        srcStore->narFromPath({storePath}, wrapperSink);
+        srcStore->narFromPath(storePath, wrapperSink);
     }, [&]() {
-        throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
+           throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri());
     });
 
     dstStore->addToStore(*info, *source, repair, checkSigs);
 }
 
 
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
+void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
-    PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
+    auto valid = dstStore->queryValidPaths(storePaths, substitute);
 
     PathSet missing;
     for (auto & path : storePaths)
-        if (!valid.count(path)) missing.insert(path);
+        if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
 
     if (missing.empty()) return;
 
@@ -677,23 +628,25 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
         PathSet(missing.begin(), missing.end()),
 
         [&](const Path & storePath) {
-            if (dstStore->isValidPath(storePath)) {
+            if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
                 nrDone++;
                 showProgress();
                 return PathSet();
             }
 
-            auto info = srcStore->queryPathInfo(storePath);
+            auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
 
             bytesExpected += info->narSize;
             act.setExpected(actCopyPath, bytesExpected);
 
-            return info->references;
+            return srcStore->printStorePathSet(info->references);
         },
 
-        [&](const Path & storePath) {
+        [&](const Path & storePathS) {
             checkInterrupt();
 
+            auto storePath = dstStore->parseStorePath(storePathS);
+
             if (!dstStore->isValidPath(storePath)) {
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
@@ -703,7 +656,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
                     nrFailed++;
                     if (!settings.keepGoing)
                         throw e;
-                    logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
+                    logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
                     showProgress();
                     return;
                 }
@@ -716,20 +669,36 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
 
 
 void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
+    const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
     SubstituteFlag substitute)
 {
-    PathSet closure;
-    srcStore->computeFSClosure({storePaths}, closure);
+    StorePathSet closure;
+    srcStore->computeFSClosure(storePaths, closure);
     copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
 }
 
 
-ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
+ValidPathInfo::ValidPathInfo(const ValidPathInfo & other)
+    : path(other.path.clone())
+    , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{})
+    , narHash(other.narHash)
+    , references(cloneStorePathSet(other.references))
+    , registrationTime(other.registrationTime)
+    , narSize(other.narSize)
+    , id(other.id)
+    , ultimate(other.ultimate)
+    , sigs(other.sigs)
+    , ca(other.ca)
+{
+}
+
+
+std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven)
 {
-    ValidPathInfo info;
-    getline(str, info.path);
-    if (str.eof()) { info.path = ""; return info; }
+    std::string path;
+    getline(str, path);
+    if (str.eof()) { return {}; }
+    ValidPathInfo info(store.parseStorePath(path));
     if (hashGiven) {
         string s;
         getline(str, s);
@@ -737,16 +706,29 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
         getline(str, s);
         if (!string2Int(s, info.narSize)) throw Error("number expected");
     }
-    getline(str, info.deriver);
+    std::string deriver;
+    getline(str, deriver);
+    if (deriver != "") info.deriver = store.parseStorePath(deriver);
     string s; int n;
     getline(str, s);
     if (!string2Int(s, n)) throw Error("number expected");
     while (n--) {
         getline(str, s);
-        info.references.insert(s);
+        info.references.insert(store.parseStorePath(s));
     }
     if (!str || str.eof()) throw Error("missing input");
-    return info;
+    return std::optional<ValidPathInfo>(std::move(info));
+}
+
+
+std::string Store::showPaths(const StorePathSet & paths)
+{
+    std::string s;
+    for (auto & i : paths) {
+        if (s.size() != 0) s += ", ";
+        s += "'" + printStorePath(i) + "'";
+    }
+    return s;
 }
 
 
@@ -761,34 +743,34 @@ string showPaths(const PathSet & paths)
 }
 
 
-std::string ValidPathInfo::fingerprint() const
+std::string ValidPathInfo::fingerprint(const Store & store) const
 {
     if (narSize == 0 || !narHash)
-        throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known")
-            % path);
+        throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known",
+            store.printStorePath(path));
     return
-        "1;" + path + ";"
+        "1;" + store.printStorePath(path) + ";"
         + narHash.to_string(Base32) + ";"
         + std::to_string(narSize) + ";"
-        + concatStringsSep(",", references);
+        + concatStringsSep(",", store.printStorePathSet(references));
 }
 
 
-void ValidPathInfo::sign(const SecretKey & secretKey)
+void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
 {
-    sigs.insert(secretKey.signDetached(fingerprint()));
+    sigs.insert(secretKey.signDetached(fingerprint(store)));
 }
 
 
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
     auto warn = [&]() {
-        printError(format("warning: path '%s' claims to be content-addressed but isn't") % path);
+        printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
     };
 
     if (hasPrefix(ca, "text:")) {
         Hash hash(std::string(ca, 5));
-        if (store.makeTextPath(storePathToName(path), hash, references) == path)
+        if (store.makeTextPath(path.name(), hash, references) == path)
             return true;
         else
             warn();
@@ -797,9 +779,13 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     else if (hasPrefix(ca, "fixed:")) {
         bool recursive = ca.compare(6, 2, "r:") == 0;
         Hash hash(std::string(ca, recursive ? 8 : 6));
-        auto refs = references;
-        replaceInSet(refs, path, std::string("self"));
-        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
+        auto refs = cloneStorePathSet(references);
+        bool hasSelfReference = false;
+        if (refs.count(path)) {
+            hasSelfReference = true;
+            refs.erase(path);
+        }
+        if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
             return true;
         else
             warn();
@@ -815,15 +801,15 @@ size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & pu
 
     size_t good = 0;
     for (auto & sig : sigs)
-        if (checkSignature(publicKeys, sig))
+        if (checkSignature(store, publicKeys, sig))
             good++;
     return good;
 }
 
 
-bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
+bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
 {
-    return verifyDetached(fingerprint(), sig, publicKeys);
+    return verifyDetached(fingerprint(store), sig, publicKeys);
 }
 
 
@@ -831,7 +817,7 @@ Strings ValidPathInfo::shortRefs() const
 {
     Strings refs;
     for (auto & r : references)
-        refs.push_back(baseNameOf(r));
+        refs.push_back(std::string(r.to_string()));
     return refs;
 }
 
@@ -944,7 +930,7 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
+    switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
         case tDaemon:
             return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
         case tLocal: {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8e4b133d575..552bbd67d47 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "path.hh"
 #include "hash.hh"
 #include "serialise.hh"
 #include "crypto.hh"
@@ -42,14 +43,11 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
 enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 
 
-/* Size of the hash part of store paths, in base-32 characters. */
-const size_t storePathHashLen = 32; // i.e. 160 bits
-
 /* Magic header of exportPath() output (obsolete). */
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
+typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
 
 
 struct GCOptions
@@ -83,7 +81,7 @@ struct GCOptions
     bool ignoreLiveness{false};
 
     /* For `gcDeleteSpecific', the paths to delete. */
-    PathSet pathsToDelete;
+    StorePathSet pathsToDelete;
 
     /* Stop after at least `maxFreed' bytes have been freed. */
     unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
@@ -104,21 +102,21 @@ struct GCResults
 
 struct SubstitutablePathInfo
 {
-    Path deriver;
-    PathSet references;
+    std::optional<StorePath> deriver;
+    StorePathSet references;
     unsigned long long downloadSize; /* 0 = unknown or inapplicable */
     unsigned long long narSize; /* 0 = unknown */
 };
 
-typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
 
 
 struct ValidPathInfo
 {
-    Path path;
-    Path deriver;
+    StorePath path;
+    std::optional<StorePath> deriver;
     Hash narHash;
-    PathSet references;
+    StorePathSet references;
     time_t registrationTime = 0;
     uint64_t narSize = 0; // 0 = unknown
     uint64_t id; // internal use only
@@ -143,7 +141,7 @@ struct ValidPathInfo
 
        Ideally, the content-addressability assertion would just be a
        Boolean, and the store path would be computed from
-       ‘storePathToName(path)’, ‘narHash’ and ‘references’. However,
+       the name component, ‘narHash’ and ‘references’. However,
        1) we've accumulated several types of content-addressed paths
        over the years; and 2) fixed-output derivations support
        multiple hash algorithms and serialisation methods (flat file
@@ -171,9 +169,9 @@ struct ValidPathInfo
        the NAR, and the sorted references. The size field is strictly
        speaking superfluous, but might prevent endless/excessive data
        attacks. */
-    std::string fingerprint() const;
+    std::string fingerprint(const Store & store) const;
 
-    void sign(const SecretKey & secretKey);
+    void sign(const Store & store, const SecretKey & secretKey);
 
     /* Return true iff the path is verifiably content-addressed. */
     bool isContentAddressed(const Store & store) const;
@@ -186,10 +184,13 @@ struct ValidPathInfo
     size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
 
     /* Verify a single signature. */
-    bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
+    bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
 
     Strings shortRefs() const;
 
+    ValidPathInfo(StorePath && path) : path(std::move(path)) { }
+    explicit ValidPathInfo(const ValidPathInfo & other);
+
     virtual ~ValidPathInfo() { }
 };
 
@@ -240,6 +241,23 @@ struct BuildResult
 };
 
 
+struct StorePathWithOutputs
+{
+    StorePath path;
+    std::set<std::string> outputs;
+
+    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
+        : path(path.clone()), outputs(outputs)
+    { }
+
+    StorePathWithOutputs(const StorePathWithOutputs & other)
+        : path(other.path.clone()), outputs(other.outputs)
+    { }
+
+    std::string to_string(const Store & store) const;
+};
+
+
 class Store : public std::enable_shared_from_this<Store>, public Config
 {
 public:
@@ -258,6 +276,7 @@ protected:
 
     struct State
     {
+        // FIXME: fix key
         LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
     };
 
@@ -273,6 +292,24 @@ public:
 
     virtual std::string getUri() = 0;
 
+    StorePath parseStorePath(std::string_view path) const;
+
+    std::string printStorePath(const StorePath & path) const;
+
+    // FIXME: remove
+    StorePathSet parseStorePathSet(const PathSet & paths) const;
+
+    PathSet printStorePathSet(const StorePathSet & path) const;
+
+    /* Split a string specifying a derivation and a set of outputs
+       (/nix/store/hash-foo!out1,out2,...) into the derivation path
+       and the outputs. */
+    StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
+
+    /* Display a set of paths in human-readable form (i.e., between quotes
+       and separated by commas). */
+    std::string showPaths(const StorePathSet & paths);
+
     /* Return true if ‘path’ is in the Nix store (but not the Nix
        store itself). */
     bool isInStore(const Path & path) const;
@@ -281,9 +318,6 @@ public:
        the Nix store. */
     bool isStorePath(const Path & path) const;
 
-    /* Throw an exception if ‘path’ is not a store path. */
-    void assertStorePath(const Path & path) const;
-
     /* Chop off the parts after the top-level store name, e.g.,
        /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
     Path toStorePath(const Path & path) const;
@@ -293,26 +327,27 @@ public:
 
     /* Same as followLinksToStore(), but apply toStorePath() to the
        result. */
-    Path followLinksToStorePath(const Path & path) const;
+    StorePath followLinksToStorePath(const Path & path) const;
 
     /* Constructs a unique store path name. */
-    Path makeStorePath(const string & type,
-        const Hash & hash, const string & name) const;
+    StorePath makeStorePath(const string & type,
+        const Hash & hash, std::string_view name) const;
 
-    Path makeOutputPath(const string & id,
-        const Hash & hash, const string & name) const;
+    StorePath makeOutputPath(const string & id,
+        const Hash & hash, std::string_view name) const;
 
-    Path makeFixedOutputPath(bool recursive,
-        const Hash & hash, const string & name,
-        const PathSet & references = {}) const;
+    StorePath makeFixedOutputPath(bool recursive,
+        const Hash & hash, std::string_view name,
+        const StorePathSet & references = {},
+        bool hasSelfReference = false) const;
 
-    Path makeTextPath(const string & name, const Hash & hash,
-        const PathSet & references) const;
+    StorePath makeTextPath(std::string_view name, const Hash & hash,
+        const StorePathSet & references) const;
 
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
-    std::pair<Path, Hash> computeStorePathForPath(const string & name,
+    std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
         const Path & srcPath, bool recursive = true,
         HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
@@ -330,21 +365,21 @@ public:
        simply yield a different store path, so other users wouldn't be
        affected), but it has some backwards compatibility issues (the
        hashing scheme changes), so I'm not doing that for now. */
-    Path computeStorePathForText(const string & name, const string & s,
-        const PathSet & references) const;
+    StorePath computeStorePathForText(const string & name, const string & s,
+        const StorePathSet & references) const;
 
     /* Check whether a path is valid. */
-    bool isValidPath(const Path & path);
+    bool isValidPath(const StorePath & path);
 
 protected:
 
-    virtual bool isValidPathUncached(const Path & path);
+    virtual bool isValidPathUncached(const StorePath & path);
 
 public:
 
     /* Query which of the given paths is valid. Optionally, try to
        substitute missing paths. */
-    virtual PathSet queryValidPaths(const PathSet & paths,
+    virtual StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute);
 
     /* Query the set of all valid paths. Note that for some store
@@ -352,54 +387,54 @@ public:
        (i.e. you'll get /nix/store/<hash> rather than
        /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
        full store path. */
-    virtual PathSet queryAllValidPaths()
+    virtual StorePathSet queryAllValidPaths()
     { unsupported("queryAllValidPaths"); }
 
     /* Query information about a valid path. It is permitted to omit
        the name part of the store path. */
-    ref<const ValidPathInfo> queryPathInfo(const Path & path);
+    ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
 
     /* Asynchronous version of queryPathInfo(). */
-    void queryPathInfo(const Path & path,
+    void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
 protected:
 
-    virtual void queryPathInfoUncached(const Path & path,
+    virtual void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
 
 public:
 
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
-    virtual void queryReferrers(const Path & path, PathSet & referrers)
+    virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
     { unsupported("queryReferrers"); }
 
     /* Return all currently valid derivations that have `path' as an
        output.  (Note that the result of `queryDeriver()' is the
        derivation that was actually used to produce `path', which may
        not exist anymore.) */
-    virtual PathSet queryValidDerivers(const Path & path) { return {}; };
+    virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual PathSet queryDerivationOutputs(const Path & path)
+    virtual StorePathSet queryDerivationOutputs(const StorePath & path)
     { unsupported("queryDerivationOutputs"); }
 
     /* Query the output names of the derivation denoted by `path'. */
-    virtual StringSet queryDerivationOutputNames(const Path & path)
+    virtual StringSet queryDerivationOutputNames(const StorePath & path)
     { unsupported("queryDerivationOutputNames"); }
 
     /* Query the full store path given the hash part of a valid store
-       path, or "" if the path doesn't exist. */
-    virtual Path queryPathFromHashPart(const string & hashPart) = 0;
+       path, or empty if the path doesn't exist. */
+    virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
 
     /* Query which of the given paths have substitutes. */
-    virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; };
+    virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
 
     /* Query substitute info (i.e. references, derivers and download
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
-    virtual void querySubstitutablePathInfos(const PathSet & paths,
+    virtual void querySubstitutablePathInfos(const StorePathSet & paths,
         SubstitutablePathInfos & infos) { return; };
 
     virtual bool wantMassQuery() { return false; }
@@ -418,12 +453,12 @@ public:
        validity the resulting path.  The resulting path is returned.
        The function object `filter' can be used to exclude files (see
        libutil/archive.hh). */
-    virtual Path addToStore(const string & name, const Path & srcPath,
+    virtual StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
     // FIXME: remove?
-    virtual Path addToStoreFromDump(const string & dump, const string & name,
+    virtual StorePath addToStoreFromDump(const string & dump, const string & name,
         bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
     {
         throw Error("addToStoreFromDump() is not supported by this store");
@@ -431,11 +466,11 @@ public:
 
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */
-    virtual Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair = NoRepair) = 0;
+    virtual StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
 
     /* Write a NAR dump of a store path. */
-    virtual void narFromPath(const Path & path, Sink & sink) = 0;
+    virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
 
     /* For each path, if it's a derivation, build it.  Building a
        derivation means ensuring that the output paths are valid.  If
@@ -445,22 +480,24 @@ public:
        output paths can be created by running the builder, after
        recursively building any sub-derivations. For inputs that are
        not derivations, substitute them. */
-    virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal);
+    virtual void buildPaths(
+        const std::vector<StorePathWithOutputs> & paths,
+        BuildMode buildMode = bmNormal);
 
     /* Build a single non-materialized derivation (i.e. not from an
        on-disk .drv file). Note that ‘drvPath’ is only used for
        informational purposes. */
-    virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode = bmNormal) = 0;
 
     /* Ensure that a path is valid.  If it is not currently valid, it
        may be made valid by running a substitute (if defined for the
        path). */
-    virtual void ensurePath(const Path & path) = 0;
+    virtual void ensurePath(const StorePath & path) = 0;
 
     /* Add a store path as a temporary root of the garbage collector.
        The root disappears as soon as we exit. */
-    virtual void addTempRoot(const Path & path)
+    virtual void addTempRoot(const StorePath & path)
     { unsupported("addTempRoot"); }
 
     /* Add an indirect root, which is merely a symlink to `path' from
@@ -506,7 +543,7 @@ public:
     /* Return a string representing information about the path that
        can be loaded into the database using `nix-store --load-db' or
        `nix-store --register-validity'. */
-    string makeValidityRegistration(const PathSet & paths,
+    string makeValidityRegistration(const StorePathSet & paths,
         bool showDerivers, bool showHash);
 
     /* Write a JSON representation of store path metadata, such as the
@@ -514,14 +551,14 @@ public:
        variable elements such as the registration time are
        included. If ‘showClosureSize’ is true, the closure size of
        each path is included. */
-    void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
+    void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
         bool includeImpureInfo, bool showClosureSize,
         AllowInvalidFlag allowInvalid = DisallowInvalid);
 
     /* Return the size of the closure of the specified path, that is,
        the sum of the size of the NAR serialisation of each path in
        the closure. */
-    std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath);
+    std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath);
 
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
@@ -537,14 +574,14 @@ public:
 
     /* Add signatures to the specified store path. The signatures are
        not verified. */
-    virtual void addSignatures(const Path & storePath, const StringSet & sigs)
+    virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)
     { unsupported("addSignatures"); }
 
     /* Utility functions. */
 
     /* Read a derivation, after ensuring its existence through
        ensurePath(). */
-    Derivation derivationFromPath(const Path & drvPath);
+    Derivation derivationFromPath(const StorePath & drvPath);
 
     /* Place in `out' the set of all store paths in the file system
        closure of `storePath'; that is, all paths than can be directly
@@ -553,36 +590,36 @@ public:
        `storePath' is returned; that is, the closures under the
        `referrers' relation instead of the `references' relation is
        returned. */
-    virtual void computeFSClosure(const PathSet & paths,
-        PathSet & out, bool flipDirection = false,
+    virtual void computeFSClosure(const StorePathSet & paths,
+        StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false);
 
-    void computeFSClosure(const Path & path,
-        PathSet & out, bool flipDirection = false,
+    void computeFSClosure(const StorePath & path,
+        StorePathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false);
 
     /* Given a set of paths that are to be built, return the set of
        derivations that will be built, and the set of output paths
        that will be substituted. */
-    virtual void queryMissing(const PathSet & targets,
-        PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
         unsigned long long & downloadSize, unsigned long long & narSize);
 
     /* Sort a set of paths topologically under the references
        relation.  If p refers to q, then p precedes q in this list. */
-    Paths topoSortPaths(const PathSet & paths);
+    StorePaths topoSortPaths(const StorePathSet & paths);
 
     /* Export multiple paths in the format expected by ‘nix-store
        --import’. */
-    void exportPaths(const Paths & paths, Sink & sink);
+    void exportPaths(const StorePathSet & paths, Sink & sink);
 
-    void exportPath(const Path & path, Sink & sink);
+    void exportPath(const StorePath & path, Sink & sink);
 
     /* Import a sequence of NAR dumps created by exportPaths() into
        the Nix store. Optionally, the contents of the NARs are
        preloaded into the specified FS accessor to speed up subsequent
        access. */
-    Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
+    StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         CheckSigsFlag checkSigs = CheckSigs);
 
     struct Stats
@@ -606,7 +643,7 @@ public:
 
     /* Return the build log of the specified store path, if available,
        or null otherwise. */
-    virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
+    virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
     { return nullptr; }
 
     /* Hack to allow long-running processes like hydra-queue-runner to
@@ -672,11 +709,11 @@ public:
 
     LocalFSStore(const Params & params);
 
-    void narFromPath(const Path & path, Sink & sink) override;
+    void narFromPath(const StorePath & path, Sink & sink) override;
     ref<FSAccessor> getFSAccessor() override;
 
     /* Register a permanent GC root. */
-    Path addPermRoot(const Path & storePath,
+    Path addPermRoot(const StorePath & storePath,
         const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
 
     virtual Path getRealStoreDir() { return storeDir; }
@@ -687,25 +724,17 @@ public:
         return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
     }
 
-    std::shared_ptr<std::string> getBuildLog(const Path & path) override;
+    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
 };
 
 
-/* Extract the name part of the given store path. */
-string storePathToName(const Path & path);
-
 /* Extract the hash part of the given store path. */
 string storePathToHash(const Path & path);
 
-/* Check whether ‘name’ is a valid store path name part, i.e. contains
-   only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with
-   a dot. */
-void checkStoreName(const string & name);
-
 
 /* Copy a path from one store to another. */
 void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
-    const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
+    const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
 
 
 /* Copy store paths from one store to another. The paths may be copied
@@ -713,7 +742,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
    (i.e. if A is a reference of B, then A is copied before B), but
    the set of store paths is not automatically closed; use
    copyClosure() for that. */
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
+void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -721,7 +750,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
 
 /* Copy the closure of the specified paths from one store to another. */
 void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
-    const PathSet & storePaths,
+    const StorePathSet & storePaths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -804,7 +833,9 @@ struct RegisterStoreImplementation
 string showPaths(const PathSet & paths);
 
 
-ValidPathInfo decodeValidPathInfo(std::istream & str,
+std::optional<ValidPathInfo> decodeValidPathInfo(
+    const Store & store,
+    std::istream & str,
     bool hashGiven = false);
 
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6762b609ddb..857d54d993f 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -65,8 +65,9 @@ typedef enum {
 class Store;
 struct Source;
 
-Path readStorePath(Store & store, Source & from);
-template<class T> T readStorePaths(Store & store, Source & from);
+template<class T> T readStorePaths(const Store & store, Source & from);
+
+void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
 
 
 }
diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc
index 931d29542ee..accc5e22b97 100644
--- a/src/libutil/rust-ffi.cc
+++ b/src/libutil/rust-ffi.cc
@@ -1,12 +1,18 @@
 #include "logging.hh"
 #include "rust-ffi.hh"
 
-namespace nix {
-
 extern "C" std::exception_ptr * make_error(rust::StringSlice s)
 {
     // FIXME: leak
-    return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size))));
+    return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size))));
+}
+
+namespace rust {
+
+std::ostream & operator << (std::ostream & str, const String & s)
+{
+    str << (std::string_view) s;
+    return str;
 }
 
 }
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index 663758bfcb6..a77f83ac535 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -1,16 +1,91 @@
+#pragma once
+
 #include "serialise.hh"
 
+#include <string_view>
+#include <cstring>
+#include <array>
+
 namespace rust {
 
-// Depending on the internal representation of Rust slices is slightly
-// evil...
+typedef void (*DropFun)(void *);
+
+/* A Rust value of N bytes. It can be moved but not copied. When it
+   goes out of scope, the C++ destructor will run the drop
+   function. */
+template<std::size_t N, DropFun drop>
+struct Value
+{
+protected:
+
+    std::array<char, N> raw;
+
+    ~Value()
+    {
+        if (!isEvacuated()) {
+            drop(this);
+            evacuate();
+        }
+    }
+
+    // Must not be called directly.
+    Value()
+    { }
+
+    Value(Value && other)
+        : raw(other.raw)
+    {
+        other.evacuate();
+    }
+
+    void operator =(Value && other)
+    {
+        if (!isEvacuated())
+            drop(this);
+        raw = other.raw;
+        other.evacuate();
+    }
+
+private:
+
+    /* FIXME: optimize these (ideally in such a way that the compiler
+       can elide most calls to evacuate() / isEvacuated(). */
+    inline void evacuate()
+    {
+        for (auto & i : raw) i = 0;
+    }
+
+    inline bool isEvacuated()
+    {
+        for (auto & i : raw)
+            if (i != 0) return false;
+        return true;
+    }
+};
+
+/* A Rust vector. */
+template<typename T, DropFun drop>
+struct Vec : Value<3 * sizeof(void *), drop>
+{
+    inline size_t size() const
+    {
+        return ((const size_t *) &this->raw)[2];
+    }
+
+    const T * data() const
+    {
+        return ((const T * *) &this->raw)[0];
+    }
+};
+
+/* A Rust slice. */
 template<typename T>
 struct Slice
 {
-    T * ptr;
+    const T * ptr;
     size_t size;
 
-    Slice(T * ptr, size_t size) : ptr(ptr), size(size)
+    Slice(const T * ptr, size_t size) : ptr(ptr), size(size)
     {
         assert(ptr);
     }
@@ -18,9 +93,44 @@ struct Slice
 
 struct StringSlice : Slice<char>
 {
-    StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
+    StringSlice(const std::string & s): Slice(s.data(), s.size()) {}
+    explicit StringSlice(std::string_view s): Slice(s.data(), s.size()) {}
+    StringSlice(const char * s): Slice(s, strlen(s)) {}
+
+    operator std::string_view() const
+    {
+        return std::string_view(ptr, size);
+    }
+};
+
+/* A Rust string. */
+struct String;
+
+extern "C" {
+    void ffi_String_new(StringSlice s, String * out);
+    void ffi_String_drop(void * s);
+}
+
+struct String : Vec<char, ffi_String_drop>
+{
+    String(std::string_view s)
+    {
+        ffi_String_new(StringSlice(s), this);
+    }
+
+    String(const char * s)
+        : String({s, std::strlen(s)})
+    {
+    }
+
+    operator std::string_view() const
+    {
+        return std::string_view(data(), size());
+    }
 };
 
+std::ostream & operator << (std::ostream & str, const String & s);
+
 struct Source
 {
     size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
@@ -33,7 +143,7 @@ struct Source
     // FIXME: how to propagate exceptions?
     static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
     {
-        auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
+        auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
         return n;
     }
 };
@@ -49,11 +159,20 @@ struct Result
         std::exception_ptr * exc;
     };
 
+    ~Result()
+    {
+        if (tag == 0)
+            data.~T();
+        else if (tag == 1)
+            // FIXME: don't leak exc
+            ;
+    }
+
     /* Rethrow the wrapped exception or return the wrapped value. */
     T unwrap()
     {
         if (tag == 0)
-            return data;
+            return std::move(data);
         else if (tag == 1)
             std::rethrow_exception(*exc);
         else
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 2cc7793fde8..262bc655f16 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -17,7 +17,7 @@ void unpackTarfile(Source & source, const Path & destDir)
 void unpackTarfile(const Path & tarFile, const Path & destDir,
     std::optional<std::string> baseName)
 {
-    if (!baseName) baseName = baseNameOf(tarFile);
+    if (!baseName) baseName = std::string(baseNameOf(tarFile));
 
     auto source = sinkToSource([&](Sink & sink) {
         // FIXME: look at first few bytes to determine compression type.
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index b4d444a6746..e67bdcdebb8 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -188,22 +188,22 @@ Path dirOf(const Path & path)
 }
 
 
-string baseNameOf(const Path & path)
+std::string_view baseNameOf(std::string_view path)
 {
     if (path.empty())
         return "";
 
-    Path::size_type last = path.length() - 1;
+    auto last = path.size() - 1;
     if (path[last] == '/' && last > 0)
         last -= 1;
 
-    Path::size_type pos = path.rfind('/', last);
+    auto pos = path.rfind('/', last);
     if (pos == string::npos)
         pos = 0;
     else
         pos += 1;
 
-    return string(path, pos, last - pos + 1);
+    return path.substr(pos, last - pos + 1);
 }
 
 
@@ -1302,9 +1302,10 @@ bool hasPrefix(const string & s, const string & prefix)
 }
 
 
-bool hasSuffix(const string & s, const string & suffix)
+bool hasSuffix(std::string_view s, std::string_view suffix)
 {
-    return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix;
+    return s.size() >= suffix.size()
+        && s.substr(s.size() - suffix.size()) == suffix;
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a1acb49b525..abd91b81c52 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -62,7 +62,7 @@ Path dirOf(const Path & path);
 
 /* Return the base name of the given canonical path, i.e., everything
    following the final `/'. */
-string baseNameOf(const Path & path);
+std::string_view baseNameOf(std::string_view path);
 
 /* Check whether 'path' is a descendant of 'dir'. */
 bool isInDir(const Path & path, const Path & dir);
@@ -407,7 +407,7 @@ bool hasPrefix(const string & s, const string & prefix);
 
 
 /* Return true iff `s' ends in `suffix'. */
-bool hasSuffix(const string & s, const string & suffix);
+bool hasSuffix(std::string_view s, std::string_view suffix);
 
 
 /* Convert a string to lower case. */
@@ -450,10 +450,10 @@ string base64Decode(const string & s);
 /* Get a value for the specified key from an associate container, or a
    default value if the key doesn't exist. */
 template <class T>
-string get(const T & map, const string & key, const string & def = "")
+std::optional<std::string> get(const T & map, const std::string & key)
 {
     auto i = map.find(key);
-    return i == map.end() ? def : i->second;
+    return i == map.end() ? std::optional<std::string>() : i->second;
 }
 
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index c506f9b0a59..73e3a347bf8 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -317,11 +317,11 @@ static void _main(int argc, char * * argv)
 
     state->printStats();
 
-    auto buildPaths = [&](const PathSet & paths) {
+    auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) {
         /* Note: we do this even when !printMissing to efficiently
            fetch binary cache data. */
         unsigned long long downloadSize, narSize;
-        PathSet willBuild, willSubstitute, unknown;
+        StorePathSet willBuild, willSubstitute, unknown;
         store->queryMissing(paths,
             willBuild, willSubstitute, unknown, downloadSize, narSize);
 
@@ -337,9 +337,9 @@ static void _main(int argc, char * * argv)
             throw UsageError("nix-shell requires a single derivation");
 
         auto & drvInfo = drvs.front();
-        auto drv = store->derivationFromPath(drvInfo.queryDrvPath());
+        auto drv = store->derivationFromPath(store->parseStorePath(drvInfo.queryDrvPath()));
 
-        PathSet pathsToBuild;
+        std::vector<StorePathWithOutputs> pathsToBuild;
 
         /* Figure out what bash shell to use. If $NIX_BUILD_SHELL
            is not set, then build bashInteractive from
@@ -358,7 +358,7 @@ static void _main(int argc, char * * argv)
                 if (!drv)
                     throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation");
 
-                pathsToBuild.insert(drv->queryDrvPath());
+                pathsToBuild.emplace_back(store->parseStorePath(drv->queryDrvPath()));
 
                 shell = drv->queryOutPath() + "/bin/bash";
 
@@ -370,10 +370,11 @@ static void _main(int argc, char * * argv)
 
         // Build or fetch all dependencies of the derivation.
         for (const auto & input : drv.inputDrvs)
-            if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); }))
-                pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second));
+            if (std::all_of(envExclude.cbegin(), envExclude.cend(),
+                    [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); }))
+                pathsToBuild.emplace_back(input.first, input.second);
         for (const auto & src : drv.inputSrcs)
-            pathsToBuild.insert(src);
+            pathsToBuild.emplace_back(src);
 
         buildPaths(pathsToBuild);
 
@@ -399,7 +400,7 @@ static void _main(int argc, char * * argv)
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 
-        auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", ""));
+        auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile").value_or(""));
 
         bool keepTmp = false;
         int fileNr = 0;
@@ -468,7 +469,7 @@ static void _main(int argc, char * * argv)
 
     else {
 
-        PathSet pathsToBuild;
+        std::vector<StorePathWithOutputs> pathsToBuild;
 
         std::map<Path, Path> drvPrefixes;
         std::map<Path, Path> resultSymlinks;
@@ -482,7 +483,7 @@ static void _main(int argc, char * * argv)
             if (outputName == "")
                 throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
 
-            pathsToBuild.insert(drvPath + "!" + outputName);
+            pathsToBuild.emplace_back(store->parseStorePath(drvPath), StringSet{outputName});
 
             std::string drvPrefix;
             auto i = drvPrefixes.find(drvPath);
@@ -508,7 +509,7 @@ static void _main(int argc, char * * argv)
 
         for (auto & symlink : resultSymlinks)
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                store2->addPermRoot(symlink.second, absPath(symlink.first), true);
+                store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
 
         for (auto & path : outPaths)
             std::cout << path << '\n';
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 70aa5c96669..1b337a7125d 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -27,7 +27,7 @@ static void readChannels()
             continue;
         auto split = tokenizeString<std::vector<string>>(line, " ");
         auto url = std::regex_replace(split[0], std::regex("/*$"), "");
-        auto name = split.size() > 1 ? split[1] : baseNameOf(url);
+        auto name = split.size() > 1 ? split[1] : std::string(baseNameOf(url));
         channels[name] = url;
     }
 }
@@ -98,10 +98,9 @@ static void update(const StringSet & channelNames)
         // shows something useful).
         auto cname = name;
         std::smatch match;
-        auto urlBase = baseNameOf(url);
-        if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) {
+        auto urlBase = std::string(baseNameOf(url));
+        if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$")))
             cname = cname + (string) match[1];
-        }
 
         std::string extraAttrs;
 
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index fdcde8b076b..f8703576042 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -52,11 +52,11 @@ static int _main(int argc, char ** argv)
         auto to = toMode ? openStore(remoteUri) : openStore();
         auto from = toMode ? openStore() : openStore(remoteUri);
 
-        PathSet storePaths2;
+        StorePathSet storePaths2;
         for (auto & path : storePaths)
             storePaths2.insert(from->followLinksToStorePath(path));
 
-        PathSet closure;
+        StorePathSet closure;
         from->computeFSClosure(storePaths2, closure, false, includeOutputs);
 
         copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 73d1d11278e..1348985619e 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -286,7 +286,7 @@ static int _main(int argc, char * * argv)
                 if (chdir(socketDir.c_str()) == -1)
                     throw SysError(format("changing to socket directory '%1%'") % socketDir);
 
-                auto socketName = baseNameOf(socketPath);
+                auto socketName = std::string(baseNameOf(socketPath));
                 auto addr = sockaddr_un{};
                 addr.sun_family = AF_UNIX;
                 if (socketName.size() + 1 >= sizeof(addr.sun_path))
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index a9c7439555d..a40d0c7e6e5 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -208,10 +208,11 @@ static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
 // at a time.
 static bool isPrebuilt(EvalState & state, DrvInfo & elem)
 {
-    Path path = elem.queryOutPath();
+    auto path = state.store->parseStorePath(elem.queryOutPath());
     if (state.store->isValidPath(path)) return true;
-    PathSet ps = state.store->querySubstitutablePaths({path});
-    return ps.find(path) != ps.end();
+    StorePathSet paths;
+    paths.insert(path.clone()); // FIXME: why doesn't StorePathSet{path.clone()} work?
+    return state.store->querySubstitutablePaths(paths).count(path);
 }
 
 
@@ -371,24 +372,21 @@ static void queryInstSources(EvalState & state,
         case srcStorePaths: {
 
             for (auto & i : args) {
-                Path path = state.store->followLinksToStorePath(i);
+                auto path = state.store->followLinksToStorePath(i);
 
-                string name = baseNameOf(path);
-                string::size_type dash = name.find('-');
-                if (dash != string::npos)
-                    name = string(name, dash + 1);
+                std::string name(path.name());
 
                 DrvInfo elem(state, "", nullptr);
                 elem.setName(name);
 
-                if (isDerivation(path)) {
-                    elem.setDrvPath(path);
-                    elem.setOutPath(state.store->derivationFromPath(path).findOutput("out"));
+                if (path.isDerivation()) {
+                    elem.setDrvPath(state.store->printStorePath(path));
+                    elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out")));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)
                         name = string(name, 0, name.size() - drvExtension.size());
                 }
-                else elem.setOutPath(path);
+                else elem.setOutPath(state.store->printStorePath(path));
 
                 elems.push_back(elem);
             }
@@ -421,13 +419,13 @@ static void queryInstSources(EvalState & state,
 
 static void printMissing(EvalState & state, DrvInfos & elems)
 {
-    PathSet targets;
+    std::vector<StorePathWithOutputs> targets;
     for (auto & i : elems) {
         Path drvPath = i.queryDrvPath();
         if (drvPath != "")
-            targets.insert(drvPath);
+            targets.emplace_back(state.store->parseStorePath(drvPath));
         else
-            targets.insert(i.queryOutPath());
+            targets.emplace_back(state.store->parseStorePath(i.queryOutPath()));
     }
 
     printMissing(state.store, targets);
@@ -697,15 +695,15 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
         drv.setName(globals.forceName);
 
     if (drv.queryDrvPath() != "") {
-        PathSet paths = {drv.queryDrvPath()};
+        std::vector<StorePathWithOutputs> paths{globals.state->store->parseStorePath(drv.queryDrvPath())};
         printMissing(globals.state->store, paths);
         if (globals.dryRun) return;
         globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
-    }
-    else {
-        printMissing(globals.state->store, {drv.queryOutPath()});
+    } else {
+        printMissing(globals.state->store,
+            {globals.state->store->parseStorePath(drv.queryOutPath())});
         if (globals.dryRun) return;
-        globals.state->store->ensurePath(drv.queryOutPath());
+        globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath()));
     }
 
     debug(format("switching to new user environment"));
@@ -729,7 +727,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
             for (auto & j : selectors)
                 /* !!! the repeated calls to followLinksToStorePath()
                    are expensive, should pre-compute them. */
-                if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j))
+                if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j))
                     || DrvName(j).matches(drvName))
                 {
                     printInfo("uninstalling '%s'", i.queryName());
@@ -953,12 +951,13 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 
 
     /* Query which paths have substitutes. */
-    PathSet validPaths, substitutablePaths;
+    StorePathSet validPaths;
+    StorePathSet substitutablePaths;
     if (printStatus || globals.prebuiltOnly) {
-        PathSet paths;
+        StorePathSet paths;
         for (auto & i : elems)
             try {
-                paths.insert(i.queryOutPath());
+                paths.insert(globals.state->store->parseStorePath(i.queryOutPath()));
             } catch (AssertionError & e) {
                 printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName());
                 i.setFailed();
@@ -989,8 +988,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath);
 
             if (globals.prebuiltOnly &&
-                validPaths.find(i.queryOutPath()) == validPaths.end() &&
-                substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end())
+                !validPaths.count(globals.state->store->parseStorePath(i.queryOutPath())) &&
+                !substitutablePaths.count(globals.state->store->parseStorePath(i.queryOutPath())))
                 continue;
 
             /* For table output. */
@@ -1001,9 +1000,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 
             if (printStatus) {
                 Path outPath = i.queryOutPath();
-                bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
+                bool hasSubs = substitutablePaths.count(globals.state->store->parseStorePath(outPath));
                 bool isInstalled = installed.find(outPath) != installed.end();
-                bool isValid = validPaths.find(outPath) != validPaths.end();
+                bool isValid = validPaths.count(globals.state->store->parseStorePath(outPath));
                 if (xmlOutput) {
                     attrs["installed"] = isInstalled ? "1" : "0";
                     attrs["valid"] = isValid ? "1" : "0";
@@ -1347,7 +1346,7 @@ static int _main(int argc, char * * argv)
             using LegacyArgs::LegacyArgs;
         };
 
-        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
+        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
             Operation oldOp = op;
 
             if (*arg == "--help")
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 7b9a8828117..8e4ecda4e8c 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -32,16 +32,16 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 {
     /* Build the components in the user environment, if they don't
        exist already. */
-    PathSet drvsToBuild;
+    std::vector<StorePathWithOutputs> drvsToBuild;
     for (auto & i : elems)
         if (i.queryDrvPath() != "")
-            drvsToBuild.insert(i.queryDrvPath());
+            drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())});
 
     debug(format("building user environment dependencies"));
     state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal);
 
     /* Construct the whole top level derivation. */
-    PathSet references;
+    StorePathSet references;
     Value manifest;
     state.mkList(manifest, elems.size());
     unsigned int n = 0;
@@ -77,10 +77,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
             /* This is only necessary when installing store paths, e.g.,
                `nix-env -i /nix/store/abcd...-foo'. */
-            state.store->addTempRoot(j.second);
-            state.store->ensurePath(j.second);
+            state.store->addTempRoot(state.store->parseStorePath(j.second));
+            state.store->ensurePath(state.store->parseStorePath(j.second));
 
-            references.insert(j.second);
+            references.insert(state.store->parseStorePath(j.second));
         }
 
         // Copy the meta attributes.
@@ -95,14 +95,14 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
         vMeta.attrs->sort();
         v.attrs->sort();
 
-        if (drvPath != "") references.insert(drvPath);
+        if (drvPath != "") references.insert(state.store->parseStorePath(drvPath));
     }
 
     /* Also write a copy of the list of user environment elements to
        the store; we need it for future modifications of the
        environment. */
-    Path manifestFile = state.store->addTextToStore("env-manifest.nix",
-        (format("%1%") % manifest).str(), references);
+    auto manifestFile = state.store->addTextToStore("env-manifest.nix",
+        fmt("%s", manifest), references);
 
     /* Get the environment builder expression. */
     Value envBuilder;
@@ -113,7 +113,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     Value args, topLevel;
     state.mkAttrs(args, 3);
     mkString(*state.allocAttr(args, state.symbols.create("manifest")),
-        manifestFile, {manifestFile});
+        state.store->printStorePath(manifestFile), {state.store->printStorePath(manifestFile)});
     args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest));
     args.attrs->sort();
     mkApp(topLevel, envBuilder, args);
@@ -123,13 +123,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     state.forceValue(topLevel);
     PathSet context;
     Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath));
-    Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context);
+    auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context));
     Attr & aOutPath(*topLevel.attrs->find(state.sOutPath));
     Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context);
 
     /* Realise the resulting store expression. */
     debug("building user environment");
-    state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal);
+    std::vector<StorePathWithOutputs> topLevelDrvs;
+    topLevelDrvs.push_back(StorePathWithOutputs{topLevelDrv.clone()});
+    state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal);
 
     /* Switch the current user environment to the output path. */
     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index a736caa8f05..5a886d69d4d 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -75,9 +75,9 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                     if (++rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
                     if (store2)
-                        drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot);
+                        drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot);
                 }
-                std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : "");
+                std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : ""));
             }
         }
     }
@@ -105,7 +105,7 @@ static int _main(int argc, char * * argv)
             using LegacyArgs::LegacyArgs;
         };
 
-        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
+        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
                 showManPage("nix-instantiate");
             else if (*arg == "--version")
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 78c88383377..5040b6fb810 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -64,7 +64,7 @@ static int _main(int argc, char * * argv)
             using LegacyArgs::LegacyArgs;
         };
 
-        MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) {
+        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--help")
                 showManPage("nix-prefetch-url");
             else if (*arg == "--version")
@@ -156,17 +156,17 @@ static int _main(int argc, char * * argv)
         /* If an expected hash is given, the file may already exist in
            the store. */
         Hash hash, expectedHash(ht);
-        Path storePath;
+        std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
             storePath = store->makeFixedOutputPath(unpack, expectedHash, name);
-            if (store->isValidPath(storePath))
+            if (store->isValidPath(*storePath))
                 hash = expectedHash;
             else
-                storePath.clear();
+                storePath.reset();
         }
 
-        if (storePath.empty()) {
+        if (!storePath) {
 
             auto actualUri = resolveMirrorUri(*state, uri);
 
@@ -193,7 +193,7 @@ static int _main(int argc, char * * argv)
                 if (hasSuffix(baseNameOf(uri), ".zip"))
                     runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
                 else
-                    unpackTarfile(tmpFile, unpacked, baseNameOf(uri));
+                    unpackTarfile(tmpFile, unpacked, std::string(baseNameOf(uri)));
 
                 /* If the archive unpacks to a single file/directory, then use
                    that as the top-level. */
@@ -217,17 +217,17 @@ static int _main(int argc, char * * argv)
                into the Nix store. */
             storePath = store->addToStore(name, tmpFile, unpack, ht);
 
-            assert(storePath == store->makeFixedOutputPath(unpack, hash, name));
+            assert(*storePath == store->makeFixedOutputPath(unpack, hash, name));
         }
 
         stopProgressBar();
 
         if (!printPath)
-            printInfo(format("path is '%1%'") % storePath);
+            printInfo("path is '%s'", store->printStorePath(*storePath));
 
         std::cout << printHash16or32(hash) << std::endl;
         if (printPath)
-            std::cout << storePath << std::endl;
+            std::cout << store->printStorePath(*storePath) << std::endl;
 
         return 0;
     }
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index d448654fe3a..a9d0dbbed63 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -10,9 +10,9 @@ using std::cout;
 namespace nix {
 
 
-static string dotQuote(const string & s)
+static string dotQuote(std::string_view s)
 {
-    return "\"" + s + "\"";
+    return "\"" + std::string(s) + "\"";
 }
 
 
@@ -34,7 +34,7 @@ static string makeEdge(const string & src, const string & dst)
 }
 
 
-static string makeNode(const string & id, const string & label,
+static string makeNode(const string & id, std::string_view label,
     const string & colour)
 {
     format f = format("%1% [label = %2%, shape = box, "
@@ -44,13 +44,6 @@ static string makeNode(const string & id, const string & label,
 }
 
 
-static string symbolicName(const string & path)
-{
-    string p = baseNameOf(path);
-    return string(p, p.find('-') + 1);
-}
-
-
 #if 0
 string pathLabel(const Path & nePath, const string & elemPath)
 {
@@ -91,25 +84,24 @@ void printClosure(const Path & nePath, const StoreExpr & fs)
 #endif
 
 
-void printDotGraph(ref<Store> store, const PathSet & roots)
+void printDotGraph(ref<Store> store, StorePathSet && roots)
 {
-    PathSet workList(roots);
-    PathSet doneSet;
+    StorePathSet workList(std::move(roots));
+    StorePathSet doneSet;
 
     cout << "digraph G {\n";
 
     while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
+        auto path = std::move(workList.extract(workList.begin()).value());
 
-        if (!doneSet.insert(path).second) continue;
+        if (!doneSet.insert(path.clone()).second) continue;
 
-        cout << makeNode(path, symbolicName(path), "#ff0000");
+        cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000");
 
         for (auto & p : store->queryPathInfo(path)->references) {
             if (p != path) {
-                workList.insert(p);
-                cout << makeEdge(p, path);
+                workList.insert(p.clone());
+                cout << makeEdge(std::string(p.to_string()), std::string(p.to_string()));
             }
         }
 
diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh
index e2b5fc72fbe..73b8d06b945 100644
--- a/src/nix-store/dotgraph.hh
+++ b/src/nix-store/dotgraph.hh
@@ -1,11 +1,9 @@
 #pragma once
 
-#include "types.hh"
+#include "store-api.hh"
 
 namespace nix {
 
-class Store;
-
-void printDotGraph(ref<Store> store, const PathSet & roots);
+void printDotGraph(ref<Store> store, StorePathSet && roots);
 
 }
diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc
index 670fbe227a4..34770885104 100644
--- a/src/nix-store/graphml.cc
+++ b/src/nix-store/graphml.cc
@@ -11,7 +11,7 @@ using std::cout;
 namespace nix {
 
 
-static inline const string & xmlQuote(const string & s)
+static inline std::string_view xmlQuote(std::string_view s)
 {
     // Luckily, store paths shouldn't contain any character that needs to be
     // quoted.
@@ -19,14 +19,13 @@ static inline const string & xmlQuote(const string & s)
 }
 
 
-static string symbolicName(const string & path)
+static string symbolicName(const std::string & p)
 {
-    string p = baseNameOf(path);
     return string(p, p.find('-') + 1);
 }
 
 
-static string makeEdge(const string & src, const string & dst)
+static string makeEdge(std::string_view src, std::string_view dst)
 {
     return fmt("  <edge source=\"%1%\" target=\"%2%\"/>\n",
         xmlQuote(src), xmlQuote(dst));
@@ -41,18 +40,18 @@ static string makeNode(const ValidPathInfo & info)
         "    <data key=\"name\">%3%</data>\n"
         "    <data key=\"type\">%4%</data>\n"
         "  </node>\n",
-        info.path,
+        info.path.to_string(),
         info.narSize,
-        symbolicName(info.path),
-        (isDerivation(info.path) ? "derivation" : "output-path"));
+        symbolicName(std::string(info.path.name())),
+        (info.path.isDerivation() ? "derivation" : "output-path"));
 }
 
 
-void printGraphML(ref<Store> store, const PathSet & roots)
+void printGraphML(ref<Store> store, StorePathSet && roots)
 {
-    PathSet workList(roots);
-    PathSet doneSet;
-    std::pair<PathSet::iterator,bool> ret;
+    StorePathSet workList(std::move(roots));
+    StorePathSet doneSet;
+    std::pair<StorePathSet::iterator, bool> ret;
 
     cout << "<?xml version='1.0' encoding='utf-8'?>\n"
          << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n"
@@ -64,19 +63,18 @@ void printGraphML(ref<Store> store, const PathSet & roots)
          << "<graph id='G' edgedefault='directed'>\n";
 
     while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
+        auto path = std::move(workList.extract(workList.begin()).value());
 
-        ret = doneSet.insert(path);
+        ret = doneSet.insert(path.clone());
         if (ret.second == false) continue;
 
-        ValidPathInfo info = *(store->queryPathInfo(path));
-        cout << makeNode(info);
+        auto info = store->queryPathInfo(path);
+        cout << makeNode(*info);
 
-        for (auto & p : store->queryPathInfo(path)->references) {
+        for (auto & p : info->references) {
             if (p != path) {
-                workList.insert(p);
-                cout << makeEdge(path, p);
+                workList.insert(p.clone());
+                cout << makeEdge(path.to_string(), p.to_string());
             }
         }
 
diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh
index b78df1e49a6..78be8a3676e 100644
--- a/src/nix-store/graphml.hh
+++ b/src/nix-store/graphml.hh
@@ -1,11 +1,9 @@
 #pragma once
 
-#include "types.hh"
+#include "store-api.hh"
 
 namespace nix {
 
-class Store;
-
-void printGraphML(ref<Store> store, const PathSet & roots);
+void printGraphML(ref<Store> store, StorePathSet && roots);
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index dfa473db5e3..e2e2939f4be 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -47,38 +47,37 @@ ref<LocalStore> ensureLocalStore()
 }
 
 
-static Path useDeriver(Path path)
+static StorePath useDeriver(const StorePath & path)
 {
-    if (isDerivation(path)) return path;
-    Path drvPath = store->queryPathInfo(path)->deriver;
-    if (drvPath == "")
-        throw Error(format("deriver of path '%1%' is not known") % path);
-    return drvPath;
+    if (path.isDerivation()) return path.clone();
+    auto info = store->queryPathInfo(path);
+    if (!info->deriver)
+        throw Error("deriver of path '%s' is not known", store->printStorePath(path));
+    return info->deriver->clone();
 }
 
 
 /* Realise the given path.  For a derivation that means build it; for
    other paths it means ensure their validity. */
-static PathSet realisePath(Path path, bool build = true)
+static PathSet realisePath(StorePathWithOutputs path, bool build = true)
 {
-    DrvPathWithOutputs p = parseDrvPathWithOutputs(path);
-
     auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
 
-    if (isDerivation(p.first)) {
+    if (path.path.isDerivation()) {
         if (build) store->buildPaths({path});
-        Derivation drv = store->derivationFromPath(p.first);
+        Derivation drv = store->derivationFromPath(path.path);
         rootNr++;
 
-        if (p.second.empty())
-            for (auto & i : drv.outputs) p.second.insert(i.first);
+        if (path.outputs.empty())
+            for (auto & i : drv.outputs) path.outputs.insert(i.first);
 
         PathSet outputs;
-        for (auto & j : p.second) {
+        for (auto & j : path.outputs) {
             DerivationOutputs::iterator i = drv.outputs.find(j);
             if (i == drv.outputs.end())
-                throw Error(format("derivation '%1%' does not have an output named '%2%'") % p.first % j);
-            Path outPath = i->second.path;
+                throw Error("derivation '%s' does not have an output named '%s'",
+                    store2->printStorePath(path.path), j);
+            auto outPath = store2->printStorePath(i->second.path);
             if (store2) {
                 if (gcRoot == "")
                     printGCWarning();
@@ -86,7 +85,7 @@ static PathSet realisePath(Path path, bool build = true)
                     Path rootName = gcRoot;
                     if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
                     if (i->first != "out") rootName += "-" + i->first;
-                    outPath = store2->addPermRoot(outPath, rootName, indirectRoot);
+                    outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot);
                 }
             }
             outputs.insert(outPath);
@@ -95,8 +94,9 @@ static PathSet realisePath(Path path, bool build = true)
     }
 
     else {
-        if (build) store->ensurePath(path);
-        else if (!store->isValidPath(path)) throw Error(format("path '%1%' does not exist and cannot be created") % path);
+        if (build) store->ensurePath(path.path);
+        else if (!store->isValidPath(path.path))
+            throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path));
         if (store2) {
             if (gcRoot == "")
                 printGCWarning();
@@ -104,10 +104,10 @@ static PathSet realisePath(Path path, bool build = true)
                 Path rootName = gcRoot;
                 rootNr++;
                 if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
-                path = store2->addPermRoot(path, rootName, indirectRoot);
+                return {store2->addPermRoot(path.path, rootName, indirectRoot)};
             }
         }
-        return {path};
+        return {store->printStorePath(path.path)};
     }
 }
 
@@ -126,23 +126,20 @@ static void opRealise(Strings opFlags, Strings opArgs)
         else if (i == "--ignore-unknown") ignoreUnknown = true;
         else throw UsageError(format("unknown flag '%1%'") % i);
 
-    Paths paths;
-    for (auto & i : opArgs) {
-        DrvPathWithOutputs p = parseDrvPathWithOutputs(i);
-        paths.push_back(makeDrvPathWithOutputs(store->followLinksToStorePath(p.first), p.second));
-    }
+    std::vector<StorePathWithOutputs> paths;
+    for (auto & i : opArgs)
+        paths.push_back(store->parseDrvPathWithOutputs(i));
 
     unsigned long long downloadSize, narSize;
-    PathSet willBuild, willSubstitute, unknown;
-    store->queryMissing(PathSet(paths.begin(), paths.end()),
-        willBuild, willSubstitute, unknown, downloadSize, narSize);
+    StorePathSet willBuild, willSubstitute, unknown;
+    store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (ignoreUnknown) {
-        Paths paths2;
+        std::vector<StorePathWithOutputs> paths2;
         for (auto & i : paths)
-            if (unknown.find(i) == unknown.end()) paths2.push_back(i);
-        paths = paths2;
-        unknown = PathSet();
+            if (!unknown.count(i.path)) paths2.push_back(i);
+        paths = std::move(paths2);
+        unknown = StorePathSet();
     }
 
     if (settings.printMissing)
@@ -151,14 +148,14 @@ static void opRealise(Strings opFlags, Strings opArgs)
     if (dryRun) return;
 
     /* Build all paths at the same time to exploit parallelism. */
-    store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode);
+    store->buildPaths(paths, buildMode);
 
     if (!ignoreUnknown)
         for (auto & i : paths) {
-            PathSet paths = realisePath(i, false);
+            auto paths2 = realisePath(i, false);
             if (!noOutput)
-                for (auto & j : paths)
-                    cout << format("%1%\n") % j;
+                for (auto & j : paths2)
+                    cout << fmt("%1%\n", j);
         }
 }
 
@@ -169,7 +166,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
     for (auto & i : opArgs)
-        cout << format("%1%\n") % store->addToStore(baseNameOf(i), i);
+        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i)));
 }
 
 
@@ -190,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs)
-        cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo);
+        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
 }
 
 
@@ -211,22 +208,21 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
     string hash = *i++;
     string name = *i++;
 
-    cout << format("%1%\n") %
-        store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name);
+    cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name)));
 }
 
 
-static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise)
+static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise)
 {
     if (forceRealise) realisePath(storePath);
-    if (useOutput && isDerivation(storePath)) {
-        Derivation drv = store->derivationFromPath(storePath);
-        PathSet outputs;
+    if (useOutput && storePath.isDerivation()) {
+        auto drv = store->derivationFromPath(storePath);
+        StorePathSet outputs;
         for (auto & i : drv.outputs)
-            outputs.insert(i.second.path);
+            outputs.insert(i.second.path.clone());
         return outputs;
     }
-    else return {storePath};
+    else return singleton(storePath.clone());
 }
 
 
@@ -239,23 +235,23 @@ const string treeLine = "|   ";
 const string treeNull = "    ";
 
 
-static void printTree(const Path & path,
-    const string & firstPad, const string & tailPad, PathSet & done)
+static void printTree(const StorePath & path,
+    const string & firstPad, const string & tailPad, StorePathSet & done)
 {
-    if (!done.insert(path).second) {
-        cout << format("%1%%2% [...]\n") % firstPad % path;
+    if (!done.insert(path.clone()).second) {
+        cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path));
         return;
     }
 
-    cout << format("%1%%2%\n") % firstPad % path;
+    cout << fmt("%s%s\n", firstPad, store->printStorePath(path));
 
-    auto references = store->queryPathInfo(path)->references;
+    auto info = store->queryPathInfo(path);
 
     /* Topologically sort under the relation A < B iff A \in
        closure(B).  That is, if derivation A is an (possibly indirect)
        input of B, then A is printed first.  This has the effect of
        flattening the tree, preventing deeply nested structures.  */
-    Paths sorted = store->topoSortPaths(references);
+    auto sorted = store->topoSortPaths(info->references);
     reverse(sorted.begin(), sorted.end());
 
     for (auto i = sorted.begin(); i != sorted.end(); ++i) {
@@ -318,11 +314,11 @@ static void opQuery(Strings opFlags, Strings opArgs)
 
         case qOutputs: {
             for (auto & i : opArgs) {
-                i = store->followLinksToStorePath(i);
-                if (forceRealise) realisePath(i);
-                Derivation drv = store->derivationFromPath(i);
+                auto i2 = store->followLinksToStorePath(i);
+                if (forceRealise) realisePath(i2);
+                Derivation drv = store->derivationFromPath(i2);
                 for (auto & j : drv.outputs)
-                    cout << format("%1%\n") % j.second.path;
+                    cout << fmt("%1%\n", store->printStorePath(j.second.path));
             }
             break;
         }
@@ -331,51 +327,54 @@ static void opQuery(Strings opFlags, Strings opArgs)
         case qReferences:
         case qReferrers:
         case qReferrersClosure: {
-            PathSet paths;
+            StorePathSet paths;
             for (auto & i : opArgs) {
-                PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
+                auto ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
                 for (auto & j : ps) {
                     if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs);
                     else if (query == qReferences) {
                         for (auto & p : store->queryPathInfo(j)->references)
-                            paths.insert(p);
+                            paths.insert(p.clone());
+                    }
+                    else if (query == qReferrers) {
+                        StorePathSet tmp;
+                        store->queryReferrers(j, tmp);
+                        for (auto & i : tmp)
+                            paths.insert(i.clone());
                     }
-                    else if (query == qReferrers) store->queryReferrers(j, paths);
                     else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true);
                 }
             }
-            Paths sorted = store->topoSortPaths(paths);
-            for (Paths::reverse_iterator i = sorted.rbegin();
+            auto sorted = store->topoSortPaths(paths);
+            for (StorePaths::reverse_iterator i = sorted.rbegin();
                  i != sorted.rend(); ++i)
-                cout << format("%s\n") % *i;
+                cout << fmt("%s\n", store->printStorePath(*i));
             break;
         }
 
         case qDeriver:
             for (auto & i : opArgs) {
-                Path deriver = store->queryPathInfo(store->followLinksToStorePath(i))->deriver;
-                cout << format("%1%\n") %
-                    (deriver == "" ? "unknown-deriver" : deriver);
+                auto info = store->queryPathInfo(store->followLinksToStorePath(i));
+                cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver");
             }
             break;
 
         case qBinding:
             for (auto & i : opArgs) {
-                Path path = useDeriver(store->followLinksToStorePath(i));
+                auto path = useDeriver(store->followLinksToStorePath(i));
                 Derivation drv = store->derivationFromPath(path);
                 StringPairs::iterator j = drv.env.find(bindingName);
                 if (j == drv.env.end())
-                    throw Error(format("derivation '%1%' has no environment binding named '%2%'")
-                        % path % bindingName);
-                cout << format("%1%\n") % j->second;
+                    throw Error("derivation '%s' has no environment binding named '%s'",
+                        store->printStorePath(path), bindingName);
+                cout << fmt("%s\n", j->second);
             }
             break;
 
         case qHash:
         case qSize:
             for (auto & i : opArgs) {
-                PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
-                for (auto & j : paths) {
+                for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
                         assert(info->narHash.type == htSHA256);
@@ -387,50 +386,51 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
 
         case qTree: {
-            PathSet done;
+            StorePathSet done;
             for (auto & i : opArgs)
                 printTree(store->followLinksToStorePath(i), "", "", done);
             break;
         }
 
         case qGraph: {
-            PathSet roots;
-            for (auto & i : opArgs) {
-                PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
-                roots.insert(paths.begin(), paths.end());
-            }
-            printDotGraph(ref<Store>(store), roots);
+            StorePathSet roots;
+            for (auto & i : opArgs)
+                for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))
+                    roots.insert(j.clone());
+            printDotGraph(ref<Store>(store), std::move(roots));
             break;
         }
 
         case qGraphML: {
-            PathSet roots;
-            for (auto & i : opArgs) {
-                PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise);
-                roots.insert(paths.begin(), paths.end());
-            }
-            printGraphML(ref<Store>(store), roots);
+            StorePathSet roots;
+            for (auto & i : opArgs)
+                for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))
+                    roots.insert(j.clone());
+            printGraphML(ref<Store>(store), std::move(roots));
             break;
         }
 
         case qResolve: {
             for (auto & i : opArgs)
-                cout << format("%1%\n") % store->followLinksToStorePath(i);
+                cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i)));
             break;
         }
 
         case qRoots: {
-            PathSet referrers;
-            for (auto & i : opArgs) {
-                store->computeFSClosure(
-                    maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise),
-                    referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
-            }
+            StorePathSet args;
+            for (auto & i : opArgs)
+                for (auto & p : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise))
+                    args.insert(p.clone());
+
+            StorePathSet referrers;
+            store->computeFSClosure(
+                args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
+
             Roots roots = store->findRoots(false);
             for (auto & [target, links] : roots)
                 if (referrers.find(target) != referrers.end())
                     for (auto & link : links)
-                        cout << format("%1% -> %2%\n") % link % target;
+                        cout << fmt("%1% -> %2%\n", link, store->printStorePath(target));
             break;
         }
 
@@ -446,7 +446,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
     if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path");
 
     Path drvPath = opArgs.front();
-    Derivation drv = store->derivationFromPath(drvPath);
+    Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath));
 
     /* Print each environment variable in the derivation in a format
        that can be sourced by the shell. */
@@ -476,7 +476,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         auto log = store->getBuildLog(path);
         if (!log)
-            throw Error("build log of derivation '%s' is not available", path);
+            throw Error("build log of derivation '%s' is not available", store->printStorePath(path));
         std::cout << *log;
     }
 }
@@ -487,13 +487,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
     if (!opFlags.empty()) throw UsageError("unknown flag");
     if (!opArgs.empty()) {
         for (auto & i : opArgs)
-            i = store->followLinksToStorePath(i);
-        for (auto & i : opArgs)
-            cout << store->makeValidityRegistration({i}, true, true);
+            cout << store->makeValidityRegistration(singleton(store->followLinksToStorePath(i)), true, true);
     } else {
-        PathSet validPaths = store->queryAllValidPaths();
-        for (auto & i : validPaths)
-            cout << store->makeValidityRegistration({i}, true, true);
+        for (auto & i : store->queryAllValidPaths())
+            cout << store->makeValidityRegistration(singleton(i), true, true);
     }
 }
 
@@ -503,18 +500,18 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
     ValidPathInfos infos;
 
     while (1) {
-        ValidPathInfo info = decodeValidPathInfo(cin, hashGiven);
-        if (info.path == "") break;
-        if (!store->isValidPath(info.path) || reregister) {
+        auto info = decodeValidPathInfo(*store, cin, hashGiven);
+        if (!info) break;
+        if (!store->isValidPath(info->path) || reregister) {
             /* !!! races */
             if (canonicalise)
-                canonicalisePathMetaData(info.path, -1);
+                canonicalisePathMetaData(store->printStorePath(info->path), -1);
             if (!hashGiven) {
-                HashResult hash = hashPath(htSHA256, info.path);
-                info.narHash = hash.first;
-                info.narSize = hash.second;
+                HashResult hash = hashPath(htSHA256, store->printStorePath(info->path));
+                info->narHash = hash.first;
+                info->narSize = hash.second;
             }
-            infos.push_back(info);
+            infos.push_back(std::move(*info));
         }
     }
 
@@ -556,12 +553,12 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
         else throw UsageError(format("unknown flag '%1%'") % i);
 
     for (auto & i : opArgs) {
-        Path path = store->followLinksToStorePath(i);
+        auto path = store->followLinksToStorePath(i);
         if (!store->isValidPath(path)) {
             if (printInvalid)
-                cout << format("%1%\n") % path;
+                cout << fmt("%s\n", store->printStorePath(path));
             else
-                throw Error(format("path '%1%' is not valid") % path);
+                throw Error("path '%s' is not valid", store->printStorePath(path));
         }
     }
 }
@@ -591,13 +588,13 @@ static void opGC(Strings opFlags, Strings opArgs)
 
     if (printRoots) {
         Roots roots = store->findRoots(false);
-        std::set<std::pair<Path, Path>> roots2;
+        std::set<std::pair<Path, StorePath>> roots2;
         // Transpose and sort the roots.
         for (auto & [target, links] : roots)
             for (auto & link : links)
-                roots2.emplace(link, target);
+                roots2.emplace(link, target.clone());
         for (auto & [link, target] : roots2)
-            std::cout << link << " -> " << target << "\n";
+            std::cout << link << " -> " << store->printStorePath(target) << "\n";
     }
 
     else {
@@ -661,11 +658,13 @@ static void opExport(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         throw UsageError(format("unknown flag '%1%'") % i);
 
+    StorePathSet paths;
+
     for (auto & i : opArgs)
-        i = store->followLinksToStorePath(i);
+        paths.insert(store->followLinksToStorePath(i));
 
     FdSink sink(STDOUT_FILENO);
-    store->exportPaths(opArgs, sink);
+    store->exportPaths(paths, sink);
     sink.flush();
 }
 
@@ -678,10 +677,10 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    Paths paths = store->importPaths(source, nullptr, NoCheckSigs);
+    auto paths = store->importPaths(source, nullptr, NoCheckSigs);
 
     for (auto & i : paths)
-        cout << format("%1%\n") % i << std::flush;
+        cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
 }
 
 
@@ -726,16 +725,16 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
     int status = 0;
 
     for (auto & i : opArgs) {
-        Path path = store->followLinksToStorePath(i);
-        printMsg(lvlTalkative, format("checking path '%1%'...") % path);
+        auto path = store->followLinksToStorePath(i);
+        printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
         HashSink sink(info->narHash.type);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
             printError(
-                format("path '%1%' was modified! expected hash '%2%', got '%3%'")
-                % path % info->narHash.to_string() % current.first.to_string());
+                "path '%s' was modified! expected hash '%s', got '%s'",
+                store->printStorePath(path), info->narHash.to_string(), current.first.to_string());
             status = 1;
         }
     }
@@ -751,10 +750,8 @@ static void opRepairPath(Strings opFlags, Strings opArgs)
     if (!opFlags.empty())
         throw UsageError("no flags expected");
 
-    for (auto & i : opArgs) {
-        Path path = store->followLinksToStorePath(i);
-        ensureLocalStore()->repairPath(path);
-    }
+    for (auto & i : opArgs)
+        ensureLocalStore()->repairPath(store->followLinksToStorePath(i));
 }
 
 /* Optimise the disk space usage of the Nix store by hard-linking
@@ -818,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                PathSet paths = readStorePaths<PathSet>(*store, in);
+                auto paths = readStorePaths<StorePathSet>(*store, in);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -828,34 +825,39 @@ static void opServe(Strings opFlags, Strings opArgs)
                    flag. */
                 if (substitute && writeAllowed) {
                     /* Filter out .drv files (we don't want to build anything). */
-                    PathSet paths2;
+                    std::vector<StorePathWithOutputs> paths2;
                     for (auto & path : paths)
-                        if (!isDerivation(path)) paths2.insert(path);
+                        if (!path.isDerivation())
+                            paths2.emplace_back(path.clone());
                     unsigned long long downloadSize, narSize;
-                    PathSet willBuild, willSubstitute, unknown;
-                    store->queryMissing(PathSet(paths2.begin(), paths2.end()),
+                    StorePathSet willBuild, willSubstitute, unknown;
+                    store->queryMissing(paths2,
                         willBuild, willSubstitute, unknown, downloadSize, narSize);
                     /* FIXME: should use ensurePath(), but it only
                        does one path at a time. */
                     if (!willSubstitute.empty())
                         try {
-                            store->buildPaths(willSubstitute);
+                            std::vector<StorePathWithOutputs> subs;
+                            for (auto & p : willSubstitute) subs.emplace_back(p.clone());
+                            store->buildPaths(subs);
                         } catch (Error & e) {
                             printError(format("warning: %1%") % e.msg());
                         }
                 }
 
-                out << store->queryValidPaths(paths);
+                writeStorePaths(*store, out, store->queryValidPaths(paths));
                 break;
             }
 
             case cmdQueryPathInfos: {
-                PathSet paths = readStorePaths<PathSet>(*store, in);
+                auto paths = readStorePaths<StorePathSet>(*store, in);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
-                        out << info->path << info->deriver << info->references;
+                        out << store->printStorePath(info->path)
+                            << (info->deriver ? store->printStorePath(*info->deriver) : "");
+                        writeStorePaths(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -869,7 +871,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             }
 
             case cmdDumpStorePath:
-                store->narFromPath(readStorePath(*store, in), out);
+                store->narFromPath(store->parseStorePath(readString(in)), out);
                 break;
 
             case cmdImportPaths: {
@@ -881,14 +883,17 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(readStorePaths<Paths>(*store, in), out);
+                store->exportPaths(readStorePaths<StorePathSet>(*store, in), out);
                 break;
             }
 
             case cmdBuildPaths: {
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
-                PathSet paths = readStorePaths<PathSet>(*store, in);
+
+                std::vector<StorePathWithOutputs> paths;
+                for (auto & s : readStrings<Strings>(in))
+                    paths.emplace_back(store->parseDrvPathWithOutputs(s));
 
                 getBuildSettings();
 
@@ -907,7 +912,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
 
-                Path drvPath = readStorePath(*store, in); // informational only
+                auto drvPath = store->parseStorePath(readString(in)); // informational only
                 BasicDerivation drv;
                 readDerivation(in, *store, drv);
 
@@ -926,30 +931,29 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdQueryClosure: {
                 bool includeOutputs = readInt(in);
-                PathSet closure;
-                store->computeFSClosure(readStorePaths<PathSet>(*store, in),
+                StorePathSet closure;
+                store->computeFSClosure(readStorePaths<StorePathSet>(*store, in),
                     closure, false, includeOutputs);
-                out << closure;
+                writeStorePaths(*store, out, closure);
                 break;
             }
 
             case cmdAddToStoreNar: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
 
-                ValidPathInfo info;
-                info.path = readStorePath(*store, in);
-                in >> info.deriver;
-                if (!info.deriver.empty())
-                    store->assertStorePath(info.deriver);
+                auto path = readString(in);
+                ValidPathInfo info(store->parseStorePath(path));
+                auto deriver = readString(in);
+                if (deriver != "")
+                    info.deriver = store->parseStorePath(deriver);
                 info.narHash = Hash(readString(in), htSHA256);
-                info.references = readStorePaths<PathSet>(*store, in);
+                info.references = readStorePaths<StorePathSet>(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 in >> info.ca;
 
-                if (info.narSize == 0) {
+                if (info.narSize == 0)
                     throw Error("narInfo is too old and missing the narSize field");
-                }
 
                 SizedSource sizedSource(in, info.narSize);
 
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 296b2c7e4cd..139db3657b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -40,16 +40,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         StringSink sink;
         dumpPath(path, sink);
 
-        ValidPathInfo info;
-        info.narHash = hashString(htSHA256, *sink.s);
+        auto narHash = hashString(htSHA256, *sink.s);
+
+        ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart));
+        info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.path = store->makeFixedOutputPath(true, info.narHash, *namePart);
         info.ca = makeFixedOutputCA(true, info.narHash);
 
         if (!dryRun)
             store->addToStore(info, sink.s);
 
-        std::cout << fmt("%s\n", info.path);
+        std::cout << fmt("%s\n", store->printStorePath(info.path));
     }
 };
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 9dbc3685afc..442bc6c53ec 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -48,28 +48,29 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 
 void StorePathsCommand::run(ref<Store> store)
 {
-    Paths storePaths;
+    StorePaths storePaths;
 
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
         for (auto & p : store->queryAllValidPaths())
-            storePaths.push_back(p);
+            storePaths.push_back(p.clone());
     }
 
     else {
         for (auto & p : toStorePaths(store, realiseMode, installables))
-            storePaths.push_back(p);
+            storePaths.push_back(p.clone());
 
         if (recursive) {
-            PathSet closure;
-            store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()),
-                closure, false, false);
-            storePaths = Paths(closure.begin(), closure.end());
+            StorePathSet closure;
+            store->computeFSClosure(storePathsToSet(storePaths), closure, false, false);
+            storePaths.clear();
+            for (auto & p : closure)
+                storePaths.push_back(p.clone());
         }
     }
 
-    run(store, storePaths);
+    run(store, std::move(storePaths));
 }
 
 void StorePathCommand::run(ref<Store> store)
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 7474c5eae26..00eb46903df 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -2,6 +2,7 @@
 
 #include "args.hh"
 #include "common-eval-args.hh"
+#include "path.hh"
 
 namespace nix {
 
@@ -28,8 +29,8 @@ private:
 
 struct Buildable
 {
-    Path drvPath; // may be empty
-    std::map<std::string, Path> outputs;
+    std::optional<StorePath> drvPath;
+    std::map<std::string, StorePath> outputs;
 };
 
 typedef std::vector<Buildable> Buildables;
@@ -131,7 +132,7 @@ public:
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, Paths storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
 
     void run(ref<Store> store) override;
 
@@ -143,7 +144,7 @@ struct StorePathCommand : public InstallablesCommand
 {
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, const Path & storePath) = 0;
+    virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
     void run(ref<Store> store) override;
 };
@@ -174,13 +175,13 @@ std::shared_ptr<Installable> parseInstallable(
 Buildables build(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables);
 
-PathSet toStorePaths(ref<Store> store, RealiseMode mode,
+std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables);
 
-Path toStorePath(ref<Store> store, RealiseMode mode,
+StorePath toStorePath(ref<Store> store, RealiseMode mode,
     std::shared_ptr<Installable> installable);
 
-PathSet toDerivations(ref<Store> store,
+std::set<StorePath> toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index b1aceb15c4e..85c777d38ab 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -80,14 +80,14 @@ struct CmdCopy : StorePathsCommand
         return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
     }
 
-    void run(ref<Store> srcStore, Paths storePaths) override
+    void run(ref<Store> srcStore, StorePaths storePaths) override
     {
         if (srcUri.empty() && dstUri.empty())
             throw UsageError("you must pass '--from' and/or '--to'");
 
         ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
 
-        copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()),
+        copyPaths(srcStore, dstStore, storePathsToSet(storePaths),
             NoRepair, checkSigs, substitute);
     }
 };
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 90f1552d92d..bb741b57202 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -20,7 +20,7 @@ struct CmdDumpPath : StorePathCommand
         };
     }
 
-    void run(ref<Store> store, const Path & storePath) override
+    void run(ref<Store> store, const StorePath & storePath) override
     {
         FdSink sink(STDOUT_FILENO);
         store->narFromPath(storePath, sink);
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 0e8bba39de2..8ce6bd06ebd 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -89,15 +89,25 @@ Buildable Installable::toBuildable()
 
 struct InstallableStorePath : Installable
 {
-    Path storePath;
+    ref<Store> store;
+    StorePath storePath;
 
-    InstallableStorePath(const Path & storePath) : storePath(storePath) { }
+    InstallableStorePath(ref<Store> store, const Path & storePath)
+        : store(store), storePath(store->parseStorePath(storePath)) { }
 
-    std::string what() override { return storePath; }
+    std::string what() override { return store->printStorePath(storePath); }
 
     Buildables toBuildables() override
     {
-        return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}};
+        std::map<std::string, StorePath> outputs;
+        outputs.insert_or_assign("out", storePath.clone());
+        Buildable b{
+            .drvPath = storePath.isDerivation() ? storePath.clone() : std::optional<StorePath>(),
+            .outputs = std::move(outputs)
+        };
+        Buildables bs;
+        bs.push_back(std::move(b));
+        return bs;
     }
 };
 
@@ -120,17 +130,17 @@ struct InstallableValue : Installable
 
         Buildables res;
 
-        PathSet drvPaths;
+        StorePathSet drvPaths;
 
         for (auto & drv : drvs) {
-            Buildable b{drv.queryDrvPath()};
-            drvPaths.insert(b.drvPath);
+            Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())};
+            drvPaths.insert(b.drvPath->clone());
 
             auto outputName = drv.queryOutputName();
             if (outputName == "")
-                throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath);
+                throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
 
-            b.outputs.emplace(outputName, drv.queryOutPath());
+            b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath()));
 
             res.push_back(std::move(b));
         }
@@ -138,10 +148,13 @@ struct InstallableValue : Installable
         // Hack to recognize .all: if all drvs have the same drvPath,
         // merge the buildables.
         if (drvPaths.size() == 1) {
-            Buildable b{*drvPaths.begin()};
+            Buildable b{.drvPath = drvPaths.begin()->clone()};
             for (auto & b2 : res)
-                b.outputs.insert(b2.outputs.begin(), b2.outputs.end());
-            return {b};
+                for (auto & output : b2.outputs)
+                    b.outputs.insert_or_assign(output.first, output.second.clone());
+            Buildables bs;
+            bs.push_back(std::move(b));
+            return bs;
         } else
             return res;
     }
@@ -212,7 +225,7 @@ static std::vector<std::shared_ptr<Installable>> parseInstallables(
             auto path = store->toStorePath(store->followLinksToStore(s));
 
             if (store->isStorePath(path))
-                result.push_back(std::make_shared<InstallableStorePath>(path));
+                result.push_back(std::make_shared<InstallableStorePath>(store, path));
         }
 
         else if (s == "" || std::regex_match(s, attrPathRegex))
@@ -242,19 +255,18 @@ Buildables build(ref<Store> store, RealiseMode mode,
 
     Buildables buildables;
 
-    PathSet pathsToBuild;
+    std::vector<StorePathWithOutputs> pathsToBuild;
 
     for (auto & i : installables) {
         for (auto & b : i->toBuildables()) {
-            if (b.drvPath != "") {
+            if (b.drvPath) {
                 StringSet outputNames;
                 for (auto & output : b.outputs)
                     outputNames.insert(output.first);
-                pathsToBuild.insert(
-                    b.drvPath + "!" + concatStringsSep(",", outputNames));
+                pathsToBuild.push_back({*b.drvPath, outputNames});
             } else
                 for (auto & output : b.outputs)
-                    pathsToBuild.insert(output.second);
+                    pathsToBuild.push_back({output.second.clone()});
             buildables.push_back(std::move(b));
         }
     }
@@ -267,19 +279,19 @@ Buildables build(ref<Store> store, RealiseMode mode,
     return buildables;
 }
 
-PathSet toStorePaths(ref<Store> store, RealiseMode mode,
+StorePathSet toStorePaths(ref<Store> store, RealiseMode mode,
     std::vector<std::shared_ptr<Installable>> installables)
 {
-    PathSet outPaths;
+    StorePathSet outPaths;
 
     for (auto & b : build(store, mode, installables))
         for (auto & output : b.outputs)
-            outPaths.insert(output.second);
+            outPaths.insert(output.second.clone());
 
     return outPaths;
 }
 
-Path toStorePath(ref<Store> store, RealiseMode mode,
+StorePath toStorePath(ref<Store> store, RealiseMode mode,
     std::shared_ptr<Installable> installable)
 {
     auto paths = toStorePaths(store, mode, {installable});
@@ -287,17 +299,17 @@ Path toStorePath(ref<Store> store, RealiseMode mode,
     if (paths.size() != 1)
             throw Error("argument '%s' should evaluate to one store path", installable->what());
 
-    return *paths.begin();
+    return paths.begin()->clone();
 }
 
-PathSet toDerivations(ref<Store> store,
+StorePathSet toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
 {
-    PathSet drvPaths;
+    StorePathSet drvPaths;
 
     for (auto & i : installables)
         for (auto & b : i->toBuildables()) {
-            if (b.drvPath.empty()) {
+            if (!b.drvPath) {
                 if (!useDeriver)
                     throw Error("argument '%s' did not evaluate to a derivation", i->what());
                 for (auto & output : b.outputs) {
@@ -305,10 +317,10 @@ PathSet toDerivations(ref<Store> store,
                     if (derivers.empty())
                         throw Error("'%s' does not have a known deriver", i->what());
                     // FIXME: use all derivers?
-                    drvPaths.insert(*derivers.begin());
+                    drvPaths.insert(derivers.begin()->clone());
                 }
             } else
-                drvPaths.insert(b.drvPath);
+                drvPaths.insert(b.drvPath->clone());
         }
 
     return drvPaths;
diff --git a/src/nix/local.mk b/src/nix/local.mk
index c09efd1fc89..a34cca9fd8d 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,7 +15,7 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil
+nix_LIBS = libexpr libmain libstore libutil libnixrust
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 122a3d6905f..795991cb783 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -43,7 +43,7 @@ struct CmdLog : InstallableCommand
 
         RunPager pager;
         for (auto & sub : subs) {
-            auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr;
+            auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr;
             for (auto & output : b.outputs) {
                 if (log) break;
                 log = sub->getBuildLog(output.second);
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 9408cc9da9d..3ef1f2750b6 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -67,7 +67,7 @@ struct MixLs : virtual Args, MixJSON
         if (st.type == FSAccessor::Type::tMissing)
             throw Error(format("path '%1%' does not exist") % path);
         doPath(st, path,
-            st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path),
+            st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
             showDirectory);
     }
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5454f4b342b..41d44f43191 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -135,7 +135,7 @@ void mainWrapped(int argc, char * * argv)
     initGC();
 
     programPath = argv[0];
-    string programName = baseNameOf(programPath);
+    auto programName = std::string(baseNameOf(programPath));
 
     {
         auto legacy = (*RegisterLegacyCommand::commands)[programName];
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 5b99b5084cb..524d467c2ae 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -29,34 +29,36 @@ struct CmdMakeContentAddressable : StorePathsCommand
             },
         };
     }
-    void run(ref<Store> store, Paths storePaths) override
+    void run(ref<Store> store, StorePaths storePaths) override
     {
-        auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));
+        auto paths = store->topoSortPaths(storePathsToSet(storePaths));
 
-        paths.reverse();
+        std::reverse(paths.begin(), paths.end());
 
-        std::map<Path, Path> remappings;
+        std::map<StorePath, StorePath> remappings;
 
         for (auto & path : paths) {
+            auto pathS = store->printStorePath(path);
             auto oldInfo = store->queryPathInfo(path);
-            auto oldHashPart = storePathToHash(path);
-            auto name = storePathToName(path);
+            auto oldHashPart = storePathToHash(pathS);
 
             StringSink sink;
             store->narFromPath(path, sink);
 
             StringMap rewrites;
 
-            ValidPathInfo info;
+            StorePathSet references;
+            bool hasSelfReference = false;
             for (auto & ref : oldInfo->references) {
                 if (ref == path)
-                    info.references.insert("self");
+                    hasSelfReference = true;
                 else {
-                    auto replacement = get(remappings, ref, ref);
+                    auto i = remappings.find(ref);
+                    auto replacement = i != remappings.end() ? i->second.clone() : ref.clone();
                     // FIXME: warn about unremapped paths?
-                    info.references.insert(replacement);
                     if (replacement != ref)
-                        rewrites[storePathToHash(ref)] = storePathToHash(replacement);
+                        rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
+                    references.insert(std::move(replacement));
                 }
             }
 
@@ -65,24 +67,26 @@ struct CmdMakeContentAddressable : StorePathsCommand
             HashModuloSink hashModuloSink(htSHA256, oldHashPart);
             hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
 
-            info.narHash = hashModuloSink.finish().first;
+            auto narHash = hashModuloSink.finish().first;
+
+            ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference));
+            info.references = std::move(references);
+            if (hasSelfReference) info.references.insert(info.path.clone());
+            info.narHash = narHash;
             info.narSize = sink.s->size();
-            replaceInSet(info.references, path, std::string("self"));
-            info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references);
-            replaceInSet(info.references, std::string("self"), info.path);
             info.ca = makeFixedOutputCA(true, info.narHash);
 
-            printError("rewrote '%s' to '%s'", path, info.path);
+            printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
 
             auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink);
+                RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink);
                 rsink2((unsigned char *) sink.s->data(), sink.s->size());
                 rsink2.flush();
             });
 
             store->addToStore(info, *source);
 
-            remappings[path] = info.path;
+            remappings.insert_or_assign(std::move(path), std::move(info.path));
         }
     }
 };
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 2cb718f123a..bffa7b3564e 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -78,36 +78,36 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         std::cout << fmt("\t%6.1f%c", res, idents.at(power));
     }
 
-    void run(ref<Store> store, Paths storePaths) override
+    void run(ref<Store> store, StorePaths storePaths) override
     {
         size_t pathLen = 0;
         for (auto & storePath : storePaths)
-            pathLen = std::max(pathLen, storePath.size());
+            pathLen = std::max(pathLen, store->printStorePath(storePath).size());
 
         if (json) {
             JSONPlaceholder jsonRoot(std::cout);
             store->pathInfoToJSON(jsonRoot,
                 // FIXME: preserve order?
-                PathSet(storePaths.begin(), storePaths.end()),
+                storePathsToSet(storePaths),
                 true, showClosureSize, AllowInvalid);
         }
 
         else {
 
-            for (auto storePath : storePaths) {
+            for (auto & storePath : storePaths) {
                 auto info = store->queryPathInfo(storePath);
-                storePath = info->path; // FIXME: screws up padding
+                auto storePathS = store->printStorePath(storePath);
 
-                std::cout << storePath;
+                std::cout << storePathS;
 
                 if (showSize || showClosureSize || showSigs)
-                    std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
+                    std::cout << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' ');
 
                 if (showSize)
                     printSize(info->narSize);
 
                 if (showClosureSize)
-                    printSize(store->getClosureSize(storePath).first);
+                    printSize(store->getClosureSize(info->path).first);
 
                 if (showSigs) {
                     std::cout << '\t';
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 661966733d3..d20c09f2683 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -24,6 +24,13 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
     return fields[n].i;
 }
 
+static std::string_view storePathToName(std::string_view path)
+{
+    auto base = baseNameOf(path);
+    auto i = base.find('-');
+    return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
+}
+
 class ProgressBar : public Logger
 {
 private:
@@ -148,7 +155,7 @@ class ProgressBar : public Logger
         if (type == actBuild) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name.resize(name.size() - 4);
+                name = name.substr(name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
             auto machineName = getS(fields, 1);
             if (machineName != "")
@@ -173,7 +180,7 @@ class ProgressBar : public Logger
         if (type == actPostBuildHook) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name.resize(name.size() - 4);
+                name = name.substr(name.size() - 4);
             i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
             i->name = DrvName(name).name;
         }
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index c2b65b6eae2..e6c811a7bef 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -413,7 +413,7 @@ Path NixRepl::getDerivationPath(Value & v) {
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
     Path drvPath = drvInfo->queryDrvPath();
-    if (drvPath == "" || !state.store->isValidPath(drvPath))
+    if (drvPath == "" || !state.store->isValidPath(state.store->parseStorePath(drvPath)))
         throw Error("expression did not evaluate to a valid derivation");
     return drvPath;
 }
@@ -521,10 +521,10 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                Derivation drv = readDerivation(drvPath);
+                auto drv = readDerivation(*state.store, drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)
-                    std::cout << format("  %1% -> %2%") % i.first % i.second.path << std::endl;
+                    std::cout << fmt("  %s -> %s\n", i.first, state.store->printStorePath(i.second.path));
             }
         } else if (command == ":i") {
             runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ece4d38541c..f885c5e496b 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -119,24 +119,24 @@ struct CmdRun : InstallablesCommand
                 unsetenv(var.c_str());
         }
 
-        std::unordered_set<Path> done;
-        std::queue<Path> todo;
-        for (auto & path : outPaths) todo.push(path);
+        std::unordered_set<StorePath> done;
+        std::queue<StorePath> todo;
+        for (auto & path : outPaths) todo.push(path.clone());
 
         auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
 
         while (!todo.empty()) {
-            Path path = todo.front();
+            auto path = todo.front().clone();
             todo.pop();
-            if (!done.insert(path).second) continue;
+            if (!done.insert(path.clone()).second) continue;
 
             if (true)
-                unixPath.push_front(path + "/bin");
+                unixPath.push_front(store->printStorePath(path) + "/bin");
 
-            auto propPath = path + "/nix-support/propagated-user-env-packages";
+            auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages";
             if (accessor->stat(propPath).type == FSAccessor::tRegular) {
                 for (auto & p : tokenizeString<Paths>(readFile(propPath)))
-                    todo.push(p);
+                    todo.push(store->parseStorePath(p));
             }
         }
 
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 6065adc4de3..0ede7b46839 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -46,9 +46,9 @@ struct CmdShowDerivation : InstallablesCommand
         auto drvPaths = toDerivations(store, installables, true);
 
         if (recursive) {
-            PathSet closure;
+            StorePathSet closure;
             store->computeFSClosure(drvPaths, closure);
-            drvPaths = closure;
+            drvPaths = std::move(closure);
         }
 
         {
@@ -56,17 +56,19 @@ struct CmdShowDerivation : InstallablesCommand
         JSONObject jsonRoot(std::cout, true);
 
         for (auto & drvPath : drvPaths) {
-            if (!isDerivation(drvPath)) continue;
+            if (!drvPath.isDerivation()) continue;
 
-            auto drvObj(jsonRoot.object(drvPath));
+            auto drvPathS = store->printStorePath(drvPath);
 
-            auto drv = readDerivation(drvPath);
+            auto drvObj(jsonRoot.object(drvPathS));
+
+            auto drv = readDerivation(*store, drvPathS);
 
             {
                 auto outputsObj(drvObj.object("outputs"));
                 for (auto & output : drv.outputs) {
                     auto outputObj(outputsObj.object(output.first));
-                    outputObj.attr("path", output.second.path);
+                    outputObj.attr("path", store->printStorePath(output.second.path));
                     if (output.second.hash != "") {
                         outputObj.attr("hashAlgo", output.second.hashAlgo);
                         outputObj.attr("hash", output.second.hash);
@@ -77,13 +79,13 @@ struct CmdShowDerivation : InstallablesCommand
             {
                 auto inputsList(drvObj.list("inputSrcs"));
                 for (auto & input : drv.inputSrcs)
-                    inputsList.elem(input);
+                    inputsList.elem(store->printStorePath(input));
             }
 
             {
                 auto inputDrvsObj(drvObj.object("inputDrvs"));
                 for (auto & input : drv.inputDrvs) {
-                    auto inputList(inputDrvsObj.list(input.first));
+                    auto inputList(inputDrvsObj.list(store->printStorePath(input.first)));
                     for (auto & outputId : input.second)
                         inputList.elem(outputId);
                 }
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 23bc83ad02d..5f07448e09a 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -27,7 +27,7 @@ struct CmdCopySigs : StorePathsCommand
         return "copy path signatures from substituters (like binary caches)";
     }
 
-    void run(ref<Store> store, Paths storePaths) override
+    void run(ref<Store> store, StorePaths storePaths) override
     {
         if (substituterUris.empty())
             throw UsageError("you must specify at least one substituter using '-s'");
@@ -44,18 +44,20 @@ struct CmdCopySigs : StorePathsCommand
 
         //logger->setExpected(doneLabel, storePaths.size());
 
-        auto doPath = [&](const Path & storePath) {
+        auto doPath = [&](const Path & storePathS) {
             //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath);
 
             checkInterrupt();
 
+            auto storePath = store->parseStorePath(storePathS);
+
             auto info = store->queryPathInfo(storePath);
 
             StringSet newSigs;
 
             for (auto & store2 : substituters) {
                 try {
-                    auto info2 = store2->queryPathInfo(storePath);
+                    auto info2 = store2->queryPathInfo(info->path);
 
                     /* Don't import signatures that don't match this
                        binary. */
@@ -80,11 +82,11 @@ struct CmdCopySigs : StorePathsCommand
         };
 
         for (auto & storePath : storePaths)
-            pool.enqueue(std::bind(doPath, storePath));
+            pool.enqueue(std::bind(doPath, store->printStorePath(storePath)));
 
         pool.process();
 
-        printInfo(format("imported %d signatures") % added);
+        printInfo("imported %d signatures", added);
     }
 };
 
@@ -109,7 +111,7 @@ struct CmdSignPaths : StorePathsCommand
         return "sign the specified paths";
     }
 
-    void run(ref<Store> store, Paths storePaths) override
+    void run(ref<Store> store, StorePaths storePaths) override
     {
         if (secretKeyFile.empty())
             throw UsageError("you must specify a secret key file using '-k'");
@@ -123,7 +125,7 @@ struct CmdSignPaths : StorePathsCommand
 
             auto info2(*info);
             info2.sigs.clear();
-            info2.sign(secretKey);
+            info2.sign(*store, secretKey);
             assert(!info2.sigs.empty());
 
             if (!info->sigs.count(*info2.sigs.begin())) {
@@ -132,7 +134,7 @@ struct CmdSignPaths : StorePathsCommand
             }
         }
 
-        printInfo(format("added %d signatures") % added);
+        printInfo("added %d signatures", added);
     }
 };
 
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index e6c369a7cf3..87f1f9d1b16 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -58,13 +58,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         printInfo("upgrading Nix in profile '%s'", profileDir);
 
-        Path storePath;
-        {
-            Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
-            storePath = getLatestNix(store);
-        }
+        auto storePath = getLatestNix(store);
 
-        auto version = DrvName(storePathToName(storePath)).version;
+        auto version = DrvName(storePath.name()).version;
 
         if (dryRun) {
             stopProgressBar();
@@ -73,13 +69,13 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         }
 
         {
-            Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath));
+            Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath)));
             store->ensurePath(storePath);
         }
 
         {
-            Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath));
-            auto program = storePath + "/bin/nix-env";
+            Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", store->printStorePath(storePath)));
+            auto program = store->printStorePath(storePath) + "/bin/nix-env";
             auto s = runProgram(program, false, {"--version"});
             if (s.find("Nix") == std::string::npos)
                 throw Error("could not verify that '%s' works", program);
@@ -88,9 +84,10 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         stopProgressBar();
 
         {
-            Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir));
+            Activity act(*logger, lvlInfo, actUnknown,
+                fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir));
             runProgram(settings.nixBinDir + "/nix-env", false,
-                {"--profile", profileDir, "-i", storePath, "--no-sandbox"});
+                {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"});
         }
 
         printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
@@ -129,15 +126,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
             !hasSuffix(userEnv, "user-environment"))
             throw Error("directory '%s' does not appear to be part of a Nix profile", where);
 
-        if (!store->isValidPath(userEnv))
+        if (!store->isValidPath(store->parseStorePath(userEnv)))
             throw Error("directory '%s' is not in the Nix store", userEnv);
 
         return profileDir;
     }
 
     /* Return the store path of the latest stable Nix. */
-    Path getLatestNix(ref<Store> store)
+    StorePath getLatestNix(ref<Store> store)
     {
+        Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
+
         // FIXME: use nixos.org?
         auto req = DownloadRequest(storePathsUrl);
         auto res = getDownloader()->download(req);
@@ -148,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Bindings & bindings(*state->allocBindings(0));
         auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v);
 
-        return state->forceString(*v2);
+        return store->parseStorePath(state->forceString(*v2));
     }
 };
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index fa141419615..9b065880366 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -49,7 +49,7 @@ struct CmdVerify : StorePathsCommand
         };
     }
 
-    void run(ref<Store> store, Paths storePaths) override
+    void run(ref<Store> store, StorePaths storePaths) override
     {
         std::vector<ref<Store>> substituters;
         for (auto & s : substituterUris)
@@ -80,7 +80,7 @@ struct CmdVerify : StorePathsCommand
                 MaintainCount<std::atomic<size_t>> mcActive(active);
                 update();
 
-                auto info = store->queryPathInfo(storePath);
+                auto info = store->queryPathInfo(store->parseStorePath(storePath));
 
                 if (!noContents) {
 
@@ -88,7 +88,7 @@ struct CmdVerify : StorePathsCommand
                     if (info->ca == "")
                         hashSink = std::make_unique<HashSink>(info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(store->printStorePath(info->path)));
 
                     store->narFromPath(info->path, *hashSink);
 
@@ -96,10 +96,10 @@ struct CmdVerify : StorePathsCommand
 
                     if (hash.first != info->narHash) {
                         corrupted++;
-                        act2.result(resCorruptedPath, info->path);
+                        act2.result(resCorruptedPath, store->printStorePath(info->path));
                         printError(
-                            format("path '%s' was modified! expected hash '%s', got '%s'")
-                            % info->path % info->narHash.to_string() % hash.first.to_string());
+                            "path '%s' was modified! expected hash '%s', got '%s'",
+                            store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string());
                     }
 
                 }
@@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand
                         auto doSigs = [&](StringSet sigs) {
                             for (auto sig : sigs) {
                                 if (!sigsSeen.insert(sig).second) continue;
-                                if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig))
+                                if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig))
                                     validSigs++;
                             }
                         };
@@ -147,8 +147,8 @@ struct CmdVerify : StorePathsCommand
 
                     if (!good) {
                         untrusted++;
-                        act2.result(resUntrustedPath, info->path);
-                        printError(format("path '%s' is untrusted") % info->path);
+                        act2.result(resUntrustedPath, store->printStorePath(info->path));
+                        printError("path '%s' is untrusted", store->printStorePath(info->path));
                     }
 
                 }
@@ -164,7 +164,7 @@ struct CmdVerify : StorePathsCommand
         };
 
         for (auto & storePath : storePaths)
-            pool.enqueue(std::bind(doPath, storePath));
+            pool.enqueue(std::bind(doPath, store->printStorePath(storePath)));
 
         pool.process();
 
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index fa2a903a86f..8566e58512d 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -73,9 +73,9 @@ struct CmdWhyDepends : SourceExprCommand
         auto packagePath = toStorePath(store, Build, package);
         auto dependency = parseInstallable(*this, store, _dependency, false);
         auto dependencyPath = toStorePath(store, NoBuild, dependency);
-        auto dependencyPathHash = storePathToHash(dependencyPath);
+        auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath));
 
-        PathSet closure;
+        StorePathSet closure;
         store->computeFSClosure({packagePath}, closure, false, false);
 
         if (!closure.count(dependencyPath)) {
@@ -91,28 +91,28 @@ struct CmdWhyDepends : SourceExprCommand
 
         struct Node
         {
-            Path path;
-            PathSet refs;
-            PathSet rrefs;
+            StorePath path;
+            StorePathSet refs;
+            StorePathSet rrefs;
             size_t dist = inf;
             Node * prev = nullptr;
             bool queued = false;
             bool visited = false;
         };
 
-        std::map<Path, Node> graph;
+        std::map<StorePath, Node> graph;
 
         for (auto & path : closure)
-            graph.emplace(path, Node{path, store->queryPathInfo(path)->references});
+            graph.emplace(path.clone(), Node{path.clone(), cloneStorePathSet(store->queryPathInfo(path)->references)});
 
         // Transpose the graph.
         for (auto & node : graph)
             for (auto & ref : node.second.refs)
-                graph[ref].rrefs.insert(node.first);
+                graph.find(ref)->second.rrefs.insert(node.first.clone());
 
         /* Run Dijkstra's shortest path algorithm to get the distance
            of every path in the closure to 'dependency'. */
-        graph[dependencyPath].dist = 0;
+        graph[dependencyPath.clone()].dist = 0;
 
         std::priority_queue<Node *> queue;
 
@@ -151,12 +151,14 @@ struct CmdWhyDepends : SourceExprCommand
         struct BailOut { };
 
         printNode = [&](Node & node, const string & firstPad, const string & tailPad) {
+            auto pathS = store->printStorePath(node.path);
+
             assert(node.dist != inf);
             std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n",
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "=> " : "",
-                node.path);
+                pathS);
 
             if (node.path == dependencyPath && !all
                 && packagePath != dependencyPath)
@@ -175,7 +177,7 @@ struct CmdWhyDepends : SourceExprCommand
                 auto & node2 = graph.at(ref);
                 if (node2.dist == inf) continue;
                 refs.emplace(node2.dist, &node2);
-                hashes.insert(storePathToHash(node2.path));
+                hashes.insert(storePathToHash(store->printStorePath(node2.path)));
             }
 
             /* For each reference, find the files and symlinks that
@@ -187,7 +189,7 @@ struct CmdWhyDepends : SourceExprCommand
             visitPath = [&](const Path & p) {
                 auto st = accessor->stat(p);
 
-                auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1);
+                auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1);
 
                 auto getColour = [&](const std::string & hash) {
                     return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE;
@@ -231,11 +233,11 @@ struct CmdWhyDepends : SourceExprCommand
 
             // FIXME: should use scanForReferences().
 
-            visitPath(node.path);
+            visitPath(pathS);
 
             RunPager pager;
             for (auto & ref : refs) {
-                auto hash = storePathToHash(ref.second->path);
+                auto hash = storePathToHash(store->printStorePath(ref.second->path));
 
                 bool last = all ? ref == *refs.rbegin() : true;
 
diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk
index f9db16268be..bf65f7905f4 100644
--- a/src/resolve-system-dependencies/local.mk
+++ b/src/resolve-system-dependencies/local.mk
@@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d)
 
 resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix
 
-resolve-system-dependencies_LIBS := libstore libmain libutil
+resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust
 
 resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 45e8b679670..8f0c99c84b2 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -179,8 +179,8 @@ int main(int argc, char ** argv)
         if (std::string(argv[1]) == "--test")
             impurePaths.insert(argv[2]);
         else {
-            auto drv = store->derivationFromPath(Path(argv[1]));
-            impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps"));
+            auto drv = store->derivationFromPath(store->parseStorePath(argv[1]));
+            impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps").value_or(""));
             impurePaths.insert("/usr/lib/libSystem.dylib");
         }
 

From f64b58b45e86e15f64aa2dff67436c2d6fe4a4d8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Dec 2019 19:50:02 +0100
Subject: [PATCH 1749/2196] Speed up base32::decode()

From 1.03% to 0.19% of the runtime of 'nix-instantiate "<nixpkgs>" -A
texlive.combined.scheme-full --dry-run'.
---
 nix-rust/src/util/base32.rs | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs
index ee42a0362d8..8f6410f1690 100644
--- a/nix-rust/src/util/base32.rs
+++ b/nix-rust/src/util/base32.rs
@@ -1,5 +1,4 @@
 use crate::error::Error;
-use std::collections::HashMap;
 
 pub fn encoded_len(input_len: usize) -> usize {
     if input_len == 0 {
@@ -9,14 +8,20 @@ pub fn encoded_len(input_len: usize) -> usize {
     }
 }
 
+pub fn decoded_len(input_len: usize) -> usize {
+    input_len * 5 / 8
+}
+
 static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
 
 lazy_static! {
-    static ref BASE32_CHARS_REVERSE: HashMap<char, u8> = BASE32_CHARS
-        .iter()
-        .enumerate()
-        .map(|(n, c)| (*c as char, n as u8))
-        .collect();
+    static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = {
+        let mut xs = [0xffu8; 256];
+        for (n, c) in BASE32_CHARS.iter().enumerate() {
+            xs[*c as usize] = n as u8;
+        }
+        Box::new(xs)
+    };
 }
 
 pub fn encode(input: &[u8]) -> String {
@@ -44,14 +49,17 @@ pub fn encode(input: &[u8]) -> String {
 }
 
 pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
-    let mut res = Vec::new();
+    let mut res = Vec::with_capacity(decoded_len(input.len()));
 
     let mut nr_bits_left: usize = 0;
     let mut bits_left: u16 = 0;
 
     for c in input.chars().rev() {
-        let x = BASE32_CHARS_REVERSE.get(&c).ok_or(Error::BadBase32)?;
-        bits_left |= (*x as u16) << nr_bits_left;
+        let b = BASE32_CHARS_REVERSE[c as usize];
+        if b == 0xff {
+            return Err(Error::BadBase32);
+        }
+        bits_left |= (b as u16) << nr_bits_left;
         nr_bits_left += 5;
         if nr_bits_left >= 8 {
             res.push((bits_left & 0xff) as u8);

From f800d450b78091835ab7ca67847d76e75d877a24 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Dec 2019 20:48:36 +0100
Subject: [PATCH 1750/2196] Speed up StorePath::to_string()

1.81% -> 0.56%
---
 nix-rust/src/c.rs           |  9 +++++++--
 nix-rust/src/store/path.rs  |  4 +++-
 nix-rust/src/util/base32.rs | 19 ++++++++++++++-----
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 4c1724eb3e3..19e737a885b 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -1,6 +1,7 @@
 use super::{
     error,
     foreign::{self, CBox},
+    store::path,
     store::StorePath,
     util,
 };
@@ -53,8 +54,12 @@ pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) {
 }
 
 #[no_mangle]
-pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> String {
-    format!("{}", self_)
+pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> Vec<u8> {
+    let mut buf = vec![0; path::STORE_PATH_HASH_CHARS + 1 + self_.name.name().len()];
+    util::base32::encode_into(self_.hash.hash(), &mut buf[0..path::STORE_PATH_HASH_CHARS]);
+    buf[path::STORE_PATH_HASH_CHARS] = b'-';
+    buf[path::STORE_PATH_HASH_CHARS + 1..].clone_from_slice(self_.name.name().as_bytes());
+    buf
 }
 
 #[no_mangle]
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 3ded5c428e9..2a223247522 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -70,7 +70,9 @@ impl StorePathHash {
 
 impl fmt::Display for StorePathHash {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&base32::encode(&self.0))
+        let mut buf = vec![0; STORE_PATH_HASH_CHARS];
+        base32::encode_into(&self.0, &mut buf);
+        f.write_str(std::str::from_utf8(&buf).unwrap())
     }
 }
 
diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs
index 8f6410f1690..ba73689336c 100644
--- a/nix-rust/src/util/base32.rs
+++ b/nix-rust/src/util/base32.rs
@@ -25,27 +25,36 @@ lazy_static! {
 }
 
 pub fn encode(input: &[u8]) -> String {
-    let mut res = String::new();
-    res.reserve(encoded_len(input.len()));
+    let mut buf = vec![0; encoded_len(input.len())];
+    encode_into(input, &mut buf);
+    std::str::from_utf8(&buf).unwrap().to_string()
+}
+
+pub fn encode_into(input: &[u8], output: &mut [u8]) {
+    let len = encoded_len(input.len());
+    assert_eq!(len, output.len());
 
     let mut nr_bits_left: usize = 0;
     let mut bits_left: u16 = 0;
+    let mut pos = len;
 
     for b in input {
         bits_left |= (*b as u16) << nr_bits_left;
         nr_bits_left += 8;
         while nr_bits_left > 5 {
-            res.push(BASE32_CHARS[(bits_left & 0x1f) as usize] as char);
+            output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+            pos -= 1;
             bits_left >>= 5;
             nr_bits_left -= 5;
         }
     }
 
     if nr_bits_left > 0 {
-        res.push(BASE32_CHARS[(bits_left & 0x1f) as usize] as char);
+        output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+        pos -= 1;
     }
 
-    res.chars().rev().collect()
+    assert_eq!(pos, 0);
 }
 
 pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {

From 38b29fb72ca4a07afbec1fd5067f59ca7d7f0fab Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 12 Dec 2019 15:15:18 +0100
Subject: [PATCH 1751/2196] libstore/ssh: Improve error message on failing
 `execvp`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If the `throw` is reached, this means that execvp into `ssh` wasn’t
successful. We can hint at a usual problem, which is a missing `ssh`
executable.

Test with:

```
env PATH= ./result/bin/nix-copy-closure --builders '' unusedhost
```

and the bash version with

```
env PATH= ./result/bin/nix-copy-closure --builders '' localhost
```
---
 src/libstore/ssh.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index ac3ccd63d14..ddc99a6cd22 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -47,10 +47,13 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             throw SysError("duping over stderr");
 
         Strings args;
+        const char * execInto;
 
         if (fakeSSH) {
+            execInto = "bash";
             args = { "bash", "-c" };
         } else {
+            execInto = "ssh";
             args = { "ssh", host.c_str(), "-x", "-a" };
             addCommonSSHOpts(args);
             if (socketPath != "")
@@ -62,7 +65,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("executing '%s' on '%s'", command, host);
+        // could not exec ssh/bash
+        throw SysError("Failed to exec into %s. Is it in PATH?", execInto);
     });
 
 
@@ -108,7 +112,7 @@ Path SSHMaster::startMaster()
         addCommonSSHOpts(args);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("starting SSH master");
+        throw SysError("Failed to exec into ssh. Is it in PATH?");
     });
 
     out.writeSide = -1;

From c6295a3afd87a605f30f1de8b09d7885eb08fedb Mon Sep 17 00:00:00 2001
From: Tom Bereknyei <tom@rebelliondefense.com>
Date: Fri, 13 Dec 2019 03:29:33 -0500
Subject: [PATCH 1752/2196] Initial gzip support

Closes #3256
---
 configure.ac               |  5 +++
 release-common.nix         |  2 +-
 src/libutil/compression.cc | 64 ++++++++++++++++++++++++++++++++++++++
 src/libutil/tarfile.cc     |  1 +
 tests/tarball.sh           | 17 ++++++++++
 5 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 9dd0acd86c7..99248b90c78 100644
--- a/configure.ac
+++ b/configure.ac
@@ -209,6 +209,11 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
+# Look for zlib, a required dependency.
+PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
+AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
+LDFLAGS="-lz $LDFLAGS"
+
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
diff --git a/release-common.nix b/release-common.nix
index dd5f939d969..3765e61b30e 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -47,7 +47,7 @@ rec {
 
   buildDeps =
     [ curl
-      bzip2 xz brotli editline
+      bzip2 xz brotli zlib editline
       openssl pkgconfig sqlite boehmgc
       boost
       nlohmann_json
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 0dd84e32034..17b506d5da9 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -11,6 +11,8 @@
 #include <brotli/decode.h>
 #include <brotli/encode.h>
 
+#include <zlib.h>
+
 #include <iostream>
 
 namespace nix {
@@ -42,6 +44,66 @@ struct NoneSink : CompressionSink
     void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
 };
 
+struct GzipDecompressionSink : CompressionSink
+{
+    Sink & nextSink;
+    z_stream strm;
+    bool finished = false;
+    uint8_t outbuf[BUFSIZ];
+
+    GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        strm.zalloc = Z_NULL;
+        strm.zfree = Z_NULL;
+        strm.opaque = Z_NULL;
+        strm.avail_in = 0;
+        strm.next_in = Z_NULL;
+        strm.next_out = outbuf;
+        strm.avail_out = sizeof(outbuf);
+
+        // Enable gzip and zlib decoding (+32) with 15 windowBits
+        int ret = inflateInit2(&strm,15+32);
+        if (ret != Z_OK)
+            throw CompressionError("unable to initialise gzip encoder");
+    }
+
+    ~GzipDecompressionSink()
+    {
+        inflateEnd(&strm);
+    }
+
+    void finish() override
+    {
+        CompressionSink::flush();
+        write(nullptr, 0);
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+
+        strm.next_in = (Bytef *) data;
+        strm.avail_in = len;
+
+        while (!finished && (!data || strm.avail_in)) {
+            checkInterrupt();
+
+            int ret = inflate(&strm,Z_SYNC_FLUSH);
+            if (ret != Z_OK && ret != Z_STREAM_END)
+                throw CompressionError("error while decompressing gzip file: %d: %d: %d",ret, len, strm.avail_in);
+
+
+            finished = ret == Z_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (Bytef *) outbuf;
+                strm.avail_out = sizeof(outbuf);
+            }
+        }
+    }
+};
+
 struct XzDecompressionSink : CompressionSink
 {
     Sink & nextSink;
@@ -215,6 +277,8 @@ ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & ne
         return make_ref<XzDecompressionSink>(nextSink);
     else if (method == "bzip2")
         return make_ref<BzipDecompressionSink>(nextSink);
+    else if (method == "gzip")
+        return make_ref<GzipDecompressionSink>(nextSink);
     else if (method == "br")
         return make_ref<BrotliDecompressionSink>(nextSink);
     else
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 262bc655f16..c00673182e5 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -24,6 +24,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir,
         auto decompressor =
             // FIXME: add .gz support
             hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
+            hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
             hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
             makeDecompressionSink("none", sink);
         readFile(tarFile, *decompressor);
diff --git a/tests/tarball.sh b/tests/tarball.sh
index ba534c6261a..ec810b4d715 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -26,3 +26,20 @@ nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-ta
 (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
 
 nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.
+
+tarball=$TEST_ROOT/tarball.tar.gz
+(cd $TEST_ROOT && tar c tarball) | gzip > $tarball
+
+nix-env -f file://$tarball -qa --out-path | grep -q dependencies
+
+nix-build -o $TEST_ROOT/result file://$tarball
+
+nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
+
+nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
+
+nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.gz
+nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.gz
+(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.gz)
+
+nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.gz -I fnord=.

From d1b238ec3cd74d652af46f577f992c9a44ac8e32 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 12:53:20 +0100
Subject: [PATCH 1753/2196] Simplify

---
 src/libstore/ssh.cc | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index ddc99a6cd22..2ee7115c53c 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -47,13 +47,10 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
             throw SysError("duping over stderr");
 
         Strings args;
-        const char * execInto;
 
         if (fakeSSH) {
-            execInto = "bash";
             args = { "bash", "-c" };
         } else {
-            execInto = "ssh";
             args = { "ssh", host.c_str(), "-x", "-a" };
             addCommonSSHOpts(args);
             if (socketPath != "")
@@ -66,7 +63,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         // could not exec ssh/bash
-        throw SysError("Failed to exec into %s. Is it in PATH?", execInto);
+        throw SysError("unable to execute '%s'", args.front());
     });
 
 
@@ -112,7 +109,7 @@ Path SSHMaster::startMaster()
         addCommonSSHOpts(args);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("Failed to exec into ssh. Is it in PATH?");
+        throw SysError("unable to execute '%s'", args.front());
     });
 
     out.writeSide = -1;

From 3e787423c22c34d65fd6fe0cdf2e6ec333d81c0b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 12:55:52 +0100
Subject: [PATCH 1754/2196] Remove FIXME

---
 src/libutil/tarfile.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index c00673182e5..79fb9f721c1 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -22,7 +22,6 @@ void unpackTarfile(const Path & tarFile, const Path & destDir,
     auto source = sinkToSource([&](Sink & sink) {
         // FIXME: look at first few bytes to determine compression type.
         auto decompressor =
-            // FIXME: add .gz support
             hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
             hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
             hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :

From e8aa2290ed4517d42408909f18f1b9434261c36a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 14:42:55 +0100
Subject: [PATCH 1755/2196] Only install *.sb files on macOS

---
 src/libstore/local.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index d3254554dd3..ac68c2342b5 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -13,7 +13,9 @@ ifneq ($(OS), FreeBSD)
  libstore_LDFLAGS += -ldl
 endif
 
+ifeq ($(OS), Darwin)
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
+endif
 
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 

From d89d9958a771fb6299eae084c1b52d0ecf95901b Mon Sep 17 00:00:00 2001
From: Dima <dgoldin+github@protonmail.ch>
Date: Fri, 13 Dec 2019 14:44:14 +0100
Subject: [PATCH 1756/2196] bugfix: Adding depth limit to resolveExprPath

There is no termination condition for evaluation of cyclical
expression paths which can lead to infinite loops. This addresses
one spot in the parser in a similar fashion as utils.cc/canonPath
does.

This issue can be reproduced by something like:

```
ln -s a b
ln -s b a

nix-instantiate -E 'import ./a'
```
---
 src/libexpr/parser.y | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 10a057062e6..df55986f652 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -576,10 +576,15 @@ Path resolveExprPath(Path path)
 {
     assert(path[0] == '/');
 
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
     while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+          throw Error(format("can't resolve expression. infinite symlink recursion in path '%1%'") % path);
         if (lstat(path.c_str(), &st))
             throw SysError(format("getting status of '%1%'") % path);
         if (!S_ISLNK(st.st_mode)) break;

From 4581159e3f5a1cc38aa8e90dfd45f0a63354be44 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 17:26:58 +0100
Subject: [PATCH 1757/2196] Simplify tarball test

---
 tests/tarball.sh | 42 +++++++++++++++++-------------------------
 1 file changed, 17 insertions(+), 25 deletions(-)

diff --git a/tests/tarball.sh b/tests/tarball.sh
index ec810b4d715..7738b948d90 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -10,36 +10,28 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
-tarball=$TEST_ROOT/tarball.tar.xz
-(cd $TEST_ROOT && tar c tarball) | xz > $tarball
+test_tarball() {
+    local ext="$1"
+    local compressor="$2"
 
-nix-env -f file://$tarball -qa --out-path | grep -q dependencies
+    tarball=$TEST_ROOT/tarball.tar$ext
+    (cd $TEST_ROOT && tar c tarball) | $compressor > $tarball
 
-nix-build -o $TEST_ROOT/result file://$tarball
+    nix-env -f file://$tarball -qa --out-path | grep -q dependencies
 
-nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
+    nix-build -o $TEST_ROOT/result file://$tarball
 
-nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
+    nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
 
-nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz
-nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz
-(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
+    nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
 
-nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.
+    nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
+    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
+    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
 
-tarball=$TEST_ROOT/tarball.tar.gz
-(cd $TEST_ROOT && tar c tarball) | gzip > $tarball
+    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
+}
 
-nix-env -f file://$tarball -qa --out-path | grep -q dependencies
-
-nix-build -o $TEST_ROOT/result file://$tarball
-
-nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
-
-nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
-
-nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.gz
-nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.gz
-(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.gz)
-
-nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.gz -I fnord=.
+test_tarball '' cat
+test_tarball .xz xz
+test_tarball .gz gzip

From 5a6d6da7aea23a48126a77f98612518af66bc203 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 18:11:37 +0100
Subject: [PATCH 1758/2196] Validate tarball components

---
 nix-rust/src/c.rs            |   5 ++++-
 nix-rust/src/error.rs        |   4 ++++
 nix-rust/src/util/tarfile.rs |  18 ++++++++++++++----
 tests/bad.tar.xz             | Bin 0 -> 228 bytes
 tests/tarball.sh             |   5 +++++
 5 files changed, 27 insertions(+), 5 deletions(-)
 create mode 100644 tests/bad.tar.xz

diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 19e737a885b..3feeb71a3b4 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -11,7 +11,10 @@ pub extern "C" fn unpack_tarfile(
     source: foreign::Source,
     dest_dir: &str,
 ) -> CBox<Result<(), error::CppException>> {
-    CBox::new(util::tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
+    CBox::new(
+        util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir))
+            .map_err(|err| err.into()),
+    )
 }
 
 #[no_mangle]
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index c2ba5d5048c..586a8f53d00 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -23,6 +23,7 @@ pub enum Error {
     HttpError(hyper::error::Error),
     Misc(String),
     Foreign(CppException),
+    BadTarFileMemberName(String),
 }
 
 impl From<std::io::Error> for Error {
@@ -64,6 +65,9 @@ impl fmt::Display for Error {
             Error::HttpError(err) => write!(f, "HTTP error: {}", err),
             Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
             Error::Misc(s) => write!(f, "{}", s),
+            Error::BadTarFileMemberName(s) => {
+                write!(f, "tar archive contains illegal file name '{}'", s)
+            }
         }
     }
 }
diff --git a/nix-rust/src/util/tarfile.rs b/nix-rust/src/util/tarfile.rs
index 379d9098f1c..74d60692c4a 100644
--- a/nix-rust/src/util/tarfile.rs
+++ b/nix-rust/src/util/tarfile.rs
@@ -2,18 +2,28 @@ use crate::{foreign::Source, Error};
 use std::fs;
 use std::io;
 use std::os::unix::fs::OpenOptionsExt;
-use std::path::Path;
+use std::path::{Component, Path};
 use tar::Archive;
 
-pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
-    let dest_dir = Path::new(dest_dir);
+pub fn unpack_tarfile(source: Source, dest_dir: &Path) -> Result<(), Error> {
+    fs::create_dir_all(dest_dir)?;
 
     let mut tar = Archive::new(source);
 
     for file in tar.entries()? {
         let mut file = file?;
 
-        let dest_file = dest_dir.join(file.path()?);
+        let path = file.path()?;
+
+        for i in path.components() {
+            if let Component::Prefix(_) | Component::RootDir | Component::ParentDir = i {
+                return Err(Error::BadTarFileMemberName(
+                    file.path()?.to_str().unwrap().to_string(),
+                ));
+            }
+        }
+
+        let dest_file = dest_dir.join(path);
 
         fs::create_dir_all(dest_file.parent().unwrap())?;
 
diff --git a/tests/bad.tar.xz b/tests/bad.tar.xz
new file mode 100644
index 0000000000000000000000000000000000000000..250a5ad1a79ee088d5976160664daad6e1a136ff
GIT binary patch
literal 228
zcmV<A02}}PH+ooF000E$*0e?f03iVu0001VFXf})C;tGVT>uwgxWr*GQJ}UxbD$dT
zFZcR4Oq`4c+brLa?D6R=fMJjM$?MdlXt0Pp!zwDzlSF=e({Qq}bnc_B9C}zwW!F?<
z;h>7LqPxu=kzFWj+$Z*4&0g78Yyp1kn1+nCzFC}~s1p<w+rfbIGz<bH?U#Qzyw`;6
z`({F^v@!)B&J6Y%vn53KeRqd?V5+Eub*M94BlkPr%izV3Qzu-*Z?*3L0002dm00(D
e^?nKf0loo%Pyhhq7Z6{u#Ao{g000001X)^+32KG_

literal 0
HcmV?d00001

diff --git a/tests/tarball.sh b/tests/tarball.sh
index 7738b948d90..8adb8d72f83 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -35,3 +35,8 @@ test_tarball() {
 test_tarball '' cat
 test_tarball .xz xz
 test_tarball .gz gzip
+
+rm -rf $TEST_ROOT/tmp
+mkdir -p $TEST_ROOT/tmp
+(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path)
+(! [ -e $TEST_ROOT/tmp/bad ])

From ca87707c90c05289d0c7c1015f5750f6dd93708b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 18:29:16 +0100
Subject: [PATCH 1759/2196] Get rid of CBox

---
 nix-rust/src/c.rs       | 11 ++++++-----
 nix-rust/src/foreign.rs | 19 -------------------
 src/libutil/rust-ffi.hh | 20 --------------------
 src/libutil/tarfile.cc  |  6 ++++--
 4 files changed, 10 insertions(+), 46 deletions(-)

diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index 3feeb71a3b4..e0462742fa1 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -1,20 +1,21 @@
 use super::{
     error,
-    foreign::{self, CBox},
+    foreign::{self},
     store::path,
     store::StorePath,
     util,
 };
 
 #[no_mangle]
-pub extern "C" fn unpack_tarfile(
+pub unsafe extern "C" fn unpack_tarfile(
     source: foreign::Source,
     dest_dir: &str,
-) -> CBox<Result<(), error::CppException>> {
-    CBox::new(
+    out: *mut Result<(), error::CppException>,
+) {
+    out.write(
         util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir))
             .map_err(|err| err.into()),
-    )
+    );
 }
 
 #[no_mangle]
diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs
index 8e04280f3e6..7bce7753c05 100644
--- a/nix-rust/src/foreign.rs
+++ b/nix-rust/src/foreign.rs
@@ -12,22 +12,3 @@ impl std::io::Read for Source {
         Ok(n)
     }
 }
-
-pub struct CBox<T> {
-    pub ptr: *mut libc::c_void,
-    phantom: std::marker::PhantomData<T>,
-}
-
-impl<T> CBox<T> {
-    pub fn new(t: T) -> Self {
-        unsafe {
-            let size = std::mem::size_of::<T>();
-            let ptr = libc::malloc(size);
-            *(ptr as *mut T) = t; // FIXME: probably UB
-            Self {
-                ptr,
-                phantom: std::marker::PhantomData,
-            }
-        }
-    }
-}
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index a77f83ac535..1466eabecda 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -180,24 +180,4 @@ struct Result
     }
 };
 
-template<typename T>
-struct CBox
-{
-    T * ptr;
-
-    T * operator ->()
-    {
-        return ptr;
-    }
-
-    CBox(T * ptr) : ptr(ptr) { }
-    CBox(const CBox &) = delete;
-    CBox(CBox &&) = delete;
-
-    ~CBox()
-    {
-        free(ptr);
-    }
-};
-
 }
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 79fb9f721c1..1be0ba24c0f 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -3,7 +3,7 @@
 
 extern "C" {
     rust::Result<std::tuple<>> *
-    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir, rust::Result<std::tuple<>> & out);
 }
 
 namespace nix {
@@ -11,7 +11,9 @@ namespace nix {
 void unpackTarfile(Source & source, const Path & destDir)
 {
     rust::Source source2(source);
-    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
+    rust::Result<std::tuple<>> res;
+    unpack_tarfile(source2, destDir, res);
+    res.unwrap();
 }
 
 void unpackTarfile(const Path & tarFile, const Path & destDir,

From e6bd88878e8fc08652b77f0444adcd74321dc6b6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 18:58:34 +0100
Subject: [PATCH 1760/2196] Improve gzip error message

---
 src/libutil/compression.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 17b506d5da9..860b04adb79 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -90,8 +90,8 @@ struct GzipDecompressionSink : CompressionSink
 
             int ret = inflate(&strm,Z_SYNC_FLUSH);
             if (ret != Z_OK && ret != Z_STREAM_END)
-                throw CompressionError("error while decompressing gzip file: %d: %d: %d",ret, len, strm.avail_in);
-
+                throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
+                    zError(ret), len, strm.avail_in);
 
             finished = ret == Z_STREAM_END;
 

From b4edc3ca61c1f1a98052ec8ffc41d1b533b08fd7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 19:05:09 +0100
Subject: [PATCH 1761/2196] Don't leak exceptions

---
 nix-rust/src/error.rs   | 20 ++++++++++++++++++--
 src/libutil/rust-ffi.cc |  6 +++++-
 src/libutil/rust-ffi.hh | 31 +++++++++++++++++++++++++------
 3 files changed, 48 insertions(+), 9 deletions(-)

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 586a8f53d00..85ac926e915 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -76,7 +76,7 @@ impl From<Error> for CppException {
     fn from(err: Error) -> Self {
         match err {
             Error::Foreign(ex) => ex,
-            _ => unsafe { make_error(&err.to_string()) },
+            _ => CppException::new(&err.to_string()),
         }
     }
 }
@@ -85,7 +85,23 @@ impl From<Error> for CppException {
 #[derive(Debug)]
 pub struct CppException(*const libc::c_void); // == std::exception_ptr*
 
+impl CppException {
+    fn new(s: &str) -> Self {
+        Self(unsafe { make_error(s) })
+    }
+}
+
+impl Drop for CppException {
+    fn drop(&mut self) {
+        unsafe {
+            destroy_error(self.0);
+        }
+    }
+}
+
 extern "C" {
     #[allow(improper_ctypes)] // YOLO
-    fn make_error(s: &str) -> CppException;
+    fn make_error(s: &str) -> *const libc::c_void;
+
+    fn destroy_error(exc: *const libc::c_void);
 }
diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc
index accc5e22b97..6f36b31922c 100644
--- a/src/libutil/rust-ffi.cc
+++ b/src/libutil/rust-ffi.cc
@@ -3,10 +3,14 @@
 
 extern "C" std::exception_ptr * make_error(rust::StringSlice s)
 {
-    // FIXME: leak
     return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size))));
 }
 
+extern "C" void destroy_error(std::exception_ptr * ex)
+{
+    free(ex);
+}
+
 namespace rust {
 
 std::ostream & operator << (std::ostream & str, const String & s)
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index 1466eabecda..4fecce606ca 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -152,28 +152,47 @@ struct Source
 template<typename T>
 struct Result
 {
-    unsigned int tag;
+    enum { Ok = 0, Err = 1, Uninit = 2 } tag;
 
     union {
         T data;
         std::exception_ptr * exc;
     };
 
+    Result() : tag(Uninit) { }; // FIXME: remove
+
+    Result(const Result &) = delete;
+
+    Result(Result && other)
+        : tag(other.tag)
+    {
+        other.tag = Uninit;
+        if (tag == Ok)
+            data = std::move(other.data);
+        else if (tag == Err)
+            exc = other.exc;
+    }
+
     ~Result()
     {
-        if (tag == 0)
+        if (tag == Ok)
             data.~T();
-        else if (tag == 1)
-            // FIXME: don't leak exc
+        else if (tag == Err)
+            free(exc);
+        else if (tag == Uninit)
             ;
+        else
+            abort();
     }
 
     /* Rethrow the wrapped exception or return the wrapped value. */
     T unwrap()
     {
-        if (tag == 0)
+        if (tag == Ok) {
+            tag = Uninit;
             return std::move(data);
-        else if (tag == 1)
+        }
+        else if (tag == Err)
             std::rethrow_exception(*exc);
         else
             abort();

From ac9cc2ec08e95910ecb73745cb011596a33723f0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Dec 2019 19:10:39 +0100
Subject: [PATCH 1762/2196] Move some code

---
 src/libutil/rust-ffi.cc | 11 +++++++++++
 src/libutil/rust-ffi.hh |  7 +------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc
index 6f36b31922c..8b8b7b75d2a 100644
--- a/src/libutil/rust-ffi.cc
+++ b/src/libutil/rust-ffi.cc
@@ -19,4 +19,15 @@ std::ostream & operator << (std::ostream & str, const String & s)
     return str;
 }
 
+size_t Source::sourceWrapper(void * _this, rust::Slice<uint8_t> data)
+{
+    try {
+        // FIXME: how to propagate exceptions?
+        auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
+        return n;
+    } catch (...) {
+        abort();
+    }
+}
+
 }
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index 4fecce606ca..3b51661c2cd 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -140,12 +140,7 @@ struct Source
         : fun(sourceWrapper), _this(&_this)
     {}
 
-    // FIXME: how to propagate exceptions?
-    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
-    {
-        auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
-        return n;
-    }
+    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data);
 };
 
 /* C++ representation of Rust's Result<T, CppException>. */

From a70706b0253e3bbd9143fde16e65fb7fe1a4007d Mon Sep 17 00:00:00 2001
From: Albert Safin <xzfcpw@gmail.com>
Date: Sat, 14 Dec 2019 15:37:20 +0000
Subject: [PATCH 1763/2196] nix-shell: don't check for "nix-shell" in shebang
 script name

---
 src/nix-build/nix-build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 73e3a347bf8..205165a4ce1 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -106,7 +106,7 @@ static void _main(int argc, char * * argv)
     // Heuristic to see if we're invoked as a shebang script, namely,
     // if we have at least one argument, it's the name of an
     // executable file, and it starts with "#!".
-    if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) {
+    if (runEnv && argc > 1) {
         script = argv[1];
         try {
             auto lines = tokenizeString<Strings>(readFile(script), "\n");

From ba6d2093c74f2023394b1b8a4a4d2a1f405c70c1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 14 Dec 2019 23:19:04 +0100
Subject: [PATCH 1764/2196] Fix progress bar

---
 src/nix/progress-bar.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index d20c09f2683..c445f31ccf3 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -155,7 +155,7 @@ class ProgressBar : public Logger
         if (type == actBuild) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name = name.substr(name.size() - 4);
+                name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
             auto machineName = getS(fields, 1);
             if (machineName != "")
@@ -180,7 +180,7 @@ class ProgressBar : public Logger
         if (type == actPostBuildHook) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name = name.substr(name.size() - 4);
+                name = name.substr(0, name.size() - 4);
             i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
             i->name = DrvName(name).name;
         }

From acb71aa5c6a7a55bc1c483c2a6940d85997d482c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 15 Dec 2019 10:44:53 +0100
Subject: [PATCH 1765/2196] Tweak error message

---
 src/libexpr/parser.y | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index df55986f652..afa1fd439a9 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -584,9 +584,9 @@ Path resolveExprPath(Path path)
     while (true) {
         // Basic cycle/depth limit to avoid infinite loops.
         if (++followCount >= maxFollow)
-          throw Error(format("can't resolve expression. infinite symlink recursion in path '%1%'") % path);
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%s'", path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }

From 410acd29c046ae7296d882ee4750441d4ff29955 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 15 Dec 2019 10:47:59 +0100
Subject: [PATCH 1766/2196] Fix cargo test

---
 nix-rust/src/error.rs | 7 +++++++
 nix-rust/src/lib.rs   | 1 +
 2 files changed, 8 insertions(+)

diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 85ac926e915..9abcacc0626 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -22,6 +22,7 @@ pub enum Error {
     #[cfg(unused)]
     HttpError(hyper::error::Error),
     Misc(String),
+    #[cfg(not(test))]
     Foreign(CppException),
     BadTarFileMemberName(String),
 }
@@ -63,6 +64,7 @@ impl fmt::Display for Error {
             Error::IOError(err) => write!(f, "I/O error: {}", err),
             #[cfg(unused)]
             Error::HttpError(err) => write!(f, "HTTP error: {}", err),
+            #[cfg(not(test))]
             Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
             Error::Misc(s) => write!(f, "{}", s),
             Error::BadTarFileMemberName(s) => {
@@ -72,6 +74,7 @@ impl fmt::Display for Error {
     }
 }
 
+#[cfg(not(test))]
 impl From<Error> for CppException {
     fn from(err: Error) -> Self {
         match err {
@@ -81,16 +84,19 @@ impl From<Error> for CppException {
     }
 }
 
+#[cfg(not(test))]
 #[repr(C)]
 #[derive(Debug)]
 pub struct CppException(*const libc::c_void); // == std::exception_ptr*
 
+#[cfg(not(test))]
 impl CppException {
     fn new(s: &str) -> Self {
         Self(unsafe { make_error(s) })
     }
 }
 
+#[cfg(not(test))]
 impl Drop for CppException {
     fn drop(&mut self) {
         unsafe {
@@ -99,6 +105,7 @@ impl Drop for CppException {
     }
 }
 
+#[cfg(not(test))]
 extern "C" {
     #[allow(improper_ctypes)] // YOLO
     fn make_error(s: &str) -> *const libc::c_void;
diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index 0c3cf5276fd..9935cd27a14 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -9,6 +9,7 @@ extern crate assert_matches;
 #[macro_use]
 extern crate proptest;
 
+#[cfg(not(test))]
 mod c;
 mod error;
 mod foreign;

From 14d82baba4ebb82df28c2d4e9517f8c3a81d8f6c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 16 Dec 2019 17:41:56 +0100
Subject: [PATCH 1767/2196] StorePath::new(): Check store directory

---
 nix-rust/src/c.rs          | 3 ++-
 nix-rust/src/error.rs      | 4 ++++
 nix-rust/src/store/path.rs | 6 ++++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs
index e0462742fa1..8d2507d377e 100644
--- a/nix-rust/src/c.rs
+++ b/nix-rust/src/c.rs
@@ -34,7 +34,8 @@ pub extern "C" fn ffi_StorePath_new(
     path: &str,
     store_dir: &str,
 ) -> Result<StorePath, error::CppException> {
-    StorePath::new(std::path::Path::new(path), store_dir).map_err(|err| err.into())
+    StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
+        .map_err(|err| err.into())
 }
 
 #[no_mangle]
diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs
index 9abcacc0626..bb0c9a9338d 100644
--- a/nix-rust/src/error.rs
+++ b/nix-rust/src/error.rs
@@ -4,6 +4,7 @@ use std::fmt;
 pub enum Error {
     InvalidPath(crate::store::StorePath),
     BadStorePath(std::path::PathBuf),
+    NotInStore(std::path::PathBuf),
     BadNarInfo,
     BadBase32,
     StorePathNameEmpty,
@@ -46,6 +47,9 @@ impl fmt::Display for Error {
             Error::InvalidPath(_) => write!(f, "invalid path"),
             Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
             Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
+            Error::NotInStore(path) => {
+                write!(f, "path '{}' is not in the Nix store", path.display())
+            }
             Error::BadBase32 => write!(f, "invalid base32 string"),
             Error::StorePathNameEmpty => write!(f, "store path name is empty"),
             Error::StorePathNameTooLong => {
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 2a223247522..2a5170befec 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -13,8 +13,10 @@ pub const STORE_PATH_HASH_BYTES: usize = 20;
 pub const STORE_PATH_HASH_CHARS: usize = 32;
 
 impl StorePath {
-    pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
-        // FIXME: check store_dir
+    pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
+        if path.parent() != Some(store_dir) {
+            return Err(Error::NotInStore(path.into()));
+        }
         Self::new_from_base_name(
             path.file_name()
                 .ok_or(Error::BadStorePath(path.into()))?

From 54bf5ba4227a234f8cd5102634b9a3b535e6fbdb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 16 Dec 2019 19:11:47 +0100
Subject: [PATCH 1768/2196] nix-store -r: Handle symlinks to store paths

Fixes #3270.
---
 src/libexpr/get-drvs.cc     |  2 +-
 src/libstore/daemon.cc      |  4 ++--
 src/libstore/derivations.cc | 10 ----------
 src/libstore/path.cc        | 15 +++++++++++++++
 src/libstore/path.hh        | 24 ++++++++++++++++++++++++
 src/libstore/store-api.cc   | 13 ++++++++++---
 src/libstore/store-api.hh   | 25 +++++--------------------
 src/libutil/util.cc         |  8 ++++----
 src/libutil/util.hh         |  2 +-
 src/nix-store/nix-store.cc  |  4 ++--
 tests/nix-build.sh          |  3 +++
 11 files changed, 67 insertions(+), 43 deletions(-)

diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index a3412eab946..ca9c547faa7 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto [drvPath, selectedOutputs] = store->parseDrvPathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
 
     this->drvPath = store->printStorePath(drvPath);
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index c53dfdd56fe..8e9f9d71bf0 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopBuildPaths: {
         std::vector<StorePathWithOutputs> drvs;
         for (auto & s : readStrings<Strings>(from))
-            drvs.push_back(store->parseDrvPathWithOutputs(s));
+            drvs.push_back(store->parsePathWithOutputs(s));
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -721,7 +721,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryMissing: {
         std::vector<StorePathWithOutputs> targets;
         for (auto & s : readStrings<Strings>(from))
-            targets.push_back(store->parseDrvPathWithOutputs(s));
+            targets.push_back(store->parsePathWithOutputs(s));
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 726e34479f8..1a061149a3b 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -375,16 +375,6 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
 }
 
 
-StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s)
-{
-    size_t n = s.find("!");
-    return n == s.npos
-        ? StorePathWithOutputs{parseStorePath(s), std::set<string>()}
-        : StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)),
-            tokenizeString<std::set<string>>(string(s, n + 1), ",")};
-}
-
-
 std::string StorePathWithOutputs::to_string(const Store & store) const
 {
     return outputs.empty()
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 81ae495a1a0..cda5f996818 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -96,4 +96,19 @@ StorePathSet singleton(const StorePath & path)
     return res;
 }
 
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
+{
+    size_t n = s.find("!");
+    return n == s.npos
+        ? std::make_pair(s, std::set<string>())
+        : std::make_pair(((std::string_view) s).substr(0, n),
+            tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
+}
+
+StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
+{
+    auto [path, outputs] = nix::parsePathWithOutputs(s);
+    return {parseStorePath(path), std::move(outputs)};
+}
+
 }
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 273808f02e4..5ebb574804e 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -7,6 +7,8 @@ namespace nix {
 /* See path.rs. */
 struct StorePath;
 
+struct Store;
+
 extern "C" {
     void ffi_StorePath_drop(void *);
     bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
@@ -67,6 +69,28 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
+struct StorePathWithOutputs
+{
+    StorePath path;
+    std::set<std::string> outputs;
+
+    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
+        : path(path.clone()), outputs(outputs)
+    { }
+
+    StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs)
+        : path(std::move(path)), outputs(std::move(outputs))
+    { }
+
+    StorePathWithOutputs(const StorePathWithOutputs & other)
+        : path(other.path.clone()), outputs(other.outputs)
+    { }
+
+    std::string to_string(const Store & store) const;
+};
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
+
 }
 
 namespace std {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c98931885df..1781e0c6839 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -39,9 +39,9 @@ Path Store::toStorePath(const Path & path) const
 }
 
 
-Path Store::followLinksToStore(const Path & _path) const
+Path Store::followLinksToStore(std::string_view _path) const
 {
-    Path path = absPath(_path);
+    Path path = absPath(std::string(_path));
     while (!isInStore(path)) {
         if (!isLink(path)) break;
         string target = readLink(path);
@@ -53,12 +53,19 @@ Path Store::followLinksToStore(const Path & _path) const
 }
 
 
-StorePath Store::followLinksToStorePath(const Path & path) const
+StorePath Store::followLinksToStorePath(std::string_view path) const
 {
     return parseStorePath(toStorePath(followLinksToStore(path)));
 }
 
 
+StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
+{
+    auto [path2, outputs] = nix::parsePathWithOutputs(path);
+    return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs));
+}
+
+
 string storePathToHash(const Path & path)
 {
     auto base = baseNameOf(path);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 552bbd67d47..d77d6bad830 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -241,23 +241,6 @@ struct BuildResult
 };
 
 
-struct StorePathWithOutputs
-{
-    StorePath path;
-    std::set<std::string> outputs;
-
-    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
-        : path(path.clone()), outputs(outputs)
-    { }
-
-    StorePathWithOutputs(const StorePathWithOutputs & other)
-        : path(other.path.clone()), outputs(other.outputs)
-    { }
-
-    std::string to_string(const Store & store) const;
-};
-
-
 class Store : public std::enable_shared_from_this<Store>, public Config
 {
 public:
@@ -304,7 +287,7 @@ public:
     /* Split a string specifying a derivation and a set of outputs
        (/nix/store/hash-foo!out1,out2,...) into the derivation path
        and the outputs. */
-    StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
+    StorePathWithOutputs parsePathWithOutputs(const string & s);
 
     /* Display a set of paths in human-readable form (i.e., between quotes
        and separated by commas). */
@@ -323,11 +306,13 @@ public:
     Path toStorePath(const Path & path) const;
 
     /* Follow symlinks until we end up with a path in the Nix store. */
-    Path followLinksToStore(const Path & path) const;
+    Path followLinksToStore(std::string_view path) const;
 
     /* Same as followLinksToStore(), but apply toStorePath() to the
        result. */
-    StorePath followLinksToStorePath(const Path & path) const;
+    StorePath followLinksToStorePath(std::string_view path) const;
+
+    StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
 
     /* Constructs a unique store path name. */
     StorePath makeStorePath(const string & type,
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e67bdcdebb8..012f1d07139 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1186,7 +1186,7 @@ void _interrupted()
 //////////////////////////////////////////////////////////////////////
 
 
-template<class C> C tokenizeString(const string & s, const string & separators)
+template<class C> C tokenizeString(std::string_view s, const string & separators)
 {
     C result;
     string::size_type pos = s.find_first_not_of(separators, 0);
@@ -1200,9 +1200,9 @@ template<class C> C tokenizeString(const string & s, const string & separators)
     return result;
 }
 
-template Strings tokenizeString(const string & s, const string & separators);
-template StringSet tokenizeString(const string & s, const string & separators);
-template vector<string> tokenizeString(const string & s, const string & separators);
+template Strings tokenizeString(std::string_view s, const string & separators);
+template StringSet tokenizeString(std::string_view s, const string & separators);
+template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
 string concatStringsSep(const string & sep, const Strings & ss)
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index abd91b81c52..abf1c95d675 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -340,7 +340,7 @@ MakeError(FormatError, Error);
 
 
 /* String tokenizer. */
-template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");
+template<class C> C tokenizeString(std::string_view s, const string & separators = " \t\n\r");
 
 
 /* Concatenate the given strings with a separator between the
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e2e2939f4be..45e152c47ef 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
 
     std::vector<StorePathWithOutputs> paths;
     for (auto & i : opArgs)
-        paths.push_back(store->parseDrvPathWithOutputs(i));
+        paths.push_back(store->followLinksToStorePathWithOutputs(i));
 
     unsigned long long downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
@@ -893,7 +893,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 std::vector<StorePathWithOutputs> paths;
                 for (auto & s : readStrings<Strings>(in))
-                    paths.emplace_back(store->parseDrvPathWithOutputs(s));
+                    paths.emplace_back(store->parsePathWithOutputs(s));
 
                 getBuildSettings();
 
diff --git a/tests/nix-build.sh b/tests/nix-build.sh
index 39526486319..0eb5996087e 100644
--- a/tests/nix-build.sh
+++ b/tests/nix-build.sh
@@ -23,3 +23,6 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
 
 outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
 [[ $outPath = $outPath2 ]]
+
+outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out)
+[[ $outPath = $outPath2 ]]

From f8abbdd4565542464f31f4dc203a9c3e091b3536 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 17 Dec 2019 17:17:53 +0100
Subject: [PATCH 1769/2196] Add priority setting to stores

This allows overriding the priority of substituters, e.g.

  $ nix-store --store ~/my-nix/ -r /nix/store/df3m4da96d84ljzxx4mygfshm1p0r2n3-geeqie-1.4 \
    --substituters 'http://cache.nixos.org?priority=100 daemon?priority=10'

Fixes #3264.
---
 src/libstore/binary-cache-store.cc      |  4 ++--
 src/libstore/binary-cache-store.hh      |  9 ---------
 src/libstore/globals.cc                 |  2 +-
 src/libstore/http-binary-cache-store.cc |  7 +++++--
 src/libstore/local-store.cc             |  2 +-
 src/libstore/nar-info-disk-cache.cc     | 16 ++++++++--------
 src/libstore/nar-info-disk-cache.hh     |  9 +++++++--
 src/libstore/s3-binary-cache-store.cc   |  9 +++++----
 src/libstore/sqlite.hh                  |  4 ++--
 src/libstore/store-api.cc               |  2 +-
 src/libstore/store-api.hh               | 11 ++++-------
 src/libutil/config.cc                   | 15 ++++++++++-----
 src/libutil/config.hh                   |  8 +++++---
 13 files changed, 51 insertions(+), 47 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index aaacf828152..717faec9271 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -49,9 +49,9 @@ void BinaryCacheStore::init()
                     throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
                         % getUri() % value % storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery_ = value == "1";
+                wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
-                string2Int(value, priority);
+                priority.setDefault(fmt("%d", std::stoi(value)));
             }
         }
     }
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index fa2200ad8b6..aa13c1cb454 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -52,11 +52,6 @@ public:
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
-protected:
-
-    bool wantMassQuery_ = false;
-    int priority = 50;
-
 public:
 
     virtual void init();
@@ -79,8 +74,6 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    bool wantMassQuery() override { return wantMassQuery_; }
-
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
@@ -107,8 +100,6 @@ public:
 
     std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
 
-    int getPriority() override { return priority; }
-
 };
 
 MakeError(NoSuchBinaryCacheFile, Error);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 6fedf3d56c2..cec85edcab7 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -127,7 +127,7 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
     else throw UsageError("option '%s' has invalid value '%s'", name, str);
 }
 
-template<> std::string BaseSetting<SandboxMode>::to_string()
+template<> std::string BaseSetting<SandboxMode>::to_string() const
 {
     if (value == smEnabled) return "true";
     else if (value == smRelaxed) return "relaxed";
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 779f89e68d9..d4ae3666255 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -42,13 +42,16 @@ class HttpBinaryCacheStore : public BinaryCacheStore
     void init() override
     {
         // FIXME: do this lazily?
-        if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
+        if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
+            priority.setDefault(fmt("%d", cacheInfo->priority));
+        } else {
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
                 throw Error("'%s' does not appear to be a binary cache", cacheUri);
             }
-            diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
+            diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
         }
     }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c1380ef0977..b254d766a58 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -840,7 +840,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
     for (auto & sub : getDefaultSubstituters()) {
         if (remaining.empty()) break;
         if (sub->storeDir != storeDir) continue;
-        if (!sub->wantMassQuery()) continue;
+        if (!sub->wantMassQuery) continue;
 
         auto valid = sub->queryValidPaths(remaining);
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 3a2cde55f20..907645d8671 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -147,26 +147,26 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
         });
     }
 
-    bool cacheExists(const std::string & uri,
-        bool & wantMassQuery, int & priority) override
+    std::optional<CacheInfo> cacheExists(const std::string & uri) override
     {
-        return retrySQLite<bool>([&]() {
+        return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
             auto state(_state.lock());
 
             auto i = state->caches.find(uri);
             if (i == state->caches.end()) {
                 auto queryCache(state->queryCache.use()(uri));
-                if (!queryCache.next()) return false;
+                if (!queryCache.next())
+                    return std::nullopt;
                 state->caches.emplace(uri,
                     Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
             }
 
             auto & cache(getCache(*state, uri));
 
-            wantMassQuery = cache.wantMassQuery;
-            priority = cache.priority;
-
-            return true;
+            return CacheInfo {
+                .wantMassQuery = cache.wantMassQuery,
+                .priority = cache.priority
+            };
         });
     }
 
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index 285873f7ede..878acbb87b5 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -15,8 +15,13 @@ public:
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
 
-    virtual bool cacheExists(const std::string & uri,
-        bool & wantMassQuery, int & priority) = 0;
+    struct CacheInfo
+    {
+        bool wantMassQuery;
+        int priority;
+    };
+
+    virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
 
     virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
         const std::string & uri, const std::string & hashPart) = 0;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 58966a5b796..f2e4b63e059 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -205,11 +205,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     void init() override
     {
-        if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
-
+        if (auto cacheInfo = diskCache->cacheExists(getUri())) {
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
+            priority.setDefault(fmt("%d", cacheInfo->priority));
+        } else {
             BinaryCacheStore::init();
-
-            diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
+            diskCache->createCache(getUri(), storeDir, wantMassQuery, priority);
         }
     }
 
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index 115679b8415..bd012d9b963 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -99,8 +99,8 @@ void handleSQLiteBusy(const SQLiteBusy & e);
 
 /* Convenience function for retrying a SQLite transaction when the
    database is busy. */
-template<typename T>
-T retrySQLite(std::function<T()> fun)
+template<typename T, typename F>
+T retrySQLite(F && fun)
 {
     while (true) {
         try {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 1781e0c6839..d8f6c22bce3 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -975,7 +975,7 @@ std::list<ref<Store>> getDefaultSubstituters()
             addStore(uri);
 
         stores.sort([](ref<Store> & a, ref<Store> & b) {
-            return a->getPriority() < b->getPriority();
+            return a->priority < b->priority;
         });
 
         return stores;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d77d6bad830..743be94108d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -255,6 +255,10 @@ public:
 
     const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
 
+    Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
+
+    Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
+
 protected:
 
     struct State
@@ -422,8 +426,6 @@ public:
     virtual void querySubstitutablePathInfos(const StorePathSet & paths,
         SubstitutablePathInfos & infos) { return; };
 
-    virtual bool wantMassQuery() { return false; }
-
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
@@ -648,11 +650,6 @@ public:
         return 0;
     };
 
-    /* Get the priority of the store, used to order substituters. In
-       particular, binary caches can specify a priority field in their
-       "nix-cache-info" file. Lower value means higher priority. */
-    virtual int getPriority() { return 0; }
-
     virtual Path toRealPath(const Path & storePath)
     {
         return storePath;
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 9023cb1bb6d..7551d97d1a7 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -154,6 +154,11 @@ AbstractSetting::AbstractSetting(
 {
 }
 
+void AbstractSetting::setDefault(const std::string & str)
+{
+    if (!overriden) set(str);
+}
+
 void AbstractSetting::toJSON(JSONPlaceholder & out)
 {
     out.write(to_string());
@@ -185,7 +190,7 @@ template<> void BaseSetting<std::string>::set(const std::string & str)
     value = str;
 }
 
-template<> std::string BaseSetting<std::string>::to_string()
+template<> std::string BaseSetting<std::string>::to_string() const
 {
     return value;
 }
@@ -199,7 +204,7 @@ void BaseSetting<T>::set(const std::string & str)
 }
 
 template<typename T>
-std::string BaseSetting<T>::to_string()
+std::string BaseSetting<T>::to_string() const
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     return std::to_string(value);
@@ -215,7 +220,7 @@ template<> void BaseSetting<bool>::set(const std::string & str)
         throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
 }
 
-template<> std::string BaseSetting<bool>::to_string()
+template<> std::string BaseSetting<bool>::to_string() const
 {
     return value ? "true" : "false";
 }
@@ -239,7 +244,7 @@ template<> void BaseSetting<Strings>::set(const std::string & str)
     value = tokenizeString<Strings>(str);
 }
 
-template<> std::string BaseSetting<Strings>::to_string()
+template<> std::string BaseSetting<Strings>::to_string() const
 {
     return concatStringsSep(" ", value);
 }
@@ -256,7 +261,7 @@ template<> void BaseSetting<StringSet>::set(const std::string & str)
     value = tokenizeString<StringSet>(str);
 }
 
-template<> std::string BaseSetting<StringSet>::to_string()
+template<> std::string BaseSetting<StringSet>::to_string() const
 {
     return concatStringsSep(" ", value);
 }
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index d86c65ff033..7ea78fdaf5c 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -115,6 +115,8 @@ public:
 
     bool overriden = false;
 
+    void setDefault(const std::string & str);
+
 protected:
 
     AbstractSetting(
@@ -131,13 +133,13 @@ protected:
 
     virtual void set(const std::string & value) = 0;
 
-    virtual std::string to_string() = 0;
+    virtual std::string to_string() const = 0;
 
     virtual void toJSON(JSONPlaceholder & out);
 
     virtual void convertToArg(Args & args, const std::string & category);
 
-    bool isOverriden() { return overriden; }
+    bool isOverriden() const { return overriden; }
 };
 
 /* A setting of type T. */
@@ -174,7 +176,7 @@ public:
         value = v;
     }
 
-    std::string to_string() override;
+    std::string to_string() const override;
 
     void convertToArg(Args & args, const std::string & category) override;
 

From 4511f09b490fad4ce0dcfbcd7c4fd83b11e7df46 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 18 Dec 2019 17:39:02 +0100
Subject: [PATCH 1770/2196] nix make-content-addressable: Add --json flag

Fixes #3274.
---
 src/nix/make-content-addressable.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 524d467c2ae..f9c7fef3f01 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -1,10 +1,12 @@
 #include "command.hh"
 #include "store-api.hh"
 #include "references.hh"
+#include "common-args.hh"
+#include "json.hh"
 
 using namespace nix;
 
-struct CmdMakeContentAddressable : StorePathsCommand
+struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 {
     CmdMakeContentAddressable()
     {
@@ -37,6 +39,9 @@ struct CmdMakeContentAddressable : StorePathsCommand
 
         std::map<StorePath, StorePath> remappings;
 
+        auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
+        auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
+
         for (auto & path : paths) {
             auto pathS = store->printStorePath(path);
             auto oldInfo = store->queryPathInfo(path);
@@ -76,7 +81,8 @@ struct CmdMakeContentAddressable : StorePathsCommand
             info.narSize = sink.s->size();
             info.ca = makeFixedOutputCA(true, info.narHash);
 
-            printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
+            if (!json)
+                printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
 
             auto source = sinkToSource([&](Sink & nextSink) {
                 RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink);
@@ -86,6 +92,9 @@ struct CmdMakeContentAddressable : StorePathsCommand
 
             store->addToStore(info, *source);
 
+            if (json)
+                jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
+
             remappings.insert_or_assign(std::move(path), std::move(info.path));
         }
     }

From 9f7b4d068cc106e5d902dc6f52bf46d4a057fd00 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Dec 2019 13:17:48 +0100
Subject: [PATCH 1771/2196] Cleanup: Remove unused makeDeb/makeRPM functions

---
 release.nix | 61 -----------------------------------------------------
 1 file changed, 61 deletions(-)

diff --git a/release.nix b/release.nix
index 9cf4c74f231..9202c2ff0fd 100644
--- a/release.nix
+++ b/release.nix
@@ -255,16 +255,6 @@ let
       };
 
 
-    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
-
-
-    #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
-
-    #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
-    #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
-
-
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
       inherit nixpkgs;
@@ -381,55 +371,4 @@ let
   };
 
 
-  makeRPM_i686 = makeRPM "i686-linux";
-  makeRPM_x86_64 = makeRPM "x86_64-linux";
-
-  makeRPM =
-    system: diskImageFun: extraPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.rpmBuild rec {
-      name = "nix-rpm";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
-            ++ extraPackages; };
-      # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
-      #enableParallelBuilding = true;
-    };
-
-
-  makeDeb_i686 = makeDeb "i686-linux";
-  makeDeb_x86_64 = makeDeb "x86_64-linux";
-
-  makeDeb =
-    system: diskImageFun: extraPackages: extraDebPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.debBuild {
-      name = "nix-deb";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
-            ++ extraPackages; };
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postInstall = "make installcheck";
-      configureFlags = "--sysconfdir=/etc";
-      debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
-        ++ extraDebPackages;
-      debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
-      doInstallCheck = true;
-      #enableParallelBuilding = true;
-    };
-
-
 in jobs

From be32da0ed0233c6a8813b89eeececf9af49ad243 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Dec 2019 15:01:58 +0100
Subject: [PATCH 1772/2196] tarfile.cc: Style fixes

---
 src/libutil/tarfile.cc | 59 +++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 68e918891d9..c85a8b0cb6b 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -6,66 +6,76 @@
 namespace nix {
 
 struct TarArchive {
-    struct archive *archive;
-    Source *source;
+    struct archive * archive;
+    Source * source;
     std::vector<unsigned char> buffer;
 
-    void check(int err, const char *reason = "Failed to extract archive (%s)") {
+    void check(int err, const char * reason = "failed to extract archive: %s")
+    {
         if (err == ARCHIVE_EOF)
             throw EndOfFile("reached end of archive");
         else if (err != ARCHIVE_OK)
             throw Error(reason, archive_error_string(this->archive));
     }
 
-    TarArchive(Source& source) : buffer(4096) {
+    TarArchive(Source & source) : buffer(4096)
+    {
         this->archive = archive_read_new();
         this->source = &source;
 
         archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
-        check(archive_read_open(archive, (void *)this, TarArchive::callback_open, TarArchive::callback_read, TarArchive::callback_close), "Failed to open archive (%s)");
+        check(archive_read_open(archive,
+                (void *)this,
+                TarArchive::callback_open,
+                TarArchive::callback_read,
+                TarArchive::callback_close),
+            "failed to open archive: %s");
     }
 
-    TarArchive(const Path &path) {
+    TarArchive(const Path & path)
+    {
         this->archive = archive_read_new();
 
         archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
-        check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)");
+        check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
     }
 
-    // disable copy constructor
-    TarArchive(const TarArchive&) = delete;
+    TarArchive(const TarArchive &) = delete;
 
-    void close() {
-        check(archive_read_close(archive), "Failed to close archive (%s)");
+    void close()
+    {
+        check(archive_read_close(archive), "failed to close archive: %s");
     }
 
-    ~TarArchive() {
+    ~TarArchive()
+    {
         if (this->archive) archive_read_free(this->archive);
     }
 
 private:
-    static int callback_open(struct archive *, void *self) {
+
+    static int callback_open(struct archive *, void * self) {
         return ARCHIVE_OK;
     }
 
-    static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) {
-        TarArchive *self = (TarArchive *)_self;
+    static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+    {
+        auto self = (TarArchive *)_self;
         *buffer = self->buffer.data();
 
         try {
             return self->source->read(self->buffer.data(), 4096);
         } catch (EndOfFile &) {
             return 0;
-        } catch (std::exception &err) {
-            archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
-
+        } catch (std::exception & err) {
+            archive_set_error(archive, EIO, "source threw exception: %s", err.what());
             return -1;
         }
     }
 
-    static int callback_close(struct archive *, void *self) {
+    static int callback_close(struct archive *, void * self) {
         return ARCHIVE_OK;
     }
 };
@@ -84,26 +94,27 @@ struct PushD {
         int r = chdir(oldDir);
         free(oldDir);
         if (r != 0)
-            std::cerr << "warning: failed to change directory back after tar extraction";
+            warn("failed to change directory back after tar extraction");
         /* can't throw out of a destructor */
     }
 };
 
-static void extract_archive(TarArchive &archive, const Path & destDir) {
+static void extract_archive(TarArchive & archive, const Path & destDir)
+{
     // need to chdir back *after* archive closing
     PushD newDir(destDir);
-    struct archive_entry *entry;
     int flags = ARCHIVE_EXTRACT_FFLAGS
         | ARCHIVE_EXTRACT_PERM
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT
         | ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS;
 
-    for(;;) {
+    for (;;) {
+        struct archive_entry * entry;
         int r = archive_read_next_header(archive.archive, &entry);
         if (r == ARCHIVE_EOF) break;
         else if (r == ARCHIVE_WARN)
-            std::cerr << "warning: " << archive_error_string(archive.archive) << std::endl;
+            warn(archive_error_string(archive.archive));
         else
             archive.check(r);
 

From 2550c11373e83f4c6085049dc9efe53f8389b035 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Dec 2019 15:08:16 +0100
Subject: [PATCH 1773/2196] tarfile.cc: Don't change the cwd

Nix is multithreaded so it's not safe to change the cwd.
---
 src/libutil/tarfile.cc | 27 ++++-----------------------
 1 file changed, 4 insertions(+), 23 deletions(-)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index c85a8b0cb6b..ca19f402ee4 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -80,34 +80,12 @@ struct TarArchive {
     }
 };
 
-struct PushD {
-    char * oldDir;
-
-    PushD(const std::string &newDir)  {
-        oldDir = getcwd(0, 0);
-        if (!oldDir) throw SysError("getcwd");
-        int r = chdir(newDir.c_str());
-        if (r != 0) throw SysError("changing directory to tar output path");
-    }
-
-    ~PushD() {
-        int r = chdir(oldDir);
-        free(oldDir);
-        if (r != 0)
-            warn("failed to change directory back after tar extraction");
-        /* can't throw out of a destructor */
-    }
-};
-
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    // need to chdir back *after* archive closing
-    PushD newDir(destDir);
     int flags = ARCHIVE_EXTRACT_FFLAGS
         | ARCHIVE_EXTRACT_PERM
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
-        | ARCHIVE_EXTRACT_SECURE_NODOTDOT
-        | ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS;
+        | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
     for (;;) {
         struct archive_entry * entry;
@@ -118,6 +96,9 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         else
             archive.check(r);
 
+        archive_entry_set_pathname(entry,
+            (destDir + "/" + archive_entry_pathname(entry)).c_str());
+
         archive.check(archive_read_extract(archive.archive, entry, flags));
     }
 

From c84c843e3360459cf2c993919fb9f89474a46f38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Dec 2019 15:09:54 +0100
Subject: [PATCH 1774/2196] tarfile.cc: Restore timestamps

This is needed to get the lastModified attribute of GitHub flakes.
---
 src/libutil/tarfile.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index ca19f402ee4..c4d8a4f919f 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -84,6 +84,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
 {
     int flags = ARCHIVE_EXTRACT_FFLAGS
         | ARCHIVE_EXTRACT_PERM
+        | ARCHIVE_EXTRACT_TIME
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 

From d25923263e0eb70f37536e2fe5b59237f6fce5cc Mon Sep 17 00:00:00 2001
From: Brian McKenna <brian@brianmckenna.org>
Date: Sat, 21 Dec 2019 07:40:23 +1100
Subject: [PATCH 1775/2196] Disable use-sqlite-wal under WSL

Before:

    $ nix-channel --update
    unpacking channels...
    warning: SQLite database '/nix/var/nix/db/db.sqlite' is busy (SQLITE_PROTOCOL)
    warning: SQLite database '/nix/var/nix/db/db.sqlite' is busy (SQLITE_PROTOCOL)
    warning: SQLite database '/nix/var/nix/db/db.sqlite' is busy (SQLITE_PROTOCOL)
    warning: SQLite database '/nix/var/nix/db/db.sqlite' is busy (SQLITE_PROTOCOL)
    warning: SQLite database '/nix/var/nix/db/db.sqlite' is busy (SQLITE_PROTOCOL)

After:

    $ inst/bin/nix-channel --update
    unpacking channels...
    created 1 symlinks in user environment

I've seen complaints that "sandbox" caused problems under WSL but I'm
having no problems. I think recent changes could have fixed the issue.
---
 src/libstore/globals.cc | 9 +++++++++
 src/libstore/globals.hh | 4 +++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index cec85edcab7..5f1ae5ab580 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -117,6 +117,15 @@ void Settings::requireExperimentalFeature(const std::string & name)
         throw Error("experimental Nix feature '%s' is disabled", name);
 }
 
+bool Settings::isWSL1()
+{
+    struct utsname utsbuf;
+    uname(&utsbuf);
+    // WSL1 uses -Microsoft suffix
+    // WSL2 uses -microsoft-standard suffix
+    return hasSuffix(utsbuf.release, "-Microsoft");
+}
+
 const string nixVersion = PACKAGE_VERSION;
 
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 41e511c6c49..782870547cb 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -34,6 +34,8 @@ class Settings : public Config {
 
     StringSet getDefaultSystemFeatures();
 
+    bool isWSL1();
+
 public:
 
     Settings();
@@ -130,7 +132,7 @@ public:
     Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
         "Whether SQLite should use fsync()."};
 
-    Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal",
+    Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
         "Whether SQLite should use WAL mode."};
 
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",

From 10414d467b392f285007edeb9e5ca897057adddf Mon Sep 17 00:00:00 2001
From: Michael Forney <mforney@mforney.org>
Date: Sat, 21 Dec 2019 21:30:38 -0800
Subject: [PATCH 1776/2196] Pass -P to cp to preserve symlinks

This is commonly the default behavior with -R, but POSIX leaves the
default unspecified.
---
 scripts/install-nix-from-closure.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 3f15818547a..3fea7e05602 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -102,7 +102,7 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
         rm -rf "$i_tmp"
     fi
     if ! [ -e "$dest/store/$i" ]; then
-        cp -Rp "$self/store/$i" "$i_tmp"
+        cp -RPp "$self/store/$i" "$i_tmp"
         chmod -R a-w "$i_tmp"
         chmod +w "$i_tmp"
         mv "$i_tmp" "$dest/store/$i"

From 43eb7b6756a6441c8901342d657d299a6efdefbc Mon Sep 17 00:00:00 2001
From: Michael Forney <mforney@mforney.org>
Date: Sun, 22 Dec 2019 00:06:51 -0800
Subject: [PATCH 1777/2196] Pass -J to tar for xz decompression

Some tar implementations can't auto-detect compression formats, so
they must be specified explicitly.
---
 scripts/install.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install.in b/scripts/install.in
index 902758b138a..76cb3b1fdce 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -56,7 +56,7 @@ fi
 
 unpack=$tmpDir/unpack
 mkdir -p "$unpack"
-tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
+tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 
 script=$(echo "$unpack"/*/install)
 

From c502831a1d087a406e80dede8cf14e3e97531006 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 1 Jan 2020 20:50:40 -0500
Subject: [PATCH 1778/2196] exportReferencesGraph: support working

Before, we would get:

    [deploy@bastion:~]$ nix-store -r /nix/store/grfnl76cahwls0igd2by2pqv0dimi8h2-nixos-system-eris-19.09.20191213.03f3def.drv
    these derivations will be built:
      /nix/store/3ka4ihvwh6wsyhpd2qa9f59506mnxvx1-initrd-linux-4.19.88.drv
      /nix/store/ssxwmll7v21did1c8j027q0m8w6pg41i-unit-prometheus-alertmanager-irc-notifier.service.drv
      /nix/store/mvyvkj46ay7pp7b1znqbkck2mq98k0qd-unit-script-network-local-commands-start.drv
      /nix/store/vsl1y9mz38qfk6pyirjwnfzfggz5akg6-unit-network-local-commands.service.drv
      /nix/store/wi5ighfwwb83fdmav6z6n2fw6npm9ffl-unit-prometheus-hydra-exporter.service.drv
      /nix/store/x0qkv535n75pbl3xn6nn1w7qkrg9wwyg-unit-prometheus-packet-sd.service.drv
      /nix/store/lv491znsjxdf51xnfxh9ld7r1zg14d52-unit-script-packet-sd-env-key-pre-start.drv
      /nix/store/nw4nzlca49agsajvpibx7zg5b873gk9f-unit-script-packet-sd-env-key-start.drv
      /nix/store/x674wwabdwjrkhnykair4c8mpxa9532w-unit-packet-sd-env-key.service.drv
      /nix/store/ywivz64ilb1ywlv652pkixw3vxzfvgv8-unit-wireguard-wg0.service.drv
      /nix/store/v3b648293g3zl8pnn0m1345nvmyd8dwb-unit-script-acme-selfsigned-status.nixos.org-start.drv
      /nix/store/zci5d3zvr6fgdicz6k7jjka6lmx0v3g4-unit-acme-selfsigned-status.nixos.org.service.drv
      /nix/store/f6pwvnm63d0kw5df0v7sipd1rkhqxk5g-system-units.drv
      /nix/store/iax8071knxk9c7krpm9jqg0lcrawf4lc-etc.drv
      /nix/store/grfnl76cahwls0igd2by2pqv0dimi8h2-nixos-system-eris-19.09.20191213.03f3def.drv
    error: invalid file name 'closure-init-0' in 'exportReferencesGraph'

This was tough to debug, I didn't figure out which one was broken until I did:

    nix-store -r /nix/store/grfnl76cahwls0igd2by2pqv0dimi8h2-nixos-system-eris-19.09.20191213.03f3def.drv 2>&1 | grep  nix/store | xargs -n1 nix-store -r

and then looking at the remaining build graph:

    $ nix-store -r /nix/store/grfnl76cahwls0igd2by2pqv0dimi8h2-nixos-system-eris-19.09.20191213.03f3def.drv
    these derivations will be built:
      /nix/store/3ka4ihvwh6wsyhpd2qa9f59506mnxvx1-initrd-linux-4.19.88.drv
      /nix/store/grfnl76cahwls0igd2by2pqv0dimi8h2-nixos-system-eris-19.09.20191213.03f3def.drv
    error: invalid file name 'closure-init-0' in 'exportReferencesGraph'

and knowing the initrd build is before the system, then:

    $ nix show-derivation /nix/store/3ka4ihvwh6wsyhpd2qa9f59506mnxvx1-initrd-linux-4.19.88.drv
    {
      "/nix/store/3ka4ihvwh6wsyhpd2qa9f59506mnxvx1-initrd-linux-4.19.88.drv": {
        [...]
        "exportReferencesGraph": "closure-init-0 /nix/store/...-stage-1-init.sh closure-mdadm.conf-1 /nix/store/...-mdadm.conf closure-ubuntu.conf-2 ...",
        [...]
      }
    }

I then searched the repo for "in 'exportReferencesGraph'", found this
recently updated regex, and realized it was missing a "-".
---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index de0ce9ae163..5d681d279f6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1985,7 +1985,7 @@ void DerivationGoal::startBuilder()
             throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
         for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
             string fileName = *i++;
-            static std::regex regex("[A-Za-z_][A-Za-z0-9_.]*");
+            static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
             if (!std::regex_match(fileName, regex))
                 throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 

From 515c0a263e137a00e82f7d981284dbe54db23247 Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puck.moe>
Date: Thu, 2 Jan 2020 17:20:57 +0000
Subject: [PATCH 1779/2196] passAsFile: hash the attribute name instead of
 numbering sequentially

This makes the paths consistent without relying on ordering.

Co-authored-by: edef <edef@edef.eu>
---
 src/libstore/build.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5d681d279f6..9ee3b04e32b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2455,12 +2455,12 @@ void DerivationGoal::initTmpDir() {
     if (!parsedDrv->getStructuredAttrs()) {
 
         StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
-        int fileNr = 0;
         for (auto & i : drv->env) {
             if (passAsFile.find(i.first) == passAsFile.end()) {
                 env[i.first] = i.second;
             } else {
-                string fn = ".attr-" + std::to_string(fileNr++);
+                auto hash = hashString(htSHA256, i.first);
+                string fn = ".attr-" + hash.to_string();
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, i.second);
                 chownToBuilder(p);

From c65a6fa86aef7bdf51fb4fba7bd31d265619ba3f Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Thu, 2 Jan 2020 23:41:48 +0000
Subject: [PATCH 1780/2196] passAsFile: leave out the hash prefix

Having a colon in the path may cause issues, and having the hash
function indicated isn't actually necessary. We now verify the path
format in the tests to prevent regressions.
---
 src/libstore/build.cc | 2 +-
 tests/pass-as-file.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9ee3b04e32b..78f39fed16b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2460,7 +2460,7 @@ void DerivationGoal::initTmpDir() {
                 env[i.first] = i.second;
             } else {
                 auto hash = hashString(htSHA256, i.first);
-                string fn = ".attr-" + hash.to_string();
+                string fn = ".attr-" + hash.to_string(Base32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, i.second);
                 chownToBuilder(p);
diff --git a/tests/pass-as-file.sh b/tests/pass-as-file.sh
index 3dfe10baa23..2c0bc5031ad 100644
--- a/tests/pass-as-file.sh
+++ b/tests/pass-as-file.sh
@@ -10,6 +10,7 @@ mkDerivation {
   passAsFile = [ \"foo\" ];
   foo = [ \"xyzzy\" ];
   builder = builtins.toFile \"builder.sh\" ''
+    [ \"\$(basename \$fooPath)\" = .attr-1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89ic ]
     [ \"\$(cat \$fooPath)\" = xyzzy ]
     touch \$out
   '';

From b33fefcb92e02725fc77b6adfabab1c1f43e9aa1 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 4 Jan 2020 17:47:52 +0100
Subject: [PATCH 1781/2196] build: recover store path when replacing fails

This shouldn't happen in normal circumstances, but just in case
attempt to move the temporary path back if possible.
---
 src/libstore/build.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 78f39fed16b..257d70ca9b4 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1513,8 +1513,10 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
     Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
     if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
-    if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
+    if (rename(tmpPath.c_str(), storePath.c_str()) == -1) {
+        rename(oldPath.c_str(), storePath.c_str()); // attempt to recover
         throw SysError("moving '%s' to '%s'", tmpPath, storePath);
+    }
     deletePath(oldPath);
 }
 

From 7d448bc966021e63e9c7b96530cee885f3dcd68d Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 4 Jan 2020 20:23:59 +0100
Subject: [PATCH 1782/2196] build: fix path repairing when hash rewriting is
 required

Handle store path repairing on darwin when sandboxing is enabled. Unlike
on linux sandboxing on darwin still requires hash rewriting.
---
 src/libstore/build.cc | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 257d70ca9b4..0674f936bbc 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3567,19 +3567,6 @@ void DerivationGoal::registerOutputs()
         if (!missingPaths.count(i.second.path)) continue;
 
         Path actualPath = path;
-        if (useChroot) {
-            actualPath = chrootRootDir + path;
-            if (pathExists(actualPath)) {
-                /* Move output paths from the chroot to the Nix store. */
-                if (buildMode == bmRepair)
-                    replaceValidPath(path, actualPath);
-                else
-                    if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1)
-                        throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
-            }
-            if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
-        }
-
         if (needsHashRewrite()) {
             auto r = redirectedOutputs.find(i.second.path);
             if (r != redirectedOutputs.end()) {
@@ -3591,6 +3578,17 @@ void DerivationGoal::registerOutputs()
                 if (buildMode == bmCheck)
                     actualPath = redirected;
             }
+        } else if (useChroot) {
+            actualPath = chrootRootDir + path;
+            if (pathExists(actualPath)) {
+                /* Move output paths from the chroot to the Nix store. */
+                if (buildMode == bmRepair)
+                    replaceValidPath(path, actualPath);
+                else
+                    if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1)
+                        throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
+            }
+            if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
         }
 
         struct stat st;

From cb90e382b5b6e177ea902b3909fd1897643ae3cd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 5 Jan 2020 16:21:34 +0100
Subject: [PATCH 1783/2196] Hide FunctionCallTrace constructor/destructor

This prevents them from being inlined. On gcc 9, this reduces the
stack size needed for

  nix-instantiate '<nixpkgs>' -A texlive.combined.scheme-full --dry-run

from 12.9 MiB to 4.8 MiB.
---
 src/libexpr/eval.cc           |  4 +---
 src/libexpr/function-trace.cc | 17 +++++++++++++++++
 src/libexpr/function-trace.hh | 14 ++------------
 3 files changed, 20 insertions(+), 15 deletions(-)
 create mode 100644 src/libexpr/function-trace.cc

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 6539a346aed..1c787645dd6 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1106,9 +1106,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
 {
-    std::unique_ptr<FunctionCallTrace> trace;
-    if (evalSettings.traceFunctionCalls)
-        trace = std::make_unique<FunctionCallTrace>(pos);
+    auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr;
 
     forceValue(fun, pos);
 
diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc
new file mode 100644
index 00000000000..af1486f7897
--- /dev/null
+++ b/src/libexpr/function-trace.cc
@@ -0,0 +1,17 @@
+#include "function-trace.hh"
+
+namespace nix {
+
+FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
+}
+
+FunctionCallTrace::~FunctionCallTrace() {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
+}
+
+}
diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh
index 2c39b7430fa..472f2045ed6 100644
--- a/src/libexpr/function-trace.hh
+++ b/src/libexpr/function-trace.hh
@@ -9,17 +9,7 @@ namespace nix {
 struct FunctionCallTrace
 {
     const Pos & pos;
-
-    FunctionCallTrace(const Pos & pos) : pos(pos) {
-        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
-        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
-    }
-
-    ~FunctionCallTrace() {
-        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
-        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
-    }
+    FunctionCallTrace(const Pos & pos);
+    ~FunctionCallTrace();
 };
 }

From 66fccd5832d125e9162abc5ed351aa37708e9623 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 5 Jan 2020 00:41:18 +0100
Subject: [PATCH 1784/2196] build: fix sandboxing on darwin

Starting ba87b08f8529e4d9f8c58d8c625152058ceadb75 getEnv now returns an
std::optional which means these getEnv() != "" conditions no longer happen
if the variables are not defined.
---
 src/libstore/build.cc                    | 33 ++++++++++++++----------
 src/libstore/gc.cc                       |  2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc |  2 +-
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 78f39fed16b..c3d69f387ae 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3338,7 +3338,7 @@ void DerivationGoal::runChild()
             ;
         }
 #if __APPLE__
-        else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") {
+        else {
             /* This has to appear before import statements. */
             std::string sandboxProfile = "(version 1)\n";
 
@@ -3447,25 +3447,32 @@ void DerivationGoal::runChild()
             /* They don't like trailing slashes on subpath directives */
             if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
 
-            builder = "/usr/bin/sandbox-exec";
-            args.push_back("sandbox-exec");
-            args.push_back("-f");
-            args.push_back(sandboxFile);
-            args.push_back("-D");
-            args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-            args.push_back("-D");
-            args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
-            if (allowLocalNetworking) {
+            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
+                builder = "/usr/bin/sandbox-exec";
+                args.push_back("sandbox-exec");
+                args.push_back("-f");
+                args.push_back(sandboxFile);
+                args.push_back("-D");
+                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
                 args.push_back("-D");
-                args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+                args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
+                if (allowLocalNetworking) {
+                    args.push_back("-D");
+                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+                }
+                args.push_back(drv->builder);
+            } else {
+                printError("warning: running in sandboxing test mode, sandbox disabled");
+                builder = drv->builder.c_str();
+                args.push_back(std::string(baseNameOf(drv->builder)));
             }
-            args.push_back(drv->builder);
         }
-#endif
+#else
         else {
             builder = drv->builder.c_str();
             args.push_back(std::string(baseNameOf(drv->builder)));
         }
+#endif
 
         for (auto & i : drv->args)
             args.push_back(rewriteStrings(i, inputRewrites));
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ed81186af16..690febc5bfb 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -443,7 +443,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
     // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
     // See: https://github.com/NixOS/nix/issues/3011
     // Because of this we disable lsof when running the tests.
-    if (getEnv("_NIX_TEST_NO_LSOF") == "") {
+    if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
         try {
             std::regex lsofRegex(R"(^n(/.*)$)");
             auto lsofLines =
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 2ec0b07accc..cc089181186 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -53,7 +53,7 @@ static int _main(int argc, char * * argv)
     {
         HashType ht = htSHA256;
         std::vector<string> args;
-        bool printPath = getEnv("PRINT_PATH") != "";
+        bool printPath = getEnv("PRINT_PATH") == "1";
         bool fromExpr = false;
         string attrPath;
         bool unpack = false;

From bc22a7ee6a6d9367f3197cf31a5fdd477b155974 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Jan 2020 22:18:00 +0100
Subject: [PATCH 1785/2196] Fix use of uninitialized store path

Fixes 'building of '/nix/store/00000000000000000000000000000000-': ...'.
---
 src/libstore/build.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4c07865e9ba..e1171082b01 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1016,7 +1016,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut
     , buildMode(buildMode)
 {
     state = &DerivationGoal::getDerivation;
-    name = fmt("building of '%s'", worker.store.printStorePath(drvPath));
+    name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath));
     trace("created");
 
     mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1041,7 +1041,7 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv
 
     /* Prevent the .chroot directory from being
        garbage-collected. (See isActiveTempFile() in gc.cc.) */
-    worker.store.addTempRoot(drvPath);
+    worker.store.addTempRoot(this->drvPath);
 }
 
 
@@ -4259,7 +4259,7 @@ SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, Repa
     , repair(repair)
 {
     state = &SubstitutionGoal::init;
-    name = fmt("substitution of '%s'", worker.store.printStorePath(storePath));
+    name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
     trace("created");
     maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
 }

From cb2d348d48cfd66fa34edd5df8c255cf74170a84 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@dfinity.org>
Date: Mon, 6 Jan 2020 16:30:56 +0100
Subject: [PATCH 1786/2196] Remove redundant check in parseJSONString

---
 src/libexpr/json-to-value.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 8bae986f97f..96cd0fc72da 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -22,7 +22,6 @@ static string parseJSONString(const char * & s)
             if (*s == '"') res += '"';
             else if (*s == '\\') res += '\\';
             else if (*s == '/') res += '/';
-            else if (*s == '/') res += '/';
             else if (*s == 'b') res += '\b';
             else if (*s == 'f') res += '\f';
             else if (*s == 'n') res += '\n';

From 52a8f9295b828872586c5b9e5587064a25dae9b2 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@dfinity.org>
Date: Tue, 7 Jan 2020 00:06:49 +0100
Subject: [PATCH 1787/2196] Add support for \u escape in fromJSON

As fromTOML supports \u and \U escapes, bring fromJSON on par. As JSON defaults
to UTF-8 encoding (every JSON parser must support UTF-8), this change parses the
`\u hex hex hex hex` sequence (\u followed by 4 hexadecimal digits) into an
UTF-8 representation.

Add a test to verify correct parsing, using all escape sequences from json.org.
---
 src/libexpr/json-to-value.cc              | 88 ++++++++++++++++++++++-
 tests/lang/eval-okay-fromjson-escapes.exp |  1 +
 tests/lang/eval-okay-fromjson-escapes.nix |  3 +
 3 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 tests/lang/eval-okay-fromjson-escapes.exp
 create mode 100644 tests/lang/eval-okay-fromjson-escapes.nix

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 96cd0fc72da..47cab2bb593 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -11,6 +11,87 @@ static void skipWhitespace(const char * & s)
 }
 
 
+/*
+  Parse an unicode escape sequence (4 hex characters following \u) in JSON string
+*/
+static string parseUnicodeEscapeSequence(const char * & s)
+{
+    int codepoint = 0;
+
+    const auto factors = { 12u, 8u, 4u, 0u };
+    for (const auto factor : factors)
+    {
+        if (!*s) throw JSONParseError("got end-of-string in JSON string while parsing \\u sequence");
+
+        if (*s >= '0' and *s <= '9') {
+            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x30u) << factor);
+        } else if (*s >= 'A' and *s <= 'F') {
+            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x37u) << factor);
+        } else if (*s >= 'a' and *s <= 'f') {
+            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x57u) << factor);
+        } else {
+            throw JSONParseError(format("illegal character '%1%' in \\u escape sequence.") % *s);
+        }
+        s++;
+    }
+
+    if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) {
+        throw JSONParseError("Unicode escape sequence is not a Unicode scalar value");
+    }
+
+    // taken from cpptoml.h
+    std::string result;
+    // See Table 3-6 of the Unicode standard
+    if (codepoint <= 0x7f)
+    {
+        // 1-byte codepoints: 00000000 0xxxxxxx
+        // repr: 0xxxxxxx
+        result += static_cast<char>(codepoint & 0x7f);
+    }
+    else if (codepoint <= 0x7ff)
+    {
+        // 2-byte codepoints: 00000yyy yyxxxxxx
+        // repr: 110yyyyy 10xxxxxx
+        //
+        // 0x1f = 00011111
+        // 0xc0 = 11000000
+        //
+        result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f));
+        //
+        // 0x80 = 10000000
+        // 0x3f = 00111111
+        //
+        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+    }
+    else if (codepoint <= 0xffff)
+    {
+        // 3-byte codepoints: zzzzyyyy yyxxxxxx
+        // repr: 1110zzzz 10yyyyyy 10xxxxxx
+        //
+        // 0xe0 = 11100000
+        // 0x0f = 00001111
+        //
+        result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f));
+        result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f));
+        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+    }
+    else
+    {
+        // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx
+        // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx
+        //
+        // 0xf0 = 11110000
+        // 0x07 = 00000111
+        //
+        result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07));
+        result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f));
+        result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f));
+        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+    }
+    return result;
+}
+
+
 static string parseJSONString(const char * & s)
 {
     string res;
@@ -27,8 +108,11 @@ static string parseJSONString(const char * & s)
             else if (*s == 'n') res += '\n';
             else if (*s == 'r') res += '\r';
             else if (*s == 't') res += '\t';
-            else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
-            else throw JSONParseError("invalid escaped character in JSON string");
+            else if (*s == 'u') {
+                res += parseUnicodeEscapeSequence(++s);
+                // to neuter the outside s++
+                s--;
+            } else throw JSONParseError("invalid escaped character in JSON string");
             s++;
         } else
             res += *s++;
diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/lang/eval-okay-fromjson-escapes.exp
new file mode 100644
index 00000000000..add5505a828
--- /dev/null
+++ b/tests/lang/eval-okay-fromjson-escapes.exp
@@ -0,0 +1 @@
+"quote \" reverse solidus \\ solidus / backspace  formfeed  newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →"
diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/lang/eval-okay-fromjson-escapes.nix
new file mode 100644
index 00000000000..f0071350773
--- /dev/null
+++ b/tests/lang/eval-okay-fromjson-escapes.nix
@@ -0,0 +1,3 @@
+# This string contains all supported escapes in a JSON string, per json.org
+# \b and \f are not supported by Nix
+builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"''

From f1fac0b5c3b75efab781949fdff2b67ffdda2cb3 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Thu, 9 Jan 2020 17:38:27 +0100
Subject: [PATCH 1788/2196] builtins.fromJSON: use nlohmann/json parser instead
 of custom parser

---
 src/libexpr/json-to-value.cc | 315 ++++++++++++++---------------------
 1 file changed, 124 insertions(+), 191 deletions(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 47cab2bb593..19d9a9b9008 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -1,232 +1,165 @@
 #include "json-to-value.hh"
 
-#include <cstring>
-
-namespace nix {
+#include <variant>
+#include <nlohmann/json.hpp>
+#include <nlohmann/detail/exceptions.hpp>
 
+using json = nlohmann::json;
 
-static void skipWhitespace(const char * & s)
-{
-    while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
-}
+namespace nix {
 
+// for more information, refer to
+// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
+class JSONSax : nlohmann::json_sax<json> {
+    class JSONState {
+    protected:
+        JSONState* parent;
+        Value * v;
+    public:
+        virtual JSONState* resolve(EvalState &)
+        {
+            throw std::logic_error("tried to close toplevel json parser state");
+        };
+        explicit JSONState(JSONState* p) : parent(p), v(nullptr) {};
+        explicit JSONState(Value* v) : v(v) {};
+        JSONState(JSONState& p) = delete;
+        Value& value(EvalState & state)
+        {
+            if (v == nullptr)
+                v = state.allocValue();
+            return *v;
+        };
+        virtual ~JSONState() {};
+        virtual void add() {};
+    };
+
+    class JSONObjectState : public JSONState {
+        using JSONState::JSONState;
+        ValueMap attrs = ValueMap();
+        virtual JSONState* resolve(EvalState & state) override
+        {
+            Value& v = parent->value(state);
+            state.mkAttrs(v, attrs.size());
+            for (auto & i : attrs)
+                v.attrs->push_back(Attr(i.first, i.second));
+            return parent;
+        }
+        virtual void add() override { v = nullptr; };
+    public:
+        void key(string_t& name, EvalState & state)
+        {
+            attrs[state.symbols.create(name)] = &value(state);
+        }
+    };
+
+    class JSONListState : public JSONState {
+        ValueVector values = ValueVector();
+        virtual JSONState* resolve(EvalState & state) override
+        {
+            Value& v = parent->value(state);
+            state.mkList(v, values.size());
+            for (size_t n = 0; n < values.size(); ++n) {
+                v.listElems()[n] = values[n];
+            }
+            return parent;
+        }
+        virtual void add() override {
+            values.push_back(v);
+            v = nullptr;
+        };
+    public:
+        JSONListState(JSONState* p, std::size_t reserve) : JSONState(p)
+        {
+            values.reserve(reserve);
+        }
+    };
 
-/*
-  Parse an unicode escape sequence (4 hex characters following \u) in JSON string
-*/
-static string parseUnicodeEscapeSequence(const char * & s)
-{
-    int codepoint = 0;
+    EvalState & state;
+    JSONState* rs;
 
-    const auto factors = { 12u, 8u, 4u, 0u };
-    for (const auto factor : factors)
+    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
-        if (!*s) throw JSONParseError("got end-of-string in JSON string while parsing \\u sequence");
-
-        if (*s >= '0' and *s <= '9') {
-            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x30u) << factor);
-        } else if (*s >= 'A' and *s <= 'F') {
-            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x37u) << factor);
-        } else if (*s >= 'a' and *s <= 'f') {
-            codepoint += static_cast<int>((static_cast<unsigned int>(*s) - 0x57u) << factor);
-        } else {
-            throw JSONParseError(format("illegal character '%1%' in \\u escape sequence.") % *s);
-        }
-        s++;
+        f(rs->value(state), args...);
+        rs->add();
+        return true;
     }
 
-    if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) {
-        throw JSONParseError("Unicode escape sequence is not a Unicode scalar value");
-    }
+public:
+    JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
+    ~JSONSax() { delete rs; };
 
-    // taken from cpptoml.h
-    std::string result;
-    // See Table 3-6 of the Unicode standard
-    if (codepoint <= 0x7f)
+    bool null()
     {
-        // 1-byte codepoints: 00000000 0xxxxxxx
-        // repr: 0xxxxxxx
-        result += static_cast<char>(codepoint & 0x7f);
+        return handle_value(mkNull);
     }
-    else if (codepoint <= 0x7ff)
+
+    bool boolean(bool val)
     {
-        // 2-byte codepoints: 00000yyy yyxxxxxx
-        // repr: 110yyyyy 10xxxxxx
-        //
-        // 0x1f = 00011111
-        // 0xc0 = 11000000
-        //
-        result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f));
-        //
-        // 0x80 = 10000000
-        // 0x3f = 00111111
-        //
-        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        return handle_value(mkBool, val);
     }
-    else if (codepoint <= 0xffff)
+
+    bool number_integer(number_integer_t val)
     {
-        // 3-byte codepoints: zzzzyyyy yyxxxxxx
-        // repr: 1110zzzz 10yyyyyy 10xxxxxx
-        //
-        // 0xe0 = 11100000
-        // 0x0f = 00001111
-        //
-        result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f));
-        result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f));
-        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        return handle_value(mkInt, val);
     }
-    else
+
+    bool number_unsigned(number_unsigned_t val)
     {
-        // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx
-        // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx
-        //
-        // 0xf0 = 11110000
-        // 0x07 = 00000111
-        //
-        result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07));
-        result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f));
-        result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f));
-        result += static_cast<char>(0x80 | (codepoint & 0x3f));
+        return handle_value(mkInt, val);
     }
-    return result;
-}
 
-
-static string parseJSONString(const char * & s)
-{
-    string res;
-    if (*s++ != '"') throw JSONParseError("expected JSON string");
-    while (*s != '"') {
-        if (!*s) throw JSONParseError("got end-of-string in JSON string");
-        if (*s == '\\') {
-            s++;
-            if (*s == '"') res += '"';
-            else if (*s == '\\') res += '\\';
-            else if (*s == '/') res += '/';
-            else if (*s == 'b') res += '\b';
-            else if (*s == 'f') res += '\f';
-            else if (*s == 'n') res += '\n';
-            else if (*s == 'r') res += '\r';
-            else if (*s == 't') res += '\t';
-            else if (*s == 'u') {
-                res += parseUnicodeEscapeSequence(++s);
-                // to neuter the outside s++
-                s--;
-            } else throw JSONParseError("invalid escaped character in JSON string");
-            s++;
-        } else
-            res += *s++;
+    bool number_float(number_float_t val, const string_t& s)
+    {
+        return handle_value(mkFloat, val);
     }
-    s++;
-    return res;
-}
-
 
-static void parseJSON(EvalState & state, const char * & s, Value & v)
-{
-    skipWhitespace(s);
-
-    if (!*s) throw JSONParseError("expected JSON value");
-
-    if (*s == '[') {
-        s++;
-        ValueVector values;
-        values.reserve(128);
-        skipWhitespace(s);
-        while (1) {
-            if (values.empty() && *s == ']') break;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            values.push_back(v2);
-            skipWhitespace(s);
-            if (*s == ']') break;
-            if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
-            s++;
-        }
-        s++;
-        state.mkList(v, values.size());
-        for (size_t n = 0; n < values.size(); ++n)
-            v.listElems()[n] = values[n];
+    bool string(string_t& val)
+    {
+        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
 
-    else if (*s == '{') {
-        s++;
-        ValueMap attrs;
-        while (1) {
-            skipWhitespace(s);
-            if (attrs.empty() && *s == '}') break;
-            string name = parseJSONString(s);
-            skipWhitespace(s);
-            if (*s != ':') throw JSONParseError("expected ':' in JSON object");
-            s++;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            attrs[state.symbols.create(name)] = v2;
-            skipWhitespace(s);
-            if (*s == '}') break;
-            if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
-            s++;
-        }
-        state.mkAttrs(v, attrs.size());
-        for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second));
-        v.attrs->sort();
-        s++;
+    bool start_object(std::size_t len)
+    {
+        JSONState* old = rs;
+        rs = new JSONObjectState(old);
+        return true;
     }
 
-    else if (*s == '"') {
-        mkString(v, parseJSONString(s));
+    bool key(string_t& name)
+    {
+        dynamic_cast<JSONObjectState*>(rs)->key(name, state);
+        return true;
     }
 
-    else if (isdigit(*s) || *s == '-' || *s == '.' ) {
-        // Buffer into a string first, then use built-in C++ conversions
-        std::string tmp_number;
-        ValueType number_type = tInt;
-
-        while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
-            if (*s == '.' || *s == 'e' || *s == 'E')
-                number_type = tFloat;
-            tmp_number += *s++;
-        }
-
-        try {
-            if (number_type == tFloat)
-                mkFloat(v, stod(tmp_number));
-            else
-                mkInt(v, stol(tmp_number));
-        } catch (std::invalid_argument & e) {
-            throw JSONParseError("invalid JSON number");
-        } catch (std::out_of_range & e) {
-            throw JSONParseError("out-of-range JSON number");
-        }
+    bool end_object() {
+        JSONState* old = rs;
+        rs = old->resolve(state);
+        delete old;
+        rs->add();
+        return true;
     }
 
-    else if (strncmp(s, "true", 4) == 0) {
-        s += 4;
-        mkBool(v, true);
+    bool end_array() {
+        return end_object();
     }
 
-    else if (strncmp(s, "false", 5) == 0) {
-        s += 5;
-        mkBool(v, false);
+    bool start_array(size_t len) {
+        JSONState* old = rs;
+        rs = new JSONListState(old, len != std::numeric_limits<size_t>::max() ? len : 128);
+        return true;
     }
 
-    else if (strncmp(s, "null", 4) == 0) {
-        s += 4;
-        mkNull(v);
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+        throw JSONParseError(ex.what());
     }
-
-    else throw JSONParseError("unrecognised JSON value");
-}
-
+};
 
 void parseJSON(EvalState & state, const string & s_, Value & v)
 {
-    const char * s = s_.c_str();
-    parseJSON(state, s, v);
-    skipWhitespace(s);
-    if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
+    JSONSax parser(state, v);
+    bool res = json::sax_parse(s_, &parser);
+    if (!res)
+        throw JSONParseError("Invalid JSON Value");
 }
 
-
 }

From a350d0beb0e13d0f58698510bd6a96d894cd06fd Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Thu, 9 Jan 2020 22:46:41 +0100
Subject: [PATCH 1789/2196] json-to-value: use unique_ptr instead of raw
 pointers

---
 src/libexpr/json-to-value.cc | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 19d9a9b9008..67feb9e1f4e 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -5,6 +5,7 @@
 #include <nlohmann/detail/exceptions.hpp>
 
 using json = nlohmann::json;
+using std::unique_ptr;
 
 namespace nix {
 
@@ -13,14 +14,14 @@ namespace nix {
 class JSONSax : nlohmann::json_sax<json> {
     class JSONState {
     protected:
-        JSONState* parent;
+        unique_ptr<JSONState> parent;
         Value * v;
     public:
-        virtual JSONState* resolve(EvalState &)
+        virtual unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
         };
-        explicit JSONState(JSONState* p) : parent(p), v(nullptr) {};
+        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
         explicit JSONState(Value* v) : v(v) {};
         JSONState(JSONState& p) = delete;
         Value& value(EvalState & state)
@@ -36,13 +37,13 @@ class JSONSax : nlohmann::json_sax<json> {
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
         ValueMap attrs = ValueMap();
-        virtual JSONState* resolve(EvalState & state) override
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value& v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
-            return parent;
+            return std::move(parent);
         }
         virtual void add() override { v = nullptr; };
     public:
@@ -54,28 +55,28 @@ class JSONSax : nlohmann::json_sax<json> {
 
     class JSONListState : public JSONState {
         ValueVector values = ValueVector();
-        virtual JSONState* resolve(EvalState & state) override
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
             Value& v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
             }
-            return parent;
+            return std::move(parent);
         }
         virtual void add() override {
             values.push_back(v);
             v = nullptr;
         };
     public:
-        JSONListState(JSONState* p, std::size_t reserve) : JSONState(p)
+        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    JSONState* rs;
+    unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -86,7 +87,6 @@ class JSONSax : nlohmann::json_sax<json> {
 
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
-    ~JSONSax() { delete rs; };
 
     bool null()
     {
@@ -120,21 +120,18 @@ class JSONSax : nlohmann::json_sax<json> {
 
     bool start_object(std::size_t len)
     {
-        JSONState* old = rs;
-        rs = new JSONObjectState(old);
+        rs = std::make_unique<JSONObjectState>(std::move(rs));
         return true;
     }
 
     bool key(string_t& name)
     {
-        dynamic_cast<JSONObjectState*>(rs)->key(name, state);
+        dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;
     }
 
     bool end_object() {
-        JSONState* old = rs;
-        rs = old->resolve(state);
-        delete old;
+        rs = rs->resolve(state);
         rs->add();
         return true;
     }
@@ -144,8 +141,8 @@ class JSONSax : nlohmann::json_sax<json> {
     }
 
     bool start_array(size_t len) {
-        JSONState* old = rs;
-        rs = new JSONListState(old, len != std::numeric_limits<size_t>::max() ? len : 128);
+        rs = std::make_unique<JSONListState>(std::move(rs),
+            len != std::numeric_limits<size_t>::max() ? len : 128);
         return true;
     }
 

From 7ccfa7ca4f85d71fdd9e0fa1cb537989ec6095f2 Mon Sep 17 00:00:00 2001
From: Joe Pea <joe@trusktr.io>
Date: Thu, 9 Jan 2020 14:15:19 -0800
Subject: [PATCH 1790/2196] Add a link to official channels in the Channels
 chapter.

---
 doc/manual/packages/channels.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/manual/packages/channels.xml b/doc/manual/packages/channels.xml
index 15c119fcb1f..1ed2bba524e 100644
--- a/doc/manual/packages/channels.xml
+++ b/doc/manual/packages/channels.xml
@@ -17,6 +17,9 @@ a set of Nix expressions and a manifest.  Using the command <link
 linkend="sec-nix-channel"><command>nix-channel</command></link> you
 can automatically stay up to date with whatever is available at that
 URL.</para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,

From 3895e7879456218acb82774670273305a6b3f08c Mon Sep 17 00:00:00 2001
From: Joe Pea <joe@trusktr.io>
Date: Thu, 9 Jan 2020 14:20:08 -0800
Subject: [PATCH 1791/2196] Add link to official channels in nix-channel
 command ref

---
 doc/manual/command-ref/nix-channel.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml
index 9fea591672c..ebcf56aff89 100644
--- a/doc/manual/command-ref/nix-channel.xml
+++ b/doc/manual/command-ref/nix-channel.xml
@@ -36,6 +36,9 @@ stay up-to-date with a set of pre-built Nix expressions.  A Nix
 channel is just a URL that points to a place containing a set of Nix
 expressions.  <phrase condition="manual">See also <xref
 linkend="sec-channels" />.</phrase></para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>This command has the following operations:
 

From 307bcb9a8e7a16bfc451e055a620b766df9d3f7d Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 11 Jan 2020 15:06:57 +0100
Subject: [PATCH 1792/2196] libexpr: show expression in assertion errors

Includes the expression of the condition in the assertion message if
the assertion failed, making assertions much easier to debug. eg.

    error: assertion (withPython -> (python2Packages != null)) failed at pkgs/tools/security/nmap/default.nix:11:1
---
 src/libexpr/eval.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1c787645dd6..3a3810f243b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -531,9 +531,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun
     throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError(format(s) % pos);
+    throw AssertionError(format(s) % s1 % pos);
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
@@ -1262,8 +1262,11 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos))
-        throwAssertionError("assertion failed at %1%", pos);
+    if (!state.evalBool(env, cond, pos)) {
+        std::ostringstream out;
+        cond->show(out);
+        throwAssertionError("assertion %1% failed at %2%", out.str(), pos);
+    }
     body->eval(state, env, v);
 }
 

From 30c9ca3b055a793339a850fc787fd5974e1405a8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Jan 2020 21:11:56 +0100
Subject: [PATCH 1793/2196] Fix Nixpkgs dependency

---
 release.nix | 2 +-
 shell.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 9202c2ff0fd..287b5294a6c 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
diff --git a/shell.nix b/shell.nix
index c709f6a5fab..e5a2b2c9186 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/bb1013511e1e5edcf314df8321acf2f3c536df0d.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From 835e541144272853f1c6e5e6f342d19926ba4e6f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Jan 2020 21:34:54 +0100
Subject: [PATCH 1794/2196] Fix build

https://hydra.nixos.org/eval/1564374
---
 src/libexpr/json-to-value.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 67feb9e1f4e..1fdce198399 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -2,7 +2,6 @@
 
 #include <variant>
 #include <nlohmann/json.hpp>
-#include <nlohmann/detail/exceptions.hpp>
 
 using json = nlohmann::json;
 using std::unique_ptr;

From c86c71c2b1feac19f3363662cf20abc88ab47e03 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Jan 2020 21:44:35 +0100
Subject: [PATCH 1795/2196] Test PRECOMPILE_HEADERS=0

---
 release.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/release.nix b/release.nix
index 287b5294a6c..db2b5ef05b2 100644
--- a/release.nix
+++ b/release.nix
@@ -240,6 +240,8 @@ let
         name = "nix-build";
         src = tarball;
 
+        enableParallelBuilding = true;
+
         buildInputs = buildDeps;
 
         dontInstall = false;
@@ -252,6 +254,9 @@ let
         # syntax-check generated dot files, it still requires some
         # fonts.  So provide those.
         FONTCONFIG_FILE = texFunctions.fontsConf;
+
+        # To test building without precompiled headers.
+        makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
       };
 
 

From d64ab5131c376a0e96075ec133dfd4185505594f Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sun, 12 Jan 2020 18:12:34 -0500
Subject: [PATCH 1796/2196] unbreak build without pch

---
 src/libstore/globals.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 5f1ae5ab580..a8945996e7a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -7,6 +7,7 @@
 #include <map>
 #include <thread>
 #include <dlfcn.h>
+#include <sys/utsname.h>
 
 
 namespace nix {

From c3181e21e75d9e31254cd97c8e56858be7072d5f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 13 Jan 2020 21:52:03 +0100
Subject: [PATCH 1797/2196] Tweak error message

---
 src/libexpr/eval.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 3a3810f243b..dac32b6f563 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1265,7 +1265,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError("assertion %1% failed at %2%", out.str(), pos);
+        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
     }
     body->eval(state, env, v);
 }

From 8b3217f8321ebd367c96d254df79e827d1076b3d Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Mon, 13 Jan 2020 22:09:18 +0100
Subject: [PATCH 1798/2196] build: remove warning when in sandboxing test mode

Introduced in 66fccd5832d125e9162abc5ed351aa37708e9623, but somehow
breaks the secure-drv-outputs test.
---
 src/libstore/build.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e1171082b01..74a68e1f5ba 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3464,7 +3464,6 @@ void DerivationGoal::runChild()
                 }
                 args.push_back(drv->builder);
             } else {
-                printError("warning: running in sandboxing test mode, sandbox disabled");
                 builder = drv->builder.c_str();
                 args.push_back(std::string(baseNameOf(drv->builder)));
             }

From 5d24e18e29ea1fff8fa316701fd95be6941da770 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Tue, 14 Jan 2020 00:14:03 -0500
Subject: [PATCH 1799/2196] doc: touchup release notes for 2.3

- At the top of the release notes, we announce sandboxing is now enabled by default,
then at the bottom it says it's now disabled when missing kernel support. These
can be merged into one point for clarity.

- The point about `max-jobs` defaulting to 1 appears unrelated to sandboxing.
---
 doc/manual/release-notes/rl-2.3.xml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml
index 80fbd63966a..0ad7d641f87 100644
--- a/doc/manual/release-notes/rl-2.3.xml
+++ b/doc/manual/release-notes/rl-2.3.xml
@@ -33,9 +33,13 @@ incompatible changes:</para>
   </listitem>
 
   <listitem>
-    <para>The installer now enables sandboxing by default on
-    Linux. The <literal>max-jobs</literal> setting now defaults to
-    1.</para>
+    <para>The installer now enables sandboxing by default on Linux when the
+    system has the necessary kernel support.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>max-jobs</literal> setting now defaults to 1.</para>
   </listitem>
 
   <listitem>
@@ -82,11 +86,6 @@ incompatible changes:</para>
     the duration of Nix function calls to stderr.</para>
   </listitem>
 
-  <listitem>
-    <para>On Linux, sandboxing is now disabled by default on systems
-    that don’t have the necessary kernel support.</para>
-  </listitem>
-
 </itemizedlist>
 
 </section>

From aef635da78d33bf679f49fd10e7130d918a82549 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 21 Jan 2020 21:14:13 +0100
Subject: [PATCH 1800/2196] Fix derivation computation with __structuredAttrs
 and multiple outputs

Fixes

  error: derivation '/nix/store/klivma7r7h5lndb99f7xxmlh5whyayvg-zlib-1.2.11.drv' has incorrect output '/nix/store/fv98nnx5ykgbq8sqabilkgkbc4169q05-zlib-1.2.11-dev', should be '/nix/store/adm7pilzlj3z5k249s8b4wv3scprhzi1-zlib-1.2.11-dev'
---
 src/libexpr/primops.cc      |  2 ++
 src/libstore/derivations.cc | 32 ++++++++++----------------------
 src/libstore/nar-info.cc    |  2 +-
 src/libstore/path.cc        |  5 ++++-
 src/libstore/path.hh        |  2 ++
 tests/structured-attrs.nix  |  6 +++++-
 tests/structured-attrs.sh   |  5 +++--
 7 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 3ef827ebcc6..29302c9b6bb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -731,6 +731,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            the hash. */
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(StorePath::dummy.clone(), "", ""));
         }
 
         Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1a061149a3b..d9da8769c56 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -246,30 +246,18 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     s.reserve(65536);
     s += "Derive([";
 
-    StringSet maskedOutputs;
-
-    if (maskOutputs) {
-        bool first = true;
-        maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " ");
-        for (auto & i : maskedOutputs) {
-            if (first) first = false; else s += ',';
-            s += '('; printString(s, i);
-            s += ",\"\",\"\",\"\")";
-        }
-    } else {
-        bool first = true;
-        for (auto & i : outputs) {
-            if (first) first = false; else s += ',';
-            s += '('; printString(s, i.first);
-            s += ','; printString(s, store.printStorePath(i.second.path));
-            s += ','; printString(s, i.second.hashAlgo);
-            s += ','; printString(s, i.second.hash);
-            s += ')';
-        }
+    bool first = true;
+    for (auto & i : outputs) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i.first);
+        s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
+        s += ','; printString(s, i.second.hashAlgo);
+        s += ','; printString(s, i.second.hash);
+        s += ')';
     }
 
     s += "],[";
-    bool first = true;
+    first = true;
     if (actualInputs) {
         for (auto & i : *actualInputs) {
             if (first) first = false; else s += ',';
@@ -299,7 +287,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     for (auto & i : env) {
         if (first) first = false; else s += ',';
         s += '('; printString(s, i.first);
-        s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second);
+        s += ','; printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second);
         s += ')';
     }
 
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index fb02cf3fd5a..1375094b5bf 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -4,7 +4,7 @@
 namespace nix {
 
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
-    : ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack
+    : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack
 {
     auto corrupt = [&]() {
         throw Error(format("NAR info file '%1%' is corrupt") % whence);
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index cda5f996818..a33bec3edaf 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -46,12 +46,15 @@ std::string_view StorePath::name() const
     return ffi_StorePath_name(*this);
 }
 
+StorePath StorePath::dummy(
+    StorePath::make(
+        (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x"));
+
 StorePath Store::parseStorePath(std::string_view path) const
 {
     return StorePath::make(path, storeDir);
 }
 
-
 StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 5ebb574804e..6b29c3566cb 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -53,6 +53,8 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
     {
         return ffi_StorePath_hash_data(*this);
     }
+
+    static StorePath dummy;
 };
 
 typedef std::set<StorePath> StorePathSet;
diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix
index 6c77a43913a..c39c3a346c5 100644
--- a/tests/structured-attrs.nix
+++ b/tests/structured-attrs.nix
@@ -16,6 +16,8 @@ mkDerivation {
 
   __structuredAttrs = true;
 
+  outputs = [ "out" "dev" ];
+
   buildCommand = ''
     set -x
 
@@ -30,8 +32,9 @@ mkDerivation {
     [[ -v nothing ]]
     [[ -z $nothing ]]
 
-    mkdir ''${outputs[out]}
+    mkdir ''${outputs[out]} ''${outputs[dev]}
     echo bar > $dest
+    echo foo > $dest2
 
     json=$(cat .attrs.json)
     [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]]
@@ -57,6 +60,7 @@ mkDerivation {
   nothing = null;
 
   dest = "${placeholder "out"}/foo";
+  dest2 = "${placeholder "dev"}/foo";
 
   "foo bar" = "BAD";
   "1foobar" = "BAD";
diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh
index 9ba2672b683..646bdb876d3 100644
--- a/tests/structured-attrs.sh
+++ b/tests/structured-attrs.sh
@@ -2,6 +2,7 @@ source common.sh
 
 clearStore
 
-outPath=$(nix-build structured-attrs.nix --no-out-link)
+nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
 
-[[ $(cat $outPath/foo) = bar ]]
+[[ $(cat $TEST_ROOT/result/foo) = bar ]]
+[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]]

From d506bd587aa62dfa9e70f5ba87dd614f7286fe2d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jan 2020 21:20:01 +0100
Subject: [PATCH 1801/2196] Fix clang warning

---
 src/libstore/path.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 6b29c3566cb..c31ea21796c 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -7,7 +7,7 @@ namespace nix {
 /* See path.rs. */
 struct StorePath;
 
-struct Store;
+class Store;
 
 extern "C" {
     void ffi_StorePath_drop(void *);

From a413594baf8633b59648e69f66dcf44bd0ad0b6c Mon Sep 17 00:00:00 2001
From: Rovanion Luckey <rovanion.luckey@gmail.com>
Date: Thu, 23 Jan 2020 14:48:53 +0100
Subject: [PATCH 1802/2196] installer: Handle edge case where the nix-daemon is
 already running on the system
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On a systemd-based Linux distribution: If the user has previously had multi-user Nix installed on the system, removed it and then reinstalled multi-user Nix again the old nix-daemon.service will still be running when `scripts/install-systemd-multi-user.sh` tries to start it which results in nothing being done and the old daemon continuing its run.

When a normal user then tries to use Nix through the daemon the nix binary will fail to connect to the nix-daemon as it does not belong to the currently installed Nix system. See below for steps to reproduce the issue that motivated this change.

$ sh <(curl https://nixos.org/nix/install) --daemon

$ sudo rm -rf /etc/nix /nix /root/.nix-profile /root/.nix-defexpr /root/.nix-channels /home/nix-installer/.nix-profile /home/nix-installer/.nix-defexpr /home/nix-installer/.nix-channels ~/.nix-channels ~/.nix-defexpr/ ~/.nix-profile /etc/profile.d/nix.sh.backup-before-nix /etc/profile.d/nix.sh; sed -i '/added by Nix installer$/d' ~/.bash_profile

$ unset NIX_REMOTE

$ sh <(curl https://nixos.org/nix/install) --daemon

└$ export NIX_REMOTE=daemon

└$ nix-env -iA nixpkgs.hello
installing 'hello-2.10'
error: cannot connect to daemon at '/nix/var/nix/daemon-socket/socket': No such file or directory
(use '--show-trace' to show detailed location information)

└$ sudo systemctl restart nix-daemon.service

└$ nix-env -iA nixpkgs.hello
installing 'hello-2.10'
these paths will be fetched (6.09 MiB download, 27.04 MiB unpacked):
  /nix/store/2g75chlbpxlrqn15zlby2dfh8hr9qwbk-hello-2.10
  /nix/store/aag9d1y4wcddzzrpfmfp9lcmc7skd7jk-glibc-2.27
copying path '/nix/store/aag9d1y4wcddzzrpfmfp9lcmc7skd7jk-glibc-2.27' from 'https://cache.nixos.org'...
copying path '/nix/store/2g75chlbpxlrqn15zlby2dfh8hr9qwbk-hello-2.10' from 'https://cache.nixos.org'...
building '/nix/store/w9adagg6vlikr799nkkqc9la5hbbpgmi-user-environment.drv'...
created 2 symlinks in user environment
---
 scripts/install-systemd-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh
index bef3ac4f991..e0201d53bd7 100755
--- a/scripts/install-systemd-multi-user.sh
+++ b/scripts/install-systemd-multi-user.sh
@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

From f8dbde0813c4e8beed6dfd09b093589e027a6675 Mon Sep 17 00:00:00 2001
From: Robin Gloster <mail@glob.in>
Date: Thu, 23 Jan 2020 17:38:07 +0100
Subject: [PATCH 1803/2196] structured-attrs: chown .attrs.* files to builder

Otherwise `chmod .`'ing the build directory doesn't work anymore, which
is done in nixpkgs if sourceRoot is set to '.'.
---
 src/libstore/build.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 74a68e1f5ba..9c6aedfa51c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2579,6 +2579,7 @@ void DerivationGoal::writeStructuredAttrs()
     }
 
     writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.json");
 
     /* As a convenience to bash scripts, write a shell file that
        maps all attributes that are representable in bash -
@@ -2647,6 +2648,7 @@ void DerivationGoal::writeStructuredAttrs()
     }
 
     writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+    chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
 

From 46992e71a1c596cb9b25a615738fb0b18e63c227 Mon Sep 17 00:00:00 2001
From: Calvin Loncaric <marvinx03@gmail.com>
Date: Sun, 26 Jan 2020 17:22:47 -0800
Subject: [PATCH 1804/2196] Document that autoconf is a dependency

---
 doc/manual/installation/prerequisites-source.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml
index e7bdcf966cf..fa6da9b1ed0 100644
--- a/doc/manual/installation/prerequisites-source.xml
+++ b/doc/manual/installation/prerequisites-source.xml
@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script

From 48ddb8e481c0ba0b59b7193df4aa914ce83a9032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 29 Jan 2020 11:47:39 +0100
Subject: [PATCH 1805/2196] retry on HTTP status code 429

---
 src/libstore/download.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index cdf56e09d69..b1a7a02822b 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -357,9 +357,10 @@ struct CurlDownloader : public Downloader
                 } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
                     // Don't retry on authentication/authorization failures
                     err = Forbidden;
-                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) {
+                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
                     // Most 4xx errors are client errors and are probably not worth retrying:
                     //   * 408 means the server timed out waiting for us, so we try again
+                    //   * 429 means too many requests, so we retry (with a delay)
                     err = Misc;
                 } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
                     // Let's treat most 5xx (server) errors as transient, except for a handful:

From c5319e5d0b7fe9b93ae29dfe410e5baa4d3b7140 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 1 Feb 2020 12:37:22 +0100
Subject: [PATCH 1806/2196] Show "warning:" in yellow instead of red

---
 src/libutil/logging.cc | 2 +-
 src/libutil/util.hh    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index b379306f6ec..fa5c84a2733 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger();
 
 void Logger::warn(const std::string & msg)
 {
-    log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg);
+    log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
 }
 
 class SimpleLogger : public Logger
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index abf1c95d675..3bfebcd153a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -429,6 +429,7 @@ void ignoreException();
 #define ANSI_FAINT "\e[2m"
 #define ANSI_RED "\e[31;1m"
 #define ANSI_GREEN "\e[32;1m"
+#define ANSI_YELLOW "\e[33;1m"
 #define ANSI_BLUE "\e[34;1m"
 
 

From 8745c63d3c674871393aa3f56c8457b056af8c87 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Mon, 3 Feb 2020 23:18:34 +0100
Subject: [PATCH 1807/2196] ssh-store: add remote-store and remote-program
 query params

Brings the functionality of ssh-ng:// in sync with the legacy ssh://
implementation.  Specifying the remote store uri enables various useful
things. eg.

    $ nix copy --to ssh-ng://cache?remote-store=file://mnt/cache --all
---
 src/libstore/ssh-store.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 42ee0650197..af99ad40a2d 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -16,6 +16,8 @@ class SSHStore : public RemoteStore
 
     const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
+    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
 
     SSHStore(const std::string & host, const Params & params)
         : Store(params)
@@ -82,7 +84,9 @@ ref<FSAccessor> SSHStore::getFSAccessor()
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
-    conn->sshConn = master.startCommand("nix-daemon --stdio");
+    conn->sshConn = master.startCommand(
+        fmt("%s --stdio", remoteProgram)
+        + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
     initConnection(*conn);

From d82b78bf51b2d8f626264fa992a907dd1088389a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Feb 2020 13:01:48 +0100
Subject: [PATCH 1808/2196] Fix segfault in gcc on i686-linux

  src/libstore/ssh-store.cc: In constructor 'nix::SSHStore::SSHStore(const string&, const Params&)':
  src/libstore/ssh-store.cc:31:21: internal compiler error: Segmentation fault
               compress)
                       ^
  Please submit a full bug report,
  with preprocessed source if appropriate.

https://hydra.nixos.org/build/111545609
---
 src/libstore/ssh-store.cc | 4 ++--
 src/libstore/store-api.hh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index af99ad40a2d..caae6b596c7 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -16,8 +16,8 @@ class SSHStore : public RemoteStore
 
     const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
     const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
-    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+    const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
+    const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
 
     SSHStore(const std::string & host, const Params & params)
         : Store(params)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 743be94108d..861b9693019 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -676,7 +676,7 @@ class LocalFSStore : public virtual Store
 public:
 
     // FIXME: the (Store*) cast works around a bug in gcc that causes
-    // it to emit the call to the Option constructor. Clang works fine
+    // it to omit the call to the Setting constructor. Clang works fine
     // either way.
     const PathSetting rootDir{(Store*) this, true, "",
         "root", "directory prefixed to all other paths"};

From 1b56de8cd1dfc47729bddbafb023819086296e2e Mon Sep 17 00:00:00 2001
From: Leonhard Markert <curiousleo@users.noreply.github.com>
Date: Mon, 10 Feb 2020 09:01:43 +0100
Subject: [PATCH 1809/2196] Remove macro_use

As of Rust 2018, macro_use is no longer required in most circumstances.
I think it is generally a good idea to remove these when not needed, to
stop them from polluting the crate's global namespace.
https://doc.rust-lang.org/edition-guide/rust-2018/macros/macro-changes.html#macro_rules-style-macros
---
 nix-rust/src/lib.rs         | 11 -----------
 nix-rust/src/store/path.rs  |  1 +
 nix-rust/src/util/base32.rs |  3 +++
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs
index e62613ba82c..27ea69fbdda 100644
--- a/nix-rust/src/lib.rs
+++ b/nix-rust/src/lib.rs
@@ -1,14 +1,3 @@
-#[macro_use]
-extern crate lazy_static;
-
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-#[cfg(test)]
-#[macro_use]
-extern crate proptest;
-
 #[cfg(not(test))]
 mod c;
 mod error;
diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs
index 2a5170befec..47b5975c034 100644
--- a/nix-rust/src/store/path.rs
+++ b/nix-rust/src/store/path.rs
@@ -138,6 +138,7 @@ impl fmt::Display for StorePathName {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use assert_matches::assert_matches;
 
     #[test]
     fn test_parse() {
diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs
index ba73689336c..efd4a290129 100644
--- a/nix-rust/src/util/base32.rs
+++ b/nix-rust/src/util/base32.rs
@@ -1,4 +1,5 @@
 use crate::error::Error;
+use lazy_static::lazy_static;
 
 pub fn encoded_len(input_len: usize) -> usize {
     if input_len == 0 {
@@ -87,7 +88,9 @@ pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use assert_matches::assert_matches;
     use hex;
+    use proptest::proptest;
 
     #[test]
     fn test_encode() {

From d8972317fc4314864619cadd5620ae780da657a3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Feb 2020 16:12:16 +0100
Subject: [PATCH 1810/2196] Prevent uninitialized StorePath creation

---
 src/libstore/path.hh    | 2 ++
 src/libutil/rust-ffi.hh | 2 ++
 src/nix/why-depends.cc  | 4 ++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index c31ea21796c..c90bb1fff8c 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -18,6 +18,8 @@ extern "C" {
 
 struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
 {
+    StorePath() = delete;
+
     static StorePath make(std::string_view path, std::string_view storeDir);
 
     static StorePath make(unsigned char hash[20], std::string_view name);
diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh
index 469a5fba321..228e2eead68 100644
--- a/src/libutil/rust-ffi.hh
+++ b/src/libutil/rust-ffi.hh
@@ -113,6 +113,8 @@ extern "C" {
 
 struct String : Vec<char, ffi_String_drop>
 {
+    String() = delete;
+
     String(std::string_view s)
     {
         ffi_String_new(StringSlice(s), this);
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 8566e58512d..c24ae7c8e09 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -103,7 +103,7 @@ struct CmdWhyDepends : SourceExprCommand
         std::map<StorePath, Node> graph;
 
         for (auto & path : closure)
-            graph.emplace(path.clone(), Node{path.clone(), cloneStorePathSet(store->queryPathInfo(path)->references)});
+            graph.emplace(path.clone(), Node { .path = path.clone(), .refs = cloneStorePathSet(store->queryPathInfo(path)->references) });
 
         // Transpose the graph.
         for (auto & node : graph)
@@ -112,7 +112,7 @@ struct CmdWhyDepends : SourceExprCommand
 
         /* Run Dijkstra's shortest path algorithm to get the distance
            of every path in the closure to 'dependency'. */
-        graph[dependencyPath.clone()].dist = 0;
+        graph.emplace(dependencyPath.clone(), Node { .path = dependencyPath.clone(), .dist = 0 });
 
         std::priority_queue<Node *> queue;
 

From 9af10b753c8a636f828b148fc3a9aecd1c0067fa Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 13 Feb 2020 17:15:05 +0100
Subject: [PATCH 1811/2196] Bindings::get(): std::optional<Attr *> -> Attr *

Returning a nullable type in an optional is silly.
---
 src/libexpr/attr-set.hh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index d6af999126d..118c7bd5d1f 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -64,12 +64,12 @@ public:
         return end();
     }
 
-    std::optional<Attr *> get(const Symbol & name)
+    Attr * get(const Symbol & name)
     {
         Attr key(name, 0);
         iterator i = std::lower_bound(begin(), end(), key);
         if (i != end() && i->name == name) return &*i;
-        return {};
+        return nullptr;
     }
 
     Attr & need(const Symbol & name, const Pos & pos = noPos)
@@ -77,7 +77,7 @@ public:
         auto a = get(name);
         if (!a)
             throw Error("attribute '%s' missing, at %s", name, pos);
-        return **a;
+        return *a;
     }
 
     iterator begin() { return &attrs[0]; }

From d78141a886383c38afb9fb294eec6fac0b74986e Mon Sep 17 00:00:00 2001
From: Carlos D <carlos.dagostino@gmail.com>
Date: Fri, 14 Feb 2020 16:11:22 +1100
Subject: [PATCH 1812/2196] Pass through http proxy env vars in pure shell

---
 src/nix-build/nix-build.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 205165a4ce1..e602ea55565 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -97,7 +97,11 @@ static void _main(int argc, char * * argv)
     std::string outLink = "./result";
 
     // List of environment variables kept for --pure
-    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+    std::set<string> keepVars{
+        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM",
+        "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
+        "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"
+    };
 
     Strings args;
     for (int i = 1; i < argc; ++i)

From 3e347220c82d1537723f49aa03a93a6f9d294417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20M=C3=B6st?= <tbsmoest@gmail.com>
Date: Fri, 14 Feb 2020 07:47:48 +0100
Subject: [PATCH 1813/2196] Fix PR_SET_PDEATHSIG results in Broken pipe (#2395)

The ssh client is lazily started by the first worker thread, that
requires a ssh connection. To avoid the ssh client to be killed, when
the worker process is stopped, do not set PR_SET_PDEATHSIG.
---
 src/libstore/ssh.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 2ee7115c53c..84548a6e4eb 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -33,6 +33,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
     out.create();
 
     auto conn = std::make_unique<Connection>();
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     conn->sshPid = startProcess([&]() {
         restoreSignals();
 
@@ -64,7 +67,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
         // could not exec ssh/bash
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
 
     in.readSide = -1;
@@ -91,6 +94,9 @@ Path SSHMaster::startMaster()
     Pipe out;
     out.create();
 
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     state->sshMaster = startProcess([&]() {
         restoreSignals();
 
@@ -110,7 +116,7 @@ Path SSHMaster::startMaster()
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
     out.writeSide = -1;
 

From 762febafe2694e7e1b2276181c7ec5564fca0b85 Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sat, 15 Feb 2020 01:37:44 -0500
Subject: [PATCH 1814/2196] doc: mention how to turn on pure evaluation mode in
 manual

The flag is `--pure-eval`, which can be found by looking at the test suite; it
should be in the notes describing the feature as well, since otherwise users may
assume this is referencing something like `nix-shell --pure`.
---
 doc/manual/release-notes/rl-2.0.xml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index fc9a77b08b6..bf6a679a1aa 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

From d8fd31f50ff55f7f334a2b9e58a82570e82ec28d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 18 Feb 2020 17:47:53 +0100
Subject: [PATCH 1815/2196] Disable the progress bar if $TERM == dumb or unset

Fixes #3363.
---
 src/nix/progress-bar.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index c445f31ccf3..26631416cc2 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -446,7 +446,9 @@ class ProgressBar : public Logger
 
 void startProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO));
+    logger = new ProgressBar(
+        printBuildLogs,
+        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
 }
 
 void stopProgressBar()

From 553e584f9231394212bb4d556c4a3eebc8444a63 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Feb 2020 15:46:07 +0100
Subject: [PATCH 1816/2196] LocalStore::checkDerivationOutputs(): Improve error
 message

---
 src/libstore/local-store.cc | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b254d766a58..e59624cd393 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -547,6 +547,18 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
+    auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
+    {
+        if (actual != expected)
+            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
+        auto j = drv.env.find(varName);
+        if (j == drv.env.end() || parseStorePath(j->second) != actual)
+            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
+                printStorePath(drvPath), varName, printStorePath(actual));
+    };
+
+
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator out = drv.outputs.find("out");
         if (out == drv.outputs.end())
@@ -554,24 +566,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
         bool recursive; Hash h;
         out->second.parseHashInfo(recursive, h);
-        auto outPath = makeFixedOutputPath(recursive, h, drvName);
 
-        StringPairs::const_iterator j = drv.env.find("out");
-        if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath));
+        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
     }
 
     else {
         Hash h = hashDerivationModulo(*this, drv, true);
-
-        for (auto & i : drv.outputs) {
-            auto outPath = makeOutputPath(i.first, h, drvName);
-            StringPairs::const_iterator j = drv.env.find(i.first);
-            if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-                throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                    printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath));
-        }
+        for (auto & i : drv.outputs)
+            check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
     }
 }
 

From f46bc0e8eb079f7d2613bfabfbcff31f4401b42c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 15 Feb 2020 21:30:26 +0100
Subject: [PATCH 1817/2196] Enable debug symbols

---
 release.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/release.nix b/release.nix
index db2b5ef05b2..3dc12cd8177 100644
--- a/release.nix
+++ b/release.nix
@@ -135,6 +135,8 @@ let
 
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
+
+        separateDebugInfo = true;
       });
 
 

From 583d06385de82ab5c7fc77d26cd138d3c6d5f4b5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 15 Feb 2020 21:48:28 +0100
Subject: [PATCH 1818/2196] Build with large config Boehm GC

---
 release-common.nix | 6 +++++-
 release.nix        | 6 ++++--
 shell.nix          | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index 60c8849d1ed..eb35d791775 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -48,7 +48,7 @@ rec {
   buildDeps =
     [ curl
       bzip2 xz brotli zlib editline
-      openssl pkgconfig sqlite boehmgc
+      openssl pkgconfig sqlite
       libarchive
       boost
       nlohmann_json
@@ -73,6 +73,10 @@ rec {
         */
       }));
 
+  propagatedDeps =
+    [ (boehmgc.override { enableLargeConfig = true; })
+    ];
+
   perlDeps =
     [ perl
       perlPackages.DBDSQLite
diff --git a/release.nix b/release.nix
index 3dc12cd8177..fdf7d017410 100644
--- a/release.nix
+++ b/release.nix
@@ -67,7 +67,7 @@ let
         src = nix;
         inherit officialRelease;
 
-        buildInputs = tarballDeps ++ buildDeps;
+        buildInputs = tarballDeps ++ buildDeps ++ propagatedDeps;
 
         postUnpack = ''
           (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
@@ -111,6 +111,8 @@ let
 
         buildInputs = buildDeps;
 
+        propagatedBuildInputs = propagatedDeps;
+
         preConfigure =
           # Copy libboost_context so we don't get all of Boost in our closure.
           # https://github.com/NixOS/nixpkgs/issues/45462
@@ -244,7 +246,7 @@ let
 
         enableParallelBuilding = true;
 
-        buildInputs = buildDeps;
+        buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;
 
diff --git a/shell.nix b/shell.nix
index e5a2b2c9186..4408b34aba4 100644
--- a/shell.nix
+++ b/shell.nix
@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

From 82de90961b79636c8a4989f46cc73ee449441e0f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Feb 2020 12:26:59 +0100
Subject: [PATCH 1819/2196] Add dev output

Necessary since we're now propagating boehm-gc.
---
 release.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/release.nix b/release.nix
index fdf7d017410..211f2b56ef4 100644
--- a/release.nix
+++ b/release.nix
@@ -109,6 +109,8 @@ let
         name = "nix";
         src = tarball;
 
+        outputs = [ "out" "dev" ];
+
         buildInputs = buildDeps;
 
         propagatedBuildInputs = propagatedDeps;

From 16e9a752871560434468c88d2cc302386bfc1e5d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Feb 2020 12:32:33 +0100
Subject: [PATCH 1820/2196] Typo

---
 doc/manual/release-notes/rl-2.0.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
index bf6a679a1aa..4c683dd3d72 100644
--- a/doc/manual/release-notes/rl-2.0.xml
+++ b/doc/manual/release-notes/rl-2.0.xml
@@ -504,7 +504,7 @@
 
   <listitem>
     <para><emphasis>Pure evaluation mode</emphasis>. With the
-    <literal>--pure-eval</literal> flag, nix enables a variant of the existing
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
     restricted evaluation mode that forbids access to anything that could cause
     different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as

From 906afedd238e4d83ef9ea4cf5a3aca77e980d582 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 19 Feb 2020 12:32:45 +0100
Subject: [PATCH 1821/2196] Use Nixpkgs 20.03

---
 release.nix | 2 +-
 shell.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 211f2b56ef4..1f592424b68 100644
--- a/release.nix
+++ b/release.nix
@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
diff --git a/shell.nix b/shell.nix
index 4408b34aba4..0cabcc15ad8 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,6 +1,6 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 

From f2a03acf3f42140f40fa5141b2b6ea94a554df64 Mon Sep 17 00:00:00 2001
From: Albert Safin <xzfcpw@gmail.com>
Date: Fri, 17 Jan 2020 06:44:00 +0000
Subject: [PATCH 1822/2196] nix-shell: clean up the tmpDir and escape variables

The problem fixed: each nix-shell invocation creates a new temporary
directory (`/tmp/nix-shell-*`) and never cleans up.

And while I'm here, shellescape all variables inlined into the rcfile.
See what might happen without escaping:

    $ export TZ="';echo pwned'"
    $ nix-shell -p hello --run hello
    pwned
    Hello, world!
---
 src/nix-build/nix-build.cc | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 205165a4ce1..d9bec431ee3 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -423,13 +423,18 @@ static void _main(int argc, char * * argv)
            lose the current $PATH directories. */
         auto rcfile = (Path) tmpDir + "/rc";
         writeFile(rcfile, fmt(
-                (keepTmp ? "" : "rm -rf '%1%'; "s) +
+                R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s +
+                (keepTmp ?
+                    "trap _nix_shell_clean_tmpdir EXIT; "
+                    "exitHooks+=(_nix_shell_clean_tmpdir); "
+                    "failureHooks+=(_nix_shell_clean_tmpdir); ":
+                    "_nix_shell_clean_tmpdir; ") +
                 (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") +
                 "%2%"
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
-                "PATH=\"%4%:$PATH\"; "
+                "PATH=%4%:\"$PATH\"; "
                 "SHELL=%5%; "
                 "set +e; "
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
@@ -438,12 +443,12 @@ static void _main(int argc, char * * argv)
                 "shopt -u nullglob; "
                 "unset TZ; %6%"
                 "%7%",
-                (Path) tmpDir,
+                shellEscape(tmpDir),
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
-                dirOf(*shell),
-                *shell,
-                (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
+                shellEscape(dirOf(*shell)),
+                shellEscape(*shell),
+                (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""),
                 envCommand));
 
         Strings envStrs;

From f6fd01bd19e14b62a8326712928f77904d097f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20M=C3=B6st?= <tbsmoest@gmail.com>
Date: Thu, 20 Feb 2020 07:52:14 +0100
Subject: [PATCH 1823/2196] .dir-locals.el: Set additional lambda indentation
 to zero

---
 .dir-locals.el | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.dir-locals.el b/.dir-locals.el
index 2d1117f4bda..8b21d4e403a 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -13,4 +13,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

From 22a754c091f765061f59bef5ce091268493bb138 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 28 Feb 2020 18:07:10 +0100
Subject: [PATCH 1824/2196] Fix GC failures on bad store path names

It failed on names like '/nix/store/9ip48nkc9rfy0a4yaw98lp6gipqlib1a-'.
---
 src/libstore/gc.cc        | 40 +++++++++++++++++++--------------------
 src/libstore/path.cc      | 14 ++++++++++++++
 src/libstore/store-api.cc |  8 --------
 src/libstore/store-api.hh |  2 ++
 4 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 690febc5bfb..0c3d896112c 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -255,12 +255,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        Path storePath = toStorePath(target);
-        // FIXME
-        if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-            roots[parseStorePath(storePath)].emplace(path);
+        auto storePath = maybeParseStorePath(toStorePath(target));
+        if (storePath && isValidPath(*storePath))
+            roots[std::move(*storePath)].emplace(path);
         else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath);
+            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
     };
 
     try {
@@ -296,10 +295,9 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         }
 
         else if (type == DT_REG) {
-            Path storePath = storeDir + "/" + std::string(baseNameOf(path));
-            // FIXME
-            if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-                roots[parseStorePath(storePath)].emplace(path);
+            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
+            if (storePath && isValidPath(*storePath))
+                roots[std::move(*storePath)].emplace(path);
         }
 
     }
@@ -523,14 +521,14 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 
     unsigned long long size = 0;
 
-    // FIXME
-    if (isStorePath(path) && isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+    if (storePath && isValidPath(*storePath)) {
         StorePathSet referrers;
-        queryReferrers(parseStorePath(path), referrers);
+        queryReferrers(*storePath, referrers);
         for (auto & i : referrers)
             if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
-        size = queryPathInfo(parseStorePath(path))->narSize;
-        invalidatePathChecked(parseStorePath(path));
+        size = queryPathInfo(*storePath)->narSize;
+        invalidatePathChecked(*storePath);
     }
 
     Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
@@ -593,8 +591,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
 
     visited.insert(path.clone());
 
-    //FIXME
-    if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false;
+    if (!isValidPath(path)) return false;
 
     StorePathSet incoming;
 
@@ -637,8 +634,9 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
 
-    // FIXME
-    if (!isStorePath(path) || !isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+
+    if (!storePath || !isValidPath(*storePath)) {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
         if (isActiveTempFile(state, path, ".lock")) return;
@@ -655,7 +653,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     StorePathSet visited;
 
-    if (canReachRoot(state, visited, parseStorePath(path))) {
+    if (storePath && canReachRoot(state, visited, *storePath)) {
         debug("cannot delete '%s' because it's still reachable", path);
     } else {
         /* No path we visited was a root, so everything is garbage.
@@ -818,8 +816,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
                 Path path = storeDir + "/" + name;
-                // FIXME
-                if (isStorePath(path) && isValidPath(parseStorePath(path)))
+                auto storePath = maybeParseStorePath(path);
+                if (storePath && isValidPath(*storePath))
                     entries.push_back(path);
                 else
                     tryToDelete(state, path);
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index a33bec3edaf..70b919adc30 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -55,6 +55,20 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath::make(path, storeDir);
 }
 
+std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+{
+    try {
+        return parseStorePath(path);
+    } catch (Error &) {
+        return {};
+    }
+}
+
+bool Store::isStorePath(const Path & path) const
+{
+    return (bool) maybeParseStorePath(path);
+}
+
 StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index d8f6c22bce3..75fa5d1e60e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -19,14 +19,6 @@ bool Store::isInStore(const Path & path) const
 }
 
 
-bool Store::isStorePath(const Path & path) const
-{
-    return isInStore(path)
-        && path.size() >= storeDir.size() + 1 + storePathHashLen
-        && path.find('/', storeDir.size() + 1) == Path::npos;
-}
-
-
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 861b9693019..a1cfb314a90 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -281,6 +281,8 @@ public:
 
     StorePath parseStorePath(std::string_view path) const;
 
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
     std::string printStorePath(const StorePath & path) const;
 
     // FIXME: remove

From d700eecea9a274c1b45549141f40180ac74454ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 2 Mar 2020 17:27:48 +0100
Subject: [PATCH 1825/2196] Add test for foldl'

---
 tests/lang/eval-okay-foldlStrict.exp | 1 +
 tests/lang/eval-okay-foldlStrict.nix | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/lang/eval-okay-foldlStrict.exp
 create mode 100644 tests/lang/eval-okay-foldlStrict.nix

diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/lang/eval-okay-foldlStrict.exp
new file mode 100644
index 00000000000..837e12b406f
--- /dev/null
+++ b/tests/lang/eval-okay-foldlStrict.exp
@@ -0,0 +1 @@
+500500
diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/lang/eval-okay-foldlStrict.nix
new file mode 100644
index 00000000000..3b87188d243
--- /dev/null
+++ b/tests/lang/eval-okay-foldlStrict.nix
@@ -0,0 +1,3 @@
+with import ./lib.nix;
+
+builtins.foldl' (x: y: x + y) 0 (range 1 1000)

From 401b5bc5418f3eb6d57da9d9e66df055f8bce122 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 21 Feb 2020 19:25:49 +0100
Subject: [PATCH 1826/2196] builtins.cache: Cache regular expressions

The evaluator was spending about 1% of its time compiling a small
number of regexes over and over again.
---
 src/libexpr/eval.hh    | 4 ++++
 src/libexpr/primops.cc | 8 +++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index cabc92d15c4..34a212aa4f9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -114,6 +114,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -314,6 +317,7 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 29302c9b6bb..4cd28698c4b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1811,19 +1811,21 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        std::regex regex(re, std::regex::extended);
+        auto regex = state.regexCache.find(re);
+        if (regex == state.regexCache.end())
+            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex)) {
+        if (!std::regex_match(str, match, regex->second)) {
             mkNull(v);
             return;
         }

From 75db069f927ffaf38ac6ef2d8143926b724ca935 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sun, 23 Feb 2020 16:36:19 +0100
Subject: [PATCH 1827/2196] Optimise Derivation::unparse()

In

  nix-instantiate --dry-run '<nixpkgs/nixos/release-combined.nix>' -A nixos.tests.simple.x86_64-linux

this reduces time spent in unparse() from 9.15% to 4.31%. The main
culprit was appending characters one at a time to the destination
string. Even though the string has enough capacity, push_back() still
needs to check this on every call.
---
 src/libstore/derivations.cc | 58 ++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 17 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index d9da8769c56..205b90e55ed 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -213,15 +213,26 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
 }
 
 
-static void printString(string & res, const string & s)
+static void printString(string & res, std::string_view s)
+{
+    char buf[s.size() * 2 + 2];
+    char * p = buf;
+    *p++ = '"';
+    for (auto c : s)
+        if (c == '\"' || c == '\\') { *p++ = '\\'; *p++ = c; }
+        else if (c == '\n') { *p++ = '\\'; *p++ = 'n'; }
+        else if (c == '\r') { *p++ = '\\'; *p++ = 'r'; }
+        else if (c == '\t') { *p++ = '\\'; *p++ = 't'; }
+        else *p++ = c;
+    *p++ = '"';
+    res.append(buf, p - buf);
+}
+
+
+static void printUnquotedString(string & res, std::string_view s)
 {
     res += '"';
-    for (const char * i = s.c_str(); *i; i++)
-        if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
-        else if (*i == '\n') res += "\\n";
-        else if (*i == '\r') res += "\\r";
-        else if (*i == '\t') res += "\\t";
-        else res += *i;
+    res.append(s);
     res += '"';
 }
 
@@ -239,6 +250,19 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
 }
 
 
+template<class ForwardIterator>
+static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+    res += '[';
+    bool first = true;
+    for ( ; i != j; ++i) {
+        if (first) first = false; else res += ',';
+        printUnquotedString(res, *i);
+    }
+    res += ']';
+}
+
+
 string Derivation::unparse(const Store & store, bool maskOutputs,
     std::map<std::string, StringSet> * actualInputs) const
 {
@@ -249,10 +273,10 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     bool first = true;
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
-        s += '('; printString(s, i.first);
-        s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printString(s, i.second.hashAlgo);
-        s += ','; printString(s, i.second.hash);
+        s += '('; printUnquotedString(s, i.first);
+        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
+        s += ','; printUnquotedString(s, i.second.hashAlgo);
+        s += ','; printUnquotedString(s, i.second.hash);
         s += ')';
     }
 
@@ -261,24 +285,24 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     if (actualInputs) {
         for (auto & i : *actualInputs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, i.first);
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, i.first);
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     } else {
         for (auto & i : inputDrvs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, store.printStorePath(i.first));
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, store.printStorePath(i.first));
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     }
 
     s += "],";
     auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
-    printStrings(s, paths.begin(), paths.end());
+    printUnquotedStrings(s, paths.begin(), paths.end());
 
-    s += ','; printString(s, platform);
+    s += ','; printUnquotedString(s, platform);
     s += ','; printString(s, builder);
     s += ','; printStrings(s, args.begin(), args.end());
 

From d37dc71e3cf077fa5d24a9bf8395deae21cc4410 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 4 Mar 2020 13:55:15 +0100
Subject: [PATCH 1828/2196] nix-build: Fix !<output> handling

This was broken by 22a754c091f765061f59bef5ce091268493bb138.

https://hydra.nixos.org/eval/1573669
---
 src/libstore/path.cc       | 2 +-
 src/libstore/store-api.hh  | 2 +-
 src/nix-build/nix-build.cc | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 70b919adc30..9a28aa96ad4 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -64,7 +64,7 @@ std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
     }
 }
 
-bool Store::isStorePath(const Path & path) const
+bool Store::isStorePath(std::string_view path) const
 {
     return (bool) maybeParseStorePath(path);
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a1cfb314a90..a000757fb8d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -305,7 +305,7 @@ public:
 
     /* Return true if ‘path’ is a store path, i.e. a direct child of
        the Nix store. */
-    bool isStorePath(const Path & path) const;
+    bool isStorePath(std::string_view path) const;
 
     /* Chop off the parts after the top-level store name, e.g.,
        /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index ff95ad787d7..27ec7d0feda 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -295,7 +295,8 @@ static void _main(int argc, char * * argv)
                 try {
                     absolute = canonPath(absPath(i), true);
                 } catch (Error & e) {};
-                if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+                auto [path, outputNames] = parsePathWithOutputs(absolute);
+                if (store->isStorePath(path) && hasSuffix(path, ".drv"))
                     drvs.push_back(DrvInfo(*state, store, absolute));
                 else
                     /* If we're in a #! script, interpret filenames

From 5e086ba8c39fc5c39cbbe452b33313c4e5782cd2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Mar 2020 11:00:17 +0100
Subject: [PATCH 1829/2196] nix-perl: Fix segfault in queryPathInfo)

---
 perl/lib/Nix/Store.xs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index fefcd5ae6e8..1ca734e75bd 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -102,7 +102,7 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (info->deriver)
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));

From 983fab7ea994c3e377e82e4c2f5dbb097820990a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Mar 2020 11:06:55 +0100
Subject: [PATCH 1830/2196] dotgraph.cc: Remove dead code

---
 src/nix-store/dotgraph.cc | 71 ---------------------------------------
 1 file changed, 71 deletions(-)

diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index a9d0dbbed63..c9837b3e753 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -44,46 +44,6 @@ static string makeNode(const string & id, std::string_view label,
 }
 
 
-#if 0
-string pathLabel(const Path & nePath, const string & elemPath)
-{
-    return (string) nePath + "-" + elemPath;
-}
-
-
-void printClosure(const Path & nePath, const StoreExpr & fs)
-{
-    PathSet workList(fs.closure.roots);
-    PathSet doneSet;
-
-    for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) {
-        cout << makeEdge(pathLabel(nePath, *i), nePath);
-    }
-
-    while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
-
-        if (doneSet.insert(path).second) {
-            ClosureElems::const_iterator elem = fs.closure.elems.find(path);
-            if (elem == fs.closure.elems.end())
-                throw Error(format("bad closure, missing path '%1%'") % path);
-
-            for (StringSet::const_iterator i = elem->second.refs.begin();
-                 i != elem->second.refs.end(); ++i)
-            {
-                workList.insert(*i);
-                cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path));
-            }
-
-            cout << makeNode(pathLabel(nePath, path),
-                symbolicName(path), "#ff0000");
-        }
-    }
-}
-#endif
-
-
 void printDotGraph(ref<Store> store, StorePathSet && roots)
 {
     StorePathSet workList(std::move(roots));
@@ -104,37 +64,6 @@ void printDotGraph(ref<Store> store, StorePathSet && roots)
                 cout << makeEdge(std::string(p.to_string()), std::string(p.to_string()));
             }
         }
-
-#if 0
-        StoreExpr ne = storeExprFromPath(path);
-
-        string label, colour;
-
-        if (ne.type == StoreExpr::neDerivation) {
-            for (PathSet::iterator i = ne.derivation.inputs.begin();
-                 i != ne.derivation.inputs.end(); ++i)
-            {
-                workList.insert(*i);
-                cout << makeEdge(*i, path);
-            }
-
-            label = "derivation";
-            colour = "#00ff00";
-            for (StringPairs::iterator i = ne.derivation.env.begin();
-                 i != ne.derivation.env.end(); ++i)
-                if (i->first == "name") label = i->second;
-        }
-
-        else if (ne.type == StoreExpr::neClosure) {
-            label = "<closure>";
-            colour = "#00ffff";
-            printClosure(path, ne);
-        }
-
-        else abort();
-
-        cout << makeNode(path, label, colour);
-#endif
     }
 
     cout << "}\n";

From 8a1d8701f6efa07534e4e0500bc561f1c33fc25f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 10 Mar 2020 11:11:46 +0100
Subject: [PATCH 1831/2196] nix-store -q --graph: Fix edges

Fixes #3389.
---
 src/nix-store/dotgraph.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index c9837b3e753..667d917f510 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -61,7 +61,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots)
         for (auto & p : store->queryPathInfo(path)->references) {
             if (p != path) {
                 workList.insert(p.clone());
-                cout << makeEdge(std::string(p.to_string()), std::string(p.to_string()));
+                cout << makeEdge(std::string(p.to_string()), std::string(path.to_string()));
             }
         }
     }

From e063c71a79d43e3864468a8481c0dfd12586af58 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Mar 2020 10:33:23 +0100
Subject: [PATCH 1832/2196] nixos.org/releases -> releases.nixos.org

---
 doc/manual/installation/installing-binary.xml | 2 +-
 scripts/install.in                            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 394d8053b94..3f57f47b5bf 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>
diff --git a/scripts/install.in b/scripts/install.in
index 76cb3b1fdce..4500fd54a2e 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -30,7 +30,7 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 

From e02481ded216ffb5b06b413e3695d4e11e62e02f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Mar 2020 16:36:36 +0100
Subject: [PATCH 1833/2196] parseExprFromString(): Use std::string_view

---
 src/libexpr/eval.hh  | 4 ++--
 src/libexpr/parser.y | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 34a212aa4f9..f3e8a34bd75 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -144,8 +144,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index afa1fd439a9..9c769e80376 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -611,13 +611,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }

From 9950cdec3514949942a79c58764b1ea9bf9d5d57 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 11 Mar 2020 16:41:22 +0100
Subject: [PATCH 1834/2196] Move some corepkgs into the nix binary

---
 corepkgs/local.mk                                 |  5 ++++-
 .../libexpr}/imported-drv-to-derivation.nix       |  0
 src/libexpr/local.mk                              |  2 ++
 src/libexpr/primops.cc                            | 15 +++++++++++----
 {corepkgs => src/nix-env}/buildenv.nix            |  0
 src/nix-env/user-env.cc                           |  4 +++-
 src/nix/local.mk                                  |  2 ++
 tests/lang/eval-okay-search-path.nix              |  2 +-
 8 files changed, 23 insertions(+), 7 deletions(-)
 rename {corepkgs => src/libexpr}/imported-drv-to-derivation.nix (100%)
 rename {corepkgs => src/nix-env}/buildenv.nix (100%)

diff --git a/corepkgs/local.mk b/corepkgs/local.mk
index 362c8eb612e..2c72d3a3199 100644
--- a/corepkgs/local.mk
+++ b/corepkgs/local.mk
@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 
diff --git a/corepkgs/imported-drv-to-derivation.nix b/src/libexpr/imported-drv-to-derivation.nix
similarity index 100%
rename from corepkgs/imported-drv-to-derivation.nix
rename to src/libexpr/imported-drv-to-derivation.nix
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 26b9f14baa8..8a9b3c2ea12 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -31,3 +31,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4cd28698c4b..8de23495187 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -121,10 +121,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
         w.attrs->sort();
-        Value fun;
-        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
-        state.forceFunction(fun, pos);
-        mkApp(v, fun, w);
+
+        static Value * fun = nullptr;
+        if (!fun) {
+            fun = state.allocValue();
+            state.eval(state.parseExprFromString(
+                #include "imported-drv-to-derivation.nix.gen.hh"
+                , "/"), *fun);
+        }
+
+        state.forceFunction(*fun, pos);
+        mkApp(v, *fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
diff --git a/corepkgs/buildenv.nix b/src/nix-env/buildenv.nix
similarity index 100%
rename from corepkgs/buildenv.nix
rename to src/nix-env/buildenv.nix
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 8e4ecda4e8c..717431b7aff 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -106,7 +106,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
     /* Get the environment builder expression. */
     Value envBuilder;
-    state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder);
+    state.eval(state.parseExprFromString(
+        #include "buildenv.nix.gen.hh"
+            , "/"), envBuilder);
 
     /* Construct a Nix expression that calls the user environment
        builder with the manifest as argument. */
diff --git a/src/nix/local.mk b/src/nix/local.mk
index a34cca9fd8d..a486a37cc4f 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -23,3 +23,5 @@ $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
+
+src/nix-env/nix-env.cc: src/nix-env/buildenv.nix.gen.hh
diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix
index cca41f821f8..c5a123d042a 100644
--- a/tests/lang/eval-okay-search-path.nix
+++ b/tests/lang/eval-okay-search-path.nix
@@ -1,7 +1,7 @@
 with import ./lib.nix;
 with builtins;
 
-assert pathExists <nix/buildenv.nix>;
+assert pathExists <nix/fetchurl.nix>;
 
 assert length __nixPath == 6;
 assert length (filter (x: x.prefix == "nix") __nixPath) == 1;

From 9080d5d924884ffde31536272ab925f784c8d7de Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 11 Mar 2020 19:41:22 +0100
Subject: [PATCH 1835/2196] README, error msg: http -> https

---
 README.md                           | 6 +++---
 scripts/install-nix-from-closure.sh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 48cb1685c7b..a953c0f7159 100644
--- a/README.md
+++ b/README.md
@@ -9,11 +9,11 @@ appear with Nix.
 
 To find out more about the tool, usage and installation instructions, please
 read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+<https://nixos.org/nix/manual>.
 
 ## Contributing
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
+Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
 of the manual. It helps you to get started with building Nix from source.
 
 ## License
@@ -21,4 +21,4 @@ of the manual. It helps you to get started with building Nix from source.
 Nix is released under the LGPL v2.1
 
 This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).
+use in the [OpenSSL Toolkit](https://www.OpenSSL.org/).
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 3fea7e05602..e00708f6cdb 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

From 15edd2349e9e343dad761d513ffdcb121688777c Mon Sep 17 00:00:00 2001
From: Will Dietz <w@wdtz.org>
Date: Wed, 11 Mar 2020 18:39:30 -0500
Subject: [PATCH 1836/2196] local.mk: fix user-env.cc dep on
 buildenv.nix.gen.hh, resolve occasional build failure

---
 src/nix/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index a486a37cc4f..51dad101f56 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -24,4 +24,4 @@ $(foreach name, \
   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 
-src/nix-env/nix-env.cc: src/nix-env/buildenv.nix.gen.hh
+src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh

From 3f55f8a8fb5e885297e288c078adc173316addf2 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 11 Mar 2020 20:04:47 +0100
Subject: [PATCH 1837/2196] pathInfoCache: Respect disk cache TTLs #3398

---
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/local-store.cc        |  3 ++-
 src/libstore/store-api.cc          | 24 ++++++++++++++++--------
 src/libstore/store-api.hh          | 21 ++++++++++++++++++++-
 4 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 717faec9271..3a2d848615b 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -106,7 +106,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
+        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e59624cd393..c5929a41c02 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -622,7 +622,8 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info));
+        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+            PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
     return id;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 75fa5d1e60e..b043feb0afc 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -226,6 +226,14 @@ std::string Store::getUri()
     return "";
 }
 
+bool Store::PathInfoCacheValue::isKnownNow()
+{
+    std::chrono::duration ttl = didExist()
+        ? std::chrono::seconds(settings.ttlPositiveNarInfoCache)
+        : std::chrono::seconds(settings.ttlNegativeNarInfoCache);
+
+    return std::chrono::steady_clock::now() < time_point + ttl;
+}
 
 bool Store::isValidPath(const StorePath & storePath)
 {
@@ -234,9 +242,9 @@ bool Store::isValidPath(const StorePath & storePath)
     {
         auto state_(state.lock());
         auto res = state_->pathInfoCache.get(hashPart);
-        if (res) {
+        if (res && res->isKnownNow()) {
             stats.narInfoReadAverted++;
-            return *res != 0;
+            return res->didExist();
         }
     }
 
@@ -246,7 +254,7 @@ bool Store::isValidPath(const StorePath & storePath)
             stats.narInfoReadAverted++;
             auto state_(state.lock());
             state_->pathInfoCache.upsert(hashPart,
-                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
             return res.first == NarInfoDiskCache::oValid;
         }
     }
@@ -301,11 +309,11 @@ void Store::queryPathInfo(const StorePath & storePath,
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
-            if (res) {
+            if (res && res->isKnownNow()) {
                 stats.narInfoReadAverted++;
-                if (!*res)
+                if (!res->didExist())
                     throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                return callback(ref<const ValidPathInfo>(*res));
+                return callback(ref<const ValidPathInfo>(res->value));
             }
         }
 
@@ -316,7 +324,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                 {
                     auto state_(state.lock());
                     state_->pathInfoCache.upsert(hashPart,
-                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                        res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
                         res.second->path != storePath)
                         throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
@@ -340,7 +348,7 @@ void Store::queryPathInfo(const StorePath & storePath,
 
                 {
                     auto state_(state.lock());
-                    state_->pathInfoCache.upsert(hashPart, info);
+                    state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
                 }
 
                 if (!info || info->path != parseStorePath(storePath)) {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index a000757fb8d..e0484de133c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -16,6 +16,7 @@
 #include <unordered_set>
 #include <memory>
 #include <string>
+#include <chrono>
 
 
 namespace nix {
@@ -261,10 +262,28 @@ public:
 
 protected:
 
+    struct PathInfoCacheValue {
+
+        // Time of cache entry creation or update
+        std::chrono::time_point<std::chrono::steady_clock> time_point = std::chrono::steady_clock::now();
+
+        // Null if missing
+        std::shared_ptr<const ValidPathInfo> value;
+
+        // Whether the value is valid as a cache entry. The path may not exist.
+        bool isKnownNow();
+
+        // Past tense, because a path can only be assumed to exists when
+        // isKnownNow() && didExist()
+        inline bool didExist() {
+          return value != nullptr;
+        }
+    };
+
     struct State
     {
         // FIXME: fix key
-        LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
+        LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
     };
 
     Sync<State> state;

From b816515f613b7013ba0d5489841568b8f666f224 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 12:23:19 +0100
Subject: [PATCH 1838/2196] Fix ca-references feature check

Fixes #3406.
---
 src/libstore/local-store.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c5929a41c02..cd2e86f29fe 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1004,16 +1004,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
+            if (info.ca != "" &&
+                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
+                    || info.references.empty()))
+                settings.requireExperimentalFeature("ca-references");
+
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "")
+            if (info.ca == "" || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
-            else {
-                if (!info.references.empty())
-                    settings.requireExperimentalFeature("ca-references");
+            else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
-            }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1268,7 +1270,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "")
+                if (info->ca == "" || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
                     hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));

From cc5c81822d298544489ea0861d266c3a5137759a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 14:50:51 +0100
Subject: [PATCH 1839/2196] mk/README.md: Remove

The make-rules repo is not maintained.
---
 mk/README.md | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 mk/README.md

diff --git a/mk/README.md b/mk/README.md
deleted file mode 100644
index e4cd742b4c7..00000000000
--- a/mk/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

From 9c7e90f414067eb59170bde952d5b8ac03c8f46c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 15:00:08 +0100
Subject: [PATCH 1840/2196] style.css: Remove

This file is licensed under the GPL. Originally, Nix was also
GPL-licensed so that was fine. However, we later changed the license
to the LGPL but missed the fact that style.css has an incompatible
license.

Since the Nix manual at nixos.org uses its own styling, we can remove
this file.

Fixes #3392.
---
 doc/manual/local.mk  |   3 +-
 doc/manual/style.css | 263 -------------------------------------------
 2 files changed, 1 insertion(+), 265 deletions(-)
 delete mode 100644 doc/manual/style.css

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 4376c3644d3..b4e7f35d501 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -4,7 +4,6 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
@@ -66,7 +65,7 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
diff --git a/doc/manual/style.css b/doc/manual/style.css
deleted file mode 100644
index 592583ab086..00000000000
--- a/doc/manual/style.css
+++ /dev/null
@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

From 30962d21beb2133b5d986b8aebfe64408be5e80e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 13 Mar 2020 11:57:58 +0100
Subject: [PATCH 1841/2196] Add CI with github actions

---
 .github/workflows/test.yml | 14 ++++++++++++++
 release.nix                |  4 ++--
 2 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/test.yml

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 00000000000..f261cd0bc54
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,14 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v8
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A build -A binaryTarball -A perlBindings -A installerScript
diff --git a/release.nix b/release.nix
index 1f592424b68..45c2f14ba9a 100644
--- a/release.nix
+++ b/release.nix
@@ -348,9 +348,9 @@ let
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
               (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+              systems
             } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+            --replace '@nixVersion@' ${build.${builtins.head systems}.src.version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';

From 90b805ef251a3e3244b2cddcf815bf36c2f4e2ab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 15:56:25 +0100
Subject: [PATCH 1842/2196] Remove build and binaryTarball since they're
 included in installerScript

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f261cd0bc54..87997414d77 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,4 +11,4 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: cachix/install-nix-action@v8
-    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A build -A binaryTarball -A perlBindings -A installerScript
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

From 858ad7a4b3eed6a5f950ef7fa06497cd23635e8f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 16:32:43 +0100
Subject: [PATCH 1843/2196] Remove callout graphics

Fixes #3396.
---
 doc/manual/images/callouts/1.gif  | Bin 889 -> 0 bytes
 doc/manual/images/callouts/10.gif | Bin 929 -> 0 bytes
 doc/manual/images/callouts/11.gif | Bin 202 -> 0 bytes
 doc/manual/images/callouts/12.gif | Bin 210 -> 0 bytes
 doc/manual/images/callouts/13.gif | Bin 209 -> 0 bytes
 doc/manual/images/callouts/14.gif | Bin 205 -> 0 bytes
 doc/manual/images/callouts/15.gif | Bin 210 -> 0 bytes
 doc/manual/images/callouts/2.gif  | Bin 907 -> 0 bytes
 doc/manual/images/callouts/3.gif  | Bin 914 -> 0 bytes
 doc/manual/images/callouts/4.gif  | Bin 907 -> 0 bytes
 doc/manual/images/callouts/5.gif  | Bin 916 -> 0 bytes
 doc/manual/images/callouts/6.gif  | Bin 218 -> 0 bytes
 doc/manual/images/callouts/7.gif  | Bin 907 -> 0 bytes
 doc/manual/images/callouts/8.gif  | Bin 918 -> 0 bytes
 doc/manual/images/callouts/9.gif  | Bin 923 -> 0 bytes
 doc/manual/local.mk               |   4 +---
 16 files changed, 1 insertion(+), 3 deletions(-)
 delete mode 100644 doc/manual/images/callouts/1.gif
 delete mode 100644 doc/manual/images/callouts/10.gif
 delete mode 100644 doc/manual/images/callouts/11.gif
 delete mode 100644 doc/manual/images/callouts/12.gif
 delete mode 100644 doc/manual/images/callouts/13.gif
 delete mode 100644 doc/manual/images/callouts/14.gif
 delete mode 100644 doc/manual/images/callouts/15.gif
 delete mode 100644 doc/manual/images/callouts/2.gif
 delete mode 100644 doc/manual/images/callouts/3.gif
 delete mode 100644 doc/manual/images/callouts/4.gif
 delete mode 100644 doc/manual/images/callouts/5.gif
 delete mode 100644 doc/manual/images/callouts/6.gif
 delete mode 100644 doc/manual/images/callouts/7.gif
 delete mode 100644 doc/manual/images/callouts/8.gif
 delete mode 100644 doc/manual/images/callouts/9.gif

diff --git a/doc/manual/images/callouts/1.gif b/doc/manual/images/callouts/1.gif
deleted file mode 100644
index 9e7a87f75461ce41cc91ee68246c7bfb47d37ca3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 889
zcmV-<1BU!ZNk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0RaC1EC2ui01N;O000P90RIW3lmh~U9GnCyi~#@uf)ov7ENlpX
zku!=C88uXL?%qX;j2KcZ^)Vs>iU2s3ba;}YN|XphHhhUuqsx_y{>fxi3!_4aC;>=R
P_%9>^1UV=v3<v-_&&Q&6

diff --git a/doc/manual/images/callouts/10.gif b/doc/manual/images/callouts/10.gif
deleted file mode 100644
index e80f7f8e632e72463866c08beba2bfb3b17221e9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 929
zcmV;S177?`Nk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000Pn0R0K%lmi2V9E=1i%mD!d00b!@#8`-+
zR;?7?dYzM!!e6Nn@BUe=1z?Uh0F0D8bXLpS8~`!?g^1x00Ly!00ATo+13^ue?{bFV
z2|%UITFwXo$ng(Dj8XuOGO`ANk~5V4ascSX2|}-X)?#!Ns5R<C0Am$W>IspO!hirf
Dy0E(c

diff --git a/doc/manual/images/callouts/11.gif b/doc/manual/images/callouts/11.gif
deleted file mode 100644
index 67f91a239d66d622f8d254539b6fd580efa42b02..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 202
zcmZ?wbhEHb<YC}pXpv<A0&Z?@VPRn@DJc~d6-`Y|BO@bgYinm`=iuPr)YR0>%*^WQ
z>bADF{{H^y)2Gj!J9p8dMeEkB+q`-6jvYIW9656K?Ac3~F5SL;`|;z)Z{EE5@ZrP1
zfB*i&fZ|UUMn(pC1|5)1AUhdY-5;oT-OT9fVPH5?qIP(rnuCecF;(8lY%CHJA0|%J
x(XwN3776fTXisr6DR7$1b@7l?>`pt$t64n;p&ZEvzI<6|A|t-7MudsM8UU#vRD1vc

diff --git a/doc/manual/images/callouts/12.gif b/doc/manual/images/callouts/12.gif
deleted file mode 100644
index 54c4b42f1901629a81924c2f0f59338104adeedd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 210
zcmZ?wbhEHb<YC}pXpv?B0v;Y7At50tDJgAjZ6hNiD=RA(7Z-nj|KQ-@xVX6F<mBAk
z+>(-#rlzKzo}S5*C(oKSYw6OZn>TOXy?ghOBS+4hIdl8=?Z=NFfBN+4-@kwVQGw!5
z7Dh$}c?KP@O+Y&tSp6TUNBM5@Ncv^za9iq?NB~b`s?Q`L(SI(T+y_1#WZ*J!SfIpc
y)1^D>K$0MX%p%4Hwu?TqL@qKHD9DJsXySU9#jCf<|A~QyiNVJ`b^kdT8LR=FrC4|X

diff --git a/doc/manual/images/callouts/13.gif b/doc/manual/images/callouts/13.gif
deleted file mode 100644
index dd5d7d9b6439affca376bcd60785d528a24ce425..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 209
zcmZ?wbhEHb<YC}pXpv?B0&Z?@At51ISy_F3eN$6Y8yg#UcXwZ3-@w4Y=;-Lg#KfGO
zoU*dAmX?;TuCB?GCof#MaP#KPJ9g|iapJ_~%a?E6y7lPMqc?BfeEj(F-@kwVQGw!5
z7Dh$}c?KPjO&~iNSp6QT`}rL5N&2Pfa9eAYa!5yGY};hEX>B&V9x4V*B03xgoIf)x
xP}6;KaKZ)#6?t!lhL!-83*8xv4m?aeZ4A@2c0{pEJ&`E!<jIdcaUx6%)&Q}PSd;(&

diff --git a/doc/manual/images/callouts/14.gif b/doc/manual/images/callouts/14.gif
deleted file mode 100644
index 3d7a952a310523200521c1fa87b943637b5d1347..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 205
zcmZ?wbhEHb<YC}pXpv$70$yHTAt50tDJfG^Q%g%rH#avQAD`&x=!ArX>gwv|=H~A1
z?x|C!&Y3f3>C&Z}Hf`Fwcki)d$4;F(b@l4iTeogKe*F0D+qWM-e*E|E-+v6C_>+Z^
zkwKn82V@h-P6k%52kLdAD!mz@3~xRxVRG8g@v!IkBvpn+%|^iofu1ZH;zuVO=~9sr
r<5W2yU@F+%-l|kl%h$jrvB98Ei)orw&!RI?aaY=tCf$Ck$Y2cs9-Uok

diff --git a/doc/manual/images/callouts/15.gif b/doc/manual/images/callouts/15.gif
deleted file mode 100644
index 1c9183d5bb619eb608a5b7543f042ae7fd18684b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 210
zcmZ?wbhEHb<YC}pXpvz60)BpeVPRouX=w!o1!H4lOG`^fM@K(DzmSlS)YR1M?Cj#=
z;@aBU=H}*}o}S5*C(oQYbH$1kTefW3vuDqtLx-+hxpMR7%?A%2ynXxj%a<?z{{8!p
z1QdU=Ffua8Gw6V90@=yH>i<B!ZkkE&i%<rg2Mg{us5o5KS-(R^$|R?s?ST@PiAMhs
zH{rt0h>7h7121rhDKRRXS|PKBUy%7g1c%!6qXjM*TZ@!<7MXS&a1%XU+t0<wU=0BK
C2U#Ql

diff --git a/doc/manual/images/callouts/2.gif b/doc/manual/images/callouts/2.gif
deleted file mode 100644
index 94d42a30f99b87314127dcd44e6162c2fdf26e6e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 907
zcmV;619bdHNk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0RaC1EC2ui01N;O000PR0RIW3lmh~U9GnCyi~#^ZiVzKBECdk3
z+PQbeAOMias9`B<6z_Eez=$EqJq^9piin`0%NY0OG(^zSrIK^CY9cU6@*c{UBiGy;
h6C(zOhK&9(<aDm-Ig}Cw@_eaKU`d)eC@BmG06P*-u}lB}

diff --git a/doc/manual/images/callouts/3.gif b/doc/manual/images/callouts/3.gif
deleted file mode 100644
index dd3541a1bc25307e41416082525903a2337a2a43..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 914
zcmV;D18w|ANk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000PY0R0K1lmh~U9GnCyi~#^ajvNR8#8?Ob
zf;Vg4oI#N2j}ZU>bIu`v@NdmWMhq*as95r2jEElrv^r-ECIXlrbBwuXvSCUs9sxM3
o<uWLvcQJCjBXrUqLkO+z-D?$bqQI36V_H>+a%2oj3IhTFJI8abOaK4?

diff --git a/doc/manual/images/callouts/4.gif b/doc/manual/images/callouts/4.gif
deleted file mode 100644
index 4bcbf7e31a17497e65fa0ccc9756961130be6ac7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 907
zcmV;619bdHNk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000PR0R0JUB_%})C8Y!^R05!33J3slENlos
z$-g-bCAOjn;oL=vlo&Q_Rq7qahA}WS)N&5Oy^{bCs0?8;#XUwUV=^4`uV>FvD1lN0
h@RA}Ue>sMFTjNk+(xCuSC=|#k1_T5#jtUG206WA?slNaK

diff --git a/doc/manual/images/callouts/5.gif b/doc/manual/images/callouts/5.gif
deleted file mode 100644
index 1c62b4f920936c063c93d8551158a80500dfefbe..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 916
zcmV;F18e+8Nk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000Pa0R0K%lmi2V9E=1i%z+?9iV!If#8?O*
z$G>+Nl>kWLuM~ia{}6l}$w(u}R?Y~C1P}wmM~j4%oa6KnflB}(2z_MquBJ%{T0T-b
q_c4Y?mj9H5@J4B;zZ@S4y}MTm!bXY!5Uk{=&>tm+5)%px2mm_@=(0fo

diff --git a/doc/manual/images/callouts/6.gif b/doc/manual/images/callouts/6.gif
deleted file mode 100644
index 23bc5555d2a467d6c3025d7f334a2b5546bd4fd9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 218
zcmZ?wbhEHb<YC}pXpv+90uBxi0RaIC2?<3-MNLgjLqkJLOG_^=ub`ly?Ck8~;^LN;
zmY$xTDO0A*nKNhU(xq$HuHCzL@3CXY&Ye4V@7}%V&!4}4|Ni^;@Bgs@#h)yUj12M&
zIv|@sb~3PrKTvn`3{;q~LW#jZ<n6|77gBv072Mwz@iw1c_%wl0ZNeeN>qb0L)3{T$
zGFay|aIp&P&{^Zgxx_Wm<T%68R)+%;((@G$G#;G6$U4>Lz?s!{{N9Xg1xktx)&MH`
BUFZM+

diff --git a/doc/manual/images/callouts/7.gif b/doc/manual/images/callouts/7.gif
deleted file mode 100644
index e55ce89585a8d5f80cc1a83df537578983a4d8e5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 907
zcmV;619bdHNk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000PR0R0K9B_%})C9MQ1lmY=mLkJLZDAWR=
z)xV4XtO1DOuNQ!iA2FPx2!IkvkhMI1%n<`504eucencSiqqSNt5uD8NPm0SJI6pGF
hhrlHxe@Z^q3i>gHLLn}jDip{m2Lc2sjtUG206SSyt84%O

diff --git a/doc/manual/images/callouts/8.gif b/doc/manual/images/callouts/8.gif
deleted file mode 100644
index 49375e09f4cc6397837fbb494c6c3cb1bca7091b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 918
zcmV;H18Mw6Nk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000Pc0R0JUB_%}&DWwD|R0041gb)pLC=}AL
zls9X=AOMKruT(=N_cWfH(~uHF0760{c(<lu3=9nc)H-(&K|=&E0cg1=MG4EBCrkBe
s*lOrTj{xSpGkA_rNq-a(P&)UHl1+{RPsRiwQDH|VAwmdAC@>%ZJL2W6v;Y7A

diff --git a/doc/manual/images/callouts/9.gif b/doc/manual/images/callouts/9.gif
deleted file mode 100644
index da12a4fe2825716c78f2d23c2f87afde98d3dd3e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 923
zcmV;M17!S1Nk%w1VGIBa0O$Vz000010RaL60s{jB1Ox;H1qB8M1_uWR2nYxX2?+`c
z3JVJh3=9kn4Gj(s4i66x5D*X%5fKs+5)%^>6ciK{6%`g178e&67#J8C85tTH8XFrM
z92^`S9UUGX9v>ecARr(iAt53nA|oRsBqSsyB_$>%CMPE+C@3f?DJd!{Dl021EG#T7
zEiEoCE-x=HFfcGNF)=bSGBYzXG&D3dH8nOiHa9mnI5;>tIXOByIy*Z%JUl!-Jv}}?
zK0iM{KtMo2K|w-7LPJACL_|bIMMXwNMn^|SNJvOYNl8jdN=r*iOiWBoO-)WtPESuy
zP*6}&QBhJ-Qd3h?R8&+|RaI72R##V7SXfwDSy@_IT3cINTwGjTU0q&YUSD5dU|?Wj
zVPRroVq;@tWMpJzWo2e&W@l$-XlQ6@X=!R|YHMq2Y;0_8ZEbFDZf|dIaBy&OadC2T
za&vQYbaZreb#-=jc6WDoczAeud3kzzdV70&e0+R;eSLm@et&;|fPjF3fq{a8f`fyD
zgoK2Jg@uNOhKGlTh=_=ZiHVAeii?YjjEszpjg5|uj*pLzkdTm(k&%*;l9Q8@l$4Z}
zm6ev3mY0{8n3$NEnVFiJnwy)OoSdAUot>VZo}ZteprD|kp`oIpqNAguq@<*!rKP5(
zrl+T;sHmu^si~@}s;jH3tgNi9t*x%EuCK4Ju&}VPv9YqUva_?Zw6wIfwY9dkwzs#p
zxVX5vxw*Q!y1To(yu7@<y}iD^zQ4b}z`(%4!NJ19!o$PE#KgqK#l^<P#>dCU$jHda
z$;ryf%FD~k%*@Qq&CSlv&d<-!(9qD)(b3Y<($mw^)YR0~)z#M4*4Nk9*x1<F+1c9K
z+S}XP+}zyV-QC{a-rwKf;Nall;o;)q;^X7v<mBY#<>lt)=I7_<=;-L_>FMg~>g((4
z?Ck9A?d|UF?(gsK@bK{Q@$vHV^7Hfa^z`)g_4W4l_V@Sq`1ttw`T6?#`uqF){QUg=
z{r&#_{{R2~A^8LW0Ra90EC2ui01N;O000Ph0R0JUB_%}&DXj!56aoMM1PBdtC{zLf
zK`U$LN=5Y7s^J`O2r%5!(vT8G0Hrztu%;0}4h#+VfE3xQVaSFMdc8Y$u9i)PHW47O
xMyX+pmO}1@Q1UNFLq`6b0%-N`-I|6e6vnXG5I_uX3jI-v0Rce_r~(5506VSIvSt7P

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index b4e7f35d501..ce05c62348a 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -7,7 +7,7 @@ XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -69,8 +69,6 @@ $(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docd
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

From c0a3ff7d47f5a0d0322689d446d71e2372de37d1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 16:39:35 +0100
Subject: [PATCH 1844/2196] Fix macOS

---
 release.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/release.nix b/release.nix
index 45c2f14ba9a..b1cb1e437aa 100644
--- a/release.nix
+++ b/release.nix
@@ -340,8 +340,7 @@ let
 
     installerScript =
       pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
+        { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
         ''
           mkdir -p $out/nix-support
 

From b6d794fb8df09b9b5d86f3fe2a2789631215f8b5 Mon Sep 17 00:00:00 2001
From: YI <uuuuuu@protonmail.com>
Date: Sun, 1 Mar 2020 16:11:22 +0800
Subject: [PATCH 1845/2196] display attr-path only when queried available

---
 doc/manual/command-ref/nix-env.xml | 3 ++-
 src/nix-env/nix-env.cc             | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index d257a5e49c7..9c03ccce11c 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -1066,7 +1066,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index a40d0c7e6e5..106dfe0b6d8 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -914,6 +914,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
+    if (printAttrPath && source != sAvailable)
+        throw UsageError("--attr-path(-P) only works with --available");
 
     /* Obtain derivation information from the specified source. */
     DrvInfos availElems, installedElems;

From 7c39201bcb181653af2bafee2fc02d1bcc58691b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 14:05:49 +0100
Subject: [PATCH 1846/2196] Remove the tarball job

Source tarballs are not very useful anymore. People who want to build
from source can also just build from the Git repository. Once upon a
time, the source tarball also saved users from needing a few
dependencies (e.g. bison and flex) but those are dwarfed by the other
dependencies, so it's no longer worth it.

Note: the release script should be updated to copy the vendoredCrates
tarball.
---
 release-common.nix |   6 +--
 release.nix        | 102 +++++++++++++++++++--------------------------
 shell.nix          |   2 +-
 3 files changed, 45 insertions(+), 65 deletions(-)

diff --git a/release-common.nix b/release-common.nix
index eb35d791775..b6c8f3d818d 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -34,7 +34,7 @@ rec {
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
-  tarballDeps =
+  buildDeps =
     [ bison
       flex
       libxml2
@@ -43,10 +43,8 @@ rec {
       docbook_xsl_ns
       autoconf-archive
       autoreconfHook
-    ];
 
-  buildDeps =
-    [ curl
+      curl
       bzip2 xz brotli zlib editline
       openssl pkgconfig sqlite
       libarchive
diff --git a/release.nix b/release.nix
index b1cb1e437aa..be9f8b98174 100644
--- a/release.nix
+++ b/release.nix
@@ -8,11 +8,14 @@ let
 
   pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
+  version =
+    builtins.readFile ./.version
+    + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
+
   jobs = rec {
 
     # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock, and include it in the Nix tarball. This allows Nix
-    # to be built without network access.
+    # Cargo.lock. This allows Nix to be built without network access.
     vendoredCrates =
       let
         lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
@@ -55,48 +58,6 @@ let
         '';
 
 
-    tarball =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
-
-        buildInputs = tarballDeps ++ buildDeps ++ propagatedDeps;
-
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
-        '';
-
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
-
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -105,20 +66,21 @@ let
 
       with import ./release-common.nix { inherit pkgs; };
 
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
+      stdenv.mkDerivation {
+        name = "nix-${version}";
 
-        outputs = [ "out" "dev" ];
+        src = nix;
+
+        outputs = [ "out" "dev" "doc" ];
 
         buildInputs = buildDeps;
 
         propagatedBuildInputs = propagatedDeps;
 
         preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
           ''
+            # Copy libboost_context so we don't get all of Boost in our closure.
+            # https://github.com/NixOS/nixpkgs/issues/45462
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
@@ -126,6 +88,10 @@ let
               chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
+
+            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+
+            (cd perl; autoreconf --install --force --verbose)
           '';
 
         configureFlags = configureFlags ++
@@ -137,10 +103,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        doCheck = true;
+
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
 
         separateDebugInfo = true;
+
+        preDist = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
       });
 
 
@@ -149,11 +122,21 @@ let
       let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
+        name = "nix-perl-${version}";
+
+        src = nix;
 
         buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          [ autoconf-archive
+            autoreconfHook
+            jobs.build.${system}
+            curl
+            bzip2
+            xz
+            pkgconfig
+            pkgs.perl
+            boost
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -173,7 +156,6 @@ let
 
       let
         toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
         installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
@@ -243,8 +225,9 @@ let
       with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
+        name = "nix-coverage-${version}";
+
+        src = nix;
 
         enableParallelBuilding = true;
 
@@ -349,7 +332,7 @@ let
               (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
               systems
             } \
-            --replace '@nixVersion@' ${build.${builtins.head systems}.src.version}
+            --replace '@nixVersion@' ${version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
@@ -357,11 +340,10 @@ let
 
     # Aggregate job containing the release-critical jobs.
     release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
+      name = "nix-${version}";
       meta.description = "Release-critical builds";
       constituents =
-        [ tarball
-          build.i686-linux
+        [ build.i686-linux
           build.x86_64-darwin
           build.x86_64-linux
           build.aarch64-linux
diff --git a/shell.nix b/shell.nix
index 0cabcc15ad8..e3b422c7c9d 100644
--- a/shell.nix
+++ b/shell.nix
@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ propagatedDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

From 5392884eb17db70a9fb2aa67fae486cda3c7ce92 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 18:34:10 +0100
Subject: [PATCH 1847/2196] Remove the 'release' job

Unless the 'tested' job in the Nixpkgs/NixOS jobsets, this job isn't
actually used for anything (e.g. we don't update a channel based on
whether 'release' succeeds).
---
 release.nix | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/release.nix b/release.nix
index be9f8b98174..9b591229ada 100644
--- a/release.nix
+++ b/release.nix
@@ -337,29 +337,6 @@ let
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
 
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
   };
 
 

From 59c37112a92b66663ac32a328f7b547f43bb2e72 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 13 Mar 2020 18:42:53 +0100
Subject: [PATCH 1848/2196] README.md: Remove reference to OpenSSL

The OpenSSL files were removed in a6ca68a70c3bc0b2e6abad70346c99642a896f9f.

https://salsa.debian.org/debian/nix/issues/3
---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index a953c0f7159..61054f8f2bc 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,3 @@ of the manual. It helps you to get started with building Nix from source.
 ## License
 
 Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](https://www.OpenSSL.org/).

From 68fe0d9809a269341901d95f33aebeacbed34afc Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 13 Mar 2020 21:24:35 -0400
Subject: [PATCH 1849/2196] Add missing `#include <regex>`

---
 src/libexpr/eval.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f3e8a34bd75..eac53b89450 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -7,6 +7,7 @@
 #include "hash.hh"
 #include "config.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>

From b244e65cdbc2949af70bd539bf8f3bd2fa952d07 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Mar 2020 13:50:01 +0100
Subject: [PATCH 1850/2196] nix repl: Scan NixRepl for GC roots

Fixes #3175.
---
 src/nix/repl.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index e6c811a7bef..49418add20b 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -32,6 +32,9 @@ extern "C" {
 #include "command.hh"
 #include "finally.hh"
 
+#define GC_INCLUDE_NEW
+#include <gc/gc_cpp.h>
+
 namespace nix {
 
 #define ESC_RED "\033[31m"
@@ -42,7 +45,7 @@ namespace nix {
 #define ESC_CYA "\033[36m"
 #define ESC_END "\033[0m"
 
-struct NixRepl
+struct NixRepl : gc
 {
     string curDir;
     EvalState state;

From ef74fafc0368944e6cfc3b804b4bcdddd6bcf9c0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 19 Mar 2020 13:52:28 +0100
Subject: [PATCH 1851/2196] nix repl: Put EvalState on the heap

See 0629601da1d163a059fa19004256961f8ecdeb78.
---
 src/nix/repl.cc | 64 ++++++++++++++++++++++++-------------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 49418add20b..27727bd2550 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -48,7 +48,7 @@ namespace nix {
 struct NixRepl : gc
 {
     string curDir;
-    EvalState state;
+    std::unique_ptr<EvalState> state;
     Bindings * autoArgs;
 
     Strings loadedFiles;
@@ -126,8 +126,8 @@ string removeWhitespace(string s)
 
 
 NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store)
-    : state(searchPath, store)
-    , staticEnv(false, &state.staticBaseEnv)
+    : state(std::make_unique<EvalState>(searchPath, store))
+    , staticEnv(false, &state->staticBaseEnv)
     , historyFile(getDataDir() + "/nix/repl-history")
 {
     curDir = absPath(".");
@@ -356,8 +356,8 @@ StringSet NixRepl::completePrefix(string prefix)
 
             Expr * e = parseString(expr);
             Value v;
-            e->eval(state, *env, v);
-            state.forceAttrs(v);
+            e->eval(*state, *env, v);
+            state->forceAttrs(v);
 
             for (auto & i : *v.attrs) {
                 string name = i.name;
@@ -412,11 +412,11 @@ bool isVarName(const string & s)
 
 
 Path NixRepl::getDerivationPath(Value & v) {
-    auto drvInfo = getDerivation(state, v, false);
+    auto drvInfo = getDerivation(*state, v, false);
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
     Path drvPath = drvInfo->queryDrvPath();
-    if (drvPath == "" || !state.store->isValidPath(state.store->parseStorePath(drvPath)))
+    if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath)))
         throw Error("expression did not evaluate to a valid derivation");
     return drvPath;
 }
@@ -462,12 +462,12 @@ bool NixRepl::processLine(string line)
     }
 
     else if (command == ":l" || command == ":load") {
-        state.resetFileCache();
+        state->resetFileCache();
         loadFile(arg);
     }
 
     else if (command == ":r" || command == ":reload") {
-        state.resetFileCache();
+        state->resetFileCache();
         reloadFiles();
     }
 
@@ -479,13 +479,13 @@ bool NixRepl::processLine(string line)
 
         if (v.type == tPath || v.type == tString) {
             PathSet context;
-            auto filename = state.coerceToString(noPos, v, context);
-            pos.file = state.symbols.create(filename);
+            auto filename = state->coerceToString(noPos, v, context);
+            pos.file = state->symbols.create(filename);
         } else if (v.type == tLambda) {
             pos = v.lambda.fun->pos;
         } else {
             // assume it's a derivation
-            pos = findDerivationFilename(state, v, arg);
+            pos = findDerivationFilename(*state, v, arg);
         }
 
         // Open in EDITOR
@@ -495,7 +495,7 @@ bool NixRepl::processLine(string line)
         runProgram(editor, args);
 
         // Reload right after exiting the editor
-        state.resetFileCache();
+        state->resetFileCache();
         reloadFiles();
     }
 
@@ -508,7 +508,7 @@ bool NixRepl::processLine(string line)
         Value v, f, result;
         evalString(arg, v);
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
-        state.callFunction(f, v, result, Pos());
+        state->callFunction(f, v, result, Pos());
 
         Path drvPath = getDerivationPath(result);
         runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
@@ -524,10 +524,10 @@ bool NixRepl::processLine(string line)
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
             if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto drv = readDerivation(*state.store, drvPath);
+                auto drv = readDerivation(*state->store, drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
                 for (auto & i : drv.outputs)
-                    std::cout << fmt("  %s -> %s\n", i.first, state.store->printStorePath(i.second.path));
+                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.path));
             }
         } else if (command == ":i") {
             runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
@@ -557,11 +557,11 @@ bool NixRepl::processLine(string line)
             isVarName(name = removeWhitespace(string(line, 0, p))))
         {
             Expr * e = parseString(string(line, p + 1));
-            Value & v(*state.allocValue());
+            Value & v(*state->allocValue());
             v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = e;
-            addVarToScope(state.symbols.create(name), v);
+            addVarToScope(state->symbols.create(name), v);
         } else {
             Value v;
             evalString(line, v);
@@ -578,21 +578,21 @@ void NixRepl::loadFile(const Path & path)
     loadedFiles.remove(path);
     loadedFiles.push_back(path);
     Value v, v2;
-    state.evalFile(lookupFileArg(state, path), v);
-    state.autoCallFunction(*autoArgs, v, v2);
+    state->evalFile(lookupFileArg(*state, path), v);
+    state->autoCallFunction(*autoArgs, v, v2);
     addAttrsToScope(v2);
 }
 
 
 void NixRepl::initEnv()
 {
-    env = &state.allocEnv(envSize);
-    env->up = &state.baseEnv;
+    env = &state->allocEnv(envSize);
+    env->up = &state->baseEnv;
     displ = 0;
     staticEnv.vars.clear();
 
     varNames.clear();
-    for (auto & i : state.staticBaseEnv.vars)
+    for (auto & i : state->staticBaseEnv.vars)
         varNames.insert(i.first);
 }
 
@@ -616,7 +616,7 @@ void NixRepl::reloadFiles()
 
 void NixRepl::addAttrsToScope(Value & attrs)
 {
-    state.forceAttrs(attrs);
+    state->forceAttrs(attrs);
     for (auto & i : *attrs.attrs)
         addVarToScope(i.name, *i.value);
     std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
@@ -635,7 +635,7 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v)
 
 Expr * NixRepl::parseString(string s)
 {
-    Expr * e = state.parseExprFromString(s, curDir, staticEnv);
+    Expr * e = state->parseExprFromString(s, curDir, staticEnv);
     return e;
 }
 
@@ -643,8 +643,8 @@ Expr * NixRepl::parseString(string s)
 void NixRepl::evalString(string s, Value & v)
 {
     Expr * e = parseString(s);
-    e->eval(state, *env, v);
-    state.forceValue(v);
+    e->eval(*state, *env, v);
+    state->forceValue(v);
 }
 
 
@@ -674,7 +674,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     str.flush();
     checkInterrupt();
 
-    state.forceValue(v);
+    state->forceValue(v);
 
     switch (v.type) {
 
@@ -703,13 +703,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     case tAttrs: {
         seen.insert(&v);
 
-        bool isDrv = state.isDerivation(v);
+        bool isDrv = state->isDerivation(v);
 
         if (isDrv) {
             str << "«derivation ";
-            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+            Bindings::iterator i = v.attrs->find(state->sDrvPath);
             PathSet context;
-            Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???";
+            Path drvPath = i != v.attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "???";
             str << drvPath << "»";
         }
 
@@ -812,7 +812,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
     void run(ref<Store> store) override
     {
         auto repl = std::make_unique<NixRepl>(searchPath, openStore());
-        repl->autoArgs = getAutoArgs(repl->state);
+        repl->autoArgs = getAutoArgs(*repl->state);
         repl->mainLoop(files);
     }
 };

From c5a488afc05372dd46e05559ef04ac9969808507 Mon Sep 17 00:00:00 2001
From: jakobrs <jakobrs100@gmail.com>
Date: Thu, 19 Mar 2020 19:15:55 +0100
Subject: [PATCH 1852/2196] Remove the --delete option for --gc

Running `nix-store --gc --delete` will, as of Nix 2.3.3, simply fail
because the --delete option conflicts with the --delete operation.

  $ nix-store --gc --delete
  error: only one operation may be specified
  Try 'nix-store --help' for more information.

Furthermore, it has been broken since at least Nix 0.16 (which was
released sometime in 2010), which means that any scripts which depend
on it should have been broken at least nine years ago. This commit
simply formally removes the option. There should be no actual difference
in behaviour as far as the user is concerned: it errors with the exact
same error message. The manual has been edited to remove any references
to the (now gone) --delete option.

Other information:
* Path for Nix 0.16 used:
  /nix/store/rp3sgmskn0p0pj1ia2qwd5al6f6pinz4-nix-0.16
---
 doc/manual/command-ref/nix-store.xml | 17 ++++-------------
 src/nix-store/nix-store.cc           |  1 -
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 113a3c2e41e..e8bbd16e849 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 45e152c47ef..c14d900aead 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -577,7 +577,6 @@ static void opGC(Strings opFlags, Strings opArgs)
         if (*i == "--print-roots") printRoots = true;
         else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
         else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
-        else if (*i == "--delete") options.action = GCOptions::gcDeleteDead;
         else if (*i == "--max-freed") {
             long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
             options.maxFreed = maxFreed >= 0 ? maxFreed : 0;

From afb78ebd34bff9a701d70041abc2ff211390584e Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 20 Mar 2020 21:21:56 +0100
Subject: [PATCH 1853/2196] libstore: disable resolve-system-dependencies hook

This is used to determine the dependency tree of impure libraries so nix
knows what paths to open in the sandbox.  With the less restrictive
defaults it isn't needed anymore.
---
 src/libstore/globals.hh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 782870547cb..3aa3653f31d 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -311,12 +311,7 @@ public:
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this,
-#if __APPLE__
-        nixLibexecDir + "/nix/resolve-system-dependencies",
-#else
-        "",
-#endif
+    Setting<std::string> preBuildHook{this, "",
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 

From 7f2df903d91cd21ab05223344ee4dec0a7d52c41 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 20 Mar 2020 21:31:20 +0100
Subject: [PATCH 1854/2196] libstore: relax default sandbox-paths on darwin

---
 src/libstore/globals.cc | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index a8945996e7a..1a2fcbe2236 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -20,13 +20,6 @@ namespace nix {
    must be deleted and recreated on startup.) */
 #define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
-/* chroot-like behavior from Apple's sandbox */
-#if __APPLE__
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
-#else
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
-#endif
-
 Settings settings;
 
 static GlobalConfig::Register r1(&settings);
@@ -68,7 +61,12 @@ Settings::Settings()
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
 #endif
 
-    allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
+
+/* chroot-like behavior from Apple's sandbox */
+#if __APPLE__
+    sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib");
+    allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
+#endif
 }
 
 void loadConfFile()

From f6c122aaeb08cc3d9e89465b440b25c7e0c87d9e Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 20 Mar 2020 21:58:45 +0100
Subject: [PATCH 1855/2196] sandbox: allow pty devices

Nix now runs builds with a pseudo-terminal to enable colored build
output.
---
 src/libstore/sandbox-defaults.sb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index 0299d1ee45d..c09ce17296f 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -71,6 +71,12 @@
        (literal "/dev/zero")
        (subpath "/dev/fd"))
 
+; Allow pseudo-terminals.
+(allow file*
+       (literal "/dev/ptmx")
+       (regex #"^/dev/pty[a-z]+")
+       (regex #"^/dev/ttys[0-9]+"))
+
 ; Does nothing, but reduces build noise.
 (allow file* (literal "/dev/dtracehelper"))
 

From 2e9bc1245c125f96ce53210751940067d4cf3f1c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 20 Mar 2020 22:12:30 +0100
Subject: [PATCH 1856/2196] sandbox: fix /bin/sh on catalina

Sadly 10.15 changed /bin/sh to a shim which executes bash, this means it
can't be used anymore without also opening up the sandbox to allow bash.

    Failed to exec /bin/bash as variant for /bin/sh (1: Operation not permitted).
---
 src/libstore/globals.cc          | 2 +-
 src/libstore/sandbox-defaults.sb | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 1a2fcbe2236..7e97f3c2235 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -64,7 +64,7 @@ Settings::Settings()
 
 /* chroot-like behavior from Apple's sandbox */
 #if __APPLE__
-    sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib");
+    sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
     allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
 #endif
 }
diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index c09ce17296f..351037822fd 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -91,3 +91,7 @@
        (literal "/etc")
        (literal "/var")
        (literal "/private/var/tmp"))
+
+; This is used by /bin/sh on macOS 10.15 and later.
+(allow file*
+       (literal "/private/var/select/sh"))

From 9450dece24edd34613d887c7c0a1f2e58d86ecb4 Mon Sep 17 00:00:00 2001
From: Philipp Middendorf <middendorf@plapadoo.de>
Date: Sat, 21 Mar 2020 09:31:39 +0100
Subject: [PATCH 1857/2196] installer: also test for xz to unpack

---
 scripts/install.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/install.in b/scripts/install.in
index 4500fd54a2e..6709f00d4a8 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -36,6 +36,7 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

From f694f43d7ddd270706e6e3a5f034336a5a83fb86 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 22 Mar 2020 12:25:47 -0600
Subject: [PATCH 1858/2196] straightforward port of rust mockup code

---
 src/libutil/error.cc | 137 +++++++++++++++++++++++++++++++++++++++++++
 src/libutil/error.hh |  57 ++++++++++++++++++
 2 files changed, 194 insertions(+)
 create mode 100644 src/libutil/error.cc
 create mode 100644 src/libutil/error.hh

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
new file mode 100644
index 00000000000..4af4691d486
--- /dev/null
+++ b/src/libutil/error.cc
@@ -0,0 +1,137 @@
+#include "error.hh"
+
+#include <iostream>
+
+namespace nix {
+
+using std::cout;
+using std::endl;
+
+// return basic_format?
+string showErrLine(ErrLine &errLine) 
+{
+  if (errLine.columnRange.has_value()) 
+  {
+    return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
+  }
+  else
+  {
+    return (format("(%1%)") % errLine.lineNumber).str();
+  };
+}
+
+void print_code_lines(string &prefix, NixCode &nix_code) 
+{
+  if (nix_code.errLine.has_value()) 
+  {
+    // previous line of code.
+    if (nix_code.errLine->prevLineOfCode.has_value()) { 
+      cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber - 1) % *nix_code.errLine->prevLineOfCode;
+    }
+
+    // line of code containing the error.
+    cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber) % nix_code.errLine->errLineOfCode;
+
+    // error arrows for the column range.
+    if (nix_code.errLine->columnRange.has_value()) 
+    {
+      int start = nix_code.errLine->columnRange->start;
+      std::string spaces;
+      for (int i = 0; i < start; ++i)
+      {
+        spaces.append(" ");
+      }
+
+      int len = nix_code.errLine->columnRange->len;
+      std::string arrows;
+      for (int i = 0; i < len; ++i)
+      {
+        arrows.append("^");
+      }
+    
+      cout << format("%1%     |%2%%3%") % prefix % spaces % arrows;
+    }
+
+
+    // next line of code.
+    if (nix_code.errLine->nextLineOfCode.has_value()) { 
+      cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber + 1) % *nix_code.errLine->nextLineOfCode;
+    }
+
+  }
+  else 
+  {
+  }
+    
+}
+
+void print_error(ErrorInfo &einfo) 
+{
+  int errwidth = 80;
+  string prefix = "  ";
+
+  string level_string;
+  switch (einfo.level) 
+  {
+    case ErrLevel::elError: 
+      {
+        level_string = "error:";  // TODO make red.
+        break;
+      }
+    case ErrLevel::elWarning: 
+      {
+        level_string = "warning:";  // TODO make yellow.
+        break;
+      }
+  }
+
+  int ndl = level_string.length() + 3 + einfo.errName.length() + einfo.toolName.length();
+  int dashwidth = errwidth - 3 ? 3 : 80 - ndl; 
+
+  string dashes;
+  for (int i = 0; i < dashwidth; ++i)
+    dashes.append("-");
+
+  // divider.
+  cout << format("%1%%2% %3% %4% %5% %6%")
+    % prefix
+    % level_string
+    % "---"
+    % einfo.errName
+    % dashes
+    % einfo.toolName;
+
+  // filename.
+  if (einfo.nixCode.has_value()) 
+  {
+    if (einfo.nixCode->nixFile.has_value()) 
+    {
+      string eline = einfo.nixCode->errLine.has_value()  
+        ? string(" ") + showErrLine(*einfo.nixCode->errLine)
+        : "";
+
+      cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline;
+      cout << prefix << endl;
+    }
+    else
+    {
+      cout << format("%1%from command line argument") % prefix << endl;
+      cout << prefix << endl;
+    }
+  }
+
+  // description
+  cout << prefix << einfo.description << endl;
+  cout << prefix << endl;
+
+  // lines of code.
+  if (einfo.nixCode.has_value())
+    print_code_lines(prefix, *einfo.nixCode);
+
+  // hint
+  cout << prefix << einfo.hint << endl;
+  cout << prefix << endl;
+
+}
+
+}
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
new file mode 100644
index 00000000000..0bc1f4e2439
--- /dev/null
+++ b/src/libutil/error.hh
@@ -0,0 +1,57 @@
+#pragma once
+
+#include "types.hh"
+#include <string>
+#include <optional>
+// #include <boost/format.hpp>
+
+using std::string;
+using std::optional;
+// using boost::format;
+
+namespace nix {
+
+enum ErrLevel 
+  { elWarning
+  , elError
+  };
+
+class ColumnRange { 
+  public:
+  unsigned int start;
+  unsigned int len;
+};
+
+class ErrLine { 
+  public:
+  int lineNumber;
+  optional<ColumnRange> columnRange;
+  optional<string> prevLineOfCode;
+  string errLineOfCode;
+  optional<string> nextLineOfCode;
+};
+
+class NixCode { 
+  public:
+  optional<string> nixFile;
+  optional<ErrLine> errLine;
+};
+
+
+class ErrorInfo { 
+  public:
+  ErrLevel level;
+  string errName;
+  string description;
+  string toolName;
+  optional<NixCode> nixCode;
+  string hint;
+};
+
+string showErrLine(ErrLine &errLine);
+
+void print_code_lines(string &prefix, NixCode &nix_code); 
+
+void print_error(ErrorInfo &einfo);
+}
+

From 26851dd2c22690838d391ef85b90a99fc00bf9ea Mon Sep 17 00:00:00 2001
From: Greg Price <gnprice@gmail.com>
Date: Sat, 21 Mar 2020 22:03:58 -0700
Subject: [PATCH 1859/2196] installer: Set files read-only when copying into
 store

After installing Nix, I found that all the files and directories
initially copied into the store were writable, with mode 644 or 755:

  drwxr-xr-x 9 root root 4096 Dec 31  1969 /nix/store/ddmmzn4ggz1f66lwxjy64n89864yj9w9-nix-2.3.3

The reason is that that's how they were in the unpacked tarball, and
the install-multi-user script used `rsync -p` without doing anything
else to affect the permissions.

The plain `install` script for a single-user install takes care to
do a `chmod -R a-w` on each store path copied.  We could do the same
here with one more command; or we can pass `--chmod` to rsync, to
have it write the files with the desired modes in the first place.

Tested the new `rsync` command on both a Linux machine with a
reasonably-modern rsync (3.1.3) and a Mac with its default, ancient,
rsync 2.6.9, and it works as expected on both.  Thankfully the latter
is just new enough to have `--chmod`, which dates to rsync 2.6.7.
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 13762cba389..35341543e75 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -567,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

From aadd59d005bdabbb801f622dc1a0e3b12a5e5286 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 23 Mar 2020 15:29:49 -0600
Subject: [PATCH 1860/2196] error test

---
 src/libutil/error.cc | 36 +++++++++++++++++++++++++-----------
 src/libutil/error.hh |  2 --
 tests/errors/main.cc | 39 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 13 deletions(-)
 create mode 100644 tests/errors/main.cc

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 4af4691d486..71f422622a9 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -8,7 +8,7 @@ using std::cout;
 using std::endl;
 
 // return basic_format?
-string showErrLine(ErrLine &errLine) 
+string showErrLine(ErrLine &errLine)
 {
   if (errLine.columnRange.has_value()) 
   {
@@ -22,15 +22,26 @@ string showErrLine(ErrLine &errLine)
 
 void print_code_lines(string &prefix, NixCode &nix_code) 
 {
+  
   if (nix_code.errLine.has_value()) 
   {
     // previous line of code.
     if (nix_code.errLine->prevLineOfCode.has_value()) { 
-      cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber - 1) % *nix_code.errLine->prevLineOfCode;
+      cout << format("%1% %|2$5d|| %3%")
+        %  prefix
+        % (nix_code.errLine->lineNumber - 1)
+        % *nix_code.errLine->prevLineOfCode
+        << endl;
     }
 
-    // line of code containing the error.
-    cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber) % nix_code.errLine->errLineOfCode;
+    // line of code containing the error.%2$+5d%
+    cout << format("%1% %|2$5d|| %3%")
+      %  prefix
+      % (nix_code.errLine->lineNumber)
+      % nix_code.errLine->errLineOfCode
+      << endl;
+    
+
 
     // error arrows for the column range.
     if (nix_code.errLine->columnRange.has_value()) 
@@ -49,19 +60,21 @@ void print_code_lines(string &prefix, NixCode &nix_code)
         arrows.append("^");
       }
     
-      cout << format("%1%     |%2%%3%") % prefix % spaces % arrows;
+      cout << format("%1%      |%2%%3%") % prefix % spaces % arrows;
     }
 
 
+
     // next line of code.
     if (nix_code.errLine->nextLineOfCode.has_value()) { 
-      cout << format("%1% %2$+5d%| %3%") %  prefix % (nix_code.errLine->lineNumber + 1) % *nix_code.errLine->nextLineOfCode;
+      cout << format("%1% %|2$5d|| %3%")
+      %  prefix
+      % (nix_code.errLine->lineNumber + 1)
+      % *nix_code.errLine->nextLineOfCode
+      << endl;
     }
 
   }
-  else 
-  {
-  }
     
 }
 
@@ -99,7 +112,8 @@ void print_error(ErrorInfo &einfo)
     % "---"
     % einfo.errName
     % dashes
-    % einfo.toolName;
+    % einfo.toolName
+    << endl;
 
   // filename.
   if (einfo.nixCode.has_value()) 
@@ -110,7 +124,7 @@ void print_error(ErrorInfo &einfo)
         ? string(" ") + showErrLine(*einfo.nixCode->errLine)
         : "";
 
-      cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline;
+      cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline << endl;
       cout << prefix << endl;
     }
     else
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 0bc1f4e2439..8dac1508e5c 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -3,11 +3,9 @@
 #include "types.hh"
 #include <string>
 #include <optional>
-// #include <boost/format.hpp>
 
 using std::string;
 using std::optional;
-// using boost::format;
 
 namespace nix {
 
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
new file mode 100644
index 00000000000..1c998103e20
--- /dev/null
+++ b/tests/errors/main.cc
@@ -0,0 +1,39 @@
+#include "error.hh"
+
+#include <optional>
+#include <iostream>
+
+using std::optional;
+
+int main() {
+
+using namespace nix; 
+
+  ColumnRange columnRange;
+  columnRange.start = 24;
+  columnRange.len = 14;
+
+  ErrLine errline;
+  errline.lineNumber = 7;
+  errline.columnRange = optional(columnRange);
+  errline.errLineOfCode = "line of code where the error occurred";
+
+  NixCode nixcode;
+  nixcode.nixFile = optional("myfile.nix");
+  nixcode.errLine = errline; 
+  
+  ErrorInfo generic;
+  generic.level = elError;
+  generic.errName = "error name";
+  generic.description = "general error description";
+  generic.toolName = "nixtool.exe";
+  generic.nixCode = nixcode;
+
+  print_error(generic);
+
+  return 0;
+}
+
+
+
+

From e40e01c1dd58c12a763d9614ff38dd7a984c9787 Mon Sep 17 00:00:00 2001
From: Greg Price <gnprice@gmail.com>
Date: Sat, 21 Mar 2020 22:20:02 -0700
Subject: [PATCH 1861/2196] doc: Files in the store have modes 444/555, not
 644/755

This line has been this way since it was written, in 9e08f5efe
in 2006.

I think it was just a small mistake then; Eelco's thesis earlier
that year says the permission on each file is set to 0444 or 0555
in a derivation's output as part of the build process.  In any
case I'm pretty sure that's the behavior now.
---
 doc/manual/command-ref/nix-store.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index e8bbd16e849..1ddb5408d5d 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -1139,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

From c34a20e1f60486140f0ddc5657c077d2fdc9c7f5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 26 Nov 2018 19:55:48 +0100
Subject: [PATCH 1862/2196] EvalState::allocAttr(): Add convenience method

(cherry picked from commit c02da997570ac0d9b595d787bea8cb5a4e3cc1f5)
---
 src/libexpr/attr-set.cc | 6 ++++++
 src/libexpr/eval.hh     | 1 +
 2 files changed, 7 insertions(+)

diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index 0785897d251..b1d61a28519 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index eac53b89450..5efad870775 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -272,6 +272,7 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
 
     Bindings * allocBindings(size_t capacity);
 

From 5a7e7fc35f934872a9e3e9e9043ff6894b36637a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 17 Feb 2020 15:53:59 +0100
Subject: [PATCH 1863/2196] Use std::string_view

(cherry picked from commit 6529490cc10018d5191e50c482ac1180b96b1a3c)
---
 src/libstore/derivations.cc | 6 +++---
 src/libstore/derivations.hh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 205b90e55ed..973ddc86abf 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair)
+    const Derivation & drv, std::string_view name, RepairFlag repair)
 {
     auto references = cloneStorePathSet(drv.inputSrcs);
     for (auto & i : drv.inputDrvs)
@@ -73,8 +73,8 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    string suffix = name + drvExtension;
-    string contents = drv.unparse(*store, false);
+    auto suffix = std::string(name) + drvExtension;
+    auto contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references, repair);
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index c2df66229ad..7222d25e5c3 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -79,7 +79,7 @@ class Store;
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
+    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Store & store, const Path & drvPath);

From edc34cc1a2865777cee6b325f705fb218955b593 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 May 2019 21:09:52 +0200
Subject: [PATCH 1864/2196] Add function for quoting strings

(cherry picked from commit 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7)
---
 src/libstore/build.cc     |  2 +-
 src/libstore/store-api.cc |  7 +------
 src/libutil/util.cc       | 22 ----------------------
 src/libutil/util.hh       | 22 ++++++++++++++++++++--
 4 files changed, 22 insertions(+), 31 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9c6aedfa51c..7efa81a62b7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1904,7 +1904,7 @@ void DerivationGoal::startBuilder()
             concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
             worker.store.printStorePath(drvPath),
             settings.thisSystem,
-            concatStringsSep(", ", settings.systemFeatures));
+            concatStringsSep<StringSet>(", ", settings.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b043feb0afc..aacbe5e0bfe 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -741,12 +741,7 @@ std::string Store::showPaths(const StorePathSet & paths)
 
 string showPaths(const PathSet & paths)
 {
-    string s;
-    for (auto & i : paths) {
-        if (s.size() != 0) s += ", ";
-        s += "'" + i + "'";
-    }
-    return s;
+    return concatStringsSep(", ", quoteStrings(paths));
 }
 
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 012f1d07139..52a1394f20d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1205,28 +1205,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
 template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
-string concatStringsSep(const string & sep, const Strings & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
-string concatStringsSep(const string & sep, const StringSet & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
 string chomp(const string & s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3bfebcd153a..559ed4332d2 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -345,8 +345,26 @@ template<class C> C tokenizeString(std::string_view s, const string & separators
 
 /* Concatenate the given strings with a separator between the
    elements. */
-string concatStringsSep(const string & sep, const Strings & ss);
-string concatStringsSep(const string & sep, const StringSet & ss);
+template<class C>
+string concatStringsSep(const string & sep, const C & ss)
+{
+    string s;
+    for (auto & i : ss) {
+        if (s.size() != 0) s += sep;
+        s += i;
+    }
+    return s;
+}
+
+
+/* Add quotes around a collection of strings. */
+template<class C> Strings quoteStrings(const C & c)
+{
+    Strings res;
+    for (auto & s : c)
+        res.push_back("'" + s + "'");
+    return res;
+}
 
 
 /* Remove trailing whitespace from a string. */

From 1eb952d27ac0714a99a5f7b0e8c7034e7ac0bc0d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 Apr 2019 14:20:58 +0200
Subject: [PATCH 1865/2196] findAlongAttrPath(): Throw AttrPathNotFound

(cherry picked from commit 6b0ca8e803710342af70e257935724c5ad84ca04)
---
 src/libexpr/attr-path.cc | 4 ++--
 src/libexpr/attr-path.hh | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 06b472d8bfa..843585631f0 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -70,7 +70,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
             v = &*a->value;
         }
 
@@ -82,7 +82,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
         }
diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh
index 716e5ba2746..fcccc39c879 100644
--- a/src/libexpr/attr-path.hh
+++ b/src/libexpr/attr-path.hh
@@ -7,6 +7,8 @@
 
 namespace nix {
 
+MakeError(AttrPathNotFound, Error);
+
 Value * findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 

From c1ca4f0accdb96295b27dadacd20b3db745e1d2d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Feb 2020 14:08:24 +0100
Subject: [PATCH 1866/2196] findAlongAttrPath(): Return position

(cherry picked from commit 0b013a54dc570395bed887369f8dd622b8ce337b)
---
 src/libexpr/attr-path.cc                 | 9 ++++++---
 src/libexpr/attr-path.hh                 | 2 +-
 src/nix-build/nix-build.cc               | 2 +-
 src/nix-env/nix-env.cc                   | 4 ++--
 src/nix-instantiate/nix-instantiate.cc   | 2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc | 2 +-
 src/nix/installables.cc                  | 2 +-
 src/nix/upgrade-nix.cc                   | 2 +-
 8 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 843585631f0..1e22f570828 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -41,6 +41,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
         Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
     Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -72,6 +73,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
             if (a == v->attrs->end())
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
             v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
@@ -85,11 +87,12 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
 }
 
 
@@ -98,7 +101,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
     Value * v2;
     try {
         auto dummyArgs = state.allocBindings(0);
-        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
         throw Error("package '%s' has no source location information", what);
     }
diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh
index fcccc39c879..3e78e2899a8 100644
--- a/src/libexpr/attr-path.hh
+++ b/src/libexpr/attr-path.hh
@@ -9,7 +9,7 @@ namespace nix {
 
 MakeError(AttrPathNotFound, Error);
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 27ec7d0feda..1bda159d027 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -314,7 +314,7 @@ static void _main(int argc, char * * argv)
         state->eval(e, vRoot);
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot));
+            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first);
             state->forceValue(v);
             getDerivations(*state, v, "", *autoArgs, drvs, false);
         }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 106dfe0b6d8..c7e553d0cb6 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -178,7 +178,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
     Value vRoot;
     loadSourceExpr(state, nixExprPath, vRoot);
 
-    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot));
+    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first);
 
     getDerivations(state, v, pathPrefix, autoArgs, elems, true);
 
@@ -408,7 +408,7 @@ static void queryInstSources(EvalState & state,
             Value vRoot;
             loadSourceExpr(state, instSource.nixExprPath, vRoot);
             for (auto & i : args) {
-                Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot));
+                Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first);
                 getDerivations(state, v, "", *instSource.autoArgs, elems, true);
             }
             break;
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 5a886d69d4d..617d927a4c1 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -39,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
     state.eval(e, vRoot);
 
     for (auto & i : attrPaths) {
-        Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot));
+        Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first);
         state.forceValue(v);
 
         PathSet context;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index cc089181186..18ced94b1a7 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -120,7 +120,7 @@ static int _main(int argc, char * * argv)
             Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
             Value vRoot;
             state->evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot));
+            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
             state->forceAttrs(v);
 
             /* Extract the URI. */
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 8ce6bd06ebd..eb6fca62b1c 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -193,7 +193,7 @@ struct InstallableAttrPath : InstallableValue
 
         Bindings & autoArgs = *cmd.getAutoArgs(state);
 
-        Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source);
+        auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first;
         state.forceValue(*v);
 
         return v;
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 87f1f9d1b16..c05c29517b2 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -145,7 +145,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         auto v = state->allocValue();
         state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v);
         Bindings & bindings(*state->allocBindings(0));
-        auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v);
+        auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first;
 
         return store->parseStorePath(state->forceString(*v2));
     }

From 231a8aa2c2196a3ba6d6bd3dc9d6c50e23e20a8d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 7 Feb 2020 14:22:01 +0100
Subject: [PATCH 1867/2196] nix edit: Support non-derivation attributes

E.g.

  $ nix edit .#nixosConfigurations.bla

now works.

(cherry picked from commit d2032edb2f86e955a8a7724a27c0c3225f386500)
---
 src/libexpr/attr-path.cc |  2 +-
 src/libexpr/attr-path.hh |  1 +
 src/nix/command.hh       |  9 ++-------
 src/nix/edit.cc          | 10 ++++++++--
 src/nix/eval.cc          |  2 +-
 src/nix/installables.cc  | 10 +++++-----
 6 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 1e22f570828..4545bfd7262 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -103,7 +103,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
         auto dummyArgs = state.allocBindings(0);
         v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
-        throw Error("package '%s' has no source location information", what);
+        throw NoPositionInfo("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this
diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh
index 3e78e2899a8..fce160da727 100644
--- a/src/libexpr/attr-path.hh
+++ b/src/libexpr/attr-path.hh
@@ -8,6 +8,7 @@
 namespace nix {
 
 MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
 
 std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 00eb46903df..a954a7d04fe 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -3,17 +3,12 @@
 #include "args.hh"
 #include "common-eval-args.hh"
 #include "path.hh"
+#include "eval.hh"
 
 namespace nix {
 
 extern std::string programPath;
 
-struct Value;
-class Bindings;
-class EvalState;
-struct Pos;
-class Store;
-
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
@@ -48,7 +43,7 @@ struct Installable
 
     Buildable toBuildable();
 
-    virtual Value * toValue(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index ca410cd1f80..1683eada0b8 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -29,9 +29,15 @@ struct CmdEdit : InstallableCommand
     {
         auto state = getEvalState();
 
-        auto v = installable->toValue(*state);
+        auto [v, pos] = installable->toValue(*state);
 
-        Pos pos = findDerivationFilename(*state, *v, installable->what());
+        try {
+            pos = findDerivationFilename(*state, *v, installable->what());
+        } catch (NoPositionInfo &) {
+        }
+
+        if (pos == noPos)
+            throw Error("cannot find position information for '%s", installable->what());
 
         stopProgressBar();
 
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 276fdf00336..6398fc58e46 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -52,7 +52,7 @@ struct CmdEval : MixJSON, InstallableCommand
 
         auto state = getEvalState();
 
-        auto v = installable->toValue(*state);
+        auto v = installable->toValue(*state).first;
         PathSet context;
 
         stopProgressBar();
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index eb6fca62b1c..013218cd924 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -121,7 +121,7 @@ struct InstallableValue : Installable
     {
         auto state = cmd.getEvalState();
 
-        auto v = toValue(*state);
+        auto v = toValue(*state).first;
 
         Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
@@ -169,11 +169,11 @@ struct InstallableExpr : InstallableValue
 
     std::string what() override { return text; }
 
-    Value * toValue(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto v = state.allocValue();
         state.eval(state.parseExprFromString(text, absPath(".")), *v);
-        return v;
+        return {v, noPos};
     }
 };
 
@@ -187,7 +187,7 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    Value * toValue(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto source = cmd.getSourceExpr(state);
 
@@ -196,7 +196,7 @@ struct InstallableAttrPath : InstallableValue
         auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first;
         state.forceValue(*v);
 
-        return v;
+        return {v, noPos};
     }
 };
 

From 76e7d958ed8a1502cb060300093c120311d10cac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Jan 2020 21:58:07 +0100
Subject: [PATCH 1868/2196] Fix coverage build

https://hydra.nixos.org/build/110757285
(cherry picked from commit b430a81a1fbf6c792ba49e3aefe46256263430e5)
---
 src/libexpr/function-trace.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc
index af1486f7897..c6057b3842f 100644
--- a/src/libexpr/function-trace.cc
+++ b/src/libexpr/function-trace.cc
@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

From f9611c7ae48abd4506a93691c25f067a0e8e73b4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 21 Oct 2019 17:17:15 +0200
Subject: [PATCH 1869/2196] buildenv: Eliminate global variables, other cleanup

(cherry picked from commit b82f75464d1e5ae9a00d8004e5dd7b1ca05059e4)
---
 src/libstore/build.cc             |   1 +
 src/libstore/builtins.hh          |   1 -
 src/libstore/builtins/buildenv.cc | 120 ++++++++++++++----------------
 src/libstore/builtins/buildenv.hh |  21 ++++++
 4 files changed, 77 insertions(+), 66 deletions(-)
 create mode 100644 src/libstore/builtins/buildenv.hh

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 7efa81a62b7..dc6a2c60608 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "builtins/buildenv.hh"
 #include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh
index 87d6ce66533..66597e456c4 100644
--- a/src/libstore/builtins.hh
+++ b/src/libstore/builtins.hh
@@ -6,7 +6,6 @@ namespace nix {
 
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
-void builtinBuildenv(const BasicDerivation & drv);
 void builtinUnpackChannel(const BasicDerivation & drv);
 
 }
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 0965938862e..c1c85d0bf99 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -1,4 +1,4 @@
-#include "builtins.hh"
+#include "buildenv.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -7,16 +7,14 @@
 
 namespace nix {
 
-typedef std::map<Path,int> Priorities;
-
-// FIXME: change into local variables.
-
-static Priorities priorities;
-
-static unsigned long symlinks;
+struct State
+{
+    std::map<Path, int> priorities;
+    unsigned long symlinks = 0;
+};
 
 /* For each activated package, create symlinks */
-static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
+static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
     DirEntries srcFiles;
 
@@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISDIR(dstSt.st_mode)) {
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 } else if (S_ISLNK(dstSt.st_mode)) {
                     auto target = canonPath(dstFile, true);
@@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                         throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
                         throw SysError(format("creating directory '%1%'"));
-                    createLinks(target, dstFile, priorities[dstFile]);
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, target, dstFile, state.priorities[dstFile]);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
@@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISLNK(dstSt.st_mode)) {
-                    auto prevPriority = priorities[dstFile];
+                    auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
                                 "packages '%1%' and '%2%' have the same priority %3%; "
@@ -109,41 +107,57 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
         }
 
         createSymlink(srcFile, dstFile);
-        priorities[dstFile] = priority;
-        symlinks++;
+        state.priorities[dstFile] = priority;
+        state.symlinks++;
     }
 }
 
-typedef std::set<Path> FileProp;
+void buildProfile(const Path & out, Packages && pkgs)
+{
+    State state;
 
-static FileProp done;
-static FileProp postponed = FileProp{};
+    std::set<Path> done, postponed;
 
-static Path out;
+    auto addPkg = [&](const Path & pkgDir, int priority) {
+        if (!done.insert(pkgDir).second) return;
+        createLinks(state, pkgDir, out, priority);
 
-static void addPkg(const Path & pkgDir, int priority)
-{
-    if (!done.insert(pkgDir).second) return;
-    createLinks(pkgDir, out, priority);
+        try {
+            for (const auto & p : tokenizeString<std::vector<string>>(
+                    readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
+                if (!done.count(p))
+                    postponed.insert(p);
+        } catch (SysError & e) {
+            if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
+        }
+    };
 
-    try {
-        for (const auto & p : tokenizeString<std::vector<string>>(
-                readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
-            if (!done.count(p))
-                postponed.insert(p);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
-    }
-}
+    /* Symlink to the packages that have been installed explicitly by the
+     * user. Process in priority order to reduce unnecessary
+     * symlink/unlink steps.
+     */
+    std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
+        return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
+    });
+    for (const auto & pkg : pkgs)
+        if (pkg.active)
+            addPkg(pkg.path, pkg.priority);
 
-struct Package {
-    Path path;
-    bool active;
-    int priority;
-    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
-};
+    /* Symlink to the packages that have been "propagated" by packages
+     * installed by the user (i.e., package X declares that it wants Y
+     * installed as well). We do these later because they have a lower
+     * priority in case of collisions.
+     */
+    auto priorityCounter = 1000;
+    while (!postponed.empty()) {
+        std::set<Path> pkgDirs;
+        postponed.swap(pkgDirs);
+        for (const auto & pkgDir : pkgDirs)
+            addPkg(pkgDir, priorityCounter++);
+    }
 
-typedef std::vector<Package> Packages;
+    printError("created %d symlinks in user environment", state.symlinks);
+}
 
 void builtinBuildenv(const BasicDerivation & drv)
 {
@@ -153,7 +167,7 @@ void builtinBuildenv(const BasicDerivation & drv)
         return i->second;
     };
 
-    out = getAttr("out");
+    Path out = getAttr("out");
     createDirs(out);
 
     /* Convert the stuff we get from the environment back into a
@@ -171,31 +185,7 @@ void builtinBuildenv(const BasicDerivation & drv)
         }
     }
 
-    /* Symlink to the packages that have been installed explicitly by the
-     * user. Process in priority order to reduce unnecessary
-     * symlink/unlink steps.
-     */
-    std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
-        return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
-    });
-    for (const auto & pkg : pkgs)
-        if (pkg.active)
-            addPkg(pkg.path, pkg.priority);
-
-    /* Symlink to the packages that have been "propagated" by packages
-     * installed by the user (i.e., package X declares that it wants Y
-     * installed as well). We do these later because they have a lower
-     * priority in case of collisions.
-     */
-    auto priorityCounter = 1000;
-    while (!postponed.empty()) {
-        auto pkgDirs = postponed;
-        postponed = FileProp{};
-        for (const auto & pkgDir : pkgDirs)
-            addPkg(pkgDir, priorityCounter++);
-    }
-
-    printError("created %d symlinks in user environment", symlinks);
+    buildProfile(out, std::move(pkgs));
 
     createSymlink(getAttr("manifest"), out + "/manifest.nix");
 }
diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
new file mode 100644
index 00000000000..0a37459b0f0
--- /dev/null
+++ b/src/libstore/builtins/buildenv.hh
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "derivations.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct Package {
+    Path path;
+    bool active;
+    int priority;
+    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+};
+
+typedef std::vector<Package> Packages;
+
+void buildProfile(const Path & out, Packages && pkgs);
+
+void builtinBuildenv(const BasicDerivation & drv);
+
+}

From 4260a22a55d7afc931b22e8c41282fbd0ed18a77 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 8 Jan 2020 15:34:06 +0100
Subject: [PATCH 1870/2196] absPath(): Use std::optional

(cherry picked from commit 1bf9eb21b75f0d93d9c1633ea2e6fdf840047e79)
---
 src/libutil/util.cc | 6 +++---
 src/libutil/util.hh | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 52a1394f20d..097ff210aff 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -97,10 +97,10 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
 }
 
 
-Path absPath(Path path, Path dir)
+Path absPath(Path path, std::optional<Path> dir)
 {
     if (path[0] != '/') {
-        if (dir == "") {
+        if (!dir) {
 #ifdef __GNU__
             /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
                lengths and doesn't define `PATH_MAX'.  */
@@ -116,7 +116,7 @@ Path absPath(Path path, Path dir)
             free(buf);
 #endif
         }
-        path = dir + "/" + path;
+        path = *dir + "/" + path;
     }
     return canonPath(path);
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 559ed4332d2..8c2cef9e130 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -46,7 +46,7 @@ void clearEnv();
 /* Return an absolutized path, resolving paths relative to the
    specified directory, or the current directory otherwise.  The path
    is also canonicalised. */
-Path absPath(Path path, Path dir = "");
+Path absPath(Path path, std::optional<Path> dir = {});
 
 /* Canonicalise a path by removing all `.' or `..' components and
    double or trailing slashes.  Optionally resolves all symlink

From 7a8de57d3e7e53a4f6d8060c7201f3fbbe6cd325 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Mar 2020 14:17:10 +0100
Subject: [PATCH 1871/2196] Pretty-print 'nix why-depends' / 'nix-store -q
 --tree' output

Extracted from 678301072f05b650dc15c5edb4c25f08f0d6cace.
---
 src/libutil/util.hh        | 34 ++++++++++++++++++++++++++++++++++
 src/nix-store/nix-store.cc | 15 +++++----------
 src/nix/why-depends.cc     |  7 +------
 tests/dependencies.sh      |  8 ++++----
 4 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 8c2cef9e130..acef682cdb9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -451,6 +451,13 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
+/* Tree formatting. */
+constexpr char treeConn[] = "├───";
+constexpr char treeLast[] = "└───";
+constexpr char treeLine[] = "│   ";
+constexpr char treeNull[] = "    ";
+
+
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,
    some escape sequences (such as colour setting) are copied but not
@@ -576,4 +583,31 @@ extern PathFilter defaultPathFilter;
 AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
 
 
+// A Rust/Python-like enumerate() iterator adapter.
+// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.
+template <typename T,
+          typename TIter = decltype(std::begin(std::declval<T>())),
+          typename = decltype(std::end(std::declval<T>()))>
+constexpr auto enumerate(T && iterable)
+{
+    struct iterator
+    {
+        size_t i;
+        TIter iter;
+        bool operator != (const iterator & other) const { return iter != other.iter; }
+        void operator ++ () { ++i; ++iter; }
+        auto operator * () const { return std::tie(i, *iter); }
+    };
+
+    struct iterable_wrapper
+    {
+        T iterable;
+        auto begin() { return iterator{ 0, std::begin(iterable) }; }
+        auto end() { return iterator{ 0, std::end(iterable) }; }
+    };
+
+    return iterable_wrapper{ std::forward<T>(iterable) };
+}
+
+
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index c14d900aead..806ab756381 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -229,12 +229,6 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
 /* Some code to print a tree representation of a derivation dependency
    graph.  Topological sorting is used to keep the tree relatively
    flat. */
-
-const string treeConn = "+---";
-const string treeLine = "|   ";
-const string treeNull = "    ";
-
-
 static void printTree(const StorePath & path,
     const string & firstPad, const string & tailPad, StorePathSet & done)
 {
@@ -254,10 +248,11 @@ static void printTree(const StorePath & path,
     auto sorted = store->topoSortPaths(info->references);
     reverse(sorted.begin(), sorted.end());
 
-    for (auto i = sorted.begin(); i != sorted.end(); ++i) {
-        auto j = i; ++j;
-        printTree(*i, tailPad + treeConn,
-            j == sorted.end() ? tailPad + treeNull : tailPad + treeLine,
+    for (const auto &[n, i] : enumerate(sorted)) {
+        bool last = n + 1 == sorted.size();
+        printTree(i,
+            tailPad + (last ? treeLast : treeConn),
+            tailPad + (last ? treeNull : treeLine),
             done);
     }
 }
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index c24ae7c8e09..d3b7a674a32 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -143,11 +143,6 @@ struct CmdWhyDepends : SourceExprCommand
            and `dependency`. */
         std::function<void(Node &, const string &, const string &)> printNode;
 
-        const string treeConn = "╠═══";
-        const string treeLast = "╚═══";
-        const string treeLine = "║   ";
-        const string treeNull = "    ";
-
         struct BailOut { };
 
         printNode = [&](Node & node, const string & firstPad, const string & tailPad) {
@@ -157,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand
             std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n",
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
-                firstPad != "" ? "=> " : "",
+                firstPad != "" ? "→ " : "",
                 pathS);
 
             if (node.path == dependencyPath && !all
diff --git a/tests/dependencies.sh b/tests/dependencies.sh
index df204d185dd..8d0fdc10f0b 100644
--- a/tests/dependencies.sh
+++ b/tests/dependencies.sh
@@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix)
 
 echo "derivation is $drvPath"
 
-nix-store -q --tree "$drvPath" | grep '   +---.*builder1.sh'
+nix-store -q --tree "$drvPath" | grep '───.*builder1.sh'
 
 # Test Graphviz graph generation.
 nix-store -q --graph "$drvPath" > $TEST_ROOT/graph
@@ -22,9 +22,9 @@ nix-store -q --graph "$outPath" > $TEST_ROOT/graph
 if test -n "$dot"; then
     # Does it parse?
     $dot < $TEST_ROOT/graph
-fi    
+fi
 
-nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2'
+nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2'
 
 echo "output path is $outPath"
 
@@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath"
 
 # Check that the derivers are set properly.
 test $(nix-store -q --deriver "$outPath") = "$drvPath"
-nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" 
+nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv"

From 777e21e596ec85f18eddf8b286d9ba4995010835 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 11 Feb 2020 23:50:16 +0100
Subject: [PATCH 1872/2196] nix path-info --json: Print hash in SRI format

(cherry picked from commit 442e665d6d3fcbdee7dece2f62a597142f8784b1)
---
 src/libstore/store-api.cc | 6 ++++--
 src/libstore/store-api.hh | 1 +
 src/nix/path-info.cc      | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index aacbe5e0bfe..b9e894a9a3f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -441,7 +441,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
 
 
 void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
-    bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
+    bool includeImpureInfo, bool showClosureSize,
+    Base hashBase,
+    AllowInvalidFlag allowInvalid)
 {
     auto jsonList = jsonOut.list();
 
@@ -453,7 +455,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash.to_string())
+                .attr("narHash", info->narHash.to_string(hashBase))
                 .attr("narSize", info->narSize);
 
             {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e0484de133c..c57a6935ef3 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -561,6 +561,7 @@ public:
        each path is included. */
     void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
         bool includeImpureInfo, bool showClosureSize,
+        Base hashBase = Base32,
         AllowInvalidFlag allowInvalid = DisallowInvalid);
 
     /* Return the size of the closure of the specified path, that is,
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index bffa7b3564e..45ec297d2fb 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -89,7 +89,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             store->pathInfoToJSON(jsonRoot,
                 // FIXME: preserve order?
                 storePathsToSet(storePaths),
-                true, showClosureSize, AllowInvalid);
+                true, showClosureSize, SRI, AllowInvalid);
         }
 
         else {

From 6b824c78f1f6c2de95b594a440118e259d28def0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 2 Dec 2019 15:56:37 +0100
Subject: [PATCH 1873/2196] nix: Add --refresh as an alias for --tarball-ttl 0

(cherry picked from commit e721f99817bb7154d8098c902e25f84521a90b7f)
---
 src/nix/main.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 41d44f43191..5cb1ad25e23 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -92,6 +92,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName("no-net")
             .description("disable substituters and consider all previously downloaded files up-to-date")
             .handler([&]() { useNet = false; });
+
+        mkFlag()
+            .longName("refresh")
+            .description("consider all previously downloaded files out-of-date")
+            .handler([&]() { settings.tarballTtl = 0; });
     }
 
     void printFlags(std::ostream & out) override

From c85097da7c1ef06e2f1912c922110ff358de3aa2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Jan 2020 17:34:48 +0100
Subject: [PATCH 1874/2196] Fix --refresh with --no-net

https://hydra.nixos.org/build/110879699
(cherry picked from commit 5bbe793abf18414878a069399d1759673d693fb6)
---
 src/nix/main.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5cb1ad25e23..3b5f5516fdf 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -55,6 +55,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     bool printBuildLogs = false;
     bool useNet = true;
+    bool refresh = false;
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
@@ -96,7 +97,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         mkFlag()
             .longName("refresh")
             .description("consider all previously downloaded files out-of-date")
-            .handler([&]() { settings.tarballTtl = 0; });
+            .handler([&]() { refresh = true; });
     }
 
     void printFlags(std::ostream & out) override
@@ -181,6 +182,9 @@ void mainWrapped(int argc, char * * argv)
             downloadSettings.connectTimeout = 1;
     }
 
+    if (args.refresh)
+        settings.tarballTtl = 0;
+
     args.command->prepare();
     args.command->run();
 }

From 0a10854f857e519d1455ff0f0be1f8401c40a11c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Mar 2020 14:26:13 +0100
Subject: [PATCH 1875/2196] Misc changes from the flakes branch

---
 .gitignore                              |  2 +-
 Makefile.config.in                      |  1 +
 src/libexpr/eval.hh                     |  2 +-
 src/libstore/build.cc                   | 16 ++++++++--------
 src/libstore/builtins/buildenv.cc       |  2 +-
 src/libstore/http-binary-cache-store.cc |  5 +++--
 src/libstore/local-store.cc             | 13 +++----------
 src/libstore/nar-info-disk-cache.cc     |  7 +------
 src/libstore/nar-info-disk-cache.hh     |  2 +-
 src/libstore/parsed-derivations.cc      |  6 +++++-
 src/libstore/parsed-derivations.hh      | 10 ++++++----
 src/libstore/profiles.cc                | 18 ++++++++++++++++++
 src/libstore/profiles.hh                |  4 ++++
 src/libstore/sqlite.cc                  | 25 +++++++++++++++++++++++--
 src/libstore/sqlite.hh                  | 10 +++++++---
 src/libstore/store-api.hh               |  5 +++++
 src/libutil/hash.hh                     | 12 ++++++++++++
 src/libutil/util.hh                     |  5 +++--
 src/nix-env/nix-env.cc                  | 17 ++---------------
 tests/binary-cache.sh                   |  4 ++--
 tests/config.nix.in                     |  2 +-
 tests/fetchGit.sh                       |  2 ++
 22 files changed, 110 insertions(+), 60 deletions(-)

diff --git a/.gitignore b/.gitignore
index e10c7541894..e3186fa769a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
 
 /src/nix/nix
 
diff --git a/Makefile.config.in b/Makefile.config.in
index fe609ce0654..e7a12089a86 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -37,6 +37,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 5efad870775..1485dc7fe29 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -368,7 +368,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dc6a2c60608..0e3a23a4dc8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1398,7 +1398,7 @@ void DerivationGoal::tryToBuild()
        few seconds and then retry this goal. */
     PathSet lockFiles;
     for (auto & outPath : drv->outputPaths())
-        lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath)));
+        lockFiles.insert(worker.store.Store::toRealPath(outPath));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         worker.waitForAWhile(shared_from_this());
@@ -1429,7 +1429,7 @@ void DerivationGoal::tryToBuild()
     for (auto & i : drv->outputs) {
         if (worker.store.isValidPath(i.second.path)) continue;
         debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
-        deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path)));
+        deletePath(worker.store.Store::toRealPath(i.second.path));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1686,7 +1686,7 @@ void DerivationGoal::buildDone()
 
         /* Delete unused redirected outputs (when doing hash rewriting). */
         for (auto & i : redirectedOutputs)
-            deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second)));
+            deletePath(worker.store.Store::toRealPath(i.second));
 
         /* Delete the chroot (if we were using one). */
         autoDelChroot.reset(); /* this runs the destructor */
@@ -2072,7 +2072,7 @@ void DerivationGoal::startBuilder()
            environment using bind-mounts.  We put it in the Nix store
            to ensure that we can create hard-links to non-directory
            inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot";
+        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
         deletePath(chrootRootDir);
 
         /* Clean up the chroot directory automatically. */
@@ -2551,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
+    auto structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
 
     auto json = *structuredAttrs;
@@ -2917,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path)
 
         #if __linux__
 
-            Path source = worker.store.toRealPath(worker.store.printStorePath(path));
+            Path source = worker.store.Store::toRealPath(path);
             Path target = chrootRootDir + worker.store.printStorePath(path);
             debug("bind-mounting %s -> %s", target, source);
 
@@ -3579,7 +3579,7 @@ void DerivationGoal::registerOutputs()
         if (needsHashRewrite()) {
             auto r = redirectedOutputs.find(i.second.path);
             if (r != redirectedOutputs.end()) {
-                auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second));
+                auto redirected = worker.store.Store::toRealPath(r->second);
                 if (buildMode == bmRepair
                     && redirectedBadOutputs.count(i.second.path)
                     && pathExists(redirected))
@@ -3672,7 +3672,7 @@ void DerivationGoal::registerOutputs()
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
 
-                Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest));
+                Path actualDest = worker.store.Store::toRealPath(dest);
 
                 if (worker.store.isValidPath(dest))
                     std::rethrow_exception(delayedException);
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index c1c85d0bf99..1b802d90817 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs)
             addPkg(pkgDir, priorityCounter++);
     }
 
-    printError("created %d symlinks in user environment", state.symlinks);
+    debug("created %d symlinks in user environment", state.symlinks);
 }
 
 void builtinBuildenv(const BasicDerivation & drv)
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index d4ae3666255..011794c62b5 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
+    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
     if (std::string(uri, 0, 7) != "http://" &&
         std::string(uri, 0, 8) != "https://" &&
-        (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
-        ) return 0;
+        (!forceHttp || std::string(uri, 0, 7) != "file://"))
+        return 0;
     auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
     store->init();
     return store;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index cd2e86f29fe..ae7513ad815 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create)
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
-    if (sqlite3_open_v2(dbPath.c_str(), &db.db,
-            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open Nix database '%1%'") % dbPath);
+    state.db = SQLite(dbPath, create);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls
@@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create)
     SetDllDirectoryW(L"");
 #endif
 
-    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
-        throwSQLiteError(db, "setting timeout");
-
-    db.exec("pragma foreign_keys = 1");
-
     /* !!! check whether sqlite has been built with foreign key
        support */
 
@@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create)
 
     /* Initialise the database schema, if necessary. */
     if (create) {
-        const char * schema =
+        static const char schema[] =
 #include "schema.sql.gen.hh"
             ;
         db.exec(schema);
@@ -1275,7 +1268,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 else
                     hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
 
-                dumpPath(toRealPath(printStorePath(i)), *hashSink);
+                dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 907645d8671..442541330a2 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -78,12 +78,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
 
         state->db = SQLite(dbPath);
 
-        if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
-            throwSQLiteError(state->db, "setting timeout");
-
-        // We can always reproduce the cache.
-        state->db.exec("pragma synchronous = off");
-        state->db.exec("pragma main.journal_mode = truncate");
+        state->db.isCache();
 
         state->db.exec(schema);
 
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index 878acbb87b5..04de2c5eb3c 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -10,7 +10,7 @@ class NarInfoDiskCache
 public:
     typedef enum { oValid, oInvalid, oUnknown } Outcome;
 
-    virtual ~NarInfoDiskCache() { };
+    virtual ~NarInfoDiskCache() { }
 
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index d0f289a0f57..45c033c667d 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -1,5 +1,7 @@
 #include "parsed-derivations.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
@@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
     auto jsonAttr = drv.env.find("__json");
     if (jsonAttr != drv.env.end()) {
         try {
-            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+            structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
         } catch (std::exception & e) {
             throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
         }
     }
 }
 
+ParsedDerivation::~ParsedDerivation() { }
+
 std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
 {
     if (structuredAttrs) {
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index cec868754d9..f4df5dd5434 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,6 +1,6 @@
 #include "derivations.hh"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
@@ -8,15 +8,17 @@ class ParsedDerivation
 {
     StorePath drvPath;
     BasicDerivation & drv;
-    std::optional<nlohmann::json> structuredAttrs;
+    std::unique_ptr<nlohmann::json> structuredAttrs;
 
 public:
 
     ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
 
-    const std::optional<nlohmann::json> & getStructuredAttrs() const
+    ~ParsedDerivation();
+
+    const nlohmann::json * getStructuredAttrs() const
     {
-        return structuredAttrs;
+        return structuredAttrs.get();
     }
 
     std::optional<std::string> getStringAttr(const std::string & name) const;
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index dae3f2d3251..2bef518786c 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile)
 }
 
 
+Path getDefaultProfile()
+{
+    Path profileLink = getHome() + "/.nix-profile";
+    try {
+        if (!pathExists(profileLink)) {
+            replaceSymlink(
+                getuid() == 0
+                ? settings.nixStateDir + "/profiles/default"
+                : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
+                profileLink);
+        }
+        return absPath(readLink(profileLink), dirOf(profileLink));
+    } catch (Error &) {
+        return profileLink;
+    }
+}
+
+
 }
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 5fa1533de31..78645d8b6a4 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile);
    rebuilt. */
 string optimisticLockProfile(const Path & profile);
 
+/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
+   it. */
+Path getDefaultProfile();
+
 }
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index a061d64f36d..eb1daafc5b8 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -25,11 +25,16 @@ namespace nix {
         throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
 }
 
-SQLite::SQLite(const Path & path)
+SQLite::SQLite(const Path & path, bool create)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
-            SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
         throw Error(format("cannot open SQLite database '%s'") % path);
+
+    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
+        throwSQLiteError(db, "setting timeout");
+
+    exec("pragma foreign_keys = 1");
 }
 
 SQLite::~SQLite()
@@ -42,6 +47,12 @@ SQLite::~SQLite()
     }
 }
 
+void SQLite::isCache()
+{
+    exec("pragma synchronous = off");
+    exec("pragma main.journal_mode = truncate");
+}
+
 void SQLite::exec(const std::string & stmt)
 {
     retrySQLite<void>([&]() {
@@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
     return *this;
 }
 
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
+{
+    if (notNull) {
+        if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
+            throwSQLiteError(stmt.db, "binding argument");
+    } else
+        bind();
+    return *this;
+}
+
 SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
 {
     if (notNull) {
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index bd012d9b963..fd04c9b079a 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -5,8 +5,8 @@
 
 #include "types.hh"
 
-class sqlite3;
-class sqlite3_stmt;
+struct sqlite3;
+struct sqlite3_stmt;
 
 namespace nix {
 
@@ -15,13 +15,16 @@ struct SQLite
 {
     sqlite3 * db = 0;
     SQLite() { }
-    SQLite(const Path & path);
+    SQLite(const Path & path, bool create = true);
     SQLite(const SQLite & from) = delete;
     SQLite& operator = (const SQLite & from) = delete;
     SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
     ~SQLite();
     operator sqlite3 * () { return db; }
 
+    /* Disable synchronous mode, set truncate journal mode. */
+    void isCache();
+
     void exec(const std::string & stmt);
 };
 
@@ -52,6 +55,7 @@ struct SQLiteStmt
 
         /* Bind the next parameter. */
         Use & operator () (const std::string & value, bool notNull = true);
+        Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
         Use & operator () (int64_t value, bool notNull = true);
         Use & bind(); // null
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index c57a6935ef3..0fa59be6a61 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -677,6 +677,11 @@ public:
         return storePath;
     }
 
+    Path toRealPath(const StorePath & storePath)
+    {
+        return toRealPath(printStorePath(storePath));
+    }
+
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index ffa43ecf5a1..ea9fca3e74f 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -80,6 +80,18 @@ struct Hash
        or base-64. By default, this is prefixed by the hash type
        (e.g. "sha256:"). */
     std::string to_string(Base base = Base32, bool includeType = true) const;
+
+    std::string gitRev() const
+    {
+        assert(type == htSHA1);
+        return to_string(Base16, false);
+    }
+
+    std::string gitShortRev() const
+    {
+        assert(type == htSHA1);
+        return std::string(to_string(Base16, false), 0, 7);
+    }
 };
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index acef682cdb9..7c3a3024253 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -476,10 +476,11 @@ string base64Decode(const string & s);
 /* Get a value for the specified key from an associate container, or a
    default value if the key doesn't exist. */
 template <class T>
-std::optional<std::string> get(const T & map, const std::string & key)
+std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
 {
     auto i = map.find(key);
-    return i == map.end() ? std::optional<std::string>() : i->second;
+    if (i == map.end()) return {};
+    return std::optional<typename T::mapped_type>(i->second);
 }
 
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index c7e553d0cb6..e47b00acf3b 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1428,21 +1428,8 @@ static int _main(int argc, char * * argv)
         if (globals.profile == "")
             globals.profile = getEnv("NIX_PROFILE").value_or("");
 
-        if (globals.profile == "") {
-            Path profileLink = getHome() + "/.nix-profile";
-            try {
-                if (!pathExists(profileLink)) {
-                    replaceSymlink(
-                        getuid() == 0
-                        ? settings.nixStateDir + "/profiles/default"
-                        : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
-                        profileLink);
-                }
-                globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
-            } catch (Error &) {
-                globals.profile = profileLink;
-            }
-        }
+        if (globals.profile == "")
+            globals.profile = getDefaultProfile();
 
         op(globals, opFlags, opArgs);
 
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index eb58ae7c12a..a3c3c78472f 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -48,7 +48,7 @@ basicTests
 
 
 # Test HttpBinaryCacheStore.
-export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1
+export _NIX_FORCE_HTTP=1
 basicTests
 
 
@@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
 res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
 otherKey="$(cat $TEST_ROOT/pk3)"
 
-_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
+_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
 
 
 # Downloading should fail if we don't provide a key.
diff --git a/tests/config.nix.in b/tests/config.nix.in
index 51aed539cb5..0ec2eba6ba6 100644
--- a/tests/config.nix.in
+++ b/tests/config.nix.in
@@ -3,7 +3,7 @@ rec {
 
   path = "@coreutils@";
 
-  system = builtins.currentSystem;
+  system = "@system@";
 
   shared = builtins.getEnv "_NIX_TEST_SHARED";
 
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index 4c46bdf0465..ed8fa14d60e 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -9,6 +9,8 @@ clearStore
 
 repo=$TEST_ROOT/git
 
+export _NIX_FORCE_HTTP=1
+
 rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
 
 git init $repo

From 4171ab4bbd95e9bb4b9d6c6e8c143002d92b0c06 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 24 Mar 2020 09:18:23 -0600
Subject: [PATCH 1876/2196] renaming

---
 src/libutil/error.cc |  8 +++-----
 src/libutil/error.hh | 31 +++++++++++++++++++++++++++++--
 tests/errors/main.cc |  9 +++++++--
 3 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 71f422622a9..81c1f1805d2 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -41,8 +41,6 @@ void print_code_lines(string &prefix, NixCode &nix_code)
       % nix_code.errLine->errLineOfCode
       << endl;
     
-
-
     // error arrows for the column range.
     if (nix_code.errLine->columnRange.has_value()) 
     {
@@ -98,7 +96,7 @@ void print_error(ErrorInfo &einfo)
       }
   }
 
-  int ndl = level_string.length() + 3 + einfo.errName.length() + einfo.toolName.length();
+  int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length();
   int dashwidth = errwidth - 3 ? 3 : 80 - ndl; 
 
   string dashes;
@@ -110,9 +108,9 @@ void print_error(ErrorInfo &einfo)
     % prefix
     % level_string
     % "---"
-    % einfo.errName
+    % einfo.name
     % dashes
-    % einfo.toolName
+    % einfo.program
     << endl;
 
   // filename.
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 8dac1508e5c..69776eb8ca3 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -39,13 +39,40 @@ class NixCode {
 class ErrorInfo { 
   public:
   ErrLevel level;
-  string errName;
+  string name;
   string description;
-  string toolName;
+  string program;
   optional<NixCode> nixCode;
   string hint;
+  ErrorInfo& GetEI() { return *this; }
 };
 
+template <class T>
+class AddName : private T
+{
+  public:
+    T& name(const std::string &name){
+      GetEI().name = name;
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+template <class T>
+class AddDescription : private T 
+{
+  public:
+    T& description(const std::string &description){
+      GetEI().description = description;
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+typedef AddName<AddDescription<ErrorInfo>> StandardError;
+
 string showErrLine(ErrLine &errLine);
 
 void print_code_lines(string &prefix, NixCode &nix_code); 
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 1c998103e20..0e6d781c9f7 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -24,13 +24,18 @@ using namespace nix;
   
   ErrorInfo generic;
   generic.level = elError;
-  generic.errName = "error name";
+  generic.name = "error name";
   generic.description = "general error description";
-  generic.toolName = "nixtool.exe";
+  generic.program = "nixtool.exe";
   generic.nixCode = nixcode;
 
   print_error(generic);
 
+
+  StandardError standardError;
+
+  print_error(standardError.name("blah").description("blah"));
+
   return 0;
 }
 

From 0166e7ab6d1d4530e9d6d9562693a58ab54c36ba Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 24 Mar 2020 11:21:35 -0600
Subject: [PATCH 1877/2196] MkNixCode, MkErrLine approach

---
 src/libutil/error.cc |   6 +-
 src/libutil/error.hh | 187 +++++++++++++++++++++++++++++++++++++++----
 tests/errors/main.cc |  38 ++++++++-
 3 files changed, 211 insertions(+), 20 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 81c1f1805d2..f258d6a8379 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -94,7 +94,11 @@ void print_error(ErrorInfo &einfo)
         level_string = "warning:";  // TODO make yellow.
         break;
       }
-  }
+    default: 
+      {
+        level_string = "wat:";  
+        break;
+      }}
 
   int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length();
   int dashwidth = errwidth - 3 ? 3 : 80 - ndl; 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 69776eb8ca3..9449540afd0 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -3,6 +3,7 @@
 #include "types.hh"
 #include <string>
 #include <optional>
+#include <iostream>
 
 using std::string;
 using std::optional;
@@ -16,35 +17,171 @@ enum ErrLevel
 
 class ColumnRange { 
   public:
-  unsigned int start;
-  unsigned int len;
+    unsigned int start;
+    unsigned int len;
 };
 
+class ErrorInfo;
+
+// -------------------------------------------------
+// forward declarations before ErrLine.
+template <class T> 
+class AddLineNumber;
+
+template <class T> 
+class AddColumnRange;
+
+template <class T> 
+class AddLOC;
+
 class ErrLine { 
   public:
-  int lineNumber;
-  optional<ColumnRange> columnRange;
-  optional<string> prevLineOfCode;
-  string errLineOfCode;
-  optional<string> nextLineOfCode;
+    int lineNumber;
+    optional<ColumnRange> columnRange;
+    optional<string> prevLineOfCode;
+    string errLineOfCode;
+    optional<string> nextLineOfCode;
+
+    friend AddLineNumber<ErrLine>;
+    friend AddColumnRange<ErrLine>;
+    friend AddLOC<ErrLine>;
+    ErrLine& GetEL() { return *this; }
+  private:
+    ErrLine() {}
+};
+
+template <class T> 
+class AddLineNumber : public T
+{
+  public:
+    T& lineNumber(int lineNumber) { 
+      GetEL().lineNumber = lineNumber;
+      return *this;
+    }
+  protected:
+    ErrLine& GetEL() { return T::GetEL(); }
+};
+
+template <class T> 
+class AddColumnRange : public T
+{
+  public:
+    T& columnRange(unsigned int start, unsigned int len) { 
+      GetEL().columnRange = { start, len };
+      return *this;
+    }
+  protected:
+    ErrLine& GetEL() { return T::GetEL(); }
+};
+
+template <class T> 
+class AddLOC : public T
+{
+  public:
+    T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
+      GetEL().prevLineOfCode = prevloc;
+      GetEL().errLineOfCode = loc;
+      GetEL().nextLineOfCode = nextloc;
+      return *this;
+    }
+  protected:
+    ErrLine& GetEL() { return T::GetEL(); }
 };
 
+typedef AddLineNumber<AddColumnRange<AddLOC<ErrLine>>> MkErrLine;
+MkErrLine mkErrLine;
+
+
+// -------------------------------------------------
+// NixCode.
+
+template <class T>
+class AddNixFile;
+
+template <class T>
+class AddErrLine;
+
 class NixCode { 
   public:
-  optional<string> nixFile;
-  optional<ErrLine> errLine;
+    optional<string> nixFile;
+    optional<ErrLine> errLine;
+
+    friend AddNixFile<NixCode>;
+    friend AddErrLine<NixCode>;
+    friend ErrorInfo;
+    NixCode& GetNC() { return *this; }
+  private:
+    NixCode() {}
+};
+
+template <class T> 
+class AddNixFile : public T
+{
+  public:
+    T& nixFile(string filename) { 
+      GetNC().nixFile = filename;
+      return *this;
+    }
+  protected:
+    NixCode& GetNC() { return T::GetNC(); }
+};
+
+template <class T> 
+class AddErrLine : public T
+{
+  public:
+    T& errLine(ErrLine errline) { 
+      GetNC().errLine = errline;
+      return *this;
+    }
+  protected:
+    NixCode& GetNC() { return T::GetNC(); }
 };
 
+typedef AddNixFile<AddErrLine<NixCode>> MkNixCode;
+
+// -------------------------------------------------
+// ErrorInfo.
+
+template <class T>
+class AddName;
+
+template <class T>
+class AddDescription;
+
+template <class T>
+class AddNixCode;
 
 class ErrorInfo { 
   public:
-  ErrLevel level;
-  string name;
-  string description;
-  string program;
-  optional<NixCode> nixCode;
-  string hint;
-  ErrorInfo& GetEI() { return *this; }
+    ErrLevel level;
+    string name;
+    string description;
+    string program;
+    optional<NixCode> nixCode;
+    string hint;
+    ErrorInfo& GetEI() { return *this; }
+
+    // give these access to the private constructor, 
+    // when they are direct descendants.
+    friend AddName<ErrorInfo>;
+    friend AddDescription<ErrorInfo>;
+    friend AddNixCode<ErrorInfo>;
+    
+  protected:
+    ErrorInfo(ErrLevel level) { this->level = level; }
+};
+
+class EIError : public ErrorInfo
+{
+  protected:
+    EIError() : ErrorInfo(elError) {}
+};
+
+class EIWarning : public ErrorInfo
+{
+  protected:
+    EIWarning() : ErrorInfo(elWarning) {}
 };
 
 template <class T>
@@ -71,7 +208,23 @@ class AddDescription : private T
     ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
-typedef AddName<AddDescription<ErrorInfo>> StandardError;
+template <class T>
+class AddNixCode : private T 
+{
+  public:
+    T& nixcode(const NixCode &nixcode){
+      GetEI().nixCode = nixcode;
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+typedef AddName<AddDescription<EIError>> StandardError;
+typedef AddName<AddDescription<EIWarning>> StandardWarning;
+
+typedef AddName<AddDescription<AddNixCode<EIError>>> MkNixError;
+typedef AddName<AddDescription<AddNixCode<EIWarning>>> MkNixWarning;
 
 string showErrLine(ErrLine &errLine);
 
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 0e6d781c9f7..20dba046b81 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -4,11 +4,13 @@
 #include <iostream>
 
 using std::optional;
+using std::nullopt;
 
 int main() {
 
 using namespace nix; 
 
+  /*
   ColumnRange columnRange;
   columnRange.start = 24;
   columnRange.len = 14;
@@ -30,11 +32,43 @@ using namespace nix;
   generic.nixCode = nixcode;
 
   print_error(generic);
-
+  */
 
   StandardError standardError;
 
-  print_error(standardError.name("blah").description("blah"));
+  print_error(standardError
+                .name("name")
+                .description("description"));
+
+  StandardWarning standardWarning;
+
+  print_error(standardWarning
+                .name("warning name")
+                .description("warning description"));
+
+  print_error(MkNixError()
+                .name("name")
+                .description("description")
+                .nixcode(
+                  MkNixCode()
+                    .nixFile("myfile.nix")
+                    .errLine(MkErrLine().lineNumber(40)
+                           .columnRange(50,10)
+                           .linesOfCode(nullopt
+                                      ,"this is the problem line of code"
+                                      ,nullopt))));
+
+  print_error(MkNixWarning()
+                .name("name")
+                .description("description")
+                .nixcode(
+                  MkNixCode()
+                    .nixFile("myfile.nix")
+                    .errLine(MkErrLine().lineNumber(40)
+                           .columnRange(50,10)
+                           .linesOfCode(nullopt
+                                      ,"this is the problem line of code"
+                                      ,nullopt))));
 
   return 0;
 }

From 657c08c852ce94123984c475bc9d5f094f7d8d46 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 24 Mar 2020 13:26:20 -0600
Subject: [PATCH 1878/2196] fix column range

---
 tests/errors/main.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 20dba046b81..c2c4d00605f 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -53,7 +53,7 @@ using namespace nix;
                   MkNixCode()
                     .nixFile("myfile.nix")
                     .errLine(MkErrLine().lineNumber(40)
-                           .columnRange(50,10)
+                           .columnRange(13,7)
                            .linesOfCode(nullopt
                                       ,"this is the problem line of code"
                                       ,nullopt))));
@@ -65,7 +65,7 @@ using namespace nix;
                   MkNixCode()
                     .nixFile("myfile.nix")
                     .errLine(MkErrLine().lineNumber(40)
-                           .columnRange(50,10)
+                           .columnRange(13,7)
                            .linesOfCode(nullopt
                                       ,"this is the problem line of code"
                                       ,nullopt))));

From fc310eda3afdf97c55a1a5ac6756f8d54a2a5ad6 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 24 Mar 2020 14:24:57 -0600
Subject: [PATCH 1879/2196] switch to one level of builder function, not
 subobject functions

---
 src/libutil/error.hh | 197 ++++++++++++++++++++-----------------------
 tests/errors/main.cc |  47 +++++------
 2 files changed, 112 insertions(+), 132 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 9449540afd0..25f3e8ee853 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -23,17 +23,6 @@ class ColumnRange {
 
 class ErrorInfo;
 
-// -------------------------------------------------
-// forward declarations before ErrLine.
-template <class T> 
-class AddLineNumber;
-
-template <class T> 
-class AddColumnRange;
-
-template <class T> 
-class AddLOC;
-
 class ErrLine { 
   public:
     int lineNumber;
@@ -42,107 +31,25 @@ class ErrLine {
     string errLineOfCode;
     optional<string> nextLineOfCode;
 
-    friend AddLineNumber<ErrLine>;
-    friend AddColumnRange<ErrLine>;
-    friend AddLOC<ErrLine>;
-    ErrLine& GetEL() { return *this; }
-  private:
-    ErrLine() {}
-};
-
-template <class T> 
-class AddLineNumber : public T
-{
-  public:
-    T& lineNumber(int lineNumber) { 
-      GetEL().lineNumber = lineNumber;
-      return *this;
-    }
-  protected:
-    ErrLine& GetEL() { return T::GetEL(); }
-};
-
-template <class T> 
-class AddColumnRange : public T
-{
-  public:
-    T& columnRange(unsigned int start, unsigned int len) { 
-      GetEL().columnRange = { start, len };
-      return *this;
-    }
-  protected:
-    ErrLine& GetEL() { return T::GetEL(); }
 };
 
-template <class T> 
-class AddLOC : public T
-{
-  public:
-    T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
-      GetEL().prevLineOfCode = prevloc;
-      GetEL().errLineOfCode = loc;
-      GetEL().nextLineOfCode = nextloc;
-      return *this;
-    }
-  protected:
-    ErrLine& GetEL() { return T::GetEL(); }
-};
-
-typedef AddLineNumber<AddColumnRange<AddLOC<ErrLine>>> MkErrLine;
-MkErrLine mkErrLine;
-
-
-// -------------------------------------------------
-// NixCode.
-
-template <class T>
-class AddNixFile;
-
-template <class T>
-class AddErrLine;
-
 class NixCode { 
   public:
     optional<string> nixFile;
     optional<ErrLine> errLine;
 
-    friend AddNixFile<NixCode>;
-    friend AddErrLine<NixCode>;
-    friend ErrorInfo;
-    NixCode& GetNC() { return *this; }
-  private:
-    NixCode() {}
-};
-
-template <class T> 
-class AddNixFile : public T
-{
-  public:
-    T& nixFile(string filename) { 
-      GetNC().nixFile = filename;
-      return *this;
+    ErrLine& ensureErrLine() 
+    { 
+      if (!this->errLine.has_value())
+         this->errLine = optional(ErrLine());
+      return *this->errLine; 
     }
-  protected:
-    NixCode& GetNC() { return T::GetNC(); }
 };
 
-template <class T> 
-class AddErrLine : public T
-{
-  public:
-    T& errLine(ErrLine errline) { 
-      GetNC().errLine = errline;
-      return *this;
-    }
-  protected:
-    NixCode& GetNC() { return T::GetNC(); }
-};
-
-typedef AddNixFile<AddErrLine<NixCode>> MkNixCode;
-
 // -------------------------------------------------
 // ErrorInfo.
 
+// Forward friend class declarations.  "builder classes"
 template <class T>
 class AddName;
 
@@ -152,6 +59,22 @@ class AddDescription;
 template <class T>
 class AddNixCode;
 
+template <class T>
+class AddNixFile;
+
+template <class T>
+class AddErrLine;
+
+template <class T> 
+class AddLineNumber;
+
+template <class T> 
+class AddColumnRange;
+
+template <class T> 
+class AddLOC;
+
+// The error info class itself.
 class ErrorInfo { 
   public:
     ErrLevel level;
@@ -167,9 +90,23 @@ class ErrorInfo {
     friend AddName<ErrorInfo>;
     friend AddDescription<ErrorInfo>;
     friend AddNixCode<ErrorInfo>;
+    friend AddNixFile<ErrorInfo>;
+    friend AddErrLine<ErrorInfo>;
+    friend AddLineNumber<ErrorInfo>;
+    friend AddColumnRange<ErrorInfo>;
+    friend AddLOC<ErrorInfo>;
     
+    NixCode& ensureNixCode() 
+    { 
+      if (!this->nixCode.has_value())
+         this->nixCode = optional(NixCode());
+      return *this->nixCode; 
+    }
   protected:
+    // constructor is protected, so only the builder classes can create an ErrorInfo.
     ErrorInfo(ErrLevel level) { this->level = level; }
+
+
 };
 
 class EIError : public ErrorInfo
@@ -208,23 +145,73 @@ class AddDescription : private T
     ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
-template <class T>
-class AddNixCode : private T 
+template <class T> 
+class AddNixFile : public T
+{
+  public:
+    T& nixFile(string filename) { 
+     GetEI().ensureNixCode().nixFile = filename;
+     return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+template <class T> 
+class AddLineNumber : public T
 {
   public:
-    T& nixcode(const NixCode &nixcode){
-      GetEI().nixCode = nixcode;
+    T& lineNumber(int lineNumber) { 
+      GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
       return *this;
     }
   protected:
     ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
+template <class T> 
+class AddColumnRange : public T
+{
+  public:
+    T& columnRange(unsigned int start, unsigned int len) { 
+      GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+template <class T> 
+class AddLOC : public T
+{
+  public:
+    T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
+      GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
+      GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
+      GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+typedef AddNixFile<AddErrLine<NixCode>> MkNixCode;
+
 typedef AddName<AddDescription<EIError>> StandardError;
 typedef AddName<AddDescription<EIWarning>> StandardWarning;
 
-typedef AddName<AddDescription<AddNixCode<EIError>>> MkNixError;
-typedef AddName<AddDescription<AddNixCode<EIWarning>>> MkNixWarning;
+typedef AddName<
+        AddDescription<
+        AddNixFile<
+        AddLineNumber<
+        AddColumnRange<
+        AddLOC<EIError>>>>>> MkNixError;
+typedef AddName<
+        AddDescription<
+        AddNixFile<
+        AddLineNumber<
+        AddColumnRange<
+        AddLOC<EIWarning>>>>>> MkNixWarning;
 
 string showErrLine(ErrLine &errLine);
 
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index c2c4d00605f..6b024d287a2 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -34,41 +34,34 @@ using namespace nix;
   print_error(generic);
   */
 
-  StandardError standardError;
-
-  print_error(standardError
+  print_error(StandardError()
                 .name("name")
                 .description("description"));
 
-  StandardWarning standardWarning;
-
-  print_error(standardWarning
+  print_error(StandardWarning()
                 .name("warning name")
                 .description("warning description"));
 
-  print_error(MkNixError()
-                .name("name")
-                .description("description")
-                .nixcode(
-                  MkNixCode()
-                    .nixFile("myfile.nix")
-                    .errLine(MkErrLine().lineNumber(40)
-                           .columnRange(13,7)
-                           .linesOfCode(nullopt
-                                      ,"this is the problem line of code"
-                                      ,nullopt))));
 
   print_error(MkNixWarning()
-                .name("name")
-                .description("description")
-                .nixcode(
-                  MkNixCode()
-                    .nixFile("myfile.nix")
-                    .errLine(MkErrLine().lineNumber(40)
-                           .columnRange(13,7)
-                           .linesOfCode(nullopt
-                                      ,"this is the problem line of code"
-                                      ,nullopt))));
+                .name("warning name")
+                .description("warning description")
+                .nixFile("myfile.nix")
+                .lineNumber(40)
+                .columnRange(13,7)
+                .linesOfCode(nullopt
+                            ,"this is the problem line of code"
+                            ,nullopt));
+
+  print_error(MkNixError()
+                .name("error name")
+                .description("warning description")
+                .nixFile("myfile.nix")
+                .lineNumber(40)
+                .columnRange(13,7)
+                .linesOfCode(nullopt
+                            ,"this is the problem line of code"
+                            ,nullopt));
 
   return 0;
 }

From 7313aa267b5be1e5264e4577e7bc3daec2fef282 Mon Sep 17 00:00:00 2001
From: Greg Price <gnprice@gmail.com>
Date: Tue, 24 Mar 2020 21:15:01 -0700
Subject: [PATCH 1880/2196] installer: Fix terminal colors.

The install-multi-user script uses blue, green, and red colors, as
well as bold and underline, to add helpful formatting that helps
structure its rather voluminous output.

Unfortunately, the terminal escape sequences it uses are not quite
well-formed.  The relevant information is all there, just obscured
by some extra noise, a leading parameter `38`.  Empirically, the
result is:

 * On macOS, in both Terminal.app and iTerm2, the spurious `38` is
   ignored, the rest of the escape sequence is applied, and the colors
   show up as intended.

 * On Linux, in at least gnome-terminal and xterm, the spurious `38`
   and the next parameter after it are ignored, and what's left is
   applied.  So in the sequence `38;4;32`, the 4 (underline) is
   ignored but the 32 (green) takes effect; in a more typical sequence
   like `38;34`, the 34 (blue) is ignored and nothing happens.

These codes are all unchanged since this script's origins as a
Darwin-only script -- so the fact that they work fine in common macOS
terminals goes some way to explain how the bug arose.

Happily, we can make the colors work as intended by just deleting the
extra `38;`.  Tested in all four terminals mentioned above; the new
codes work correctly on all of them, and on the two macOS terminals
they work exactly the same as before.

---

In a bit more technical detail -- perhaps more than anyone, me
included, ever wanted to know, but now that I've gone and learned it
I'll write it down anyway :) -- here's what's happening in these codes:

An ECMA-48 "control sequence" begins with `\033[` aka "CSI", contains
any number of parameters as semicolon-separated decimal numbers (plus
sometimes other wrinkles), and ends with a byte from 0x40..0x7e.  In
our case, with `m` aka "SGR", "Select Graphic Rendition".

An SGR control sequence `\033[...m` sets colors, fonts, text styles,
etc.  In particular a parameter `31` means red, `32` green, `34` blue,
`4` underline, and `0` means reset to normal.  Those are all we use.

There is also a `38`.  This is used for setting colors too... but it
needs arguments.  `38;5;nn` is color nn from a 256-color palette, and
`38;2;rr;gg;bb` has the given RGB values.

There is no meaning defined for `38;1` or `38;34` etc.  On seeing a
parameter `38` followed by an unrecognized argument for it, apparently
some implementations (as seen on macOS) discard only the `38` and
others (as seen on Linux) discard the argument too before resuming.
---
 scripts/install-multi-user.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 35341543e75..a0f1deb987f 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"

From 3582dc3c88d38d5c46091b4a23601d6f56a4c81e Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 25 Mar 2020 10:52:03 -0600
Subject: [PATCH 1881/2196] programName as static member var

---
 src/libutil/error.cc | 13 +++++++++----
 src/libutil/error.hh |  6 +++++-
 tests/errors/main.cc |  4 ++++
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index f258d6a8379..4abee052d58 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -1,11 +1,15 @@
 #include "error.hh"
 
 #include <iostream>
+#include <optional>
 
 namespace nix {
 
 using std::cout;
 using std::endl;
+using std::nullopt;
+
+optional<string> ErrorInfo::programName = nullopt;
 
 // return basic_format?
 string showErrLine(ErrLine &errLine)
@@ -98,10 +102,11 @@ void print_error(ErrorInfo &einfo)
       {
         level_string = "wat:";  
         break;
-      }}
+      }
+  }
 
-  int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length();
-  int dashwidth = errwidth - 3 ? 3 : 80 - ndl; 
+  int ndl = prefix.length() + level_string.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+  int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; 
 
   string dashes;
   for (int i = 0; i < dashwidth; ++i)
@@ -114,7 +119,7 @@ void print_error(ErrorInfo &einfo)
     % "---"
     % einfo.name
     % dashes
-    % einfo.program
+    % einfo.programName.value_or("")
     << endl;
 
   // filename.
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 25f3e8ee853..7260dfd5bf6 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -80,11 +80,12 @@ class ErrorInfo {
     ErrLevel level;
     string name;
     string description;
-    string program;
     optional<NixCode> nixCode;
     string hint;
     ErrorInfo& GetEI() { return *this; }
 
+    static optional<string> programName;
+
     // give these access to the private constructor, 
     // when they are direct descendants.
     friend AddName<ErrorInfo>;
@@ -109,18 +110,21 @@ class ErrorInfo {
 
 };
 
+// Init as error
 class EIError : public ErrorInfo
 {
   protected:
     EIError() : ErrorInfo(elError) {}
 };
 
+// Init as warning
 class EIWarning : public ErrorInfo
 {
   protected:
     EIWarning() : ErrorInfo(elWarning) {}
 };
 
+// Builder class definitions.
 template <class T>
 class AddName : private T
 {
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 6b024d287a2..5ac429b4624 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -5,11 +5,15 @@
 
 using std::optional;
 using std::nullopt;
+using std::cout;
+using std::endl;
 
 int main() {
 
 using namespace nix; 
 
+  ErrorInfo::programName = optional("errorTest");
+
   /*
   ColumnRange columnRange;
   columnRange.start = 24;

From d44c9c55817839a41a7b4c509da027d7b4176ca5 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 25 Mar 2020 11:20:44 -0600
Subject: [PATCH 1882/2196] some colors

---
 src/libutil/error.cc | 15 ++++++++++-----
 tests/errors/main.cc |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 4abee052d58..9fbd234e033 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -62,7 +62,7 @@ void print_code_lines(string &prefix, NixCode &nix_code)
         arrows.append("^");
       }
     
-      cout << format("%1%      |%2%%3%") % prefix % spaces % arrows;
+      cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows;
     }
 
 
@@ -90,12 +90,16 @@ void print_error(ErrorInfo &einfo)
   {
     case ErrLevel::elError: 
       {
-        level_string = "error:";  // TODO make red.
+        level_string = ANSI_RED;
+        level_string += "error:";
+        level_string += ANSI_NORMAL;
         break;
       }
     case ErrLevel::elWarning: 
       {
-        level_string = "warning:";  // TODO make yellow.
+        level_string = ANSI_YELLOW;
+        level_string += "warning:";  
+        level_string += ANSI_NORMAL;
         break;
       }
     default: 
@@ -113,7 +117,7 @@ void print_error(ErrorInfo &einfo)
     dashes.append("-");
 
   // divider.
-  cout << format("%1%%2% %3% %4% %5% %6%")
+  cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
     % prefix
     % level_string
     % "---"
@@ -131,7 +135,8 @@ void print_error(ErrorInfo &einfo)
         ? string(" ") + showErrLine(*einfo.nixCode->errLine)
         : "";
 
-      cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline << endl;
+      cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
+        % prefix % *einfo.nixCode->nixFile % eline << endl;
       cout << prefix << endl;
     }
     else
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 5ac429b4624..f53101629c5 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -59,7 +59,7 @@ using namespace nix;
 
   print_error(MkNixError()
                 .name("error name")
-                .description("warning description")
+                .description("error description")
                 .nixFile("myfile.nix")
                 .lineNumber(40)
                 .columnRange(13,7)

From a3ef00be6c668bd59a879a4acdb1599225d86b44 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 27 Mar 2020 10:03:02 -0600
Subject: [PATCH 1883/2196] camelcase; optional hint

---
 src/libutil/error.cc | 60 +++++++++++++++++---------------
 src/libutil/error.hh | 82 +++++++++++++++++++++++++++++++++++++-------
 tests/errors/main.cc | 34 +++++++++---------
 3 files changed, 119 insertions(+), 57 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 9fbd234e033..d6595070f17 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -24,55 +24,55 @@ string showErrLine(ErrLine &errLine)
   };
 }
 
-void print_code_lines(string &prefix, NixCode &nix_code) 
+void printCodeLines(string &prefix, NixCode &nixCode) 
 {
   
-  if (nix_code.errLine.has_value()) 
+  if (nixCode.errLine.has_value()) 
   {
     // previous line of code.
-    if (nix_code.errLine->prevLineOfCode.has_value()) { 
+    if (nixCode.errLine->prevLineOfCode.has_value()) { 
       cout << format("%1% %|2$5d|| %3%")
         %  prefix
-        % (nix_code.errLine->lineNumber - 1)
-        % *nix_code.errLine->prevLineOfCode
+        % (nixCode.errLine->lineNumber - 1)
+        % *nixCode.errLine->prevLineOfCode
         << endl;
     }
 
     // line of code containing the error.%2$+5d%
     cout << format("%1% %|2$5d|| %3%")
       %  prefix
-      % (nix_code.errLine->lineNumber)
-      % nix_code.errLine->errLineOfCode
+      % (nixCode.errLine->lineNumber)
+      % nixCode.errLine->errLineOfCode
       << endl;
     
     // error arrows for the column range.
-    if (nix_code.errLine->columnRange.has_value()) 
+    if (nixCode.errLine->columnRange.has_value()) 
     {
-      int start = nix_code.errLine->columnRange->start;
+      int start = nixCode.errLine->columnRange->start;
       std::string spaces;
       for (int i = 0; i < start; ++i)
       {
         spaces.append(" ");
       }
 
-      int len = nix_code.errLine->columnRange->len;
+      int len = nixCode.errLine->columnRange->len;
       std::string arrows;
       for (int i = 0; i < len; ++i)
       {
         arrows.append("^");
       }
     
-      cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows;
+      cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl;
     }
 
 
 
     // next line of code.
-    if (nix_code.errLine->nextLineOfCode.has_value()) { 
+    if (nixCode.errLine->nextLineOfCode.has_value()) { 
       cout << format("%1% %|2$5d|| %3%")
       %  prefix
-      % (nix_code.errLine->lineNumber + 1)
-      % *nix_code.errLine->nextLineOfCode
+      % (nixCode.errLine->lineNumber + 1)
+      % *nixCode.errLine->nextLineOfCode
       << endl;
     }
 
@@ -80,36 +80,36 @@ void print_code_lines(string &prefix, NixCode &nix_code)
     
 }
 
-void print_error(ErrorInfo &einfo) 
+void printErrorInfo(ErrorInfo &einfo) 
 {
   int errwidth = 80;
   string prefix = "  ";
 
-  string level_string;
+  string levelString;
   switch (einfo.level) 
   {
     case ErrLevel::elError: 
       {
-        level_string = ANSI_RED;
-        level_string += "error:";
-        level_string += ANSI_NORMAL;
+        levelString = ANSI_RED;
+        levelString += "error:";
+        levelString += ANSI_NORMAL;
         break;
       }
     case ErrLevel::elWarning: 
       {
-        level_string = ANSI_YELLOW;
-        level_string += "warning:";  
-        level_string += ANSI_NORMAL;
+        levelString = ANSI_YELLOW;
+        levelString += "warning:";  
+        levelString += ANSI_NORMAL;
         break;
       }
     default: 
       {
-        level_string = "wat:";  
+        levelString = "wat:";  
         break;
       }
   }
 
-  int ndl = prefix.length() + level_string.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+  int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
   int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; 
 
   string dashes;
@@ -119,7 +119,7 @@ void print_error(ErrorInfo &einfo)
   // divider.
   cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
     % prefix
-    % level_string
+    % levelString
     % "---"
     % einfo.name
     % dashes
@@ -152,11 +152,15 @@ void print_error(ErrorInfo &einfo)
 
   // lines of code.
   if (einfo.nixCode.has_value())
-    print_code_lines(prefix, *einfo.nixCode);
+    printCodeLines(prefix, *einfo.nixCode);
 
   // hint
-  cout << prefix << einfo.hint << endl;
-  cout << prefix << endl;
+  if (einfo.hint.has_value()) 
+  {
+    cout << prefix << endl;
+    cout << prefix << *einfo.hint << endl;
+    cout << prefix << endl;
+  }
 
 }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 7260dfd5bf6..f02d7288abe 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -1,12 +1,16 @@
 #pragma once
 
-#include "types.hh"
+#include "util.hh"
 #include <string>
 #include <optional>
 #include <iostream>
+#include <iomanip>
 
 using std::string;
 using std::optional;
+using boost::format;
+using std::cout;
+using std::endl;
 
 namespace nix {
 
@@ -81,13 +85,13 @@ class ErrorInfo {
     string name;
     string description;
     optional<NixCode> nixCode;
-    string hint;
+    optional<string> hint;
     ErrorInfo& GetEI() { return *this; }
 
     static optional<string> programName;
 
     // give these access to the private constructor, 
-    // when they are direct descendants.
+    // when they are direct descendants (children but not grandchildren).
     friend AddName<ErrorInfo>;
     friend AddDescription<ErrorInfo>;
     friend AddNixCode<ErrorInfo>;
@@ -150,7 +154,7 @@ class AddDescription : private T
 };
 
 template <class T> 
-class AddNixFile : public T
+class AddNixFile : private T
 {
   public:
     T& nixFile(string filename) { 
@@ -162,7 +166,7 @@ class AddNixFile : public T
 };
 
 template <class T> 
-class AddLineNumber : public T
+class AddLineNumber : private T
 {
   public:
     T& lineNumber(int lineNumber) { 
@@ -174,7 +178,7 @@ class AddLineNumber : public T
 };
 
 template <class T> 
-class AddColumnRange : public T
+class AddColumnRange : private T
 {
   public:
     T& columnRange(unsigned int start, unsigned int len) { 
@@ -186,7 +190,7 @@ class AddColumnRange : public T
 };
 
 template <class T> 
-class AddLOC : public T
+class AddLOC : private T
 {
   public:
     T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
@@ -199,17 +203,64 @@ class AddLOC : public T
     ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
-typedef AddNixFile<AddErrLine<NixCode>> MkNixCode;
+template <class T>
+class yellowify
+{
+public:
+    yellowify(T &s) : value(s) {}
+    T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
+{
+  return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+}
+
+// hint format shows templated values in yellow.
+class hintfmt 
+{
+  public:
+    hintfmt(string format) :fmt(format) {}
+    template<class T>
+      hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; }
+
+    template <typename U>
+    friend class AddHint;
+  private:
+    format fmt;
+      
+};
+
+template <class T> 
+class AddHint : private T
+{
+  public:
+    T& hint(hintfmt &hfmt) {
+      GetEI().hint = optional(hfmt.fmt.str());
+      return *this;
+    }
+    T& nohint() {
+      GetEI().hint = std::nullopt;
+      return *this;
+    }
+  protected:
+    ErrorInfo& GetEI() { return T::GetEI(); }
+};
+
+// --------------------------------------------------------
+// error types
 
-typedef AddName<AddDescription<EIError>> StandardError;
-typedef AddName<AddDescription<EIWarning>> StandardWarning;
+typedef AddName<AddDescription<AddHint<EIError>>> StandardError;
+typedef AddName<AddDescription<AddHint<EIWarning>>> StandardWarning;
 
 typedef AddName<
         AddDescription<
         AddNixFile<
         AddLineNumber<
         AddColumnRange<
-        AddLOC<EIError>>>>>> MkNixError;
+        AddLOC<
+        AddHint<EIError>>>>>>> MkNixError;
 typedef AddName<
         AddDescription<
         AddNixFile<
@@ -217,10 +268,15 @@ typedef AddName<
         AddColumnRange<
         AddLOC<EIWarning>>>>>> MkNixWarning;
 
+
+// --------------------------------------------------------
+// error printing
+
+void printErrorInfo(ErrorInfo &einfo);
+
 string showErrLine(ErrLine &errLine);
 
-void print_code_lines(string &prefix, NixCode &nix_code); 
+void printCodeLines(string &prefix, NixCode &nixCode); 
 
-void print_error(ErrorInfo &einfo);
 }
 
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index f53101629c5..35d2a1cdfe0 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -8,9 +8,9 @@ using std::nullopt;
 using std::cout;
 using std::endl;
 
-int main() {
-
-using namespace nix; 
+int main() 
+{
+  using namespace nix; 
 
   ErrorInfo::programName = optional("errorTest");
 
@@ -35,19 +35,23 @@ using namespace nix;
   generic.program = "nixtool.exe";
   generic.nixCode = nixcode;
 
-  print_error(generic);
+  printErrorInfo(generic);
   */
 
-  print_error(StandardError()
+  printErrorInfo(StandardError()
                 .name("name")
-                .description("description"));
+                .description("description")
+                .nohint()
+                );
 
-  print_error(StandardWarning()
+  printErrorInfo(StandardWarning()
                 .name("warning name")
-                .description("warning description"));
+                .description("warning description")
+                .nohint()
+                );
 
 
-  print_error(MkNixWarning()
+  printErrorInfo(MkNixWarning()
                 .name("warning name")
                 .description("warning description")
                 .nixFile("myfile.nix")
@@ -57,19 +61,17 @@ using namespace nix;
                             ,"this is the problem line of code"
                             ,nullopt));
 
-  print_error(MkNixError()
+  printErrorInfo(MkNixError()
                 .name("error name")
                 .description("error description")
                 .nixFile("myfile.nix")
                 .lineNumber(40)
                 .columnRange(13,7)
-                .linesOfCode(nullopt
+                .linesOfCode(optional("previous line of code")
                             ,"this is the problem line of code"
-                            ,nullopt));
+                            ,optional("next line of code"))
+                .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+                );
 
   return 0;
 }
-
-
-
-

From 00eb3fcb7a19a38d5ca94f4dcbda149f16e73a6e Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 27 Mar 2020 10:13:46 -0600
Subject: [PATCH 1884/2196] more cleanup

---
 src/libutil/error.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index d6595070f17..9194539a5a0 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -104,13 +104,13 @@ void printErrorInfo(ErrorInfo &einfo)
       }
     default: 
       {
-        levelString = "wat:";  
+        levelString = format("invalid error level: %1%") % einfo.level;  
         break;
       }
   }
 
   int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
-  int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; 
+  int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; 
 
   string dashes;
   for (int i = 0; i < dashwidth; ++i)
@@ -161,7 +161,6 @@ void printErrorInfo(ErrorInfo &einfo)
     cout << prefix << *einfo.hint << endl;
     cout << prefix << endl;
   }
-
 }
 
 }

From 759f39800bb80eaa297ea2532b1640959487ed9c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 27 Mar 2020 10:55:09 -0600
Subject: [PATCH 1885/2196] remove util.hh from deps

---
 src/libutil/ansicolor.hh | 13 +++++++++++++
 src/libutil/error.cc     |  2 +-
 src/libutil/error.hh     | 25 ++++++++++++++++---------
 src/libutil/util.hh      | 10 +---------
 4 files changed, 31 insertions(+), 19 deletions(-)
 create mode 100644 src/libutil/ansicolor.hh

diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
new file mode 100644
index 00000000000..390bd4d17c3
--- /dev/null
+++ b/src/libutil/ansicolor.hh
@@ -0,0 +1,13 @@
+#pragma once 
+
+namespace nix 
+{
+  /* Some ANSI escape sequences. */
+  #define ANSI_NORMAL "\e[0m"
+  #define ANSI_BOLD "\e[1m"
+  #define ANSI_FAINT "\e[2m"
+  #define ANSI_RED "\e[31;1m"
+  #define ANSI_GREEN "\e[32;1m"
+  #define ANSI_YELLOW "\e[33;1m"
+  #define ANSI_BLUE "\e[34;1m"
+}
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 9194539a5a0..c986affccba 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -104,7 +104,7 @@ void printErrorInfo(ErrorInfo &einfo)
       }
     default: 
       {
-        levelString = format("invalid error level: %1%") % einfo.level;  
+        levelString = (format("invalid error level: %1%") % einfo.level).str();  
         break;
       }
   }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index f02d7288abe..88980afb778 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -1,11 +1,13 @@
 #pragma once
 
-#include "util.hh"
+#include "ansicolor.hh"
 #include <string>
 #include <optional>
 #include <iostream>
 #include <iomanip>
 
+#include <boost/format.hpp>
+
 using std::string;
 using std::optional;
 using boost::format;
@@ -251,8 +253,14 @@ class AddHint : private T
 // --------------------------------------------------------
 // error types
 
-typedef AddName<AddDescription<AddHint<EIError>>> StandardError;
-typedef AddName<AddDescription<AddHint<EIWarning>>> StandardWarning;
+typedef AddName<
+        AddDescription<
+        AddHint<
+        EIError>>> StandardError;
+typedef AddName<
+        AddDescription<
+        AddHint<
+        EIWarning>>> StandardWarning;
 
 typedef AddName<
         AddDescription<
@@ -260,13 +268,16 @@ typedef AddName<
         AddLineNumber<
         AddColumnRange<
         AddLOC<
-        AddHint<EIError>>>>>>> MkNixError;
+        AddHint<
+        EIError>>>>>>> MkNixError;
 typedef AddName<
         AddDescription<
         AddNixFile<
         AddLineNumber<
         AddColumnRange<
-        AddLOC<EIWarning>>>>>> MkNixWarning;
+        AddLOC<
+        AddHint<
+        EIWarning>>>>>>> MkNixWarning;
 
 
 // --------------------------------------------------------
@@ -274,9 +285,5 @@ typedef AddName<
 
 void printErrorInfo(ErrorInfo &einfo);
 
-string showErrLine(ErrLine &errLine);
-
-void printCodeLines(string &prefix, NixCode &nixCode); 
-
 }
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3bfebcd153a..d69e2915879 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "logging.hh"
+#include "ansicolor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -423,15 +424,6 @@ std::string shellEscape(const std::string & s);
 void ignoreException();
 
 
-/* Some ANSI escape sequences. */
-#define ANSI_NORMAL "\e[0m"
-#define ANSI_BOLD "\e[1m"
-#define ANSI_FAINT "\e[2m"
-#define ANSI_RED "\e[31;1m"
-#define ANSI_GREEN "\e[32;1m"
-#define ANSI_YELLOW "\e[33;1m"
-#define ANSI_BLUE "\e[34;1m"
-
 
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,

From 225e62a56a7cebb030bebffb8d2bd7afe21cc64a Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Sun, 29 Mar 2020 01:04:55 -0400
Subject: [PATCH 1886/2196] Replace some `bool recursive` with a new
 `FileIngestionMethod` enum

---
 perl/lib/Nix/Store.xs                    |  4 ++--
 src/libexpr/eval.cc                      |  2 +-
 src/libexpr/primops.cc                   | 16 ++++++++--------
 src/libexpr/primops/fetchGit.cc          |  2 +-
 src/libexpr/primops/fetchMercurial.cc    |  2 +-
 src/libstore/binary-cache-store.cc       |  4 ++--
 src/libstore/binary-cache-store.hh       |  2 +-
 src/libstore/build.cc                    | 12 ++++++------
 src/libstore/daemon.cc                   | 24 ++++++++++++++++--------
 src/libstore/derivations.cc              |  6 +++---
 src/libstore/derivations.hh              |  2 +-
 src/libstore/download.cc                 |  9 +++++----
 src/libstore/legacy-ssh-store.cc         |  2 +-
 src/libstore/local-store.cc              | 12 ++++++------
 src/libstore/local-store.hh              |  4 ++--
 src/libstore/path.hh                     |  5 +++++
 src/libstore/remote-store.cc             | 12 +++++++-----
 src/libstore/remote-store.hh             |  2 +-
 src/libstore/store-api.cc                | 18 +++++++++---------
 src/libstore/store-api.hh                | 11 +++++------
 src/nix-prefetch-url/nix-prefetch-url.cc |  9 ++++++---
 src/nix-store/nix-store.cc               |  8 ++++----
 src/nix/add-to-store.cc                  |  4 ++--
 src/nix/command.cc                       | 10 +++++-----
 src/nix/command.hh                       |  4 ++--
 src/nix/copy.cc                          |  2 +-
 src/nix/make-content-addressable.cc      |  4 ++--
 27 files changed, 105 insertions(+), 87 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 1ca734e75bd..eab8ccacb5c 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -274,7 +274,7 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, (FileIngestionMethod) recursive, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -285,7 +285,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            auto path = store()->makeFixedOutputPath((FileIngestionMethod) recursive, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dac32b6f563..88bbf3b3206 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1661,7 +1661,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
     else {
         auto p = settings.readOnlyMode
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
-            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8de23495187..7d45733f444 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    bool outputHashRecursive = false;
+    FileIngestionMethod outputHashRecursive = FileIngestionMethod::Flat;
 
     StringSet outputs;
     outputs.insert("out");
@@ -574,8 +574,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
-            if (s == "recursive") outputHashRecursive = true;
-            else if (s == "flat") outputHashRecursive = false;
+            if (s == "recursive") outputHashRecursive = FileIngestionMethod::Recursive;
+            else if (s == "flat") outputHashRecursive = FileIngestionMethod::Flat;
             else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
         };
 
@@ -725,7 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
-                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+                (static_cast<bool>(outputHashRecursive) ? "r:" : "") + printHashType(h.type),
                 h.to_string(Base16, false)));
     }
 
@@ -1038,7 +1038,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
 
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
+    Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v)
 {
     const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
@@ -1095,7 +1095,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1104,7 +1104,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Path path;
     string name;
     Value * filterFun = nullptr;
-    auto recursive = true;
+    auto recursive = FileIngestionMethod::Recursive;
     Hash expectedHash;
 
     for (auto & attr : *args[0]->attrs) {
@@ -1120,7 +1120,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             state.forceValue(*attr.value);
             filterFun = attr.value;
         } else if (n == "recursive")
-            recursive = state.forceBool(*attr.value, *attr.pos);
+            recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
         else if (n == "sha256")
             expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 4aee1073eeb..9f2b673adc1 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -69,7 +69,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, FileIngestionMethod::Recursive, htSHA256, filter));
 
             return gitInfo;
         }
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index db274fa4f70..548f4e392e1 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -63,7 +63,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, FileIngestionMethod::Recursive, htSHA256, filter));
 
             return hgInfo;
         }
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 3a2d848615b..a61af4a001c 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (recursive) {
+    if (recursive == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index aa13c1cb454..1a1ea636c7c 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -79,7 +79,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0e3a23a4dc8..224633106ce 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2712,7 +2712,7 @@ struct RestrictedStore : public LocalFSStore
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
     { throw Error("addToStore"); }
 
@@ -2725,7 +2725,7 @@ struct RestrictedStore : public LocalFSStore
     }
 
     StorePath addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+        FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
     {
         auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
         goal.addDependency(path);
@@ -3647,10 +3647,10 @@ void DerivationGoal::registerOutputs()
 
         if (fixedOutput) {
 
-            bool recursive; Hash h;
+            FileIngestionMethod recursive; Hash h;
             i.second.parseHashInfo(recursive, h);
 
-            if (!recursive) {
+            if (!static_cast<bool>(recursive)) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
@@ -3659,7 +3659,7 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
-            Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
+            Hash h2 = static_cast<bool>(recursive) ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
 
             auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
 
@@ -3912,7 +3912,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
                 auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
 
-                auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
+                auto used = static_cast<bool>(recursive) ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
 
                 if (recursive && checks.ignoreSelfRefs)
                     used.erase(info.path);
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 8e9f9d71bf0..f1afdff699e 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -355,20 +355,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
-        bool fixed, recursive;
         std::string s, baseName;
-        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-        /* Compatibility hack. */
-        if (!fixed) {
-            s = "sha256";
-            recursive = true;
+        FileIngestionMethod method;
+        {
+            bool fixed, recursive;
+            from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+            method = FileIngestionMethod { recursive };
+            /* Compatibility hack. */
+            if (!fixed) {
+                s = "sha256";
+                method = FileIngestionMethod::Recursive;
+            }
         }
         HashType hashAlgo = parseHashType(s);
 
         TeeSource savedNAR(from);
         RetrieveRegularNARSink savedRegular;
 
-        if (recursive) {
+        if (method == FileIngestionMethod::Recursive) {
             /* Get the entire NAR dump from the client and save it to
                a string so that we can pass it to
                addToStoreFromDump(). */
@@ -380,7 +384,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (!savedRegular.regular) throw Error("regular file expected");
 
-        auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
+        auto path = store->addToStoreFromDump(
+            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
+            baseName,
+            method,
+            hashAlgo);
         logger->stopWork();
 
         to << store->printStorePath(path);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 973ddc86abf..5934c1912ce 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -9,13 +9,13 @@
 namespace nix {
 
 
-void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
+void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
 {
-    recursive = false;
+    recursive = FileIngestionMethod::Flat;
     string algo = hashAlgo;
 
     if (string(algo, 0, 2) == "r:") {
-        recursive = true;
+        recursive = FileIngestionMethod::Recursive;
         algo = string(algo, 2);
     }
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 7222d25e5c3..b1224b93bbb 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -22,7 +22,7 @@ struct DerivationOutput
         , hashAlgo(std::move(hashAlgo))
         , hash(std::move(hash))
     { }
-    void parseHashInfo(bool & recursive, Hash & hash) const;
+    void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 149c8476529..8a9b658990d 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -814,7 +814,8 @@ CachedDownloadResult Downloader::downloadCached(
 
     std::optional<StorePath> expectedStorePath;
     if (request.expectedHash) {
-        expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
+        auto method = request.unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+        expectedStorePath = store->makeFixedOutputPath(method, request.expectedHash, name);
         if (store->isValidPath(*expectedStorePath)) {
             CachedDownloadResult result;
             result.storePath = store->printStorePath(*expectedStorePath);
@@ -875,10 +876,10 @@ CachedDownloadResult Downloader::downloadCached(
                 StringSink sink;
                 dumpString(*res.data, sink);
                 Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
-                ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
+                ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
                 info.narHash = hashString(htSHA256, *sink.s);
                 info.narSize = sink.s->size();
-                info.ca = makeFixedOutputCA(false, hash);
+                info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
                 store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
                 storePath = info.path.clone();
             }
@@ -914,7 +915,7 @@ CachedDownloadResult Downloader::downloadCached(
             if (members.size() != 1)
                 throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
             auto topDir = tmpDir + "/" + members.begin()->name;
-            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
+            unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
         storePath = std::move(*unpackedStorePath);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 458266be00f..af20d389b06 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override
     { unsupported("addToStore"); }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ae7513ad815..746f81beb78 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -557,7 +557,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        bool recursive; Hash h;
+        FileIngestionMethod recursive; Hash h;
         out->second.parseHashInfo(recursive, h);
 
         check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
@@ -1043,7 +1043,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
-    bool recursive, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
@@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (recursive) {
+            if (recursive == FileIngestionMethod::Recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
             } else
@@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
                above (if called with recursive == true and hashAlgo ==
                sha256); otherwise, compute it here. */
             HashResult hash;
-            if (recursive) {
+            if (recursive == FileIngestionMethod::Recursive) {
                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
                 hash.second = dump.size();
             } else
@@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
 
 StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
 
@@ -1111,7 +1111,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (recursive)
+    if (recursive == FileIngestionMethod::Recursive)
         dumpPath(srcPath, sink, filter);
     else
         sink.s = make_ref<std::string>(readFile(srcPath));
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 16aeab0ad9e..c1e75390c5d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -149,7 +149,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     /* Like addToStore(), but the contents of the path are contained
@@ -157,7 +157,7 @@ public:
        true) or simply the contents of a regular file (if recursive ==
        false). */
     StorePath addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index c90bb1fff8c..73c4b8e2922 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -73,6 +73,11 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
+enum struct FileIngestionMethod : bool {
+    Flat = false,
+    Recursive = true
+};
+
 struct StorePathWithOutputs
 {
     StorePath path;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8c55da2686e..5c36693e699 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -484,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
@@ -492,10 +492,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
 
     Path srcPath(absPath(_srcPath));
 
-    conn->to << wopAddToStore << name
-       << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
-       << (recursive ? 1 : 0)
-       << printHashType(hashAlgo);
+    conn->to
+        << wopAddToStore
+        << name
+        << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+        << (method == FileIngestionMethod::Recursive ? 1 : 0)
+        << printHashType(hashAlgo);
 
     try {
         conn->to.written = 0;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index f301a97d8c4..3c86b4524eb 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -65,7 +65,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b9e894a9a3f..06aa0883c10 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -171,18 +171,18 @@ static std::string makeType(
 
 
 StorePath Store::makeFixedOutputPath(
-    bool recursive,
+    FileIngestionMethod recursive,
     const Hash & hash,
     std::string_view name,
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256 && recursive) {
+    if (hash.type == htSHA256  && recursive == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
         return makeStorePath("output:out", hashString(htSHA256,
-                "fixed:out:" + (recursive ? (string) "r:" : "") +
+                "fixed:out:" + (static_cast<bool>(recursive) ? (string) "r:" : "") +
                 hash.to_string(Base16) + ":"), name);
     }
 }
@@ -200,9 +200,9 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
 
 
 std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
-    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
+    const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const
 {
-    Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
+    Hash h = static_cast<bool>(recursive) ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
     return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
 }
 
@@ -781,8 +781,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     }
 
     else if (hasPrefix(ca, "fixed:")) {
-        bool recursive = ca.compare(6, 2, "r:") == 0;
-        Hash hash(std::string(ca, recursive ? 8 : 6));
+        FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
+        Hash hash(std::string(ca, static_cast<bool>(recursive) ? 8 : 6));
         auto refs = cloneStorePathSet(references);
         bool hasSelfReference = false;
         if (refs.count(path)) {
@@ -826,9 +826,9 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
-std::string makeFixedOutputCA(bool recursive, const Hash & hash)
+std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 {
-    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
+    return "fixed:" + (static_cast<bool>(recursive) ? (std::string) "r:" : "") + hash.to_string();
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0fa59be6a61..32c24c5009b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -43,7 +43,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
 enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 
-
 /* Magic header of exportPath() output (obsolete). */
 const uint32_t exportMagic = 0x4558494e;
 
@@ -346,7 +345,7 @@ public:
     StorePath makeOutputPath(const string & id,
         const Hash & hash, std::string_view name) const;
 
-    StorePath makeFixedOutputPath(bool recursive,
+    StorePath makeFixedOutputPath(FileIngestionMethod method,
         const Hash & hash, std::string_view name,
         const StorePathSet & references = {},
         bool hasSelfReference = false) const;
@@ -358,7 +357,7 @@ public:
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
     std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
-        const Path & srcPath, bool recursive = true,
+        const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
         HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
@@ -462,12 +461,12 @@ public:
        The function object `filter' can be used to exclude files (see
        libutil/archive.hh). */
     virtual StorePath addToStore(const string & name, const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
     {
         throw Error("addToStoreFromDump() is not supported by this store");
     }
@@ -850,7 +849,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
 
 /* Compute the content-addressability assertion (ValidPathInfo::ca)
    for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(bool recursive, const Hash & hash);
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
 
 
 /* Split URI into protocol+hierarchy part and its parameter set. */
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 18ced94b1a7..8c364a86a02 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -159,7 +159,8 @@ static int _main(int argc, char * * argv)
         std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
-            storePath = store->makeFixedOutputPath(unpack, expectedHash, name);
+            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
             if (store->isValidPath(*storePath))
                 hash = expectedHash;
             else
@@ -208,13 +209,15 @@ static int _main(int argc, char * * argv)
             if (expectedHash != Hash(ht) && expectedHash != hash)
                 throw Error(format("hash mismatch for '%1%'") % uri);
 
+            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+
             /* Copy the file to the Nix store. FIXME: if RemoteStore
                implemented addToStoreFromDump() and downloadFile()
                supported a sink, we could stream the download directly
                into the Nix store. */
-            storePath = store->addToStore(name, tmpFile, unpack, ht);
+            storePath = store->addToStore(name, tmpFile, recursive, ht);
 
-            assert(*storePath == store->makeFixedOutputPath(unpack, hash, name));
+            assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
         }
 
         stopProgressBar();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 806ab756381..9a26eb57f1d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    bool recursive = false;
+    FileIngestionMethod recursive = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
-        if (i == "--recursive") recursive = true;
+        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
         else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (opArgs.empty())
@@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
 /* Hack to support caching in `nix-prefetch-url'. */
 static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 {
-    bool recursive = false;
+    FileIngestionMethod recursive = FileIngestionMethod::Flat;
 
     for (auto i : opFlags)
-        if (i == "--recursive") recursive = true;
+        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
         else throw UsageError(format("unknown flag '%1%'") % i);
 
     if (opArgs.size() != 3)
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 139db3657b5..e82ab844ed2 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -42,10 +42,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         auto narHash = hashString(htSHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart));
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
         info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(true, info.narHash);
+        info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
 
         if (!dryRun)
             store->addToStore(info, sink.s);
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 442bc6c53ec..fce6c391cfc 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -28,20 +28,20 @@ void StoreCommand::run()
     run(getStore());
 }
 
-StorePathsCommand::StorePathsCommand(bool recursive)
+StorePathsCommand::StorePathsCommand(FileIngestionMethod recursive)
     : recursive(recursive)
 {
-    if (recursive)
+    if (recursive == FileIngestionMethod::Recursive)
         mkFlag()
             .longName("no-recursive")
             .description("apply operation to specified paths only")
-            .set(&this->recursive, false);
+            .set(&this->recursive, FileIngestionMethod::Flat);
     else
         mkFlag()
             .longName("recursive")
             .shortName('r')
             .description("apply operation to closure of the specified paths")
-            .set(&this->recursive, true);
+            .set(&this->recursive, FileIngestionMethod::Recursive);
 
     mkFlag(0, "all", "apply operation to the entire store", &all);
 }
@@ -61,7 +61,7 @@ void StorePathsCommand::run(ref<Store> store)
         for (auto & p : toStorePaths(store, realiseMode, installables))
             storePaths.push_back(p.clone());
 
-        if (recursive) {
+        if (recursive == FileIngestionMethod::Recursive) {
             StorePathSet closure;
             store->computeFSClosure(storePathsToSet(storePaths), closure, false, false);
             storePaths.clear();
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a954a7d04fe..4cdda9f799d 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -114,7 +114,7 @@ struct StorePathsCommand : public InstallablesCommand
 {
 private:
 
-    bool recursive = false;
+    FileIngestionMethod recursive = FileIngestionMethod::Flat;
     bool all = false;
 
 protected:
@@ -123,7 +123,7 @@ protected:
 
 public:
 
-    StorePathsCommand(bool recursive = false);
+    StorePathsCommand(FileIngestionMethod recursive = FileIngestionMethod::Flat);
 
     using StoreCommand::run;
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 85c777d38ab..a8ee3fce3fe 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -17,7 +17,7 @@ struct CmdCopy : StorePathsCommand
     SubstituteFlag substitute = NoSubstitute;
 
     CmdCopy()
-        : StorePathsCommand(true)
+        : StorePathsCommand(FileIngestionMethod::Recursive)
     {
         mkFlag()
             .longName("from")
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index f9c7fef3f01..93eddbb1fad 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -74,12 +74,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
             auto narHash = hashModuloSink.finish().first;
 
-            ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference));
+            ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference));
             info.references = std::move(references);
             if (hasSelfReference) info.references.insert(info.path.clone());
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.ca = makeFixedOutputCA(true, info.narHash);
+            info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
 
             if (!json)
                 printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));

From f58604ac32d0ab0718a0e68b3c1b050e4ab121cb Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Fri, 25 Oct 2019 17:45:14 +0200
Subject: [PATCH 1887/2196] Add fetchSubmodules to builtins.fetchGit

There are some downsides to this features:

 - Submodules are not cached (unlike the root repo),
 - Full checkouts are created in a temporary directory.
---
 src/libexpr/primops/fetchGit.cc | 45 +++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 4aee1073eeb..b08e484041c 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -22,13 +22,14 @@ struct GitInfo
     std::string rev;
     std::string shortRev;
     uint64_t revCount = 0;
+    bool submodules = false;
 };
 
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::optional<std::string> ref, std::string rev,
-    const std::string & name)
+    const std::string & name, bool fetchSubmodules)
 {
     if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
@@ -145,7 +146,8 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
 
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev
+        + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";
     PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
 
@@ -165,16 +167,35 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
-    unpackTarfile(*source, tmpDir);
+    // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root
+    // repo is cached.
+    if (fetchSubmodules) {
+        Path tmpGitDir = createTempDir();
+        AutoDelete delTmpGitDir(tmpGitDir, true);
+
+        runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+        runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
+                                  "--", cacheDir, fmt("%s:%s", *ref, *ref) });
+
+        runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
+        runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
+        runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" });
+
+        deletePath(tmpDir + "/.git");
+
+        gitInfo.submodules = true;
+    } else {
+        auto source = sinkToSource([&](Sink & sink) {
+            RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+            gitOptions.standardOut = &sink;
+            runProgram2(gitOptions);
+        });
+
+        unpackTarfile(*source, tmpDir);
+    }
 
     gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
@@ -198,6 +219,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
+    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -216,6 +238,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "fetchSubmodules")
+                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
@@ -230,13 +254,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules);
 
     state.mkAttrs(v, 8);
     mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
     mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules);
     v.attrs->sort();
 
     if (state.allowedPaths)

From c2a24c2b8876a0862ede39f63e684cb875cc7aab Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Sat, 26 Oct 2019 11:09:50 +0200
Subject: [PATCH 1888/2196] Add test for fetchGit submodule support

---
 tests/fetchGitSubmodules.sh | 50 +++++++++++++++++++++++++++++++++++++
 tests/local.mk              |  1 +
 2 files changed, 51 insertions(+)
 create mode 100644 tests/fetchGitSubmodules.sh

diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
new file mode 100644
index 00000000000..c821d2caa0b
--- /dev/null
+++ b/tests/fetchGitSubmodules.sh
@@ -0,0 +1,50 @@
+source common.sh
+
+set -u
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git submodule tests"
+    exit 99
+fi
+
+clearStore
+
+rootRepo=$TEST_ROOT/gitSubmodulesRoot
+subRepo=$TEST_ROOT/gitSubmodulesSub
+
+rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2
+
+initGitRepo() {
+    git init $1
+    git -C $1 config user.email "foobar@example.com"
+    git -C $1 config user.name "Foobar"
+}
+
+addGitContent() {
+    echo "lorem ipsum" > $1/content
+    git -C $1 add content
+    git -C $1 commit -m "Initial commit"
+}
+
+initGitRepo $subRepo
+addGitContent $subRepo
+
+initGitRepo $rootRepo
+
+git -C $rootRepo submodule init
+git -C $rootRepo submodule add $subRepo sub
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Add submodule"
+
+rev=$(git -C $rootRepo rev-parse HEAD)
+
+pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
+
+# The resulting store path cannot be the same.
+[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
+
+[[ ! -e $pathWithoutSubmodules/sub/content ]]
+[[ -e $pathWithSubmodules/sub/content ]]
+
+
diff --git a/tests/local.mk b/tests/local.mk
index dab3a23b6b9..01fac4fcdb1 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -17,6 +17,7 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
+  fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \
   run.sh \

From ea861be292d790db9835c08abee734baa77b9cf7 Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Sat, 26 Oct 2019 11:13:08 +0200
Subject: [PATCH 1889/2196] Add documentation for submodule support in fetchGit

---
 doc/manual/expressions/builtins.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 394e1fc32c9..838edae3c9a 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term>fetchSubmodules</term>
+          <listitem>
+            <para>
+              A boolean parameter that specifies whether submodules
+              should be checked out. Defaults to
+              <literal>false</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <example>

From c8d33de77781fe7850d04a7374c6c22a2e995b70 Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Mon, 2 Mar 2020 14:42:19 +0100
Subject: [PATCH 1890/2196] Add git submodule fixes from @bjornfor

This fixes fetching repositories with no submodules and also cleans up
.git files in checkouts.
---
 src/libexpr/local.mk            | 1 +
 src/libexpr/primops/fetchGit.cc | 9 +++++++--
 tests/fetchGitSubmodules.sh     | 3 ++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 8a9b3c2ea12..9b5fcc561ca 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
+
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index b08e484041c..c58d28bbce8 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -6,6 +6,7 @@
 #include "hash.hh"
 #include "tarfile.hh"
 
+#include <filesystem>
 #include <sys/time.h>
 
 #include <regex>
@@ -182,9 +183,13 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
-        runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" });
+        runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
 
-        deletePath(tmpDir + "/.git");
+        for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) {
+            if (p.path().filename() == ".git") {
+                std::filesystem::remove_all(p.path());
+            }
+        }
 
         gitInfo.submodules = true;
     } else {
diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index c821d2caa0b..987bfd69efc 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -47,4 +47,5 @@ pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo
 [[ ! -e $pathWithoutSubmodules/sub/content ]]
 [[ -e $pathWithSubmodules/sub/content ]]
 
-
+# No .git directory or submodule reference files must be left
+test "$(find "$pathWithSubmodules" -name .git)" = ""

From c846abb5cc900d5fcb5aeb48427a1eac646cc0f3 Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Mon, 2 Mar 2020 14:46:19 +0100
Subject: [PATCH 1891/2196] Add more test for git submodule functionality

---
 tests/fetchGitSubmodules.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 987bfd69efc..c252d42f74a 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -40,12 +40,23 @@ rev=$(git -C $rootRepo rev-parse HEAD)
 
 pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
 pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
 
 # The resulting store path cannot be the same.
 [[ $pathWithoutSubmodules != $pathWithSubmodules ]]
 
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
+
+# The submodules flag is actually honored.
 [[ ! -e $pathWithoutSubmodules/sub/content ]]
 [[ -e $pathWithSubmodules/sub/content ]]
 
 # No .git directory or submodule reference files must be left
 test "$(find "$pathWithSubmodules" -name .git)" = ""
+
+# Git repos without submodules can be fetched with submodules = true.
+noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
+
+[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]

From 435366ed3c292756bb74fd12f47ff96e93cd48ac Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Mon, 2 Mar 2020 14:50:13 +0100
Subject: [PATCH 1892/2196] Rename fetchGit fetchSubmodules to just submodules

---
 src/libexpr/primops/fetchGit.cc | 2 +-
 tests/fetchGitSubmodules.sh     | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index c58d28bbce8..e8905b54839 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -243,7 +243,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "fetchSubmodules")
+            else if (n == "submodules")
                 fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index c252d42f74a..6e4ef270b7a 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -39,8 +39,8 @@ git -C $rootRepo commit -m "Add submodule"
 rev=$(git -C $rootRepo rev-parse HEAD)
 
 pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
-pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
-pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
 
 # The resulting store path cannot be the same.
 [[ $pathWithoutSubmodules != $pathWithSubmodules ]]
@@ -57,6 +57,6 @@ test "$(find "$pathWithSubmodules" -name .git)" = ""
 
 # Git repos without submodules can be fetched with submodules = true.
 noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath")
-noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath")
 
 [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]

From 6c00a9545f2162496032c490aadbadd85856662c Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Mon, 2 Mar 2020 15:50:39 +0100
Subject: [PATCH 1893/2196] Fix typo in submodule test

---
 tests/fetchGitSubmodules.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 6e4ef270b7a..9a21421cc37 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -56,7 +56,8 @@ pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$roo
 test "$(find "$pathWithSubmodules" -name .git)" = ""
 
 # Git repos without submodules can be fetched with submodules = true.
-noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath")
-noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath")
+subRev=$(git -C $subRepo rev-parse HEAD)
+noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
 
 [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]

From cc4fe977e5eedf00d8e3d267ccbc3676b3fac1a0 Mon Sep 17 00:00:00 2001
From: Julian Stecklina <julian.stecklina@cyberus-technology.de>
Date: Mon, 9 Mar 2020 14:07:26 +0100
Subject: [PATCH 1894/2196] Link to stdc++fs

Some platforms seem to still require linking with stdc++fs to enable
STL std::filesystem support.
---
 src/libexpr/local.mk | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 9b5fcc561ca..b6ee82424e4 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -8,7 +8,11 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 
 libexpr_LIBS = libutil libstore libnixrust
 
+ifeq ($(CXX), g++)
+libexpr_LDFLAGS = -lstdc++fs
+else
 libexpr_LDFLAGS =
+endif
 
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl

From 002a3a95dcb7391a8372ba7a74f4e7e9bd48b59b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Mon, 16 Mar 2020 14:47:21 +0100
Subject: [PATCH 1895/2196] fetchGit: fix "fatal: couldn't find remote ref
 refs/heads/master" issue with submodules

---
 src/libexpr/primops/fetchGit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index e8905b54839..66259d778e6 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -179,7 +179,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
         runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                                  "--", cacheDir, fmt("%s:%s", *ref, *ref) });
+                                  "--", cacheDir, fmt("%s", *ref) });
 
         runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });

From 587e259bfdda248fb2d81389faca816b8efdb5f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Mon, 16 Mar 2020 14:47:55 +0100
Subject: [PATCH 1896/2196] tests/fetchGitSubmodules: add more tests

---
 tests/fetchGitSubmodules.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 9a21421cc37..8ee1bb0679a 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -42,6 +42,25 @@ pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootR
 pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
 pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
 
+r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
+r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r2 ]]
+[[ $r2 != $r3 ]]
+
+r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath")
+r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath")
+r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r4 ]]
+[[ $r4 == $r5 ]]
+[[ $r3 == $r6 ]]
+[[ $r6 == $r7 ]]
+[[ $r7 == $r8 ]]
+
 # The resulting store path cannot be the same.
 [[ $pathWithoutSubmodules != $pathWithSubmodules ]]
 

From 6864ad7cf510d96f0cfc0cdbeffc1188a4eab44c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Tue, 17 Mar 2020 14:27:14 +0100
Subject: [PATCH 1897/2196] fetchGit: fix submodule output attribute

Before this change it would be false for all evaluations but the first.
Now it follows the input argument (as it should).
---
 src/libexpr/primops/fetchGit.cc | 7 +++----
 tests/fetchGitSubmodules.sh     | 9 +++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 66259d778e6..653d99b53a4 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -32,6 +32,9 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::optional<std::string> ref, std::string rev,
     const std::string & name, bool fetchSubmodules)
 {
+    GitInfo gitInfo;
+    gitInfo.submodules = fetchSubmodules;
+
     if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
@@ -49,7 +52,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (!clean) {
 
             /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
@@ -141,7 +143,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     }
 
     // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
     gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
     gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
@@ -190,8 +191,6 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 std::filesystem::remove_all(p.path());
             }
         }
-
-        gitInfo.submodules = true;
     } else {
         auto source = sinkToSource([&](Sink & sink) {
             RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 8ee1bb0679a..2d625c376bd 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -61,6 +61,15 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo
 [[ $r6 == $r7 ]]
 [[ $r7 == $r8 ]]
 
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
+[[ $have_submodules == true ]]
+
 # The resulting store path cannot be the same.
 [[ $pathWithoutSubmodules != $pathWithSubmodules ]]
 

From 369fffd6f1f7d939e528b3591dca49989c51ae35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Tue, 17 Mar 2020 14:27:58 +0100
Subject: [PATCH 1898/2196] fetchGit: add submodules attribute to the .link
 file

The .link file is used as a lock, so I think we should put the
"submodule" attribute in there since turning on submodules creates a new
.link file path.
---
 src/libexpr/primops/fetchGit.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 653d99b53a4..969b1a4edf0 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -211,6 +211,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     json["name"] = name;
     json["rev"] = gitInfo.rev;
     json["revCount"] = gitInfo.revCount;
+    json["submodules"] = gitInfo.submodules;
 
     writeFile(storeLink, json.dump());
 

From be84049baf047fd2eee08def5c8013797c0832a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Tue, 17 Mar 2020 20:14:19 +0100
Subject: [PATCH 1899/2196] tests/fetchGitSubmodules.sh: more checks

---
 tests/fetchGitSubmodules.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 2d625c376bd..30a8737bd7e 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -38,10 +38,6 @@ git -C $rootRepo commit -m "Add submodule"
 
 rev=$(git -C $rootRepo rev-parse HEAD)
 
-pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
-pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
-pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
-
 r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
 r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
 r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
@@ -70,16 +66,26 @@ have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"
 have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
 [[ $have_submodules == true ]]
 
+pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+
 # The resulting store path cannot be the same.
 [[ $pathWithoutSubmodules != $pathWithSubmodules ]]
 
 # Checking out the same repo with submodules returns in the same store path.
 [[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
 
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]
+
 # The submodules flag is actually honored.
 [[ ! -e $pathWithoutSubmodules/sub/content ]]
 [[ -e $pathWithSubmodules/sub/content ]]
 
+[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]
+
 # No .git directory or submodule reference files must be left
 test "$(find "$pathWithSubmodules" -name .git)" = ""
 

From b306b7039ef3f2b76dbcc8020dcf67ed27442e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Wed, 18 Mar 2020 08:44:37 +0100
Subject: [PATCH 1900/2196] fetchGit: checkout rev instead of latest ref

Major bugfix for the submodules = true code path.

TODO: Add tests.
---
 src/libexpr/primops/fetchGit.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 969b1a4edf0..0c6bc6dc020 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -182,7 +182,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
                                   "--", cacheDir, fmt("%s", *ref) });
 
-        runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
+        runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev });
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
         runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
 

From cc522d0d23b71faeac95998d264abe14ddceeed5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Wed, 18 Mar 2020 08:45:31 +0100
Subject: [PATCH 1901/2196] fetchGit: fix submodules = true for dirty trees

---
 src/libexpr/primops/fetchGit.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 0c6bc6dc020..66e1d7b9857 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -55,8 +55,12 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
+            auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" });
+            if (fetchSubmodules) {
+                gitOpts.emplace_back("--recurse-submodules");
+            }
             auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
+                runProgram("git", true, gitOpts), "\0"s);
 
             PathFilter filter = [&](const Path & p) -> bool {
                 assert(hasPrefix(p, uri));

From f686efeed4d6305101c56136855e2d6b87e649e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= <bjorn.forsman@gmail.com>
Date: Fri, 20 Mar 2020 14:36:10 +0100
Subject: [PATCH 1902/2196] fetchGit: fix submodule corner case by fetching all
 refs from cacheDir

Due to fetchGit not checking if rev is an ancestor of ref (there is even
a FIXME comment about it in the code), the cache repo might not have the
ref even though it has the rev. This doesn't matter when submodule =
false, but the submodule = true code blows up because it tries to fetch
the (missing) ref from the cache repo.

Fix this in the simplest way possible: fetch all refs from the local
cache repo when submodules = true.

TODO: Add tests.
---
 src/libexpr/primops/fetchGit.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 66e1d7b9857..f138b755c8b 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -183,8 +183,11 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         AutoDelete delTmpGitDir(tmpGitDir, true);
 
         runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+        // TODO: the cacheDir repo might lack the ref (it only checks if rev
+        // exists, see FIXME above) so use a big hammer and fetch everything to
+        // ensure we get the rev.
         runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                                  "--", cacheDir, fmt("%s", *ref) });
+                                  "--update-head-ok", "--", cacheDir, "refs/*:refs/*" });
 
         runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev });
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });

From 40c023ecfe49fea6e66db34c5f841fcf7001cbeb Mon Sep 17 00:00:00 2001
From: Julian Stecklina <js@alien8.de>
Date: Sun, 29 Mar 2020 23:47:48 +0200
Subject: [PATCH 1903/2196] fetchGit: don't use std::filesystem to filter git
 repos

Using std::filesystem means also having to link with -lstdc++fs on
some platforms and it's hard to discover for what platforms this is
needed. As all the functionality is already implemented as utilities,
use those instead.
---
 src/libexpr/local.mk            |  4 ----
 src/libexpr/primops/fetchGit.cc | 17 ++++++++++-------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index b6ee82424e4..9b5fcc561ca 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -8,11 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 
 libexpr_LIBS = libutil libstore libnixrust
 
-ifeq ($(CXX), g++)
-libexpr_LDFLAGS = -lstdc++fs
-else
 libexpr_LDFLAGS =
-endif
 
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index f138b755c8b..d60e6b803ff 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -6,7 +6,6 @@
 #include "hash.hh"
 #include "tarfile.hh"
 
-#include <filesystem>
 #include <sys/time.h>
 
 #include <regex>
@@ -28,6 +27,13 @@ struct GitInfo
 
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
+static bool isNotDotGitDirectory(const Path & path)
+{
+  static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
+
+  return not std::regex_match(path, gitDirRegex);
+}
+
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::optional<std::string> ref, std::string rev,
     const std::string & name, bool fetchSubmodules)
@@ -175,6 +181,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
+    PathFilter filter = defaultPathFilter;
 
     // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root
     // repo is cached.
@@ -193,11 +200,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
         runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
 
-        for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) {
-            if (p.path().filename() == ".git") {
-                std::filesystem::remove_all(p.path());
-            }
-        }
+	filter = isNotDotGitDirectory;
     } else {
         auto source = sinkToSource([&](Sink & sink) {
             RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
@@ -208,7 +211,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         unpackTarfile(*source, tmpDir);
     }
 
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter));
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
 

From e322a16523c3c0644a900453abbe32bd0fd982e3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Mar 2020 14:29:29 +0200
Subject: [PATCH 1904/2196] Remove global -I flags

(cherry picked from commit 2c692a3b144523bca68dd6de618124ba6c9bb332)
---
 local.mk                                       | 4 +++-
 mk/precompiled-headers.mk                      | 4 ++--
 src/build-remote/build-remote.cc               | 2 +-
 src/libexpr/local.mk                           | 2 ++
 src/libexpr/primops/fromTOML.cc                | 2 +-
 src/libmain/local.mk                           | 2 ++
 src/libstore/local.mk                          | 3 ++-
 src/nix-build/nix-build.cc                     | 2 +-
 src/nix-channel/nix-channel.cc                 | 2 +-
 src/nix-collect-garbage/nix-collect-garbage.cc | 2 +-
 src/nix-copy-closure/nix-copy-closure.cc       | 2 +-
 src/nix-daemon/nix-daemon.cc                   | 2 +-
 src/nix-env/nix-env.cc                         | 2 +-
 src/nix-instantiate/nix-instantiate.cc         | 2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc       | 4 ++--
 src/nix-store/nix-store.cc                     | 2 +-
 src/nix/local.mk                               | 2 ++
 tests/plugins/local.mk                         | 2 ++
 18 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/local.mk b/local.mk
index 55f85a04406..d254c10fe9b 100644
--- a/local.mk
+++ b/local.mk
@@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 1a727ba1b70..1c0452dc2a5 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 69d1c6f7e46..00340b7872a 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 8a9b3c2ea12..a4ccab376a4 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,6 +6,8 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+
 libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index a84e569e944..c43324dbb51 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 
diff --git a/src/libmain/local.mk b/src/libmain/local.mk
index 0c80f5a0a03..a8eed6c65fe 100644
--- a/src/libmain/local.mk
+++ b/src/libmain/local.mk
@@ -6,6 +6,8 @@ libmain_DIR := $(d)
 
 libmain_SOURCES := $(wildcard $(d)/*.cc)
 
+libmain_CXXFLAGS += -I src/libutil -I src/libstore
+
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index ac68c2342b5..91acef368fe 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
-libstore_CXXFLAGS = \
+libstore_CXXFLAGS += \
+ -I src/libutil -I src/libstore \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 1bda159d027..0a058a31b49 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -16,7 +16,7 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 1b337a7125d..a2639579dd9 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -2,7 +2,7 @@
 #include "globals.hh"
 #include "download.hh"
 #include "store-api.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <fcntl.h>
 #include <regex>
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index d4060ac937f..aa5ada3a62b 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -2,7 +2,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <iostream>
 #include <cerrno>
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index f8703576042..b10184718d1 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,6 +1,6 @@
 #include "shared.hh"
 #include "store-api.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 1348985619e..e68d1b1bed6 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -6,7 +6,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "finally.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 #include "daemon.hh"
 
 #include <algorithm>
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index e47b00acf3b..1a2bb42a3c2 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -13,7 +13,7 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <cerrno>
 #include <ctime>
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 617d927a4c1..6c99d118180 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -9,7 +9,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <map>
 #include <iostream>
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 18ced94b1a7..2b9254659a7 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -6,9 +6,9 @@
 #include "eval-inline.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
 #include "finally.hh"
-#include "progress-bar.hh"
+#include "../nix/legacy.hh"
+#include "../nix/progress-bar.hh"
 #include "tarfile.hh"
 
 #include <iostream>
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 806ab756381..fcc00175a99 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,7 +9,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "graphml.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <iostream>
 #include <algorithm>
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 51dad101f56..50a18efd79d 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,6 +15,8 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain
+
 nix_LIBS = libexpr libmain libstore libutil libnixrust
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk
index 1d2bac052fd..82ad9940286 100644
--- a/tests/plugins/local.mk
+++ b/tests/plugins/local.mk
@@ -7,3 +7,5 @@ libplugintest_SOURCES := $(d)/plugintest.cc
 libplugintest_ALLOW_UNDEFINED := 1
 
 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
+
+libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr

From 367577d9a6ec465d773f7b3ba53a5656253f514d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Mar 2020 17:00:40 +0200
Subject: [PATCH 1905/2196] Fix macOS build

---
 src/resolve-system-dependencies/local.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk
index bf65f7905f4..f0e82e02369 100644
--- a/src/resolve-system-dependencies/local.mk
+++ b/src/resolve-system-dependencies/local.mk
@@ -6,6 +6,8 @@ resolve-system-dependencies_DIR := $(d)
 
 resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix
 
+resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain
+
 resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust
 
 resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc

From 35c7bab09ac97efd3561d3c5a0c827e4dc391141 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 30 Mar 2020 09:14:29 -0600
Subject: [PATCH 1906/2196] build with make

---
 Makefile              |  3 ++-
 src/libutil/util.hh   |  1 +
 tests/errors/local.mk | 17 +++++++++++++++++
 tests/errors/main.cc  |  6 ++++--
 4 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 tests/errors/local.mk

diff --git a/Makefile b/Makefile
index 469070533b5..2d70aeb27c7 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,8 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk
+  tests/plugins/local.mk \
+  tests/errors/local.mk
 
 -include Makefile.config
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index d69e2915879..0b55c67882c 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "logging.hh"
 #include "ansicolor.hh"
+#include "error.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/tests/errors/local.mk b/tests/errors/local.mk
new file mode 100644
index 00000000000..c5b9cc5c8fb
--- /dev/null
+++ b/tests/errors/local.mk
@@ -0,0 +1,17 @@
+programs += error-test
+
+error-test_DIR := $(d)
+
+error-test_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+error-test_LIBS = libutil
+
+error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
+
+# $(foreach name, \
+#   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
+#   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
+# $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
+
+# src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 35d2a1cdfe0..39adc2ece9a 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -12,7 +12,7 @@ int main()
 {
   using namespace nix; 
 
-  ErrorInfo::programName = optional("errorTest");
+  ErrorInfo::programName = optional("error-test");
 
   /*
   ColumnRange columnRange;
@@ -59,7 +59,9 @@ int main()
                 .columnRange(13,7)
                 .linesOfCode(nullopt
                             ,"this is the problem line of code"
-                            ,nullopt));
+                            ,nullopt)
+                .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+                );
 
   printErrorInfo(MkNixError()
                 .name("error name")

From 28d073e810d5e692a4f6c57d23afc011e6fcc153 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 30 Mar 2020 09:15:21 -0600
Subject: [PATCH 1907/2196] remove cruft

---
 tests/errors/local.mk | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/errors/local.mk b/tests/errors/local.mk
index c5b9cc5c8fb..0d64c8dbdd5 100644
--- a/tests/errors/local.mk
+++ b/tests/errors/local.mk
@@ -8,10 +8,3 @@ error-test_SOURCES := \
 error-test_LIBS = libutil
 
 error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
-
-# $(foreach name, \
-#   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
-#   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
-# $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
-
-# src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh

From e1a94ad852e91fcdcf4efb60fe7e9b9e328df7ac Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Mar 2020 19:14:17 +0200
Subject: [PATCH 1908/2196] Backport 'nix dev-shell' from the flakes branch

This also adds a '--profile' option to 'nix build' (replacing 'nix-env
--set').
---
 src/libutil/util.cc     |  11 ++
 src/libutil/util.hh     |  12 +-
 src/nix/build.cc        |  17 ++-
 src/nix/command.cc      |  94 ++++++++++++
 src/nix/command.hh      |  62 ++++----
 src/nix/installables.cc |   5 +
 src/nix/installables.hh |  45 ++++++
 src/nix/run.cc          | 120 ++++++---------
 src/nix/shell.cc        | 319 ++++++++++++++++++++++++++++++++++++++++
 9 files changed, 569 insertions(+), 116 deletions(-)
 create mode 100644 src/nix/installables.hh
 create mode 100644 src/nix/shell.cc

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 097ff210aff..332c1c43a76 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -478,6 +478,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 }
 
 
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    return {std::move(fd), tmpl};
+}
+
+
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 7c3a3024253..1f85c7c462e 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -122,10 +122,6 @@ void deletePath(const Path & path);
 
 void deletePath(const Path & path, unsigned long long & bytesFreed);
 
-/* Create a temporary directory. */
-Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
-    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
-
 std::string getUserName();
 
 /* Return $HOME or the user's home directory from /etc/passwd. */
@@ -205,6 +201,14 @@ public:
 };
 
 
+/* Create a temporary directory. */
+Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
+    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
+
+/* Create a temporary file, returning a file handle and its path. */
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
+
+
 class Pipe
 {
 public:
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 3c9d2df3954..0b07628364c 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -5,7 +5,7 @@
 
 using namespace nix;
 
-struct CmdBuild : MixDryRun, InstallablesCommand
+struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 {
     Path outLink = "result";
 
@@ -40,6 +40,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand
                 "To build the build.x86_64-linux attribute from release.nix:",
                 "nix build -f release.nix build.x86_64-linux"
             },
+            Example{
+                "To make a profile point at GNU Hello:",
+                "nix build --profile /tmp/profile nixpkgs.hello"
+            },
         };
     }
 
@@ -49,18 +53,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
         if (dryRun) return;
 
-        for (size_t i = 0; i < buildables.size(); ++i) {
-            auto & b(buildables[i]);
-
-            if (outLink != "")
-                for (auto & output : b.outputs)
+        if (outLink != "") {
+            for (size_t i = 0; i < buildables.size(); ++i) {
+                for (auto & output : buildables[i].outputs)
                     if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
                         std::string symlink = outLink;
                         if (i) symlink += fmt("-%d", i);
                         if (output.first != "out") symlink += fmt("-%s", output.first);
                         store2->addPermRoot(output.second, absPath(symlink), true);
                     }
+            }
         }
+
+        updateProfile(buildables);
     }
 };
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 442bc6c53ec..99b24d2a2e3 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -2,6 +2,9 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "nixexpr.hh"
+#include "profiles.hh"
+
+extern char * * environ;
 
 namespace nix {
 
@@ -96,4 +99,95 @@ Strings editorFor(const Pos & pos)
     return args;
 }
 
+MixProfile::MixProfile()
+{
+    mkFlag()
+        .longName("profile")
+        .description("profile to update")
+        .labels({"path"})
+        .dest(&profile);
+}
+
+void MixProfile::updateProfile(const StorePath & storePath)
+{
+    if (!profile) return;
+    auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
+    if (!store) throw Error("'--profile' is not supported for this Nix store");
+    auto profile2 = absPath(*profile);
+    switchLink(profile2,
+        createGeneration(
+            ref<LocalFSStore>(store),
+            profile2, store->printStorePath(storePath)));
+}
+
+void MixProfile::updateProfile(const Buildables & buildables)
+{
+    if (!profile) return;
+
+    std::optional<StorePath> result;
+
+    for (auto & buildable : buildables) {
+        for (auto & output : buildable.outputs) {
+            if (result)
+                throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
+            result = output.second.clone();
+        }
+    }
+
+    if (!result)
+        throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
+
+    updateProfile(*result);
+}
+
+MixDefaultProfile::MixDefaultProfile()
+{
+    profile = getDefaultProfile();
+}
+
+MixEnvironment::MixEnvironment() : ignoreEnvironment(false) {
+    mkFlag()
+        .longName("ignore-environment")
+        .shortName('i')
+        .description("clear the entire environment (except those specified with --keep)")
+        .set(&ignoreEnvironment, true);
+
+    mkFlag()
+        .longName("keep")
+        .shortName('k')
+        .description("keep specified environment variable")
+        .arity(1)
+        .labels({"name"})
+        .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
+
+    mkFlag()
+        .longName("unset")
+        .shortName('u')
+        .description("unset specified environment variable")
+        .arity(1)
+        .labels({"name"})
+        .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
+}
+
+void MixEnvironment::setEnviron() {
+    if (ignoreEnvironment) {
+        if (!unset.empty())
+            throw UsageError("--unset does not make sense with --ignore-environment");
+
+        for (const auto & var : keep) {
+            auto val = getenv(var.c_str());
+            if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val));
+        }
+
+        vectorEnv = stringsToCharPtrs(stringsEnv);
+        environ = vectorEnv.data();
+    } else {
+        if (!keep.empty())
+            throw UsageError("--keep does not make sense without --ignore-environment");
+
+        for (const auto & var : unset)
+            unsetenv(var.c_str());
+    }
+}
+
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a954a7d04fe..23f5c989864 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "installables.hh"
 #include "args.hh"
 #include "common-eval-args.hh"
 #include "path.hh"
@@ -22,34 +23,7 @@ private:
     std::shared_ptr<Store> _store;
 };
 
-struct Buildable
-{
-    std::optional<StorePath> drvPath;
-    std::map<std::string, StorePath> outputs;
-};
-
-typedef std::vector<Buildable> Buildables;
-
-struct Installable
-{
-    virtual ~Installable() { }
-
-    virtual std::string what() = 0;
-
-    virtual Buildables toBuildables()
-    {
-        throw Error("argument '%s' cannot be built", what());
-    }
-
-    Buildable toBuildable();
-
-    virtual std::pair<Value *, Pos> toValue(EvalState & state)
-    {
-        throw Error("argument '%s' cannot be evaluated", what());
-    }
-};
-
-struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs
+struct SourceExprCommand : virtual StoreCommand, MixEvalArgs
 {
     Path file;
 
@@ -184,4 +158,36 @@ std::set<StorePath> toDerivations(ref<Store> store,
    filename:lineno. */
 Strings editorFor(const Pos & pos);
 
+struct MixProfile : virtual StoreCommand
+{
+    std::optional<Path> profile;
+
+    MixProfile();
+
+    /* If 'profile' is set, make it point at 'storePath'. */
+    void updateProfile(const StorePath & storePath);
+
+    /* If 'profile' is set, make it point at the store path produced
+       by 'buildables'. */
+    void updateProfile(const Buildables & buildables);
+};
+
+struct MixDefaultProfile : MixProfile
+{
+    MixDefaultProfile();
+};
+
+struct MixEnvironment : virtual Args {
+
+    StringSet keep, unset;
+    Strings stringsEnv;
+    std::vector<char*> vectorEnv;
+    bool ignoreEnvironment;
+
+    MixEnvironment();
+
+    /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
+    void setEnviron();
+};
+
 }
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 013218cd924..f464d0aa1ed 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -109,6 +109,11 @@ struct InstallableStorePath : Installable
         bs.push_back(std::move(b));
         return bs;
     }
+
+    std::optional<StorePath> getStorePath() override
+    {
+        return storePath.clone();
+    }
 };
 
 struct InstallableValue : Installable
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
new file mode 100644
index 00000000000..503984220bc
--- /dev/null
+++ b/src/nix/installables.hh
@@ -0,0 +1,45 @@
+#pragma once
+
+#include "util.hh"
+#include "path.hh"
+#include "eval.hh"
+
+#include <optional>
+
+namespace nix {
+
+struct Buildable
+{
+    std::optional<StorePath> drvPath;
+    std::map<std::string, StorePath> outputs;
+};
+
+typedef std::vector<Buildable> Buildables;
+
+struct Installable
+{
+    virtual ~Installable() { }
+
+    virtual std::string what() = 0;
+
+    virtual Buildables toBuildables()
+    {
+        throw Error("argument '%s' cannot be built", what());
+    }
+
+    Buildable toBuildable();
+
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
+    {
+        throw Error("argument '%s' cannot be evaluated", what());
+    }
+
+    /* Return a value only if this installable is a store path or a
+       symlink to it. */
+    virtual std::optional<StorePath> getStorePath()
+    {
+        return {};
+    }
+};
+
+}
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f885c5e496b..8e30264c023 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -8,6 +8,7 @@
 #include "fs-accessor.hh"
 #include "progress-bar.hh"
 #include "affinity.hh"
+#include "eval.hh"
 
 #if __linux__
 #include <sys/mount.h>
@@ -19,11 +20,46 @@ using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
 
-struct CmdRun : InstallablesCommand
+struct RunCommon : virtual Command
+{
+    void runProgram(ref<Store> store,
+        const std::string & program,
+        const Strings & args)
+    {
+        stopProgressBar();
+
+        restoreSignals();
+
+        restoreAffinity();
+
+        /* If this is a diverted store (i.e. its "logical" location
+           (typically /nix/store) differs from its "physical" location
+           (e.g. /home/eelco/nix/store), then run the command in a
+           chroot. For non-root users, this requires running it in new
+           mount and user namespaces. Unfortunately,
+           unshare(CLONE_NEWUSER) doesn't work in a multithreaded
+           program (which "nix" is), so we exec() a single-threaded
+           helper program (chrootHelper() below) to do the work. */
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+
+        if (store2 && store->storeDir != store2->realStoreDir) {
+            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
+            for (auto & arg : args) helperArgs.push_back(arg);
+
+            execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
+
+            throw SysError("could not execute chroot helper");
+        }
+
+        execvp(program.c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("unable to execute '%s'", program);
+    }
+};
+
+struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
 {
     std::vector<std::string> command = { "bash" };
-    StringSet keep, unset;
-    bool ignoreEnvironment = false;
 
     CmdRun()
     {
@@ -37,28 +73,6 @@ struct CmdRun : InstallablesCommand
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
             });
-
-        mkFlag()
-            .longName("ignore-environment")
-            .shortName('i')
-            .description("clear the entire environment (except those specified with --keep)")
-            .set(&ignoreEnvironment, true);
-
-        mkFlag()
-            .longName("keep")
-            .shortName('k')
-            .description("keep specified environment variable")
-            .arity(1)
-            .labels({"name"})
-            .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
-
-        mkFlag()
-            .longName("unset")
-            .shortName('u')
-            .description("unset specified environment variable")
-            .arity(1)
-            .labels({"name"})
-            .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
     }
 
     std::string description() override
@@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand
 
         auto accessor = store->getFSAccessor();
 
-        if (ignoreEnvironment) {
-
-            if (!unset.empty())
-                throw UsageError("--unset does not make sense with --ignore-environment");
-
-            std::map<std::string, std::string> kept;
-            for (auto & var : keep) {
-                auto s = getenv(var.c_str());
-                if (s) kept[var] = s;
-            }
-
-            clearEnv();
-
-            for (auto & var : kept)
-                setenv(var.first.c_str(), var.second.c_str(), 1);
-
-        } else {
-
-            if (!keep.empty())
-                throw UsageError("--keep does not make sense without --ignore-environment");
-
-            for (auto & var : unset)
-                unsetenv(var.c_str());
-        }
 
         std::unordered_set<StorePath> done;
         std::queue<StorePath> todo;
         for (auto & path : outPaths) todo.push(path.clone());
 
+        setEnviron();
+
         auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
 
         while (!todo.empty()) {
@@ -142,38 +134,10 @@ struct CmdRun : InstallablesCommand
 
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
-        std::string cmd = *command.begin();
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        stopProgressBar();
-
-        restoreSignals();
-
-        restoreAffinity();
-
-        /* If this is a diverted store (i.e. its "logical" location
-           (typically /nix/store) differs from its "physical" location
-           (e.g. /home/eelco/nix/store), then run the command in a
-           chroot. For non-root users, this requires running it in new
-           mount and user namespaces. Unfortunately,
-           unshare(CLONE_NEWUSER) doesn't work in a multithreaded
-           program (which "nix" is), so we exec() a single-threaded
-           helper program (chrootHelper() below) to do the work. */
-        auto store2 = store.dynamic_pointer_cast<LocalStore>();
-
-        if (store2 && store->storeDir != store2->realStoreDir) {
-            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
-            for (auto & arg : args) helperArgs.push_back(arg);
-
-            execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
-
-            throw SysError("could not execute chroot helper");
-        }
-
-        execvp(cmd.c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("unable to exec '%s'", cmd);
+        runProgram(store, *command.begin(), args);
     }
 };
 
diff --git a/src/nix/shell.cc b/src/nix/shell.cc
new file mode 100644
index 00000000000..82b1da5520f
--- /dev/null
+++ b/src/nix/shell.cc
@@ -0,0 +1,319 @@
+#include "eval.hh"
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+#include "affinity.hh"
+#include "progress-bar.hh"
+
+#include <regex>
+
+using namespace nix;
+
+struct Var
+{
+    bool exported;
+    std::string value; // quoted string or array
+};
+
+struct BuildEnvironment
+{
+    std::map<std::string, Var> env;
+    std::string bashFunctions;
+};
+
+BuildEnvironment readEnvironment(const Path & path)
+{
+    BuildEnvironment res;
+
+    std::set<std::string> exported;
+
+    debug("reading environment file '%s'", path);
+
+    auto file = readFile(path);
+
+    auto pos = file.cbegin();
+
+    static std::string varNameRegex =
+        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+
+    static std::regex declareRegex(
+        "^declare -x (" + varNameRegex + ")" +
+        R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");
+
+    static std::string simpleStringRegex =
+        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+
+    static std::string quotedStringRegex =
+        R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
+
+    static std::string arrayRegex =
+        R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re";
+
+    static std::regex varRegex(
+        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n");
+
+    static std::regex functionRegex(
+        "^" + varNameRegex + " \\(\\) *\n");
+
+    while (pos != file.end()) {
+
+        std::smatch match;
+
+        if (std::regex_search(pos, file.cend(), match, declareRegex)) {
+            pos = match[0].second;
+            exported.insert(match[1]);
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, varRegex)) {
+            pos = match[0].second;
+            res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }});
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
+            res.bashFunctions = std::string(pos, file.cend());
+            break;
+        }
+
+        else throw Error("shell environment '%s' has unexpected line '%s'",
+            path, file.substr(pos - file.cbegin(), 60));
+    }
+
+    return res;
+}
+
+/* Given an existing derivation, return the shell environment as
+   initialised by stdenv's setup script. We do this by building a
+   modified derivation with the same dependencies and nearly the same
+   initial environment variables, that just writes the resulting
+   environment to a file and exits. */
+StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
+{
+    auto builder = baseNameOf(drv.builder);
+    if (builder != "bash")
+        throw Error("'nix shell' only works on derivations that use 'bash' as their builder");
+
+    drv.args = {
+        "-c",
+        "set -e; "
+        "export IN_NIX_SHELL=impure; "
+        "export dontAddDisableDepTrack=1; "
+        "if [[ -n $stdenv ]]; then "
+        "  source $stdenv/setup; "
+        "fi; "
+        "export > $out; "
+        "set >> $out "};
+
+    /* Remove derivation checks. */
+    drv.env.erase("allowedReferences");
+    drv.env.erase("allowedRequisites");
+    drv.env.erase("disallowedReferences");
+    drv.env.erase("disallowedRequisites");
+
+    // FIXME: handle structured attrs
+
+    /* Rehash and write the derivation. FIXME: would be nice to use
+       'buildDerivation', but that's privileged. */
+    auto drvName = drv.env["name"] + "-env";
+    for (auto & output : drv.outputs)
+        drv.env.erase(output.first);
+    drv.env["out"] = "";
+    drv.env["outputs"] = "out";
+    Hash h = hashDerivationModulo(*store, drv, true);
+    auto shellOutPath = store->makeOutputPath("out", h, drvName);
+    drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", ""));
+    drv.env["out"] = store->printStorePath(shellOutPath);
+    auto shellDrvPath2 = writeDerivation(store, drv, drvName);
+
+    /* Build the derivation. */
+    store->buildPaths({shellDrvPath2});
+
+    assert(store->isValidPath(shellOutPath));
+
+    return shellOutPath;
+}
+
+struct Common : InstallableCommand, MixProfile
+{
+    std::set<string> ignoreVars{
+        "BASHOPTS",
+        "EUID",
+        "HOME", // FIXME: don't ignore in pure mode?
+        "NIX_BUILD_TOP",
+        "NIX_ENFORCE_PURITY",
+        "NIX_LOG_FD",
+        "PPID",
+        "PWD",
+        "SHELLOPTS",
+        "SHLVL",
+        "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
+        "TEMP",
+        "TEMPDIR",
+        "TERM",
+        "TMP",
+        "TMPDIR",
+        "TZ",
+        "UID",
+    };
+
+    void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
+    {
+        out << "nix_saved_PATH=\"$PATH\"\n";
+
+        for (auto & i : buildEnvironment.env) {
+            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
+                out << fmt("%s=%s\n", i.first, i.second.value);
+                if (i.second.exported)
+                    out << fmt("export %s\n", i.first);
+            }
+        }
+
+        out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
+
+        out << buildEnvironment.bashFunctions << "\n";
+
+        // FIXME: set outputs
+
+        out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
+        for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
+            out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
+
+        out << "eval \"$shellHook\"\n";
+    }
+
+    StorePath getShellOutPath(ref<Store> store)
+    {
+        auto path = installable->getStorePath();
+        if (path && hasSuffix(path->to_string(), "-env"))
+            return path->clone();
+        else {
+            auto drvs = toDerivations(store, {installable});
+
+            if (drvs.size() != 1)
+                throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations",
+                    installable->what(), drvs.size());
+
+            auto & drvPath = *drvs.begin();
+
+            return getDerivationEnvironment(store, store->derivationFromPath(drvPath));
+        }
+    }
+
+    BuildEnvironment getBuildEnvironment(ref<Store> store)
+    {
+        auto shellOutPath = getShellOutPath(store);
+
+        updateProfile(shellOutPath);
+
+        return readEnvironment(store->printStorePath(shellOutPath));
+    }
+};
+
+struct CmdDevShell : Common, MixEnvironment
+{
+    std::vector<std::string> command;
+
+    CmdDevShell()
+    {
+        mkFlag()
+            .longName("command")
+            .shortName('c')
+            .description("command and arguments to be executed insted of an interactive shell")
+            .labels({"command", "args"})
+            .arity(ArityAny)
+            .handler([&](std::vector<std::string> ss) {
+                if (ss.empty()) throw UsageError("--command requires at least one argument");
+                command = ss;
+            });
+    }
+
+    std::string description() override
+    {
+        return "run a bash shell that provides the build environment of a derivation";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To get the build environment of GNU hello:",
+                "nix dev-shell nixpkgs.hello"
+            },
+            Example{
+                "To store the build environment in a profile:",
+                "nix dev-shell --profile /tmp/my-shell nixpkgs.hello"
+            },
+            Example{
+                "To use a build environment previously recorded in a profile:",
+                "nix dev-shell /tmp/my-shell"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto buildEnvironment = getBuildEnvironment(store);
+
+        auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
+
+        std::ostringstream ss;
+        makeRcScript(buildEnvironment, ss);
+
+        ss << fmt("rm -f '%s'\n", rcFilePath);
+
+        if (!command.empty()) {
+            std::vector<std::string> args;
+            for (auto s : command)
+                args.push_back(shellEscape(s));
+            ss << fmt("exec %s\n", concatStringsSep(" ", args));
+        }
+
+        writeFull(rcFileFd.get(), ss.str());
+
+        stopProgressBar();
+
+        auto shell = getEnv("SHELL").value_or("bash");
+
+        setEnviron();
+
+        auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
+
+        restoreAffinity();
+        restoreSignals();
+
+        execvp(shell.c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("executing shell '%s'", shell);
+    }
+};
+
+struct CmdPrintDevEnv : Common
+{
+    std::string description() override
+    {
+        return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To apply the build environment of GNU hello to the current shell:",
+                ". <(nix print-dev-env nixpkgs.hello)"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto buildEnvironment = getBuildEnvironment(store);
+
+        stopProgressBar();
+
+        makeRcScript(buildEnvironment, std::cout);
+    }
+};
+
+static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
+static auto r2 = registerCommand<CmdDevShell>("dev-shell");

From bbbb7c1bc7a66e1169931cda8bd863b68236726a Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Mon, 30 Mar 2020 18:15:55 -0400
Subject: [PATCH 1909/2196] Use `auto` with some `FileIngestionMethod` local
 variables

---
 src/libexpr/primops.cc     | 2 +-
 src/nix-store/nix-store.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7d45733f444..71a511c41af 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    FileIngestionMethod outputHashRecursive = FileIngestionMethod::Flat;
+    auto outputHashRecursive = FileIngestionMethod::Flat;
 
     StringSet outputs;
     outputs.insert("out");
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 9a26eb57f1d..cde7d561bc1 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -174,7 +174,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    FileIngestionMethod recursive = FileIngestionMethod::Flat;
+    auto recursive = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
         if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
@@ -194,7 +194,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
 /* Hack to support caching in `nix-prefetch-url'. */
 static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 {
-    FileIngestionMethod recursive = FileIngestionMethod::Flat;
+    auto recursive = FileIngestionMethod::Flat;
 
     for (auto i : opFlags)
         if (i == "--recursive") recursive = FileIngestionMethod::Recursive;

From 51afea3af23640a3fb2cfe8cfae5cf3cc27eb3a1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Mar 2020 22:31:51 +0000
Subject: [PATCH 1910/2196] Never cast `FileIngestionMethod` to or from boolean

---
 src/libexpr/primops.cc    |  9 ++++++---
 src/libstore/build.cc     | 10 +++++++---
 src/libstore/path.hh      |  2 +-
 src/libstore/store-api.cc | 21 ++++++++++++++-------
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 71a511c41af..4d5aaf86a7b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -724,9 +724,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
-        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
-                (static_cast<bool>(outputHashRecursive) ? "r:" : "") + printHashType(h.type),
-                h.to_string(Base16, false)));
+        drv.outputs.insert_or_assign("out", DerivationOutput {
+            std::move(outPath),
+            (outputHashRecursive == FileIngestionMethod::Recursive ? "r:" : "")
+                + printHashType(h.type),
+            h.to_string(Base16, false),
+        });
     }
 
     else {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 224633106ce..be52e637e79 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3650,7 +3650,7 @@ void DerivationGoal::registerOutputs()
             FileIngestionMethod recursive; Hash h;
             i.second.parseHashInfo(recursive, h);
 
-            if (!static_cast<bool>(recursive)) {
+            if (recursive == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
@@ -3659,7 +3659,9 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
-            Hash h2 = static_cast<bool>(recursive) ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
+            Hash h2 = recursive == FileIngestionMethod::Recursive
+                ? hashPath(h.type, actualPath).first
+                : hashFile(h.type, actualPath);
 
             auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
 
@@ -3912,7 +3914,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
 
                 auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
 
-                auto used = static_cast<bool>(recursive) ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
+                auto used = recursive
+                    ? cloneStorePathSet(getClosure(info.path).first)
+                    : cloneStorePathSet(info.references);
 
                 if (recursive && checks.ignoreSelfRefs)
                     used.erase(info.path);
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 73c4b8e2922..5122e742290 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -73,7 +73,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
-enum struct FileIngestionMethod : bool {
+enum struct FileIngestionMethod : uint8_t {
     Flat = false,
     Recursive = true
 };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 06aa0883c10..ca14095885e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -177,13 +177,16 @@ StorePath Store::makeFixedOutputPath(
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256  && recursive == FileIngestionMethod::Recursive) {
+    if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
-        return makeStorePath("output:out", hashString(htSHA256,
-                "fixed:out:" + (static_cast<bool>(recursive) ? (string) "r:" : "") +
-                hash.to_string(Base16) + ":"), name);
+        return makeStorePath("output:out",
+            hashString(htSHA256,
+                "fixed:out:"
+                + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+                + hash.to_string(Base16) + ":"),
+            name);
     }
 }
 
@@ -202,7 +205,9 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
 std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
     const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const
 {
-    Hash h = static_cast<bool>(recursive) ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
+    Hash h = recursive == FileIngestionMethod::Recursive
+        ? hashPath(hashAlgo, srcPath, filter).first
+        : hashFile(hashAlgo, srcPath);
     return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
 }
 
@@ -782,7 +787,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
 
     else if (hasPrefix(ca, "fixed:")) {
         FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
-        Hash hash(std::string(ca, static_cast<bool>(recursive) ? 8 : 6));
+        Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6));
         auto refs = cloneStorePathSet(references);
         bool hasSelfReference = false;
         if (refs.count(path)) {
@@ -828,7 +833,9 @@ Strings ValidPathInfo::shortRefs() const
 
 std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 {
-    return "fixed:" + (static_cast<bool>(recursive) ? (std::string) "r:" : "") + hash.to_string();
+    return "fixed:"
+        + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+        + hash.to_string();
 }
 
 

From 7e9a2718f0c39fde53a3447c3b7f3617a4a9b958 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Mar 2020 22:36:15 +0000
Subject: [PATCH 1911/2196] s/outputHashRecursive/ingestionMethod/c

---
 src/libexpr/primops.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4d5aaf86a7b..415e77fe7b5 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
-    auto outputHashRecursive = FileIngestionMethod::Flat;
+    auto ingestionMethod = FileIngestionMethod::Flat;
 
     StringSet outputs;
     outputs.insert("out");
@@ -574,8 +574,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
-            if (s == "recursive") outputHashRecursive = FileIngestionMethod::Recursive;
-            else if (s == "flat") outputHashRecursive = FileIngestionMethod::Flat;
+            if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
+            else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
             else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
         };
 
@@ -722,11 +722,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
             std::move(outPath),
-            (outputHashRecursive == FileIngestionMethod::Recursive ? "r:" : "")
+            (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
                 + printHashType(h.type),
             h.to_string(Base16, false),
         });

From 8aa46cd340c1294c3d06cd52f85c906bdf749070 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 30 Mar 2020 22:40:41 +0000
Subject: [PATCH 1912/2196] Get rid of FileIngestionMethod casts in perl
 bindings, too

---
 perl/lib/Nix/Store.xs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index eab8ccacb5c..890310b3e80 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, (FileIngestionMethod) recursive, parseHashType(algo));
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            auto path = store()->makeFixedOutputPath((FileIngestionMethod) recursive, h, name);
+            auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            auto path = store()->makeFixedOutputPath(method, h, name);
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

From 12556e570924315eb25ad6057b6c2c5162955e4f Mon Sep 17 00:00:00 2001
From: mlatus <wqseleven@gmail.com>
Date: Tue, 31 Mar 2020 19:40:16 +0800
Subject: [PATCH 1913/2196] fix placeholder not substituted in passAsFile

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0e3a23a4dc8..527d7ac42fb 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2465,7 +2465,7 @@ void DerivationGoal::initTmpDir() {
                 auto hash = hashString(htSHA256, i.first);
                 string fn = ".attr-" + hash.to_string(Base32, false);
                 Path p = tmpDir + "/" + fn;
-                writeFile(p, i.second);
+                writeFile(p, rewriteStrings(i.second, inputRewrites));
                 chownToBuilder(p);
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }

From 3166b97174ab879dac3b19eb3f257e73a943fef0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Mar 2020 13:45:28 +0200
Subject: [PATCH 1914/2196] nix shell -> nix dev-shell

---
 src/nix/shell.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index 82b1da5520f..71e640667fc 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -92,7 +92,7 @@ StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
 {
     auto builder = baseNameOf(drv.builder);
     if (builder != "bash")
-        throw Error("'nix shell' only works on derivations that use 'bash' as their builder");
+        throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");
 
     drv.args = {
         "-c",

From 09652f597cb44fbb452353f9eeaba2d9bc70a7f3 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 31 Mar 2020 09:36:20 -0600
Subject: [PATCH 1915/2196] enum style

---
 src/libutil/error.hh | 4 ++--
 tests/errors/main.cc | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 88980afb778..1acc70d9d42 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -16,10 +16,10 @@ using std::endl;
 
 namespace nix {
 
-enum ErrLevel 
+typedef enum  
   { elWarning
   , elError
-  };
+  } ErrLevel;
 
 class ColumnRange { 
   public:
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 39adc2ece9a..ffce334b738 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -40,7 +40,7 @@ int main()
 
   printErrorInfo(StandardError()
                 .name("name")
-                .description("description")
+                .description("error description")
                 .nohint()
                 );
 

From 9e7b89bf100500cf252f785e2dd77f83ee5cf7a4 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 31 Mar 2020 11:56:37 -0600
Subject: [PATCH 1916/2196] rename errors/warnings

---
 src/libutil/error.hh |  8 ++++----
 tests/errors/main.cc | 32 ++++----------------------------
 2 files changed, 8 insertions(+), 32 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 1acc70d9d42..cd2b2832dcd 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -256,11 +256,11 @@ class AddHint : private T
 typedef AddName<
         AddDescription<
         AddHint<
-        EIError>>> StandardError;
+        EIError>>> ProgramError;
 typedef AddName<
         AddDescription<
         AddHint<
-        EIWarning>>> StandardWarning;
+        EIWarning>>> ProgramWarning;
 
 typedef AddName<
         AddDescription<
@@ -269,7 +269,7 @@ typedef AddName<
         AddColumnRange<
         AddLOC<
         AddHint<
-        EIError>>>>>>> MkNixError;
+        EIError>>>>>>> NixLangError;
 typedef AddName<
         AddDescription<
         AddNixFile<
@@ -277,7 +277,7 @@ typedef AddName<
         AddColumnRange<
         AddLOC<
         AddHint<
-        EIWarning>>>>>>> MkNixWarning;
+        EIWarning>>>>>>> NixLangWarning;
 
 
 // --------------------------------------------------------
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index ffce334b738..101d44a7eae 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -14,44 +14,20 @@ int main()
 
   ErrorInfo::programName = optional("error-test");
 
-  /*
-  ColumnRange columnRange;
-  columnRange.start = 24;
-  columnRange.len = 14;
-
-  ErrLine errline;
-  errline.lineNumber = 7;
-  errline.columnRange = optional(columnRange);
-  errline.errLineOfCode = "line of code where the error occurred";
-
-  NixCode nixcode;
-  nixcode.nixFile = optional("myfile.nix");
-  nixcode.errLine = errline; 
-  
-  ErrorInfo generic;
-  generic.level = elError;
-  generic.name = "error name";
-  generic.description = "general error description";
-  generic.program = "nixtool.exe";
-  generic.nixCode = nixcode;
-
-  printErrorInfo(generic);
-  */
-
-  printErrorInfo(StandardError()
+  printErrorInfo(ProgramError()
                 .name("name")
                 .description("error description")
                 .nohint()
                 );
 
-  printErrorInfo(StandardWarning()
+  printErrorInfo(ProgramWarning()
                 .name("warning name")
                 .description("warning description")
                 .nohint()
                 );
 
 
-  printErrorInfo(MkNixWarning()
+  printErrorInfo(NixLangWarning()
                 .name("warning name")
                 .description("warning description")
                 .nixFile("myfile.nix")
@@ -63,7 +39,7 @@ int main()
                 .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
                 );
 
-  printErrorInfo(MkNixError()
+  printErrorInfo(NixLangError()
                 .name("error name")
                 .description("error description")
                 .nixFile("myfile.nix")

From 5b3aefff857d038d49ad9a5d92371b4f6b19e8b9 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 31 Mar 2020 12:42:41 -0600
Subject: [PATCH 1917/2196] add some explanatory comments

---
 src/libutil/error.cc |  4 +-
 src/libutil/error.hh |  2 +
 tests/errors/main.cc | 91 ++++++++++++++++++++++++++------------------
 3 files changed, 60 insertions(+), 37 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index c986affccba..5384248f841 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -152,12 +152,14 @@ void printErrorInfo(ErrorInfo &einfo)
 
   // lines of code.
   if (einfo.nixCode.has_value())
+  {
     printCodeLines(prefix, *einfo.nixCode);
+    cout << prefix << endl;
+  }
 
   // hint
   if (einfo.hint.has_value()) 
   {
-    cout << prefix << endl;
     cout << prefix << *einfo.hint << endl;
     cout << prefix << endl;
   }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index cd2b2832dcd..00539f06a12 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -257,6 +257,7 @@ typedef AddName<
         AddDescription<
         AddHint<
         EIError>>> ProgramError;
+
 typedef AddName<
         AddDescription<
         AddHint<
@@ -270,6 +271,7 @@ typedef AddName<
         AddLOC<
         AddHint<
         EIError>>>>>>> NixLangError;
+
 typedef AddName<
         AddDescription<
         AddNixFile<
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 101d44a7eae..1aff5a01613 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -8,48 +8,67 @@ using std::nullopt;
 using std::cout;
 using std::endl;
 
+
 int main() 
 {
   using namespace nix; 
 
+  // In each program where errors occur, this has to be set.
   ErrorInfo::programName = optional("error-test");
 
-  printErrorInfo(ProgramError()
-                .name("name")
-                .description("error description")
-                .nohint()
-                );
-
-  printErrorInfo(ProgramWarning()
-                .name("warning name")
-                .description("warning description")
-                .nohint()
-                );
-
-
-  printErrorInfo(NixLangWarning()
-                .name("warning name")
-                .description("warning description")
-                .nixFile("myfile.nix")
-                .lineNumber(40)
-                .columnRange(13,7)
-                .linesOfCode(nullopt
-                            ,"this is the problem line of code"
-                            ,nullopt)
-                .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-                );
-
-  printErrorInfo(NixLangError()
-                .name("error name")
-                .description("error description")
-                .nixFile("myfile.nix")
-                .lineNumber(40)
-                .columnRange(13,7)
-                .linesOfCode(optional("previous line of code")
-                            ,"this is the problem line of code"
-                            ,optional("next line of code"))
-                .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-                );
+  // There are currently four error types - 
+  // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
+  // Each error type is created with a specific sequence of builder functions.
+  // Unlike with a constructor, each parameter is clearly named.
+  // If the sequence of function calls isn't followed, then there's a type error.
+  // This should make for a consistent look in the code when errors are created.
+
+  // ProgramError takes name, description, and an optional hint.
+  printErrorInfo(
+    ProgramError()
+      .name("name")
+      .description("error description")
+      .nohint()
+      );
+
+  // ProgramWarning takes name, description, and an optional hint.
+  // The hint is in the form of a hintfmt class, which wraps boost::format(), and 
+  // makes all the substituted text yellow.
+  printErrorInfo(
+    ProgramWarning()
+      .name("warning name")
+      .description("warning description")
+      .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow.
+      );
+
+  // NixLangWarning adds nix file, line number, column range, and the lines of code 
+  // where a warning occurred.
+  printErrorInfo(
+    NixLangWarning()
+      .name("warning name")
+      .description("warning description")
+      .nixFile("myfile.nix")
+      .lineNumber(40)
+      .columnRange(13,7)
+      .linesOfCode(nullopt
+                  ,"this is the problem line of code"
+                  ,nullopt)
+      .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+      );
+
+  // NixLangError is just the same as NixLangWarning, except for the Error flag.
+  printErrorInfo(
+    NixLangError()
+      .name("error name")
+      .description("error description")
+      .nixFile("myfile.nix")
+      .lineNumber(40)
+      .columnRange(13,7)
+      .linesOfCode(optional("previous line of code")
+                  ,"this is the problem line of code"
+                  ,optional("next line of code"))
+      .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+      );
 
   return 0;
 }

From a72b6b2ec80a8500015315acd36991aad9817907 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 31 Mar 2020 18:29:41 -0600
Subject: [PATCH 1918/2196] examples of invalid errors

---
 tests/errors/main.cc | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 1aff5a01613..9d08e82eaad 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -16,8 +16,10 @@ int main()
   // In each program where errors occur, this has to be set.
   ErrorInfo::programName = optional("error-test");
 
-  // There are currently four error types - 
+  // There are currently four error types:
+  //  
   // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
+  // 			
   // Each error type is created with a specific sequence of builder functions.
   // Unlike with a constructor, each parameter is clearly named.
   // If the sequence of function calls isn't followed, then there's a type error.
@@ -41,6 +43,27 @@ int main()
       .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow.
       );
 
+  /*
+    // some invalid errors:
+    
+    // type error: no hint function.
+    ProgramError()
+      .name("name")
+      .description("error description");
+
+    // type error:  description before name.
+    ProgramError()
+      .description("error description")
+      .name("name")
+      .nohint();
+
+    // type error: hint function with regular boost format, not special hintfmt.
+    ProgramError()
+      .description("error description")
+      .name("name")
+      .hint(format("there was a %1%") % "warning");
+  */
+
   // NixLangWarning adds nix file, line number, column range, and the lines of code 
   // where a warning occurred.
   printErrorInfo(

From 8713aeac5e29f019286df6dc6c52a2da956972e7 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 1 Apr 2020 15:51:14 -0600
Subject: [PATCH 1919/2196] remove using std::*, switch to include guard

---
 src/libutil/error.cc |   6 +-
 src/libutil/error.hh |  14 ++--
 tests/errors/main.cc | 150 +++++++++++++++++++++----------------------
 3 files changed, 78 insertions(+), 92 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 5384248f841..2b7a079dc45 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -5,11 +5,7 @@
 
 namespace nix {
 
-using std::cout;
-using std::endl;
-using std::nullopt;
-
-optional<string> ErrorInfo::programName = nullopt;
+optional<string> ErrorInfo::programName = std::nullopt;
 
 // return basic_format?
 string showErrLine(ErrLine &errLine)
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 00539f06a12..614f08a35be 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -1,4 +1,5 @@
-#pragma once
+#ifndef error_hh
+#define error_hh
 
 #include "ansicolor.hh"
 #include <string>
@@ -8,12 +9,6 @@
 
 #include <boost/format.hpp>
 
-using std::string;
-using std::optional;
-using boost::format;
-using std::cout;
-using std::endl;
-
 namespace nix {
 
 typedef enum  
@@ -36,7 +31,6 @@ class ErrLine {
     optional<string> prevLineOfCode;
     string errLineOfCode;
     optional<string> nextLineOfCode;
-
 };
 
 class NixCode { 
@@ -112,8 +106,6 @@ class ErrorInfo {
   protected:
     // constructor is protected, so only the builder classes can create an ErrorInfo.
     ErrorInfo(ErrLevel level) { this->level = level; }
-
-
 };
 
 // Init as error
@@ -285,7 +277,9 @@ typedef AddName<
 // --------------------------------------------------------
 // error printing
 
+// just to cout for now.
 void printErrorInfo(ErrorInfo &einfo);
 
 }
 
+#endif
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 9d08e82eaad..f11390c7329 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -3,95 +3,91 @@
 #include <optional>
 #include <iostream>
 
-using std::optional;
-using std::nullopt;
-using std::cout;
-using std::endl;
 
 
 int main() 
 {
-  using namespace nix; 
+    using namespace nix; 
 
-  // In each program where errors occur, this has to be set.
-  ErrorInfo::programName = optional("error-test");
+    // In each program where errors occur, this has to be set.
+    ErrorInfo::programName = optional("error-test");
 
-  // There are currently four error types:
-  //  
-  // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
-  // 			
-  // Each error type is created with a specific sequence of builder functions.
-  // Unlike with a constructor, each parameter is clearly named.
-  // If the sequence of function calls isn't followed, then there's a type error.
-  // This should make for a consistent look in the code when errors are created.
+    // There are currently four error types:
+    //  
+    // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
+    // 			
+    // Each error type is created with a specific sequence of builder functions.
+    // Unlike with a constructor, each parameter is clearly named.
+    // If the sequence of function calls isn't followed, then there's a type error.
+    // This should make for a consistent look in the code when errors are created.
 
-  // ProgramError takes name, description, and an optional hint.
-  printErrorInfo(
-    ProgramError()
-      .name("name")
-      .description("error description")
-      .nohint()
-      );
+    // ProgramError takes name, description, and an optional hint.
+    printErrorInfo(
+        ProgramError()
+            .name("name")
+            .description("error description")
+            .nohint()
+            );
 
-  // ProgramWarning takes name, description, and an optional hint.
-  // The hint is in the form of a hintfmt class, which wraps boost::format(), and 
-  // makes all the substituted text yellow.
-  printErrorInfo(
-    ProgramWarning()
-      .name("warning name")
-      .description("warning description")
-      .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow.
-      );
+    // ProgramWarning takes name, description, and an optional hint.
+    // The hint is in the form of a hintfmt class, which wraps boost::format(), and 
+    // makes all the substituted text yellow.
+    printErrorInfo(
+        ProgramWarning()
+            .name("warning name")
+            .description("warning description")
+            .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow.
+            );
 
-  /*
-    // some invalid errors:
-    
-    // type error: no hint function.
-    ProgramError()
-      .name("name")
-      .description("error description");
+    /*
+      // some invalid errors:
+      
+      // type error: no hint function.
+      ProgramError()
+        .name("name")
+        .description("error description");
 
-    // type error:  description before name.
-    ProgramError()
-      .description("error description")
-      .name("name")
-      .nohint();
+      // type error:  description before name.
+      ProgramError()
+        .description("error description")
+        .name("name")
+        .nohint();
 
-    // type error: hint function with regular boost format, not special hintfmt.
-    ProgramError()
-      .description("error description")
-      .name("name")
-      .hint(format("there was a %1%") % "warning");
-  */
+      // type error: hint function with regular boost format, not special hintfmt.
+      ProgramError()
+        .description("error description")
+        .name("name")
+        .hint(format("there was a %1%") % "warning");
+    */
 
-  // NixLangWarning adds nix file, line number, column range, and the lines of code 
-  // where a warning occurred.
-  printErrorInfo(
-    NixLangWarning()
-      .name("warning name")
-      .description("warning description")
-      .nixFile("myfile.nix")
-      .lineNumber(40)
-      .columnRange(13,7)
-      .linesOfCode(nullopt
-                  ,"this is the problem line of code"
-                  ,nullopt)
-      .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-      );
+    // NixLangWarning adds nix file, line number, column range, and the lines of code 
+    // where a warning occurred.
+    printErrorInfo(
+        NixLangWarning()
+            .name("warning name")
+            .description("warning description")
+            .nixFile("myfile.nix")
+            .lineNumber(40)
+            .columnRange(13,7)
+            .linesOfCode(std::nullopt
+                        ,"this is the problem line of code"
+                        ,std::nullopt)
+            .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+            );
 
-  // NixLangError is just the same as NixLangWarning, except for the Error flag.
-  printErrorInfo(
-    NixLangError()
-      .name("error name")
-      .description("error description")
-      .nixFile("myfile.nix")
-      .lineNumber(40)
-      .columnRange(13,7)
-      .linesOfCode(optional("previous line of code")
-                  ,"this is the problem line of code"
-                  ,optional("next line of code"))
-      .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-      );
+    // NixLangError is just the same as NixLangWarning, except for the Error flag.
+    printErrorInfo(
+        NixLangError()
+            .name("error name")
+            .description("error description")
+            .nixFile("myfile.nix")
+            .lineNumber(40)
+            .columnRange(13,7)
+            .linesOfCode(optional("previous line of code")
+                        ,"this is the problem line of code"
+                        ,optional("next line of code"))
+            .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
+            );
 
-  return 0;
+    return 0;
 }

From dd7b8183a5065bc0279d9bd87c27dd0ac91ccd8a Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 1 Apr 2020 16:20:20 -0600
Subject: [PATCH 1920/2196] indenting

---
 src/libutil/error.cc | 258 +++++++++++++++++++++----------------------
 src/libutil/error.hh | 252 +++++++++++++++++++++---------------------
 2 files changed, 255 insertions(+), 255 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 2b7a079dc45..4480e80bf1e 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -10,155 +10,155 @@ optional<string> ErrorInfo::programName = std::nullopt;
 // return basic_format?
 string showErrLine(ErrLine &errLine)
 {
-  if (errLine.columnRange.has_value()) 
-  {
-    return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
-  }
-  else
-  {
-    return (format("(%1%)") % errLine.lineNumber).str();
-  };
+    if (errLine.columnRange.has_value()) 
+    {
+        return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
+    }
+    else
+    {
+        return (format("(%1%)") % errLine.lineNumber).str();
+    };
 }
 
 void printCodeLines(string &prefix, NixCode &nixCode) 
 {
-  
-  if (nixCode.errLine.has_value()) 
-  {
-    // previous line of code.
-    if (nixCode.errLine->prevLineOfCode.has_value()) { 
-      cout << format("%1% %|2$5d|| %3%")
-        %  prefix
-        % (nixCode.errLine->lineNumber - 1)
-        % *nixCode.errLine->prevLineOfCode
-        << endl;
-    }
 
-    // line of code containing the error.%2$+5d%
-    cout << format("%1% %|2$5d|| %3%")
-      %  prefix
-      % (nixCode.errLine->lineNumber)
-      % nixCode.errLine->errLineOfCode
-      << endl;
-    
-    // error arrows for the column range.
-    if (nixCode.errLine->columnRange.has_value()) 
+  if (nixCode.errLine.has_value()) 
     {
-      int start = nixCode.errLine->columnRange->start;
-      std::string spaces;
-      for (int i = 0; i < start; ++i)
-      {
-        spaces.append(" ");
-      }
-
-      int len = nixCode.errLine->columnRange->len;
-      std::string arrows;
-      for (int i = 0; i < len; ++i)
-      {
-        arrows.append("^");
-      }
-    
-      cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl;
-    }
+        // previous line of code.
+        if (nixCode.errLine->prevLineOfCode.has_value()) { 
+            cout << format("%1% %|2$5d|| %3%")
+                %    prefix
+                % (nixCode.errLine->lineNumber - 1)
+                % *nixCode.errLine->prevLineOfCode
+                << endl;
+        }
+
+        // line of code containing the error.%2$+5d%
+        cout << format("%1% %|2$5d|| %3%")
+            %    prefix
+            % (nixCode.errLine->lineNumber)
+            % nixCode.errLine->errLineOfCode
+            << endl;
+
+      // error arrows for the column range.
+        if (nixCode.errLine->columnRange.has_value()) 
+        {
+            int start = nixCode.errLine->columnRange->start;
+            std::string spaces;
+            for (int i = 0; i < start; ++i)
+            {
+                spaces.append(" ");
+            }
+
+            int len = nixCode.errLine->columnRange->len;
+            std::string arrows;
+            for (int i = 0; i < len; ++i)
+            {
+                arrows.append("^");
+            }
+
+          cout << format("%1%            |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl;
+        }
+
+
+
+        // next line of code.
+        if (nixCode.errLine->nextLineOfCode.has_value()) { 
+            cout << format("%1% %|2$5d|| %3%")
+            %    prefix
+            % (nixCode.errLine->lineNumber + 1)
+            % *nixCode.errLine->nextLineOfCode
+            << endl;
+        }
 
-
-
-    // next line of code.
-    if (nixCode.errLine->nextLineOfCode.has_value()) { 
-      cout << format("%1% %|2$5d|| %3%")
-      %  prefix
-      % (nixCode.errLine->lineNumber + 1)
-      % *nixCode.errLine->nextLineOfCode
-      << endl;
     }
 
-  }
-    
 }
 
 void printErrorInfo(ErrorInfo &einfo) 
 {
-  int errwidth = 80;
-  string prefix = "  ";
-
-  string levelString;
-  switch (einfo.level) 
-  {
-    case ErrLevel::elError: 
-      {
-        levelString = ANSI_RED;
-        levelString += "error:";
-        levelString += ANSI_NORMAL;
-        break;
-      }
-    case ErrLevel::elWarning: 
-      {
-        levelString = ANSI_YELLOW;
-        levelString += "warning:";  
-        levelString += ANSI_NORMAL;
-        break;
-      }
-    default: 
-      {
-        levelString = (format("invalid error level: %1%") % einfo.level).str();  
-        break;
-      }
-  }
-
-  int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
-  int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; 
-
-  string dashes;
-  for (int i = 0; i < dashwidth; ++i)
-    dashes.append("-");
-
-  // divider.
-  cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
-    % prefix
-    % levelString
-    % "---"
-    % einfo.name
-    % dashes
-    % einfo.programName.value_or("")
-    << endl;
-
-  // filename.
-  if (einfo.nixCode.has_value()) 
-  {
-    if (einfo.nixCode->nixFile.has_value()) 
-    {
-      string eline = einfo.nixCode->errLine.has_value()  
-        ? string(" ") + showErrLine(*einfo.nixCode->errLine)
-        : "";
+    int errwidth = 80;
+    string prefix = "    ";
 
-      cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
-        % prefix % *einfo.nixCode->nixFile % eline << endl;
-      cout << prefix << endl;
-    }
-    else
+    string levelString;
+    switch (einfo.level) 
     {
-      cout << format("%1%from command line argument") % prefix << endl;
-      cout << prefix << endl;
+        case ErrLevel::elError: 
+            {
+                levelString = ANSI_RED;
+                levelString += "error:";
+                levelString += ANSI_NORMAL;
+                break;
+            }
+        case ErrLevel::elWarning: 
+            {
+                levelString = ANSI_YELLOW;
+                levelString += "warning:";    
+                levelString += ANSI_NORMAL;
+                break;
+            }
+        default: 
+            {
+                levelString = (format("invalid error level: %1%") % einfo.level).str();    
+                break;
+            }
     }
-  }
 
-  // description
-  cout << prefix << einfo.description << endl;
-  cout << prefix << endl;
+    int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; 
+
+    string dashes;
+    for (int i = 0; i < dashwidth; ++i)
+        dashes.append("-");
+
+    // divider.
+    cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
+        % prefix
+        % levelString
+        % "---"
+        % einfo.name
+        % dashes
+        % einfo.programName.value_or("")
+        << endl;
 
-  // lines of code.
-  if (einfo.nixCode.has_value())
-  {
-    printCodeLines(prefix, *einfo.nixCode);
-    cout << prefix << endl;
-  }
+    // filename.
+    if (einfo.nixCode.has_value()) 
+    {
+        if (einfo.nixCode->nixFile.has_value()) 
+        {
+            string eline = einfo.nixCode->errLine.has_value()
+                ? string(" ") + showErrLine(*einfo.nixCode->errLine)
+                : "";
+
+            cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
+                % prefix % *einfo.nixCode->nixFile % eline << endl;
+            cout << prefix << endl;
+        }
+        else
+        {
+            cout << format("%1%from command line argument") % prefix << endl;
+            cout << prefix << endl;
+        }
+    }
 
-  // hint
-  if (einfo.hint.has_value()) 
-  {
-    cout << prefix << *einfo.hint << endl;
+    // description
+    cout << prefix << einfo.description << endl;
     cout << prefix << endl;
-  }
+
+    // lines of code.
+    if (einfo.nixCode.has_value())
+    {
+        printCodeLines(prefix, *einfo.nixCode);
+        cout << prefix << endl;
+    }
+
+    // hint
+    if (einfo.hint.has_value()) 
+    {
+        cout << prefix << *einfo.hint << endl;
+        cout << prefix << endl;
+    }
 }
 
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 614f08a35be..00c32ff21da 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -11,45 +11,45 @@
 
 namespace nix {
 
-typedef enum  
-  { elWarning
-  , elError
-  } ErrLevel;
+typedef enum { 
+    elWarning
+    , elError
+    } ErrLevel;
 
 class ColumnRange { 
-  public:
-    unsigned int start;
-    unsigned int len;
+    public:
+        unsigned int start;
+        unsigned int len;
 };
 
 class ErrorInfo;
 
 class ErrLine { 
-  public:
-    int lineNumber;
-    optional<ColumnRange> columnRange;
-    optional<string> prevLineOfCode;
-    string errLineOfCode;
-    optional<string> nextLineOfCode;
+    public:
+        int lineNumber;
+        optional<ColumnRange> columnRange;
+        optional<string> prevLineOfCode;
+        string errLineOfCode;
+        optional<string> nextLineOfCode;
 };
 
 class NixCode { 
-  public:
-    optional<string> nixFile;
-    optional<ErrLine> errLine;
-
-    ErrLine& ensureErrLine() 
-    { 
-      if (!this->errLine.has_value())
-         this->errLine = optional(ErrLine());
-      return *this->errLine; 
-    }
+    public:
+        optional<string> nixFile;
+        optional<ErrLine> errLine;
+
+        ErrLine& ensureErrLine() 
+        { 
+            if (!this->errLine.has_value())
+                 this->errLine = optional(ErrLine());
+            return *this->errLine; 
+        }
 };
 
 // -------------------------------------------------
 // ErrorInfo.
 
-// Forward friend class declarations.  "builder classes"
+// Forward friend class declarations.    "builder classes"
 template <class T>
 class AddName;
 
@@ -76,170 +76,170 @@ class AddLOC;
 
 // The error info class itself.
 class ErrorInfo { 
-  public:
-    ErrLevel level;
-    string name;
-    string description;
-    optional<NixCode> nixCode;
-    optional<string> hint;
-    ErrorInfo& GetEI() { return *this; }
-
-    static optional<string> programName;
-
-    // give these access to the private constructor, 
-    // when they are direct descendants (children but not grandchildren).
-    friend AddName<ErrorInfo>;
-    friend AddDescription<ErrorInfo>;
-    friend AddNixCode<ErrorInfo>;
-    friend AddNixFile<ErrorInfo>;
-    friend AddErrLine<ErrorInfo>;
-    friend AddLineNumber<ErrorInfo>;
-    friend AddColumnRange<ErrorInfo>;
-    friend AddLOC<ErrorInfo>;
-    
-    NixCode& ensureNixCode() 
-    { 
-      if (!this->nixCode.has_value())
-         this->nixCode = optional(NixCode());
-      return *this->nixCode; 
-    }
-  protected:
-    // constructor is protected, so only the builder classes can create an ErrorInfo.
-    ErrorInfo(ErrLevel level) { this->level = level; }
+    public:
+        ErrLevel level;
+        string name;
+        string description;
+        optional<NixCode> nixCode;
+        optional<string> hint;
+        ErrorInfo& GetEI() { return *this; }
+
+        static optional<string> programName;
+
+        // give these access to the private constructor, 
+        // when they are direct descendants (children but not grandchildren).
+        friend AddName<ErrorInfo>;
+        friend AddDescription<ErrorInfo>;
+        friend AddNixCode<ErrorInfo>;
+        friend AddNixFile<ErrorInfo>;
+        friend AddErrLine<ErrorInfo>;
+        friend AddLineNumber<ErrorInfo>;
+        friend AddColumnRange<ErrorInfo>;
+        friend AddLOC<ErrorInfo>;
+
+        NixCode& ensureNixCode() 
+        { 
+            if (!this->nixCode.has_value())
+                 this->nixCode = optional(NixCode());
+            return *this->nixCode; 
+        }
+    protected:
+        // constructor is protected, so only the builder classes can create an ErrorInfo.
+        ErrorInfo(ErrLevel level) { this->level = level; }
 };
 
 // Init as error
 class EIError : public ErrorInfo
 {
-  protected:
-    EIError() : ErrorInfo(elError) {}
+    protected:
+        EIError() : ErrorInfo(elError) {}
 };
 
 // Init as warning
 class EIWarning : public ErrorInfo
 {
-  protected:
-    EIWarning() : ErrorInfo(elWarning) {}
+    protected:
+        EIWarning() : ErrorInfo(elWarning) {}
 };
 
 // Builder class definitions.
 template <class T>
 class AddName : private T
 {
-  public:
-    T& name(const std::string &name){
-      GetEI().name = name;
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& name(const std::string &name){
+            GetEI().name = name;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T>
 class AddDescription : private T 
 {
-  public:
-    T& description(const std::string &description){
-      GetEI().description = description;
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& description(const std::string &description){
+            GetEI().description = description;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T> 
 class AddNixFile : private T
 {
-  public:
-    T& nixFile(string filename) { 
-     GetEI().ensureNixCode().nixFile = filename;
-     return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& nixFile(string filename) { 
+            GetEI().ensureNixCode().nixFile = filename;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T> 
 class AddLineNumber : private T
 {
-  public:
-    T& lineNumber(int lineNumber) { 
-      GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& lineNumber(int lineNumber) { 
+            GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T> 
 class AddColumnRange : private T
 {
-  public:
-    T& columnRange(unsigned int start, unsigned int len) { 
-      GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& columnRange(unsigned int start, unsigned int len) { 
+            GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T> 
 class AddLOC : private T
 {
-  public:
-    T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
-      GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
-      GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
-      GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
+            GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
+            GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
+            GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 template <class T>
 class yellowify
 {
 public:
-    yellowify(T &s) : value(s) {}
-    T &value;
+        yellowify(T &s) : value(s) {}
+        T &value;
 };
 
 template <class T>
 std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
 {
-  return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
 
 // hint format shows templated values in yellow.
 class hintfmt 
 {
-  public:
-    hintfmt(string format) :fmt(format) {}
-    template<class T>
-      hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; }
-
-    template <typename U>
-    friend class AddHint;
-  private:
-    format fmt;
-      
+    public:
+        hintfmt(string format) :fmt(format) {}
+        template<class T>
+            hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; }
+
+        template <typename U>
+        friend class AddHint;
+    private:
+        format fmt;
+
 };
 
 template <class T> 
 class AddHint : private T
 {
-  public:
-    T& hint(hintfmt &hfmt) {
-      GetEI().hint = optional(hfmt.fmt.str());
-      return *this;
-    }
-    T& nohint() {
-      GetEI().hint = std::nullopt;
-      return *this;
-    }
-  protected:
-    ErrorInfo& GetEI() { return T::GetEI(); }
+    public:
+        T& hint(hintfmt &hfmt) {
+            GetEI().hint = optional(hfmt.fmt.str());
+            return *this;
+        }
+        T& nohint() {
+            GetEI().hint = std::nullopt;
+            return *this;
+        }
+    protected:
+        ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
 // --------------------------------------------------------

From e697884f654fb48f0fee05de25910b27fa147f07 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 1 Apr 2020 21:30:19 -0600
Subject: [PATCH 1921/2196] using std:: everywhere; fix a formatting error; add
 exception flags

---
 src/libutil/error.cc | 41 ++++++++++++++++++++---------------------
 src/libutil/error.hh | 35 +++++++++++++++++++++--------------
 tests/errors/main.cc |  8 +++-----
 3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 4480e80bf1e..04da075edf7 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -5,9 +5,8 @@
 
 namespace nix {
 
-optional<string> ErrorInfo::programName = std::nullopt;
+std::optional<string> ErrorInfo::programName = std::nullopt;
 
-// return basic_format?
 string showErrLine(ErrLine &errLine)
 {
     if (errLine.columnRange.has_value()) 
@@ -27,19 +26,19 @@ void printCodeLines(string &prefix, NixCode &nixCode)
     {
         // previous line of code.
         if (nixCode.errLine->prevLineOfCode.has_value()) { 
-            cout << format("%1% %|2$5d|| %3%")
+            std::cout << format("%1% %|2$5d|| %3%")
                 %    prefix
                 % (nixCode.errLine->lineNumber - 1)
                 % *nixCode.errLine->prevLineOfCode
-                << endl;
+                << std::endl;
         }
 
         // line of code containing the error.%2$+5d%
-        cout << format("%1% %|2$5d|| %3%")
+        std::cout << format("%1% %|2$5d|| %3%")
             %    prefix
             % (nixCode.errLine->lineNumber)
             % nixCode.errLine->errLineOfCode
-            << endl;
+            << std::endl;
 
       // error arrows for the column range.
         if (nixCode.errLine->columnRange.has_value()) 
@@ -58,18 +57,18 @@ void printCodeLines(string &prefix, NixCode &nixCode)
                 arrows.append("^");
             }
 
-          cout << format("%1%            |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl;
+          std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
         }
 
 
 
         // next line of code.
         if (nixCode.errLine->nextLineOfCode.has_value()) { 
-            cout << format("%1% %|2$5d|| %3%")
+            std::cout << format("%1% %|2$5d|| %3%")
             %    prefix
             % (nixCode.errLine->lineNumber + 1)
             % *nixCode.errLine->nextLineOfCode
-            << endl;
+            << std::endl;
         }
 
     }
@@ -113,14 +112,14 @@ void printErrorInfo(ErrorInfo &einfo)
         dashes.append("-");
 
     // divider.
-    cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
+    std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
         % prefix
         % levelString
         % "---"
         % einfo.name
         % dashes
         % einfo.programName.value_or("")
-        << endl;
+        << std::endl;
 
     // filename.
     if (einfo.nixCode.has_value()) 
@@ -131,33 +130,33 @@ void printErrorInfo(ErrorInfo &einfo)
                 ? string(" ") + showErrLine(*einfo.nixCode->errLine)
                 : "";
 
-            cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
-                % prefix % *einfo.nixCode->nixFile % eline << endl;
-            cout << prefix << endl;
+            std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
+                % prefix % *einfo.nixCode->nixFile % eline << std::endl;
+            std::cout << prefix << std::endl;
         }
         else
         {
-            cout << format("%1%from command line argument") % prefix << endl;
-            cout << prefix << endl;
+            std::cout << format("%1%from command line argument") % prefix << std::endl;
+            std::cout << prefix << std::endl;
         }
     }
 
     // description
-    cout << prefix << einfo.description << endl;
-    cout << prefix << endl;
+    std::cout << prefix << einfo.description << std::endl;
+    std::cout << prefix << std::endl;
 
     // lines of code.
     if (einfo.nixCode.has_value())
     {
         printCodeLines(prefix, *einfo.nixCode);
-        cout << prefix << endl;
+        std::cout << prefix << std::endl;
     }
 
     // hint
     if (einfo.hint.has_value()) 
     {
-        cout << prefix << *einfo.hint << endl;
-        cout << prefix << endl;
+        std::cout << prefix << *einfo.hint << std::endl;
+        std::cout << prefix << std::endl;
     }
 }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 00c32ff21da..9dcb42e1470 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -27,21 +27,21 @@ class ErrorInfo;
 class ErrLine { 
     public:
         int lineNumber;
-        optional<ColumnRange> columnRange;
-        optional<string> prevLineOfCode;
+        std::optional<ColumnRange> columnRange;
+        std::optional<string> prevLineOfCode;
         string errLineOfCode;
-        optional<string> nextLineOfCode;
+        std::optional<string> nextLineOfCode;
 };
 
 class NixCode { 
     public:
-        optional<string> nixFile;
-        optional<ErrLine> errLine;
+        std::optional<string> nixFile;
+        std::optional<ErrLine> errLine;
 
         ErrLine& ensureErrLine() 
         { 
             if (!this->errLine.has_value())
-                 this->errLine = optional(ErrLine());
+                 this->errLine = std::optional(ErrLine());
             return *this->errLine; 
         }
 };
@@ -80,11 +80,11 @@ class ErrorInfo {
         ErrLevel level;
         string name;
         string description;
-        optional<NixCode> nixCode;
-        optional<string> hint;
+        std::optional<NixCode> nixCode;
+        std::optional<string> hint;
         ErrorInfo& GetEI() { return *this; }
 
-        static optional<string> programName;
+        static std::optional<string> programName;
 
         // give these access to the private constructor, 
         // when they are direct descendants (children but not grandchildren).
@@ -100,7 +100,7 @@ class ErrorInfo {
         NixCode& ensureNixCode() 
         { 
             if (!this->nixCode.has_value())
-                 this->nixCode = optional(NixCode());
+                 this->nixCode = std::optional(NixCode());
             return *this->nixCode; 
         }
     protected:
@@ -187,7 +187,7 @@ template <class T>
 class AddLOC : private T
 {
     public:
-        T& linesOfCode(optional<string> prevloc, string loc, optional<string> nextloc) { 
+        T& linesOfCode(std::optional<string> prevloc, string loc, std::optional<string> nextloc) { 
             GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
             GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
             GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
@@ -197,6 +197,11 @@ class AddLOC : private T
         ErrorInfo& GetEI() { return T::GetEI(); }
 };
 
+
+// ----------------------------------------------------------------
+// format for hints.  same as boost format, except templated values 
+// are always in yellow.
+
 template <class T>
 class yellowify
 {
@@ -211,11 +216,12 @@ std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
     return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
 
-// hint format shows templated values in yellow.
 class hintfmt 
 {
     public:
-        hintfmt(string format) :fmt(format) {}
+        hintfmt(string format) :fmt(format) {
+            fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+          }
         template<class T>
             hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; }
 
@@ -226,12 +232,13 @@ class hintfmt
 
 };
 
+// the template layer for adding a hint.
 template <class T> 
 class AddHint : private T
 {
     public:
         T& hint(hintfmt &hfmt) {
-            GetEI().hint = optional(hfmt.fmt.str());
+            GetEI().hint = std::optional(hfmt.fmt.str());
             return *this;
         }
         T& nohint() {
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index f11390c7329..870af75d3f9 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -3,14 +3,12 @@
 #include <optional>
 #include <iostream>
 
-
-
 int main() 
 {
     using namespace nix; 
 
     // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = optional("error-test");
+    ErrorInfo::programName = std::optional("error-test");
 
     // There are currently four error types:
     //  
@@ -83,9 +81,9 @@ int main()
             .nixFile("myfile.nix")
             .lineNumber(40)
             .columnRange(13,7)
-            .linesOfCode(optional("previous line of code")
+            .linesOfCode(std::optional("previous line of code")
                         ,"this is the problem line of code"
-                        ,optional("next line of code"))
+                        ,std::optional("next line of code"))
             .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
             );
 

From c34e96f7e0d53894f302134b2f90f83b20ffd22a Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Thu, 2 Apr 2020 05:03:58 +0200
Subject: [PATCH 1922/2196] Make function arguments retain position info

This allows querying the location of function arguments. E.g.

  builtins.unsafeGetAttrPos "x" (builtins.functionArgs ({ x }: null))

  => { column = 57; file = "/home/infinisil/src/nix/inst/test.nix"; line = 1; }
---
 src/libexpr/nixexpr.hh                           | 3 ++-
 src/libexpr/parser.y                             | 4 ++--
 src/libexpr/primops.cc                           | 7 +++++--
 tests/lang/eval-okay-getattrpos-functionargs.exp | 1 +
 tests/lang/eval-okay-getattrpos-functionargs.nix | 4 ++++
 5 files changed, 14 insertions(+), 5 deletions(-)
 create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.exp
 create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.nix

diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index f7e9105a401..8c96de37cbe 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -209,9 +209,10 @@ struct ExprList : Expr
 
 struct Formal
 {
+    Pos pos;
     Symbol name;
     Expr * def;
-    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+    Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
 };
 
 struct Formals
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 9c769e80376..08cec96dca4 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -531,8 +531,8 @@ formals
   ;
 
 formal
-  : ID { $$ = new Formal(data->symbols.create($1), 0); }
-  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
+  : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
   ;
 
 %%
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8de23495187..81c378dff8c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1354,9 +1354,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     }
 
     state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size());
-    for (auto & i : args[0]->lambda.fun->formals->formals)
+    for (auto & i : args[0]->lambda.fun->formals->formals) {
         // !!! should optimise booleans (allocate only once)
-        mkBool(*state.allocAttr(v, i.name), i.def);
+        Value * value = state.allocValue();
+        v.attrs->push_back(Attr(i.name, value, &i.pos));
+        mkBool(*value, i.def);
+    }
     v.attrs->sort();
 }
 
diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp
new file mode 100644
index 00000000000..7f9ac40e81b
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-functionargs.exp
@@ -0,0 +1 @@
+{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; }
diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix
new file mode 100644
index 00000000000..11d6bb0e3ac
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-functionargs.nix
@@ -0,0 +1,4 @@
+let
+  fun = { foo }: {};
+  pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun);
+in { inherit (pos) column line; file = baseNameOf pos.file; }

From 4fc4eb6c93511c61a278ba0e5555ffb24435ab59 Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Thu, 2 Apr 2020 17:04:00 +0200
Subject: [PATCH 1923/2196] libexpr: remove unused attrError

The attrError variable is no longer used but still allocated on every
call to the findAlongAttrPath function.
---
 src/libexpr/attr-path.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 4545bfd7262..76d101b98e9 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -37,9 +37,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 {
     Strings tokens = parseAttrPath(attrPath);
 
-    Error attrError =
-        Error(format("attribute selection path '%1%' does not match expression") % attrPath);
-
     Value * v = &vIn;
     Pos pos = noPos;
 

From b85ba3e30da663a00847a30fd9ceb1bc8284bafc Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 2 Apr 2020 14:08:05 -0600
Subject: [PATCH 1924/2196] full include path

---
 tests/errors/main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 870af75d3f9..5de36a37d80 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -1,4 +1,4 @@
-#include "error.hh"
+#include "../../src/libutil/error.hh"
 
 #include <optional>
 #include <iostream>

From 1c329ca433e78ba9018a54f5d499c7cabe3c7b66 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 2 Apr 2020 14:25:43 -0600
Subject: [PATCH 1925/2196] indenting

---
 src/libutil/error.hh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 9dcb42e1470..32571d97f63 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -12,9 +12,9 @@
 namespace nix {
 
 typedef enum { 
-    elWarning
-    , elError
-    } ErrLevel;
+    elWarning,
+    elError
+} ErrLevel;
 
 class ColumnRange { 
     public:

From c6b3fcddb0d05718ec11807ad847d7bf60cc703f Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 2 Apr 2020 16:02:40 -0600
Subject: [PATCH 1926/2196] formatted with astyle

---
 src/libutil/error.cc | 138 ++++++++---------
 src/libutil/error.hh | 361 ++++++++++++++++++++++++-------------------
 tests/errors/main.cc | 103 ++++++------
 3 files changed, 314 insertions(+), 288 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 04da075edf7..8ab4a2dea9c 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -3,109 +3,99 @@
 #include <iostream>
 #include <optional>
 
-namespace nix {
+namespace nix
+{
 
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
 string showErrLine(ErrLine &errLine)
 {
-    if (errLine.columnRange.has_value()) 
-    {
+    if (errLine.columnRange.has_value()) {
         return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
-    }
-    else
-    {
+    } else {
         return (format("(%1%)") % errLine.lineNumber).str();
     };
 }
 
-void printCodeLines(string &prefix, NixCode &nixCode) 
+void printCodeLines(string &prefix, NixCode &nixCode)
 {
 
-  if (nixCode.errLine.has_value()) 
-    {
+    if (nixCode.errLine.has_value()) {
         // previous line of code.
-        if (nixCode.errLine->prevLineOfCode.has_value()) { 
+        if (nixCode.errLine->prevLineOfCode.has_value()) {
             std::cout << format("%1% %|2$5d|| %3%")
-                %    prefix
-                % (nixCode.errLine->lineNumber - 1)
-                % *nixCode.errLine->prevLineOfCode
-                << std::endl;
+                      %    prefix
+                      % (nixCode.errLine->lineNumber - 1)
+                      % *nixCode.errLine->prevLineOfCode
+                      << std::endl;
         }
 
         // line of code containing the error.%2$+5d%
         std::cout << format("%1% %|2$5d|| %3%")
-            %    prefix
-            % (nixCode.errLine->lineNumber)
-            % nixCode.errLine->errLineOfCode
-            << std::endl;
-
-      // error arrows for the column range.
-        if (nixCode.errLine->columnRange.has_value()) 
-        {
+                  %    prefix
+                  % (nixCode.errLine->lineNumber)
+                  % nixCode.errLine->errLineOfCode
+                  << std::endl;
+
+        // error arrows for the column range.
+        if (nixCode.errLine->columnRange.has_value()) {
             int start = nixCode.errLine->columnRange->start;
             std::string spaces;
-            for (int i = 0; i < start; ++i)
-            {
+            for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
             }
 
             int len = nixCode.errLine->columnRange->len;
             std::string arrows;
-            for (int i = 0; i < len; ++i)
-            {
+            for (int i = 0; i < len; ++i) {
                 arrows.append("^");
             }
 
-          std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
+            std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
         }
 
 
 
         // next line of code.
-        if (nixCode.errLine->nextLineOfCode.has_value()) { 
+        if (nixCode.errLine->nextLineOfCode.has_value()) {
             std::cout << format("%1% %|2$5d|| %3%")
-            %    prefix
-            % (nixCode.errLine->lineNumber + 1)
-            % *nixCode.errLine->nextLineOfCode
-            << std::endl;
+                      %    prefix
+                      % (nixCode.errLine->lineNumber + 1)
+                      % *nixCode.errLine->nextLineOfCode
+                      << std::endl;
         }
 
     }
 
 }
 
-void printErrorInfo(ErrorInfo &einfo) 
+void printErrorInfo(ErrorInfo &einfo)
 {
     int errwidth = 80;
     string prefix = "    ";
 
     string levelString;
-    switch (einfo.level) 
-    {
-        case ErrLevel::elError: 
-            {
-                levelString = ANSI_RED;
-                levelString += "error:";
-                levelString += ANSI_NORMAL;
-                break;
-            }
-        case ErrLevel::elWarning: 
-            {
-                levelString = ANSI_YELLOW;
-                levelString += "warning:";    
-                levelString += ANSI_NORMAL;
-                break;
-            }
-        default: 
-            {
-                levelString = (format("invalid error level: %1%") % einfo.level).str();    
-                break;
-            }
+    switch (einfo.level) {
+    case ErrLevel::elError: {
+        levelString = ANSI_RED;
+        levelString += "error:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    case ErrLevel::elWarning: {
+        levelString = ANSI_YELLOW;
+        levelString += "warning:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    default: {
+        levelString = (format("invalid error level: %1%") % einfo.level).str();
+        break;
+    }
     }
 
     int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
-    int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; 
+    int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
 
     string dashes;
     for (int i = 0; i < dashwidth; ++i)
@@ -113,29 +103,25 @@ void printErrorInfo(ErrorInfo &einfo)
 
     // divider.
     std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
-        % prefix
-        % levelString
-        % "---"
-        % einfo.name
-        % dashes
-        % einfo.programName.value_or("")
-        << std::endl;
+              % prefix
+              % levelString
+              % "---"
+              % einfo.name
+              % dashes
+              % einfo.programName.value_or("")
+              << std::endl;
 
     // filename.
-    if (einfo.nixCode.has_value()) 
-    {
-        if (einfo.nixCode->nixFile.has_value()) 
-        {
+    if (einfo.nixCode.has_value()) {
+        if (einfo.nixCode->nixFile.has_value()) {
             string eline = einfo.nixCode->errLine.has_value()
-                ? string(" ") + showErrLine(*einfo.nixCode->errLine)
-                : "";
+                           ? string(" ") + showErrLine(*einfo.nixCode->errLine)
+                           : "";
 
-            std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) 
-                % prefix % *einfo.nixCode->nixFile % eline << std::endl;
+            std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL)
+                      % prefix % *einfo.nixCode->nixFile % eline << std::endl;
             std::cout << prefix << std::endl;
-        }
-        else
-        {
+        } else {
             std::cout << format("%1%from command line argument") % prefix << std::endl;
             std::cout << prefix << std::endl;
         }
@@ -146,15 +132,13 @@ void printErrorInfo(ErrorInfo &einfo)
     std::cout << prefix << std::endl;
 
     // lines of code.
-    if (einfo.nixCode.has_value())
-    {
+    if (einfo.nixCode.has_value()) {
         printCodeLines(prefix, *einfo.nixCode);
         std::cout << prefix << std::endl;
     }
 
     // hint
-    if (einfo.hint.has_value()) 
-    {
+    if (einfo.hint.has_value()) {
         std::cout << prefix << *einfo.hint << std::endl;
         std::cout << prefix << std::endl;
     }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 32571d97f63..70884c189bb 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -9,41 +9,45 @@
 
 #include <boost/format.hpp>
 
-namespace nix {
+namespace nix
+{
 
-typedef enum { 
+typedef enum {
     elWarning,
     elError
 } ErrLevel;
 
-class ColumnRange { 
-    public:
-        unsigned int start;
-        unsigned int len;
+class ColumnRange
+{
+public:
+    unsigned int start;
+    unsigned int len;
 };
 
 class ErrorInfo;
 
-class ErrLine { 
-    public:
-        int lineNumber;
-        std::optional<ColumnRange> columnRange;
-        std::optional<string> prevLineOfCode;
-        string errLineOfCode;
-        std::optional<string> nextLineOfCode;
+class ErrLine
+{
+public:
+    int lineNumber;
+    std::optional<ColumnRange> columnRange;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
 };
 
-class NixCode { 
-    public:
-        std::optional<string> nixFile;
-        std::optional<ErrLine> errLine;
-
-        ErrLine& ensureErrLine() 
-        { 
-            if (!this->errLine.has_value())
-                 this->errLine = std::optional(ErrLine());
-            return *this->errLine; 
-        }
+class NixCode
+{
+public:
+    std::optional<string> nixFile;
+    std::optional<ErrLine> errLine;
+
+    ErrLine& ensureErrLine()
+    {
+        if (!this->errLine.has_value())
+            this->errLine = std::optional(ErrLine());
+        return *this->errLine;
+    }
 };
 
 // -------------------------------------------------
@@ -65,149 +69,180 @@ class AddNixFile;
 template <class T>
 class AddErrLine;
 
-template <class T> 
+template <class T>
 class AddLineNumber;
 
-template <class T> 
+template <class T>
 class AddColumnRange;
 
-template <class T> 
+template <class T>
 class AddLOC;
 
 // The error info class itself.
-class ErrorInfo { 
-    public:
-        ErrLevel level;
-        string name;
-        string description;
-        std::optional<NixCode> nixCode;
-        std::optional<string> hint;
-        ErrorInfo& GetEI() { return *this; }
-
-        static std::optional<string> programName;
-
-        // give these access to the private constructor, 
-        // when they are direct descendants (children but not grandchildren).
-        friend AddName<ErrorInfo>;
-        friend AddDescription<ErrorInfo>;
-        friend AddNixCode<ErrorInfo>;
-        friend AddNixFile<ErrorInfo>;
-        friend AddErrLine<ErrorInfo>;
-        friend AddLineNumber<ErrorInfo>;
-        friend AddColumnRange<ErrorInfo>;
-        friend AddLOC<ErrorInfo>;
-
-        NixCode& ensureNixCode() 
-        { 
-            if (!this->nixCode.has_value())
-                 this->nixCode = std::optional(NixCode());
-            return *this->nixCode; 
-        }
-    protected:
-        // constructor is protected, so only the builder classes can create an ErrorInfo.
-        ErrorInfo(ErrLevel level) { this->level = level; }
+class ErrorInfo
+{
+public:
+    ErrLevel level;
+    string name;
+    string description;
+    std::optional<NixCode> nixCode;
+    std::optional<string> hint;
+    ErrorInfo& GetEI()
+    {
+        return *this;
+    }
+
+    static std::optional<string> programName;
+
+    // give these access to the private constructor,
+    // when they are direct descendants (children but not grandchildren).
+    friend AddName<ErrorInfo>;
+    friend AddDescription<ErrorInfo>;
+    friend AddNixCode<ErrorInfo>;
+    friend AddNixFile<ErrorInfo>;
+    friend AddErrLine<ErrorInfo>;
+    friend AddLineNumber<ErrorInfo>;
+    friend AddColumnRange<ErrorInfo>;
+    friend AddLOC<ErrorInfo>;
+
+    NixCode& ensureNixCode()
+    {
+        if (!this->nixCode.has_value())
+            this->nixCode = std::optional(NixCode());
+        return *this->nixCode;
+    }
+protected:
+    // constructor is protected, so only the builder classes can create an ErrorInfo.
+    ErrorInfo(ErrLevel level)
+    {
+        this->level = level;
+    }
 };
 
 // Init as error
 class EIError : public ErrorInfo
 {
-    protected:
-        EIError() : ErrorInfo(elError) {}
+protected:
+    EIError() : ErrorInfo(elError) {}
 };
 
 // Init as warning
 class EIWarning : public ErrorInfo
 {
-    protected:
-        EIWarning() : ErrorInfo(elWarning) {}
+protected:
+    EIWarning() : ErrorInfo(elWarning) {}
 };
 
 // Builder class definitions.
 template <class T>
 class AddName : private T
 {
-    public:
-        T& name(const std::string &name){
-            GetEI().name = name;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& name(const std::string &name)
+    {
+        GetEI().name = name;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
 template <class T>
-class AddDescription : private T 
+class AddDescription : private T
 {
-    public:
-        T& description(const std::string &description){
-            GetEI().description = description;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& description(const std::string &description)
+    {
+        GetEI().description = description;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
-template <class T> 
+template <class T>
 class AddNixFile : private T
 {
-    public:
-        T& nixFile(string filename) { 
-            GetEI().ensureNixCode().nixFile = filename;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& nixFile(string filename)
+    {
+        GetEI().ensureNixCode().nixFile = filename;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
-template <class T> 
+template <class T>
 class AddLineNumber : private T
 {
-    public:
-        T& lineNumber(int lineNumber) { 
-            GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& lineNumber(int lineNumber)
+    {
+        GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
-template <class T> 
+template <class T>
 class AddColumnRange : private T
 {
-    public:
-        T& columnRange(unsigned int start, unsigned int len) { 
-            GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& columnRange(unsigned int start, unsigned int len)
+    {
+        GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
-template <class T> 
+template <class T>
 class AddLOC : private T
 {
-    public:
-        T& linesOfCode(std::optional<string> prevloc, string loc, std::optional<string> nextloc) { 
-            GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
-            GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
-            GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& linesOfCode(std::optional<string> prevloc, string loc, std::optional<string> nextloc)
+    {
+        GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
+        GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
+        GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
 
 // ----------------------------------------------------------------
-// format for hints.  same as boost format, except templated values 
+// format for hints.  same as boost format, except templated values
 // are always in yellow.
 
 template <class T>
 class yellowify
 {
 public:
-        yellowify(T &s) : value(s) {}
-        T &value;
+    yellowify(T &s) : value(s) {}
+    T &value;
 };
 
 template <class T>
@@ -216,69 +251,79 @@ std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
     return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
 
-class hintfmt 
+class hintfmt
 {
-    public:
-        hintfmt(string format) :fmt(format) {
-            fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-          }
-        template<class T>
-            hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; }
-
-        template <typename U>
-        friend class AddHint;
-    private:
-        format fmt;
+public:
+    hintfmt(string format) :fmt(format)
+    {
+        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    }
+    template<class T>
+    hintfmt& operator%(const T &value)
+    {
+        fmt % yellowify(value);
+        return *this;
+    }
+
+    template <typename U>
+    friend class AddHint;
+private:
+    format fmt;
 
 };
 
 // the template layer for adding a hint.
-template <class T> 
+template <class T>
 class AddHint : private T
 {
-    public:
-        T& hint(hintfmt &hfmt) {
-            GetEI().hint = std::optional(hfmt.fmt.str());
-            return *this;
-        }
-        T& nohint() {
-            GetEI().hint = std::nullopt;
-            return *this;
-        }
-    protected:
-        ErrorInfo& GetEI() { return T::GetEI(); }
+public:
+    T& hint(hintfmt &hfmt)
+    {
+        GetEI().hint = std::optional(hfmt.fmt.str());
+        return *this;
+    }
+    T& nohint()
+    {
+        GetEI().hint = std::nullopt;
+        return *this;
+    }
+protected:
+    ErrorInfo& GetEI()
+    {
+        return T::GetEI();
+    }
 };
 
 // --------------------------------------------------------
 // error types
 
 typedef AddName<
-        AddDescription<
-        AddHint<
-        EIError>>> ProgramError;
+AddDescription<
+AddHint<
+EIError>>> ProgramError;
 
 typedef AddName<
-        AddDescription<
-        AddHint<
-        EIWarning>>> ProgramWarning;
+AddDescription<
+AddHint<
+EIWarning>>> ProgramWarning;
 
 typedef AddName<
-        AddDescription<
-        AddNixFile<
-        AddLineNumber<
-        AddColumnRange<
-        AddLOC<
-        AddHint<
-        EIError>>>>>>> NixLangError;
+AddDescription<
+AddNixFile<
+AddLineNumber<
+AddColumnRange<
+AddLOC<
+AddHint<
+EIError>>>>>>> NixLangError;
 
 typedef AddName<
-        AddDescription<
-        AddNixFile<
-        AddLineNumber<
-        AddColumnRange<
-        AddLOC<
-        AddHint<
-        EIWarning>>>>>>> NixLangWarning;
+AddDescription<
+AddNixFile<
+AddLineNumber<
+AddColumnRange<
+AddLOC<
+AddHint<
+EIWarning>>>>>>> NixLangWarning;
 
 
 // --------------------------------------------------------
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index 5de36a37d80..e035882e7b5 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -1,45 +1,45 @@
 #include "../../src/libutil/error.hh"
 
-#include <optional>
 #include <iostream>
+#include <optional>
 
-int main() 
+int main()
 {
-    using namespace nix; 
+    using namespace nix;
 
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-test");
 
     // There are currently four error types:
-    //  
+    //
     // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
-    // 			
+    //
     // Each error type is created with a specific sequence of builder functions.
     // Unlike with a constructor, each parameter is clearly named.
-    // If the sequence of function calls isn't followed, then there's a type error.
-    // This should make for a consistent look in the code when errors are created.
+    // If the sequence of function calls isn't followed, then there's a type
+    // error. This should make for a consistent look in the code when errors are
+    // created.
 
     // ProgramError takes name, description, and an optional hint.
-    printErrorInfo(
-        ProgramError()
-            .name("name")
-            .description("error description")
-            .nohint()
-            );
+    printErrorInfo( ProgramError()
+                    .name("name")
+                    .description("error description")
+                    .nohint()
+                  );
 
     // ProgramWarning takes name, description, and an optional hint.
-    // The hint is in the form of a hintfmt class, which wraps boost::format(), and 
-    // makes all the substituted text yellow.
-    printErrorInfo(
-        ProgramWarning()
-            .name("warning name")
-            .description("warning description")
-            .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow.
-            );
+    // The hint is in the form of a hintfmt class, which wraps boost::format(),
+    // and makes all the substituted text yellow.
+    printErrorInfo( ProgramWarning()
+                    .name("warning name")
+                    .description("warning description")
+                    .hint(hintfmt("there was a %1%") %
+                          "warning") // 'warning' will be yellow.
+                  );
 
     /*
       // some invalid errors:
-      
+
       // type error: no hint function.
       ProgramError()
         .name("name")
@@ -51,41 +51,38 @@ int main()
         .name("name")
         .nohint();
 
-      // type error: hint function with regular boost format, not special hintfmt.
-      ProgramError()
-        .description("error description")
-        .name("name")
+      // type error: hint function with regular boost format, not special
+      hintfmt. ProgramError() .description("error description") .name("name")
         .hint(format("there was a %1%") % "warning");
     */
 
-    // NixLangWarning adds nix file, line number, column range, and the lines of code 
-    // where a warning occurred.
-    printErrorInfo(
-        NixLangWarning()
-            .name("warning name")
-            .description("warning description")
-            .nixFile("myfile.nix")
-            .lineNumber(40)
-            .columnRange(13,7)
-            .linesOfCode(std::nullopt
-                        ,"this is the problem line of code"
-                        ,std::nullopt)
-            .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-            );
+    // NixLangWarning adds nix file, line number, column range, and the lines of
+    // code where a warning occurred.
+    printErrorInfo(NixLangWarning()
+                   .name("warning name")
+                   .description("warning description")
+                   .nixFile("myfile.nix")
+                   .lineNumber(40)
+                   .columnRange(13, 7)
+                   .linesOfCode(std::nullopt,
+                                "this is the problem line of code",
+                                std::nullopt)
+                   .hint(hintfmt("this hint has %1% templated %2%!!") %
+                         "yellow" % "values"));
 
-    // NixLangError is just the same as NixLangWarning, except for the Error flag.
-    printErrorInfo(
-        NixLangError()
-            .name("error name")
-            .description("error description")
-            .nixFile("myfile.nix")
-            .lineNumber(40)
-            .columnRange(13,7)
-            .linesOfCode(std::optional("previous line of code")
-                        ,"this is the problem line of code"
-                        ,std::optional("next line of code"))
-            .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values")
-            );
+    // NixLangError is just the same as NixLangWarning, except for the Error
+    // flag.
+    printErrorInfo(NixLangError()
+                   .name("error name")
+                   .description("error description")
+                   .nixFile("myfile.nix")
+                   .lineNumber(40)
+                   .columnRange(13, 7)
+                   .linesOfCode(std::optional("previous line of code"),
+                                "this is the problem line of code",
+                                std::optional("next line of code"))
+                   .hint(hintfmt("this hint has %1% templated %2%!!") %
+                         "yellow" % "values"));
 
     return 0;
 }

From 7b7801d3f0e0e1cd32b1279979970ad71b66b879 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 3 Apr 2020 08:48:20 -0600
Subject: [PATCH 1927/2196] variadic args for hint format

---
 src/libutil/error.hh | 26 ++++++++++++++++++++------
 src/libutil/types.hh | 10 ++++++----
 tests/errors/main.cc | 10 ++++------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 70884c189bb..1c5d6d13cbf 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -7,6 +7,7 @@
 #include <iostream>
 #include <iomanip>
 
+#include "types.hh"
 #include <boost/format.hpp>
 
 namespace nix
@@ -234,7 +235,7 @@ protected:
 
 
 // ----------------------------------------------------------------
-// format for hints.  same as boost format, except templated values
+// format for hints.  same as fmt, except templated values
 // are always in yellow.
 
 template <class T>
@@ -251,20 +252,25 @@ std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
     return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
 
-class hintfmt
+class hintformat
 {
 public:
-    hintfmt(string format) :fmt(format)
+    hintformat(string format) :fmt(format)
     {
         fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
     }
     template<class T>
-    hintfmt& operator%(const T &value)
+    hintformat& operator%(const T &value)
     {
         fmt % yellowify(value);
         return *this;
     }
 
+    std::string str() const
+    {
+        return fmt.str();
+    }
+
     template <typename U>
     friend class AddHint;
 private:
@@ -272,14 +278,22 @@ private:
 
 };
 
+template<typename... Args>
+inline hintformat hintfmt(const std::string & fs, const Args & ... args)
+{
+    hintformat f(fs);
+    formatHelper(f, args...);
+    return f;
+}
+
 // the template layer for adding a hint.
 template <class T>
 class AddHint : private T
 {
 public:
-    T& hint(hintfmt &hfmt)
+    T& hint(const hintformat &hf)
     {
-        GetEI().hint = std::optional(hfmt.fmt.str());
+        GetEI().hint = std::optional(hf.str());
         return *this;
     }
     T& nohint()
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 20b96a85ced..981af528b23 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -41,7 +41,8 @@ struct FormatOrString
 {
     string s;
     FormatOrString(const string & s) : s(s) { };
-    FormatOrString(const format & f) : s(f.str()) { };
+    template<class F>
+    FormatOrString(const F & f) : s(f.str()) { };
     FormatOrString(const char * s) : s(s) { };
 };
 
@@ -51,12 +52,13 @@ struct FormatOrString
    ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
    takes place). */
 
-inline void formatHelper(boost::format & f)
+template<class F>
+inline void formatHelper(F & f)
 {
 }
 
-template<typename T, typename... Args>
-inline void formatHelper(boost::format & f, const T & x, const Args & ... args)
+template<class F, typename T, typename... Args>
+inline void formatHelper(F & f, const T & x, const Args & ... args)
 {
     formatHelper(f % x, args...);
 }
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index e035882e7b5..be98bd5b845 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -33,8 +33,8 @@ int main()
     printErrorInfo( ProgramWarning()
                     .name("warning name")
                     .description("warning description")
-                    .hint(hintfmt("there was a %1%") %
-                          "warning") // 'warning' will be yellow.
+                    // the templated value, 'warning', is automatically colored yellow.
+                    .hint(hintfmt("there was a %1%", "warning")) 
                   );
 
     /*
@@ -67,8 +67,7 @@ int main()
                    .linesOfCode(std::nullopt,
                                 "this is the problem line of code",
                                 std::nullopt)
-                   .hint(hintfmt("this hint has %1% templated %2%!!") %
-                         "yellow" % "values"));
+                   .hint(hintfmt("this hint has %1% templated %2%!!", "yellow" , "values")));
 
     // NixLangError is just the same as NixLangWarning, except for the Error
     // flag.
@@ -81,8 +80,7 @@ int main()
                    .linesOfCode(std::optional("previous line of code"),
                                 "this is the problem line of code",
                                 std::optional("next line of code"))
-                   .hint(hintfmt("this hint has %1% templated %2%!!") %
-                         "yellow" % "values"));
+                   .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
 
     return 0;
 }

From 63fa92605b2b0a8a6d948a27e1d331cc46b199bd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Apr 2020 11:54:48 +0200
Subject: [PATCH 1928/2196] nix-env: Refuse to operate on a new-style profile

This prevents users from accidentally nuking their profile via
nix-env.

(cherry picked from commit 021634e3e3edb327089d33ab41b743f0a40126da)
---
 src/nix-env/user-env.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 717431b7aff..f852916d896 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -15,6 +15,8 @@ namespace nix {
 DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
 {
     DrvInfos elems;
+    if (pathExists(userEnv + "/manifest.json"))
+        throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv);
     Path manifestFile = userEnv + "/manifest.nix";
     if (pathExists(manifestFile)) {
         Value v;

From 5e7ccdc9e3ddd61dc85e20c898001345bfb497a5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 3 Apr 2020 20:06:26 +0200
Subject: [PATCH 1929/2196] Publish a tarball containing the crates we depend
 on

This is needed since we no longer produce a source tarball.

(cherry picked from commit bf70a047a0b2da606f65a88f13ce2994065cd9c4)
---
 release.nix | 94 ++++++++++++++++++++++++++++++-----------------------
 1 file changed, 53 insertions(+), 41 deletions(-)

diff --git a/release.nix b/release.nix
index 9b591229ada..65b035957db 100644
--- a/release.nix
+++ b/release.nix
@@ -12,52 +12,64 @@ let
     builtins.readFile ./.version
     + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
 
-  jobs = rec {
-
-    # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock. This allows Nix to be built without network access.
-    vendoredCrates =
-      let
-        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
-
-        files = map (pkg: import <nix/fetchurl.nix> {
-          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
-          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
-        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
-
-      in pkgs.runCommand "cargo-vendor-dir" {}
-        ''
-          mkdir -p $out/vendor
-
-          cat > $out/vendor/config <<EOF
-          [source.crates-io]
-          replace-with = "vendored-sources"
-
-          [source.vendored-sources]
-          directory = "vendor"
-          EOF
-
-          ${toString (builtins.map (file: ''
-            mkdir $out/vendor/tmp
-            tar xvf ${file} -C $out/vendor/tmp
-            dir=$(echo $out/vendor/tmp/*)
+  # Create a "vendor" directory that contains the crates listed in
+  # Cargo.lock. This allows Nix to be built without network access.
+  vendoredCrates' =
+    let
+      lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+      files = map (pkg: import <nix/fetchurl.nix> {
+        url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+        sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+      }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+    in pkgs.runCommand "cargo-vendor-dir" {}
+      ''
+        mkdir -p $out/vendor
+
+        cat > $out/vendor/config <<EOF
+        [source.crates-io]
+        replace-with = "vendored-sources"
+
+        [source.vendored-sources]
+        directory = "vendor"
+        EOF
+
+        ${toString (builtins.map (file: ''
+          mkdir $out/vendor/tmp
+          tar xvf ${file} -C $out/vendor/tmp
+          dir=$(echo $out/vendor/tmp/*)
+
+          # Add just enough metadata to keep Cargo happy.
+          printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+          # Clean up some cruft from the winapi crates. FIXME: find
+          # a way to remove winapi* from our dependencies.
+          if [[ $dir =~ /winapi ]]; then
+            find $dir -name "*.a" -print0 | xargs -0 rm -f --
+          fi
 
-            # Add just enough metadata to keep Cargo happy.
-            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+          mv "$dir" $out/vendor/
 
-            # Clean up some cruft from the winapi crates. FIXME: find
-            # a way to remove winapi* from our dependencies.
-            if [[ $dir =~ /winapi ]]; then
-              find $dir -name "*.a" -print0 | xargs -0 rm -f --
-            fi
+          rm -rf $out/vendor/tmp
+        '') files)}
+      '';
 
-            mv "$dir" $out/vendor/
+  jobs = rec {
 
-            rm -rf $out/vendor/tmp
-          '') files)}
+    vendoredCrates =
+      with pkgs;
+      runCommand "vendored-crates" {}
+        ''
+          mkdir -p $out/nix-support
+          name=nix-vendored-crates-${version}
+          fn=$out/$name.tar.xz
+            tar cvfJ $fn -C ${vendoredCrates'} vendor \
+              --owner=0 --group=0 --mode=u+rw,uga+r \
+              --transform "s,vendor,$name,"
+            echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products
         '';
 
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -89,7 +101,7 @@ let
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
 
-            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+            ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor
 
             (cd perl; autoreconf --install --force --verbose)
           '';

From 9bb528d3920559454a1ce0f8f4ac3ba58b18f6d1 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 3 Apr 2020 13:15:59 -0600
Subject: [PATCH 1930/2196] handle Pos instead of individual
 file/line/columnrange args

---
 src/libutil/error.hh | 98 +++++++++++---------------------------------
 tests/errors/main.cc | 13 +++---
 2 files changed, 31 insertions(+), 80 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 1c5d6d13cbf..57a9944be13 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -8,7 +8,6 @@
 #include <iomanip>
 
 #include "types.hh"
-#include <boost/format.hpp>
 
 namespace nix
 {
@@ -62,19 +61,7 @@ template <class T>
 class AddDescription;
 
 template <class T>
-class AddNixCode;
-
-template <class T>
-class AddNixFile;
-
-template <class T>
-class AddErrLine;
-
-template <class T>
-class AddLineNumber;
-
-template <class T>
-class AddColumnRange;
+class AddPos;
 
 template <class T>
 class AddLOC;
@@ -99,11 +86,7 @@ public:
     // when they are direct descendants (children but not grandchildren).
     friend AddName<ErrorInfo>;
     friend AddDescription<ErrorInfo>;
-    friend AddNixCode<ErrorInfo>;
-    friend AddNixFile<ErrorInfo>;
-    friend AddErrLine<ErrorInfo>;
-    friend AddLineNumber<ErrorInfo>;
-    friend AddColumnRange<ErrorInfo>;
+    friend AddPos<ErrorInfo>;
     friend AddLOC<ErrorInfo>;
 
     NixCode& ensureNixCode()
@@ -168,44 +151,15 @@ protected:
 };
 
 template <class T>
-class AddNixFile : private T
-{
-public:
-    T& nixFile(string filename)
-    {
-        GetEI().ensureNixCode().nixFile = filename;
-        return *this;
-    }
-protected:
-    ErrorInfo& GetEI()
-    {
-        return T::GetEI();
-    }
-};
-
-template <class T>
-class AddLineNumber : private T
-{
-public:
-    T& lineNumber(int lineNumber)
-    {
-        GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber;
-        return *this;
-    }
-protected:
-    ErrorInfo& GetEI()
-    {
-        return T::GetEI();
-    }
-};
-
-template <class T>
-class AddColumnRange : private T
+class AddPos : private T
 {
 public:
-    T& columnRange(unsigned int start, unsigned int len)
+    template <class P>
+    T& pos(const P &aPos)
     {
-        GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len };
+        GetEI().ensureNixCode().nixFile = aPos.file;
+        GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line;
+        GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 };
         return *this;
     }
 protected:
@@ -312,32 +266,28 @@ protected:
 // error types
 
 typedef AddName<
-AddDescription<
-AddHint<
-EIError>>> ProgramError;
+        AddDescription<
+        AddHint<
+        EIError>>> ProgramError;
 
 typedef AddName<
-AddDescription<
-AddHint<
-EIWarning>>> ProgramWarning;
+        AddDescription<
+        AddHint<
+        EIWarning>>> ProgramWarning;
 
 typedef AddName<
-AddDescription<
-AddNixFile<
-AddLineNumber<
-AddColumnRange<
-AddLOC<
-AddHint<
-EIError>>>>>>> NixLangError;
+        AddDescription<
+        AddPos<
+        AddLOC<
+        AddHint<
+        EIError>>>>> NixLangError;
 
 typedef AddName<
-AddDescription<
-AddNixFile<
-AddLineNumber<
-AddColumnRange<
-AddLOC<
-AddHint<
-EIWarning>>>>>>> NixLangWarning;
+        AddDescription<
+        AddPos<
+        AddLOC<
+        AddHint<
+        EIWarning>>>>> NixLangWarning;
 
 
 // --------------------------------------------------------
diff --git a/tests/errors/main.cc b/tests/errors/main.cc
index be98bd5b845..cd6ca7caf4d 100644
--- a/tests/errors/main.cc
+++ b/tests/errors/main.cc
@@ -1,4 +1,5 @@
 #include "../../src/libutil/error.hh"
+#include "../../src/libexpr/nixexpr.hh"
 
 #include <iostream>
 #include <optional>
@@ -58,12 +59,14 @@ int main()
 
     // NixLangWarning adds nix file, line number, column range, and the lines of
     // code where a warning occurred.
+
+    SymbolTable testTable;
+    auto problem_symbol = testTable.create("problem");
+
     printErrorInfo(NixLangWarning()
                    .name("warning name")
                    .description("warning description")
-                   .nixFile("myfile.nix")
-                   .lineNumber(40)
-                   .columnRange(13, 7)
+                   .pos(Pos(problem_symbol, 40, 13))
                    .linesOfCode(std::nullopt,
                                 "this is the problem line of code",
                                 std::nullopt)
@@ -74,9 +77,7 @@ int main()
     printErrorInfo(NixLangError()
                    .name("error name")
                    .description("error description")
-                   .nixFile("myfile.nix")
-                   .lineNumber(40)
-                   .columnRange(13, 7)
+                   .pos(Pos(problem_symbol, 40, 13))
                    .linesOfCode(std::optional("previous line of code"),
                                 "this is the problem line of code",
                                 std::optional("next line of code"))

From 9a8b3e97477f1621d8dc5a968217e4b44eaf59a8 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 3 Apr 2020 14:55:26 -0600
Subject: [PATCH 1931/2196] move out of tests/

---
 Makefile                                             |  2 +-
 tests/errors/main.cc => src/error-demo/error-demo.cc |  6 +++---
 src/error-demo/local.mk                              | 10 ++++++++++
 tests/errors/local.mk                                | 10 ----------
 4 files changed, 14 insertions(+), 14 deletions(-)
 rename tests/errors/main.cc => src/error-demo/error-demo.cc (95%)
 create mode 100644 src/error-demo/local.mk
 delete mode 100644 tests/errors/local.mk

diff --git a/Makefile b/Makefile
index 2d70aeb27c7..4fac27c473c 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,7 @@ makefiles = \
   doc/manual/local.mk \
   tests/local.mk \
   tests/plugins/local.mk \
-  tests/errors/local.mk
+  src/error-demo/local.mk
 
 -include Makefile.config
 
diff --git a/tests/errors/main.cc b/src/error-demo/error-demo.cc
similarity index 95%
rename from tests/errors/main.cc
rename to src/error-demo/error-demo.cc
index cd6ca7caf4d..8fb20d38c85 100644
--- a/tests/errors/main.cc
+++ b/src/error-demo/error-demo.cc
@@ -1,5 +1,5 @@
-#include "../../src/libutil/error.hh"
-#include "../../src/libexpr/nixexpr.hh"
+#include "error.hh"
+#include "nixexpr.hh"
 
 #include <iostream>
 #include <optional>
@@ -9,7 +9,7 @@ int main()
     using namespace nix;
 
     // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = std::optional("error-test");
+    ErrorInfo::programName = std::optional("error-demo");
 
     // There are currently four error types:
     //
diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk
new file mode 100644
index 00000000000..121cab9e367
--- /dev/null
+++ b/src/error-demo/local.mk
@@ -0,0 +1,10 @@
+programs += error-demo
+
+error-demo_DIR := $(d)
+
+error-demo_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+error-demo_LIBS = libutil
+
+error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
diff --git a/tests/errors/local.mk b/tests/errors/local.mk
deleted file mode 100644
index 0d64c8dbdd5..00000000000
--- a/tests/errors/local.mk
+++ /dev/null
@@ -1,10 +0,0 @@
-programs += error-test
-
-error-test_DIR := $(d)
-
-error-test_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-
-error-test_LIBS = libutil
-
-error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

From 1221ae3dd07959d47d9f27e9d2271671003d2bed Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 5 Apr 2020 07:12:16 -0600
Subject: [PATCH 1932/2196] libexpr

---
 src/error-demo/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk
index 121cab9e367..2cb2a0dc43d 100644
--- a/src/error-demo/local.mk
+++ b/src/error-demo/local.mk
@@ -5,6 +5,6 @@ error-demo_DIR := $(d)
 error-demo_SOURCES := \
   $(wildcard $(d)/*.cc) \
 
-error-demo_LIBS = libutil
+error-demo_LIBS = libutil libexpr
 
 error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

From c976cb0b8ab5d0f2c4ab8c9826fc7db56e2f1b3e Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Sat, 4 Apr 2020 12:27:39 -0700
Subject: [PATCH 1933/2196] Don't retry on "unsupported protocol" error

When encountering an unsupported protocol, there's no need to retry.
Chances are, it won't suddenly be supported between retry attempts;
error instead. Otherwise, you see something like the following:

    $ nix-env -i -f git://git@github.com/foo/bar
    warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 335 ms
    warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 604 ms
    warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 1340 ms
    warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 2685 ms

With this change, you now see:

    $ nix-env -i -f git://git@github.com/foo/bar
    error: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1)
---
 src/libstore/download.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 149c8476529..5967d04258b 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -390,6 +390,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_SSL_CACERT_BADFILE:
                         case CURLE_TOO_MANY_REDIRECTS:
                         case CURLE_WRITE_ERROR:
+                        case CURLE_UNSUPPORTED_PROTOCOL:
                             err = Misc;
                             break;
                         default: // Shut up warnings

From 85f14c4582ae3e64d7423ac48daaa0ea437fc144 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 6 Apr 2020 11:15:01 -0600
Subject: [PATCH 1934/2196] add libutil, libexpr include dirs

---
 src/error-demo/local.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk
index 2cb2a0dc43d..2c528490a2a 100644
--- a/src/error-demo/local.mk
+++ b/src/error-demo/local.mk
@@ -5,6 +5,8 @@ error-demo_DIR := $(d)
 error-demo_SOURCES := \
   $(wildcard $(d)/*.cc) \
 
+error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
+
 error-demo_LIBS = libutil libexpr
 
 error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

From 2248cc6716ad5a9a56f9368e3cf6784a01cbb646 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 6 Apr 2020 12:05:17 -0600
Subject: [PATCH 1935/2196] ignore error-demo

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index e3186fa769a..ad568412313 100644
--- a/.gitignore
+++ b/.gitignore
@@ -75,6 +75,8 @@ perl/Makefile.config
 
 /src/nix-copy-closure/nix-copy-closure
 
+/src/error-demo/error-demo
+
 /src/build-remote/build-remote
 
 # /tests/

From ec449c845056dad43eeec4d6ff983002f038cf69 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 6 Apr 2020 19:43:22 -0600
Subject: [PATCH 1936/2196] constructor style basically working

---
 src/error-demo/error-demo.cc |  73 +++++-----
 src/libutil/error.cc         |  42 +++++-
 src/libutil/error.hh         | 251 +++++++++++++++++------------------
 3 files changed, 204 insertions(+), 162 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 8fb20d38c85..fdb574b25a9 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -22,21 +22,25 @@ int main()
     // created.
 
     // ProgramError takes name, description, and an optional hint.
-    printErrorInfo( ProgramError()
-                    .name("name")
-                    .description("error description")
-                    .nohint()
-                  );
+    printErrorInfo(
+        ErrorInfo::ProgramError("name",
+                                "error description",
+                                std::nullopt));
 
     // ProgramWarning takes name, description, and an optional hint.
     // The hint is in the form of a hintfmt class, which wraps boost::format(),
     // and makes all the substituted text yellow.
-    printErrorInfo( ProgramWarning()
-                    .name("warning name")
-                    .description("warning description")
-                    // the templated value, 'warning', is automatically colored yellow.
-                    .hint(hintfmt("there was a %1%", "warning")) 
-                  );
+    printErrorInfo(
+        ErrorInfo::ProgramWarning("name",
+                                  "warning description",
+                                  std::optional(hintfmt("there was a %1%", "warning"))));
+
+    // printErrorInfo( ProgramWarning()
+    //                 .name("warning name")
+    //                 .description("warning description")
+    //                 // the templated value, 'warning', is automatically colored yellow.
+    //                 .hint(hintfmt("there was a %1%", "warning"))
+    //               );
 
     /*
       // some invalid errors:
@@ -63,25 +67,34 @@ int main()
     SymbolTable testTable;
     auto problem_symbol = testTable.create("problem");
 
-    printErrorInfo(NixLangWarning()
-                   .name("warning name")
-                   .description("warning description")
-                   .pos(Pos(problem_symbol, 40, 13))
-                   .linesOfCode(std::nullopt,
-                                "this is the problem line of code",
-                                std::nullopt)
-                   .hint(hintfmt("this hint has %1% templated %2%!!", "yellow" , "values")));
-
-    // NixLangError is just the same as NixLangWarning, except for the Error
-    // flag.
-    printErrorInfo(NixLangError()
-                   .name("error name")
-                   .description("error description")
-                   .pos(Pos(problem_symbol, 40, 13))
-                   .linesOfCode(std::optional("previous line of code"),
-                                "this is the problem line of code",
-                                std::optional("next line of code"))
-                   .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+    printErrorInfo(ErrorInfo::NixLangWarning(
+                       "warning name",
+                       "warning description",
+                       Pos(problem_symbol, 40, 13),
+                       std::nullopt,
+                       "this is the problem line of code",
+                       std::nullopt,
+                       hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+
+    // // NixLangError is just the same as NixLangWarning, except for the Error
+    // // flag.
+    // printErrorInfo(NixLangError()
+    //                .name("error name")
+    //                .description("error description")
+    //                .pos(Pos(problem_symbol, 40, 13))
+    //                .linesOfCode(std::optional("previous line of code"),
+    //                             "this is the problem line of code",
+    //                             std::optional("next line of code"))
+    //                .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+    printErrorInfo(ErrorInfo::NixLangError(
+                       "error name",
+                       "error description",
+                       Pos(problem_symbol, 40, 13),
+                       std::optional("previous line of code"),
+                       "this is the problem line of code",
+                       std::optional("next line of code"),
+                       hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+
 
     return 0;
 }
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 8ab4a2dea9c..41fabbbcf95 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -8,7 +8,43 @@ namespace nix
 
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
-string showErrLine(ErrLine &errLine)
+ErrorInfo ErrorInfo::ProgramError(const string &name,
+                                  const string &description,
+                                  const std::optional<hintformat> &hf)
+{
+    return ProgramEI(elError, name, description, hf);
+}
+
+ErrorInfo ErrorInfo::ProgramWarning(const string &name,
+                                    const string &description,
+                                    const std::optional<hintformat> &hf)
+{
+    return ProgramEI(elWarning, name, description, hf);
+}
+
+
+
+ErrorInfo ErrorInfo::ProgramEI(ErrLevel level,
+                               const string &name,
+                               const string &description,
+                               const std::optional<hintformat> &hf)
+{
+    ErrorInfo ei(elError);
+    ei.name = name;
+    ei.description = description;
+    if (hf.has_value())
+        ei.hint = std::optional<string>(hf->str());
+    else
+        ei.hint = std::nullopt;
+    return ei;
+}
+
+
+
+
+
+
+string showErrLine(const ErrLine &errLine)
 {
     if (errLine.columnRange.has_value()) {
         return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
@@ -17,7 +53,7 @@ string showErrLine(ErrLine &errLine)
     };
 }
 
-void printCodeLines(string &prefix, NixCode &nixCode)
+void printCodeLines(const string &prefix, const NixCode &nixCode)
 {
 
     if (nixCode.errLine.has_value()) {
@@ -69,7 +105,7 @@ void printCodeLines(string &prefix, NixCode &nixCode)
 
 }
 
-void printErrorInfo(ErrorInfo &einfo)
+void printErrorInfo(const ErrorInfo &einfo)
 {
     int errwidth = 80;
     string prefix = "    ";
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 57a9944be13..a73b6639eac 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -41,32 +41,61 @@ class NixCode
 public:
     std::optional<string> nixFile;
     std::optional<ErrLine> errLine;
-
-    ErrLine& ensureErrLine()
-    {
-        if (!this->errLine.has_value())
-            this->errLine = std::optional(ErrLine());
-        return *this->errLine;
-    }
 };
 
-// -------------------------------------------------
-// ErrorInfo.
+// ----------------------------------------------------------------
+// format for hints.  same as fmt, except templated values
+// are always in yellow.
 
-// Forward friend class declarations.    "builder classes"
 template <class T>
-class AddName;
+class yellowify
+{
+public:
+    yellowify(T &s) : value(s) {}
+    T &value;
+};
 
 template <class T>
-class AddDescription;
+std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
+{
+    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+}
 
-template <class T>
-class AddPos;
+class hintformat
+{
+public:
+    hintformat(string format) :fmt(format)
+    {
+        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    }
+    template<class T>
+    hintformat& operator%(const T &value)
+    {
+        fmt % yellowify(value);
+        return *this;
+    }
 
-template <class T>
-class AddLOC;
+    std::string str() const
+    {
+        return fmt.str();
+    }
+
+    template <typename U>
+    friend class AddHint;
+private:
+    format fmt;
+};
+
+template<typename... Args>
+inline hintformat hintfmt(const std::string & fs, const Args & ... args)
+{
+    hintformat f(fs);
+    formatHelper(f, args...);
+    return f;
+}
 
-// The error info class itself.
+// -------------------------------------------------
+// ErrorInfo.
 class ErrorInfo
 {
 public:
@@ -75,81 +104,94 @@ public:
     string description;
     std::optional<NixCode> nixCode;
     std::optional<string> hint;
-    ErrorInfo& GetEI()
-    {
-        return *this;
-    }
 
     static std::optional<string> programName;
 
-    // give these access to the private constructor,
-    // when they are direct descendants (children but not grandchildren).
-    friend AddName<ErrorInfo>;
-    friend AddDescription<ErrorInfo>;
-    friend AddPos<ErrorInfo>;
-    friend AddLOC<ErrorInfo>;
+    static ErrorInfo ProgramError(const string &name,
+                                  const string &description,
+                                  const std::optional<hintformat> &hf);
 
-    NixCode& ensureNixCode()
-    {
-        if (!this->nixCode.has_value())
-            this->nixCode = std::optional(NixCode());
-        return *this->nixCode;
-    }
-protected:
-    // constructor is protected, so only the builder classes can create an ErrorInfo.
-    ErrorInfo(ErrLevel level)
-    {
-        this->level = level;
-    }
-};
 
-// Init as error
-class EIError : public ErrorInfo
-{
-protected:
-    EIError() : ErrorInfo(elError) {}
-};
+    static ErrorInfo ProgramWarning(const string &name,
+                                    const string &description,
+                                    const std::optional<hintformat> &hf);
 
-// Init as warning
-class EIWarning : public ErrorInfo
-{
-protected:
-    EIWarning() : ErrorInfo(elWarning) {}
-};
 
-// Builder class definitions.
-template <class T>
-class AddName : private T
-{
-public:
-    T& name(const std::string &name)
+    template <class P>
+    static ErrorInfo NixLangError(const string &name,
+                                  const string &description,
+                                  const P &pos,
+                                  std::optional<string> prevloc,
+                                  string loc,
+                                  std::optional<string> nextloc,
+                                  const std::optional<hintformat> &hf)
     {
-        GetEI().name = name;
-        return *this;
+        return NixLangEI(elError, name, description, pos, prevloc, loc, nextloc, hf);
     }
-protected:
-    ErrorInfo& GetEI()
+
+
+    template <class P>
+    static ErrorInfo NixLangWarning(const string &name,
+                                    const string &description,
+                                    const P &pos,
+                                    std::optional<string> prevloc,
+                                    string loc,
+                                    std::optional<string> nextloc,
+                                    const std::optional<hintformat> &hf)
     {
-        return T::GetEI();
+        return NixLangEI(elWarning, name, description, pos, prevloc, loc, nextloc, hf);
     }
-};
 
-template <class T>
-class AddDescription : private T
-{
-public:
-    T& description(const std::string &description)
+
+
+private:
+    template <class P>
+    static ErrorInfo NixLangEI(ErrLevel level,
+                               const string &name,
+                               const string &description,
+                               const P &pos,
+                               std::optional<string> prevloc,
+                               string loc,
+                               std::optional<string> nextloc,
+                               const std::optional<hintformat> &hf)
     {
-        GetEI().description = description;
-        return *this;
+        ErrorInfo ei(level);
+        ei.name = name;
+        ei.description = description;
+        if (hf.has_value())
+            ei.hint = std::optional<string>(hf->str());
+        else
+            ei.hint = std::nullopt;
+
+        ErrLine errline;
+        errline.lineNumber = pos.line;
+        errline.columnRange = { .start = pos.column, .len = 1 };
+        errline.prevLineOfCode = prevloc;
+        errline.errLineOfCode = loc;
+        errline.nextLineOfCode = nextloc;
+        NixCode nixcode;
+        nixcode.nixFile = pos.file;
+        nixcode.errLine = std::optional(errline);
+        ei.nixCode = std::optional(nixcode);
+
+        return ei;
     }
-protected:
-    ErrorInfo& GetEI()
+
+    static ErrorInfo ProgramEI(ErrLevel level,
+                               const string &name,
+                               const string &description,
+                               const std::optional<hintformat> &hf);
+
+
+
+    // constructor is protected, so only the builder classes can create an ErrorInfo.
+    ErrorInfo(ErrLevel level)
     {
-        return T::GetEI();
+        this->level = level;
     }
 };
 
+/*
 template <class T>
 class AddPos : private T
 {
@@ -186,60 +228,9 @@ protected:
         return T::GetEI();
     }
 };
+*/
 
-
-// ----------------------------------------------------------------
-// format for hints.  same as fmt, except templated values
-// are always in yellow.
-
-template <class T>
-class yellowify
-{
-public:
-    yellowify(T &s) : value(s) {}
-    T &value;
-};
-
-template <class T>
-std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
-{
-    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
-}
-
-class hintformat
-{
-public:
-    hintformat(string format) :fmt(format)
-    {
-        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    }
-    template<class T>
-    hintformat& operator%(const T &value)
-    {
-        fmt % yellowify(value);
-        return *this;
-    }
-
-    std::string str() const
-    {
-        return fmt.str();
-    }
-
-    template <typename U>
-    friend class AddHint;
-private:
-    format fmt;
-
-};
-
-template<typename... Args>
-inline hintformat hintfmt(const std::string & fs, const Args & ... args)
-{
-    hintformat f(fs);
-    formatHelper(f, args...);
-    return f;
-}
-
+/*
 // the template layer for adding a hint.
 template <class T>
 class AddHint : private T
@@ -261,11 +252,12 @@ protected:
         return T::GetEI();
     }
 };
+*/
 
 // --------------------------------------------------------
 // error types
 
-typedef AddName<
+/*typedef AddName<
         AddDescription<
         AddHint<
         EIError>>> ProgramError;
@@ -290,11 +282,12 @@ typedef AddName<
         EIWarning>>>>> NixLangWarning;
 
 
+*/
 // --------------------------------------------------------
 // error printing
 
 // just to cout for now.
-void printErrorInfo(ErrorInfo &einfo);
+void printErrorInfo(const ErrorInfo &einfo);
 
 }
 

From 55c96b64e4de2b7e3443124bb0aa17ecc9188940 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 6 Apr 2020 20:14:48 -0600
Subject: [PATCH 1937/2196] comment cleanup

---
 src/error-demo/error-demo.cc | 86 ++++++++++-----------------------
 src/libutil/error.hh         | 92 ------------------------------------
 2 files changed, 24 insertions(+), 154 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index fdb574b25a9..ef8b5630868 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -11,15 +11,10 @@ int main()
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
-    // There are currently four error types:
+    // There are currently four constructor functions:
     //
     // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
     //
-    // Each error type is created with a specific sequence of builder functions.
-    // Unlike with a constructor, each parameter is clearly named.
-    // If the sequence of function calls isn't followed, then there's a type
-    // error. This should make for a consistent look in the code when errors are
-    // created.
 
     // ProgramError takes name, description, and an optional hint.
     printErrorInfo(
@@ -33,68 +28,35 @@ int main()
     printErrorInfo(
         ErrorInfo::ProgramWarning("name",
                                   "warning description",
-                                  std::optional(hintfmt("there was a %1%", "warning"))));
-
-    // printErrorInfo( ProgramWarning()
-    //                 .name("warning name")
-    //                 .description("warning description")
-    //                 // the templated value, 'warning', is automatically colored yellow.
-    //                 .hint(hintfmt("there was a %1%", "warning"))
-    //               );
-
-    /*
-      // some invalid errors:
-
-      // type error: no hint function.
-      ProgramError()
-        .name("name")
-        .description("error description");
-
-      // type error:  description before name.
-      ProgramError()
-        .description("error description")
-        .name("name")
-        .nohint();
-
-      // type error: hint function with regular boost format, not special
-      hintfmt. ProgramError() .description("error description") .name("name")
-        .hint(format("there was a %1%") % "warning");
-    */
+                                  std::optional(
+                                      hintfmt("there was a %1%", "warning"))));
 
     // NixLangWarning adds nix file, line number, column range, and the lines of
     // code where a warning occurred.
-
     SymbolTable testTable;
     auto problem_symbol = testTable.create("problem");
 
-    printErrorInfo(ErrorInfo::NixLangWarning(
-                       "warning name",
-                       "warning description",
-                       Pos(problem_symbol, 40, 13),
-                       std::nullopt,
-                       "this is the problem line of code",
-                       std::nullopt,
-                       hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
-
-    // // NixLangError is just the same as NixLangWarning, except for the Error
-    // // flag.
-    // printErrorInfo(NixLangError()
-    //                .name("error name")
-    //                .description("error description")
-    //                .pos(Pos(problem_symbol, 40, 13))
-    //                .linesOfCode(std::optional("previous line of code"),
-    //                             "this is the problem line of code",
-    //                             std::optional("next line of code"))
-    //                .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
-    printErrorInfo(ErrorInfo::NixLangError(
-                       "error name",
-                       "error description",
-                       Pos(problem_symbol, 40, 13),
-                       std::optional("previous line of code"),
-                       "this is the problem line of code",
-                       std::optional("next line of code"),
-                       hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
-
+    printErrorInfo(
+        ErrorInfo::NixLangWarning(
+            "warning name",
+            "warning description",
+            Pos(problem_symbol, 40, 13),
+            std::nullopt,
+            "this is the problem line of code",
+            std::nullopt,
+            hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+
+    // NixLangError is just the same as NixLangWarning, except for the Error
+    // flag.
+    printErrorInfo(
+        ErrorInfo::NixLangError(
+            "error name",
+            "error description",
+            Pos(problem_symbol, 40, 13),
+            std::optional("previous line of code"),
+            "this is the problem line of code",
+            std::optional("next line of code"),
+            hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
 
     return 0;
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index a73b6639eac..a8a1afbca09 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -191,98 +191,6 @@ private:
     }
 };
 
-/*
-template <class T>
-class AddPos : private T
-{
-public:
-    template <class P>
-    T& pos(const P &aPos)
-    {
-        GetEI().ensureNixCode().nixFile = aPos.file;
-        GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line;
-        GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 };
-        return *this;
-    }
-protected:
-    ErrorInfo& GetEI()
-    {
-        return T::GetEI();
-    }
-};
-
-template <class T>
-class AddLOC : private T
-{
-public:
-    T& linesOfCode(std::optional<string> prevloc, string loc, std::optional<string> nextloc)
-    {
-        GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc;
-        GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc;
-        GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc;
-        return *this;
-    }
-protected:
-    ErrorInfo& GetEI()
-    {
-        return T::GetEI();
-    }
-};
-*/
-
-/*
-// the template layer for adding a hint.
-template <class T>
-class AddHint : private T
-{
-public:
-    T& hint(const hintformat &hf)
-    {
-        GetEI().hint = std::optional(hf.str());
-        return *this;
-    }
-    T& nohint()
-    {
-        GetEI().hint = std::nullopt;
-        return *this;
-    }
-protected:
-    ErrorInfo& GetEI()
-    {
-        return T::GetEI();
-    }
-};
-*/
-
-// --------------------------------------------------------
-// error types
-
-/*typedef AddName<
-        AddDescription<
-        AddHint<
-        EIError>>> ProgramError;
-
-typedef AddName<
-        AddDescription<
-        AddHint<
-        EIWarning>>> ProgramWarning;
-
-typedef AddName<
-        AddDescription<
-        AddPos<
-        AddLOC<
-        AddHint<
-        EIError>>>>> NixLangError;
-
-typedef AddName<
-        AddDescription<
-        AddPos<
-        AddLOC<
-        AddHint<
-        EIWarning>>>>> NixLangWarning;
-
-
-*/
 // --------------------------------------------------------
 // error printing
 

From 462421d34588c227eb736b07f7b40a38aa0972a6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 30 Mar 2020 16:04:18 +0200
Subject: [PATCH 1938/2196] Backport libfetchers from the flakes branch

This provides a pluggable mechanism for defining new fetchers. It adds
a builtin function 'fetchTree' that generalizes existing fetchers like
'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a
set of attributes, e.g.

  fetchTree {
    type = "git";
    url = "https://example.org/repo.git";
    ref = "some-branch";
    rev = "abcdef...";
  }

The existing fetchers are just wrappers around this. Note that the
input attributes to fetchTree are the same as flake input
specifications and flake lock file entries.

All fetchers share a common cache stored in
~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching
mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}).

This also adds support for Git worktrees (c169ea59049f861aaba429f48b828d0820b74d1d).
---
 Makefile                              |   1 +
 src/libexpr/common-eval-args.cc       |   8 +-
 src/libexpr/local.mk                  |   4 +-
 src/libexpr/parser.y                  |   6 +-
 src/libexpr/primops.cc                |  67 -----
 src/libexpr/primops.hh                |   1 +
 src/libexpr/primops/fetchGit.cc       | 223 ++------------
 src/libexpr/primops/fetchMercurial.cc | 207 +++----------
 src/libexpr/primops/fetchTree.cc      | 165 +++++++++++
 src/libfetchers/attrs.cc              |  92 ++++++
 src/libfetchers/attrs.hh              |  37 +++
 src/libfetchers/cache.cc              | 121 ++++++++
 src/libfetchers/cache.hh              |  34 +++
 src/libfetchers/fetchers.cc           |  73 +++++
 src/libfetchers/fetchers.hh           | 103 +++++++
 src/libfetchers/git.cc                | 401 ++++++++++++++++++++++++++
 src/libfetchers/github.cc             | 195 +++++++++++++
 src/libfetchers/local.mk              |  11 +
 src/libfetchers/mercurial.cc          | 303 +++++++++++++++++++
 src/libfetchers/tarball.cc            | 277 ++++++++++++++++++
 src/libfetchers/tree-info.cc          |  14 +
 src/libfetchers/tree-info.hh          |  29 ++
 src/libstore/derivations.cc           |   2 +-
 src/libstore/download.cc              | 137 ---------
 src/libstore/download.hh              |  31 +-
 src/libstore/globals.hh               |   9 +
 src/libstore/store-api.cc             |  25 +-
 src/libstore/store-api.hh             |   1 +
 src/libutil/types.hh                  |   8 +
 src/libutil/url.cc                    | 137 +++++++++
 src/libutil/url.hh                    |  62 ++++
 src/nix-channel/nix-channel.cc        |  17 +-
 src/nix/local.mk                      |   4 +-
 tests/fetchGit.sh                     |  32 +-
 tests/init.sh                         |   2 +-
 tests/tarball.sh                      |   7 +
 36 files changed, 2199 insertions(+), 647 deletions(-)
 create mode 100644 src/libexpr/primops/fetchTree.cc
 create mode 100644 src/libfetchers/attrs.cc
 create mode 100644 src/libfetchers/attrs.hh
 create mode 100644 src/libfetchers/cache.cc
 create mode 100644 src/libfetchers/cache.hh
 create mode 100644 src/libfetchers/fetchers.cc
 create mode 100644 src/libfetchers/fetchers.hh
 create mode 100644 src/libfetchers/git.cc
 create mode 100644 src/libfetchers/github.cc
 create mode 100644 src/libfetchers/local.mk
 create mode 100644 src/libfetchers/mercurial.cc
 create mode 100644 src/libfetchers/tarball.cc
 create mode 100644 src/libfetchers/tree-info.cc
 create mode 100644 src/libfetchers/tree-info.hh
 create mode 100644 src/libutil/url.cc
 create mode 100644 src/libutil/url.hh

diff --git a/Makefile b/Makefile
index 469070533b5..e3057c36ca2 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ makefiles = \
   nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
+  src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/nix/local.mk \
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 13950ab8d16..82bfeac36a1 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -3,6 +3,8 @@
 #include "download.hh"
 #include "util.hh"
 #include "eval.hh"
+#include "fetchers.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -46,9 +48,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 Path lookupFileArg(EvalState & state, string s)
 {
     if (isUri(s)) {
-        CachedDownloadRequest request(s);
-        request.unpack = true;
-        return getDownloader()->downloadCached(state.store, request).path;
+        return state.store->toRealPath(
+            fetchers::downloadTarball(
+                state.store, resolveUri(s), "source", false).storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index a4ccab376a4..917e8a1c7df 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -6,9 +6,9 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
 
-libexpr_LIBS = libutil libstore libnixrust
+libexpr_LIBS = libutil libstore libfetchers libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 9c769e80376..a30fb44b59e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -545,6 +545,7 @@ formal
 
 #include "eval.hh"
 #include "download.hh"
+#include "fetchers.hh"
 #include "store-api.hh"
 
 
@@ -687,9 +688,8 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     if (isUri(elem.second)) {
         try {
-            CachedDownloadRequest request(elem.second);
-            request.unpack = true;
-            res = { true, getDownloader()->downloadCached(store, request).path };
+            res = { true, store->toRealPath(fetchers::downloadTarball(
+                        store, resolveUri(elem.second), "source", false).storePath) };
         } catch (DownloadError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8de23495187..629f3da15b3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1,6 +1,5 @@
 #include "archive.hh"
 #include "derivations.hh"
-#include "download.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "globals.hh"
@@ -2045,68 +2044,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 }
 
 
-/*************************************************************
- * Networking
- *************************************************************/
-
-
-void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
-    const string & who, bool unpack, const std::string & defaultName)
-{
-    CachedDownloadRequest request("");
-    request.unpack = unpack;
-    request.name = defaultName;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-
-        state.forceAttrs(*args[0], pos);
-
-        for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
-            if (n == "url")
-                request.uri = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256")
-                request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-            else if (n == "name")
-                request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else
-                throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
-        }
-
-        if (request.uri.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
-
-    } else
-        request.uri = state.forceStringNoCtx(*args[0], pos);
-
-    state.checkURI(request.uri);
-
-    if (evalSettings.pureEval && !request.expectedHash)
-        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
-
-    auto res = getDownloader()->downloadCached(state.store, request);
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(res.path);
-
-    mkString(v, res.storePath, PathSet({res.storePath}));
-}
-
-
-static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchurl", false, "");
-}
-
-
-static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    fetch(state, pos, args, v, "fetchTarball", true, "source");
-}
-
-
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -2289,10 +2226,6 @@ void EvalState::createBaseEnv()
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
     addPrimOp("placeholder", 1, prim_placeholder);
 
-    // Networking
-    addPrimOp("__fetchurl", 1, prim_fetchurl);
-    addPrimOp("fetchTarball", 1, prim_fetchTarball);
-
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index c790b30f6d0..05d0792efc7 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -20,6 +20,7 @@ struct RegisterPrimOp
    them. */
 /* Load a ValueInitializer from a DSO and return whatever it initializes */
 void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
 /* Execute a program and parse its output */
 void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);
 
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 4aee1073eeb..812de9d91de 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -1,202 +1,17 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
 #include "hash.hh"
-#include "tarfile.hh"
-
-#include <sys/time.h>
-
-#include <regex>
-
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
+#include "fetchers.hh"
+#include "url.hh"
 
 namespace nix {
 
-struct GitInfo
-{
-    Path storePath;
-    std::string rev;
-    std::string shortRev;
-    uint64_t revCount = 0;
-};
-
-std::regex revRegex("^[0-9a-fA-F]{40}$");
-
-GitInfo exportGit(ref<Store> store, const std::string & uri,
-    std::optional<std::string> ref, std::string rev,
-    const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
-
-        bool clean = true;
-
-        try {
-            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError & e) {
-            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-            clean = false;
-        }
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
-            gitInfo.rev = "0000000000000000000000000000000000000000";
-            gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return gitInfo;
-        }
-
-        // clean working tree, but no ref or rev specified.  Use 'HEAD'.
-        rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
-        ref = "HEAD"s;
-    }
-
-    if (!ref) ref = "HEAD"s;
-
-    if (rev != "" && !std::regex_match(rev, revRegex))
-        throw Error("invalid Git revision '%s'", rev);
-
-    deletePath(getCacheDir() + "/nix/git");
-
-    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
-
-    if (!pathExists(cacheDir)) {
-        createDirs(dirOf(cacheDir));
-        runProgram("git", true, { "init", "--bare", cacheDir });
-    }
-
-    Path localRefFile;
-    if (ref->compare(0, 5, "refs/") == 0)
-        localRefFile = cacheDir + "/" + *ref;
-    else
-        localRefFile = cacheDir + "/refs/heads/" + *ref;
-
-    bool doFetch;
-    time_t now = time(0);
-    /* If a rev was specified, we need to fetch if it's not in the
-       repo. */
-    if (rev != "") {
-        try {
-            runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
-            doFetch = false;
-        } catch (ExecError & e) {
-            if (WIFEXITED(e.status)) {
-                doFetch = true;
-            } else {
-                throw;
-            }
-        }
-    } else {
-        /* If the local ref is older than ‘tarball-ttl’ seconds, do a
-           git fetch to update the local ref to the remote ref. */
-        struct stat st;
-        doFetch = stat(localRefFile.c_str(), &st) != 0 ||
-            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
-    }
-    if (doFetch)
-    {
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
-
-        struct timeval times[2];
-        times[0].tv_sec = now;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = now;
-        times[1].tv_usec = 0;
-
-        utimes(localRefFile.c_str(), times);
-    }
-
-    // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
-    gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
-    gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
-
-    printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
-    Path storeLink = cacheDir + "/" + storeLinkName + ".link";
-    PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == gitInfo.rev);
-
-        gitInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
-            gitInfo.revCount = json["revCount"];
-            return gitInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    unpackTarfile(*source, tmpDir);
-
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
-
-    nlohmann::json json;
-    json["storePath"] = gitInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["rev"] = gitInfo.rev;
-    json["revCount"] = gitInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return gitInfo;
-}
-
 static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
     std::optional<std::string> ref;
-    std::string rev;
+    std::optional<Hash> rev;
     std::string name = "source";
     PathSet context;
 
@@ -213,7 +28,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "ref")
                 ref = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
@@ -230,17 +45,35 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
+
+    auto parsedUrl = parseURL(
+        url.find("://") != std::string::npos
+        ? "git+" + url
+        : "git+file://" + url);
+    if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
+    if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
+    // FIXME: use name
+    auto input = fetchers::inputFromURL(parsedUrl);
+
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
+    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
+    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
+        tree.info.revCount.value_or(0));
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index db274fa4f70..f18351646e4 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -1,174 +1,18 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "download.hh"
 #include "store-api.hh"
-#include "pathlocks.hh"
-
-#include <sys/time.h>
+#include "fetchers.hh"
+#include "url.hh"
 
 #include <regex>
 
-#include <nlohmann/json.hpp>
-
-using namespace std::string_literals;
-
 namespace nix {
 
-struct HgInfo
-{
-    Path storePath;
-    std::string branch;
-    std::string rev;
-    uint64_t revCount = 0;
-};
-
-std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
-
-HgInfo exportMercurial(ref<Store> store, const std::string & uri,
-    std::string rev, const std::string & name)
-{
-    if (evalSettings.pureEval && rev == "")
-        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
-
-    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
-
-        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
-
-        if (!clean) {
-
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
-            printTalkative("copying unclean Mercurial working tree '%s'", uri);
-
-            HgInfo hgInfo;
-            hgInfo.rev = "0000000000000000000000000000000000000000";
-            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
-
-            auto files = tokenizeString<std::set<std::string>>(
-                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
-
-            PathFilter filter = [&](const Path & p) -> bool {
-                assert(hasPrefix(p, uri));
-                std::string file(p, uri.size() + 1);
-
-                auto st = lstat(p);
-
-                if (S_ISDIR(st.st_mode)) {
-                    auto prefix = file + "/";
-                    auto i = files.lower_bound(prefix);
-                    return i != files.end() && hasPrefix(*i, prefix);
-                }
-
-                return files.count(file);
-            };
-
-            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
-
-            return hgInfo;
-        }
-    }
-
-    if (rev == "") rev = "default";
-
-    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
-
-    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
-
-    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-       do so now. */
-    time_t now = time(0);
-    struct stat st;
-    if (stat(stampFile.c_str(), &st) != 0 ||
-        (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
-    {
-        /* Except that if this is a commit hash that we already have,
-           we don't have to pull again. */
-        if (!(std::regex_match(rev, commitHashRegex)
-                && pathExists(cacheDir)
-                && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
-                    .killStderr(true)).second == "1"))
-        {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
-
-            if (pathExists(cacheDir)) {
-                try {
-                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                }
-                catch (ExecError & e) {
-                    string transJournal = cacheDir + "/.hg/store/journal";
-                    /* hg throws "abandoned transaction" error only if this file exists */
-                    if (pathExists(transJournal)) {
-                        runProgram("hg", true, { "recover", "-R", cacheDir });
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
-                    } else {
-                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                    }
-                }
-            } else {
-                createDirs(dirOf(cacheDir));
-                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
-            }
-        }
-
-        writeFile(stampFile, "");
-    }
-
-    auto tokens = tokenizeString<std::vector<std::string>>(
-        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
-    assert(tokens.size() == 3);
-
-    HgInfo hgInfo;
-    hgInfo.rev = tokens[0];
-    hgInfo.revCount = std::stoull(tokens[1]);
-    hgInfo.branch = tokens[2];
-
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
-    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
-
-    try {
-        auto json = nlohmann::json::parse(readFile(storeLink));
-
-        assert(json["name"] == name && json["rev"] == hgInfo.rev);
-
-        hgInfo.storePath = json["storePath"];
-
-        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
-            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
-            return hgInfo;
-        }
-
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT) throw;
-    }
-
-    Path tmpDir = createTempDir();
-    AutoDelete delTmpDir(tmpDir, true);
-
-    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
-
-    deletePath(tmpDir + "/.hg_archival.txt");
-
-    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
-
-    nlohmann::json json;
-    json["storePath"] = hgInfo.storePath;
-    json["uri"] = uri;
-    json["name"] = name;
-    json["branch"] = hgInfo.branch;
-    json["rev"] = hgInfo.rev;
-    json["revCount"] = hgInfo.revCount;
-
-    writeFile(storeLink, json.dump());
-
-    return hgInfo;
-}
-
 static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     std::string url;
-    std::string rev;
+    std::optional<Hash> rev;
+    std::optional<std::string> ref;
     std::string name = "source";
     PathSet context;
 
@@ -182,8 +26,15 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             string n(attr.name);
             if (n == "url")
                 url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "rev")
-                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "rev") {
+                // Ugly: unlike fetchGit, here the "rev" attribute can
+                // be both a revision or a branch/tag name.
+                auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
+                if (std::regex_match(value, revRegex))
+                    rev = Hash(value, htSHA1);
+                else
+                    ref = value;
+            }
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
@@ -200,18 +51,36 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto hgInfo = exportMercurial(state.store, url, rev, name);
+    if (evalSettings.pureEval && !rev)
+        throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
+
+    auto parsedUrl = parseURL(
+        url.find("://") != std::string::npos
+        ? "hg+" + url
+        : "hg+file://" + url);
+    if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
+    if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
+    // FIXME: use name
+    auto input = fetchers::inputFromURL(parsedUrl);
+
+    auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);
-    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
-    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
+    auto storePath = state.store->printStorePath(tree.storePath);
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    if (input2->getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    // Backward compatibility: set 'rev' to
+    // 0000000000000000000000000000000000000000 for a dirty tree.
+    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
+    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath));
+        state.allowedPaths->insert(tree.actualPath);
 }
 
 static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
new file mode 100644
index 00000000000..9586f71ed5a
--- /dev/null
+++ b/src/libexpr/primops/fetchTree.cc
@@ -0,0 +1,165 @@
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "download.hh"
+
+#include <ctime>
+#include <iomanip>
+
+namespace nix {
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    std::shared_ptr<const fetchers::Input> input,
+    Value & v)
+{
+    state.mkAttrs(v, 8);
+
+    auto storePath = state.store->printStorePath(tree.storePath);
+
+    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+
+    assert(tree.info.narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        tree.info.narHash.to_string(SRI));
+
+    if (input->getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    }
+
+    if (tree.info.revCount)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+
+    if (tree.info.lastModified)
+        mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
+            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+
+    v.attrs->sort();
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    std::shared_ptr<const fetchers::Input> input;
+    PathSet context;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+        state.forceAttrs(*args[0], pos);
+
+        fetchers::Attrs attrs;
+
+        for (auto & attr : *args[0]->attrs) {
+            state.forceValue(*attr.value);
+            if (attr.value->type == tString)
+                attrs.emplace(attr.name, attr.value->string.s);
+            else if (attr.value->type == tBool)
+                attrs.emplace(attr.name, attr.value->boolean);
+            else
+                throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
+                    attr.name, showType(*attr.value));
+        }
+
+        if (!attrs.count("type"))
+            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
+
+        input = fetchers::inputFromAttrs(attrs);
+    } else
+        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+
+    if (evalSettings.pureEval && !input->isImmutable())
+        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
+
+    // FIXME: use fetchOrSubstituteTree
+    auto [tree, input2] = input->fetchTree(state.store);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    emitTreeAttrs(state, tree, input2, v);
+}
+
+static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+
+static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack, std::string name)
+{
+    std::optional<std::string> url;
+    std::optional<Hash> expectedHash;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string n(attr.name);
+            if (n == "url")
+                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "sha256")
+                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
+                name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError("unsupported argument '%s' to '%s', at %s",
+                    attr.name, who, attr.pos);
+        }
+
+        if (!url)
+            throw EvalError("'url' argument required, at %s", pos);
+
+    } else
+        url = state.forceStringNoCtx(*args[0], pos);
+
+    url = resolveUri(*url);
+
+    state.checkURI(*url);
+
+    if (name == "")
+        name = baseNameOf(*url);
+
+    if (evalSettings.pureEval && !expectedHash)
+        throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
+
+    auto storePath =
+        unpack
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+
+    auto path = state.store->toRealPath(storePath);
+
+    if (expectedHash) {
+        auto hash = unpack
+            ? state.store->queryPathInfo(storePath)->narHash
+            : hashFile(htSHA256, path);
+        if (hash != *expectedHash)
+            throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
+                *url, expectedHash->to_string(), hash.to_string());
+    }
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(path);
+
+    mkString(v, path, PathSet({path}));
+}
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false, "");
+}
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true, "source");
+}
+
+static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
+static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+
+}
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
new file mode 100644
index 00000000000..40c02de4225
--- /dev/null
+++ b/src/libfetchers/attrs.cc
@@ -0,0 +1,92 @@
+#include "attrs.hh"
+#include "fetchers.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+Attrs jsonToAttrs(const nlohmann::json & json)
+{
+    Attrs attrs;
+
+    for (auto & i : json.items()) {
+        if (i.value().is_number())
+            attrs.emplace(i.key(), i.value().get<int64_t>());
+        else if (i.value().is_string())
+            attrs.emplace(i.key(), i.value().get<std::string>());
+        else if (i.value().is_boolean())
+            attrs.emplace(i.key(), i.value().get<bool>());
+        else
+            throw Error("unsupported input attribute type in lock file");
+    }
+
+    return attrs;
+}
+
+nlohmann::json attrsToJson(const Attrs & attrs)
+{
+    nlohmann::json json;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            json[attr.first] = v->t;
+        } else abort();
+    }
+    return json;
+}
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<std::string>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
+}
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetStrAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not an integer", name);
+}
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetIntAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a Boolean", name);
+}
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetBoolAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+}
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
new file mode 100644
index 00000000000..2c9e772d25b
--- /dev/null
+++ b/src/libfetchers/attrs.hh
@@ -0,0 +1,37 @@
+#pragma once
+
+#include "types.hh"
+
+#include <variant>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::fetchers {
+
+/* Wrap bools to prevent string literals (i.e. 'char *') from being
+   cast to a bool in Attr. */
+template<typename T>
+struct Explicit {
+    T t;
+};
+
+typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::map<std::string, Attr> Attrs;
+
+Attrs jsonToAttrs(const nlohmann::json & json);
+
+nlohmann::json attrsToJson(const Attrs & attrs);
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
+
+bool getBoolAttr(const Attrs & attrs, const std::string & name);
+
+}
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
new file mode 100644
index 00000000000..e1c7f3dee32
--- /dev/null
+++ b/src/libfetchers/cache.cc
@@ -0,0 +1,121 @@
+#include "cache.hh"
+#include "sqlite.hh"
+#include "sync.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+static const char * schema = R"sql(
+
+create table if not exists Cache (
+    input     text not null,
+    info      text not null,
+    path      text not null,
+    immutable integer not null,
+    timestamp integer not null,
+    primary key (input)
+);
+)sql";
+
+struct CacheImpl : Cache
+{
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt add, lookup;
+    };
+
+    Sync<State> _state;
+
+    CacheImpl()
+    {
+        auto state(_state.lock());
+
+        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
+        createDirs(dirOf(dbPath));
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->add.create(state->db,
+            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+
+        state->lookup.create(state->db,
+            "select info, path, immutable, timestamp from Cache where input = ?");
+    }
+
+    void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) override
+    {
+        _state.lock()->add.use()
+            (attrsToJson(inAttrs).dump())
+            (attrsToJson(infoAttrs).dump())
+            (store->printStorePath(storePath))
+            (immutable)
+            (time(0)).exec();
+    }
+
+    std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(store, inAttrs)) {
+            if (!res->expired)
+                return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
+            debug("ignoring expired cache entry '%s'",
+                attrsToJson(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJson = attrsToJson(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJson));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJson);
+            return {};
+        }
+
+        auto infoJson = stmt.getStr(0);
+        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto immutable = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        store->addTempRoot(storePath);
+        if (!store->isValidPath(storePath)) {
+            // FIXME: we could try to substitute 'storePath'.
+            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+            return {};
+        }
+
+        debug("using cache entry '%s' -> '%s', '%s'",
+            inAttrsJson, infoJson, store->printStorePath(storePath));
+
+        return Result {
+            .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
+            .storePath = std::move(storePath)
+        };
+    }
+};
+
+ref<Cache> getCache()
+{
+    static auto cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(cache);
+}
+
+}
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
new file mode 100644
index 00000000000..d76ab12331d
--- /dev/null
+++ b/src/libfetchers/cache.hh
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+struct Cache
+{
+    virtual void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) = 0;
+
+    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+
+    struct Result
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+        StorePath storePath;
+    };
+
+    virtual std::optional<Result> lookupExpired(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+};
+
+ref<Cache> getCache();
+
+}
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
new file mode 100644
index 00000000000..25d3da431da
--- /dev/null
+++ b/src/libfetchers/fetchers.cc
@@ -0,0 +1,73 @@
+#include "fetchers.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+
+void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+{
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    inputSchemes->push_back(std::move(inputScheme));
+}
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+{
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromURL(url);
+        if (res) return res;
+    }
+    throw Error("input '%s' is unsupported", url.url);
+}
+
+std::unique_ptr<Input> inputFromURL(const std::string & url)
+{
+    return inputFromURL(parseURL(url));
+}
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
+{
+    for (auto & inputScheme : *inputSchemes) {
+        auto res = inputScheme->inputFromAttrs(attrs);
+        if (res) {
+            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
+                // FIXME: require SRI hash.
+                res->narHash = Hash(*narHash);
+            return res;
+        }
+    }
+    throw Error("input '%s' is unsupported", attrsToJson(attrs));
+}
+
+Attrs Input::toAttrs() const
+{
+    auto attrs = toAttrsInternal();
+    if (narHash)
+        attrs.emplace("narHash", narHash->to_string(SRI));
+    attrs.emplace("type", type());
+    return attrs;
+}
+
+std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+{
+    auto [tree, input] = fetchTreeInternal(store);
+
+    if (tree.actualPath == "")
+        tree.actualPath = store->toRealPath(tree.storePath);
+
+    if (!tree.info.narHash)
+        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+    if (input->narHash)
+        assert(input->narHash == tree.info.narHash);
+
+    if (narHash && narHash != input->narHash)
+        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+            to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
+
+    return {std::move(tree), input};
+}
+
+}
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
new file mode 100644
index 00000000000..59a58ae6781
--- /dev/null
+++ b/src/libfetchers/fetchers.hh
@@ -0,0 +1,103 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "path.hh"
+#include "tree-info.hh"
+#include "attrs.hh"
+#include "url.hh"
+
+#include <memory>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Input;
+
+struct Tree
+{
+    Path actualPath;
+    StorePath storePath;
+    TreeInfo info;
+};
+
+struct Input : std::enable_shared_from_this<Input>
+{
+    std::optional<Hash> narHash; // FIXME: implement
+
+    virtual std::string type() const = 0;
+
+    virtual ~Input() { }
+
+    virtual bool operator ==(const Input & other) const { return false; }
+
+    /* Check whether this is a "direct" input, that is, not
+       one that goes through a registry. */
+    virtual bool isDirect() const { return true; }
+
+    /* Check whether this is an "immutable" input, that is,
+       one that contains a commit hash or content hash. */
+    virtual bool isImmutable() const { return (bool) narHash; }
+
+    virtual bool contains(const Input & other) const { return false; }
+
+    virtual std::optional<std::string> getRef() const { return {}; }
+
+    virtual std::optional<Hash> getRev() const { return {}; }
+
+    virtual ParsedURL toURL() const = 0;
+
+    std::string to_string() const
+    {
+        return toURL().to_string();
+    }
+
+    Attrs toAttrs() const;
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+
+private:
+
+    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+
+    virtual Attrs toAttrsInternal() const = 0;
+};
+
+struct InputScheme
+{
+    virtual ~InputScheme() { }
+
+    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+
+    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
+};
+
+std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+
+std::unique_ptr<Input> inputFromURL(const std::string & url);
+
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
+
+void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+
+struct DownloadFileResult
+{
+    StorePath storePath;
+    std::string etag;
+    std::string effectiveUrl;
+};
+
+DownloadFileResult downloadFile(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+Tree downloadTarball(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable);
+
+}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
new file mode 100644
index 00000000000..cee0713f42e
--- /dev/null
+++ b/src/libfetchers/git.cc
@@ -0,0 +1,401 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+static std::string readHead(const Path & path)
+{
+    return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
+}
+
+struct GitInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+    bool shallow = false;
+
+    GitInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "git"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        if (shallow) url2.query.insert_or_assign("shallow", "1");
+        return url2;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        if (shallow)
+            attrs.emplace("shallow", true);
+        return attrs;
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        // Don't clone file:// URIs (but otherwise treat them the
+        // same as remote URIs, i.e. don't use the working tree or
+        // HEAD).
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        bool isLocal = url.scheme == "file" && !forceHttp;
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<GitInput>(*this);
+
+        assert(!rev || rev->type == htSHA1);
+
+        auto cacheType = shallow ? "git-shallow" : "git";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", cacheType},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree {
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
+                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input->ref && !input->rev && isLocal) {
+            bool clean = false;
+
+            /* Check whether this repo has any commits. There are
+               probably better ways to do this. */
+            auto gitDir = actualUrl + "/.git";
+            auto commonGitDir = chomp(runProgram(
+                "git",
+                true,
+                { "-C", actualUrl, "rev-parse", "--git-common-dir" }
+            ));
+            if (commonGitDir != ".git")
+                    gitDir = commonGitDir;
+
+            bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
+
+            try {
+                if (haveCommits) {
+                    runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+                    clean = true;
+                }
+            } catch (ExecError & e) {
+                if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
+            }
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Git tree '%s' is dirty", actualUrl);
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+
+                auto tree = Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        // FIXME: maybe we should use the timestamp of the last
+                        // modified dirty file?
+                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
+                    }
+                };
+
+                return {std::move(tree), input};
+            }
+        }
+
+        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+
+        Attrs mutableAttrs({
+            {"type", cacheType},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        Path repoDir;
+
+        if (isLocal) {
+
+            if (!input->rev)
+                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+
+            repoDir = actualUrl;
+
+        } else {
+
+            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+                if (!rev || rev == rev2) {
+                    input->rev = rev2;
+                    return makeResult(res->first, std::move(res->second));
+                }
+            }
+
+            Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
+            repoDir = cacheDir;
+
+            if (!pathExists(cacheDir)) {
+                createDirs(dirOf(cacheDir));
+                runProgram("git", true, { "init", "--bare", repoDir });
+            }
+
+            Path localRefFile =
+                input->ref->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input->ref
+                : cacheDir + "/refs/heads/" + *input->ref;
+
+            bool doFetch;
+            time_t now = time(0);
+
+            /* If a rev was specified, we need to fetch if it's not in the
+               repo. */
+            if (input->rev) {
+                try {
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    doFetch = false;
+                } catch (ExecError & e) {
+                    if (WIFEXITED(e.status)) {
+                        doFetch = true;
+                    } else {
+                        throw;
+                    }
+                }
+            } else {
+                /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+                   git fetch to update the local ref to the remote ref. */
+                struct stat st;
+                doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                    (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+            }
+
+            if (doFetch) {
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+
+                // FIXME: git stderr messes up our progress indicator, so
+                // we're using --quiet for now. Should process its stderr.
+                try {
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                } catch (Error & e) {
+                    if (!pathExists(localRefFile)) throw;
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                }
+
+                struct timeval times[2];
+                times[0].tv_sec = now;
+                times[0].tv_usec = 0;
+                times[1].tv_sec = now;
+                times[1].tv_usec = 0;
+
+                utimes(localRefFile.c_str(), times);
+            }
+
+            if (!input->rev)
+                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+        }
+
+        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
+
+        if (isShallow && !shallow)
+            throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
+
+        // FIXME: check whether rev is an ancestor of ref.
+
+        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+
+        /* Now that we know the ref, check again whether we have it in
+           the store. */
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        // FIXME: should pipe this, or find some better way to extract a
+        // revision.
+        auto source = sinkToSource([&](Sink & sink) {
+            RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+            gitOptions.standardOut = &sink;
+            runProgram2(gitOptions);
+        });
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        unpackTarfile(*source, tmpDir);
+
+        auto storePath = store->addToStore(name, tmpDir);
+
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"lastModified", lastModified},
+        });
+
+        if (!shallow)
+            infoAttrs.insert_or_assign("revCount",
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct GitInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return nullptr;
+
+        auto url2(url);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "git");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow")
+                throw Error("unsupported Git input attribute '%s'", name);
+
+        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Git branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+
+        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+
+}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
new file mode 100644
index 00000000000..3fc95ff51ec
--- /dev/null
+++ b/src/libfetchers/github.cc
@@ -0,0 +1,195 @@
+#include "download.hh"
+#include "cache.hh"
+#include "fetchers.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct GitHubInput : Input
+{
+    std::string owner;
+    std::string repo;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    std::string type() const override { return "github"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const GitHubInput *>(&other);
+        return
+            other2
+            && owner == other2->owner
+            && repo == other2->repo
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        auto path = owner + "/" + repo;
+        assert(!(ref && rev));
+        if (ref) path += "/" + *ref;
+        if (rev) path += "/" + rev->to_string(Base16, false);
+        return ParsedURL {
+            .scheme = "github",
+            .path = path,
+        };
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("owner", owner);
+        attrs.emplace("repo", repo);
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto rev = this->rev;
+        auto ref = this->ref.value_or("master");
+
+        if (!rev) {
+            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
+                owner, repo, ref);
+            auto json = nlohmann::json::parse(
+                readFile(
+                    store->toRealPath(
+                        downloadFile(store, url, "source", false).storePath)));
+            rev = Hash(json["sha"], htSHA1);
+            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        }
+
+        auto input = std::make_shared<GitHubInput>(*this);
+        input->ref = {};
+        input->rev = *rev;
+
+        Attrs immutableAttrs({
+            {"type", "git-tarball"},
+            {"rev", rev->gitRev()},
+        });
+
+        if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(res->second),
+                    .storePath = std::move(res->second),
+                    .info = TreeInfo {
+                        .lastModified = getIntAttr(res->first, "lastModified"),
+                    },
+                },
+                input
+            };
+        }
+
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+
+        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
+            owner, repo, rev->to_string(Base16, false));
+
+        std::string accessToken = settings.githubAccessToken.get();
+        if (accessToken != "")
+            url += "?access_token=" + accessToken;
+
+        auto tree = downloadTarball(store, url, "source", true);
+
+        getCache()->add(
+            store,
+            immutableAttrs,
+            {
+                {"rev", rev->gitRev()},
+                {"lastModified", *tree.info.lastModified}
+            },
+            tree.storePath,
+            true);
+
+        return {std::move(tree), input};
+    }
+};
+
+struct GitHubInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "github") return nullptr;
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+        auto input = std::make_unique<GitHubInput>();
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                input->rev = Hash(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                input->ref = path[2];
+            else
+                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (input->rev)
+                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
+                input->rev = Hash(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
+                if (input->ref)
+                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
+                input->ref = value;
+            }
+        }
+
+        if (input->ref && input->rev)
+            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+
+        input->owner = path[0];
+        input->repo = path[1];
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "github") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
+                throw Error("unsupported GitHub input attribute '%s'", name);
+
+        auto input = std::make_unique<GitHubInput>();
+        input->owner = getStrAttr(attrs, "owner");
+        input->repo = getStrAttr(attrs, "repo");
+        input->ref = maybeGetStrAttr(attrs, "ref");
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+
+}
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
new file mode 100644
index 00000000000..d7143d8a60c
--- /dev/null
+++ b/src/libfetchers/local.mk
@@ -0,0 +1,11 @@
+libraries += libfetchers
+
+libfetchers_NAME = libnixfetchers
+
+libfetchers_DIR := $(d)
+
+libfetchers_SOURCES := $(wildcard $(d)/*.cc)
+
+libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+
+libfetchers_LIBS = libutil libstore libnixrust
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
new file mode 100644
index 00000000000..790f7c7534f
--- /dev/null
+++ b/src/libfetchers/mercurial.cc
@@ -0,0 +1,303 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "globals.hh"
+#include "tarfile.hh"
+#include "store-api.hh"
+
+#include <sys/time.h>
+
+using namespace std::string_literals;
+
+namespace nix::fetchers {
+
+struct MercurialInput : Input
+{
+    ParsedURL url;
+    std::optional<std::string> ref;
+    std::optional<Hash> rev;
+
+    MercurialInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "hg"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const MercurialInput *>(&other);
+        return
+            other2
+            && url == other2->url
+            && rev == other2->rev
+            && ref == other2->ref;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) rev;
+    }
+
+    std::optional<std::string> getRef() const override { return ref; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        ParsedURL url2(url);
+        url2.scheme = "hg+" + url2.scheme;
+        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
+        if (ref) url2.query.insert_or_assign("ref", *ref);
+        return url;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (ref)
+            attrs.emplace("ref", *ref);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        return attrs;
+    }
+
+    std::pair<bool, std::string> getActualUrl() const
+    {
+        bool isLocal = url.scheme == "file";
+        return {isLocal, isLocal ? url.path : url.base};
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto name = "source";
+
+        auto input = std::make_shared<MercurialInput>(*this);
+
+        auto [isLocal, actualUrl_] = getActualUrl();
+        auto actualUrl = actualUrl_; // work around clang bug
+
+        // FIXME: return lastModified.
+
+        // FIXME: don't clone local repositories.
+
+        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+
+            bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+
+            if (!clean) {
+
+                /* This is an unclean working tree. So copy all tracked
+                   files. */
+
+                if (!settings.allowDirty)
+                    throw Error("Mercurial tree '%s' is unclean", actualUrl);
+
+                if (settings.warnDirty)
+                    warn("Mercurial tree '%s' is unclean", actualUrl);
+
+                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+
+                auto files = tokenizeString<std::set<std::string>>(
+                    runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+
+                PathFilter filter = [&](const Path & p) -> bool {
+                    assert(hasPrefix(p, actualUrl));
+                    std::string file(p, actualUrl.size() + 1);
+
+                    auto st = lstat(p);
+
+                    if (S_ISDIR(st.st_mode)) {
+                        auto prefix = file + "/";
+                        auto i = files.lower_bound(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
+                    }
+
+                    return files.count(file);
+                };
+
+                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+
+                return {Tree {
+                    .actualPath = store->printStorePath(storePath),
+                    .storePath = std::move(storePath),
+                }, input};
+            }
+        }
+
+        if (!input->ref) input->ref = "default";
+
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", "hg"},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            assert(input->rev);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = getIntAttr(infoAttrs, "revCount"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (input->rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
+
+        assert(input->rev || input->ref);
+        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+
+        Attrs mutableAttrs({
+            {"type", "hg"},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
+
+        if (auto res = getCache()->lookup(store, mutableAttrs)) {
+            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+            if (!rev || rev == rev2) {
+                input->rev = rev2;
+                return makeResult(res->first, std::move(res->second));
+            }
+        }
+
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
+
+        /* If this is a commit hash that we already have, we don't
+           have to pull again. */
+        if (!(input->rev
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    .killStderr(true)).second == "1"))
+        {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
+
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+            }
+        }
+
+        auto tokens = tokenizeString<std::vector<std::string>>(
+            runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+        assert(tokens.size() == 3);
+
+        input->rev = Hash(tokens[0], htSHA1);
+        auto revCount = std::stoull(tokens[1]);
+        input->ref = tokens[2];
+
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+
+        deletePath(tmpDir + "/.hg_archival.txt");
+
+        auto storePath = store->addToStore(name, tmpDir);
+
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"revCount", (int64_t) revCount},
+        });
+
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
+
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
+    }
+};
+
+struct MercurialInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "hg+http" &&
+            url.scheme != "hg+https" &&
+            url.scheme != "hg+ssh" &&
+            url.scheme != "hg+file") return nullptr;
+
+        auto url2(url);
+        url2.scheme = std::string(url2.scheme, 3);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "hg");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
+        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+            input->ref = *ref;
+        }
+        if (auto rev = maybeGetStrAttr(attrs, "rev"))
+            input->rev = Hash(*rev, htSHA1);
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+
+}
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
new file mode 100644
index 00000000000..b0a83001ec6
--- /dev/null
+++ b/src/libfetchers/tarball.cc
@@ -0,0 +1,277 @@
+#include "fetchers.hh"
+#include "cache.hh"
+#include "download.hh"
+#include "globals.hh"
+#include "store-api.hh"
+#include "archive.hh"
+#include "tarfile.hh"
+
+namespace nix::fetchers {
+
+DownloadFileResult downloadFile(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable)
+{
+    // FIXME: check store
+
+    Attrs inAttrs({
+        {"type", "file"},
+        {"url", url},
+        {"name", name},
+    });
+
+    auto cached = getCache()->lookupExpired(store, inAttrs);
+
+    auto useCached = [&]() -> DownloadFileResult
+    {
+        return {
+            .storePath = std::move(cached->storePath),
+            .etag = getStrAttr(cached->infoAttrs, "etag"),
+            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
+        };
+    };
+
+    if (cached && !cached->expired)
+        return useCached();
+
+    DownloadRequest request(url);
+    if (cached)
+        request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
+    DownloadResult res;
+    try {
+        res = getDownloader()->download(request);
+    } catch (DownloadError & e) {
+        if (cached) {
+            warn("%s; using cached version", e.msg());
+            return useCached();
+        } else
+            throw;
+    }
+
+    // FIXME: write to temporary file.
+
+    Attrs infoAttrs({
+        {"etag", res.etag},
+        {"url", res.effectiveUri},
+    });
+
+    std::optional<StorePath> storePath;
+
+    if (res.cached) {
+        assert(cached);
+        assert(request.expectedETag == res.etag);
+        storePath = std::move(cached->storePath);
+    } else {
+        StringSink sink;
+        dumpString(*res.data, sink);
+        auto hash = hashString(htSHA256, *res.data);
+        ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
+        info.narHash = hashString(htSHA256, *sink.s);
+        info.narSize = sink.s->size();
+        info.ca = makeFixedOutputCA(false, hash);
+        store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
+        storePath = std::move(info.path);
+    }
+
+    getCache()->add(
+        store,
+        inAttrs,
+        infoAttrs,
+        *storePath,
+        immutable);
+
+    if (url != res.effectiveUri)
+        getCache()->add(
+            store,
+            {
+                {"type", "file"},
+                {"url", res.effectiveUri},
+                {"name", name},
+            },
+            infoAttrs,
+            *storePath,
+            immutable);
+
+    return {
+        .storePath = std::move(*storePath),
+        .etag = res.etag,
+        .effectiveUrl = res.effectiveUri,
+    };
+}
+
+Tree downloadTarball(
+    ref<Store> store,
+    const std::string & url,
+    const std::string & name,
+    bool immutable)
+{
+    Attrs inAttrs({
+        {"type", "tarball"},
+        {"url", url},
+        {"name", name},
+    });
+
+    auto cached = getCache()->lookupExpired(store, inAttrs);
+
+    if (cached && !cached->expired)
+        return Tree {
+            .actualPath = store->toRealPath(cached->storePath),
+            .storePath = std::move(cached->storePath),
+            .info = TreeInfo {
+                .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
+            },
+        };
+
+    auto res = downloadFile(store, url, name, immutable);
+
+    std::optional<StorePath> unpackedStorePath;
+    time_t lastModified;
+
+    if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
+        unpackedStorePath = std::move(cached->storePath);
+        lastModified = getIntAttr(cached->infoAttrs, "lastModified");
+    } else {
+        Path tmpDir = createTempDir();
+        AutoDelete autoDelete(tmpDir, true);
+        unpackTarfile(store->toRealPath(res.storePath), tmpDir);
+        auto members = readDirectory(tmpDir);
+        if (members.size() != 1)
+            throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
+        auto topDir = tmpDir + "/" + members.begin()->name;
+        lastModified = lstat(topDir).st_mtime;
+        unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
+    }
+
+    Attrs infoAttrs({
+        {"lastModified", lastModified},
+        {"etag", res.etag},
+    });
+
+    getCache()->add(
+        store,
+        inAttrs,
+        infoAttrs,
+        *unpackedStorePath,
+        immutable);
+
+    return Tree {
+        .actualPath = store->toRealPath(*unpackedStorePath),
+        .storePath = std::move(*unpackedStorePath),
+        .info = TreeInfo {
+            .lastModified = lastModified,
+        },
+    };
+}
+
+struct TarballInput : Input
+{
+    ParsedURL url;
+    std::optional<Hash> hash;
+
+    TarballInput(const ParsedURL & url) : url(url)
+    { }
+
+    std::string type() const override { return "tarball"; }
+
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const TarballInput *>(&other);
+        return
+            other2
+            && to_string() == other2->to_string()
+            && hash == other2->hash;
+    }
+
+    bool isImmutable() const override
+    {
+        return hash || narHash;
+    }
+
+    ParsedURL toURL() const override
+    {
+        auto url2(url);
+        // NAR hashes are preferred over file hashes since tar/zip files
+        // don't have a canonical representation.
+        if (narHash)
+            url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
+        else if (hash)
+            url2.query.insert_or_assign("hash", hash->to_string(SRI));
+        return url2;
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("url", url.to_string());
+        if (narHash)
+            attrs.emplace("narHash", narHash->to_string(SRI));
+        else if (hash)
+            attrs.emplace("hash", hash->to_string(SRI));
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto tree = downloadTarball(store, url.to_string(), "source", false);
+
+        auto input = std::make_shared<TarballInput>(*this);
+        input->narHash = store->queryPathInfo(tree.storePath)->narHash;
+
+        return {std::move(tree), input};
+    }
+};
+
+struct TarballInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+
+        if (!hasSuffix(url.path, ".zip")
+            && !hasSuffix(url.path, ".tar")
+            && !hasSuffix(url.path, ".tar.gz")
+            && !hasSuffix(url.path, ".tar.xz")
+            && !hasSuffix(url.path, ".tar.bz2"))
+            return nullptr;
+
+        auto input = std::make_unique<TarballInput>(url);
+
+        auto hash = input->url.query.find("hash");
+        if (hash != input->url.query.end()) {
+            // FIXME: require SRI hash.
+            input->hash = Hash(hash->second);
+            input->url.query.erase(hash);
+        }
+
+        auto narHash = input->url.query.find("narHash");
+        if (narHash != input->url.query.end()) {
+            // FIXME: require SRI hash.
+            input->narHash = Hash(narHash->second);
+            input->url.query.erase(narHash);
+        }
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "hash" && name != "narHash")
+                throw Error("unsupported tarball input attribute '%s'", name);
+
+        auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
+        if (auto hash = maybeGetStrAttr(attrs, "hash"))
+            // FIXME: require SRI hash.
+            input->hash = Hash(*hash);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+
+}
diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc
new file mode 100644
index 00000000000..5788e94a197
--- /dev/null
+++ b/src/libfetchers/tree-info.cc
@@ -0,0 +1,14 @@
+#include "tree-info.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+StorePath TreeInfo::computeStorePath(Store & store) const
+{
+    assert(narHash);
+    return store.makeFixedOutputPath(true, narHash, "source");
+}
+
+}
diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh
new file mode 100644
index 00000000000..2c734728135
--- /dev/null
+++ b/src/libfetchers/tree-info.hh
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct TreeInfo
+{
+    Hash narHash;
+    std::optional<uint64_t> revCount;
+    std::optional<time_t> lastModified;
+
+    bool operator ==(const TreeInfo & other) const
+    {
+        return
+            narHash == other.narHash
+            && revCount == other.revCount
+            && lastModified == other.lastModified;
+    }
+
+    StorePath computeStorePath(Store & store) const;
+};
+
+}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 973ddc86abf..0067032af71 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
         if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
             h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
-                readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
+                readDerivation(store, store.toRealPath(i.first)), false)).first;
         }
         inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 5967d04258b..215046b7249 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -35,10 +35,6 @@ DownloadSettings downloadSettings;
 
 static GlobalConfig::Register r1(&downloadSettings);
 
-CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
-    : uri(uri), ttl(settings.tarballTtl)
-{ }
-
 std::string resolveUri(const std::string & uri)
 {
     if (uri.compare(0, 8, "channel:") == 0)
@@ -802,139 +798,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-CachedDownloadResult Downloader::downloadCached(
-    ref<Store> store, const CachedDownloadRequest & request)
-{
-    auto url = resolveUri(request.uri);
-
-    auto name = request.name;
-    if (name == "") {
-        auto p = url.rfind('/');
-        if (p != string::npos) name = string(url, p + 1);
-    }
-
-    std::optional<StorePath> expectedStorePath;
-    if (request.expectedHash) {
-        expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
-        if (store->isValidPath(*expectedStorePath)) {
-            CachedDownloadResult result;
-            result.storePath = store->printStorePath(*expectedStorePath);
-            result.path = store->toRealPath(result.storePath);
-            return result;
-        }
-    }
-
-    Path cacheDir = getCacheDir() + "/nix/tarballs";
-    createDirs(cacheDir);
-
-    string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
-
-    Path dataFile = cacheDir + "/" + urlHash + ".info";
-    Path fileLink = cacheDir + "/" + urlHash + "-file";
-
-    PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
-
-    std::optional<StorePath> storePath;
-
-    string expectedETag;
-
-    bool skip = false;
-
-    CachedDownloadResult result;
-
-    if (pathExists(fileLink) && pathExists(dataFile)) {
-        storePath = store->parseStorePath(readLink(fileLink));
-        // FIXME
-        store->addTempRoot(*storePath);
-        if (store->isValidPath(*storePath)) {
-            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
-            if (ss.size() >= 3 && ss[0] == url) {
-                time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
-                    skip = true;
-                    result.effectiveUri = request.uri;
-                    result.etag = ss[1];
-                } else if (!ss[1].empty()) {
-                    debug(format("verifying previous ETag '%1%'") % ss[1]);
-                    expectedETag = ss[1];
-                }
-            }
-        } else
-            storePath.reset();
-    }
-
-    if (!skip) {
-
-        try {
-            DownloadRequest request2(url);
-            request2.expectedETag = expectedETag;
-            auto res = download(request2);
-            result.effectiveUri = res.effectiveUri;
-            result.etag = res.etag;
-
-            if (!res.cached) {
-                StringSink sink;
-                dumpString(*res.data, sink);
-                Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
-                ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
-                info.narHash = hashString(htSHA256, *sink.s);
-                info.narSize = sink.s->size();
-                info.ca = makeFixedOutputCA(false, hash);
-                store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
-                storePath = info.path.clone();
-            }
-
-            assert(storePath);
-            replaceSymlink(store->printStorePath(*storePath), fileLink);
-
-            writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
-        } catch (DownloadError & e) {
-            if (!storePath) throw;
-            warn("warning: %s; using cached result", e.msg());
-            result.etag = expectedETag;
-        }
-    }
-
-    if (request.unpack) {
-        Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
-        PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
-        std::optional<StorePath> unpackedStorePath;
-        if (pathExists(unpackedLink)) {
-            unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
-            // FIXME
-            store->addTempRoot(*unpackedStorePath);
-            if (!store->isValidPath(*unpackedStorePath))
-                unpackedStorePath.reset();
-        }
-        if (!unpackedStorePath) {
-            printInfo("unpacking '%s'...", url);
-            Path tmpDir = createTempDir();
-            AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
-            auto members = readDirectory(tmpDir);
-            if (members.size() != 1)
-                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
-            auto topDir = tmpDir + "/" + members.begin()->name;
-            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
-        }
-        replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
-        storePath = std::move(*unpackedStorePath);
-    }
-
-    if (expectedStorePath && *storePath != *expectedStorePath) {
-        unsigned int statusCode = 102;
-        Hash gotHash = request.unpack
-            ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
-            : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
-        throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, request.expectedHash.to_string(), gotHash.to_string());
-    }
-
-    result.storePath = store->printStorePath(*storePath);
-    result.path = store->toRealPath(result.storePath);
-    return result;
-}
-
 
 bool isUri(const string & s)
 {
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 5a131c70450..28c8a916212 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -65,28 +65,6 @@ struct DownloadResult
     uint64_t bodySize = 0;
 };
 
-struct CachedDownloadRequest
-{
-    std::string uri;
-    bool unpack = false;
-    std::string name;
-    Hash expectedHash;
-    unsigned int ttl;
-
-    CachedDownloadRequest(const std::string & uri);
-    CachedDownloadRequest() = delete;
-};
-
-struct CachedDownloadResult
-{
-    // Note: 'storePath' may be different from 'path' when using a
-    // chroot store.
-    Path storePath;
-    Path path;
-    std::optional<std::string> etag;
-    std::string effectiveUri;
-};
-
 class Store;
 
 struct Downloader
@@ -108,12 +86,6 @@ struct Downloader
        invoked on the thread of the caller. */
     void download(DownloadRequest && request, Sink & sink);
 
-    /* Check if the specified file is already in ~/.cache/nix/tarballs
-       and is more recent than ‘tarball-ttl’ seconds. Otherwise,
-       use the recorded ETag to verify if the server has a more
-       recent version, and if so, download it to the Nix store. */
-    CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
-
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
 
@@ -135,4 +107,7 @@ public:
 
 bool isUri(const string & s);
 
+/* Resolve deprecated 'channel:<foo>' URLs. */
+std::string resolveUri(const std::string & uri);
+
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3aa3653f31d..40f350f0b75 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -351,12 +351,21 @@ public:
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 
+    Setting<std::string> githubAccessToken{this, "", "github-access-token",
+        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
     bool isExperimentalFeatureEnabled(const std::string & name);
 
     void requireExperimentalFeature(const std::string & name);
+
+    Setting<bool> allowDirty{this, true, "allow-dirty",
+        "Whether to allow dirty Git/Mercurial trees."};
+
+    Setting<bool> warnDirty{this, true, "warn-dirty",
+        "Whether to warn about dirty Git/Mercurial trees."};
 };
 
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b9e894a9a3f..e5282bb3057 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,6 +6,7 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
+#include "url.hh"
 
 #include <future>
 
@@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw NotInStore("path '%1%' is not in the Nix store", path);
     return path;
 }
 
@@ -866,27 +867,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
-        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
-            auto e = s.find('=');
-            if (e != std::string::npos) {
-                auto value = s.substr(e + 1);
-                std::string decoded;
-                for (size_t i = 0; i < value.size(); ) {
-                    if (value[i] == '%') {
-                        if (i + 2 >= value.size())
-                            throw Error("invalid URI parameter '%s'", value);
-                        try {
-                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
-                            i += 3;
-                        } catch (...) {
-                            throw Error("invalid URI parameter '%s'", value);
-                        }
-                    } else
-                        decoded += value[i++];
-                }
-                params[s.substr(0, e)] = decoded;
-            }
-        }
+        params = decodeQuery(uri.substr(q + 1));
         uri = uri_.substr(0, q);
     }
     return {uri, params};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0fa59be6a61..81014763d02 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
+MakeError(NotInStore, Error);
 
 
 struct BasicDerivation;
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 20b96a85ced..a1ce7b37218 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -157,4 +157,12 @@ typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
 
+/* Helper class to run code at startup. */
+template<typename T>
+struct OnStartup
+{
+    OnStartup(T && t) { t(); }
+};
+
+
 }
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
new file mode 100644
index 00000000000..5d5328e5de6
--- /dev/null
+++ b/src/libutil/url.cc
@@ -0,0 +1,137 @@
+#include "url.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::regex refRegex(refRegexS, std::regex::ECMAScript);
+std::regex revRegex(revRegexS, std::regex::ECMAScript);
+std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
+
+ParsedURL parseURL(const std::string & url)
+{
+    static std::regex uriRegex(
+        "((" + schemeRegex + "):"
+        + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    std::smatch match;
+
+    if (std::regex_match(url, match, uriRegex)) {
+        auto & base = match[1];
+        std::string scheme = match[2];
+        auto authority = match[3].matched
+            ? std::optional<std::string>(match[3]) : std::nullopt;
+        std::string path = match[4].matched ? match[4] : match[5];
+        auto & query = match[6];
+        auto & fragment = match[7];
+
+        auto isFile = scheme.find("file") != std::string::npos;
+
+        if (authority && *authority != "" && isFile)
+            throw Error("file:// URL '%s' has unexpected authority '%s'",
+                url, *authority);
+
+        if (isFile && path.empty())
+            path = "/";
+
+        return ParsedURL{
+            .url = url,
+            .base = base,
+            .scheme = scheme,
+            .authority = authority,
+            .path = path,
+            .query = decodeQuery(query),
+            .fragment = percentDecode(std::string(fragment))
+        };
+    }
+
+    else
+        throw BadURL("'%s' is not a valid URL", url);
+}
+
+std::string percentDecode(std::string_view in)
+{
+    std::string decoded;
+    for (size_t i = 0; i < in.size(); ) {
+        if (in[i] == '%') {
+            if (i + 2 >= in.size())
+                throw BadURL("invalid URI parameter '%s'", in);
+            try {
+                decoded += std::stoul(std::string(in, i + 1, 2), 0, 16);
+                i += 3;
+            } catch (...) {
+                throw BadURL("invalid URI parameter '%s'", in);
+            }
+        } else
+            decoded += in[i++];
+    }
+    return decoded;
+}
+
+std::map<std::string, std::string> decodeQuery(const std::string & query)
+{
+    std::map<std::string, std::string> result;
+
+    for (auto s : tokenizeString<Strings>(query, "&")) {
+        auto e = s.find('=');
+        if (e != std::string::npos)
+            result.emplace(
+                s.substr(0, e),
+                percentDecode(std::string_view(s).substr(e + 1)));
+    }
+
+    return result;
+}
+
+std::string percentEncode(std::string_view s)
+{
+    std::string res;
+    for (auto & c : s)
+        if ((c >= 'a' && c <= 'z')
+            || (c >= 'A' && c <= 'Z')
+            || (c >= '0' && c <= '9')
+            || strchr("-._~!$&'()*+,;=:@", c))
+            res += c;
+        else
+            res += fmt("%%%02x", (unsigned int) c);
+    return res;
+}
+
+std::string encodeQuery(const std::map<std::string, std::string> & ss)
+{
+    std::string res;
+    bool first = true;
+    for (auto & [name, value] : ss) {
+        if (!first) res += '&';
+        first = false;
+        res += percentEncode(name);
+        res += '=';
+        res += percentEncode(value);
+    }
+    return res;
+}
+
+std::string ParsedURL::to_string() const
+{
+    return
+        scheme
+        + ":"
+        + (authority ? "//" + *authority : "")
+        + path
+        + (query.empty() ? "" : "?" + encodeQuery(query))
+        + (fragment.empty() ? "" : "#" + percentEncode(fragment));
+}
+
+bool ParsedURL::operator ==(const ParsedURL & other) const
+{
+    return
+        scheme == other.scheme
+        && authority == other.authority
+        && path == other.path
+        && query == other.query
+        && fragment == other.fragment;
+}
+
+}
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
new file mode 100644
index 00000000000..1503023a2fc
--- /dev/null
+++ b/src/libutil/url.hh
@@ -0,0 +1,62 @@
+#pragma once
+
+#include "types.hh"
+
+#include <regex>
+
+namespace nix {
+
+struct ParsedURL
+{
+    std::string url;
+    std::string base; // URL without query/fragment
+    std::string scheme;
+    std::optional<std::string> authority;
+    std::string path;
+    std::map<std::string, std::string> query;
+    std::string fragment;
+
+    std::string to_string() const;
+
+    bool operator ==(const ParsedURL & other) const;
+};
+
+MakeError(BadURL, Error);
+
+std::string percentDecode(std::string_view in);
+
+std::map<std::string, std::string> decodeQuery(const std::string & query);
+
+ParsedURL parseURL(const std::string & url);
+
+// URI stuff.
+const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
+const static std::string schemeRegex = "(?:[a-z+]+)";
+const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
+const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
+const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
+const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
+const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
+const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
+const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
+const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
+const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
+const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
+const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
+
+// A Git ref (i.e. branch or tag name).
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+extern std::regex refRegex;
+
+// A Git revision (a SHA-1 commit hash).
+const static std::string revRegexS = "[0-9a-fA-F]{40}";
+extern std::regex revRegex;
+
+// A ref or revision, or a ref followed by a revision.
+const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
+
+const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
+extern std::regex flakeIdRegex;
+
+}
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index a2639579dd9..2a9defb4e7b 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -3,6 +3,7 @@
 #include "download.hh"
 #include "store-api.hh"
 #include "../nix/legacy.hh"
+#include "fetchers.hh"
 
 #include <fcntl.h>
 #include <regex>
@@ -86,12 +87,9 @@ static void update(const StringSet & channelNames)
         // We want to download the url to a file to see if it's a tarball while also checking if we
         // got redirected in the process, so that we can grab the various parts of a nix channel
         // definition from a consistent location if the redirect changes mid-download.
-        CachedDownloadRequest request(url);
-        request.ttl = 0;
-        auto dl = getDownloader();
-        auto result = dl->downloadCached(store, request);
-        auto filename = result.path;
-        url = chomp(result.effectiveUri);
+        auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false);
+        auto filename = store->toRealPath(result.storePath);
+        url = result.effectiveUrl;
 
         // If the URL contains a version number, append it to the name
         // attribute (so that "nix-env -q" on the channels profile
@@ -114,11 +112,10 @@ static void update(const StringSet & channelNames)
         if (!unpacked) {
             // Download the channel tarball.
             try {
-                filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path;
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
             } catch (DownloadError & e) {
-                filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path;
+                filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
             }
-            chomp(filename);
         }
 
         // Regardless of where it came from, add the expression representing this channel to accumulated expression
@@ -185,6 +182,8 @@ static int _main(int argc, char ** argv)
             } else if (*arg == "--rollback") {
                 cmd = cRollback;
             } else {
+                if (hasPrefix(*arg, "-"))
+                    throw UsageError("unsupported argument '%s'", *arg);
                 args.push_back(std::move(*arg));
             }
             return true;
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 50a18efd79d..033675e893c 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,9 +15,9 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
 
-nix_LIBS = libexpr libmain libstore libutil libnixrust
+nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index ed8fa14d60e..d9c9874f568 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -11,7 +11,7 @@ repo=$TEST_ROOT/git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow
 
 git init $repo
 git -C $repo config user.email "foobar@example.com"
@@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD)
 
 echo world > $repo/hello
 git -C $repo commit -m 'Bla2' -a
+git -C $repo worktree add $TEST_ROOT/worktree
+echo hello >> $TEST_ROOT/worktree/hello
 rev2=$(git -C $repo rev-parse HEAD)
 
+# Fetch a worktree
+unset _NIX_FORCE_HTTP
+path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+export _NIX_FORCE_HTTP=1
+[[ $(tail -n 1 $path0/hello) = "hello" ]]
+
 # Fetch the default branch.
 path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
@@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
 [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
 
-# But with TTL 0, it should fail.
-(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv)
-
 # Fetching with a explicit hash should succeed.
 path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
@@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar
 git -C $repo add dir1/foo
 git -C $repo rm hello
 
+unset _NIX_FORCE_HTTP
 path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
@@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 git -C $repo checkout $rev2 -b dev
 echo dev > $repo/hello
 
-# File URI uses 'master' unless specified otherwise
+# File URI uses dirty tree unless specified otherwise
 path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
-[[ $path = $path2 ]]
+[ $(cat $path2/hello) = dev ]
 
 # Using local path with branch other than 'master' should work when clean or dirty
 path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
@@ -131,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP
 
 
 # Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/gitv2
+rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH
+# Try again, but without 'git' on PATH. This should fail.
 NIX=$(command -v nix)
 # This should fail
 (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
@@ -141,3 +147,13 @@ NIX=$(command -v nix)
 # Try again, with 'git' available.  This should work.
 path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
+
+# Fetching a shallow repo shouldn't work by default, because we can't
+# return a revCount.
+git clone --depth 1 file://$repo $TEST_ROOT/shallow
+(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")
+
+# But you can request a shallow clone, which won't return a revCount.
+path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
+[[ $path3 = $path6 ]]
+[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
diff --git a/tests/init.sh b/tests/init.sh
index 6a119aad0a4..c62c4856a93 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command
+experimental-features = nix-command flakes
 include nix.conf.extra
 EOF
 
diff --git a/tests/tarball.sh b/tests/tarball.sh
index 8adb8d72f83..b3ec16d4080 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -10,6 +10,8 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
+hash=$(nix hash-path $tarroot)
+
 test_tarball() {
     local ext="$1"
     local compressor="$2"
@@ -25,6 +27,11 @@ test_tarball() {
 
     nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
 
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
     nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
     nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
     (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)

From 670feb000a9fac76f0996711f061ec466a53dc97 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Apr 2020 14:56:20 +0200
Subject: [PATCH 1939/2196] Add 'path' fetcher

This fetchers copies a plain directory (i.e. not a Git/Mercurial
repository) to the store (or does nothing if the path is already a
store path).

One use case is to pin the 'nixpkgs' flake used to build the current
NixOS system, and prevent it from being garbage-collected, via a
system registry entry like this:

  {
      "from": {
          "id": "nixpkgs",
          "type": "indirect"
      },
      "to": {
          "type": "path",
          "path": "/nix/store/rralhl3wj4rdwzjn16g7d93mibvlr521-source",
          "lastModified": 1585388205,
          "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a"
      },
      "exact": true
  }

Note the fake "lastModified" and "rev" attributes that ensure that the
flake gives the same evaluation results as the corresponding
Git/GitHub inputs.

(cherry picked from commit 12f9379123eba828f2ae06f7978a37b7045c2b23)
---
 src/libfetchers/attrs.cc |  15 +++++
 src/libfetchers/attrs.hh |   2 +
 src/libfetchers/path.cc  | 130 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 147 insertions(+)
 create mode 100644 src/libfetchers/path.cc

diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index 40c02de4225..feb0a608567 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -89,4 +89,19 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name)
     return *s;
 }
 
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
+{
+    std::map<std::string, std::string> query;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            query.insert_or_assign(attr.first, fmt("%d", *v));
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            query.insert_or_assign(attr.first, *v);
+        } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
+            query.insert_or_assign(attr.first, v->t ? "1" : "0");
+        } else abort();
+    }
+    return query;
+}
+
 }
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 2c9e772d25b..d6e0ae00053 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -34,4 +34,6 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
 
 bool getBoolAttr(const Attrs & attrs, const std::string & name);
 
+std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
+
 }
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
new file mode 100644
index 00000000000..9801d9b8655
--- /dev/null
+++ b/src/libfetchers/path.cc
@@ -0,0 +1,130 @@
+#include "fetchers.hh"
+#include "store-api.hh"
+
+namespace nix::fetchers {
+
+struct PathInput : Input
+{
+    Path path;
+
+    /* Allow the user to pass in "fake" tree info attributes. This is
+       useful for making a pinned tree work the same as the repository
+       from which is exported
+       (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+    std::optional<Hash> rev;
+    std::optional<uint64_t> revCount;
+    std::optional<time_t> lastModified;
+
+    std::string type() const override { return "path"; }
+
+    std::optional<Hash> getRev() const override { return rev; }
+
+    ParsedURL toURL() const override
+    {
+        auto query = attrsToQuery(toAttrsInternal());
+        query.erase("path");
+        return ParsedURL {
+            .scheme = "path",
+            .path = path,
+            .query = query,
+        };
+    }
+
+    Attrs toAttrsInternal() const override
+    {
+        Attrs attrs;
+        attrs.emplace("path", path);
+        if (rev)
+            attrs.emplace("rev", rev->gitRev());
+        if (revCount)
+            attrs.emplace("revCount", *revCount);
+        if (lastModified)
+            attrs.emplace("lastModified", *lastModified);
+        return attrs;
+    }
+
+    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    {
+        auto input = std::make_shared<PathInput>(*this);
+
+        auto storePath = store->maybeParseStorePath(path);
+
+        if (storePath)
+            store->addTempRoot(*storePath);
+
+        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
+            // FIXME: try to substitute storePath.
+            storePath = store->addToStore("name", path);
+
+        return
+            {
+                Tree {
+                    .actualPath = store->toRealPath(*storePath),
+                    .storePath = std::move(*storePath),
+                    .info = TreeInfo {
+                        .revCount = revCount,
+                        .lastModified = lastModified
+                    }
+                },
+                input
+            };
+    }
+
+};
+
+struct PathInputScheme : InputScheme
+{
+    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "path") return nullptr;
+
+        auto input = std::make_unique<PathInput>();
+        input->path = url.path;
+
+        for (auto & [name, value] : url.query)
+            if (name == "rev")
+                input->rev = Hash(value, htSHA1);
+            else if (name == "revCount") {
+                uint64_t revCount;
+                if (!string2Int(value, revCount))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input->revCount = revCount;
+            }
+            else if (name == "lastModified") {
+                time_t lastModified;
+                if (!string2Int(value, lastModified))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input->lastModified = lastModified;
+            }
+            else
+                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
+
+        return input;
+    }
+
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+        auto input = std::make_unique<PathInput>();
+        input->path = getStrAttr(attrs, "path");
+
+        for (auto & [name, value] : attrs)
+            if (name == "rev")
+                input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
+            else if (name == "revCount")
+                input->revCount = getIntAttr(attrs, "revCount");
+            else if (name == "lastModified")
+                input->lastModified = getIntAttr(attrs, "lastModified");
+            else if (name == "type" || name == "path")
+                ;
+            else
+                throw Error("unsupported path input attribute '%s'", name);
+
+        return input;
+    }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
+
+}

From f58a9b0e62879e28cc7cac59b489a52c924bbc36 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Apr 2020 19:04:33 +0200
Subject: [PATCH 1940/2196] Respect the narHash attribute in more input types

(cherry picked from commit a6ff66b658b61aef80d936f0183447fe4cb46000)
---
 src/libfetchers/fetchers.cc  | 4 +++-
 src/libfetchers/git.cc       | 2 +-
 src/libfetchers/github.cc    | 2 +-
 src/libfetchers/mercurial.cc | 2 +-
 src/libfetchers/tarball.cc   | 6 ++----
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 25d3da431da..94ac30e3800 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -29,8 +29,10 @@ std::unique_ptr<Input> inputFromURL(const std::string & url)
 
 std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
 {
+    auto attrs2(attrs);
+    attrs2.erase("narHash");
     for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromAttrs(attrs);
+        auto res = inputScheme->inputFromAttrs(attrs2);
         if (res) {
             if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
                 // FIXME: require SRI hash.
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index cee0713f42e..3f94d9bdd9c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -39,7 +39,7 @@ struct GitInput : Input
 
     bool isImmutable() const override
     {
-        return (bool) rev;
+        return (bool) rev || narHash;
     }
 
     std::optional<std::string> getRef() const override { return ref; }
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 3fc95ff51ec..ef27eaa76c6 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -33,7 +33,7 @@ struct GitHubInput : Input
 
     bool isImmutable() const override
     {
-        return (bool) rev;
+        return (bool) rev || narHash;
     }
 
     std::optional<std::string> getRef() const override { return ref; }
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 790f7c7534f..1d65715714e 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -33,7 +33,7 @@ struct MercurialInput : Input
 
     bool isImmutable() const override
     {
-        return (bool) rev;
+        return (bool) rev || narHash;
     }
 
     std::optional<std::string> getRef() const override { return ref; }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b0a83001ec6..4c4e5828e74 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -205,9 +205,7 @@ struct TarballInput : Input
     {
         Attrs attrs;
         attrs.emplace("url", url.to_string());
-        if (narHash)
-            attrs.emplace("narHash", narHash->to_string(SRI));
-        else if (hash)
+        if (hash)
             attrs.emplace("hash", hash->to_string(SRI));
         return attrs;
     }
@@ -260,7 +258,7 @@ struct TarballInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "hash" && name != "narHash")
+            if (name != "type" && name != "url" && name != "hash")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));

From a6dfa3cb85b59a0979de7fe02c9d67939c4ac217 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 2 Apr 2020 19:04:27 +0200
Subject: [PATCH 1941/2196] PathInput: Add some methods

(cherry picked from commit 78ad5b3d91507427fa563f3474dc52da608ad224)
---
 src/libfetchers/path.cc | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 9801d9b8655..037404726a0 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -19,6 +19,22 @@ struct PathInput : Input
 
     std::optional<Hash> getRev() const override { return rev; }
 
+    bool operator ==(const Input & other) const override
+    {
+        auto other2 = dynamic_cast<const PathInput *>(&other);
+        return
+            other2
+            && path == other2->path
+            && rev == other2->rev
+            && revCount == other2->revCount
+            && lastModified == other2->lastModified;
+    }
+
+    bool isImmutable() const override
+    {
+        return (bool) narHash;
+    }
+
     ParsedURL toURL() const override
     {
         auto query = attrsToQuery(toAttrsInternal());

From 26aeeb7653fa051ddec913bdc2c578b9066bc08f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 6 Apr 2020 14:28:37 +0200
Subject: [PATCH 1942/2196] Add FIXME

(cherry picked from commit 2f494531b7811b45f6b76787f225495a14d28a7f)
---
 src/libfetchers/path.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 037404726a0..7c7e20f4e82 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -63,6 +63,8 @@ struct PathInput : Input
     {
         auto input = std::make_shared<PathInput>(*this);
 
+        // FIXME: check whether access to 'path' is allowed.
+
         auto storePath = store->maybeParseStorePath(path);
 
         if (storePath)

From cd39709003eee4b85a31799b8bc2de59721930ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Apr 2020 09:27:17 +0200
Subject: [PATCH 1943/2196] Cleanup

---
 src/libstore/download.cc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 215046b7249..af69699a866 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -1,14 +1,10 @@
 #include "download.hh"
 #include "util.hh"
 #include "globals.hh"
-#include "hash.hh"
 #include "store-api.hh"
-#include "archive.hh"
 #include "s3.hh"
 #include "compression.hh"
-#include "pathlocks.hh"
 #include "finally.hh"
-#include "tarfile.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>

From 9d04b5da17898fd564308423bc7d10fc929efe18 Mon Sep 17 00:00:00 2001
From: mlatus <wqseleven@gmail.com>
Date: Tue, 7 Apr 2020 20:29:40 +0800
Subject: [PATCH 1944/2196] `nix run` using $SHELL as default command

---
 src/nix/run.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/run.cc b/src/nix/run.cc
index 8e30264c023..5334531fd99 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -59,14 +59,14 @@ struct RunCommon : virtual Command
 
 struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
 {
-    std::vector<std::string> command = { "bash" };
+    std::vector<std::string> command = { getEnv("SHELL").value_or("bash") };
 
     CmdRun()
     {
         mkFlag()
             .longName("command")
             .shortName('c')
-            .description("command and arguments to be executed; defaults to 'bash'")
+            .description("command and arguments to be executed; defaults to '$SHELL'")
             .labels({"command", "args"})
             .arity(ArityAny)
             .handler([&](std::vector<std::string> ss) {

From 20c0984a46b6fbc1be0b073f14f5e20b7c82a26a Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 7 Apr 2020 10:14:15 -0600
Subject: [PATCH 1945/2196] remove columnrange; switch to fmt in error.cc

---
 src/libutil/error.cc | 68 ++++++++++++++++++++++----------------------
 src/libutil/error.hh | 11 ++-----
 2 files changed, 36 insertions(+), 43 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 8ab4a2dea9c..106ea127e51 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -10,10 +10,10 @@ std::optional<string> ErrorInfo::programName = std::nullopt;
 
 string showErrLine(ErrLine &errLine)
 {
-    if (errLine.columnRange.has_value()) {
-        return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
+    if (errLine.column > 0) {
+        return fmt("(%1%:%2%)", errLine.lineNumber, errLine.column);
     } else {
-        return (format("(%1%)") % errLine.lineNumber).str();
+        return fmt("(%1%)", errLine.lineNumber);
     };
 }
 
@@ -23,45 +23,45 @@ void printCodeLines(string &prefix, NixCode &nixCode)
     if (nixCode.errLine.has_value()) {
         // previous line of code.
         if (nixCode.errLine->prevLineOfCode.has_value()) {
-            std::cout << format("%1% %|2$5d|| %3%")
-                      %    prefix
-                      % (nixCode.errLine->lineNumber - 1)
-                      % *nixCode.errLine->prevLineOfCode
+            std::cout << fmt("%1% %|2$5d|| %3%",
+                             prefix,
+                             (nixCode.errLine->lineNumber - 1),
+                             *nixCode.errLine->prevLineOfCode)
                       << std::endl;
         }
 
         // line of code containing the error.%2$+5d%
-        std::cout << format("%1% %|2$5d|| %3%")
-                  %    prefix
-                  % (nixCode.errLine->lineNumber)
-                  % nixCode.errLine->errLineOfCode
+        std::cout << fmt("%1% %|2$5d|| %3%",
+                         prefix,
+                         (nixCode.errLine->lineNumber),
+                         nixCode.errLine->errLineOfCode)
                   << std::endl;
 
         // error arrows for the column range.
-        if (nixCode.errLine->columnRange.has_value()) {
-            int start = nixCode.errLine->columnRange->start;
+        if (nixCode.errLine->column > 0) {
+            int start = nixCode.errLine->column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
             }
 
-            int len = nixCode.errLine->columnRange->len;
-            std::string arrows;
-            for (int i = 0; i < len; ++i) {
-                arrows.append("^");
-            }
+            // for now, length of 1.
+            std::string arrows("^");
 
-            std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
+            std::cout << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                             prefix,
+                             spaces,
+                             arrows) << std::endl;
         }
 
 
 
         // next line of code.
         if (nixCode.errLine->nextLineOfCode.has_value()) {
-            std::cout << format("%1% %|2$5d|| %3%")
-                      %    prefix
-                      % (nixCode.errLine->lineNumber + 1)
-                      % *nixCode.errLine->nextLineOfCode
+            std::cout << fmt("%1% %|2$5d|| %3%",
+                             prefix,
+                             (nixCode.errLine->lineNumber + 1),
+                             *nixCode.errLine->nextLineOfCode)
                       << std::endl;
         }
 
@@ -89,7 +89,7 @@ void printErrorInfo(ErrorInfo &einfo)
         break;
     }
     default: {
-        levelString = (format("invalid error level: %1%") % einfo.level).str();
+        levelString = fmt("invalid error level: %1%", einfo.level);
         break;
     }
     }
@@ -102,13 +102,13 @@ void printErrorInfo(ErrorInfo &einfo)
         dashes.append("-");
 
     // divider.
-    std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
-              % prefix
-              % levelString
-              % "---"
-              % einfo.name
-              % dashes
-              % einfo.programName.value_or("")
+    std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL
+                     , prefix
+                     , levelString
+                     , "---"
+                     , einfo.name
+                     , dashes
+                     , einfo.programName.value_or(""))
               << std::endl;
 
     // filename.
@@ -118,11 +118,11 @@ void printErrorInfo(ErrorInfo &einfo)
                            ? string(" ") + showErrLine(*einfo.nixCode->errLine)
                            : "";
 
-            std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL)
-                      % prefix % *einfo.nixCode->nixFile % eline << std::endl;
+            std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL
+                             , prefix, *einfo.nixCode->nixFile, eline) << std::endl;
             std::cout << prefix << std::endl;
         } else {
-            std::cout << format("%1%from command line argument") % prefix << std::endl;
+            std::cout << fmt("%1%from command line argument", prefix) << std::endl;
             std::cout << prefix << std::endl;
         }
     }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 57a9944be13..664bfbb5ab3 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -17,20 +17,13 @@ typedef enum {
     elError
 } ErrLevel;
 
-class ColumnRange
-{
-public:
-    unsigned int start;
-    unsigned int len;
-};
-
 class ErrorInfo;
 
 class ErrLine
 {
 public:
     int lineNumber;
-    std::optional<ColumnRange> columnRange;
+    int column;
     std::optional<string> prevLineOfCode;
     string errLineOfCode;
     std::optional<string> nextLineOfCode;
@@ -159,7 +152,7 @@ public:
     {
         GetEI().ensureNixCode().nixFile = aPos.file;
         GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line;
-        GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 };
+        GetEI().ensureNixCode().ensureErrLine().column = aPos.column;
         return *this;
     }
 protected:

From 00c507cc52ceb0d879c43e88e70de0028ff47fc6 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 7 Apr 2020 14:36:32 -0600
Subject: [PATCH 1946/2196] columnRange -> column

---
 src/libutil/error.cc | 14 +++++---------
 src/libutil/error.hh | 13 ++++---------
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 41fabbbcf95..db8821a5c45 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -46,8 +46,8 @@ ErrorInfo ErrorInfo::ProgramEI(ErrLevel level,
 
 string showErrLine(const ErrLine &errLine)
 {
-    if (errLine.columnRange.has_value()) {
-        return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str();
+    if (errLine.column > 0) {
+        return (format("(%1%:%2%)") % errLine.lineNumber % errLine.column).str();
     } else {
         return (format("(%1%)") % errLine.lineNumber).str();
     };
@@ -74,18 +74,14 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
                   << std::endl;
 
         // error arrows for the column range.
-        if (nixCode.errLine->columnRange.has_value()) {
-            int start = nixCode.errLine->columnRange->start;
+        if (nixCode.errLine->column > 0) {
+            int start = nixCode.errLine->column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
             }
 
-            int len = nixCode.errLine->columnRange->len;
-            std::string arrows;
-            for (int i = 0; i < len; ++i) {
-                arrows.append("^");
-            }
+            std::string arrows("^");
 
             std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
         }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index a8a1afbca09..e3bb2c1ddb8 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -17,20 +17,13 @@ typedef enum {
     elError
 } ErrLevel;
 
-class ColumnRange
-{
-public:
-    unsigned int start;
-    unsigned int len;
-};
-
 class ErrorInfo;
 
 class ErrLine
 {
 public:
     int lineNumber;
-    std::optional<ColumnRange> columnRange;
+    int column;
     std::optional<string> prevLineOfCode;
     string errLineOfCode;
     std::optional<string> nextLineOfCode;
@@ -107,6 +100,8 @@ public:
 
     static std::optional<string> programName;
 
+    ErrorInfo& set_name(const string &name) { this->name = name; return *this; }
+
     static ErrorInfo ProgramError(const string &name,
                                   const string &description,
                                   const std::optional<hintformat> &hf);
@@ -165,7 +160,7 @@ private:
 
         ErrLine errline;
         errline.lineNumber = pos.line;
-        errline.columnRange = { .start = pos.column, .len = 1 };
+        errline.column = pos.column;
         errline.prevLineOfCode = prevloc;
         errline.errLineOfCode = loc;
         errline.nextLineOfCode = nextloc;

From 1ab8d6ac1861e9405ae34af3deb681020c03e82d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 8 Apr 2020 15:18:41 +0200
Subject: [PATCH 1947/2196] Downloader: Only write data to the sink on a 200
 response

Hopefully fixes #3278.
---
 src/libstore/download.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index af69699a866..bc875f65348 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -83,8 +83,15 @@ struct CurlDownloader : public Downloader
             , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
                 if (this->request.dataCallback) {
-                    writtenToSink += len;
-                    this->request.dataCallback((char *) data, len);
+                    long httpStatus = 0;
+                    curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+
+                    /* Only write data to the sink if this is a
+                       successful response. */
+                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                        writtenToSink += len;
+                        this->request.dataCallback((char *) data, len);
+                    }
                 } else
                     this->result.data->append((char *) data, len);
               })

From 47ed067d45d3de7786cdb55f187b0db2eb6289c1 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 8 Apr 2020 09:07:58 -0600
Subject: [PATCH 1948/2196] initializer style

---
 src/error-demo/error-demo.cc |  22 ++---
 src/libutil/error.cc         |  85 ++++++--------------
 src/libutil/error.hh         | 150 +++++++++++++----------------------
 3 files changed, 94 insertions(+), 163 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index ef8b5630868..7eef0b1623b 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -18,22 +18,26 @@ int main()
 
     // ProgramError takes name, description, and an optional hint.
     printErrorInfo(
-        ErrorInfo::ProgramError("name",
-                                "error description",
-                                std::nullopt));
+        ErrorInfo { .level = elError,
+                    .name = "name",
+                    .description = "error description",
+                  });
 
     // ProgramWarning takes name, description, and an optional hint.
     // The hint is in the form of a hintfmt class, which wraps boost::format(),
     // and makes all the substituted text yellow.
     printErrorInfo(
-        ErrorInfo::ProgramWarning("name",
-                                  "warning description",
-                                  std::optional(
-                                      hintfmt("there was a %1%", "warning"))));
+        ErrorInfo { .level = elWarning,
+                    .name = "name",
+                    .description = "error description",
+                    .hint =  std::optional(
+                                      hintfmt("there was a %1%", "warning"))
+                  });
+
 
     // NixLangWarning adds nix file, line number, column range, and the lines of
     // code where a warning occurred.
-    SymbolTable testTable;
+/*    SymbolTable testTable;
     auto problem_symbol = testTable.create("problem");
 
     printErrorInfo(
@@ -58,5 +62,5 @@ int main()
             std::optional("next line of code"),
             hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
 
-    return 0;
+*/    return 0;
 }
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index db8821a5c45..24ed4df2e36 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -8,74 +8,37 @@ namespace nix
 
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
-ErrorInfo ErrorInfo::ProgramError(const string &name,
-                                  const string &description,
-                                  const std::optional<hintformat> &hf)
+string showErrPos(const ErrPos &errPos)
 {
-    return ProgramEI(elError, name, description, hf);
-}
-
-ErrorInfo ErrorInfo::ProgramWarning(const string &name,
-                                    const string &description,
-                                    const std::optional<hintformat> &hf)
-{
-    return ProgramEI(elWarning, name, description, hf);
-}
-
-
-
-ErrorInfo ErrorInfo::ProgramEI(ErrLevel level,
-                               const string &name,
-                               const string &description,
-                               const std::optional<hintformat> &hf)
-{
-    ErrorInfo ei(elError);
-    ei.name = name;
-    ei.description = description;
-    if (hf.has_value())
-        ei.hint = std::optional<string>(hf->str());
-    else
-        ei.hint = std::nullopt;
-    return ei;
-}
-
-
-
-
-
-
-string showErrLine(const ErrLine &errLine)
-{
-    if (errLine.column > 0) {
-        return (format("(%1%:%2%)") % errLine.lineNumber % errLine.column).str();
+    if (errPos.column > 0) {
+        return (format("(%1%:%2%)") % errPos.lineNumber % errPos.column).str();
     } else {
-        return (format("(%1%)") % errLine.lineNumber).str();
+        return (format("(%1%)") % errPos.lineNumber).str();
     };
 }
 
-void printCodeLines(const string &prefix, const NixCode &nixCode)
+void printCodeLines(const string &prefix, const ErrorInfo &einfo)
 {
-
-    if (nixCode.errLine.has_value()) {
+    if (einfo.errPos.has_value()) {
         // previous line of code.
-        if (nixCode.errLine->prevLineOfCode.has_value()) {
+        if (einfo.prevLineOfCode.has_value()) {
             std::cout << format("%1% %|2$5d|| %3%")
-                      %    prefix
-                      % (nixCode.errLine->lineNumber - 1)
-                      % *nixCode.errLine->prevLineOfCode
+                      % prefix
+                      % (einfo.errPos->lineNumber - 1)
+                      % *einfo.prevLineOfCode
                       << std::endl;
         }
 
         // line of code containing the error.%2$+5d%
         std::cout << format("%1% %|2$5d|| %3%")
                   %    prefix
-                  % (nixCode.errLine->lineNumber)
-                  % nixCode.errLine->errLineOfCode
+                  % (einfo.errPos->lineNumber)
+                  % einfo.errLineOfCode
                   << std::endl;
 
         // error arrows for the column range.
-        if (nixCode.errLine->column > 0) {
-            int start = nixCode.errLine->column;
+        if (einfo.errPos->column > 0) {
+            int start = einfo.errPos->column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
@@ -89,11 +52,11 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
 
 
         // next line of code.
-        if (nixCode.errLine->nextLineOfCode.has_value()) {
+        if (einfo.nextLineOfCode.has_value()) {
             std::cout << format("%1% %|2$5d|| %3%")
                       %    prefix
-                      % (nixCode.errLine->lineNumber + 1)
-                      % *nixCode.errLine->nextLineOfCode
+                      % (einfo.errPos->lineNumber + 1)
+                      % *einfo.nextLineOfCode
                       << std::endl;
         }
 
@@ -144,14 +107,14 @@ void printErrorInfo(const ErrorInfo &einfo)
               << std::endl;
 
     // filename.
-    if (einfo.nixCode.has_value()) {
-        if (einfo.nixCode->nixFile.has_value()) {
-            string eline = einfo.nixCode->errLine.has_value()
-                           ? string(" ") + showErrLine(*einfo.nixCode->errLine)
+    if (einfo.errPos.has_value()) {
+        if (einfo.errPos->nixFile != "") {
+            string eline = einfo.errLineOfCode != ""
+                           ? string(" ") + showErrPos(*einfo.errPos)
                            : "";
 
             std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL)
-                      % prefix % *einfo.nixCode->nixFile % eline << std::endl;
+                      % prefix % einfo.errPos->nixFile % eline << std::endl;
             std::cout << prefix << std::endl;
         } else {
             std::cout << format("%1%from command line argument") % prefix << std::endl;
@@ -164,8 +127,8 @@ void printErrorInfo(const ErrorInfo &einfo)
     std::cout << prefix << std::endl;
 
     // lines of code.
-    if (einfo.nixCode.has_value()) {
-        printCodeLines(prefix, *einfo.nixCode);
+    if (einfo.errLineOfCode != "") {
+        printCodeLines(prefix, einfo);
         std::cout << prefix << std::endl;
     }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index e3bb2c1ddb8..b687bde8199 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -17,23 +17,21 @@ typedef enum {
     elError
 } ErrLevel;
 
-class ErrorInfo;
-
-class ErrLine
+class ErrPos
 {
 public:
     int lineNumber;
     int column;
-    std::optional<string> prevLineOfCode;
-    string errLineOfCode;
-    std::optional<string> nextLineOfCode;
-};
+    string nixFile;
 
-class NixCode
-{
-public:
-    std::optional<string> nixFile;
-    std::optional<ErrLine> errLine;
+    template <class P>
+    ErrPos& operator=(const P &pos)
+    {
+        lineNumber = pos.line;
+        column = pos.column;
+        nixFile = pos.file.str();
+        return *this;
+    }
 };
 
 // ----------------------------------------------------------------
@@ -79,6 +77,11 @@ private:
     format fmt;
 };
 
+std::ostream& operator<<(std::ostream &os, const hintformat &hf)
+{
+    return os << hf.str();
+}
+
 template<typename... Args>
 inline hintformat hintfmt(const std::string & fs, const Args & ... args)
 {
@@ -95,95 +98,56 @@ public:
     ErrLevel level;
     string name;
     string description;
-    std::optional<NixCode> nixCode;
-    std::optional<string> hint;
+    std::optional<hintformat> hint;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
+    std::optional<ErrPos> errPos;
 
     static std::optional<string> programName;
 
-    ErrorInfo& set_name(const string &name) { this->name = name; return *this; }
-
-    static ErrorInfo ProgramError(const string &name,
-                                  const string &description,
-                                  const std::optional<hintformat> &hf);
-
-
-    static ErrorInfo ProgramWarning(const string &name,
-                                    const string &description,
-                                    const std::optional<hintformat> &hf);
-
-
-    template <class P>
-    static ErrorInfo NixLangError(const string &name,
-                                  const string &description,
-                                  const P &pos,
-                                  std::optional<string> prevloc,
-                                  string loc,
-                                  std::optional<string> nextloc,
-                                  const std::optional<hintformat> &hf)
-    {
-        return NixLangEI(elError, name, description, pos, prevloc, loc, nextloc, hf);
-    }
-
-
-    template <class P>
-    static ErrorInfo NixLangWarning(const string &name,
-                                    const string &description,
-                                    const P &pos,
-                                    std::optional<string> prevloc,
-                                    string loc,
-                                    std::optional<string> nextloc,
-                                    const std::optional<hintformat> &hf)
-    {
-        return NixLangEI(elWarning, name, description, pos, prevloc, loc, nextloc, hf);
-    }
-
-
-
 private:
-    template <class P>
-    static ErrorInfo NixLangEI(ErrLevel level,
-                               const string &name,
-                               const string &description,
-                               const P &pos,
-                               std::optional<string> prevloc,
-                               string loc,
-                               std::optional<string> nextloc,
-                               const std::optional<hintformat> &hf)
-    {
-        ErrorInfo ei(level);
-        ei.name = name;
-        ei.description = description;
-        if (hf.has_value())
-            ei.hint = std::optional<string>(hf->str());
-        else
-            ei.hint = std::nullopt;
-
-        ErrLine errline;
-        errline.lineNumber = pos.line;
-        errline.column = pos.column;
-        errline.prevLineOfCode = prevloc;
-        errline.errLineOfCode = loc;
-        errline.nextLineOfCode = nextloc;
-        NixCode nixcode;
-        nixcode.nixFile = pos.file;
-        nixcode.errLine = std::optional(errline);
-        ei.nixCode = std::optional(nixcode);
-
-        return ei;
-    }
-
-    static ErrorInfo ProgramEI(ErrLevel level,
-                               const string &name,
-                               const string &description,
-                               const std::optional<hintformat> &hf);
+    // template <class P>
+    // static ErrorInfo NixLangEI(ErrLevel level,
+    //                            const string &name,
+    //                            const string &description,
+    //                            const P &pos,
+    //                            std::optional<string> prevloc,
+    //                            string loc,
+    //                            std::optional<string> nextloc,
+    //                            const std::optional<hintformat> &hf)
+    // {
+    //     ErrorInfo ei(level);
+    //     ei.name = name;
+    //     ei.description = description;
+    //     if (hf.has_value())
+    //         ei.hint = std::optional<string>(hf->str());
+    //     else
+    //         ei.hint = std::nullopt;
+
+    //     ErrLine errline;
+    //     errline.lineNumber = pos.line;
+    //     errline.column = pos.column;
+    //     errline.prevLineOfCode = prevloc;
+    //     errline.errLineOfCode = loc;
+    //     errline.nextLineOfCode = nextloc;
+    //     NixCode nixcode;
+    //     nixcode.nixFile = pos.file;
+    //     nixcode.errLine = std::optional(errline);
+    //     ei.nixCode = std::optional(nixcode);
+
+    //     return ei;
+    // }
+
+    // static ErrorInfo ProgramEI(ErrLevel level,
+    //                            const string &name,
+    //                            const string &description,
+    //                            const std::optional<hintformat> &hf);
 
 
 
     // constructor is protected, so only the builder classes can create an ErrorInfo.
-    ErrorInfo(ErrLevel level)
-    {
-        this->level = level;
-    }
+
 };
 
 // --------------------------------------------------------

From 54f91923c844a98c4f8fd4c06feab9421a879ad7 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 8 Apr 2020 09:48:21 -0600
Subject: [PATCH 1949/2196] return of NixCode

---
 src/error-demo/error-demo.cc | 47 +++++++++++++---------
 src/libutil/error.cc         | 78 +++++++++++++++++-------------------
 src/libutil/error.hh         | 63 ++++++++---------------------
 3 files changed, 80 insertions(+), 108 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 7eef0b1623b..f8ec9553360 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -31,36 +31,43 @@ int main()
                     .name = "name",
                     .description = "error description",
                     .hint =  std::optional(
-                                      hintfmt("there was a %1%", "warning"))
+                                 hintfmt("there was a %1%", "warning")),
                   });
 
 
     // NixLangWarning adds nix file, line number, column range, and the lines of
     // code where a warning occurred.
-/*    SymbolTable testTable;
-    auto problem_symbol = testTable.create("problem");
+    SymbolTable testTable;
+    auto problem_file = testTable.create("myfile.nix");
 
     printErrorInfo(
-        ErrorInfo::NixLangWarning(
-            "warning name",
-            "warning description",
-            Pos(problem_symbol, 40, 13),
-            std::nullopt,
-            "this is the problem line of code",
-            std::nullopt,
-            hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+        ErrorInfo{
+            .level = elWarning,
+            .name = "warning name",
+            .description = "warning description",
+            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
+            .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = std::nullopt,
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = std::nullopt
+            }});
 
     // NixLangError is just the same as NixLangWarning, except for the Error
     // flag.
     printErrorInfo(
-        ErrorInfo::NixLangError(
-            "error name",
-            "error description",
-            Pos(problem_symbol, 40, 13),
-            std::optional("previous line of code"),
-            "this is the problem line of code",
-            std::optional("next line of code"),
-            hintfmt("this hint has %1% templated %2%!!", "yellow", "values")));
+        ErrorInfo{
+            .level = elError,
+            .name = "error name",
+            .description = "error description",
+            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
+            .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = std::optional("previous line of code"),
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = std::optional("next line of code"),
+            }});
 
-*/    return 0;
+
+    return 0;
 }
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 24ed4df2e36..1b983605955 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -17,51 +17,45 @@ string showErrPos(const ErrPos &errPos)
     };
 }
 
-void printCodeLines(const string &prefix, const ErrorInfo &einfo)
+void printCodeLines(const string &prefix, const NixCode &nixCode)
 {
-    if (einfo.errPos.has_value()) {
-        // previous line of code.
-        if (einfo.prevLineOfCode.has_value()) {
-            std::cout << format("%1% %|2$5d|| %3%")
-                      % prefix
-                      % (einfo.errPos->lineNumber - 1)
-                      % *einfo.prevLineOfCode
-                      << std::endl;
-        }
-
-        // line of code containing the error.%2$+5d%
+    // previous line of code.
+    if (nixCode.prevLineOfCode.has_value()) {
         std::cout << format("%1% %|2$5d|| %3%")
-                  %    prefix
-                  % (einfo.errPos->lineNumber)
-                  % einfo.errLineOfCode
+                  % prefix
+                  % (nixCode.errPos.lineNumber - 1)
+                  % *nixCode.prevLineOfCode
                   << std::endl;
+    }
 
-        // error arrows for the column range.
-        if (einfo.errPos->column > 0) {
-            int start = einfo.errPos->column;
-            std::string spaces;
-            for (int i = 0; i < start; ++i) {
-                spaces.append(" ");
-            }
-
-            std::string arrows("^");
+    // line of code containing the error.%2$+5d%
+    std::cout << format("%1% %|2$5d|| %3%")
+              % prefix
+              % (nixCode.errPos.lineNumber)
+              % nixCode.errLineOfCode
+              << std::endl;
 
-            std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
+    // error arrows for the column range.
+    if (nixCode.errPos.column > 0) {
+        int start = nixCode.errPos.column;
+        std::string spaces;
+        for (int i = 0; i < start; ++i) {
+            spaces.append(" ");
         }
 
+        std::string arrows("^");
 
-
-        // next line of code.
-        if (einfo.nextLineOfCode.has_value()) {
-            std::cout << format("%1% %|2$5d|| %3%")
-                      %    prefix
-                      % (einfo.errPos->lineNumber + 1)
-                      % *einfo.nextLineOfCode
-                      << std::endl;
-        }
-
+        std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
     }
 
+    // next line of code.
+    if (nixCode.nextLineOfCode.has_value()) {
+        std::cout << format("%1% %|2$5d|| %3%")
+                  %    prefix
+                  % (nixCode.errPos.lineNumber + 1)
+                  % *nixCode.nextLineOfCode
+                  << std::endl;
+    }
 }
 
 void printErrorInfo(const ErrorInfo &einfo)
@@ -107,14 +101,14 @@ void printErrorInfo(const ErrorInfo &einfo)
               << std::endl;
 
     // filename.
-    if (einfo.errPos.has_value()) {
-        if (einfo.errPos->nixFile != "") {
-            string eline = einfo.errLineOfCode != ""
-                           ? string(" ") + showErrPos(*einfo.errPos)
+    if (einfo.nixCode.has_value()) {
+        if (einfo.nixCode->errPos.nixFile != "") {
+            string eline = einfo.nixCode->errLineOfCode != ""
+                           ? string(" ") + showErrPos(einfo.nixCode->errPos)
                            : "";
 
             std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL)
-                      % prefix % einfo.errPos->nixFile % eline << std::endl;
+                      % prefix % einfo.nixCode->errPos.nixFile % eline << std::endl;
             std::cout << prefix << std::endl;
         } else {
             std::cout << format("%1%from command line argument") % prefix << std::endl;
@@ -127,8 +121,8 @@ void printErrorInfo(const ErrorInfo &einfo)
     std::cout << prefix << std::endl;
 
     // lines of code.
-    if (einfo.errLineOfCode != "") {
-        printCodeLines(prefix, einfo);
+    if (einfo.nixCode->errLineOfCode != "") {
+        printCodeLines(prefix, *einfo.nixCode);
         std::cout << prefix << std::endl;
     }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index b687bde8199..7e76c007936 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -29,9 +29,24 @@ public:
     {
         lineNumber = pos.line;
         column = pos.column;
-        nixFile = pos.file.str();
+        nixFile = pos.file;
         return *this;
     }
+
+    template <class P>
+    ErrPos(const P &p)
+    {
+      *this = p;
+    }
+};
+
+class NixCode
+{
+public:
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
 };
 
 // ----------------------------------------------------------------
@@ -99,55 +114,11 @@ public:
     string name;
     string description;
     std::optional<hintformat> hint;
-    std::optional<string> prevLineOfCode;
-    string errLineOfCode;
-    std::optional<string> nextLineOfCode;
-    std::optional<ErrPos> errPos;
+    std::optional<NixCode> nixCode;
 
     static std::optional<string> programName;
 
 private:
-    // template <class P>
-    // static ErrorInfo NixLangEI(ErrLevel level,
-    //                            const string &name,
-    //                            const string &description,
-    //                            const P &pos,
-    //                            std::optional<string> prevloc,
-    //                            string loc,
-    //                            std::optional<string> nextloc,
-    //                            const std::optional<hintformat> &hf)
-    // {
-    //     ErrorInfo ei(level);
-    //     ei.name = name;
-    //     ei.description = description;
-    //     if (hf.has_value())
-    //         ei.hint = std::optional<string>(hf->str());
-    //     else
-    //         ei.hint = std::nullopt;
-
-    //     ErrLine errline;
-    //     errline.lineNumber = pos.line;
-    //     errline.column = pos.column;
-    //     errline.prevLineOfCode = prevloc;
-    //     errline.errLineOfCode = loc;
-    //     errline.nextLineOfCode = nextloc;
-    //     NixCode nixcode;
-    //     nixcode.nixFile = pos.file;
-    //     nixcode.errLine = std::optional(errline);
-    //     ei.nixCode = std::optional(nixcode);
-
-    //     return ei;
-    // }
-
-    // static ErrorInfo ProgramEI(ErrLevel level,
-    //                            const string &name,
-    //                            const string &description,
-    //                            const std::optional<hintformat> &hf);
-
-
-
-    // constructor is protected, so only the builder classes can create an ErrorInfo.
-
 };
 
 // --------------------------------------------------------

From 555baa8fb07d4296c271ebc263a0a8fbe30268ee Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 8 Apr 2020 09:56:10 -0600
Subject: [PATCH 1950/2196] comments

---
 src/error-demo/error-demo.cc | 17 +++++------------
 src/libutil/error.hh         |  2 +-
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index f8ec9553360..a9ff6057c65 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -11,21 +11,15 @@ int main()
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
-    // There are currently four constructor functions:
-    //
-    // 			ProgramError, ProgramWarning, NixLangError, NixLangWarning.
-    //
-
-    // ProgramError takes name, description, and an optional hint.
+    // Error in a program; no hint and no nix code.
     printErrorInfo(
         ErrorInfo { .level = elError,
                     .name = "name",
                     .description = "error description",
                   });
 
-    // ProgramWarning takes name, description, and an optional hint.
-    // The hint is in the form of a hintfmt class, which wraps boost::format(),
-    // and makes all the substituted text yellow.
+    // Warning with name, description, and hint.
+    // The hintfmt function makes all the substituted text yellow.
     printErrorInfo(
         ErrorInfo { .level = elWarning,
                     .name = "name",
@@ -35,7 +29,7 @@ int main()
                   });
 
 
-    // NixLangWarning adds nix file, line number, column range, and the lines of
+    // Warning with nix file, line number, column, and the lines of
     // code where a warning occurred.
     SymbolTable testTable;
     auto problem_file = testTable.create("myfile.nix");
@@ -53,8 +47,7 @@ int main()
                 .nextLineOfCode = std::nullopt
             }});
 
-    // NixLangError is just the same as NixLangWarning, except for the Error
-    // flag.
+    // Error with previous and next lines of code.
     printErrorInfo(
         ErrorInfo{
             .level = elError,
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 7e76c007936..8286eaab11a 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -50,7 +50,7 @@ public:
 };
 
 // ----------------------------------------------------------------
-// format for hints.  same as fmt, except templated values
+// format function for hints.  same as fmt, except templated values
 // are always in yellow.
 
 template <class T>

From bf81b3155998152582abdad1550277cf8ca32e92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 8 Apr 2020 18:27:10 +0200
Subject: [PATCH 1951/2196] build.cc: improve message if home directory exists

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 527d7ac42fb..632982ecd75 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2161,7 +2161,7 @@ void DerivationGoal::startBuilder()
     if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
-            throw Error(format("directory '%1%' exists; please remove it") % homeDir);
+            throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir);
 
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete

From 8c2bf15c4fbf5cd18d3ee0b88f6b510e8881a622 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 8 Apr 2020 11:17:02 -0600
Subject: [PATCH 1952/2196] format -> fmt

---
 src/libutil/error.cc | 57 +++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 1b983605955..138c50d2b4e 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -11,9 +11,9 @@ std::optional<string> ErrorInfo::programName = std::nullopt;
 string showErrPos(const ErrPos &errPos)
 {
     if (errPos.column > 0) {
-        return (format("(%1%:%2%)") % errPos.lineNumber % errPos.column).str();
+        return fmt("(%1%:%2%)", errPos.lineNumber, errPos.column);
     } else {
-        return (format("(%1%)") % errPos.lineNumber).str();
+        return fmt("(%1%)", errPos.lineNumber);
     };
 }
 
@@ -21,18 +21,18 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
 {
     // previous line of code.
     if (nixCode.prevLineOfCode.has_value()) {
-        std::cout << format("%1% %|2$5d|| %3%")
-                  % prefix
-                  % (nixCode.errPos.lineNumber - 1)
-                  % *nixCode.prevLineOfCode
+        std::cout << fmt("%1% %|2$5d|| %3%",
+                         prefix,
+                         (nixCode.errPos.lineNumber - 1),
+                         *nixCode.prevLineOfCode)
                   << std::endl;
     }
 
     // line of code containing the error.%2$+5d%
-    std::cout << format("%1% %|2$5d|| %3%")
-              % prefix
-              % (nixCode.errPos.lineNumber)
-              % nixCode.errLineOfCode
+    std::cout << fmt("%1% %|2$5d|| %3%",
+                     prefix,
+                     (nixCode.errPos.lineNumber),
+                     nixCode.errLineOfCode)
               << std::endl;
 
     // error arrows for the column range.
@@ -45,15 +45,18 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
 
         std::string arrows("^");
 
-        std::cout << format("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl;
+        std::cout << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                         prefix,
+                         spaces,
+                         arrows) << std::endl;
     }
 
     // next line of code.
     if (nixCode.nextLineOfCode.has_value()) {
-        std::cout << format("%1% %|2$5d|| %3%")
-                  %    prefix
-                  % (nixCode.errPos.lineNumber + 1)
-                  % *nixCode.nextLineOfCode
+        std::cout << fmt("%1% %|2$5d|| %3%",
+                         prefix,
+                         (nixCode.errPos.lineNumber + 1),
+                         *nixCode.nextLineOfCode)
                   << std::endl;
     }
 }
@@ -78,8 +81,8 @@ void printErrorInfo(const ErrorInfo &einfo)
         break;
     }
     default: {
-        levelString = (format("invalid error level: %1%") % einfo.level).str();
-        break;
+        levelString = fmt("invalid error level: %1%", einfo.level);
+                       break;
     }
     }
 
@@ -91,13 +94,13 @@ void printErrorInfo(const ErrorInfo &einfo)
         dashes.append("-");
 
     // divider.
-    std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL)
-              % prefix
-              % levelString
-              % "---"
-              % einfo.name
-              % dashes
-              % einfo.programName.value_or("")
+    std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
+                     prefix,
+                     levelString,
+                     "---",
+                     einfo.name,
+                     dashes,
+                     einfo.programName.value_or(""))
               << std::endl;
 
     // filename.
@@ -107,11 +110,11 @@ void printErrorInfo(const ErrorInfo &einfo)
                            ? string(" ") + showErrPos(einfo.nixCode->errPos)
                            : "";
 
-            std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL)
-                      % prefix % einfo.nixCode->errPos.nixFile % eline << std::endl;
+            std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL
+                             , prefix, einfo.nixCode->errPos.nixFile, eline) << std::endl;
             std::cout << prefix << std::endl;
         } else {
-            std::cout << format("%1%from command line argument") % prefix << std::endl;
+            std::cout << fmt("%1%from command line argument", prefix) << std::endl;
             std::cout << prefix << std::endl;
         }
     }

From 805ffe1bc93b14ad8d1132c20179e85a914c91e6 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 8 Apr 2020 11:33:46 -0600
Subject: [PATCH 1953/2196] indention

---
 src/libutil/error.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 138c50d2b4e..665361a2a64 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -82,7 +82,7 @@ void printErrorInfo(const ErrorInfo &einfo)
     }
     default: {
         levelString = fmt("invalid error level: %1%", einfo.level);
-                       break;
+        break;
     }
     }
 

From 65ef57e0cbded52652510355304303a7b71586af Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 13:30:45 +0200
Subject: [PATCH 1954/2196] DownloadRequest -> DataTransferRequest

---
 src/libstore/builtins/fetchurl.cc        |  2 +-
 src/libstore/download.cc                 | 13 ++++++-------
 src/libstore/download.hh                 | 12 ++++++------
 src/libstore/http-binary-cache-store.cc  |  8 ++++----
 src/nix-prefetch-url/nix-prefetch-url.cc |  2 +-
 src/nix/upgrade-nix.cc                   |  2 +-
 6 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index f6ae5d2e622..a939f040fee 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -36,7 +36,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
-            DownloadRequest request(url);
+            DataTransferRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index bc875f65348..77f3c527a1a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -49,7 +49,7 @@ struct CurlDownloader : public Downloader
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
         CurlDownloader & downloader;
-        DownloadRequest request;
+        DataTransferRequest request;
         DownloadResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
@@ -73,7 +73,7 @@ struct CurlDownloader : public Downloader
         curl_off_t writtenToSink = 0;
 
         DownloadItem(CurlDownloader & downloader,
-            const DownloadRequest & request,
+            const DataTransferRequest & request,
             Callback<DownloadResult> && callback)
             : downloader(downloader)
             , request(request)
@@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader
     }
 #endif
 
-    void enqueueDownload(const DownloadRequest & request,
+    void enqueueDownload(const DataTransferRequest & request,
         Callback<DownloadResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
@@ -687,7 +687,7 @@ ref<Downloader> makeDownloader()
     return make_ref<CurlDownloader>();
 }
 
-std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
+std::future<DownloadResult> Downloader::enqueueDownload(const DataTransferRequest & request)
 {
     auto promise = std::make_shared<std::promise<DownloadResult>>();
     enqueueDownload(request,
@@ -701,12 +701,12 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
     return promise->get_future();
 }
 
-DownloadResult Downloader::download(const DownloadRequest & request)
+DownloadResult Downloader::download(const DataTransferRequest & request)
 {
     return enqueueDownload(request).get();
 }
 
-void Downloader::download(DownloadRequest && request, Sink & sink)
+void Downloader::download(DataTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
        that would cause the sink to execute on the downloader
@@ -801,7 +801,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-
 bool isUri(const string & s)
 {
     if (s.compare(0, 8, "channel:") == 0) return true;
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 28c8a916212..89d3fce7fce 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -33,7 +33,7 @@ struct DownloadSettings : Config
 
 extern DownloadSettings downloadSettings;
 
-struct DownloadRequest
+struct DataTransferRequest
 {
     std::string uri;
     std::string expectedETag;
@@ -47,7 +47,7 @@ struct DownloadRequest
     std::string mimeType;
     std::function<void(char *, size_t)> dataCallback;
 
-    DownloadRequest(const std::string & uri)
+    DataTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()
@@ -74,17 +74,17 @@ struct Downloader
     /* Enqueue a download request, returning a future to the result of
        the download. The future may throw a DownloadError
        exception. */
-    virtual void enqueueDownload(const DownloadRequest & request,
+    virtual void enqueueDownload(const DataTransferRequest & request,
         Callback<DownloadResult> callback) = 0;
 
-    std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
+    std::future<DownloadResult> enqueueDownload(const DataTransferRequest & request);
 
     /* Synchronously download a file. */
-    DownloadResult download(const DownloadRequest & request);
+    DownloadResult download(const DataTransferRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
-    void download(DownloadRequest && request, Sink & sink);
+    void download(DataTransferRequest && request, Sink & sink);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 011794c62b5..e152e75246d 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -85,7 +85,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         checkEnabled();
 
         try {
-            DownloadRequest request(cacheUri + "/" + path);
+            DataTransferRequest request(cacheUri + "/" + path);
             request.head = true;
             getDownloader()->download(request);
             return true;
@@ -103,7 +103,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         const std::string & data,
         const std::string & mimeType) override
     {
-        auto req = DownloadRequest(cacheUri + "/" + path);
+        auto req = DataTransferRequest(cacheUri + "/" + path);
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
@@ -113,9 +113,9 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         }
     }
 
-    DownloadRequest makeRequest(const std::string & path)
+    DataTransferRequest makeRequest(const std::string & path)
     {
-        DownloadRequest request(cacheUri + "/" + path);
+        DataTransferRequest request(cacheUri + "/" + path);
         return request;
     }
 
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 2b9254659a7..bfd65da982a 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -180,7 +180,7 @@ static int _main(int argc, char * * argv)
 
                 FdSink sink(fd.get());
 
-                DownloadRequest req(actualUri);
+                DataTransferRequest req(actualUri);
                 req.decompress = false;
                 getDownloader()->download(std::move(req), sink);
             }
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index c05c29517b2..aeaf7b09c53 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -138,7 +138,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
 
         // FIXME: use nixos.org?
-        auto req = DownloadRequest(storePathsUrl);
+        auto req = DataTransferRequest(storePathsUrl);
         auto res = getDownloader()->download(req);
 
         auto state = std::make_unique<EvalState>(Strings(), store);

From 741e9012d3fa6f0fb41a4a5662c6b8b38ecfaa1f Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 15:10:58 +0200
Subject: [PATCH 1955/2196] Rename src/lib/download.* to src/lib/datatransfer.*

---
 src/libexpr/common-eval-args.cc               | 2 +-
 src/libexpr/eval.cc                           | 2 +-
 src/libexpr/parser.y                          | 2 +-
 src/libstore/build.cc                         | 2 +-
 src/libstore/builtins/fetchurl.cc             | 2 +-
 src/libstore/{download.cc => datatransfer.cc} | 2 +-
 src/libstore/{download.hh => datatransfer.hh} | 0
 src/libstore/http-binary-cache-store.cc       | 2 +-
 src/libstore/s3-binary-cache-store.cc         | 2 +-
 src/nix-channel/nix-channel.cc                | 2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc      | 2 +-
 src/nix/main.cc                               | 2 +-
 src/nix/upgrade-nix.cc                        | 2 +-
 13 files changed, 12 insertions(+), 12 deletions(-)
 rename src/libstore/{download.cc => datatransfer.cc} (99%)
 rename src/libstore/{download.hh => datatransfer.hh} (100%)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 82bfeac36a1..26460601d15 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -1,6 +1,6 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dac32b6f563..7a20c6fa616 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -5,7 +5,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 6f25f5cf0f8..96b6f0ecd4c 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -544,7 +544,7 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "fetchers.hh"
 #include "store-api.hh"
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 632982ecd75..db01d95108e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -7,7 +7,7 @@
 #include "affinity.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index a939f040fee..01d21fa9892 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -1,5 +1,5 @@
 #include "builtins.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "compression.hh"
diff --git a/src/libstore/download.cc b/src/libstore/datatransfer.cc
similarity index 99%
rename from src/libstore/download.cc
rename to src/libstore/datatransfer.cc
index 77f3c527a1a..30ad48577b6 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/datatransfer.cc
@@ -1,4 +1,4 @@
-#include "download.hh"
+#include "datatransfer.hh"
 #include "util.hh"
 #include "globals.hh"
 #include "store-api.hh"
diff --git a/src/libstore/download.hh b/src/libstore/datatransfer.hh
similarity index 100%
rename from src/libstore/download.hh
rename to src/libstore/datatransfer.hh
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index e152e75246d..1fccee518d6 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -1,5 +1,5 @@
 #include "binary-cache-store.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f2e4b63e059..ed3c690b696 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -6,7 +6,7 @@
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
 #include "compression.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 2a9defb4e7b..d0719194d5b 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -1,6 +1,6 @@
 #include "shared.hh"
 #include "globals.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "store-api.hh"
 #include "../nix/legacy.hh"
 #include "fetchers.hh"
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index bfd65da982a..58a180e7279 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -1,6 +1,6 @@
 #include "hash.hh"
 #include "shared.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 3b5f5516fdf..ad410220163 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -8,7 +8,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "finally.hh"
 
 #include <sys/types.h>
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index aeaf7b09c53..414689ec5a9 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,7 +1,7 @@
 #include "command.hh"
 #include "common-args.hh"
 #include "store-api.hh"
-#include "download.hh"
+#include "datatransfer.hh"
 #include "eval.hh"
 #include "attr-path.hh"
 #include "names.hh"

From e5cc53beec62a36ebdc219e26b87f501ddb84a4f Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 15:14:49 +0200
Subject: [PATCH 1956/2196] DownloadSettings -> DataTransferSettings

---
 src/libstore/datatransfer.cc | 14 +++++++-------
 src/libstore/datatransfer.hh |  6 +++---
 src/nix/main.cc              |  8 ++++----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 30ad48577b6..62f546cfa0b 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -27,9 +27,9 @@ using namespace std::string_literals;
 
 namespace nix {
 
-DownloadSettings downloadSettings;
+DataTransferSettings dataTransferSettings;
 
-static GlobalConfig::Register r1(&downloadSettings);
+static GlobalConfig::Register r1(&dataTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -257,12 +257,12 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
+                    (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (downloadSettings.enableHttp2)
+            if (dataTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
@@ -297,10 +297,10 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
@@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            downloadSettings.httpConnections.get());
+            dataTransferSettings.httpConnections.get());
         #endif
 
         wakeupPipe.create();
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index 89d3fce7fce..16955ff06f9 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -9,7 +9,7 @@
 
 namespace nix {
 
-struct DownloadSettings : Config
+struct DataTransferSettings : Config
 {
     Setting<bool> enableHttp2{this, true, "http2",
         "Whether to enable HTTP/2 support."};
@@ -31,7 +31,7 @@ struct DownloadSettings : Config
         "How often Nix will attempt to download a file before giving up."};
 };
 
-extern DownloadSettings downloadSettings;
+extern DataTransferSettings dataTransferSettings;
 
 struct DataTransferRequest
 {
@@ -39,7 +39,7 @@ struct DataTransferRequest
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = downloadSettings.tries;
+    size_t tries = dataTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ad410220163..6bb04d8b609 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv)
             settings.useSubstitutes = false;
         if (!settings.tarballTtl.overriden)
             settings.tarballTtl = std::numeric_limits<unsigned int>::max();
-        if (!downloadSettings.tries.overriden)
-            downloadSettings.tries = 0;
-        if (!downloadSettings.connectTimeout.overriden)
-            downloadSettings.connectTimeout = 1;
+        if (!dataTransferSettings.tries.overriden)
+            dataTransferSettings.tries = 0;
+        if (!dataTransferSettings.connectTimeout.overriden)
+            dataTransferSettings.connectTimeout = 1;
     }
 
     if (args.refresh)

From 142ed7fe45a2b1d566e4401b3974cb29b8944fc6 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 15:18:51 +0200
Subject: [PATCH 1957/2196] DownloadResult -> DataTransferResult

---
 src/libstore/datatransfer.cc            | 20 ++++++++++----------
 src/libstore/datatransfer.hh            |  8 ++++----
 src/libstore/http-binary-cache-store.cc |  2 +-
 src/libstore/s3-binary-cache-store.cc   |  4 ++--
 src/libstore/s3.hh                      |  4 ++--
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 62f546cfa0b..06aa86d970c 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -50,10 +50,10 @@ struct CurlDownloader : public Downloader
     {
         CurlDownloader & downloader;
         DataTransferRequest request;
-        DownloadResult result;
+        DataTransferResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        Callback<DownloadResult> callback;
+        Callback<DataTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -74,7 +74,7 @@ struct CurlDownloader : public Downloader
 
         DownloadItem(CurlDownloader & downloader,
             const DataTransferRequest & request,
-            Callback<DownloadResult> && callback)
+            Callback<DataTransferResult> && callback)
             : downloader(downloader)
             , request(request)
             , act(*logger, lvlTalkative, actDownload,
@@ -642,7 +642,7 @@ struct CurlDownloader : public Downloader
 #endif
 
     void enqueueDownload(const DataTransferRequest & request,
-        Callback<DownloadResult> callback) override
+        Callback<DataTransferResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
@@ -660,7 +660,7 @@ struct CurlDownloader : public Downloader
 
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
-                DownloadResult res;
+                DataTransferResult res;
                 if (!s3Res.data)
                     throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
@@ -687,11 +687,11 @@ ref<Downloader> makeDownloader()
     return make_ref<CurlDownloader>();
 }
 
-std::future<DownloadResult> Downloader::enqueueDownload(const DataTransferRequest & request)
+std::future<DataTransferResult> Downloader::enqueueDownload(const DataTransferRequest & request)
 {
-    auto promise = std::make_shared<std::promise<DownloadResult>>();
+    auto promise = std::make_shared<std::promise<DataTransferResult>>();
     enqueueDownload(request,
-        {[promise](std::future<DownloadResult> fut) {
+        {[promise](std::future<DataTransferResult> fut) {
             try {
                 promise->set_value(fut.get());
             } catch (...) {
@@ -701,7 +701,7 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DataTransferReques
     return promise->get_future();
 }
 
-DownloadResult Downloader::download(const DataTransferRequest & request)
+DataTransferResult Downloader::download(const DataTransferRequest & request)
 {
     return enqueueDownload(request).get();
 }
@@ -756,7 +756,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink)
     };
 
     enqueueDownload(request,
-        {[_state](std::future<DownloadResult> fut) {
+        {[_state](std::future<DataTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
             try {
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index 16955ff06f9..b9e55655f0e 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -56,7 +56,7 @@ struct DataTransferRequest
     }
 };
 
-struct DownloadResult
+struct DataTransferResult
 {
     bool cached = false;
     std::string etag;
@@ -75,12 +75,12 @@ struct Downloader
        the download. The future may throw a DownloadError
        exception. */
     virtual void enqueueDownload(const DataTransferRequest & request,
-        Callback<DownloadResult> callback) = 0;
+        Callback<DataTransferResult> callback) = 0;
 
-    std::future<DownloadResult> enqueueDownload(const DataTransferRequest & request);
+    std::future<DataTransferResult> enqueueDownload(const DataTransferRequest & request);
 
     /* Synchronously download a file. */
-    DownloadResult download(const DataTransferRequest & request);
+    DataTransferResult download(const DataTransferRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 1fccee518d6..24a98d07364 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -143,7 +143,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
         getDownloader()->enqueueDownload(request,
-            {[callbackPtr, this](std::future<DownloadResult> result) {
+            {[callbackPtr, this](std::future<DataTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
                 } catch (DownloadError & e) {
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ed3c690b696..478cb9f84a3 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -132,7 +132,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
     return res;
 }
 
-S3Helper::DownloadResult S3Helper::getObject(
+S3Helper::DataTransferResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
     debug("fetching 's3://%s/%s'...", bucketName, key);
@@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject(
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
-    DownloadResult res;
+    DataTransferResult res;
 
     auto now1 = std::chrono::steady_clock::now();
 
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index ef5f23d0f25..d7d30924319 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -18,13 +18,13 @@ struct S3Helper
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    struct DownloadResult
+    struct DataTransferResult
     {
         std::shared_ptr<std::string> data;
         unsigned int durationMs;
     };
 
-    DownloadResult getObject(
+    DataTransferResult getObject(
         const std::string & bucketName, const std::string & key);
 };
 

From 2df2741ec6e9411739b2049d7487b7b8a26d4547 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 15:37:33 +0200
Subject: [PATCH 1958/2196] enqueueDownload -> enqueueDataTransfer

---
 src/libstore/datatransfer.cc            | 10 +++++-----
 src/libstore/datatransfer.hh            |  4 ++--
 src/libstore/http-binary-cache-store.cc |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 06aa86d970c..9475c24ecae 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader
     }
 #endif
 
-    void enqueueDownload(const DataTransferRequest & request,
+    void enqueueDataTransfer(const DataTransferRequest & request,
         Callback<DataTransferResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
@@ -687,10 +687,10 @@ ref<Downloader> makeDownloader()
     return make_ref<CurlDownloader>();
 }
 
-std::future<DataTransferResult> Downloader::enqueueDownload(const DataTransferRequest & request)
+std::future<DataTransferResult> Downloader::enqueueDataTransfer(const DataTransferRequest & request)
 {
     auto promise = std::make_shared<std::promise<DataTransferResult>>();
-    enqueueDownload(request,
+    enqueueDataTransfer(request,
         {[promise](std::future<DataTransferResult> fut) {
             try {
                 promise->set_value(fut.get());
@@ -703,7 +703,7 @@ std::future<DataTransferResult> Downloader::enqueueDownload(const DataTransferRe
 
 DataTransferResult Downloader::download(const DataTransferRequest & request)
 {
-    return enqueueDownload(request).get();
+    return enqueueDataTransfer(request).get();
 }
 
 void Downloader::download(DataTransferRequest && request, Sink & sink)
@@ -755,7 +755,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink)
         state->avail.notify_one();
     };
 
-    enqueueDownload(request,
+    enqueueDataTransfer(request,
         {[_state](std::future<DataTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index b9e55655f0e..53159bdb46e 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -74,10 +74,10 @@ struct Downloader
     /* Enqueue a download request, returning a future to the result of
        the download. The future may throw a DownloadError
        exception. */
-    virtual void enqueueDownload(const DataTransferRequest & request,
+    virtual void enqueueDataTransfer(const DataTransferRequest & request,
         Callback<DataTransferResult> callback) = 0;
 
-    std::future<DataTransferResult> enqueueDownload(const DataTransferRequest & request);
+    std::future<DataTransferResult> enqueueDataTransfer(const DataTransferRequest & request);
 
     /* Synchronously download a file. */
     DataTransferResult download(const DataTransferRequest & request);
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 24a98d07364..861287a33da 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -142,7 +142,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        getDownloader()->enqueueDownload(request,
+        getDownloader()->enqueueDataTransfer(request,
             {[callbackPtr, this](std::future<DataTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);

From cd391206e6d29608bc3e25301b9921f1ac189086 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 22:57:09 +0200
Subject: [PATCH 1959/2196] {get,make,new}Downloader -> DataTransfer

---
 src/libstore/builtins/fetchurl.cc        |  6 ++--
 src/libstore/datatransfer.cc             | 36 ++++++++++++------------
 src/libstore/datatransfer.hh             | 18 ++++++------
 src/libstore/http-binary-cache-store.cc  | 14 ++++-----
 src/nix-prefetch-url/nix-prefetch-url.cc |  2 +-
 src/nix/upgrade-nix.cc                   |  2 +-
 6 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 01d21fa9892..0b6db8f0ca7 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto mainUrl = getAttr("url");
     bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
-    /* Note: have to use a fresh downloader here because we're in
+    /* Note: have to use a fresh dataTransfer here because we're in
        a forked process. */
-    auto downloader = makeDownloader();
+    auto dataTransfer = makeDataTransfer();
 
     auto fetch = [&](const std::string & url) {
 
@@ -42,7 +42,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             auto decompressor = makeDecompressionSink(
                 unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
-            downloader->download(std::move(request), *decompressor);
+            dataTransfer->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 9475c24ecae..6ea9f932417 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-struct CurlDownloader : public Downloader
+struct curlDataTransfer : public DataTransfer
 {
     CURLM * curlm = 0;
 
@@ -48,7 +48,7 @@ struct CurlDownloader : public Downloader
 
     struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
     {
-        CurlDownloader & downloader;
+        curlDataTransfer & dataTransfer;
         DataTransferRequest request;
         DataTransferResult result;
         Activity act;
@@ -72,10 +72,10 @@ struct CurlDownloader : public Downloader
 
         curl_off_t writtenToSink = 0;
 
-        DownloadItem(CurlDownloader & downloader,
+        DownloadItem(curlDataTransfer & dataTransfer,
             const DataTransferRequest & request,
             Callback<DataTransferResult> && callback)
-            : downloader(downloader)
+            : dataTransfer(dataTransfer)
             , request(request)
             , act(*logger, lvlTalkative, actDownload,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
@@ -106,7 +106,7 @@ struct CurlDownloader : public Downloader
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(downloader.curlm, req);
+                    curl_multi_remove_handle(dataTransfer.curlm, req);
                 curl_easy_cleanup(req);
             }
             if (requestHeaders) curl_slist_free_all(requestHeaders);
@@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
+                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
-                    downloader.enqueueItem(shared_from_this());
+                    dataTransfer.enqueueItem(shared_from_this());
                 }
                 else
                     fail(exc);
@@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader
 
     std::thread workerThread;
 
-    CurlDownloader()
+    curlDataTransfer()
         : mt19937(rd())
     {
         static std::once_flag globalInit;
@@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
-    ~CurlDownloader()
+    ~curlDataTransfer()
     {
         stopWorkerThread();
 
@@ -676,18 +676,18 @@ struct CurlDownloader : public Downloader
     }
 };
 
-ref<Downloader> getDownloader()
+ref<DataTransfer> getDataTransfer()
 {
-    static ref<Downloader> downloader = makeDownloader();
-    return downloader;
+    static ref<DataTransfer> dataTransfer = makeDataTransfer();
+    return dataTransfer;
 }
 
-ref<Downloader> makeDownloader()
+ref<DataTransfer> makeDataTransfer()
 {
-    return make_ref<CurlDownloader>();
+    return make_ref<curlDataTransfer>();
 }
 
-std::future<DataTransferResult> Downloader::enqueueDataTransfer(const DataTransferRequest & request)
+std::future<DataTransferResult> DataTransfer::enqueueDataTransfer(const DataTransferRequest & request)
 {
     auto promise = std::make_shared<std::promise<DataTransferResult>>();
     enqueueDataTransfer(request,
@@ -701,15 +701,15 @@ std::future<DataTransferResult> Downloader::enqueueDataTransfer(const DataTransf
     return promise->get_future();
 }
 
-DataTransferResult Downloader::download(const DataTransferRequest & request)
+DataTransferResult DataTransfer::download(const DataTransferRequest & request)
 {
     return enqueueDataTransfer(request).get();
 }
 
-void Downloader::download(DataTransferRequest && request, Sink & sink)
+void DataTransfer::download(DataTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
-       that would cause the sink to execute on the downloader
+       that would cause the sink to execute on the dataTransfer
        thread. If 'sink' is a coroutine, this will fail. Also, if the
        sink is expensive (e.g. one that does decompression and writing
        to the Nix store), it would stall the download thread too much.
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index 53159bdb46e..e1c5196e08b 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -67,11 +67,11 @@ struct DataTransferResult
 
 class Store;
 
-struct Downloader
+struct DataTransfer
 {
-    virtual ~Downloader() { }
+    virtual ~DataTransfer() { }
 
-    /* Enqueue a download request, returning a future to the result of
+    /* Enqueue a data transfer request, returning a future to the result of
        the download. The future may throw a DownloadError
        exception. */
     virtual void enqueueDataTransfer(const DataTransferRequest & request,
@@ -89,18 +89,18 @@ struct Downloader
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
 
-/* Return a shared Downloader object. Using this object is preferred
+/* Return a shared DataTransfer object. Using this object is preferred
    because it enables connection reuse and HTTP/2 multiplexing. */
-ref<Downloader> getDownloader();
+ref<DataTransfer> getDataTransfer();
 
-/* Return a new Downloader object. */
-ref<Downloader> makeDownloader();
+/* Return a new DataTransfer object. */
+ref<DataTransfer> makeDataTransfer();
 
 class DownloadError : public Error
 {
 public:
-    Downloader::Error error;
-    DownloadError(Downloader::Error error, const FormatOrString & fs)
+    DataTransfer::Error error;
+    DownloadError(DataTransfer::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
 };
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 861287a33da..9a19af5e0ca 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -87,12 +87,12 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         try {
             DataTransferRequest request(cacheUri + "/" + path);
             request.head = true;
-            getDownloader()->download(request);
+            getDataTransfer()->download(request);
             return true;
         } catch (DownloadError & e) {
             /* S3 buckets return 403 if a file doesn't exist and the
                bucket is unlistable, so treat 403 as 404. */
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
                 return false;
             maybeDisable();
             throw;
@@ -107,7 +107,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
-            getDownloader()->download(req);
+            getDataTransfer()->download(req);
         } catch (DownloadError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
@@ -124,9 +124,9 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         checkEnabled();
         auto request(makeRequest(path));
         try {
-            getDownloader()->download(std::move(request), sink);
+            getDataTransfer()->download(std::move(request), sink);
         } catch (DownloadError & e) {
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
             maybeDisable();
             throw;
@@ -142,12 +142,12 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        getDownloader()->enqueueDataTransfer(request,
+        getDataTransfer()->enqueueDataTransfer(request,
             {[callbackPtr, this](std::future<DataTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
                 } catch (DownloadError & e) {
-                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                    if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
                         return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
                     callbackPtr->rethrow();
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 58a180e7279..4c81e7d82dd 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -182,7 +182,7 @@ static int _main(int argc, char * * argv)
 
                 DataTransferRequest req(actualUri);
                 req.decompress = false;
-                getDownloader()->download(std::move(req), sink);
+                getDataTransfer()->download(std::move(req), sink);
             }
 
             /* Optionally unpack the file. */
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 414689ec5a9..a4eda90b416 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -139,7 +139,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         // FIXME: use nixos.org?
         auto req = DataTransferRequest(storePathsUrl);
-        auto res = getDownloader()->download(req);
+        auto res = getDataTransfer()->download(req);
 
         auto state = std::make_unique<EvalState>(Strings(), store);
         auto v = state->allocValue();

From 213d124277460b316cfa03d43e4ec279bba86219 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 22:59:36 +0200
Subject: [PATCH 1960/2196] DownloadItem -> TransferItem

---
 src/libstore/datatransfer.cc | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 6ea9f932417..452af107d17 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -46,7 +46,7 @@ struct curlDataTransfer : public DataTransfer
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
         curlDataTransfer & dataTransfer;
         DataTransferRequest request;
@@ -72,7 +72,7 @@ struct curlDataTransfer : public DataTransfer
 
         curl_off_t writtenToSink = 0;
 
-        DownloadItem(curlDataTransfer & dataTransfer,
+        TransferItem(curlDataTransfer & dataTransfer,
             const DataTransferRequest & request,
             Callback<DataTransferResult> && callback)
             : dataTransfer(dataTransfer)
@@ -102,7 +102,7 @@ struct curlDataTransfer : public DataTransfer
                 requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
-        ~DownloadItem()
+        ~TransferItem()
         {
             if (req) {
                 if (active)
@@ -156,7 +156,7 @@ struct curlDataTransfer : public DataTransfer
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
         }
 
         size_t headerCallback(void * contents, size_t size, size_t nmemb)
@@ -198,7 +198,7 @@ struct curlDataTransfer : public DataTransfer
 
         static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
         int progressCallback(double dltotal, double dlnow)
@@ -213,7 +213,7 @@ struct curlDataTransfer : public DataTransfer
 
         static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
         {
-            return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
+            return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
         }
 
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@@ -237,7 +237,7 @@ struct curlDataTransfer : public DataTransfer
 
         static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
         {
-            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+            return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
         void init()
@@ -248,7 +248,7 @@ struct curlDataTransfer : public DataTransfer
 
             if (verbosity >= lvlVomit) {
                 curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
-                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
+                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
             }
 
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@@ -267,9 +267,9 @@ struct curlDataTransfer : public DataTransfer
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             #endif
-            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
-            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
             curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@@ -439,12 +439,12 @@ struct curlDataTransfer : public DataTransfer
     struct State
     {
         struct EmbargoComparator {
-            bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
+            bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
                 return i1->embargo > i2->embargo;
             }
         };
         bool quit = false;
-        std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
+        std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
     };
 
     Sync<State> state_;
@@ -504,7 +504,7 @@ struct curlDataTransfer : public DataTransfer
             stopWorkerThread();
         });
 
-        std::map<CURL *, std::shared_ptr<DownloadItem>> items;
+        std::map<CURL *, std::shared_ptr<TransferItem>> items;
 
         bool quit = false;
 
@@ -561,7 +561,7 @@ struct curlDataTransfer : public DataTransfer
                     throw SysError("reading curl wakeup socket");
             }
 
-            std::vector<std::shared_ptr<DownloadItem>> incoming;
+            std::vector<std::shared_ptr<TransferItem>> incoming;
             auto now = std::chrono::steady_clock::now();
 
             {
@@ -609,7 +609,7 @@ struct curlDataTransfer : public DataTransfer
         }
     }
 
-    void enqueueItem(std::shared_ptr<DownloadItem> item)
+    void enqueueItem(std::shared_ptr<TransferItem> item)
     {
         if (item->request.data
             && !hasPrefix(item->request.uri, "http://")
@@ -672,7 +672,7 @@ struct curlDataTransfer : public DataTransfer
             return;
         }
 
-        enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
+        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
     }
 };
 

From c4c1ae0a00e8dd3258af58c15b664828b592133e Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 23:00:43 +0200
Subject: [PATCH 1961/2196] DownloadError -> DataTransferError

---
 src/libexpr/parser.y                    |  2 +-
 src/libstore/datatransfer.cc            | 10 +++++-----
 src/libstore/datatransfer.hh            |  6 +++---
 src/libstore/http-binary-cache-store.cc |  8 ++++----
 src/nix-channel/nix-channel.cc          |  2 +-
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 96b6f0ecd4c..8fbe59301ab 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -690,7 +690,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
                         store, resolveUri(elem.second), "source", false).storePath) };
-        } catch (DownloadError & e) {
+        } catch (DataTransferError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }
diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 452af107d17..be5dd27cda6 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -112,7 +112,7 @@ struct curlDataTransfer : public DataTransfer
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+                    ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
-                    ? DownloadError(err,
+                    ? DataTransferError(err,
                         fmt("unable to %s '%s': HTTP error %d",
                             request.verb(), request.uri, httpStatus)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
-                    : DownloadError(err,
+                    : DataTransferError(err,
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -662,7 +662,7 @@ struct curlDataTransfer : public DataTransfer
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 DataTransferResult res;
                 if (!s3Res.data)
-                    throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+                    throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index e1c5196e08b..f2ea53ae392 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -72,7 +72,7 @@ struct DataTransfer
     virtual ~DataTransfer() { }
 
     /* Enqueue a data transfer request, returning a future to the result of
-       the download. The future may throw a DownloadError
+       the download. The future may throw a DataTransferError
        exception. */
     virtual void enqueueDataTransfer(const DataTransferRequest & request,
         Callback<DataTransferResult> callback) = 0;
@@ -96,11 +96,11 @@ ref<DataTransfer> getDataTransfer();
 /* Return a new DataTransfer object. */
 ref<DataTransfer> makeDataTransfer();
 
-class DownloadError : public Error
+class DataTransferError : public Error
 {
 public:
     DataTransfer::Error error;
-    DownloadError(DataTransfer::Error error, const FormatOrString & fs)
+    DataTransferError(DataTransfer::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
 };
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 9a19af5e0ca..5e446ddf282 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -89,7 +89,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
             request.head = true;
             getDataTransfer()->download(request);
             return true;
-        } catch (DownloadError & e) {
+        } catch (DataTransferError & e) {
             /* S3 buckets return 403 if a file doesn't exist and the
                bucket is unlistable, so treat 403 as 404. */
             if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
@@ -108,7 +108,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         req.mimeType = mimeType;
         try {
             getDataTransfer()->download(req);
-        } catch (DownloadError & e) {
+        } catch (DataTransferError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
@@ -125,7 +125,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         auto request(makeRequest(path));
         try {
             getDataTransfer()->download(std::move(request), sink);
-        } catch (DownloadError & e) {
+        } catch (DataTransferError & e) {
             if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
             maybeDisable();
@@ -146,7 +146,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
             {[callbackPtr, this](std::future<DataTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
-                } catch (DownloadError & e) {
+                } catch (DataTransferError & e) {
                     if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
                         return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index d0719194d5b..0f8024d7bb9 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -113,7 +113,7 @@ static void update(const StringSet & channelNames)
             // Download the channel tarball.
             try {
                 filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
-            } catch (DownloadError & e) {
+            } catch (DataTransferError & e) {
                 filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
             }
         }

From a0c5931208042da39bb6a5e80a4b27cf50f665d6 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 23:08:39 +0200
Subject: [PATCH 1962/2196] actDownload -> actDataTransfer

---
 src/libstore/build.cc        | 2 +-
 src/libstore/datatransfer.cc | 2 +-
 src/libutil/logging.cc       | 2 +-
 src/libutil/logging.hh       | 2 +-
 src/nix/progress-bar.cc      | 6 +++---
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index db01d95108e..f091a000d23 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -361,7 +361,7 @@ class Worker
     {
         actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
         actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize);
+        act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize);
         act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
     }
 };
diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index be5dd27cda6..9ed06b5c106 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -77,7 +77,7 @@ struct curlDataTransfer : public DataTransfer
             Callback<DataTransferResult> && callback)
             : dataTransfer(dataTransfer)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload,
+            , act(*logger, lvlTalkative, actDataTransfer,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index fa5c84a2733..001a3fb281d 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg,
 
         if (action == "start") {
             auto type = (ActivityType) json["type"];
-            if (trusted || type == actDownload)
+            if (trusted || type == actDataTransfer)
                 activities.emplace(std::piecewise_construct,
                     std::forward_as_tuple(json["id"]),
                     std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index beb5e6b64e6..423ed95daa1 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -17,7 +17,7 @@ typedef enum {
 typedef enum {
     actUnknown = 0,
     actCopyPath = 100,
-    actDownload = 101,
+    actDataTransfer = 101,
     actRealise = 102,
     actCopyPaths = 103,
     actBuilds = 104,
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 26631416cc2..d108896ad44 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -190,8 +190,8 @@ class ProgressBar : public Logger
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
         }
 
-        if ((type == actDownload && hasAncestor(*state, actCopyPath, parent))
-            || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent))
+        if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent))
+            || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent))
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
@@ -416,7 +416,7 @@ class ProgressBar : public Logger
             if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
         }
 
-        showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
+        showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB);
 
         {
             auto s = renderActivity(actOptimiseStore, "%s paths optimised");

From 7848372b0f683a5a4db5f86fd998e8df5fa22715 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 23:34:31 +0200
Subject: [PATCH 1963/2196] Add upload method

---
 src/libstore/datatransfer.cc            | 6 ++++++
 src/libstore/datatransfer.hh            | 3 +++
 src/libstore/http-binary-cache-store.cc | 3 +--
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index 9ed06b5c106..dbdf07ad428 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -706,6 +706,12 @@ DataTransferResult DataTransfer::download(const DataTransferRequest & request)
     return enqueueDataTransfer(request).get();
 }
 
+DataTransferResult DataTransfer::upload(const DataTransferRequest & request)
+{
+    /* Note: this method is the same as download, but helps in readability */
+    return enqueueDataTransfer(request).get();
+}
+
 void DataTransfer::download(DataTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index f2ea53ae392..68d97ceb111 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -82,6 +82,9 @@ struct DataTransfer
     /* Synchronously download a file. */
     DataTransferResult download(const DataTransferRequest & request);
 
+    /* Synchronously upload a file. */
+    DataTransferResult upload(const DataTransferRequest & request);
+
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
     void download(DataTransferRequest && request, Sink & sink);
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 5e446ddf282..c75846bc3f4 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -107,7 +107,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
-            getDataTransfer()->download(req);
+            getDataTransfer()->upload(req);
         } catch (DataTransferError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
@@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([](
 });
 
 }
-

From c330109bfa38370f7cef6449efb88309a78e1684 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 23:43:43 +0200
Subject: [PATCH 1964/2196] DataTransfer -> FileTransfer

---
 src/libstore/build.cc                    |  2 +-
 src/libstore/builtins/fetchurl.cc        |  8 +-
 src/libstore/datatransfer.cc             | 94 ++++++++++++------------
 src/libstore/datatransfer.hh             | 44 +++++------
 src/libstore/http-binary-cache-store.cc  | 32 ++++----
 src/libstore/s3-binary-cache-store.cc    |  4 +-
 src/libstore/s3.hh                       |  4 +-
 src/libutil/logging.cc                   |  2 +-
 src/libutil/logging.hh                   |  2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc |  4 +-
 src/nix/main.cc                          |  8 +-
 src/nix/progress-bar.cc                  |  6 +-
 src/nix/upgrade-nix.cc                   |  4 +-
 13 files changed, 107 insertions(+), 107 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f091a000d23..9baa2cf1aec 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -361,7 +361,7 @@ class Worker
     {
         actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
         actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize);
+        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
         act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
     }
 };
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 0b6db8f0ca7..1432de9d704 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto mainUrl = getAttr("url");
     bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
-    /* Note: have to use a fresh dataTransfer here because we're in
+    /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */
-    auto dataTransfer = makeDataTransfer();
+    auto fileTransfer = makeFileTransfer();
 
     auto fetch = [&](const std::string & url) {
 
@@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
-            DataTransferRequest request(url);
+            FileTransferRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
                 unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
-            dataTransfer->download(std::move(request), *decompressor);
+            fileTransfer->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc
index dbdf07ad428..5cc63a1d8c7 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/datatransfer.cc
@@ -27,9 +27,9 @@ using namespace std::string_literals;
 
 namespace nix {
 
-DataTransferSettings dataTransferSettings;
+FileTransferSettings fileTransferSettings;
 
-static GlobalConfig::Register r1(&dataTransferSettings);
+static GlobalConfig::Register r1(&fileTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-struct curlDataTransfer : public DataTransfer
+struct curlFileTransfer : public FileTransfer
 {
     CURLM * curlm = 0;
 
@@ -48,12 +48,12 @@ struct curlDataTransfer : public DataTransfer
 
     struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
-        curlDataTransfer & dataTransfer;
-        DataTransferRequest request;
-        DataTransferResult result;
+        curlFileTransfer & fileTransfer;
+        FileTransferRequest request;
+        FileTransferResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        Callback<DataTransferResult> callback;
+        Callback<FileTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -72,12 +72,12 @@ struct curlDataTransfer : public DataTransfer
 
         curl_off_t writtenToSink = 0;
 
-        TransferItem(curlDataTransfer & dataTransfer,
-            const DataTransferRequest & request,
-            Callback<DataTransferResult> && callback)
-            : dataTransfer(dataTransfer)
+        TransferItem(curlFileTransfer & fileTransfer,
+            const FileTransferRequest & request,
+            Callback<FileTransferResult> && callback)
+            : fileTransfer(fileTransfer)
             , request(request)
-            , act(*logger, lvlTalkative, actDataTransfer,
+            , act(*logger, lvlTalkative, actFileTransfer,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
@@ -106,13 +106,13 @@ struct curlDataTransfer : public DataTransfer
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(dataTransfer.curlm, req);
+                    curl_multi_remove_handle(fileTransfer.curlm, req);
                 curl_easy_cleanup(req);
             }
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -257,12 +257,12 @@ struct curlDataTransfer : public DataTransfer
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str());
+                    (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (dataTransferSettings.enableHttp2)
+            if (fileTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
@@ -297,10 +297,10 @@ struct curlDataTransfer : public DataTransfer
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get());
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
@@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+                    ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
-                    ? DataTransferError(err,
+                    ? FileTransferError(err,
                         fmt("unable to %s '%s': HTTP error %d",
                             request.verb(), request.uri, httpStatus)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
-                    : DataTransferError(err,
+                    : FileTransferError(err,
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -422,13 +422,13 @@ struct curlDataTransfer : public DataTransfer
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937));
+                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
-                    dataTransfer.enqueueItem(shared_from_this());
+                    fileTransfer.enqueueItem(shared_from_this());
                 }
                 else
                     fail(exc);
@@ -456,7 +456,7 @@ struct curlDataTransfer : public DataTransfer
 
     std::thread workerThread;
 
-    curlDataTransfer()
+    curlFileTransfer()
         : mt19937(rd())
     {
         static std::once_flag globalInit;
@@ -469,7 +469,7 @@ struct curlDataTransfer : public DataTransfer
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            dataTransferSettings.httpConnections.get());
+            fileTransferSettings.httpConnections.get());
         #endif
 
         wakeupPipe.create();
@@ -478,7 +478,7 @@ struct curlDataTransfer : public DataTransfer
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
-    ~curlDataTransfer()
+    ~curlFileTransfer()
     {
         stopWorkerThread();
 
@@ -641,8 +641,8 @@ struct curlDataTransfer : public DataTransfer
     }
 #endif
 
-    void enqueueDataTransfer(const DataTransferRequest & request,
-        Callback<DataTransferResult> callback) override
+    void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
@@ -660,9 +660,9 @@ struct curlDataTransfer : public DataTransfer
 
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
-                DataTransferResult res;
+                FileTransferResult res;
                 if (!s3Res.data)
-                    throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+                    throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
@@ -676,22 +676,22 @@ struct curlDataTransfer : public DataTransfer
     }
 };
 
-ref<DataTransfer> getDataTransfer()
+ref<FileTransfer> getFileTransfer()
 {
-    static ref<DataTransfer> dataTransfer = makeDataTransfer();
-    return dataTransfer;
+    static ref<FileTransfer> fileTransfer = makeFileTransfer();
+    return fileTransfer;
 }
 
-ref<DataTransfer> makeDataTransfer()
+ref<FileTransfer> makeFileTransfer()
 {
-    return make_ref<curlDataTransfer>();
+    return make_ref<curlFileTransfer>();
 }
 
-std::future<DataTransferResult> DataTransfer::enqueueDataTransfer(const DataTransferRequest & request)
+std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
 {
-    auto promise = std::make_shared<std::promise<DataTransferResult>>();
-    enqueueDataTransfer(request,
-        {[promise](std::future<DataTransferResult> fut) {
+    auto promise = std::make_shared<std::promise<FileTransferResult>>();
+    enqueueFileTransfer(request,
+        {[promise](std::future<FileTransferResult> fut) {
             try {
                 promise->set_value(fut.get());
             } catch (...) {
@@ -701,21 +701,21 @@ std::future<DataTransferResult> DataTransfer::enqueueDataTransfer(const DataTran
     return promise->get_future();
 }
 
-DataTransferResult DataTransfer::download(const DataTransferRequest & request)
+FileTransferResult FileTransfer::download(const FileTransferRequest & request)
 {
-    return enqueueDataTransfer(request).get();
+    return enqueueFileTransfer(request).get();
 }
 
-DataTransferResult DataTransfer::upload(const DataTransferRequest & request)
+FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
 {
     /* Note: this method is the same as download, but helps in readability */
-    return enqueueDataTransfer(request).get();
+    return enqueueFileTransfer(request).get();
 }
 
-void DataTransfer::download(DataTransferRequest && request, Sink & sink)
+void FileTransfer::download(FileTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
-       that would cause the sink to execute on the dataTransfer
+       that would cause the sink to execute on the fileTransfer
        thread. If 'sink' is a coroutine, this will fail. Also, if the
        sink is expensive (e.g. one that does decompression and writing
        to the Nix store), it would stall the download thread too much.
@@ -761,8 +761,8 @@ void DataTransfer::download(DataTransferRequest && request, Sink & sink)
         state->avail.notify_one();
     };
 
-    enqueueDataTransfer(request,
-        {[_state](std::future<DataTransferResult> fut) {
+    enqueueFileTransfer(request,
+        {[_state](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
             try {
diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh
index 68d97ceb111..2347f363d69 100644
--- a/src/libstore/datatransfer.hh
+++ b/src/libstore/datatransfer.hh
@@ -9,7 +9,7 @@
 
 namespace nix {
 
-struct DataTransferSettings : Config
+struct FileTransferSettings : Config
 {
     Setting<bool> enableHttp2{this, true, "http2",
         "Whether to enable HTTP/2 support."};
@@ -31,15 +31,15 @@ struct DataTransferSettings : Config
         "How often Nix will attempt to download a file before giving up."};
 };
 
-extern DataTransferSettings dataTransferSettings;
+extern FileTransferSettings fileTransferSettings;
 
-struct DataTransferRequest
+struct FileTransferRequest
 {
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = dataTransferSettings.tries;
+    size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
@@ -47,7 +47,7 @@ struct DataTransferRequest
     std::string mimeType;
     std::function<void(char *, size_t)> dataCallback;
 
-    DataTransferRequest(const std::string & uri)
+    FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()
@@ -56,7 +56,7 @@ struct DataTransferRequest
     }
 };
 
-struct DataTransferResult
+struct FileTransferResult
 {
     bool cached = false;
     std::string etag;
@@ -67,43 +67,43 @@ struct DataTransferResult
 
 class Store;
 
-struct DataTransfer
+struct FileTransfer
 {
-    virtual ~DataTransfer() { }
+    virtual ~FileTransfer() { }
 
     /* Enqueue a data transfer request, returning a future to the result of
-       the download. The future may throw a DataTransferError
+       the download. The future may throw a FileTransferError
        exception. */
-    virtual void enqueueDataTransfer(const DataTransferRequest & request,
-        Callback<DataTransferResult> callback) = 0;
+    virtual void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) = 0;
 
-    std::future<DataTransferResult> enqueueDataTransfer(const DataTransferRequest & request);
+    std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 
     /* Synchronously download a file. */
-    DataTransferResult download(const DataTransferRequest & request);
+    FileTransferResult download(const FileTransferRequest & request);
 
     /* Synchronously upload a file. */
-    DataTransferResult upload(const DataTransferRequest & request);
+    FileTransferResult upload(const FileTransferRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
-    void download(DataTransferRequest && request, Sink & sink);
+    void download(FileTransferRequest && request, Sink & sink);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
 
-/* Return a shared DataTransfer object. Using this object is preferred
+/* Return a shared FileTransfer object. Using this object is preferred
    because it enables connection reuse and HTTP/2 multiplexing. */
-ref<DataTransfer> getDataTransfer();
+ref<FileTransfer> getFileTransfer();
 
-/* Return a new DataTransfer object. */
-ref<DataTransfer> makeDataTransfer();
+/* Return a new FileTransfer object. */
+ref<FileTransfer> makeFileTransfer();
 
-class DataTransferError : public Error
+class FileTransferError : public Error
 {
 public:
-    DataTransfer::Error error;
-    DataTransferError(DataTransfer::Error error, const FormatOrString & fs)
+    FileTransfer::Error error;
+    FileTransferError(FileTransfer::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
 };
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index c75846bc3f4..ce0c5af3bc6 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -85,14 +85,14 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         checkEnabled();
 
         try {
-            DataTransferRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(cacheUri + "/" + path);
             request.head = true;
-            getDataTransfer()->download(request);
+            getFileTransfer()->download(request);
             return true;
-        } catch (DataTransferError & e) {
+        } catch (FileTransferError & e) {
             /* S3 buckets return 403 if a file doesn't exist and the
                bucket is unlistable, so treat 403 as 404. */
-            if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 return false;
             maybeDisable();
             throw;
@@ -103,19 +103,19 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         const std::string & data,
         const std::string & mimeType) override
     {
-        auto req = DataTransferRequest(cacheUri + "/" + path);
+        auto req = FileTransferRequest(cacheUri + "/" + path);
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
-            getDataTransfer()->upload(req);
-        } catch (DataTransferError & e) {
+            getFileTransfer()->upload(req);
+        } catch (FileTransferError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
 
-    DataTransferRequest makeRequest(const std::string & path)
+    FileTransferRequest makeRequest(const std::string & path)
     {
-        DataTransferRequest request(cacheUri + "/" + path);
+        FileTransferRequest request(cacheUri + "/" + path);
         return request;
     }
 
@@ -124,9 +124,9 @@ class HttpBinaryCacheStore : public BinaryCacheStore
         checkEnabled();
         auto request(makeRequest(path));
         try {
-            getDataTransfer()->download(std::move(request), sink);
-        } catch (DataTransferError & e) {
-            if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
+            getFileTransfer()->download(std::move(request), sink);
+        } catch (FileTransferError & e) {
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
             maybeDisable();
             throw;
@@ -142,12 +142,12 @@ class HttpBinaryCacheStore : public BinaryCacheStore
 
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        getDataTransfer()->enqueueDataTransfer(request,
-            {[callbackPtr, this](std::future<DataTransferResult> result) {
+        getFileTransfer()->enqueueFileTransfer(request,
+            {[callbackPtr, this](std::future<FileTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
-                } catch (DataTransferError & e) {
-                    if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden)
+                } catch (FileTransferError & e) {
+                    if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                         return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
                     callbackPtr->rethrow();
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 478cb9f84a3..fccf010a431 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -132,7 +132,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
     return res;
 }
 
-S3Helper::DataTransferResult S3Helper::getObject(
+S3Helper::FileTransferResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
     debug("fetching 's3://%s/%s'...", bucketName, key);
@@ -146,7 +146,7 @@ S3Helper::DataTransferResult S3Helper::getObject(
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
-    DataTransferResult res;
+    FileTransferResult res;
 
     auto now1 = std::chrono::steady_clock::now();
 
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index d7d30924319..2042bffcf94 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -18,13 +18,13 @@ struct S3Helper
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    struct DataTransferResult
+    struct FileTransferResult
     {
         std::shared_ptr<std::string> data;
         unsigned int durationMs;
     };
 
-    DataTransferResult getObject(
+    FileTransferResult getObject(
         const std::string & bucketName, const std::string & key);
 };
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 001a3fb281d..bb437cf1c65 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg,
 
         if (action == "start") {
             auto type = (ActivityType) json["type"];
-            if (trusted || type == actDataTransfer)
+            if (trusted || type == actFileTransfer)
                 activities.emplace(std::piecewise_construct,
                     std::forward_as_tuple(json["id"]),
                     std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 423ed95daa1..108b5dcb082 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -17,7 +17,7 @@ typedef enum {
 typedef enum {
     actUnknown = 0,
     actCopyPath = 100,
-    actDataTransfer = 101,
+    actFileTransfer = 101,
     actRealise = 102,
     actCopyPaths = 103,
     actBuilds = 104,
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 4c81e7d82dd..e136bb95ad2 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -180,9 +180,9 @@ static int _main(int argc, char * * argv)
 
                 FdSink sink(fd.get());
 
-                DataTransferRequest req(actualUri);
+                FileTransferRequest req(actualUri);
                 req.decompress = false;
-                getDataTransfer()->download(std::move(req), sink);
+                getFileTransfer()->download(std::move(req), sink);
             }
 
             /* Optionally unpack the file. */
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 6bb04d8b609..140c963e087 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv)
             settings.useSubstitutes = false;
         if (!settings.tarballTtl.overriden)
             settings.tarballTtl = std::numeric_limits<unsigned int>::max();
-        if (!dataTransferSettings.tries.overriden)
-            dataTransferSettings.tries = 0;
-        if (!dataTransferSettings.connectTimeout.overriden)
-            dataTransferSettings.connectTimeout = 1;
+        if (!fileTransferSettings.tries.overriden)
+            fileTransferSettings.tries = 0;
+        if (!fileTransferSettings.connectTimeout.overriden)
+            fileTransferSettings.connectTimeout = 1;
     }
 
     if (args.refresh)
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index d108896ad44..adc9b9a5dca 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -190,8 +190,8 @@ class ProgressBar : public Logger
             i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
         }
 
-        if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent))
-            || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent))
+        if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent))
+            || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent))
             || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
             i->visible = false;
 
@@ -416,7 +416,7 @@ class ProgressBar : public Logger
             if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
         }
 
-        showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB);
+        showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
 
         {
             auto s = renderActivity(actOptimiseStore, "%s paths optimised");
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index a4eda90b416..574f7d8c27e 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -138,8 +138,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version");
 
         // FIXME: use nixos.org?
-        auto req = DataTransferRequest(storePathsUrl);
-        auto res = getDataTransfer()->download(req);
+        auto req = FileTransferRequest(storePathsUrl);
+        auto res = getFileTransfer()->download(req);
 
         auto state = std::make_unique<EvalState>(Strings(), store);
         auto v = state->allocValue();

From f5095594e723d02c9bf22fe24ad838ebff4f9165 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Mon, 6 Apr 2020 23:57:28 +0200
Subject: [PATCH 1965/2196] datatransfer.{cc,hh} -> filetransfer.{cc,hh}

---
 src/libexpr/common-eval-args.cc                   | 2 +-
 src/libexpr/eval.cc                               | 2 +-
 src/libexpr/parser.y                              | 2 +-
 src/libstore/build.cc                             | 2 +-
 src/libstore/builtins/fetchurl.cc                 | 2 +-
 src/libstore/{datatransfer.cc => filetransfer.cc} | 2 +-
 src/libstore/{datatransfer.hh => filetransfer.hh} | 0
 src/libstore/http-binary-cache-store.cc           | 2 +-
 src/libstore/s3-binary-cache-store.cc             | 2 +-
 src/nix-channel/nix-channel.cc                    | 2 +-
 src/nix-prefetch-url/nix-prefetch-url.cc          | 2 +-
 src/nix/main.cc                                   | 2 +-
 src/nix/upgrade-nix.cc                            | 2 +-
 13 files changed, 12 insertions(+), 12 deletions(-)
 rename src/libstore/{datatransfer.cc => filetransfer.cc} (99%)
 rename src/libstore/{datatransfer.hh => filetransfer.hh} (100%)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 26460601d15..10c9c16bbf7 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -1,6 +1,6 @@
 #include "common-eval-args.hh"
 #include "shared.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7a20c6fa616..b91a021b40c 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -5,7 +5,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 8fbe59301ab..235ed5b1d76 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -544,7 +544,7 @@ formal
 #include <unistd.h>
 
 #include "eval.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "fetchers.hh"
 #include "store-api.hh"
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9baa2cf1aec..0febb8dfbdd 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -7,7 +7,7 @@
 #include "affinity.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 1432de9d704..486babf1427 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -1,5 +1,5 @@
 #include "builtins.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "compression.hh"
diff --git a/src/libstore/datatransfer.cc b/src/libstore/filetransfer.cc
similarity index 99%
rename from src/libstore/datatransfer.cc
rename to src/libstore/filetransfer.cc
index 5cc63a1d8c7..e9684b3d495 100644
--- a/src/libstore/datatransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -1,4 +1,4 @@
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "globals.hh"
 #include "store-api.hh"
diff --git a/src/libstore/datatransfer.hh b/src/libstore/filetransfer.hh
similarity index 100%
rename from src/libstore/datatransfer.hh
rename to src/libstore/filetransfer.hh
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index ce0c5af3bc6..451a647859c 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -1,5 +1,5 @@
 #include "binary-cache-store.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index fccf010a431..b24e7b7d621 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -6,7 +6,7 @@
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
 #include "compression.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 0f8024d7bb9..7d584f891bb 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -1,6 +1,6 @@
 #include "shared.hh"
 #include "globals.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "store-api.hh"
 #include "../nix/legacy.hh"
 #include "fetchers.hh"
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index e136bb95ad2..748554b9ca7 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -1,6 +1,6 @@
 #include "hash.hh"
 #include "shared.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 140c963e087..2c64c74767c 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -8,7 +8,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "finally.hh"
 
 #include <sys/types.h>
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 574f7d8c27e..4fcc6a7210b 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,7 +1,7 @@
 #include "command.hh"
 #include "common-args.hh"
 #include "store-api.hh"
-#include "datatransfer.hh"
+#include "filetransfer.hh"
 #include "eval.hh"
 #include "attr-path.hh"
 #include "names.hh"

From 7867685dcdf1ba3c9290b777e0975ff0b775f667 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Wed, 8 Apr 2020 14:12:22 +0200
Subject: [PATCH 1966/2196] after flake rebase

---
 src/libexpr/parser.y             |  2 +-
 src/libexpr/primops/fetchTree.cc |  2 +-
 src/libfetchers/github.cc        |  2 +-
 src/libfetchers/tarball.cc       | 10 +++++-----
 src/nix-channel/nix-channel.cc   |  2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 235ed5b1d76..b33ab908b9f 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -690,7 +690,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
                         store, resolveUri(elem.second), "source", false).storePath) };
-        } catch (DataTransferError & e) {
+        } catch (FileTransferError & e) {
             printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
             res = { false, "" };
         }
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 9586f71ed5a..43c58485ae4 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -2,7 +2,7 @@
 #include "eval-inline.hh"
 #include "store-api.hh"
 #include "fetchers.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 
 #include <ctime>
 #include <iomanip>
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index ef27eaa76c6..8675a5a662d 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -1,4 +1,4 @@
-#include "download.hh"
+#include "filetransfer.hh"
 #include "cache.hh"
 #include "fetchers.hh"
 #include "globals.hh"
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 4c4e5828e74..695525b3108 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -1,6 +1,6 @@
 #include "fetchers.hh"
 #include "cache.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "archive.hh"
@@ -36,13 +36,13 @@ DownloadFileResult downloadFile(
     if (cached && !cached->expired)
         return useCached();
 
-    DownloadRequest request(url);
+    FileTransferRequest request(url);
     if (cached)
         request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
-    DownloadResult res;
+    FileTransferResult res;
     try {
-        res = getDownloader()->download(request);
-    } catch (DownloadError & e) {
+        res = getFileTransfer()->download(request);
+    } catch (FileTransferError & e) {
         if (cached) {
             warn("%s; using cached version", e.msg());
             return useCached();
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 7d584f891bb..abd39041483 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -113,7 +113,7 @@ static void update(const StringSet & channelNames)
             // Download the channel tarball.
             try {
                 filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath);
-            } catch (DataTransferError & e) {
+            } catch (FileTransferError & e) {
                 filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath);
             }
         }

From a693a9fa4b92df0d3283a9de8b1e968378a026d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 9 Apr 2020 09:45:15 +0200
Subject: [PATCH 1967/2196] Attach pos to if expression errors

---
 src/libexpr/eval.cc    | 2 +-
 src/libexpr/nixexpr.hh | 3 ++-
 src/libexpr/parser.y   | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b91a021b40c..f963a42ca10 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1256,7 +1256,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
-    (state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
 }
 
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 8c96de37cbe..25798cac66e 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -262,8 +262,9 @@ struct ExprWith : Expr
 
 struct ExprIf : Expr
 {
+    Pos pos;
     Expr * cond, * then, * else_;
-    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
 };
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index b33ab908b9f..1993fa6c103 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -335,7 +335,7 @@ expr_function
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
   | expr_op
   ;
 

From a364b1551ae8724657369d68a09626c3cbef25d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 9 Apr 2020 15:03:05 +0200
Subject: [PATCH 1968/2196] remote .travis.yml as we migrated to github actions

---
 .travis.yml | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index ee4ea1ac65a..00000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-matrix:
-  include:
-    - language: osx
-      script: ./tests/install-darwin.sh
-    - language: nix
-      script: nix-build release.nix -A build.x86_64-linux
-notifications:
-  email: false

From 30d4618cc944a41c2ca202babd0be0da4a1cd9d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 9 Apr 2020 15:04:51 +0200
Subject: [PATCH 1969/2196] README: add github actions badge

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 61054f8f2bc..9c0c8788742 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
+[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
 Nix, the purely functional package manager
 ------------------------------------------

From 04bedda0b605dcecacb23b65d5a05c8f4aecfbdd Mon Sep 17 00:00:00 2001
From: Philipp Middendorf <middendorf@plapadoo.de>
Date: Thu, 9 Apr 2020 17:05:29 +0200
Subject: [PATCH 1970/2196] gc.cc: Ignore hidden files in temproots

---
 src/libstore/gc.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0c3d896112c..6bab1e37c90 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
+        if (i.name[0] == '.') {
+            // Ignore hidden files. Some package managers (notably portage) create
+            // those to keep the directory alive.
+            continue;
+        }
         Path path = tempRootsDir + "/" + i.name;
 
         pid_t pid = std::stoi(i.name);

From 16a4864759a80aa03b9c87b8aeaf7a4c31c03e39 Mon Sep 17 00:00:00 2001
From: Bruce Toll <4109762+tollb@users.noreply.github.com>
Date: Sun, 17 Feb 2019 16:26:49 -0500
Subject: [PATCH 1971/2196] Delete temporary directory on successful build

With --check and the --keep-failed (-K) flag, the temporary directory
was being retained regardless of whether the build was successful and
reproducible.  This removes the temporary directory, as expected, on
a reproducible check build.

Added tests to verify that temporary build directories are not
retained unnecessarily, particularly when using --check with
--keep-failed.
---
 src/libstore/build.cc |  1 +
 tests/check.nix       | 33 +++++++++++++++++++++++++++
 tests/check.sh        | 52 +++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0febb8dfbdd..760663ac978 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1680,6 +1680,7 @@ void DerivationGoal::buildDone()
         }
 
         if (buildMode == bmCheck) {
+            deleteTmpDir(true);
             done(BuildResult::Built);
             return;
         }
diff --git a/tests/check.nix b/tests/check.nix
index 56c82e565a8..bca04fdafa1 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -1,12 +1,45 @@
+{checkBuildId ? 0}:
+
 with import ./config.nix;
 
 {
   nondeterministic = mkDerivation {
+    inherit checkBuildId;
     name = "nondeterministic";
     buildCommand =
       ''
         mkdir $out
         date +%s.%N > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  deterministic = mkDerivation {
+    inherit checkBuildId;
+    name = "deterministic";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  failed = mkDerivation {
+    inherit checkBuildId;
+    name = "failed";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+        false
       '';
   };
 
diff --git a/tests/check.sh b/tests/check.sh
index bc23a6634ca..b423dc0b51c 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -1,14 +1,62 @@
 source common.sh
 
+checkBuildTempDirRemoved ()
+{
+    buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
+    checkBuildIdFile=${buildDir}/checkBuildId
+    [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile
+}
+
+# written to build temp directories to verify created by this instance
+checkBuildId=$(date +%s%N)
+
 clearStore
 
 nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
-nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
+# check for dangling temporary build directories
+# only retain if build fails and --keep-failed is specified, or...
+# ...build is non-deterministic and --check and --keep-failed are both specified
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log
+if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check 2> $TEST_ROOT/log || status=$?
+grep 'may not be deterministic' $TEST_ROOT/log
+[ "$status" = "104" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
+
+# The above nix-build fails with status=1 on darwin (not sure why)
+# ...but the primary purpose of the test case is to verify the temp directory is retained
+if [ "$(uname -s)" != "Darwin" ]; then
 grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
+fi
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
 clearStore
 

From 3abf6d03c611417b309fdedf4323c08e6afbcd9c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 10 Apr 2020 17:27:35 +0200
Subject: [PATCH 1972/2196] Update release script

---
 maintainers/upload-release.pl | 86 ++++++++++++++++++++++-------------
 1 file changed, 55 insertions(+), 31 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index 77534babba2..cb584d4278b 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1
+#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
 
 use strict;
 use Data::Dumper;
@@ -9,12 +9,16 @@
 use File::Copy;
 use JSON::PP;
 use LWP::UserAgent;
+use Net::Amazon::S3;
 
 my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
 
-my $releasesDir = "/home/eelco/mnt/releases";
+my $releasesBucketName = "nix-releases";
+my $channelsBucketName = "nix-channels";
 my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
+my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -42,13 +46,31 @@ sub fetch {
 
 print STDERR "Nix revision is $nixRev, version is $version\n";
 
-File::Path::make_path($releasesDir);
-if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
-}
+my $releaseDir = "nix/$releaseName";
+
+my $tmpDir = "$TMPDIR/nix-release/$releaseName";
+File::Path::make_path($tmpDir);
+
+# S3 setup.
+my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
+my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
+
+my $s3 = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+      host                  => "s3-eu-west-1.amazonaws.com",
+    });
+
+my $releasesBucket = $s3->bucket($releasesBucketName) or die;
+
+my $s3_us = Net::Amazon::S3->new(
+    { aws_access_key_id     => $aws_access_key_id,
+      aws_secret_access_key => $aws_secret_access_key,
+      retry                 => 1,
+    });
 
-my $releaseDir = "$releasesDir/nix/$releaseName";
-File::Path::make_path($releaseDir);
+my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -57,40 +79,49 @@ sub downloadFile {
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
-    my $dstFile = "$releaseDir/" . $dstName;
+    my $tmpFile = "$tmpDir/$dstName";
 
-    if (! -e $dstFile) {
-        print STDERR "downloading $srcFile to $dstFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
+    if (!-e $tmpFile) {
+        print STDERR "downloading $srcFile to $tmpFile...\n";
+        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
-        rename("$dstFile.tmp", $dstFile) or die;
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
+        print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$dstFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_expected);
 
-    if (! -e "$dstFile.asc") {
-        system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n";
+    if (! -e "$tmpFile.asc") {
+        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
     }
 
-    return ($dstFile, $sha256_expected);
+    return $sha256_expected;
 }
 
 downloadFile("tarball", "2"); # .tar.bz2
-my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz
+my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("installerScript", "1");
 
+for my $fn (glob "$tmpDir/*") {
+    my $name = basename($fn);
+    my $dstKey = "$releaseDir/" . $name;
+    unless (defined $releasesBucket->head_key($dstKey)) {
+        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+        $releasesBucket->add_key_filename($dstKey, $fn)
+            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
+    }
+}
+
 exit if $version =~ /pre/;
 
 # Update Nixpkgs in a very hacky way.
@@ -125,18 +156,11 @@ sub getStorePath {
 
 system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
 
-# Extract the HTML manual.
-File::Path::make_path("$releaseDir/manual");
-
-system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
-
-if (! -e "$releaseDir/manual/index.html") {
-    symlink("manual.html", "$releaseDir/manual/index.html") or die;
-}
-
 # Update the "latest" symlink.
-symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die;
-rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die;
+$channelsBucket->add_key(
+    "nix-latest/install", "",
+    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;

From 8132d0a12e1b1d30973ae2c543622a46c24ec075 Mon Sep 17 00:00:00 2001
From: Bruce Toll <4109762+tollb@users.noreply.github.com>
Date: Sun, 17 Feb 2019 14:34:31 -0500
Subject: [PATCH 1973/2196] Fix nix-build --check -K in sandbox w/o root

Temporarily add user-write permission to build directory so that it
can be moved out of the sandbox to the store with a .check suffix.

This is necessary because the build directory has already had its
permissions set read-only, but write permission is required
to update the directory's parent link to move it out of the sandbox.

Updated the related --check "derivation may not be deterministic"
messages to consistently use the real store paths.

Added test for non-root sandbox nix-build --check -K to demonstrate
issue and help prevent regressions.
---
 src/libstore/build.cc  | 30 ++++++++++++++++++++++++++----
 tests/linux-sandbox.sh |  7 +++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 760663ac978..b4207e1b818 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3537,6 +3537,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
 }
 
 
+static void moveCheckToStore(const Path & src, const Path & dst)
+{
+    /* For the rename of directory to succeed, we must be running as root or
+       the directory must be made temporarily writable (to update the
+       directory's parent link ".."). */
+    struct stat st;
+    if (lstat(src.c_str(), &st) == -1) {
+        throw SysError(format("getting attributes of path '%1%'") % src);
+    }
+
+    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+    if (changePerm)
+        chmod_(src, st.st_mode | S_IWUSR);
+
+    if (rename(src.c_str(), dst.c_str()))
+        throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
+
+    if (changePerm)
+        chmod_(dst, st.st_mode);
+}
+
+
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
@@ -3715,8 +3738,7 @@ void DerivationGoal::registerOutputs()
                 if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
-                    if (rename(actualPath.c_str(), dst.c_str()))
-                        throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+                    moveCheckToStore(actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
@@ -3724,10 +3746,10 @@ void DerivationGoal::registerOutputs()
                         path, dst, worker.store.printStorePath(drvPath), tmpDir);
 
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), path, dst);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
                 } else
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), path);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
             }
 
             /* Since we verified the build, it's now ultimately trusted. */
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 52967d07dda..16abd974c5e 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
 nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
+
+# Test that sandboxed builds with --check and -K can move .check directory to store
+nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
+
+(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
+if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
+grep -q 'may not be deterministic' $TEST_ROOT/log

From e8bd1bc732ff19f33e6915d14e15df1d7de612c0 Mon Sep 17 00:00:00 2001
From: Bruce Toll <4109762+tollb@users.noreply.github.com>
Date: Fri, 10 Apr 2020 18:20:12 -0400
Subject: [PATCH 1974/2196] Add test case for temporary directories on darwin

A test case for correct handling of temporary directory deletion that
was added to check.sh as part of PR #2689 was initially disabled for
Darwin because of a directory permission issue in PR #2688.

Now that the issue in PR #2688 is fixed, this commit enables the test
case for Darwin.
---
 tests/check.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/check.sh b/tests/check.sh
index b423dc0b51c..5f25d04cb2c 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -49,13 +49,8 @@ checkBuildTempDirRemoved $TEST_ROOT/log
 
 nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
     --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
-
-# The above nix-build fails with status=1 on darwin (not sure why)
-# ...but the primary purpose of the test case is to verify the temp directory is retained
-if [ "$(uname -s)" != "Darwin" ]; then
 grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
-fi
 if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
 clearStore

From fc14213d2da7302b237dfe5ca51f6f4d9af34e5c Mon Sep 17 00:00:00 2001
From: DavHau <42246742+DavHau@users.noreply.github.com>
Date: Wed, 25 Mar 2020 07:19:00 +0000
Subject: [PATCH 1975/2196] improve toFile error message when containing
 potential drv path

---
 src/libexpr/primops.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8de23495187..6d7622900a7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1021,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
+            throw EvalError(format(
+                "in 'toFile': the file named '%1%' must not contain a reference "
+                "to a derivation but contains (%2%), at %3%") % name % path % pos);
         refs.insert(state.store->parseStorePath(path));
     }
 

From 4d9db420ffc9bd48da107a61c093b0d65d9d8db1 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 12 Apr 2020 09:57:22 +0200
Subject: [PATCH 1976/2196] never use /var/folders for TMPDIR on darwin

This doesn't just cause problems for nix-store --serve but also results
in certain build failures. Builds that use unix domain sockets in their
tests often fail because the /var/folders prefix already consumes more
than half of the maximum length of socket paths.

    struct sockaddr_un {
       sa_family_t sun_family;               /* AF_UNIX */
       char        sun_path[108];            /* Pathname */
    };
---
 src/libmain/shared.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d41e772e9a7..72ba717e51a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -155,7 +155,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }

From d2c371927e45c1d4e4f7e4ad331a3f68d67d5469 Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Mon, 13 Apr 2020 21:09:47 +0200
Subject: [PATCH 1977/2196] SourceExprCommand: allocate the vSourceExpr via
 uncollectable memory

Previously the memory would occasionally be collected during eval since
the GC doesn't consider the member variable as alive / doesn't scan the
region of memory where the pointer lives.

By using the traceable_allocator<T> allocator provided by Boehm GC we
can ensure the memory isn't collected. It should be properly freed when
SourceExprCommand goes out of scope.
---
 src/nix/command.hh      |  2 +-
 src/nix/installables.cc | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/nix/command.hh b/src/nix/command.hh
index 23f5c989864..2c2303208ee 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -41,7 +41,7 @@ private:
 
     std::shared_ptr<EvalState> evalState;
 
-    Value * vSourceExpr = 0;
+    std::shared_ptr<Value> vSourceExpr;
 };
 
 enum RealiseMode { Build, NoBuild, DryRun };
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index f464d0aa1ed..902383bff07 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -8,10 +8,13 @@
 #include "store-api.hh"
 #include "shared.hh"
 
+#include <gc/gc.h>
+
 #include <regex>
 
 namespace nix {
 
+
 SourceExprCommand::SourceExprCommand()
 {
     mkFlag()
@@ -24,11 +27,14 @@ SourceExprCommand::SourceExprCommand()
 
 Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
-    if (vSourceExpr) return vSourceExpr;
+    if (vSourceExpr) return vSourceExpr.get();
 
     auto sToplevel = state.symbols.create("_toplevel");
 
-    vSourceExpr = state.allocValue();
+    // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't
+    // consider the member variable "alive" during execution causing it to be
+    // GC'ed in the middle of evaluation.
+    vSourceExpr = std::allocate_shared<Value>(traceable_allocator<Value>());
 
     if (file != "")
         state.evalFile(lookupFileArg(state, file), *vSourceExpr);
@@ -69,7 +75,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
         vSourceExpr->attrs->sort();
     }
 
-    return vSourceExpr;
+    return vSourceExpr.get();
 }
 
 ref<EvalState> SourceExprCommand::getEvalState()

From 895516cadf0739dd8359a38d503e0aaf5068dc4b Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Mon, 30 Mar 2020 15:31:14 +0200
Subject: [PATCH 1978/2196] add NIX_USER_CONF_FILES

Motivation: maintain project-level configuration files.

Document the whole situation a bit better so that it corresponds to the
implementation, and add NIX_USER_CONF_FILES that allows overriding
which user files Nix will load during startup.
---
 doc/manual/command-ref/conf-file.xml    | 36 ++++++++++++++-----------
 doc/manual/command-ref/env-common.xml   | 13 ++++++---
 src/libmain/shared.cc                   |  5 +++-
 src/libstore/globals.cc                 | 23 +++++++++++++---
 src/libstore/globals.hh                 |  8 +++++-
 src/libutil/util.hh                     |  1 +
 tests/common.sh.in                      |  3 +++
 tests/config.sh                         | 18 +++++++++++++
 tests/config/nix-with-substituters.conf |  2 ++
 tests/local.mk                          |  1 +
 10 files changed, 86 insertions(+), 24 deletions(-)
 create mode 100644 tests/config.sh
 create mode 100644 tests/config/nix-with-substituters.conf

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 48dce7c950d..1820598e51d 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -19,26 +19,30 @@
 
 <refsection><title>Description</title>
 
-<para>Nix reads settings from two configuration files:</para>
+<para>By default Nix reads settings from the following places:</para>
+
+<para>The system-wide configuration file
+<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
+(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
+<filename>$NIX_CONF_DIR/nix.conf</filename> if
+<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
+client assumes that the daemon has already loaded them.
+</para>
 
-<itemizedlist>
+<para>User-specific configuration files:</para>
 
-  <listitem>
-    <para>The system-wide configuration file
-    <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
-    (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
-    <filename>$NIX_CONF_DIR/nix.conf</filename> if
-    <envar>NIX_CONF_DIR</envar> is set.</para>
-  </listitem>
+<para>
+  If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
+  <literal>:</literal> will be loaded in reverse order.
+</para>
 
-  <listitem>
-    <para>The user configuration file
-    <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or
-    <filename>~/.config/nix/nix.conf</filename> if
-    <envar>XDG_CONFIG_HOME</envar> is not set.</para>
-  </listitem>
+<para>
+  Otherwise it will look for <filename>nix/nix.conf</filename> files in
+  <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
 
-</itemizedlist>
+  The default location is <filename>$HOME/.config/nix.conf</filename> if
+  those environment variables are unset.
+</para>
 
 <para>The configuration files consist of
 <literal><replaceable>name</replaceable> =
diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 696d68c3451..0217de7b2ab 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -33,7 +33,7 @@
 
     will cause Nix to look for paths relative to
     <filename>/home/eelco/Dev</filename> and
-    <filename>/etc/nixos</filename>, in that order.  It is also
+    <filename>/etc/nixos</filename>, in this order.  It is also
     possible to match paths against a prefix.  For example, the value
 
     <screen>
@@ -59,7 +59,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</sc
     15.09 channel.</para>
 
     <para>A following shorthand can be used to refer to the official channels:
-    
+
     <screen>nixpkgs=channel:nixos-15.09</screen>
     </para>
 
@@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
 
-  <listitem><para>Overrides the location of the Nix configuration
+  <listitem><para>Overrides the location of the system Nix configuration
   directory (default
   <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
 
 </varlistentry>
 
+<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
+
+  <listitem><para>Overrides the location of the user Nix configuration files
+  to load from (defaults to the XDG spec locations). The variable is treated
+  as a list separated by the <literal>:</literal> token.</para></listitem>
+
+</varlistentry>
 
 <varlistentry><term><envar>TMPDIR</envar></term>
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d41e772e9a7..3c141a5faa2 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -260,7 +260,10 @@ void printVersion(const string & programName)
         cfg.push_back("signed-caches");
 #endif
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
-        std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
+        std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
+        std::cout << "User configuration files: " <<
+            concatStringsSep(":", settings.nixUserConfFiles)
+            << "\n";
         std::cout << "Store directory: " << settings.nixStore << "\n";
         std::cout << "State directory: " << settings.nixStateDir << "\n";
     }
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 7e97f3c2235..d6ea0318ec0 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -31,6 +31,7 @@ Settings::Settings()
     , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
     , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
+    , nixUserConfFiles(getUserConfigFiles())
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
@@ -77,11 +78,27 @@ void loadConfFile()
        ~/.nix/nix.conf or the command line. */
     globalConfig.resetOverriden();
 
+    auto files = settings.nixUserConfFiles;
+    for (auto file = files.rbegin(); file != files.rend(); file++) {
+        globalConfig.applyConfigFile(*file);
+    }
+}
+
+std::vector<Path> getUserConfigFiles()
+{
+    // Use the paths specified in NIX_USER_CONF_FILES if it has been defined
+    auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
+    if (nixConfFiles.has_value()) {
+        return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
+    }
+
+    // Use the paths specified by the XDG spec
+    std::vector<Path> files;
     auto dirs = getConfigDirs();
-    // Iterate over them in reverse so that the ones appearing first in the path take priority
-    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
-        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+    for (auto & dir : dirs) {
+        files.insert(files.end(), dir + "/nix/nix.conf");
     }
+    return files;
 }
 
 unsigned int Settings::getDefaultCores()
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 40f350f0b75..da95fd3ae91 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -53,9 +53,12 @@ public:
     /* The directory where state is stored. */
     Path nixStateDir;
 
-    /* The directory where configuration files are stored. */
+    /* The directory where system configuration files are stored. */
     Path nixConfDir;
 
+    /* A list of user configuration files to load. */
+    std::vector<Path> nixUserConfFiles;
+
     /* The directory where internal helper programs are stored. */
     Path nixLibexecDir;
 
@@ -378,6 +381,9 @@ void initPlugins();
 
 void loadConfFile();
 
+// Used by the Settings constructor
+std::vector<Path> getUserConfigFiles();
+
 extern const string nixVersion;
 
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 1f85c7c462e..815b1f2884f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -16,6 +16,7 @@
 #include <sstream>
 #include <optional>
 #include <future>
+#include <iterator>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 15d7b1ef911..dd7e6182290 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
+unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
+unset XDG_CONFIG_HOME
+unset XDG_CONFIG_DIRS
 unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 
diff --git a/tests/config.sh b/tests/config.sh
new file mode 100644
index 00000000000..8fa349f11a3
--- /dev/null
+++ b/tests/config.sh
@@ -0,0 +1,18 @@
+source common.sh
+
+# Test that files are loaded from XDG by default
+export XDG_CONFIG_HOME=/tmp/home
+export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]]
+
+# Test that setting NIX_USER_CONF_FILES overrides all the default user config files
+export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]]
+
+# Test that it's possible to load the config from a custom location
+here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
+export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
+var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
+[[ $var == https://example.com ]]
diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf
new file mode 100644
index 00000000000..90f359a6fa9
--- /dev/null
+++ b/tests/config/nix-with-substituters.conf
@@ -0,0 +1,2 @@
+experimental-features = nix-command
+substituters = https://example.com
diff --git a/tests/local.mk b/tests/local.mk
index 01fac4fcdb1..c5602773d75 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -1,5 +1,6 @@
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
+  config.sh \
   gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \

From 057e5b6b2e8bf7977c1f57d6f80cc17aa40096d9 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 15 Apr 2020 10:09:43 -0600
Subject: [PATCH 1979/2196] move implementation to cc

---
 src/libutil/error.cc | 5 +++++
 src/libutil/error.hh | 5 +----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 40354cc8795..a5571d4ecb6 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -8,6 +8,11 @@ namespace nix
 
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
+std::ostream& operator<<(std::ostream &os, const hintformat &hf)
+{
+    return os << hf.str();
+}
+
 string showErrPos(const ErrPos &errPos)
 {
     if (errPos.column > 0) {
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 8286eaab11a..5658e6335be 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -92,10 +92,7 @@ private:
     format fmt;
 };
 
-std::ostream& operator<<(std::ostream &os, const hintformat &hf)
-{
-    return os << hf.str();
-}
+std::ostream& operator<<(std::ostream &os, const hintformat &hf);
 
 template<typename... Args>
 inline hintformat hintfmt(const std::string & fs, const Args & ... args)

From b865b5b40c13cfff32e1a0ab685baff1bef5ae5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 16 Apr 2020 12:32:07 +0200
Subject: [PATCH 1980/2196] pass Pos to forceValue to improve infinite
 recursion error

---
 src/libexpr/eval-inline.hh |  4 +--
 src/libexpr/eval.cc        |  6 ++--
 src/libexpr/primops.cc     | 64 +++++++++++++++++++-------------------
 3 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index c27116e3b44..942cda1ea12 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -57,7 +57,7 @@ inline void EvalState::forceAttrs(Value & v)
 
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tAttrs)
         throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
 }
@@ -73,7 +73,7 @@ inline void EvalState::forceList(Value & v)
 
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (!v.isList())
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f963a42ca10..3b4d9bfdcb5 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1502,7 +1502,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tBool)
         throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
     return v.boolean;
@@ -1517,7 +1517,7 @@ bool EvalState::isFunctor(Value & fun)
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
-    forceValue(v);
+    forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
         throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
 }
@@ -1594,7 +1594,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     bool coerceMore, bool copyToStore)
 {
-    forceValue(v);
+    forceValue(v, pos);
 
     string s;
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index fc6c8296b4a..a00b936766b 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -241,7 +241,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
 /* Return a string representing the type of the expression. */
 static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     string t;
     switch (args[0]->type) {
         case tInt: t = "int"; break;
@@ -269,7 +269,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is the null value. */
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tNull);
 }
 
@@ -277,7 +277,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Determine whether the argument is a function. */
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     bool res;
     switch (args[0]->type) {
         case tLambda:
@@ -296,21 +296,21 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args,
 /* Determine whether the argument is an integer. */
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tInt);
 }
 
 /* Determine whether the argument is a float. */
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tFloat);
 }
 
 /* Determine whether the argument is a string. */
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tString);
 }
 
@@ -318,14 +318,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va
 /* Determine whether the argument is a Boolean. */
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tBool);
 }
 
 /* Determine whether the argument is a path. */
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tPath);
 }
 
@@ -382,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
         throw EvalError(format("attribute 'operator' required, at %1%") % pos);
-    state.forceValue(*op->value);
+    state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
        `workSet', adding the result to `workSet', continuing until
@@ -401,7 +401,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
-        state.forceValue(*key->value);
+        state.forceValue(*key->value, pos);
 
         if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
@@ -413,7 +413,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
 
         /* Add the values returned by the operator to the work set. */
         for (unsigned int n = 0; n < call.listSize(); ++n) {
-            state.forceValue(*call.listElems()[n]);
+            state.forceValue(*call.listElems()[n], pos);
             workSet.push_back(call.listElems()[n]);
         }
     }
@@ -445,7 +445,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value
 static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     try {
-        state.forceValue(*args[1]);
+        state.forceValue(*args[1], pos);
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
@@ -461,7 +461,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 {
     state.mkAttrs(v, 2);
     try {
-        state.forceValue(*args[0]);
+        state.forceValue(*args[0], pos);
         v.attrs->push_back(Attr(state.sValue, args[0]));
         mkBool(*state.allocAttr(v, state.symbols.create("success")), true);
     } catch (AssertionError & e) {
@@ -483,8 +483,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu
 /* Evaluate the first argument, then return the second argument. */
 static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -494,7 +494,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value &
 static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValueDeep(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -503,12 +503,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
    return the second expression.  Useful for debugging. */
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type == tString)
         printError(format("trace: %1%") % args[0]->string.s);
     else
         printError(format("trace: %1%") % *args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[1], pos);
     v = *args[1];
 }
 
@@ -599,7 +599,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         try {
 
             if (ignoreNulls) {
-                state.forceValue(*i->value);
+                state.forceValue(*i->value, pos);
                 if (i->value->type == tNull) continue;
             }
 
@@ -1092,7 +1092,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
@@ -1118,7 +1118,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
-            state.forceValue(*attr.value);
+            state.forceValue(*attr.value, pos);
             filterFun = attr.value;
         } else if (n == "recursive")
             recursive = state.forceBool(*attr.value, *attr.pos);
@@ -1189,7 +1189,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
-    state.forceValue(*i->value);
+    state.forceValue(*i->value, pos);
     v = *i->value;
 }
 
@@ -1219,7 +1219,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val
 /* Determine whether the argument is a set. */
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->type == tAttrs);
 }
 
@@ -1345,7 +1345,7 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 */
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
         throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
 
@@ -1391,7 +1391,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
 /* Determine whether the argument is a list. */
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
     mkBool(v, args[0]->isList());
 }
 
@@ -1401,7 +1401,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
-    state.forceValue(*list.listElems()[n]);
+    state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
 }
 
@@ -1524,9 +1524,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
-        state.forceValue(v);
+        state.forceValue(v, pos);
     } else {
-        state.forceValue(*args[1]);
+        state.forceValue(*args[1], pos);
         v = *args[1];
     }
 }
@@ -1591,7 +1591,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     auto len = args[1]->listSize();
     state.mkList(v, len);
     for (unsigned int n = 0; n < len; ++n) {
-        state.forceValue(*args[1]->listElems()[n]);
+        state.forceValue(*args[1]->listElems()[n], pos);
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
@@ -1626,7 +1626,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     for (unsigned int n = 0; n < len; ++n) {
         auto vElem = args[1]->listElems()[n];
-        state.forceValue(*vElem);
+        state.forceValue(*vElem, pos);
         Value res;
         state.callFunction(*args[0], *vElem, res, pos);
         if (state.forceBool(res, pos))
@@ -1757,8 +1757,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu
 
 static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.forceValue(*args[0]);
-    state.forceValue(*args[1]);
+    state.forceValue(*args[0], pos);
+    state.forceValue(*args[1], pos);
     CompareValues comp;
     mkBool(v, comp(args[0], args[1]));
 }

From 1290411c2d0c62dd1761485f55292dc944eae55d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 7 Apr 2020 14:00:12 +0200
Subject: [PATCH 1981/2196] fetchMercurial: Use inputFromAttrs()

---
 src/libexpr/primops/fetchMercurial.cc | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index f18351646e4..0a1ba49d5e4 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -54,15 +54,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     if (evalSettings.pureEval && !rev)
         throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
 
-    auto parsedUrl = parseURL(
-        url.find("://") != std::string::npos
-        ? "hg+" + url
-        : "hg+file://" + url);
-    if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
-    if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
-    // FIXME: use name
-    auto input = fetchers::inputFromURL(parsedUrl);
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "hg");
+    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    if (ref) attrs.insert_or_assign("ref", *ref);
+    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
+    auto input = fetchers::inputFromAttrs(attrs);
 
+    // FIXME: use name
     auto [tree, input2] = input->fetchTree(state.store);
 
     state.mkAttrs(v, 8);

From b3e5eea4a91400fb2a12aba4b07a94d03ba54605 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 16:28:07 +0200
Subject: [PATCH 1982/2196] Add function to allocate a Value in traceable
 memory

---
 src/libexpr/eval.cc  | 8 ++++++++
 src/libexpr/value.hh | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index f963a42ca10..2f5b1db4789 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -22,6 +22,8 @@
 
 #if HAVE_BOEHMGC
 
+#define GC_INCLUDE_NEW
+
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
@@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size)
 }
 
 
+RootValue allocRootValue(Value * v)
+{
+    return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+}
+
+
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 68937387317..9cf2bf06a54 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -261,4 +261,9 @@ typedef std::map<Symbol, Value *> ValueMap;
 #endif
 
 
+/* A value allocated in traceable memory. */
+typedef std::shared_ptr<Value *> RootValue;
+
+RootValue allocRootValue(Value * v);
+
 }

From 10e17eaa5802a3c368eee8408821828072e76ba7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 17:24:28 +0200
Subject: [PATCH 1983/2196] ValueMap, VectorVector: Use traceable_allocator

We want to *trace* the 'Value *' arrays, not garbage-collect them!
Otherwise the vectors/maps can end up pointing to nowhere.

Fixes #3377. Closes #3384.
---
 src/libexpr/value.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 9cf2bf06a54..71025824ef0 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -253,8 +253,8 @@ void mkPath(Value & v, const char * s);
 
 
 #if HAVE_BOEHMGC
-typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
+typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;

From 9f46f54de4e55267df492456fc0393f74616366b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 17:28:32 +0200
Subject: [PATCH 1984/2196] JSONSax: Use a RootValue

More #3377.
---
 src/libexpr/json-to-value.cc | 63 ++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc
index 1fdce198399..76e1a26bff8 100644
--- a/src/libexpr/json-to-value.cc
+++ b/src/libexpr/json-to-value.cc
@@ -4,7 +4,6 @@
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
-using std::unique_ptr;
 
 namespace nix {
 
@@ -13,69 +12,69 @@ namespace nix {
 class JSONSax : nlohmann::json_sax<json> {
     class JSONState {
     protected:
-        unique_ptr<JSONState> parent;
-        Value * v;
+        std::unique_ptr<JSONState> parent;
+        RootValue v;
     public:
-        virtual unique_ptr<JSONState> resolve(EvalState &)
+        virtual std::unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
-        };
-        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
-        explicit JSONState(Value* v) : v(v) {};
-        JSONState(JSONState& p) = delete;
-        Value& value(EvalState & state)
+        }
+        explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
+        explicit JSONState(Value * v) : v(allocRootValue(v)) {}
+        JSONState(JSONState & p) = delete;
+        Value & value(EvalState & state)
         {
-            if (v == nullptr)
-                v = state.allocValue();
-            return *v;
-        };
-        virtual ~JSONState() {};
-        virtual void add() {};
+            if (!v)
+                v = allocRootValue(state.allocValue());
+            return **v;
+        }
+        virtual ~JSONState() {}
+        virtual void add() {}
     };
 
     class JSONObjectState : public JSONState {
         using JSONState::JSONState;
-        ValueMap attrs = ValueMap();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueMap attrs;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
             return std::move(parent);
         }
-        virtual void add() override { v = nullptr; };
+        void add() override { v = nullptr; }
     public:
-        void key(string_t& name, EvalState & state)
+        void key(string_t & name, EvalState & state)
         {
-            attrs[state.symbols.create(name)] = &value(state);
+            attrs.insert_or_assign(state.symbols.create(name), &value(state));
         }
     };
 
     class JSONListState : public JSONState {
-        ValueVector values = ValueVector();
-        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        ValueVector values;
+        std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value& v = parent->value(state);
+            Value & v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
             }
             return std::move(parent);
         }
-        virtual void add() override {
-            values.push_back(v);
+        void add() override {
+            values.push_back(*v);
             v = nullptr;
-        };
+        }
     public:
-        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
     };
 
     EvalState & state;
-    unique_ptr<JSONState> rs;
+    std::unique_ptr<JSONState> rs;
 
     template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
     {
@@ -107,12 +106,12 @@ class JSONSax : nlohmann::json_sax<json> {
         return handle_value(mkInt, val);
     }
 
-    bool number_float(number_float_t val, const string_t& s)
+    bool number_float(number_float_t val, const string_t & s)
     {
         return handle_value(mkFloat, val);
     }
 
-    bool string(string_t& val)
+    bool string(string_t & val)
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
@@ -123,7 +122,7 @@ class JSONSax : nlohmann::json_sax<json> {
         return true;
     }
 
-    bool key(string_t& name)
+    bool key(string_t & name)
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;

From 96262e744e4eef67d808a532b1704fceb62bca5b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 16 Apr 2020 09:55:38 -0600
Subject: [PATCH 1985/2196] switch to structs, which don't need public:

---
 src/libutil/error.hh | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 5658e6335be..417a6ad957e 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -17,9 +17,8 @@ typedef enum {
     elError
 } ErrLevel;
 
-class ErrPos
+struct ErrPos
 {
-public:
     int lineNumber;
     int column;
     string nixFile;
@@ -40,9 +39,8 @@ public:
     }
 };
 
-class NixCode
+struct NixCode
 {
-public:
     ErrPos errPos;
     std::optional<string> prevLineOfCode;
     string errLineOfCode;
@@ -54,9 +52,8 @@ public:
 // are always in yellow.
 
 template <class T>
-class yellowify
+struct yellowify
 {
-public:
     yellowify(T &s) : value(s) {}
     T &value;
 };
@@ -104,9 +101,8 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args)
 
 // -------------------------------------------------
 // ErrorInfo.
-class ErrorInfo
+struct ErrorInfo
 {
-public:
     ErrLevel level;
     string name;
     string description;
@@ -114,8 +110,6 @@ public:
     std::optional<NixCode> nixCode;
 
     static std::optional<string> programName;
-
-private:
 };
 
 // --------------------------------------------------------

From fcd048a526bd239fa615457e77d61d69d679bf03 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 16:54:34 +0200
Subject: [PATCH 1986/2196] Use RootValue

---
 src/libexpr/primops.cc  | 10 +++++-----
 src/nix/command.hh      |  2 +-
 src/nix/installables.cc | 21 ++++++++-------------
 3 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index fc6c8296b4a..a3f2b92cedf 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -121,16 +121,16 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         }
         w.attrs->sort();
 
-        static Value * fun = nullptr;
+        static RootValue fun;
         if (!fun) {
-            fun = state.allocValue();
+            fun = allocRootValue(state.allocValue());
             state.eval(state.parseExprFromString(
                 #include "imported-drv-to-derivation.nix.gen.hh"
-                , "/"), *fun);
+                , "/"), **fun);
         }
 
-        state.forceFunction(*fun, pos);
-        mkApp(v, *fun, w);
+        state.forceFunction(**fun, pos);
+        mkApp(v, **fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 2c2303208ee..bf43d950f6a 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -41,7 +41,7 @@ private:
 
     std::shared_ptr<EvalState> evalState;
 
-    std::shared_ptr<Value> vSourceExpr;
+    RootValue vSourceExpr;
 };
 
 enum RealiseMode { Build, NoBuild, DryRun };
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 902383bff07..1d70ad3d554 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -8,8 +8,6 @@
 #include "store-api.hh"
 #include "shared.hh"
 
-#include <gc/gc.h>
-
 #include <regex>
 
 namespace nix {
@@ -27,17 +25,14 @@ SourceExprCommand::SourceExprCommand()
 
 Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
-    if (vSourceExpr) return vSourceExpr.get();
+    if (vSourceExpr) return *vSourceExpr;
 
     auto sToplevel = state.symbols.create("_toplevel");
 
-    // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't
-    // consider the member variable "alive" during execution causing it to be
-    // GC'ed in the middle of evaluation.
-    vSourceExpr = std::allocate_shared<Value>(traceable_allocator<Value>());
+    vSourceExpr = allocRootValue(state.allocValue());
 
     if (file != "")
-        state.evalFile(lookupFileArg(state, file), *vSourceExpr);
+        state.evalFile(lookupFileArg(state, file), **vSourceExpr);
 
     else {
 
@@ -45,9 +40,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
         auto searchPath = state.getSearchPath();
 
-        state.mkAttrs(*vSourceExpr, 1024);
+        state.mkAttrs(**vSourceExpr, 1024);
 
-        mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true);
+        mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true);
 
         std::unordered_set<std::string> seen;
 
@@ -58,7 +53,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
             mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath"));
             Value * v2 = state.allocValue();
             mkApp(*v2, *v1, mkString(*state.allocValue(), name));
-            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)),
+            mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)),
                 state.getBuiltin("import"), *v2);
         };
 
@@ -72,10 +67,10 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
             } else
                 addEntry(i.first);
 
-        vSourceExpr->attrs->sort();
+        (*vSourceExpr)->attrs->sort();
     }
 
-    return vSourceExpr.get();
+    return *vSourceExpr;
 }
 
 ref<EvalState> SourceExprCommand::getEvalState()

From 67a5941472ab56a798556ee2d5afc62fc38a799a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 13:12:58 +0200
Subject: [PATCH 1987/2196] Logger: Add method for writing to stdout

Usually this just writes to stdout, but for ProgressBar, we need to
clear the current line, write the line to stdout, and then redraw the
progress bar.

(cherry picked from commit 696c026006a6ac46adc990ed5cb0f31535bac076)
---
 src/libutil/logging.cc  |  6 ++++++
 src/libutil/logging.hh  | 10 ++++++++++
 src/nix/progress-bar.cc | 13 +++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index bb437cf1c65..3cc4ef8f15b 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -3,6 +3,7 @@
 
 #include <atomic>
 #include <nlohmann/json.hpp>
+#include <iostream>
 
 namespace nix {
 
@@ -24,6 +25,11 @@ void Logger::warn(const std::string & msg)
     log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
 }
 
+void Logger::writeToStdout(std::string_view s)
+{
+    std::cout << s << "\n";
+}
+
 class SimpleLogger : public Logger
 {
 public:
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 108b5dcb082..18c24d50877 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -78,6 +78,16 @@ public:
     virtual void stopActivity(ActivityId act) { };
 
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
+
+    virtual void writeToStdout(std::string_view s);
+
+    template<typename... Args>
+    inline void stdout(const std::string & fs, const Args & ... args)
+    {
+        boost::format f(fs);
+        formatHelper(f, args...);
+        writeToStdout(f.str());
+    }
 };
 
 ActivityId getCurActivity();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index adc9b9a5dca..8e7ba95a341 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <iostream>
 
 namespace nix {
 
@@ -442,6 +443,18 @@ class ProgressBar : public Logger
 
         return res;
     }
+
+    void writeToStdout(std::string_view s) override
+    {
+        auto state(state_.lock());
+        if (state->active) {
+            std::cerr << "\r\e[K";
+            Logger::writeToStdout(s);
+            draw(*state);
+        } else {
+            Logger::writeToStdout(s);
+        }
+    }
 };
 
 void startProgressBar(bool printBuildLogs)

From efaffaa9d1de38efecb718aa7a99ba1f2e342ade Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 16 Apr 2020 13:46:37 +0200
Subject: [PATCH 1988/2196] Use Logger::stdout()

(cherry picked from commit 8f41847394524fcac40d3b5620139ca7e94a18e3)
---
 src/nix/add-to-store.cc |  2 +-
 src/nix/eval.cc         |  5 ++---
 src/nix/hash.cc         |  5 ++---
 src/nix/ls.cc           | 10 ++++------
 src/nix/show-config.cc  |  2 +-
 src/nix/why-depends.cc  |  2 +-
 6 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 139db3657b5..ed02227db17 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         if (!dryRun)
             store->addToStore(info, sink.s);
 
-        std::cout << fmt("%s\n", store->printStorePath(info.path));
+        logger->stdout("%s", store->printStorePath(info.path));
     }
 };
 
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 6398fc58e46..86a1e8b681f 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -55,16 +55,15 @@ struct CmdEval : MixJSON, InstallableCommand
         auto v = installable->toValue(*state).first;
         PathSet context;
 
-        stopProgressBar();
-
         if (raw) {
+            stopProgressBar();
             std::cout << state->coerceToString(noPos, *v, context);
         } else if (json) {
             JSONPlaceholder jsonOut(std::cout);
             printValueAsJSON(*state, true, *v, jsonOut, context);
         } else {
             state->forceValueDeep(*v);
-            std::cout << *v << "\n";
+            logger->stdout("%s", *v);
         }
     }
 };
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 0cc523f507e..01628cf6cb5 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -60,8 +60,7 @@ struct CmdHash : Command
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
-            std::cout << format("%1%\n") %
-                h.to_string(base, base == SRI);
+            logger->stdout(h.to_string(base, base == SRI));
         }
     }
 };
@@ -95,7 +94,7 @@ struct CmdToBase : Command
     void run() override
     {
         for (auto s : args)
-            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI));
+            logger->stdout(Hash(s, ht).to_string(base, base == SRI));
     }
 };
 
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 3ef1f2750b6..8590199d73b 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON
                         (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
                     st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
                     "dr-xr-xr-x";
-                std::cout <<
-                    (format("%s %20d %s") % tp % st.fileSize % relPath);
+                auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
                 if (st.type == FSAccessor::Type::tSymlink)
-                    std::cout << " -> " << accessor->readLink(curPath)
-                    ;
-                std::cout << "\n";
+                    line += " -> " + accessor->readLink(curPath);
+                logger->stdout(line);
                 if (recursive && st.type == FSAccessor::Type::tDirectory)
                     doPath(st, curPath, relPath, false);
             } else {
-                std::cout << relPath << "\n";
+                logger->stdout(relPath);
                 if (recursive) {
                     auto st = accessor->stat(curPath);
                     if (st.type == FSAccessor::Type::tDirectory)
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 87544f93771..6104b10bcd1 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -23,7 +23,7 @@ struct CmdShowConfig : Command, MixJSON
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);
             for (auto & s : settings)
-                std::cout << s.first + " = " + s.second.value + "\n";
+                logger->stdout("%s = %s", s.first, s.second.value);
         }
     }
 };
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index d3b7a674a32..f9acc7f13ea 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -149,7 +149,7 @@ struct CmdWhyDepends : SourceExprCommand
             auto pathS = store->printStorePath(node.path);
 
             assert(node.dist != inf);
-            std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n",
+            logger->stdout("%s%s%s%s" ANSI_NORMAL,
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "→ " : "",

From 12814806efca65a73aebf8d3109eb2fbf880e2b8 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 16 Apr 2020 10:48:15 -0600
Subject: [PATCH 1989/2196] iomanip no longer needed

---
 src/libutil/error.hh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 417a6ad957e..f402b692e68 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -5,8 +5,6 @@
 #include <string>
 #include <optional>
 #include <iostream>
-#include <iomanip>
-
 #include "types.hh"
 
 namespace nix

From 3d5b1032a193419cfa7406f0e42a7621994d70af Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 17 Apr 2020 15:07:44 -0600
Subject: [PATCH 1990/2196] logError, logWarning; Logger functions; switch to
 Verbosity enum

---
 src/error-demo/error-demo.cc | 67 ++++++++++++++++++------------------
 src/libstore/daemon.cc       |  4 +++
 src/libutil/error.cc         | 50 +++++++++++++++++++--------
 src/libutil/error.hh         | 19 +++++-----
 src/libutil/logging.cc       | 18 ++++++++++
 src/libutil/logging.hh       | 30 ++++++++++------
 src/nix/progress-bar.cc      |  6 ++++
 7 files changed, 128 insertions(+), 66 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index a9ff6057c65..b7dd4cc22fb 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -1,4 +1,4 @@
-#include "error.hh"
+#include "logging.hh"
 #include "nixexpr.hh"
 
 #include <iostream>
@@ -8,24 +8,25 @@ int main()
 {
     using namespace nix;
 
+    std::unique_ptr<Logger> logger(makeDefaultLogger());
+
+    verbosity = lvlError;
+
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
     // Error in a program; no hint and no nix code.
-    printErrorInfo(
-        ErrorInfo { .level = elError,
-                    .name = "name",
+    logError(
+        ErrorInfo { .name = "name",
                     .description = "error description",
                   });
 
     // Warning with name, description, and hint.
     // The hintfmt function makes all the substituted text yellow.
-    printErrorInfo(
-        ErrorInfo { .level = elWarning,
-                    .name = "name",
+    logWarning(
+        ErrorInfo { .name = "name",
                     .description = "error description",
-                    .hint =  std::optional(
-                                 hintfmt("there was a %1%", "warning")),
+                    .hint = hintfmt("there was a %1%", "warning"),
                   });
 
 
@@ -34,32 +35,32 @@ int main()
     SymbolTable testTable;
     auto problem_file = testTable.create("myfile.nix");
 
-    printErrorInfo(
-        ErrorInfo{
-            .level = elWarning,
-            .name = "warning name",
-            .description = "warning description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::nullopt,
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::nullopt
-            }});
+    logWarning(
+        ErrorInfo { .name = "warning name",
+                    .description = "warning description",
+                    .hint = hintfmt("this hint has %1% templated %2%!!",
+                                    "yellow",
+                                    "values"),
+                    .nixCode = NixCode {
+                        .errPos = Pos(problem_file, 40, 13),
+                        .prevLineOfCode = std::nullopt,
+                        .errLineOfCode = "this is the problem line of code",
+                        .nextLineOfCode = std::nullopt
+                    }});
 
     // Error with previous and next lines of code.
-    printErrorInfo(
-        ErrorInfo{
-            .level = elError,
-            .name = "error name",
-            .description = "error description",
-            .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"),
-            .nixCode = NixCode {
-                .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::optional("previous line of code"),
-                .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::optional("next line of code"),
-            }});
+    logError(
+        ErrorInfo { .name = "error name",
+                    .description = "error description",
+                    .hint = hintfmt("this hint has %1% templated %2%!!",
+                                    "yellow",
+                                    "values"),
+                    .nixCode = NixCode {
+                        .errPos = Pos(problem_file, 40, 13),
+                        .prevLineOfCode = std::optional("previous line of code"),
+                        .errLineOfCode = "this is the problem line of code",
+                        .nextLineOfCode = std::optional("next line of code"),
+                    }});
 
 
     return 0;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 8e9f9d71bf0..c8daef57cac 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -73,6 +73,10 @@ struct TunnelLogger : public Logger
         enqueueMsg(*buf.s);
     }
 
+    void logEI(const ErrorInfo & ei) override
+    {
+    }
+
     /* startWork() means that we're starting an operation for which we
       want to send out stderr to the client. */
     void startWork()
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index a5571d4ecb6..779f519c027 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -2,6 +2,7 @@
 
 #include <iostream>
 #include <optional>
+#include "serialise.hh"
 
 namespace nix
 {
@@ -66,25 +67,45 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
     }
 }
 
-void printErrorInfo(const ErrorInfo &einfo)
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 {
     int errwidth = 80;
     string prefix = "    ";
 
     string levelString;
     switch (einfo.level) {
-    case ErrLevel::elError: {
+    case Verbosity::lvlError: {
         levelString = ANSI_RED;
         levelString += "error:";
         levelString += ANSI_NORMAL;
         break;
     }
-    case ErrLevel::elWarning: {
+    case Verbosity::lvlWarn: {
         levelString = ANSI_YELLOW;
         levelString += "warning:";
         levelString += ANSI_NORMAL;
         break;
     }
+    case Verbosity::lvlInfo: {
+        levelString = ANSI_YELLOW;
+        levelString += "info:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    case Verbosity::lvlTalkative:
+    case Verbosity::lvlChatty:
+    case Verbosity::lvlVomit: {
+        levelString = ANSI_GREEN;
+        levelString += "info:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    case Verbosity::lvlDebug: {
+        levelString = ANSI_YELLOW;
+        levelString += "debug:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
     default: {
         levelString = fmt("invalid error level: %1%", einfo.level);
         break;
@@ -99,7 +120,7 @@ void printErrorInfo(const ErrorInfo &einfo)
         dashes.append("-");
 
     // divider.
-    std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
+    out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
                      prefix,
                      levelString,
                      "---",
@@ -115,32 +136,33 @@ void printErrorInfo(const ErrorInfo &einfo)
                            ? string(" ") + showErrPos(einfo.nixCode->errPos)
                            : "";
 
-            std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
+            out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
                              prefix, 
                              einfo.nixCode->errPos.nixFile,
                              eline) << std::endl;
-            std::cout << prefix << std::endl;
+            out << prefix << std::endl;
         } else {
-            std::cout << fmt("%1%from command line argument", prefix) << std::endl;
-            std::cout << prefix << std::endl;
+            out << fmt("%1%from command line argument", prefix) << std::endl;
+            out << prefix << std::endl;
         }
     }
 
     // description
-    std::cout << prefix << einfo.description << std::endl;
-    std::cout << prefix << std::endl;
+    out << prefix << einfo.description << std::endl;
+    out << prefix << std::endl;
 
     // lines of code.
     if (einfo.nixCode->errLineOfCode != "") {
         printCodeLines(prefix, *einfo.nixCode);
-        std::cout << prefix << std::endl;
+        out << prefix << std::endl;
     }
 
     // hint
     if (einfo.hint.has_value()) {
-        std::cout << prefix << *einfo.hint << std::endl;
-        std::cout << prefix << std::endl;
+        out << prefix << *einfo.hint << std::endl;
+        out << prefix << std::endl;
     }
-}
 
+    return out;
+}
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index f402b692e68..0419a1b5256 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -11,9 +11,14 @@ namespace nix
 {
 
 typedef enum {
-    elWarning,
-    elError
-} ErrLevel;
+    lvlError = 0,
+    lvlWarn,
+    lvlInfo,
+    lvlTalkative,
+    lvlChatty,
+    lvlDebug,
+    lvlVomit
+} Verbosity;
 
 struct ErrPos
 {
@@ -101,7 +106,7 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args)
 // ErrorInfo.
 struct ErrorInfo
 {
-    ErrLevel level;
+    Verbosity level;
     string name;
     string description;
     std::optional<hintformat> hint;
@@ -110,11 +115,7 @@ struct ErrorInfo
     static std::optional<string> programName;
 };
 
-// --------------------------------------------------------
-// error printing
-
-// just to cout for now.
-void printErrorInfo(const ErrorInfo &einfo);
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
 
 }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index fa5c84a2733..98f500a65ac 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -3,6 +3,7 @@
 
 #include <atomic>
 #include <nlohmann/json.hpp>
+#include <sstream>
 
 namespace nix {
 
@@ -57,6 +58,14 @@ class SimpleLogger : public Logger
         writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
     }
 
+    void logEI(const ErrorInfo & ei) override
+    {
+        std::stringstream oss; 
+        oss << ei;
+
+        log(ei.level, oss.str());
+    }
+
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent)
         override
@@ -135,6 +144,15 @@ struct JSONLogger : Logger
         write(json);
     }
 
+    void logEI(const ErrorInfo & ei) override
+    {
+        // nlohmann::json json;
+        // json["action"] = "msg";
+        // json["level"] = lvl;
+        // json["msg"] = fs.s;
+        // write(json);
+    }
+
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent) override
     {
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index beb5e6b64e6..ba6f39ac927 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -1,19 +1,10 @@
 #pragma once
 
 #include "types.hh"
+#include "error.hh"
 
 namespace nix {
 
-typedef enum {
-    lvlError = 0,
-    lvlWarn,
-    lvlInfo,
-    lvlTalkative,
-    lvlChatty,
-    lvlDebug,
-    lvlVomit
-} Verbosity;
-
 typedef enum {
     actUnknown = 0,
     actCopyPath = 100,
@@ -70,6 +61,13 @@ public:
         log(lvlInfo, fs);
     }
 
+    virtual void logEI(const ErrorInfo &ei) = 0;
+
+    void logEI(Verbosity lvl, ErrorInfo ei) {
+      ei.level = lvl;
+      logEI(ei);
+    }
+
     virtual void warn(const std::string & msg);
 
     virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
@@ -157,6 +155,18 @@ extern Verbosity verbosity; /* suppress msgs > this */
 #define debug(args...) printMsg(lvlDebug, args)
 #define vomit(args...) printMsg(lvlVomit, args)
 
+#define logErrorInfo(level, errorInfo...) \
+    do { \
+        if (level <= nix::verbosity) { \
+            logger->logEI(level, errorInfo); \
+        } \
+    } while (0)
+
+#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo)
+#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo)
+
+
+
 template<typename... Args>
 inline void warn(const std::string & fs, const Args & ... args)
 {
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 26631416cc2..41f3d4cd7b3 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -124,6 +124,12 @@ class ProgressBar : public Logger
         log(*state, lvl, fs.s);
     }
 
+    void logEI(const ErrorInfo &ei) override
+    {
+        auto state(state_.lock());
+        // log(*state, lvl, ei.as_str());
+    }
+
     void log(State & state, Verbosity lvl, const std::string & s)
     {
         if (state.active) {

From 4697552948b4d9b958b0e8f8ff1b68b5e9608e4e Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 17 Apr 2020 15:50:46 -0600
Subject: [PATCH 1991/2196] demoing other error levels than warn/error; rename
 line and file fields in errPos

---
 src/error-demo/error-demo.cc | 37 ++++++++++++++++++++++++--
 src/libutil/error.cc         | 50 +++++++++++++++++++++---------------
 src/libutil/error.hh         |  8 +++---
 src/libutil/logging.cc       | 39 ++++++++++++++++++++++++----
 4 files changed, 103 insertions(+), 31 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index b7dd4cc22fb..14027278d4f 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -8,13 +8,46 @@ int main()
 {
     using namespace nix;
 
-    std::unique_ptr<Logger> logger(makeDefaultLogger());
+    makeDefaultLogger();
 
-    verbosity = lvlError;
+    verbosity = lvlVomit;
 
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
+    // For completeness sake, info through vomit levels.
+    // But this is maybe a heavy format for those.
+    logger->logEI(
+        ErrorInfo { .level = lvlInfo,
+                    .name = "Info name",
+                    .description = "Info description",
+                  });
+
+    logger->logEI(
+        ErrorInfo { .level = lvlTalkative,
+                    .name = "Talkative name",
+                    .description = "Talkative description",
+                  });
+
+    logger->logEI(
+        ErrorInfo { .level = lvlChatty,
+                    .name = "Chatty name",
+                    .description = "Chatty description",
+                  });
+
+    logger->logEI(
+        ErrorInfo { .level = lvlDebug,
+                    .name = "Debug name",
+                    .description = "Debug description",
+                  });
+
+    logger->logEI(
+        ErrorInfo { .level = lvlVomit,
+                    .name = "Vomit name",
+                    .description = "Vomit description",
+                  });
+
+
     // Error in a program; no hint and no nix code.
     logError(
         ErrorInfo { .name = "name",
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 779f519c027..0e07c0d98e0 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -17,9 +17,9 @@ std::ostream& operator<<(std::ostream &os, const hintformat &hf)
 string showErrPos(const ErrPos &errPos)
 {
     if (errPos.column > 0) {
-        return fmt("(%1%:%2%)", errPos.lineNumber, errPos.column);
+        return fmt("(%1%:%2%)", errPos.line, errPos.column);
     } else {
-        return fmt("(%1%)", errPos.lineNumber);
+        return fmt("(%1%)", errPos.line);
     };
 }
 
@@ -29,7 +29,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
     if (nixCode.prevLineOfCode.has_value()) {
         std::cout << fmt("%1% %|2$5d|| %3%",
                          prefix,
-                         (nixCode.errPos.lineNumber - 1),
+                         (nixCode.errPos.line - 1),
                          *nixCode.prevLineOfCode)
                   << std::endl;
     }
@@ -37,7 +37,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
     // line of code containing the error.%2$+5d%
     std::cout << fmt("%1% %|2$5d|| %3%",
                      prefix,
-                     (nixCode.errPos.lineNumber),
+                     (nixCode.errPos.line),
                      nixCode.errLineOfCode)
               << std::endl;
 
@@ -61,7 +61,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
     if (nixCode.nextLineOfCode.has_value()) {
         std::cout << fmt("%1% %|2$5d|| %3%",
                          prefix,
-                         (nixCode.errPos.lineNumber + 1),
+                         (nixCode.errPos.line + 1),
                          *nixCode.nextLineOfCode)
                   << std::endl;
     }
@@ -87,16 +87,26 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         break;
     }
     case Verbosity::lvlInfo: {
-        levelString = ANSI_YELLOW;
+        levelString = ANSI_GREEN;
         levelString += "info:";
         levelString += ANSI_NORMAL;
         break;
     }
-    case Verbosity::lvlTalkative:
-    case Verbosity::lvlChatty:
+    case Verbosity::lvlTalkative: {
+        levelString = ANSI_GREEN;
+        levelString += "talk:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    case Verbosity::lvlChatty: {
+        levelString = ANSI_GREEN;
+        levelString += "chat:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
     case Verbosity::lvlVomit: {
         levelString = ANSI_GREEN;
-        levelString += "info:";
+        levelString += "vomit:";
         levelString += ANSI_NORMAL;
         break;
     }
@@ -121,25 +131,25 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 
     // divider.
     out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
-                     prefix,
-                     levelString,
-                     "---",
-                     einfo.name,
-                     dashes,
-                     einfo.programName.value_or(""))
-              << std::endl;
+               prefix,
+               levelString,
+               "---",
+               einfo.name,
+               dashes,
+               einfo.programName.value_or(""))
+        << std::endl;
 
     // filename.
     if (einfo.nixCode.has_value()) {
-        if (einfo.nixCode->errPos.nixFile != "") {
+        if (einfo.nixCode->errPos.file != "") {
             string eline = einfo.nixCode->errLineOfCode != ""
                            ? string(" ") + showErrPos(einfo.nixCode->errPos)
                            : "";
 
             out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
-                             prefix, 
-                             einfo.nixCode->errPos.nixFile,
-                             eline) << std::endl;
+                       prefix,
+                       einfo.nixCode->errPos.file,
+                       eline) << std::endl;
             out << prefix << std::endl;
         } else {
             out << fmt("%1%from command line argument", prefix) << std::endl;
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 0419a1b5256..63778ebda7f 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -22,16 +22,16 @@ typedef enum {
 
 struct ErrPos
 {
-    int lineNumber;
+    int line;
     int column;
-    string nixFile;
+    string file;
 
     template <class P>
     ErrPos& operator=(const P &pos)
     {
-        lineNumber = pos.line;
+        line = pos.line;
         column = pos.column;
-        nixFile = pos.file;
+        file = pos.file;
         return *this;
     }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 98f500a65ac..4a8b9864020 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -146,11 +146,40 @@ struct JSONLogger : Logger
 
     void logEI(const ErrorInfo & ei) override
     {
-        // nlohmann::json json;
-        // json["action"] = "msg";
-        // json["level"] = lvl;
-        // json["msg"] = fs.s;
-        // write(json);
+        // add fields like Pos info and etc?
+        std::ostringstream oss;
+        oss << ei;
+
+        nlohmann::json json;
+        json["action"] = "msg";
+        json["level"] = ei.level;
+        json["msg"] = oss.str();
+
+        // Extra things that COULD go into json.  Useful?  
+        // TODO: decide if useful.
+        // TODO: make a json obj that goes into json["msg"]?
+        json["name"] = ei.name;
+        json["description"] = ei.description;
+        if (ei.hint.has_value())
+        {
+          json["hint"] = ei.hint->str();
+        }
+        if (ei.nixCode.has_value())
+        {
+          if (ei.nixCode->errPos.line != 0)
+            json["line"] = ei.nixCode->errPos.line;
+          if (ei.nixCode->errPos.column != 0)
+            json["column"] = ei.nixCode->errPos.column;
+          if (ei.nixCode->errPos.file != "")
+            json["file"] = ei.nixCode->errPos.file;
+          if (ei.nixCode->prevLineOfCode.has_value())
+            json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode;
+          json["errLineOfCode"] = ei.nixCode->errLineOfCode;
+          if (ei.nixCode->nextLineOfCode.has_value())
+            json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode;
+        }
+
+        write(json);
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

From 15e9564fd1e11b3797f0188a494aa66d4622dfb0 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 19 Apr 2020 17:16:51 -0600
Subject: [PATCH 1992/2196] logEI for tunnelLogger and progressbar

---
 src/libstore/daemon.cc  | 8 ++++++++
 src/nix/progress-bar.cc | 6 +++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index c8daef57cac..d985f151b18 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -75,6 +75,14 @@ struct TunnelLogger : public Logger
 
     void logEI(const ErrorInfo & ei) override
     {
+        if (ei.level > verbosity) return;
+
+        std::stringstream oss; 
+        oss << ei;
+
+        StringSink buf;
+        buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
+        enqueueMsg(*buf.s);
     }
 
     /* startWork() means that we're starting an operation for which we
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 41f3d4cd7b3..b1226d550a9 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -127,7 +127,11 @@ class ProgressBar : public Logger
     void logEI(const ErrorInfo &ei) override
     {
         auto state(state_.lock());
-        // log(*state, lvl, ei.as_str());
+
+        std::stringstream oss; 
+        oss << ei;
+
+        log(*state, ei.level, oss.str());
     }
 
     void log(State & state, Verbosity lvl, const std::string & s)

From d3052197feababc312fd874e08ae48050d985eb3 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 21 Apr 2020 13:25:41 -0600
Subject: [PATCH 1993/2196] add ErrorInfo to BaseError

---
 src/libutil/error.cc   |   2 +-
 src/libutil/error.hh   | 122 ---------------------------------------
 src/libutil/fmt.hh     | 127 +++++++++++++++++++++++++++++++++++++++++
 src/libutil/logging.hh |   1 -
 src/libutil/types.hh   | 125 ++++++++++++++++++++--------------------
 src/libutil/util.hh    |   1 -
 6 files changed, 192 insertions(+), 186 deletions(-)
 delete mode 100644 src/libutil/error.hh
 create mode 100644 src/libutil/fmt.hh

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 0e07c0d98e0..41595b0df99 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -1,4 +1,4 @@
-#include "error.hh"
+#include "types.hh"
 
 #include <iostream>
 #include <optional>
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
deleted file mode 100644
index 63778ebda7f..00000000000
--- a/src/libutil/error.hh
+++ /dev/null
@@ -1,122 +0,0 @@
-#ifndef error_hh
-#define error_hh
-
-#include "ansicolor.hh"
-#include <string>
-#include <optional>
-#include <iostream>
-#include "types.hh"
-
-namespace nix
-{
-
-typedef enum {
-    lvlError = 0,
-    lvlWarn,
-    lvlInfo,
-    lvlTalkative,
-    lvlChatty,
-    lvlDebug,
-    lvlVomit
-} Verbosity;
-
-struct ErrPos
-{
-    int line;
-    int column;
-    string file;
-
-    template <class P>
-    ErrPos& operator=(const P &pos)
-    {
-        line = pos.line;
-        column = pos.column;
-        file = pos.file;
-        return *this;
-    }
-
-    template <class P>
-    ErrPos(const P &p)
-    {
-      *this = p;
-    }
-};
-
-struct NixCode
-{
-    ErrPos errPos;
-    std::optional<string> prevLineOfCode;
-    string errLineOfCode;
-    std::optional<string> nextLineOfCode;
-};
-
-// ----------------------------------------------------------------
-// format function for hints.  same as fmt, except templated values
-// are always in yellow.
-
-template <class T>
-struct yellowify
-{
-    yellowify(T &s) : value(s) {}
-    T &value;
-};
-
-template <class T>
-std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
-{
-    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
-}
-
-class hintformat
-{
-public:
-    hintformat(string format) :fmt(format)
-    {
-        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    }
-    template<class T>
-    hintformat& operator%(const T &value)
-    {
-        fmt % yellowify(value);
-        return *this;
-    }
-
-    std::string str() const
-    {
-        return fmt.str();
-    }
-
-    template <typename U>
-    friend class AddHint;
-private:
-    format fmt;
-};
-
-std::ostream& operator<<(std::ostream &os, const hintformat &hf);
-
-template<typename... Args>
-inline hintformat hintfmt(const std::string & fs, const Args & ... args)
-{
-    hintformat f(fs);
-    formatHelper(f, args...);
-    return f;
-}
-
-// -------------------------------------------------
-// ErrorInfo.
-struct ErrorInfo
-{
-    Verbosity level;
-    string name;
-    string description;
-    std::optional<hintformat> hint;
-    std::optional<NixCode> nixCode;
-
-    static std::optional<string> programName;
-};
-
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
-
-}
-
-#endif
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
new file mode 100644
index 00000000000..08edffee0a6
--- /dev/null
+++ b/src/libutil/fmt.hh
@@ -0,0 +1,127 @@
+#pragma once
+
+#include <boost/format.hpp>
+#include <string>
+#include "ansicolor.hh"
+
+
+namespace nix {
+
+
+/* Inherit some names from other namespaces for convenience. */
+using std::string;
+using boost::format;
+
+
+/* A variadic template that does nothing. Useful to call a function
+   for all variadic arguments but ignoring the result. */
+struct nop { template<typename... T> nop(T...) {} };
+
+
+struct FormatOrString
+{
+    string s;
+    FormatOrString(const string & s) : s(s) { };
+    template<class F>
+    FormatOrString(const F & f) : s(f.str()) { };
+    FormatOrString(const char * s) : s(s) { };
+};
+
+
+/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is
+   equivalent to ‘boost::format(format) % a_0 % ... %
+   ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
+   takes place). */
+
+template<class F>
+inline void formatHelper(F & f)
+{
+}
+
+template<class F, typename T, typename... Args>
+inline void formatHelper(F & f, const T & x, const Args & ... args)
+{
+    formatHelper(f % x, args...);
+}
+
+inline std::string fmt(const std::string & s)
+{
+    return s;
+}
+
+inline std::string fmt(const char * s)
+{
+    return s;
+}
+
+inline std::string fmt(const FormatOrString & fs)
+{
+    return fs.s;
+}
+
+template<typename... Args>
+inline std::string fmt(const std::string & fs, const Args & ... args)
+{
+    boost::format f(fs);
+    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    formatHelper(f, args...);
+    return f.str();
+}
+
+// -----------------------------------------------------------------------------
+// format function for hints in errors.  same as fmt, except templated values
+// are always in yellow.
+
+template <class T>
+struct yellowify
+{
+    yellowify(T &s) : value(s) {}
+    T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
+{
+    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+}
+
+class hintformat
+{
+public:
+    hintformat(const string &format) :fmt(format)
+    {
+        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    }
+    template<class T>
+    hintformat& operator%(const T &value)
+    {
+        fmt % yellowify(value);
+        return *this;
+    }
+    hintformat(const hintformat &hf)
+    : fmt(hf.fmt)
+    {}
+
+
+    std::string str() const
+    {
+        return fmt.str();
+    }
+
+    template <typename U>
+    friend class AddHint;
+private:
+    format fmt;
+};
+
+std::ostream& operator<<(std::ostream &os, const hintformat &hf);
+
+template<typename... Args>
+inline hintformat hintfmt(const std::string & fs, const Args & ... args)
+{
+    hintformat f(fs);
+    formatHelper(f, args...);
+    return f;
+}
+
+}
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index ba6f39ac927..8269e2e1af1 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -1,7 +1,6 @@
 #pragma once
 
 #include "types.hh"
-#include "error.hh"
 
 namespace nix {
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 981af528b23..633c6bdf760 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -3,13 +3,13 @@
 
 #include "ref.hh"
 
-#include <string>
 #include <list>
 #include <set>
 #include <memory>
 #include <map>
+#include <optional>
 
-#include <boost/format.hpp>
+#include "fmt.hh"
 
 /* Before 4.7, gcc's std::exception uses empty throw() specifiers for
  * its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -20,73 +20,66 @@
 #endif
 #endif
 
-
 namespace nix {
 
-
-/* Inherit some names from other namespaces for convenience. */
-using std::string;
 using std::list;
 using std::set;
 using std::vector;
-using boost::format;
-
-
-/* A variadic template that does nothing. Useful to call a function
-   for all variadic arguments but ignoring the result. */
-struct nop { template<typename... T> nop(T...) {} };
 
-
-struct FormatOrString
+typedef enum {
+    lvlError = 0,
+    lvlWarn,
+    lvlInfo,
+    lvlTalkative,
+    lvlChatty,
+    lvlDebug,
+    lvlVomit
+} Verbosity;
+
+struct ErrPos
 {
-    string s;
-    FormatOrString(const string & s) : s(s) { };
-    template<class F>
-    FormatOrString(const F & f) : s(f.str()) { };
-    FormatOrString(const char * s) : s(s) { };
-};
-
-
-/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is
-   equivalent to ‘boost::format(format) % a_0 % ... %
-   ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
-   takes place). */
+    int line;
+    int column;
+    string file;
 
-template<class F>
-inline void formatHelper(F & f)
-{
-}
-
-template<class F, typename T, typename... Args>
-inline void formatHelper(F & f, const T & x, const Args & ... args)
-{
-    formatHelper(f % x, args...);
-}
+    template <class P>
+    ErrPos& operator=(const P &pos)
+    {
+        line = pos.line;
+        column = pos.column;
+        file = pos.file;
+        return *this;
+    }
 
-inline std::string fmt(const std::string & s)
-{
-    return s;
-}
+    template <class P>
+    ErrPos(const P &p)
+    {
+      *this = p;
+    }
+};
 
-inline std::string fmt(const char * s)
+struct NixCode
 {
-    return s;
-}
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
 
-inline std::string fmt(const FormatOrString & fs)
+// -------------------------------------------------
+// ErrorInfo.
+struct ErrorInfo
 {
-    return fs.s;
-}
+    Verbosity level;
+    string name;
+    string description;
+    std::optional<hintformat> hint;
+    std::optional<NixCode> nixCode;
 
-template<typename... Args>
-inline std::string fmt(const std::string & fs, const Args & ... args)
-{
-    boost::format f(fs);
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    formatHelper(f, args...);
-    return f.str();
-}
+    static std::optional<string> programName;
+};
 
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
 
 /* BaseError should generally not be caught, as it has Interrupted as
    a subclass. Catch Error instead. */
@@ -94,33 +87,42 @@ class BaseError : public std::exception
 {
 protected:
     string prefix_; // used for location traces etc.
-    string err;
+    ErrorInfo err;
 public:
     unsigned int status = 1; // exit status
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err(fmt(args...))
+        : err(hintfmt(args...))
         , status(status)
     {
     }
 
     template<typename... Args>
     BaseError(const Args & ... args)
-        : err(fmt(args...))
+        : err { .level = lvlError, 
+                .hint = hintfmt(args...)
+              }
+    {
+    }
+
+    BaseError(ErrorInfo e)
+        : err(e)
     {
     }
 
 #ifdef EXCEPTION_NEEDS_THROW_SPEC
     ~BaseError() throw () { };
-    const char * what() const throw () { return err.c_str(); }
+    const char * what() const throw () { return err.description.c_str(); }
 #else
-    const char * what() const noexcept { return err.c_str(); }
+    const char * what() const noexcept { return err.description.c_str(); }
 #endif
 
-    const string & msg() const { return err; }
+    const string & msg() const { return err.description; }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
+
+    const ErrorInfo & info() const { return err; }
 };
 
 #define MakeError(newClass, superClass) \
@@ -139,7 +141,8 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-        : Error(addErrno(fmt(args...)))
+        : Error(args...)  // TODO addErrNo for hintfmt
+        // : Error(addErrno(hintfmt(args...)))
     { }
 
 private:
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index fb9a7ed383d..636f3ba6c0e 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -3,7 +3,6 @@
 #include "types.hh"
 #include "logging.hh"
 #include "ansicolor.hh"
-#include "error.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>

From e4fb9a38493a041861fe5c75bc8ddd129a2e5262 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 21 Apr 2020 17:07:07 -0600
Subject: [PATCH 1994/2196] remove 'format' from Error constructor calls

---
 src/libexpr/attr-path.cc                 |  16 +--
 src/libexpr/eval-inline.hh               |   6 +-
 src/libexpr/eval.cc                      |  23 ++---
 src/libexpr/nixexpr.cc                   |   2 +-
 src/libexpr/nixexpr.hh                   |   4 +-
 src/libexpr/parser.y                     |  32 +++---
 src/libexpr/primops.cc                   | 124 +++++++++++------------
 src/libexpr/primops/fetchGit.cc          |   2 +-
 src/libexpr/primops/fetchMercurial.cc    |   2 +-
 src/libexpr/value-to-json.cc             |   4 +-
 src/libmain/shared.cc                    |  10 +-
 src/libmain/shared.hh                    |   6 +-
 src/libstore/binary-cache-store.cc       |   8 +-
 src/libstore/build.cc                    |  98 +++++++++---------
 src/libstore/builtins/buildenv.cc        |  10 +-
 src/libstore/builtins/fetchurl.cc        |   4 +-
 src/libstore/daemon.cc                   |   2 +-
 src/libstore/derivations.cc              |   6 +-
 src/libstore/download.cc                 |   6 +-
 src/libstore/download.hh                 |   6 +-
 src/libstore/gc.cc                       |  44 ++++----
 src/libstore/local-binary-cache-store.cc |   2 +-
 src/libstore/local-fs-store.cc           |   6 +-
 src/libstore/local-store.cc              |  48 ++++-----
 src/libstore/nar-accessor.cc             |   6 +-
 src/libstore/nar-info.cc                 |   2 +-
 src/libstore/optimise-store.cc           |  24 ++---
 src/libstore/pathlocks.cc                |   8 +-
 src/libstore/profiles.cc                 |   8 +-
 src/libstore/references.cc               |   2 +-
 src/libstore/remote-fs-accessor.cc       |   2 +-
 src/libstore/remote-store.cc             |   4 +-
 src/libstore/s3-binary-cache-store.cc    |   8 +-
 src/libstore/sqlite.cc                   |   2 +-
 src/libstore/store-api.cc                |   4 +-
 src/libutil/archive.cc                   |  19 ++--
 src/libutil/args.cc                      |   9 +-
 src/libutil/compression.cc               |   2 +-
 src/libutil/fmt.hh                       |   6 +-
 src/libutil/types.hh                     |   4 +-
 src/libutil/util.cc                      |  50 ++++-----
 src/nix-channel/nix-channel.cc           |  30 ++++--
 src/nix-daemon/nix-daemon.cc             |  43 ++++----
 src/nix-env/nix-env.cc                   |  55 +++++-----
 src/nix-instantiate/nix-instantiate.cc   |   4 +-
 src/nix-prefetch-url/nix-prefetch-url.cc |  10 +-
 src/nix-store/nix-store.cc               |  38 +++----
 src/nix/cat.cc                           |   4 +-
 src/nix/hash.cc                          |   2 +-
 src/nix/ls.cc                            |   2 +-
 src/nix/repl.cc                          |   8 +-
 src/nix/run.cc                           |   4 +-
 52 files changed, 424 insertions(+), 407 deletions(-)

diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 76d101b98e9..9a9531a3f83 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error(format("missing closing quote in selection path '%1%'") % s);
+                    throw Error("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -60,11 +60,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a set but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v)); 
             if (attr.empty())
-                throw Error(format("empty attribute name in selection path '%1%'") % attrPath);
+                throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
@@ -77,9 +77,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 
             if (!v->isList())
                 throw TypeError(
-                    format("the expression selected by the selection path '%1%' should be a list but is %2%")
-                    % attrPath % showType(*v));
-
+                    "the expression selected by the selection path '%1%' should be a list but is %2%",
+                    attrPath,
+                    showType(*v)); 
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index c27116e3b44..4d82ccf0976 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -9,18 +9,18 @@ namespace nix {
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
 {
-    throw EvalError(format(s) % pos);
+    throw EvalError(s, pos);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 {
-    throw TypeError(format(s) % showType(v));
+    throw TypeError(s, showType(v));
 }
 
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
 {
-    throw TypeError(format(s) % showType(v) % pos);
+    throw TypeError(s, showType(v), pos);
 }
 
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index dac32b6f563..516c25b022b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -493,52 +493,52 @@ Value & EvalState::getBuiltin(const string & name)
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 {
-    throw EvalError(format(s) % s2);
+    throw EvalError(s, s2);
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
 {
-    throw EvalError(format(s) % s2 % pos);
+    throw EvalError(s, s2, pos);
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(format(s) % s2 % s3);
+    throw EvalError(s, s2, s3);
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
 {
-    throw EvalError(format(s) % s2 % s3 % pos);
+    throw EvalError(s, s2, s3, pos);
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
 {
-    throw EvalError(format(s) % sym % p1 % p2);
+    throw EvalError(s, sym, p1, p2);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
 {
-    throw TypeError(format(s) % pos);
+    throw TypeError(s, pos);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 {
-    throw TypeError(format(s) % s1);
+    throw TypeError(s, s1);
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
 {
-    throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
+    throw TypeError(s, fun.showNamePos(), s2, pos);
 }
 
 LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError(format(s) % s1 % pos);
+    throw AssertionError(s, s1, pos);
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
 {
-    throw UndefinedVarError(format(s) % s1 % pos);
+    throw UndefinedVarError(s, s1, pos);
 }
 
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@@ -1883,8 +1883,7 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError(format("cannot coerce %1% to a string, at %2%") %
-        showType() % pos);
+    throw TypeError("cannot coerce %1% to a string, at %2%", showType(), pos);
 }
 
 
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 63cbef1ddf8..a9955f6df7c 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -267,7 +267,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos);
+    if (withLevel == -1) throw UndefinedVarError("undefined variable '%1%' at %2%", name, pos);
 
     fromWith = true;
     this->level = withLevel;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index f7e9105a401..79e3f90e50d 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -234,9 +234,7 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-                % arg % pos);
-    };
+            throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); };
     void setName(Symbol & name);
     string showNamePos() const;
     COMMON_METHODS
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 9c769e80376..3767532d5ef 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -64,15 +64,15 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % showAttrPath(attrPath) % pos % prevPos);
+    throw ParseError("attribute '%1%' at %2% already defined at %3%",
+        showAttrPath(attrPath), pos, prevPos);
 }
 
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(format("attribute '%1%' at %2% already defined at %3%")
-        % attr % pos % prevPos);
+    throw ParseError("attribute '%1%' at %2% already defined at %3%",
+        attr, pos, prevPos);
 }
 
 
@@ -140,8 +140,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(format("duplicate formal function argument '%1%' at %2%")
-            % formal.name % pos);
+        throw ParseError("duplicate formal function argument '%1%' at %2%",
+            formal.name, pos);
     formals->formals.push_front(formal);
 }
 
@@ -327,8 +327,7 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(format("dynamic attributes not allowed in let at %1%")
-            % CUR_POS);
+        throw ParseError("dynamic attributes not allowed in let at %1%", CUR_POS);
       $$ = new ExprLet($2, $4);
     }
   | expr_if
@@ -475,8 +474,8 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(format("dynamic attributes not allowed in inherit at %1%")
-              % makeCurPos(@2, data));
+          throw ParseError("dynamic attributes not allowed in inherit at %1%",
+              makeCurPos(@2, data));
     }
   | { $$ = new AttrPath; }
   ;
@@ -670,11 +669,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    format f = format(
+    string f = 
         "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : ""));
-    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    throw ThrownError(f % path % pos);
+        + string(pos ? ", at %2%" : "");
+    throw ThrownError(f, path, pos);
 }
 
 
@@ -691,7 +689,8 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             request.unpack = true;
             res = { true, getDownloader()->downloadCached(store, request).path };
         } catch (DownloadError & e) {
-            printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
+            // TODO: change to warn()?
+            printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second);
             res = { false, "" };
         }
     } else {
@@ -699,7 +698,8 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second);
+            // TODO: change to warn()?
+            printError("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second);
             res = { false, "" };
         }
     }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8de23495187..13eb1ba5878 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -94,8 +94,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%",
+                        path, e.path, pos);
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -171,8 +171,8 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%"
+           , path, e.path, pos);
     }
 
     path = state.checkSourcePath(path);
@@ -181,17 +181,17 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
     if (!handle)
-        throw EvalError(format("could not open '%1%': %2%") % path % dlerror());
+        throw EvalError("could not open '%1%': %2%", path, dlerror());
 
     dlerror();
     ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
     if(!func) {
         char *message = dlerror();
         if (message)
-            throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message);
+            throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message);
         else
-            throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected")
-                    % sym % path);
+            throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected"
+                   , sym, path);
     }
 
     (func)(state, v);
@@ -207,7 +207,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto elems = args[0]->listElems();
     auto count = args[0]->listSize();
     if (count == 0) {
-        throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos);
+        throw EvalError("at least one argument to 'exec' required, at %1%", pos);
     }
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
@@ -218,8 +218,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%")
-            % program % e.path % pos);
+        throw EvalError("cannot execute '%1%', since path '%2%' is not valid, at %3%"
+           , program, e.path, pos);
     }
 
     auto output = runProgram(program, true, commandArgs);
@@ -227,13 +227,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos);
+        e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos);
+        e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
         throw;
     }
 }
@@ -339,7 +339,7 @@ struct CompareValues
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
         if (v1->type != v2->type)
-            throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+            throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         switch (v1->type) {
             case tInt:
                 return v1->integer < v2->integer;
@@ -350,7 +350,7 @@ struct CompareValues
             case tPath:
                 return strcmp(v1->path, v2->path) < 0;
             default:
-                throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
+                throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         }
     }
 };
@@ -371,7 +371,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError(format("attribute 'startSet' required, at %1%") % pos);
+        throw EvalError("attribute 'startSet' required, at %1%", pos);
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -382,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError(format("attribute 'operator' required, at %1%") % pos);
+        throw EvalError("attribute 'operator' required, at %1%", pos);
     state.forceValue(*op->value);
 
     /* Construct the closure by applying the operator to element of
@@ -401,7 +401,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError(format("attribute 'key' required, at %1%") % pos);
+            throw EvalError("attribute 'key' required, at %1%", pos);
         state.forceValue(*key->value);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -431,7 +431,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     string s = state.coerceToString(pos, *args[0], context);
-    throw Abort(format("evaluation aborted with the following error message: '%1%'") % s);
+    throw Abort("evaluation aborted with the following error message: '%1%'", s);
 }
 
 
@@ -506,9 +506,9 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceValue(*args[0]);
     if (args[0]->type == tString)
-        printError(format("trace: %1%") % args[0]->string.s);
+        printError("trace: %1%", args[0]->string.s);
     else
-        printError(format("trace: %1%") % *args[0]);
+        printError("trace: %1%", *args[0]);
     state.forceValue(*args[1]);
     v = *args[1];
 }
@@ -533,13 +533,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError(format("required attribute 'name' missing, at %1%") % pos);
+        throw EvalError("required attribute 'name' missing, at %1%", pos);
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName);
+        e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
         throw;
     }
 
@@ -583,18 +583,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName);
+                    throw EvalError("duplicate derivation output '%1%', at %2%", j, posDrvName);
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName);
+                    throw EvalError("invalid derivation output name 'drv', at %1%", posDrvName);
                 outputs.insert(j);
             }
             if (outputs.empty())
-                throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName);
+                throw EvalError("derivation cannot have an empty set of outputs, at %1%", posDrvName);
         };
 
         try {
@@ -706,9 +706,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName);
+        throw EvalError("required attribute 'builder' missing, at %1%", posDrvName);
     if (drv.platform == "")
-        throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
+        throw EvalError("required attribute 'system' missing, at %1%", posDrvName);
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
@@ -717,7 +717,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
-            throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
+            throw Error("multiple outputs are not supported in fixed-output derivations, at %1%", posDrvName);
 
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
@@ -818,7 +818,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
+        throw EvalError("path '%1%' is not in the Nix store, at %2%", path, pos);
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
         state.store->ensurePath(state.store->parseStorePath(path2));
@@ -834,9 +834,9 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format(
-                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError(
+                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%", 
+                path, e.path, pos);
     }
 
     try {
@@ -879,12 +879,12 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%"
+           , path, e.path, pos);
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
-        throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path);
+        throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
     mkString(v, s.c_str());
 }
 
@@ -908,7 +908,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError(format("attribute 'path' missing, at %1%") % pos);
+            throw EvalError("attribute 'path' missing, at %1%", pos);
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -916,8 +916,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%")
-                % path % e.path % pos);
+            throw EvalError("cannot find '%1%', since path '%2%' is not valid, at %3%",
+                        path, e.path, pos);
         }
 
         searchPath.emplace_back(prefix, path);
@@ -934,7 +934,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+      throw Error("unknown hash type '%1%', at %2%", type, pos);
 
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
@@ -950,8 +950,8 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%")
-            % path % e.path % pos);
+        throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%",
+                        path, e.path, pos);
     }
 
     DirEntries entries = readDirectory(state.checkSourcePath(path));
@@ -1021,7 +1021,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
+            throw EvalError("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%", name, pos);
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1089,11 +1089,11 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+        throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, pos);
 
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+        throw TypeError("first argument in call to 'filterSource' is not a function but %1%, at %2%", showType(*args[0]), pos);
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
 }
@@ -1113,7 +1113,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
-                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+                throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, *attr.pos);
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
@@ -1124,10 +1124,10 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else if (n == "sha256")
             expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
-            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+            throw EvalError("unsupported argument '%1%' to 'addPath', at %2%", attr.name, *attr.pos);
     }
     if (path.empty())
-        throw EvalError(format("'path' required, at %1%") % pos);
+        throw EvalError("'path' required, at %1%", pos);
     if (name.empty())
         name = baseNameOf(path);
 
@@ -1185,7 +1185,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos);
+        throw EvalError("attribute '%1%' missing, at %2%", attr, pos);
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value);
@@ -1265,14 +1265,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos);
+            throw TypeError("'name' attribute missing in a call to 'listToAttrs', at %1%", pos);
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
+                throw TypeError("'value' attribute missing in a call to 'listToAttrs', at %1%", pos);
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
@@ -1346,7 +1346,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError(format("'functionArgs' requires a function, at %1%") % pos);
+        throw TypeError("'functionArgs' requires a function, at %1%", pos);
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1396,7 +1396,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
 {
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
-        throw Error(format("list index %1% is out of bounds, at %2%") % n % pos);
+        throw Error("list index %1% is out of bounds, at %2%", n, pos);
     state.forceValue(*list.listElems()[n]);
     v = *list.listElems()[n];
 }
@@ -1423,7 +1423,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error(format("'tail' called on an empty list, at %1%") % pos);
+        throw Error("'tail' called on an empty list, at %1%", pos);
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
         v.listElems()[n] = args[0]->listElems()[n + 1];
@@ -1564,7 +1564,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
-        throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos);
+        throw EvalError("cannot create list of size %1%, at %2%", len, pos);
 
     state.mkList(v, len);
 
@@ -1722,7 +1722,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     state.forceValue(*args[1], pos);
 
     NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos);
+    if (f2 == 0) throw EvalError("division by zero, at %1%", pos);
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
@@ -1731,7 +1731,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         NixInt i2 = state.forceInt(*args[1], pos);
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError(format("overflow in integer division, at %1%") % pos);
+            throw EvalError("overflow in integer division, at %1%", pos);
         mkInt(v, i1 / i2);
     }
 }
@@ -1787,7 +1787,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos);
+    if (start < 0) throw EvalError("negative start position in 'substring', at %1%", pos);
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1807,7 +1807,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+      throw Error("unknown hash type '%1%', at %2%", type, pos);
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
@@ -1950,7 +1950,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos);
+        throw EvalError("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%", pos);
 
     vector<string> from;
     from.reserve(args[0]->listSize());
@@ -2072,11 +2072,11 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else if (n == "name")
                 request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos);
+                throw EvalError("unsupported argument '%1%' to '%2%', at %3%", attr.name, who, attr.pos);
         }
 
         if (request.uri.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError("'url' argument required, at %1%", pos);
 
     } else
         request.uri = state.forceStringNoCtx(*args[0], pos);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 4aee1073eeb..480cc1adcbf 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -221,7 +221,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError("'url' argument required, at %1%", pos);
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index db274fa4f70..76029f5489c 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -191,7 +191,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
         }
 
         if (url.empty())
-            throw EvalError(format("'url' argument required, at %1%") % pos);
+            throw EvalError("'url' argument required, at %1%", pos);
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index 5fe8570adeb..6ec8315bab0 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         default:
-            throw TypeError(format("cannot convert %1% to JSON") % showType(v));
+            throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }
 
@@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
-    throw TypeError(format("cannot convert %1% to JSON") % showType());
+    throw TypeError("cannot convert %1% to JSON", showType());
 }
 
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d41e772e9a7..f7ab56271d2 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -75,7 +75,7 @@ string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     return *i;
 }
 
@@ -229,7 +229,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
     Strings ss(args);
     auto pos = ss.begin();
     if (!parseArg(pos, ss.end()))
-        throw UsageError(format("unexpected argument '%1%'") % args.front());
+        throw UsageError("unexpected argument '%1%'", args.front());
     return true;
 }
 
@@ -273,7 +273,7 @@ void showManPage(const string & name)
     restoreSignals();
     setenv("MANPATH", settings.nixManDir.c_str(), 1);
     execlp("man", "man", name.c_str(), nullptr);
-    throw SysError(format("command 'man %1%' failed") % name.c_str());
+    throw SysError("command 'man %1%' failed", name.c_str());
 }
 
 
@@ -301,7 +301,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
             % e.what() % programName);
         return 1;
     } catch (BaseError & e) {
-        printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+        printError(error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         if (e.prefix() != "" && !settings.showTrace)
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
@@ -338,7 +338,7 @@ RunPager::RunPager()
         execlp("pager", "pager", nullptr);
         execlp("less", "less", nullptr);
         execlp("more", "more", nullptr);
-        throw SysError(format("executing '%1%'") % pager);
+        throw SysError("executing '%1%'", pager);
     });
 
     pid.setKillSignal(SIGINT);
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index b4957465227..f558247c0df 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
 {
     ++i;
-    if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
+    if (i == end) throw UsageError("'%1%' requires an argument", opt);
     string s = *i;
     N multiplier = 1;
     if (allowUnit && !s.empty()) {
@@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
             else if (u == 'M') multiplier = 1ULL << 20;
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier '%1%'") % u);
+            else throw UsageError("invalid unit specifier '%1%'", u);
             s.resize(s.size() - 1);
         }
     }
     N n;
     if (!string2Int(s, n))
-        throw UsageError(format("'%1%' requires an integer argument") % opt);
+        throw UsageError("'%1%' requires an integer argument", opt);
     return n * multiplier;
 }
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 3a2d848615b..4b7385c6bd3 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -40,14 +40,14 @@ void BinaryCacheStore::init()
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
-            size_t colon = line.find(':');
-            if (colon == std::string::npos) continue;
+            size_t colon= line.find(':');
+            if (colon ==std::string::npos) continue;
             auto name = line.substr(0, colon);
             auto value = trim(line.substr(colon + 1, std::string::npos));
             if (name == "StoreDir") {
                 if (value != storeDir)
-                    throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
-                        % getUri() % value % storeDir);
+                    throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
+                        getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
                 wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 527d7ac42fb..2301adb16d1 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -453,7 +453,7 @@ static void commonChildInit(Pipe & logPipe)
        that e.g. ssh cannot open /dev/tty) and it doesn't receive
        terminal signals. */
     if (setsid() == -1)
-        throw SysError(format("creating a new session"));
+        throw SysError("creating a new session");
 
     /* Dup the write side of the logger pipe into stderr. */
     if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
@@ -466,7 +466,7 @@ static void commonChildInit(Pipe & logPipe)
     /* Reroute stdin to /dev/null. */
     int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
     if (fdDevNull == -1)
-        throw SysError(format("cannot open '%1%'") % pathNullDevice);
+        throw SysError("cannot open '%1%'", pathNullDevice);
     if (dup2(fdDevNull, STDIN_FILENO) == -1)
         throw SysError("cannot dup null device into stdin");
     close(fdDevNull);
@@ -488,7 +488,7 @@ void handleDiffHook(
 
             auto diffRes = runProgram(diffHookOptions);
             if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)));
+                throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first));
 
             if (diffRes.second != "")
                 printError(chomp(diffRes.second));
@@ -534,8 +534,8 @@ UserLock::UserLock()
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
     if (!gr)
-        throw Error(format("the group '%1%' specified in 'build-users-group' does not exist")
-            % settings.buildUsersGroup);
+        throw Error("the group '%1%' specified in 'build-users-group' does not exist",
+            settings.buildUsersGroup);
     gid = gr->gr_gid;
 
     /* Copy the result of getgrnam. */
@@ -546,8 +546,8 @@ UserLock::UserLock()
     }
 
     if (users.empty())
-        throw Error(format("the build users group '%1%' has no members")
-            % settings.buildUsersGroup);
+        throw Error("the build users group '%1%' has no members",
+            settings.buildUsersGroup);
 
     /* Find a user account that isn't currently in use for another
        build. */
@@ -556,8 +556,8 @@ UserLock::UserLock()
 
         struct passwd * pw = getpwnam(i.c_str());
         if (!pw)
-            throw Error(format("the user '%1%' in the group '%2%' does not exist")
-                % i % settings.buildUsersGroup);
+            throw Error("the user '%1%' in the group '%2%' does not exist",
+                i, settings.buildUsersGroup);
 
         createDirs(settings.nixStateDir + "/userpool");
 
@@ -565,7 +565,7 @@ UserLock::UserLock()
 
         AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
         if (!fd)
-            throw SysError(format("opening user lock '%1%'") % fnUserLock);
+            throw SysError("opening user lock '%1%'", fnUserLock);
 
         if (lockFile(fd.get(), ltWrite, false)) {
             fdUserLock = std::move(fd);
@@ -574,8 +574,8 @@ UserLock::UserLock()
 
             /* Sanity check... */
             if (uid == getuid() || uid == geteuid())
-                throw Error(format("the Nix user should not be a member of '%1%'")
-                    % settings.buildUsersGroup);
+                throw Error("the Nix user should not be a member of '%1%'",
+                    settings.buildUsersGroup);
 
 #if __linux__
             /* Get the list of supplementary groups of this build user.  This
@@ -585,7 +585,7 @@ UserLock::UserLock()
             int err = getgrouplist(pw->pw_name, pw->pw_gid,
                 supplementaryGIDs.data(), &ngroups);
             if (err == -1)
-                throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name);
+                throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
 
             supplementaryGIDs.resize(ngroups);
 #endif
@@ -594,9 +594,9 @@ UserLock::UserLock()
         }
     }
 
-    throw Error(format("all build users are currently in use; "
-        "consider creating additional users and adding them to the '%1%' group")
-        % settings.buildUsersGroup);
+    throw Error("all build users are currently in use; "
+        "consider creating additional users and adding them to the '%1%' group",
+        settings.buildUsersGroup);
 }
 
 
@@ -1985,7 +1985,7 @@ void DerivationGoal::startBuilder()
         string s = get(drv->env, "exportReferencesGraph").value_or("");
         Strings ss = tokenizeString<Strings>(s);
         if (ss.size() % 2 != 0)
-            throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s);
+            throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
         for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
             string fileName = *i++;
             static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
@@ -2034,7 +2034,7 @@ void DerivationGoal::startBuilder()
                     worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
             } catch (InvalidPath & e) {
             } catch (Error & e) {
-                throw Error(format("while processing 'sandbox-paths': %s") % e.what());
+                throw Error("while processing 'sandbox-paths': %s", e.what());
             }
         for (auto & i : closure) {
             auto p = worker.store.printStorePath(i);
@@ -2081,10 +2081,10 @@ void DerivationGoal::startBuilder()
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
         if (mkdir(chrootRootDir.c_str(), 0750) == -1)
-            throw SysError(format("cannot create '%1%'") % chrootRootDir);
+            throw SysError("cannot create '%1%'", chrootRootDir);
 
         if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir);
+            throw SysError("cannot change ownership of '%1%'", chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need
            this.  (Of course they should really respect $TMPDIR
@@ -2128,7 +2128,7 @@ void DerivationGoal::startBuilder()
         chmod_(chrootStoreDir, 01775);
 
         if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
-            throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir);
+            throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
 
         for (auto & i : inputPaths) {
             auto p = worker.store.printStorePath(i);
@@ -2161,7 +2161,7 @@ void DerivationGoal::startBuilder()
     if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
-            throw Error(format("directory '%1%' exists; please remove it") % homeDir);
+            throw Error("directory '%1%' exists; please remove it", homeDir);
 
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete
@@ -2206,8 +2206,7 @@ void DerivationGoal::startBuilder()
                 if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
                     state = stExtraChrootDirs;
                 } else {
-                    throw Error(format("unknown pre-build hook command '%1%'")
-                        % line);
+                    throw Error("unknown pre-build hook command '%1%'", line);
                 }
             } else if (state == stExtraChrootDirs) {
                 if (line == "") {
@@ -2967,7 +2966,7 @@ void DerivationGoal::chownToBuilder(const Path & path)
 {
     if (!buildUser) return;
     if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
-        throw SysError(format("cannot change ownership of '%1%'") % path);
+        throw SysError("cannot change ownership of '%1%'", path);
 }
 
 
@@ -3104,7 +3103,7 @@ void DerivationGoal::runChild()
             /* Bind-mount chroot directory to itself, to treat it as a
                different filesystem from /, as needed for pivot_root. */
             if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
-                throw SysError(format("unable to bind mount '%1%'") % chrootRootDir);
+                throw SysError("unable to bind mount '%1%'", chrootRootDir);
 
             /* Bind-mount the sandbox's Nix store onto itself so that
                we can mark it as a "shared" subtree, allowing bind
@@ -3238,16 +3237,16 @@ void DerivationGoal::runChild()
 
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)
-                throw SysError(format("cannot change directory to '%1%'") % chrootRootDir);
+                throw SysError("cannot change directory to '%1%'", chrootRootDir);
 
             if (mkdir("real-root", 0) == -1)
                 throw SysError("cannot create real-root directory");
 
             if (pivot_root(".", "real-root") == -1)
-                throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root"));
+                throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
 
             if (chroot(".") == -1)
-                throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir);
+                throw SysError("cannot change root directory to '%1%'", chrootRootDir);
 
             if (umount2("real-root", MNT_DETACH) == -1)
                 throw SysError("cannot unmount real root filesystem");
@@ -3268,7 +3267,7 @@ void DerivationGoal::runChild()
 #endif
 
         if (chdir(tmpDirInSandbox.c_str()) == -1)
-            throw SysError(format("changing into '%1%'") % tmpDir);
+            throw SysError("changing into '%1%'", tmpDir);
 
         /* Close all other file descriptors. */
         closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
@@ -3407,9 +3406,9 @@ void DerivationGoal::runChild()
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
                 for (auto & i : dirsInChroot) {
                     if (i.first != i.second.source)
-                        throw Error(format(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin")
-                            % i.first % i.second.source);
+                        throw Error(
+                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
+                            i.first, i.second.source);
 
                     string path = i.first;
                     struct stat st;
@@ -3500,7 +3499,7 @@ void DerivationGoal::runChild()
                 else if (drv->builder == "builtin:unpack-channel")
                     builtinUnpackChannel(drv2);
                 else
-                    throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
+                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
                 _exit(0);
             } catch (std::exception & e) {
                 writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n");
@@ -3510,7 +3509,7 @@ void DerivationGoal::runChild()
 
         execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 
-        throw SysError(format("executing '%1%'") % drv->builder);
+        throw SysError("executing '%1%'", drv->builder);
 
     } catch (std::exception & e) {
         writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n");
@@ -3595,7 +3594,7 @@ void DerivationGoal::registerOutputs()
                     replaceValidPath(path, actualPath);
                 else
                     if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1)
-                        throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
+                        throw SysError("moving build output '%1%' from the sandbox to the Nix store", path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
         }
@@ -3616,13 +3615,13 @@ void DerivationGoal::registerOutputs()
            user. */
         if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
             (buildUser && st.st_uid != buildUser->getUID()))
-            throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path);
+            throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path);
 #endif
 
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
         if (!outputRewrites.empty()) {
-            printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path);
+            printError("warning: rewriting hashes in '%1%'; cross fingers", path);
 
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
@@ -3654,7 +3653,8 @@ void DerivationGoal::registerOutputs()
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
-                        format("output path '%1%' should be a non-executable regular file") % path);
+                        "output path '%1%' should be a non-executable regular file",
+                        path);
             }
 
             /* Check the hash. In hash mode, move the path produced by
@@ -3715,7 +3715,7 @@ void DerivationGoal::registerOutputs()
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
                     if (rename(actualPath.c_str(), dst.c_str()))
-                        throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+                        throw SysError("renaming '%1%' to '%2%'", actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
@@ -4014,7 +4014,7 @@ Path DerivationGoal::openLogFile()
         settings.compressLog ? ".bz2" : "");
 
     fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
-    if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName);
+    if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
 
     logFileSink = std::make_shared<FdSink>(fdLogFile.get());
 
@@ -4522,7 +4522,6 @@ void SubstitutionGoal::handleEOF(int fd)
     if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
 }
 
-
 //////////////////////////////////////////////////////////////////////
 
 
@@ -4739,9 +4738,9 @@ void Worker::run(const Goals & _topGoals)
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
         else {
-            if (awake.empty() && 0 == settings.maxBuildJobs) throw Error(
-                "unable to start any build; either increase '--max-jobs' "
-                "or enable remote builds");
+            if (awake.empty() && 0 == settings.maxBuildJobs) 
+                throw Error("unable to start any build; either increase '--max-jobs' "
+                            "or enable remote builds");
             assert(!awake.empty());
         }
     }
@@ -4754,7 +4753,6 @@ void Worker::run(const Goals & _topGoals)
     assert(!settings.keepGoing || children.empty());
 }
 
-
 void Worker::waitForInput()
 {
     printMsg(lvlVomit, "waiting for children");
@@ -4895,6 +4893,9 @@ void Worker::waitForInput()
 }
 
 
+
+
+
 unsigned int Worker::exitStatus()
 {
     /*
@@ -4994,7 +4995,6 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
         throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
 }
 
-
 BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
@@ -5055,4 +5055,8 @@ void LocalStore::repairPath(const StorePath & path)
 }
 
 
+
 }
+
+
+
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 1b802d90817..00076909494 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -72,15 +72,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (!S_ISDIR(lstat(target).st_mode))
                         throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
-                        throw SysError(format("creating directory '%1%'"));
+                        throw SysError("creating directory '%1%'", dstFile);
                     createLinks(state, target, dstFile, state.priorities[dstFile]);
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         else {
@@ -99,11 +99,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     if (prevPriority < priority)
                         continue;
                     if (unlink(dstFile.c_str()) == -1)
-                        throw SysError(format("unlinking '%1%'") % dstFile);
+                        throw SysError("unlinking '%1%'", dstFile);
                 } else if (S_ISDIR(dstSt.st_mode))
                     throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
             } else if (errno != ENOENT)
-                throw SysError(format("getting status of '%1%'") % dstFile);
+                throw SysError("getting status of '%1%'", dstFile);
         }
 
         createSymlink(srcFile, dstFile);
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index f6ae5d2e622..223ed1fba62 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
     auto getAttr = [&](const string & name) {
         auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
@@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
-                throw SysError(format("making '%1%' executable") % storePath);
+                throw SysError("making '%1%' executable", storePath);
         }
     };
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index d985f151b18..f8b449e5e5e 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -747,7 +747,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     default:
-        throw Error(format("invalid operation %1%") % op);
+        throw Error("invalid operation %1%", op);
     }
 }
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 973ddc86abf..f35b8cc434c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -87,7 +87,7 @@ static void expect(std::istream & str, const string & s)
     char s2[s.size()];
     str.read(s2, s.size());
     if (string(s2, s.size()) != s)
-        throw FormatError(format("expected string '%1%'") % s);
+        throw FormatError("expected string '%1%'", s);
 }
 
 
@@ -114,7 +114,7 @@ static Path parsePath(std::istream & str)
 {
     string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
-        throw FormatError(format("bad path '%1%' in derivation") % s);
+        throw FormatError("bad path '%1%' in derivation", s);
     return s;
 }
 
@@ -196,7 +196,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
     try {
         return parseDerivation(store, readFile(drvPath));
     } catch (FormatError & e) {
-        throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
+        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
     }
 }
 
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 5967d04258b..7d0cb449ba5 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -113,7 +113,7 @@ struct CurlDownloader : public Downloader
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(DownloadError(Interrupted, "download of '%s' was interrupted", request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -518,7 +518,7 @@ struct CurlDownloader : public Downloader
             int running;
             CURLMcode mc = curl_multi_perform(curlm, &running);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
 
             /* Set the promises of any finished requests. */
             CURLMsg * msg;
@@ -548,7 +548,7 @@ struct CurlDownloader : public Downloader
             vomit("download thread waiting for %d ms", sleepTimeMs);
             mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
-                throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc));
+                throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 5a131c70450..8141928d2ba 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -128,8 +128,10 @@ class DownloadError : public Error
 {
 public:
     Downloader::Error error;
-    DownloadError(Downloader::Error error, const FormatOrString & fs)
-        : Error(fs), error(error)
+
+    template<typename... Args>
+    DownloadError(Downloader::Error error, const Args & ... args)
+        : Error(args...), error(error)
     { }
 };
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0c3d896112c..27bd3f3e946 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -38,7 +38,7 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
 
     AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
     if (!fdGCLock)
-        throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
+        throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
         printError(format("waiting for the big garbage collector lock..."));
@@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target)
 
     /* Atomically replace the old one. */
     if (rename(tempLink.c_str(), link.c_str()) == -1)
-        throw SysError(format("cannot rename '%1%' to '%2%'")
-            % tempLink % link);
+        throw SysError("cannot rename '%1%' to '%2%'",
+            tempLink , link);
 }
 
 
@@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     Path gcRoot(canonPath(_gcRoot));
 
     if (isInStore(gcRoot))
-        throw Error(format(
+        throw Error(
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
-                "(are you running nix-build inside the store?)") % gcRoot);
+                "(are you running nix-build inside the store?)", gcRoot);
 
     if (indirect) {
         /* Don't clobber the link if it already exists and doesn't
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
+            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
         makeSymlink(gcRoot, printStorePath(storePath));
         addIndirectRoot(gcRoot);
     }
@@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
             Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
 
             if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(format(
+                throw Error(
                     "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'")
-                    % gcRoot % rootsDir);
+                    "it's not in the directory '%2%'",
+                    gcRoot, rootsDir);
         }
 
         if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
@@ -170,7 +170,7 @@ void LocalStore::addTempRoot(const StorePath & path)
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting '%1%'") % fnTempRoots);
+                throw SysError("statting '%1%'", fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -211,7 +211,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         if (!*fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
-            throw SysError(format("opening temporary roots file '%1%'") % path);
+            throw SysError("opening temporary roots file '%1%'", path);
         }
 
         /* This should work, but doesn't, for some reason. */
@@ -222,7 +222,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file '%1%'") % path);
+            printError("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
             writeFull(fd->get(), "d");
             continue;
@@ -398,7 +398,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (!fdDir) {
                     if (errno == ENOENT || errno == EACCES)
                         continue;
-                    throw SysError(format("opening %1%") % fdStr);
+                    throw SysError("opening %1%", fdStr);
                 }
                 struct dirent * fd_ent;
                 while (errno = 0, fd_ent = readdir(fdDir.get())) {
@@ -408,7 +408,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                 if (errno) {
                     if (errno == ESRCH)
                         continue;
-                    throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
+                    throw SysError("iterating /proc/%1%/fd", ent->d_name);
                 }
                 fdDir.reset();
 
@@ -536,7 +536,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
     struct stat st;
     if (lstat(realPath.c_str(), &st)) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of %1%") % realPath);
+        throw SysError("getting status of %1%", realPath);
     }
 
     printInfo(format("deleting '%1%'") % path);
@@ -554,10 +554,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
         // size.
         try {
             if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError(format("making '%1%' writable") % realPath);
+                throw SysError("making '%1%' writable", realPath);
             Path tmp = trashDir + "/" + std::string(baseNameOf(path));
             if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
+                throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
             state.bytesInvalidated += size;
         } catch (SysError & e) {
             if (e.errNo == ENOSPC) {
@@ -676,7 +676,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 void LocalStore::removeUnusedLinks(const GCState & state)
 {
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+    if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
     long long actualSize = 0, unsharedSize = 0;
 
@@ -689,7 +689,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
 
         struct stat st;
         if (lstat(path.c_str(), &st) == -1)
-            throw SysError(format("statting '%1%'") % path);
+            throw SysError("statting '%1%'", path);
 
         if (st.st_nlink != 1) {
             actualSize += st.st_size;
@@ -700,14 +700,14 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
 
         if (unlink(path.c_str()) == -1)
-            throw SysError(format("deleting '%1%'") % path);
+            throw SysError("deleting '%1%'", path);
 
         state.results.bytesFreed += st.st_size;
     }
 
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError(format("statting '%1%'") % linksDir);
+        throw SysError("statting '%1%'", linksDir);
     long long overhead = st.st_blocks * 512ULL;
 
     printInfo(format("note: currently hard linking saves %.2f MiB")
@@ -801,7 +801,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         try {
 
             AutoCloseDir dir(opendir(realStoreDir.c_str()));
-            if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir);
+            if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
                aren't valid.  When using --max-freed etc., deleting
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 363be144334..48aca478cc1 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s)
     AutoDelete del(tmp, false);
     writeFile(tmp, s);
     if (rename(tmp.c_str(), path.c_str()))
-        throw SysError(format("renaming '%1%' to '%2%'") % tmp % path);
+        throw SysError("renaming '%1%' to '%2%'", tmp, path);
     del.cancel();
 }
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index aa5abd835d7..2d564a0d795 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor
     {
         Path storePath = store->toStorePath(path);
         if (!store->isValidPath(store->parseStorePath(storePath)))
-            throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
+            throw InvalidPath("path '%1%' is not a valid store path", storePath);
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
@@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor
         struct stat st;
         if (lstat(realPath.c_str(), &st)) {
             if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false};
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%1%'", path);
         }
 
         if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
-            throw Error(format("file '%1%' has unsupported type") % path);
+            throw Error("file '%1%' has unsupported type", path);
 
         return {
             S_ISREG(st.st_mode) ? Type::tRegular :
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ae7513ad815..db5a3b53cca 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -92,13 +92,13 @@ LocalStore::LocalStore(const Params & params)
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
-                throw SysError(format("getting attributes of path '%1%'") % realStoreDir);
+                throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
                 if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
-                    throw SysError(format("changing ownership of path '%1%'") % realStoreDir);
+                    throw SysError("changing ownership of path '%1%'", realStoreDir);
                 if (chmod(realStoreDir.c_str(), perm) == -1)
-                    throw SysError(format("changing permissions on path '%1%'") % realStoreDir);
+                    throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
     }
@@ -109,12 +109,12 @@ LocalStore::LocalStore(const Params & params)
         struct stat st;
         while (path != "/") {
             if (lstat(path.c_str(), &st))
-                throw SysError(format("getting status of '%1%'") % path);
+                throw SysError("getting status of '%1%'", path);
             if (S_ISLNK(st.st_mode))
-                throw Error(format(
+                throw Error(
                         "the path '%1%' is a symlink; "
-                        "this is not allowed for the Nix store and its parent directories")
-                    % path);
+                        "this is not allowed for the Nix store and its parent directories",
+                        path);
             path = dirOf(path);
         }
     }
@@ -155,8 +155,8 @@ LocalStore::LocalStore(const Params & params)
        upgrade.  */
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
-        throw Error(format("current Nix store schema is version %1%, but I only support %2%")
-            % curSchema % nixSchemaVersion);
+        throw Error("current Nix store schema is version %1%, but I only support %2%",
+             curSchema, nixSchemaVersion);
 
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
@@ -284,7 +284,7 @@ int LocalStore::getSchema()
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
         if (!string2Int(s, curSchema))
-            throw Error(format("'%1%' is corrupt") % schemaPath);
+            throw Error("'%1%' is corrupt", schemaPath);
     }
     return curSchema;
 }
@@ -293,7 +293,7 @@ int LocalStore::getSchema()
 void LocalStore::openDB(State & state, bool create)
 {
     if (access(dbDir.c_str(), R_OK | W_OK))
-        throw SysError(format("Nix database directory '%1%' is not writable") % dbDir);
+        throw SysError("Nix database directory '%1%' is not writable", dbDir);
 
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
@@ -367,7 +367,7 @@ void LocalStore::makeStoreWritable()
             throw SysError("setting up a private mount namespace");
 
         if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
-            throw SysError(format("remounting %1% writable") % realStoreDir);
+            throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
 }
@@ -388,7 +388,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
                  | 0444
                  | (st.st_mode & S_IXUSR ? 0111 : 0);
             if (chmod(path.c_str(), mode) == -1)
-                throw SysError(format("changing mode of '%1%' to %2$o") % path % mode);
+                throw SysError("changing mode of '%1%' to %2$o", path, mode);
         }
 
     }
@@ -406,7 +406,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 #else
         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 #endif
-            throw SysError(format("changing modification time of '%1%'") % path);
+            throw SysError("changing modification time of '%1%'", path);
     }
 }
 
@@ -415,7 +415,7 @@ void canonicaliseTimestampAndPermissions(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
     canonicaliseTimestampAndPermissions(path, st);
 }
 
@@ -430,17 +430,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
        setattrlist() to remove other attributes as well. */
     if (lchflags(path.c_str(), 0)) {
         if (errno != ENOTSUP)
-            throw SysError(format("clearing flags of path '%1%'") % path);
+            throw SysError("clearing flags of path '%1%'", path);
     }
 #endif
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error(format("file '%1%' has an unsupported type") % path);
+        throw Error("file '%1%' has an unsupported type", path);
 
 #if __linux__
     /* Remove extended attributes / ACLs. */
@@ -474,7 +474,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
         assert(!S_ISDIR(st.st_mode));
         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
-            throw BuildError(format("invalid ownership on file '%1%'") % path);
+            throw BuildError("invalid ownership on file '%1%'", path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
         return;
@@ -498,8 +498,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
         if (!S_ISLNK(st.st_mode) &&
             chown(path.c_str(), geteuid(), getegid()) == -1)
 #endif
-            throw SysError(format("changing owner of '%1%' to %2%")
-                % path % geteuid());
+            throw SysError("changing owner of '%1%' to %2%",
+                  path, geteuid());
     }
 
     if (S_ISDIR(st.st_mode)) {
@@ -518,11 +518,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
        be a symlink, since we can't change its ownership. */
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
-        throw Error(format("wrong ownership of top-level store path '%1%'") % path);
+        throw Error("wrong ownership of top-level store path '%1%'", path);
     }
 }
 
@@ -1392,7 +1392,7 @@ static void makeMutable(const Path & path)
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
     if (fd == -1) {
         if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     }
 
     unsigned int flags = 0, old;
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index b74480684f2..ca663d83759 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor
         auto i = get(path);
 
         if (i.type != FSAccessor::Type::tDirectory)
-            throw Error(format("path '%1%' inside NAR file is not a directory") % path);
+            throw Error("path '%1%' inside NAR file is not a directory", path);
 
         StringSet res;
         for (auto & child : i.children)
@@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)
-            throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
+            throw Error("path '%1%' inside NAR file is not a regular file", path);
 
         if (getNarBytes) return getNarBytes(i.start, i.size);
 
@@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tSymlink)
-            throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
+            throw Error("path '%1%' inside NAR file is not a symlink", path);
         return i.target;
     }
 };
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 1375094b5bf..fbdcd778685 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -7,7 +7,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
     : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack
 {
     auto corrupt = [&]() {
-        throw Error(format("NAR info file '%1%' is corrupt") % whence);
+        throw Error("NAR info file '%1%' is corrupt", whence);
     };
 
     auto parseHashField = [&](const string & s) {
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 8ac382e9dbf..f9cf2bb6b87 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -19,9 +19,9 @@ static void makeWritable(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-        throw SysError(format("changing writability of '%1%'") % path);
+        throw SysError("changing writability of '%1%'", path);
 }
 
 
@@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
     InodeHash inodeHash;
 
     AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % linksDir);
+    if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
         // We don't care if we hit non-hash files, anything goes
         inodeHash.insert(dirent->d_ino);
     }
-    if (errno) throw SysError(format("reading directory '%1%'") % linksDir);
+    if (errno) throw SysError("reading directory '%1%'", linksDir);
 
     printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
 
@@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
     Strings names;
 
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+    if (!dir) throw SysError("opening directory '%1%'", path);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
         if (name == "." || name == "..") continue;
         names.push_back(name);
     }
-    if (errno) throw SysError(format("reading directory '%1%'") % path);
+    if (errno) throw SysError("reading directory '%1%'", path);
 
     return names;
 }
@@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
 #if __APPLE__
     /* HFS/macOS has some undocumented security feature disabling hardlinking for
@@ -130,7 +130,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        printError(format("skipping suspicious writable file '%1%'") % path);
+        printError("skipping suspicious writable file '%1%'", path);
         return;
     }
 
@@ -186,7 +186,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        current file with a hard link to that file. */
     struct stat stLink;
     if (lstat(linkPath.c_str(), &stLink))
-        throw SysError(format("getting attributes of path '%1%'") % linkPath);
+        throw SysError("getting attributes of path '%1%'", linkPath);
 
     if (st.st_ino == stLink.st_ino) {
         debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
@@ -194,7 +194,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        printError(format("removing corrupted link '%1%'") % linkPath);
+        printError("removing corrupted link '%1%'", linkPath);
         unlink(linkPath.c_str());
         goto retry;
     }
@@ -229,7 +229,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            printError(format("unable to unlink '%1%'") % tempLink);
+            printError("unable to unlink '%1%'", tempLink);
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
@@ -238,7 +238,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
             debug("'%s' has reached maximum number of links", linkPath);
             return;
         }
-        throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
+        throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
     }
 
     stats.filesLinked++;
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 2635e3940af..52d430ffd68 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create)
 
     fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
     if (!fd && (create || errno != ENOENT))
-        throw SysError(format("opening lock file '%1%'") % path);
+        throw SysError("opening lock file '%1%'", path);
 
     return fd;
 }
@@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
         while (flock(fd, type) != 0) {
             checkInterrupt();
             if (errno != EINTR)
-                throw SysError(format("acquiring/releasing lock"));
+                throw SysError("acquiring/releasing lock");
             else
                 return false;
         }
@@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
             checkInterrupt();
             if (errno == EWOULDBLOCK) return false;
             if (errno != EINTR)
-                throw SysError(format("acquiring/releasing lock"));
+                throw SysError("acquiring/releasing lock");
         }
     }
 
@@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths,
                hasn't been unlinked). */
             struct stat st;
             if (fstat(fd.get(), &st) == -1)
-                throw SysError(format("statting lock file '%1%'") % lockPath);
+                throw SysError("statting lock file '%1%'", lockPath);
             if (st.st_size != 0)
                 /* This lock file has been unlinked, so we're holding
                    a lock on a deleted file.  This means that other
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 2bef518786c..6cfe393a45e 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen)
             gen.number = n;
             struct stat st;
             if (lstat(gen.path.c_str(), &st) != 0)
-                throw SysError(format("statting '%1%'") % gen.path);
+                throw SysError("statting '%1%'", gen.path);
             gen.creationTime = st.st_mtime;
             gens.push_back(gen);
         }
@@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
 static void removeFile(const Path & path)
 {
     if (remove(path.c_str()) == -1)
-        throw SysError(format("cannot unlink '%1%'") % path);
+        throw SysError("cannot unlink '%1%'", path);
 }
 
 
@@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens
     Generations gens = findGenerations(profile, curGen);
 
     if (gensToDelete.find(curGen) != gensToDelete.end())
-        throw Error(format("cannot delete current generation of profile %1%'") % profile);
+        throw Error("cannot delete current generation of profile %1%'", profile);
 
     for (auto & i : gens) {
         if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
@@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
     int days;
 
     if (!string2Int(strDays, days) || days < 1)
-        throw Error(format("invalid number of days specifier '%1%'") % timeSpec);
+        throw Error("invalid number of days specifier '%1%'", timeSpec);
 
     time_t oldTime = curTime - days * 24 * 3600;
 
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 102e1592180..a10d536a355 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path,
         auto baseName = std::string(baseNameOf(i));
         string::size_type pos = baseName.find('-');
         if (pos == string::npos)
-            throw Error(format("bad reference '%1%'") % i);
+            throw Error("bad reference '%1%'", i);
         string s = string(baseName, 0, pos);
         assert(s.size() == refLength);
         assert(backMap.find(s) == backMap.end());
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 5a2d103b973..9277a8e6bc9 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     std::string restPath = std::string(path, storePath.size());
 
     if (!store->isValidPath(store->parseStorePath(storePath)))
-        throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
+        throw InvalidPath("path '%1%' is not a valid store path", storePath);
 
     auto i = nars.find(storePath);
     if (i != nars.end()) return {i->second, restPath};
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8c55da2686e..0f8126aee23 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
     if (socketPath.size() + 1 >= sizeof(addr.sun_path))
-        throw Error(format("socket path '%1%' is too long") % socketPath);
+        throw Error("socket path '%1%' is too long", socketPath);
     strcpy(addr.sun_path, socketPath.c_str());
 
     if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
-        throw SysError(format("cannot connect to daemon at '%1%'") % socketPath);
+        throw SysError("cannot connect to daemon at '%1%'", socketPath);
 
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f2e4b63e059..0326821f66b 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -32,8 +32,10 @@ namespace nix {
 struct S3Error : public Error
 {
     Aws::S3::S3Errors err;
-    S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
-        : Error(fs), err(err) { };
+
+    template<typename... Args>
+    S3Error(Aws::S3::S3Errors err, const Args & ... args)
+        : Error(args...), err(err) { };
 };
 
 /* Helper: given an Outcome<R, E>, return R in case of success, or
@@ -249,7 +251,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                 // If bucket listing is disabled, 404s turn into 403s
                 || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                 return false;
-            throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
+            throw Error("AWS error fetching '%s': %s", path, error.GetMessage());
         }
 
         return true;
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index eb1daafc5b8..3407e582639 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -29,7 +29,7 @@ SQLite::SQLite(const Path & path, bool create)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open SQLite database '%s'") % path);
+        throw Error("cannot open SQLite database '%s'", path);
 
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
         throwSQLiteError(db, "setting timeout");
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b9e894a9a3f..a04d5013ee5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -22,7 +22,7 @@ bool Store::isInStore(const Path & path) const
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw Error("path '%1%' is not in the Nix store", path);
     Path::size_type slash = path.find('/', storeDir.size() + 1);
     if (slash == Path::npos)
         return path;
@@ -40,7 +40,7 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw Error("path '%1%' is not in the Nix store", path);
     return path;
 }
 
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index db544a212ab..6a848470520 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -46,7 +46,7 @@ static void dumpContents(const Path & path, size_t size,
     sink << "contents" << size;
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file '%1%'") % path);
+    if (!fd) throw SysError("opening file '%1%'", path);
 
     std::vector<unsigned char> buf(65536);
     size_t left = size;
@@ -68,7 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
 
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path '%1%'") % path);
+        throw SysError("getting attributes of path '%1%'", path);
 
     sink << "(";
 
@@ -94,8 +94,9 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
                     name.erase(pos);
                 }
                 if (unhacked.find(name) != unhacked.end())
-                    throw Error(format("file name collision in between '%1%' and '%2%'")
-                        % (path + "/" + unhacked[name]) % (path + "/" + i.name));
+                    throw Error("file name collision in between '%1%' and '%2%'",
+                       (path + "/" + unhacked[name]),
+                       (path + "/" + i.name));
                 unhacked[name] = i.name;
             } else
                 unhacked[i.name] = i.name;
@@ -111,7 +112,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
     else if (S_ISLNK(st.st_mode))
         sink << "type" << "symlink" << "target" << readLink(path);
 
-    else throw Error(format("file '%1%' has an unsupported type") % path);
+    else throw Error("file '%1%' has an unsupported type", path);
 
     sink << ")";
 }
@@ -247,7 +248,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                 } else if (s == "name") {
                     name = readString(source);
                     if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
-                        throw Error(format("NAR contains invalid file name '%1%'") % name);
+                        throw Error("NAR contains invalid file name '%1%'", name);
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
@@ -303,14 +304,14 @@ struct RestoreSink : ParseSink
     {
         Path p = dstPath + path;
         if (mkdir(p.c_str(), 0777) == -1)
-            throw SysError(format("creating directory '%1%'") % p);
+            throw SysError("creating directory '%1%'", p);
     };
 
     void createRegularFile(const Path & path)
     {
         Path p = dstPath + path;
         fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
-        if (!fd) throw SysError(format("creating file '%1%'") % p);
+        if (!fd) throw SysError("creating file '%1%'", p);
     }
 
     void isExecutable()
@@ -332,7 +333,7 @@ struct RestoreSink : ParseSink
                OpenSolaris).  Since preallocation is just an
                optimisation, ignore it. */
             if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
-                throw SysError(format("preallocating file of %1% bytes") % len);
+                throw SysError("preallocating file of %1% bytes", len);
         }
 #endif
     }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ba15ea57109..4e705dd6496 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -47,7 +47,7 @@ void Args::parseCmdline(const Strings & _cmdline)
         }
         else if (!dashDash && std::string(arg, 0, 1) == "-") {
             if (!processFlag(pos, cmdline.end()))
-                throw UsageError(format("unrecognised flag '%1%'") % arg);
+                throw UsageError("unrecognised flag '%1%'", arg);
         }
         else {
             pendingArgs.push_back(*pos++);
@@ -104,8 +104,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         for (size_t n = 0 ; n < flag.arity; ++n) {
             if (pos == end) {
                 if (flag.arity == ArityAny) break;
-                throw UsageError(format("flag '%1%' requires %2% argument(s)")
-                    % name % flag.arity);
+                throw UsageError("flag '%1%' requires %2% argument(s)",
+                    name,
+                    flag.arity);
             }
             args.push_back(*pos++);
         }
@@ -133,7 +134,7 @@ bool Args::processArgs(const Strings & args, bool finish)
 {
     if (expectedArgs.empty()) {
         if (!args.empty())
-            throw UsageError(format("unexpected argument '%1%'") % args.front());
+            throw UsageError("unexpected argument '%1%'", args.front());
         return true;
     }
 
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 860b04adb79..a117ddc72e1 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -481,7 +481,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
     else if (method == "br")
         return make_ref<BrotliCompressionSink>(nextSink);
     else
-        throw UnknownCompressionMethod(format("unknown compression method '%s'") % method);
+        throw UnknownCompressionMethod("unknown compression method '%s'", method);
 }
 
 ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 08edffee0a6..d65215063a3 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -92,15 +92,15 @@ public:
     {
         fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
     }
+    hintformat(const hintformat &hf)
+    : fmt(hf.fmt)
+    {}
     template<class T>
     hintformat& operator%(const T &value)
     {
         fmt % yellowify(value);
         return *this;
     }
-    hintformat(const hintformat &hf)
-    : fmt(hf.fmt)
-    {}
 
 
     std::string str() const
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 633c6bdf760..2261adb8260 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -93,7 +93,9 @@ public:
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err(hintfmt(args...))
+        : err { .level = lvlError,
+                .hint = hintfmt(args...)
+              }
         , status(status)
     {
     }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 332c1c43a76..bffcbcccfd0 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -129,7 +129,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
     string s;
 
     if (path[0] != '/')
-        throw Error(format("not an absolute path: '%1%'") % path);
+        throw Error("not an absolute path: '%1%'", path);
 
     string::const_iterator i = path.begin(), end = path.end();
     string temp;
@@ -165,7 +165,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
                the symlink target might contain new symlinks). */
             if (resolveSymlinks && isLink(s)) {
                 if (++followCount >= maxFollow)
-                    throw Error(format("infinite symlink recursion in path '%1%'") % path);
+                    throw Error("infinite symlink recursion in path '%1%'", path);
                 temp = absPath(readLink(s), dirOf(s))
                     + string(i, end);
                 i = temp.begin(); /* restart */
@@ -226,7 +226,7 @@ struct stat lstat(const Path & path)
 {
     struct stat st;
     if (lstat(path.c_str(), &st))
-        throw SysError(format("getting status of '%1%'") % path);
+        throw SysError("getting status of '%1%'", path);
     return st;
 }
 
@@ -238,7 +238,7 @@ bool pathExists(const Path & path)
     res = lstat(path.c_str(), &st);
     if (!res) return true;
     if (errno != ENOENT && errno != ENOTDIR)
-        throw SysError(format("getting status of %1%") % path);
+        throw SysError("getting status of %1%", path);
     return false;
 }
 
@@ -274,7 +274,7 @@ DirEntries readDirectory(const Path & path)
     entries.reserve(64);
 
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+    if (!dir) throw SysError("opening directory '%1%'", path);
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@@ -289,7 +289,7 @@ DirEntries readDirectory(const Path & path)
 #endif
         );
     }
-    if (errno) throw SysError(format("reading directory '%1%'") % path);
+    if (errno) throw SysError("reading directory '%1%'", path);
 
     return entries;
 }
@@ -322,7 +322,7 @@ string readFile(const Path & path, bool drain)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd)
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     return drain ? drainFD(fd.get()) : readFile(fd.get());
 }
 
@@ -339,7 +339,7 @@ void writeFile(const Path & path, const string & s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     writeFull(fd.get(), s);
 }
 
@@ -348,7 +348,7 @@ void writeFile(const Path & path, Source & source, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
 
     std::vector<unsigned char> buf(64 * 1024);
 
@@ -396,7 +396,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
     struct stat st;
     if (lstat(path.c_str(), &st) == -1) {
         if (errno == ENOENT) return;
-        throw SysError(format("getting status of '%1%'") % path);
+        throw SysError("getting status of '%1%'", path);
     }
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
@@ -407,7 +407,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
         const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
         if ((st.st_mode & PERM_MASK) != PERM_MASK) {
             if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1)
-                throw SysError(format("chmod '%1%'") % path);
+                throw SysError("chmod '%1%'", path);
         }
 
         for (auto & i : readDirectory(path))
@@ -416,7 +416,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
     if (remove(path.c_str()) == -1) {
         if (errno == ENOENT) return;
-        throw SysError(format("cannot unlink '%1%'") % path);
+        throw SysError("cannot unlink '%1%'", path);
     }
 }
 
@@ -468,12 +468,12 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
                "wheel", then "tar" will fail to unpack archives that
                have the setgid bit set on directories. */
             if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError(format("setting group of directory '%1%'") % tmpDir);
+                throw SysError("setting group of directory '%1%'", tmpDir);
 #endif
             return tmpDir;
         }
         if (errno != EEXIST)
-            throw SysError(format("creating directory '%1%'") % tmpDir);
+            throw SysError("creating directory '%1%'", tmpDir);
     }
 }
 
@@ -555,15 +555,15 @@ Paths createDirs(const Path & path)
     if (lstat(path.c_str(), &st) == -1) {
         created = createDirs(dirOf(path));
         if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
-            throw SysError(format("creating directory '%1%'") % path);
+            throw SysError("creating directory '%1%'", path);
         st = lstat(path);
         created.push_back(path);
     }
 
     if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
-        throw SysError(format("statting symlink '%1%'") % path);
+        throw SysError("statting symlink '%1%'", path);
 
-    if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path);
+    if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
 
     return created;
 }
@@ -572,7 +572,7 @@ Paths createDirs(const Path & path)
 void createSymlink(const Path & target, const Path & link)
 {
     if (symlink(target.c_str(), link.c_str()))
-        throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target);
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
 }
 
 
@@ -589,7 +589,7 @@ void replaceSymlink(const Path & target, const Path & link)
         }
 
         if (rename(tmp.c_str(), link.c_str()) != 0)
-            throw SysError(format("renaming '%1%' to '%2%'") % tmp % link);
+            throw SysError("renaming '%1%' to '%2%'", tmp, link);
 
         break;
     }
@@ -694,7 +694,7 @@ AutoDelete::~AutoDelete()
                 deletePath(path);
             else {
                 if (remove(path.c_str()) == -1)
-                    throw SysError(format("cannot unlink '%1%'") % path);
+                    throw SysError("cannot unlink '%1%'", path);
             }
         }
     } catch (...) {
@@ -760,7 +760,7 @@ void AutoCloseFD::close()
     if (fd != -1) {
         if (::close(fd) == -1)
             /* This should never happen. */
-            throw SysError(format("closing file descriptor %1%") % fd);
+            throw SysError("closing file descriptor %1%", fd);
     }
 }
 
@@ -920,7 +920,7 @@ void killUser(uid_t uid)
 #endif
             if (errno == ESRCH) break; /* no more processes */
             if (errno != EINTR)
-                throw SysError(format("cannot kill processes for uid '%1%'") % uid);
+                throw SysError("cannot kill processes for uid '%1%'", uid);
         }
 
         _exit(0);
@@ -928,7 +928,7 @@ void killUser(uid_t uid)
 
     int status = pid.wait();
     if (status != 0)
-        throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status));
+        throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
 
     /* !!! We should really do some check to make sure that there are
        no processes left running under `uid', but there is no portable
@@ -1322,7 +1322,7 @@ void ignoreException()
     try {
         throw;
     } catch (std::exception & e) {
-        printError(format("error (ignored): %1%") % e.what());
+        printError("error (ignored): %1%", e.what());
     }
 }
 
@@ -1440,7 +1440,7 @@ void callFailure(const std::function<void(std::exception_ptr exc)> & failure, st
     try {
         failure(exc);
     } catch (std::exception & e) {
-        printError(format("uncaught exception: %s") % e.what());
+        printError("uncaught exception: %s", e.what());
         abort();
     }
 }
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index a2639579dd9..c403e37804d 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -37,7 +37,8 @@ static void writeChannels()
 {
     auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)};
     if (!channelsFD)
-        throw SysError(format("opening '%1%' for writing") % channelsList);
+        throw Error("");
+        // throw SysError("opening '%1%' for writing", channelsList);
     for (const auto & channel : channels)
         writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n");
 }
@@ -46,9 +47,11 @@ static void writeChannels()
 static void addChannel(const string & url, const string & name)
 {
     if (!regex_search(url, std::regex("^(file|http|https)://")))
-        throw Error(format("invalid channel URL '%1%'") % url);
+        throw Error("");
+        // throw Error("invalid channel URL '%1%'", url);
     if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$")))
-        throw Error(format("invalid channel identifier '%1%'") % name);
+        throw Error("");
+        // throw Error("invalid channel identifier '%1%'", name);
     readChannels();
     channels[name] = url;
     writeChannels();
@@ -140,9 +143,11 @@ static void update(const StringSet & channelNames)
         if (S_ISLNK(st.st_mode))
             // old-skool ~/.nix-defexpr
             if (unlink(nixDefExpr.c_str()) == -1)
-                throw SysError(format("unlinking %1%") % nixDefExpr);
+                throw Error("");
+        //         throw SysError("unlinking %1%", nixDefExpr);
     } else if (errno != ENOENT) {
-        throw SysError(format("getting status of %1%") % nixDefExpr);
+        throw Error("");
+        // throw SysError("getting status of %1%", nixDefExpr);
     }
     createDirs(nixDefExpr);
     auto channelLink = nixDefExpr + "/channels";
@@ -194,10 +199,12 @@ static int _main(int argc, char ** argv)
 
         switch (cmd) {
             case cNone:
-                throw UsageError("no command specified");
+                throw Error("");
+        //         throw UsageError("no command specified");
             case cAdd:
                 if (args.size() < 1 || args.size() > 2)
-                    throw UsageError("'--add' requires one or two arguments");
+                    throw Error("");
+        //             throw UsageError("'--add' requires one or two arguments");
                 {
                 auto url = args[0];
                 std::string name;
@@ -213,12 +220,14 @@ static int _main(int argc, char ** argv)
                 break;
             case cRemove:
                 if (args.size() != 1)
-                    throw UsageError("'--remove' requires one argument");
+                    throw Error("");
+        //             throw UsageError("'--remove' requires one argument");
                 removeChannel(args[0]);
                 break;
             case cList:
                 if (!args.empty())
-                    throw UsageError("'--list' expects no arguments");
+                    throw Error("");
+        //             throw UsageError("'--list' expects no arguments");
                 readChannels();
                 for (const auto & channel : channels)
                     std::cout << channel.first << ' ' << channel.second << '\n';
@@ -228,7 +237,8 @@ static int _main(int argc, char ** argv)
                 break;
             case cRollback:
                 if (args.size() > 1)
-                    throw UsageError("'--rollback' has at most one argument");
+                    throw Error("");
+        //             throw UsageError("'--rollback' has at most one argument");
                 Strings envArgs{"--profile", profile};
                 if (args.size() == 1) {
                     envArgs.push_back("--switch-generation");
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index e68d1b1bed6..5f78ab46409 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -36,7 +36,7 @@ using namespace nix::daemon;
 #define SPLICE_F_MOVE 0
 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags)
 {
-    /* We ignore most parameters, we just have them for conformance with the linux syscall */
+    // We ignore most parameters, we just have them for conformance with the linux syscall 
     std::vector<char> buf(8192);
     auto read_count = read(fd_in, buf.data(), buf.size());
     if (read_count == -1)
@@ -57,7 +57,7 @@ static void sigChldHandler(int sigNo)
 {
     // Ensure we don't modify errno of whatever we've interrupted
     auto saved_errno = errno;
-    /* Reap all dead children. */
+    //  Reap all dead children. 
     while (waitpid(-1, 0, WNOHANG) > 0) ;
     errno = saved_errno;
 }
@@ -106,7 +106,7 @@ struct PeerInfo
 };
 
 
-/* Get the identity of the caller, if possible. */
+//  Get the identity of the caller, if possible. 
 static PeerInfo getPeerInfo(int remote)
 {
     PeerInfo peer = { false, 0, false, 0, false, 0 };
@@ -154,13 +154,12 @@ static void daemonLoop(char * * argv)
     if (chdir("/") == -1)
         throw SysError("cannot change current directory");
 
-    /* Get rid of children automatically; don't let them become
-       zombies. */
+    //  Get rid of children automatically; don't let them become zombies. 
     setSigChldAction(true);
 
     AutoCloseFD fdSocket;
 
-    /* Handle socket-based activation by systemd. */
+    //  Handle socket-based activation by systemd. 
     auto listenFds = getEnv("LISTEN_FDS");
     if (listenFds) {
         if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1")
@@ -169,17 +168,17 @@ static void daemonLoop(char * * argv)
         closeOnExec(fdSocket.get());
     }
 
-    /* Otherwise, create and bind to a Unix domain socket. */
+    //  Otherwise, create and bind to a Unix domain socket. 
     else {
         createDirs(dirOf(settings.nixDaemonSocketFile));
         fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
     }
 
-    /* Loop accepting connections. */
+    //  Loop accepting connections. 
     while (1) {
 
         try {
-            /* Accept a connection. */
+            //  Accept a connection. 
             struct sockaddr_un remoteAddr;
             socklen_t remoteAddrLen = sizeof(remoteAddr);
 
@@ -209,13 +208,13 @@ static void daemonLoop(char * * argv)
                 trusted = Trusted;
 
             if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup)
-                throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user);
+                throw Error("user '%1%' is not allowed to connect to the Nix daemon", user);
 
             printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""))
                 % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>")
                 % (peer.uidKnown ? user : "<unknown>"));
 
-            /* Fork a child to handle the connection. */
+            //  Fork a child to handle the connection. 
             ProcessOptions options;
             options.errorPrefix = "unexpected Nix daemon error: ";
             options.dieWithParent = false;
@@ -224,20 +223,20 @@ static void daemonLoop(char * * argv)
             startProcess([&]() {
                 fdSocket = -1;
 
-                /* Background the daemon. */
+                //  Background the daemon. 
                 if (setsid() == -1)
-                    throw SysError(format("creating a new session"));
+                    throw SysError("creating a new session");
 
-                /* Restore normal handling of SIGCHLD. */
+                //  Restore normal handling of SIGCHLD. 
                 setSigChldAction(false);
 
-                /* For debugging, stuff the pid into argv[1]. */
+                //  For debugging, stuff the pid into argv[1]. 
                 if (peer.pidKnown && argv[1]) {
                     string processName = std::to_string(peer.pid);
                     strncpy(argv[1], processName.c_str(), strlen(argv[1]));
                 }
 
-                /* Handle the connection. */
+                //  Handle the connection. 
                 FdSource from(remote.get());
                 FdSink to(remote.get());
                 processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid);
@@ -248,7 +247,7 @@ static void daemonLoop(char * * argv)
         } catch (Interrupted & e) {
             return;
         } catch (Error & e) {
-            printError(format("error processing connection: %1%") % e.msg());
+            printError("error processing connection: %1%", e.msg());
         }
     }
 }
@@ -261,7 +260,7 @@ static int _main(int argc, char * * argv)
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
-                ; /* ignored for backwards compatibility */
+                ; //  ignored for backwards compatibility 
             else if (*arg == "--help")
                 showManPage("nix-daemon");
             else if (*arg == "--version")
@@ -276,7 +275,7 @@ static int _main(int argc, char * * argv)
 
         if (stdio) {
             if (getStoreType() == tDaemon) {
-                /* Forward on this connection to the real daemon */
+                //  Forward on this connection to the real daemon 
                 auto socketPath = settings.nixDaemonSocketFile;
                 auto s = socket(PF_UNIX, SOCK_STREAM, 0);
                 if (s == -1)
@@ -284,17 +283,17 @@ static int _main(int argc, char * * argv)
 
                 auto socketDir = dirOf(socketPath);
                 if (chdir(socketDir.c_str()) == -1)
-                    throw SysError(format("changing to socket directory '%1%'") % socketDir);
+                    throw SysError("changing to socket directory '%1%'", socketDir);
 
                 auto socketName = std::string(baseNameOf(socketPath));
                 auto addr = sockaddr_un{};
                 addr.sun_family = AF_UNIX;
                 if (socketName.size() + 1 >= sizeof(addr.sun_path))
-                    throw Error(format("socket name %1% is too long") % socketName);
+                    throw Error("socket name %1% is too long", socketName);
                 strcpy(addr.sun_path, socketName.c_str());
 
                 if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                    throw SysError(format("cannot connect to daemon at %1%") % socketPath);
+                    throw SysError("cannot connect to daemon at %1%", socketPath);
 
                 auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1;
                 while (true) {
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 1a2bb42a3c2..dde8875f1c8 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -25,7 +25,6 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-
 using namespace nix;
 using std::cout;
 
@@ -70,8 +69,7 @@ typedef void (* Operation) (Globals & globals,
 static string needArg(Strings::iterator & i,
     Strings & args, const string & arg)
 {
-    if (i == args.end()) throw UsageError(
-        format("'%1%' requires an argument") % arg);
+    if (i == args.end()) throw UsageError( "'%1%' requires an argument", arg);
     return *i++;
 }
 
@@ -133,7 +131,7 @@ static void getAllExprs(EvalState & state,
             Value & vArg(*state.allocValue());
             mkString(vArg, path2);
             if (v.attrs->size() == v.attrs->capacity())
-                throw Error(format("too many Nix expressions in directory '%1%'") % path);
+                throw Error("too many Nix expressions in directory '%1%'", path);
             mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg);
         }
         else if (S_ISDIR(st.st_mode))
@@ -144,11 +142,12 @@ static void getAllExprs(EvalState & state,
 }
 
 
+
 static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
 {
     struct stat st;
     if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting information about '%1%'") % path);
+        throw SysError("getting inon about '%1%'", path);
 
     if (isNixExpr(path, st))
         state.evalFile(path, v);
@@ -221,7 +220,7 @@ static void checkSelectorUse(DrvNames & selectors)
     /* Check that all selectors have been used. */
     for (auto & i : selectors)
         if (i.hits == 0 && i.fullName != "*")
-            throw Error(format("selector '%1%' matches no derivations") % i.fullName);
+            throw Error("selector '%1%' matches no derivations", i.fullName);
 }
 
 
@@ -507,7 +506,7 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs)
             globals.preserveInstalled = true;
         else if (arg == "--remove-all" || arg == "-r")
             globals.removeAll = true;
-        else throw UsageError(format("unknown flag '%1%'") % arg);
+        else throw UsageError("unknown flag '%1%'", arg);
     }
 
     installDerivations(globals, opArgs, globals.profile);
@@ -618,7 +617,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs)
         else if (arg == "--leq") upgradeType = utLeq;
         else if (arg == "--eq") upgradeType = utEq;
         else if (arg == "--always") upgradeType = utAlways;
-        else throw UsageError(format("unknown flag '%1%'") % arg);
+        else throw UsageError("unknown flag '%1%'", arg);
     }
 
     upgradeDerivations(globals, opArgs, upgradeType);
@@ -637,7 +636,7 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv,
 static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     if (opArgs.size() < 2)
         throw UsageError("not enough arguments to '--set-flag'");
 
@@ -680,7 +679,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
     for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) {
         string arg = *i++;
         if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
-        else throw UsageError(format("unknown flag '%1%'") % arg);
+        else throw UsageError("unknown flag '%1%'", arg);
     }
 
     DrvInfos elems;
@@ -748,7 +747,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
 static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     uninstallDerivations(globals, opArgs, globals.profile);
 }
 
@@ -911,7 +910,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         else if (arg == "--attr" || arg == "-A")
             attrPath = needArg(i, opFlags, arg);
         else
-            throw UsageError(format("unknown flag '%1%'") % arg);
+            throw UsageError("unknown flag '%1%'", arg);
     }
 
     if (printAttrPath && source != sAvailable)
@@ -1177,9 +1176,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     if (opArgs.size() != 1)
-        throw UsageError(format("exactly one argument expected"));
+        throw UsageError("exactly one argument expected");
 
     Path profile = absPath(opArgs.front());
     Path profileLink = getHome() + "/.nix-profile";
@@ -1207,10 +1206,10 @@ static void switchGeneration(Globals & globals, int dstGen)
 
     if (!dst) {
         if (dstGen == prevGen)
-            throw Error(format("no generation older than the current (%1%) exists")
-                % curGen);
+            throw Error("no generation older than the current (%1%) exists",
+                        curGen);
         else
-            throw Error(format("generation %1% does not exist") % dstGen);
+            throw Error("generation %1% does not exist", dstGen);
     }
 
     printInfo(format("switching from generation %1% to %2%")
@@ -1225,13 +1224,13 @@ static void switchGeneration(Globals & globals, int dstGen)
 static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     if (opArgs.size() != 1)
-        throw UsageError(format("exactly one argument expected"));
+        throw UsageError("exactly one argument expected");
 
     int dstGen;
     if (!string2Int(opArgs.front(), dstGen))
-        throw UsageError(format("expected a generation number"));
+        throw UsageError("expected a generation number");
 
     switchGeneration(globals, dstGen);
 }
@@ -1240,9 +1239,9 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
 static void opRollback(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     if (opArgs.size() != 0)
-        throw UsageError(format("no arguments expected"));
+        throw UsageError("no arguments expected");
 
     switchGeneration(globals, prevGen);
 }
@@ -1251,9 +1250,9 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs)
 static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
     if (opArgs.size() != 0)
-        throw UsageError(format("no arguments expected"));
+        throw UsageError("no arguments expected");
 
     PathLocks lock;
     lockProfile(lock, globals.profile);
@@ -1278,7 +1277,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs
 static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag '%1%'") % opFlags.front());
+        throw UsageError("unknown flag '%1%'", opFlags.front());
 
     if (opArgs.size() == 1 && opArgs.front() == "old") {
         deleteOldGenerations(globals.profile, globals.dryRun);
@@ -1286,18 +1285,18 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
     } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) {
         if(opArgs.front().size() < 2)
-            throw Error(format("invalid number of generations ‘%1%’") % opArgs.front());
+            throw Error("invalid number of generations ‘%1%’", opArgs.front());
         string str_max = string(opArgs.front(), 1, opArgs.front().size());
         int max;
         if (!string2Int(str_max, max) || max == 0)
-            throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front());
+            throw Error("invalid number of generations to keep ‘%1%’", opArgs.front());
         deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
     } else {
         std::set<unsigned int> gens;
         for (auto & i : opArgs) {
             unsigned int n;
             if (!string2Int(i, n))
-                throw UsageError(format("invalid generation number '%1%'") % i);
+                throw UsageError("invalid generation number '%1%'", i);
             gens.insert(n);
         }
         deleteGenerations(globals.profile, gens, globals.dryRun);
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 6c99d118180..bf353677a5b 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -66,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 /* What output do we want? */
                 string outputName = i.queryOutputName();
                 if (outputName == "")
-                    throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath);
+                    throw Error("derivation '%1%' lacks an 'outputName' attribute ", drvPath);
 
                 if (gcRoot == "")
                     printGCWarning();
@@ -166,7 +166,7 @@ static int _main(int argc, char * * argv)
         if (findFile) {
             for (auto & i : files) {
                 Path p = state->findFile(i);
-                if (p == "") throw Error(format("unable to find '%1%'") % i);
+                if (p == "") throw Error("unable to find '%1%'", i);
                 std::cout << p << std::endl;
             }
             return 0;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 2b9254659a7..d3d05cc50ff 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -37,11 +37,11 @@ string resolveMirrorUri(EvalState & state, string uri)
 
     auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
     if (mirrorList == vMirrors.attrs->end())
-        throw Error(format("unknown mirror name '%1%'") % mirrorName);
+        throw Error("unknown mirror name '%1%'", mirrorName);
     state.forceList(*mirrorList->value);
 
     if (mirrorList->value->listSize() < 1)
-        throw Error(format("mirror URI '%1%' did not expand to anything") % uri);
+        throw Error("mirror URI '%1%' did not expand to anything", uri);
 
     string mirror = state.forceString(*mirrorList->value->listElems()[0]);
     return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
@@ -73,7 +73,7 @@ static int _main(int argc, char * * argv)
                 string s = getArg(*arg, arg, end);
                 ht = parseHashType(s);
                 if (ht == htUnknown)
-                    throw UsageError(format("unknown hash type '%1%'") % s);
+                    throw UsageError("unknown hash type '%1%'", s);
             }
             else if (*arg == "--print-path")
                 printPath = true;
@@ -151,7 +151,7 @@ static int _main(int argc, char * * argv)
         if (name.empty())
             name = baseNameOf(uri);
         if (name.empty())
-            throw Error(format("cannot figure out file name for '%1%'") % uri);
+            throw Error("cannot figure out file name for '%1%'", uri);
 
         /* If an expected hash is given, the file may already exist in
            the store. */
@@ -206,7 +206,7 @@ static int _main(int argc, char * * argv)
             hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
-                throw Error(format("hash mismatch for '%1%'") % uri);
+                throw Error("hash mismatch for '%1%'", uri);
 
             /* Copy the file to the Nix store. FIXME: if RemoteStore
                implemented addToStoreFromDump() and downloadFile()
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index fcc00175a99..e4dd9bc96a8 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -124,7 +124,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
         else if (i == "--repair") buildMode = bmRepair;
         else if (i == "--check") buildMode = bmCheck;
         else if (i == "--ignore-unknown") ignoreUnknown = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     std::vector<StorePathWithOutputs> paths;
     for (auto & i : opArgs)
@@ -178,7 +178,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--recursive") recursive = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.empty())
         throw UsageError("first argument must be hash algorithm");
@@ -198,10 +198,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 
     for (auto i : opFlags)
         if (i == "--recursive") recursive = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.size() != 3)
-        throw UsageError(format("'--print-fixed-path' requires three arguments"));
+        throw UsageError("'--print-fixed-path' requires three arguments");
 
     Strings::iterator i = opArgs.begin();
     HashType hashAlgo = parseHashType(*i++);
@@ -296,9 +296,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (i == "--use-output" || i == "-u") useOutput = true;
         else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true;
         else if (i == "--include-outputs") includeOutputs = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
         if (prev != qDefault && prev != query)
-            throw UsageError(format("query type '%1%' conflicts with earlier flag") % i);
+            throw UsageError("query type '%1%' conflicts with earlier flag", i);
     }
 
     if (query == qDefault) query = qOutputs;
@@ -444,7 +444,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
     Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath));
 
     /* Print each environment variable in the derivation in a format
-       that can be sourced by the shell. */
+     * that can be sourced by the shell. */
     for (auto & i : drv.env)
         cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second);
 
@@ -531,7 +531,7 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         if (i == "--reregister") reregister = true;
         else if (i == "--hash-given") hashGiven = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -545,7 +545,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--print-invalid") printInvalid = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     for (auto & i : opArgs) {
         auto path = store->followLinksToStorePath(i);
@@ -576,7 +576,7 @@ static void opGC(Strings opFlags, Strings opArgs)
             long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
             options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
         }
-        else throw UsageError(format("bad sub-operation '%1%' in GC") % *i);
+        else throw UsageError("bad sub-operation '%1%' in GC", *i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -612,7 +612,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
 
     for (auto & i : opFlags)
         if (i == "--ignore-liveness") options.ignoreLiveness = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     for (auto & i : opArgs)
         options.pathsToDelete.insert(store->followLinksToStorePath(i));
@@ -650,7 +650,7 @@ static void opRestore(Strings opFlags, Strings opArgs)
 static void opExport(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag '%1%'") % i);
+        throw UsageError("unknown flag '%1%'", i);
 
     StorePathSet paths;
 
@@ -666,7 +666,7 @@ static void opExport(Strings opFlags, Strings opArgs)
 static void opImport(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag '%1%'") % i);
+        throw UsageError("unknown flag '%1%'", i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -701,7 +701,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
     for (auto & i : opFlags)
         if (i == "--check-contents") checkContents = true;
         else if (i == "--repair") repair = Repair;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     if (store->verifyStore(checkContents, repair)) {
         printError("warning: not all errors were fixed");
@@ -764,7 +764,7 @@ static void opServe(Strings opFlags, Strings opArgs)
     bool writeAllowed = false;
     for (auto & i : opFlags)
         if (i == "--write") writeAllowed = true;
-        else throw UsageError(format("unknown flag '%1%'") % i);
+        else throw UsageError("unknown flag '%1%'", i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
@@ -835,7 +835,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                             for (auto & p : willSubstitute) subs.emplace_back(p.clone());
                             store->buildPaths(subs);
                         } catch (Error & e) {
-                            printError(format("warning: %1%") % e.msg());
+                            printError("warning: %1%", e.msg());
                         }
                 }
 
@@ -906,7 +906,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
 
-                auto drvPath = store->parseStorePath(readString(in)); // informational only
+                auto drvPath = store->parseStorePath(readString(in)); // inonal ony
                 BasicDerivation drv;
                 readDerivation(in, *store, drv);
 
@@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             }
 
             default:
-                throw Error(format("unknown serve command %1%") % cmd);
+                throw Error("unknown serve command %1%", cmd);
         }
 
         out.flush();
@@ -973,7 +973,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
 {
     for (auto & i : opFlags)
-        throw UsageError(format("unknown flag '%1%'") % i);
+        throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.size() != 3) throw UsageError("three arguments expected");
     auto i = opArgs.begin();
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 851f90abd1b..4e14f50ed75 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -13,9 +13,9 @@ struct MixCat : virtual Args
     {
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
-            throw Error(format("path '%1%' does not exist") % path);
+            throw Error("path '%1%' does not exist", path);
         if (st.type != FSAccessor::Type::tRegular)
-            throw Error(format("path '%1%' is not a regular file") % path);
+            throw Error("path '%1%' is not a regular file", path);
 
         std::cout << accessor->readFile(path);
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 0cc523f507e..128708339c0 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -126,7 +126,7 @@ static int compatNixHash(int argc, char * * argv)
             string s = getArg(*arg, arg, end);
             ht = parseHashType(s);
             if (ht == htUnknown)
-                throw UsageError(format("unknown hash type '%1%'") % s);
+                throw UsageError("unknown hash type '%1%'", s);
         }
         else if (*arg == "--to-base16") op = opTo16;
         else if (*arg == "--to-base32") op = opTo32;
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 3ef1f2750b6..5d55bd4d537 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -65,7 +65,7 @@ struct MixLs : virtual Args, MixJSON
 
         auto st = accessor->stat(path);
         if (st.type == FSAccessor::Type::tMissing)
-            throw Error(format("path '%1%' does not exist") % path);
+            throw Error("path '%1%' does not exist", path);
         doPath(st, path,
             st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)),
             showDirectory);
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 27727bd2550..c770fd4a4db 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -252,12 +252,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
                 // input without clearing the input so far.
                 continue;
             } else {
-              printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+              printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
             }
         } catch (Error & e) {
-            printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         } catch (Interrupted & e) {
-            printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg());
+            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         }
 
         // We handled the current input fully, so we should clear it
@@ -546,7 +546,7 @@ bool NixRepl::processLine(string line)
         return false;
 
     else if (command != "")
-        throw Error(format("unknown command '%1%'") % command);
+        throw Error("unknown command '%1%'", command);
 
     else {
         size_t p = line.find('=');
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 8e30264c023..d790979a4f1 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -197,10 +197,10 @@ void chrootHelper(int argc, char * * argv)
         Finally freeCwd([&]() { free(cwd); });
 
         if (chroot(tmpDir.c_str()) == -1)
-            throw SysError(format("chrooting into '%s'") % tmpDir);
+            throw SysError("chrooting into '%s'", tmpDir);
 
         if (chdir(cwd) == -1)
-            throw SysError(format("chdir to '%s' in chroot") % cwd);
+            throw SysError("chdir to '%s' in chroot", cwd);
     } else
         if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1)
             throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);

From c0d940978a66841df638038e3ca85501bcee5734 Mon Sep 17 00:00:00 2001
From: Dustin DeWeese <dustin.deweese@gmail.com>
Date: Mon, 20 Apr 2020 17:32:50 -0700
Subject: [PATCH 1995/2196] Replace select() with poll() to allow waiting on
 more than FD_SETSIZE fds

---
 src/libstore/build.cc | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b4207e1b818..57263476573 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -33,7 +33,6 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/utsname.h>
-#include <sys/select.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -43,6 +42,7 @@
 #include <errno.h>
 #include <cstring>
 #include <termios.h>
+#include <poll.h>
 
 #include <pwd.h>
 #include <grp.h>
@@ -4789,8 +4789,7 @@ void Worker::waitForInput()
        terminated. */
 
     bool useTimeout = false;
-    struct timeval timeout;
-    timeout.tv_usec = 0;
+    long timeout = 0;
     auto before = steady_time_point::clock::now();
 
     /* If we're monitoring for silence on stdout/stderr, or if there
@@ -4808,7 +4807,7 @@ void Worker::waitForInput()
             nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
     }
     if (nearest != steady_time_point::max()) {
-        timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
+        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
         useTimeout = true;
     }
 
@@ -4819,30 +4818,28 @@ void Worker::waitForInput()
         if (lastWokenUp == steady_time_point::min())
             printError("waiting for locks or build slots...");
         if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max(1L,
+        timeout = std::max(1L,
             (long) std::chrono::duration_cast<std::chrono::seconds>(
                 lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
     } else lastWokenUp = steady_time_point::min();
 
     if (useTimeout)
-        vomit("sleeping %d seconds", timeout.tv_sec);
+        vomit("sleeping %d seconds", timeout);
 
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
-    fd_set fds;
-    FD_ZERO(&fds);
-    int fdMax = 0;
+    std::vector<struct pollfd> pollStatus;
+    std::map <int, int> fdToPollStatus;
     for (auto & i : children) {
         for (auto & j : i.fds) {
-            if (j >= FD_SETSIZE)
-                throw Error("reached FD_SETSIZE limit");
-            FD_SET(j, &fds);
-            if (j >= fdMax) fdMax = j + 1;
+            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
+            fdToPollStatus[j] = pollStatus.size() - 1;
         }
     }
 
-    if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) {
+    if (poll(pollStatus.data(), pollStatus.size(),
+            useTimeout ? timeout * 1000 : -1) == -1) {
         if (errno == EINTR) return;
         throw SysError("waiting for input");
     }
@@ -4863,7 +4860,7 @@ void Worker::waitForInput()
         set<int> fds2(j->fds);
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
-            if (FD_ISSET(k, &fds)) {
+            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
                 ssize_t rd = read(k, buffer.data(), buffer.size());
                 // FIXME: is there a cleaner way to handle pt close
                 // than EIO? Is this even standard?

From 2ea4d45449ea676e27cc678145ef39af7ac05ca8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Apr 2020 10:15:32 +0200
Subject: [PATCH 1996/2196] Path fetcher: Fix store path name

(cherry picked from commit c7af247beacd418e6f2c4d33dffc35299101cd12)
---
 src/libfetchers/path.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 7c7e20f4e82..ba2cc192e54 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -72,7 +72,7 @@ struct PathInput : Input
 
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
             // FIXME: try to substitute storePath.
-            storePath = store->addToStore("name", path);
+            storePath = store->addToStore("source", path);
 
         return
             {

From 7114f088fceb366e9ff0fb60e2970a11861d11fd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Apr 2020 14:54:16 +0200
Subject: [PATCH 1997/2196] Don't install error-demo

---
 Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index f6b4a3101e9..e3057c36ca2 100644
--- a/Makefile
+++ b/Makefile
@@ -16,8 +16,7 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk \
-  src/error-demo/local.mk
+  tests/plugins/local.mk
 
 -include Makefile.config
 

From c9d0cf7e02d4b3af3c027d7d74d5caa3c8963d26 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Apr 2020 15:03:41 +0200
Subject: [PATCH 1998/2196] Don't include error.hh in util.hh to prevent header
 bloat

---
 src/libutil/util.hh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 8e24ef968fb..d7ee62bcca4 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -3,7 +3,6 @@
 #include "types.hh"
 #include "logging.hh"
 #include "ansicolor.hh"
-#include "error.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>

From 3bc9155dfc14434f5e9566fd15c79141899f6d07 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 22 Apr 2020 15:00:11 -0600
Subject: [PATCH 1999/2196] a few more 'format's rremoved

---
 src/libexpr/lexer.l | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index c34e5c38392..85376a08f1b 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -127,14 +127,14 @@ or          { return OR_KW; }
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(format("invalid integer '%1%'") % yytext);
+                  throw ParseError("invalid integer '%1%'", yytext);
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError(format("invalid float '%1%'") % yytext);
+                  throw ParseError("invalid float '%1%'", yytext);
               return FLOAT;
             }
 

From 833501f6f11095ae226c13948eb3dc1de2a246ea Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 23 Apr 2020 15:55:34 -0600
Subject: [PATCH 2000/2196] 'what' string

---
 src/libutil/types.hh | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 2261adb8260..141bc6b3c46 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -88,6 +88,13 @@ class BaseError : public std::exception
 protected:
     string prefix_; // used for location traces etc.
     ErrorInfo err;
+    string what_;
+    void initWhat() 
+    {
+      std::ostringstream oss;
+      oss << err;
+      what_ = oss.str();
+    }
 public:
     unsigned int status = 1; // exit status
 
@@ -97,30 +104,27 @@ public:
                 .hint = hintfmt(args...)
               }
         , status(status)
-    {
-    }
+    { initWhat(); }
 
     template<typename... Args>
     BaseError(const Args & ... args)
         : err { .level = lvlError, 
                 .hint = hintfmt(args...)
               }
-    {
-    }
+    { initWhat(); }
 
     BaseError(ErrorInfo e)
         : err(e)
-    {
-    }
+    { initWhat(); }
 
 #ifdef EXCEPTION_NEEDS_THROW_SPEC
     ~BaseError() throw () { };
-    const char * what() const throw () { return err.description.c_str(); }
+    const char * what() const throw () { return what_.c_str(); }
 #else
-    const char * what() const noexcept { return err.description.c_str(); }
+    const char * what() const noexcept { return what_.c_str(); }
 #endif
 
-    const string & msg() const { return err.description; }
+    const string & msg() const { return what_; }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
 

From d9632765a81604d270bf4693bddd53c22ee189d4 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 24 Apr 2020 12:44:23 -0600
Subject: [PATCH 2001/2196] add has_value check; remove obslete friend class

---
 src/libutil/error.cc   | 2 +-
 src/libutil/fmt.hh     | 5 ++---
 src/libutil/logging.hh | 9 +++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 41595b0df99..c6bca7135e2 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -162,7 +162,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     out << prefix << std::endl;
 
     // lines of code.
-    if (einfo.nixCode->errLineOfCode != "") {
+    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") {
         printCodeLines(prefix, *einfo.nixCode);
         out << prefix << std::endl;
     }
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index d65215063a3..d182adc3a14 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -92,9 +92,11 @@ public:
     {
         fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
     }
+
     hintformat(const hintformat &hf)
     : fmt(hf.fmt)
     {}
+
     template<class T>
     hintformat& operator%(const T &value)
     {
@@ -102,14 +104,11 @@ public:
         return *this;
     }
 
-
     std::string str() const
     {
         return fmt.str();
     }
 
-    template <typename U>
-    friend class AddHint;
 private:
     format fmt;
 };
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 8269e2e1af1..89fd9841977 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -62,11 +62,12 @@ public:
 
     virtual void logEI(const ErrorInfo &ei) = 0;
 
-    void logEI(Verbosity lvl, ErrorInfo ei) {
-      ei.level = lvl;
-      logEI(ei);
+    void logEI(Verbosity lvl, ErrorInfo ei) 
+    {
+        ei.level = lvl;
+        logEI(ei);
     }
-
+    
     virtual void warn(const std::string & msg);
 
     virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

From d8d4844b883dcea67cccb9bd75eecb14e60ae496 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 24 Apr 2020 14:57:51 -0600
Subject: [PATCH 2002/2196] all things error to error.hh

---
 src/libmain/shared.cc |   9 ++-
 src/libutil/error.cc  |  18 +++++
 src/libutil/types.hh  | 151 ++----------------------------------------
 src/libutil/util.cc   |  18 -----
 src/libutil/util.hh   |   1 +
 5 files changed, 30 insertions(+), 167 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index f7ab56271d2..8551eb0484a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -281,6 +281,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
+    ErrorInfo::programName = programName;
+
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
@@ -296,12 +298,15 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
+        // TODO: switch to logError
+        // logError(e.info());
         printError(
-            format(error + "%1%\nTry '%2% --help' for more information.")
+            format("%1%\nTry '%2% --help' for more information.")
             % e.what() % programName);
         return 1;
     } catch (BaseError & e) {
-        printError(error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+        // logError(e.info());
+        printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         if (e.prefix() != "" && !settings.showTrace)
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index c6bca7135e2..20b97ea3d08 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -7,6 +7,24 @@
 namespace nix
 {
 
+
+const std::string nativeSystem = SYSTEM;
+
+
+BaseError & BaseError::addPrefix(const FormatOrString & fs)
+{
+    prefix_ = fs.s + prefix_;
+    return *this;
+}
+
+
+std::string SysError::addErrno(const std::string & s)
+{
+    errNo = errno;
+    return s + ": " + strerror(errNo);
+}
+
+
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
 std::ostream& operator<<(std::ostream &os, const hintformat &hf)
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 141bc6b3c46..f11256f6167 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -5,20 +5,8 @@
 
 #include <list>
 #include <set>
-#include <memory>
 #include <map>
-#include <optional>
 
-#include "fmt.hh"
-
-/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
- * its (virtual) destructor and what() in c++11 mode, in violation of spec
- */
-#ifdef __GNUC__
-#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7)
-#define EXCEPTION_NEEDS_THROW_SPEC
-#endif
-#endif
 
 namespace nix {
 
@@ -26,144 +14,13 @@ using std::list;
 using std::set;
 using std::vector;
 
-typedef enum {
-    lvlError = 0,
-    lvlWarn,
-    lvlInfo,
-    lvlTalkative,
-    lvlChatty,
-    lvlDebug,
-    lvlVomit
-} Verbosity;
-
-struct ErrPos
-{
-    int line;
-    int column;
-    string file;
-
-    template <class P>
-    ErrPos& operator=(const P &pos)
-    {
-        line = pos.line;
-        column = pos.column;
-        file = pos.file;
-        return *this;
-    }
-
-    template <class P>
-    ErrPos(const P &p)
-    {
-      *this = p;
-    }
-};
-
-struct NixCode
-{
-    ErrPos errPos;
-    std::optional<string> prevLineOfCode;
-    string errLineOfCode;
-    std::optional<string> nextLineOfCode;
-};
-
-// -------------------------------------------------
-// ErrorInfo.
-struct ErrorInfo
-{
-    Verbosity level;
-    string name;
-    string description;
-    std::optional<hintformat> hint;
-    std::optional<NixCode> nixCode;
-
-    static std::optional<string> programName;
-};
-
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
-
-/* BaseError should generally not be caught, as it has Interrupted as
-   a subclass. Catch Error instead. */
-class BaseError : public std::exception
-{
-protected:
-    string prefix_; // used for location traces etc.
-    ErrorInfo err;
-    string what_;
-    void initWhat() 
-    {
-      std::ostringstream oss;
-      oss << err;
-      what_ = oss.str();
-    }
-public:
-    unsigned int status = 1; // exit status
-
-    template<typename... Args>
-    BaseError(unsigned int status, const Args & ... args)
-        : err { .level = lvlError,
-                .hint = hintfmt(args...)
-              }
-        , status(status)
-    { initWhat(); }
-
-    template<typename... Args>
-    BaseError(const Args & ... args)
-        : err { .level = lvlError, 
-                .hint = hintfmt(args...)
-              }
-    { initWhat(); }
-
-    BaseError(ErrorInfo e)
-        : err(e)
-    { initWhat(); }
-
-#ifdef EXCEPTION_NEEDS_THROW_SPEC
-    ~BaseError() throw () { };
-    const char * what() const throw () { return what_.c_str(); }
-#else
-    const char * what() const noexcept { return what_.c_str(); }
-#endif
-
-    const string & msg() const { return what_; }
-    const string & prefix() const { return prefix_; }
-    BaseError & addPrefix(const FormatOrString & fs);
-
-    const ErrorInfo & info() const { return err; }
-};
-
-#define MakeError(newClass, superClass) \
-    class newClass : public superClass                  \
-    {                                                   \
-    public:                                             \
-        using superClass::superClass;                   \
-    }
-
-MakeError(Error, BaseError);
-
-class SysError : public Error
-{
-public:
-    int errNo;
-
-    template<typename... Args>
-    SysError(const Args & ... args)
-        : Error(args...)  // TODO addErrNo for hintfmt
-        // : Error(addErrno(hintfmt(args...)))
-    { }
-
-private:
-
-    std::string addErrno(const std::string & s);
-};
-
-
-typedef list<string> Strings;
-typedef set<string> StringSet;
+typedef list<std::string> Strings;
+typedef set<std::string> StringSet;
 typedef std::map<std::string, std::string> StringMap;
 
-
 /* Paths are just strings. */
-typedef string Path;
+
+typedef std::string Path;
 typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index bffcbcccfd0..747a9e991ec 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -40,24 +40,6 @@ extern char * * environ;
 
 namespace nix {
 
-
-const std::string nativeSystem = SYSTEM;
-
-
-BaseError & BaseError::addPrefix(const FormatOrString & fs)
-{
-    prefix_ = fs.s + prefix_;
-    return *this;
-}
-
-
-std::string SysError::addErrno(const std::string & s)
-{
-    errNo = errno;
-    return s + ": " + strerror(errNo);
-}
-
-
 std::optional<std::string> getEnv(const std::string & key)
 {
     char * value = getenv(key.c_str());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 636f3ba6c0e..978e14a30f7 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "types.hh"
+#include "error.hh"
 #include "logging.hh"
 #include "ansicolor.hh"
 

From cdac083dc54b65f66f88caf193c617c1dbf3bbd6 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 24 Apr 2020 21:40:13 -0600
Subject: [PATCH 2003/2196] don't print blank lines for blank description

---
 src/libutil/error.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 20b97ea3d08..c800fe9a8b4 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -88,7 +88,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
 std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 {
     int errwidth = 80;
-    string prefix = "    ";
+    string prefix = "";
 
     string levelString;
     switch (einfo.level) {
@@ -176,8 +176,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     }
 
     // description
-    out << prefix << einfo.description << std::endl;
-    out << prefix << std::endl;
+    if (einfo.description != "") {
+        out << prefix << einfo.description << std::endl;
+        out << prefix << std::endl;
+    }
 
     // lines of code.
     if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") {

From f59404e1a619934b9430b3a5bf352dbfffef2c31 Mon Sep 17 00:00:00 2001
From: Linus Heckemann <git@sphalerite.org>
Date: Sat, 25 Apr 2020 13:18:39 +0200
Subject: [PATCH 2004/2196] nix-env: refactor uninstallDerivations

Reduces the number of store queries it performs. Also prints a warning
if any of the selectors did not match any installed derivations.

UX Caveats:
- Will print a warning that nothing matched if a previous selector
  already removed the path
- Will not do anything if no selectors were provided (no change from
  before).

Fixes #3531
---
 src/nix-env/nix-env.cc | 47 ++++++++++++++++++++++++++----------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 1a2bb42a3c2..6c9cd9548a1 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -718,28 +718,39 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
     while (true) {
         string lockToken = optimisticLockProfile(profile);
 
-        DrvInfos installedElems = queryInstalled(*globals.state, profile);
-        DrvInfos newElems;
-
-        for (auto & i : installedElems) {
-            DrvName drvName(i.queryName());
-            bool found = false;
-            for (auto & j : selectors)
-                /* !!! the repeated calls to followLinksToStorePath()
-                   are expensive, should pre-compute them. */
-                if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j))
-                    || DrvName(j).matches(drvName))
-                {
-                    printInfo("uninstalling '%s'", i.queryName());
-                    found = true;
-                    break;
-                }
-            if (!found) newElems.push_back(i);
+        DrvInfos workingElems = queryInstalled(*globals.state, profile);
+
+        for (auto & selector : selectors) {
+            DrvInfos::iterator split = workingElems.begin();
+            if (isPath(selector)) {
+                StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector);
+                split = std::partition(
+                    workingElems.begin(), workingElems.end(),
+                    [&selectorStorePath, globals](auto &elem) {
+                        return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath());
+                    }
+                );
+            } else {
+                DrvName selectorName(selector);
+                split = std::partition(
+                    workingElems.begin(), workingElems.end(),
+                    [&selectorName](auto &elem){
+                        DrvName elemName(elem.queryName());
+                        return !selectorName.matches(elemName);
+                    }
+                );
+            }
+            if (split == workingElems.end())
+                warn("Selector '%s' matched no installed paths", selector);
+            for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) {
+                printInfo("uninstalling '%s'", removedElem->queryName());
+            }
+            workingElems.erase(split, workingElems.end());
         }
 
         if (globals.dryRun) return;
 
-        if (createUserEnv(*globals.state, newElems,
+        if (createUserEnv(*globals.state, workingElems,
                 profile, settings.envKeepDerivations, lockToken)) break;
     }
 }

From d4fd7b543ee8dcea2a06c2f857b7e4dd72d34ccc Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sat, 25 Apr 2020 12:05:26 -0600
Subject: [PATCH 2005/2196] print dashes instead of empty name string

---
 src/libutil/error.cc | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index c800fe9a8b4..2c34325acf8 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -148,14 +148,21 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         dashes.append("-");
 
     // divider.
-    out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
-               prefix,
-               levelString,
-               "---",
-               einfo.name,
-               dashes,
-               einfo.programName.value_or(""))
-        << std::endl;
+    if (einfo.name != "")
+      out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
+                 prefix,
+                 levelString,
+                 einfo.name,
+                 dashes,
+                 einfo.programName.value_or(""))
+          << std::endl;
+    else
+      out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
+                 prefix,
+                 levelString,
+                 dashes,
+                 einfo.programName.value_or(""))
+          << std::endl;
 
     // filename.
     if (einfo.nixCode.has_value()) {

From 1ff42722ce44f485c7dfabc6bb370494080f2755 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 26 Apr 2020 14:47:41 -0600
Subject: [PATCH 2006/2196] error.hh

---
 src/libutil/error.hh | 157 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 157 insertions(+)
 create mode 100644 src/libutil/error.hh

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
new file mode 100644
index 00000000000..e33b1d3d619
--- /dev/null
+++ b/src/libutil/error.hh
@@ -0,0 +1,157 @@
+#pragma once
+
+
+#include "ref.hh"
+
+#include <list>
+#include <memory>
+#include <map>
+#include <optional>
+
+#include "fmt.hh"
+
+/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
+ * its (virtual) destructor and what() in c++11 mode, in violation of spec
+ */
+#ifdef __GNUC__
+#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7)
+#define EXCEPTION_NEEDS_THROW_SPEC
+#endif
+#endif
+
+namespace nix {
+
+using std::list;
+using std::vector;
+
+typedef enum {
+    lvlError = 0,
+    lvlWarn,
+    lvlInfo,
+    lvlTalkative,
+    lvlChatty,
+    lvlDebug,
+    lvlVomit
+} Verbosity;
+
+struct ErrPos
+{
+    int line;
+    int column;
+    string file;
+
+    template <class P>
+    ErrPos& operator=(const P &pos)
+    {
+        line = pos.line;
+        column = pos.column;
+        file = pos.file;
+        return *this;
+    }
+
+    template <class P>
+    ErrPos(const P &p)
+    {
+      *this = p;
+    }
+};
+
+struct NixCode
+{
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
+
+// -------------------------------------------------
+// ErrorInfo.
+struct ErrorInfo
+{
+    Verbosity level;
+    string name;
+    string description;
+    std::optional<hintformat> hint;
+    std::optional<NixCode> nixCode;
+
+    static std::optional<string> programName;
+};
+
+std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
+
+/* BaseError should generally not be caught, as it has Interrupted as
+   a subclass. Catch Error instead. */
+class BaseError : public std::exception
+{
+protected:
+    string prefix_; // used for location traces etc.
+    ErrorInfo err;
+    string what_;
+    void initWhat() 
+    {
+        std::ostringstream oss;
+        oss << err;
+        what_ = oss.str();
+    }
+public:
+    unsigned int status = 1; // exit status
+
+    template<typename... Args>
+    BaseError(unsigned int status, const Args & ... args)
+        : err { .level = lvlError,
+                .hint = hintfmt(args...)
+              }
+        , status(status)
+    { initWhat(); }
+
+    template<typename... Args>
+    BaseError(const Args & ... args)
+        : err { .level = lvlError, 
+                .hint = hintfmt(args...)
+              }
+    { initWhat(); }
+
+    BaseError(ErrorInfo e)
+        : err(e)
+    { initWhat(); }
+
+#ifdef EXCEPTION_NEEDS_THROW_SPEC
+    ~BaseError() throw () { };
+    const char * what() const throw () { return what_.c_str(); }
+#else
+    const char * what() const noexcept { return what_.c_str(); }
+#endif
+
+    const string & msg() const { return what_; }
+    const string & prefix() const { return prefix_; }
+    BaseError & addPrefix(const FormatOrString & fs);
+
+    const ErrorInfo & info() const { return err; }
+};
+
+#define MakeError(newClass, superClass) \
+    class newClass : public superClass                  \
+    {                                                   \
+    public:                                             \
+        using superClass::superClass;                   \
+    }
+
+MakeError(Error, BaseError);
+
+class SysError : public Error
+{
+public:
+    int errNo;
+
+    template<typename... Args>
+    SysError(const Args & ... args)
+        : Error(args...)  // TODO addErrNo for hintfmt
+        // : Error(addErrno(hintfmt(args...)))
+    { }
+
+private:
+
+    std::string addErrno(const std::string & s);
+};
+
+}

From a3bc695e7dd60a0f1860942ba0343572b85e05e4 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Mon, 27 Apr 2020 11:12:54 -0600
Subject: [PATCH 2007/2196] Set GCROOT to store path to prevent garbage
 collection

---
 src/nix/shell.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index 71e640667fc..bd07104cfd3 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -200,13 +200,15 @@ struct Common : InstallableCommand, MixProfile
         }
     }
 
-    BuildEnvironment getBuildEnvironment(ref<Store> store)
+    std::pair<BuildEnvironment, std::string> getBuildEnvironment(ref<Store> store)
     {
         auto shellOutPath = getShellOutPath(store);
 
+        auto strPath = store->printStorePath(shellOutPath);
+
         updateProfile(shellOutPath);
 
-        return readEnvironment(store->printStorePath(shellOutPath));
+        return {readEnvironment(strPath), strPath};
     }
 };
 
@@ -253,7 +255,7 @@ struct CmdDevShell : Common, MixEnvironment
 
     void run(ref<Store> store) override
     {
-        auto buildEnvironment = getBuildEnvironment(store);
+        auto [buildEnvironment, gcroot] = getBuildEnvironment(store);
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
@@ -276,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment
         auto shell = getEnv("SHELL").value_or("bash");
 
         setEnviron();
+        setenv("GCROOT", gcroot.data(), 1);
 
         auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 
@@ -307,7 +310,7 @@ struct CmdPrintDevEnv : Common
 
     void run(ref<Store> store) override
     {
-        auto buildEnvironment = getBuildEnvironment(store);
+        auto buildEnvironment = getBuildEnvironment(store).first;
 
         stopProgressBar();
 

From 9e95b95a5da59252d140fb61cb1177645935dcd6 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Mon, 27 Apr 2020 13:18:26 -0600
Subject: [PATCH 2008/2196] comment

---
 src/nix/shell.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index bd07104cfd3..fa8fe2e9bdf 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -278,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment
         auto shell = getEnv("SHELL").value_or("bash");
 
         setEnviron();
+        // prevent garbage collection until shell exits
         setenv("GCROOT", gcroot.data(), 1);
 
         auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};

From c05e20daa1abb3446e378331697938b78af2b3d7 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <hi@alyssa.is>
Date: Mon, 27 Apr 2020 14:15:15 +0000
Subject: [PATCH 2009/2196] Fix long paths permanently breaking GC

Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a,
long enough that everything after "/nix/store/" is longer than 4096
(MAX_PATH) bytes.

Nix will happily allow such a path to be inserted into the store,
because it doesn't look at all the nested structure.  It just cares
about the /nix/store/[hash]-[name] part.  But, when the path is deleted,
we encounter a problem.  Nix will move the path to /nix/store/trash, but
then when it's trying to recursively delete the trash directory, it will
at some point try to unlink
/nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a.  This will fail,
because the path is too long.  After this has failed, any store deletion
operation will never work again, because Nix needs to delete the trash
directory before recreating it to move new things to it.  (I assume this
is because otherwise a path being deleted could already exist in the
trash, and then moving it would fail.)

This means that if I can trick somebody into just fetching a tarball
containing a path of the right length, they won't be able to delete
store paths or garbage collect ever again, until the offending path is
manually removed from /nix/store/trash.  (And even fixing this manually
is quite difficult if you don't understand the issue, because the
absolute path that Nix says it failed to remove is also too long for
rm(1).)

This patch fixes the issue by making Nix's recursive delete operation
use unlinkat(2).  This function takes a relative path and a directory
file descriptor.  We ensure that the relative path is always just the
name of the directory entry, and therefore its length will never exceed
255 bytes.  This means that it will never even come close to AX_PATH,
and Nix will therefore be able to handle removing arbitrarily deep
directory hierachies.

Since the directory file descriptor is used for recursion after being
used in readDirectory, I made a variant of readDirectory that takes an
already open directory stream, to avoid the directory being opened
multiple times.  As we have seen from this issue, the less we have to
interact with paths, the better, and so it's good to reuse file
descriptors where possible.

I left _deletePath as succeeding even if the parent directory doesn't
exist, even though that feels wrong to me, because without that early
return, the linux-sandbox test failed.

Reported-by: Alyssa Ross <hi@alyssa.is>
Thanks-to: Puck Meerburg <puck@puckipedia.com>
Tested-by: Puck Meerburg <puck@puckipedia.com>
Reviewed-by: Puck Meerburg <puck@puckipedia.com>
---
 src/libutil/util.cc | 54 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 43 insertions(+), 11 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 332c1c43a76..615a7656c55 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -268,16 +268,13 @@ bool isLink(const Path & path)
 }
 
 
-DirEntries readDirectory(const Path & path)
+DirEntries readDirectory(DIR *dir, const Path & path)
 {
     DirEntries entries;
     entries.reserve(64);
 
-    AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
-
     struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir.get())) { /* sic */
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
         checkInterrupt();
         string name = dirent->d_name;
         if (name == "." || name == "..") continue;
@@ -294,6 +291,14 @@ DirEntries readDirectory(const Path & path)
     return entries;
 }
 
+DirEntries readDirectory(const Path & path)
+{
+    AutoCloseDir dir(opendir(path.c_str()));
+    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+
+    return readDirectory(dir.get(), path);
+}
+
 
 unsigned char getFileType(const Path & path)
 {
@@ -389,12 +394,14 @@ void writeLine(int fd, string s)
 }
 
 
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
 {
     checkInterrupt();
 
+    string name(baseNameOf(path));
+
     struct stat st;
-    if (lstat(path.c_str(), &st) == -1) {
+    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
         if (errno == ENOENT) return;
         throw SysError(format("getting status of '%1%'") % path);
     }
@@ -406,20 +413,45 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
         /* Make the directory accessible. */
         const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
         if ((st.st_mode & PERM_MASK) != PERM_MASK) {
-            if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1)
+            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
                 throw SysError(format("chmod '%1%'") % path);
         }
 
-        for (auto & i : readDirectory(path))
-            _deletePath(path + "/" + i.name, bytesFreed);
+        int fd = openat(parentfd, path.c_str(), O_RDONLY);
+        if (!fd)
+            throw SysError(format("opening directory '%1%'") % path);
+        AutoCloseDir dir(fdopendir(fd));
+        if (!dir)
+            throw SysError(format("opening directory '%1%'") % path);
+        for (auto & i : readDirectory(dir.get(), path))
+            _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
     }
 
-    if (remove(path.c_str()) == -1) {
+    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
+    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
         if (errno == ENOENT) return;
         throw SysError(format("cannot unlink '%1%'") % path);
     }
 }
 
+static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+{
+    Path dir = dirOf(path);
+    if (dir == "")
+        dir = "/";
+
+    AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
+    if (!dirfd) {
+        // This really shouldn't fail silently, but it's left this way
+        // for backwards compatibility.
+        if (errno == ENOENT) return;
+
+        throw SysError(format("opening directory '%1%'") % path);
+    }
+
+    _deletePath(dirfd.get(), path, bytesFreed);
+}
+
 
 void deletePath(const Path & path)
 {

From e51a757720ade58e48128730134400cf12f54851 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 27 Apr 2020 15:15:08 -0600
Subject: [PATCH 2010/2196] astyle format

---
 src/libutil/error.hh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index e33b1d3d619..b715c3e45c8 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -24,7 +24,8 @@ namespace nix {
 using std::list;
 using std::vector;
 
-typedef enum {
+typedef enum
+{
     lvlError = 0,
     lvlWarn,
     lvlInfo,
@@ -52,7 +53,7 @@ struct ErrPos
     template <class P>
     ErrPos(const P &p)
     {
-      *this = p;
+        *this = p;
     }
 };
 
@@ -87,7 +88,7 @@ protected:
     string prefix_; // used for location traces etc.
     ErrorInfo err;
     string what_;
-    void initWhat() 
+    void initWhat()
     {
         std::ostringstream oss;
         oss << err;
@@ -106,7 +107,7 @@ public:
 
     template<typename... Args>
     BaseError(const Args & ... args)
-        : err { .level = lvlError, 
+        : err { .level = lvlError,
                 .hint = hintfmt(args...)
               }
     { initWhat(); }
@@ -146,7 +147,7 @@ public:
     template<typename... Args>
     SysError(const Args & ... args)
         : Error(args...)  // TODO addErrNo for hintfmt
-        // : Error(addErrno(hintfmt(args...)))
+          // : Error(addErrno(hintfmt(args...)))
     { }
 
 private:

From 52a3ca823dd8aca8da32a198be2b1238cb321e6c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 28 Apr 2020 17:56:01 +0200
Subject: [PATCH 2011/2196] Tweak warning message

---
 src/nix-env/nix-env.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 6c9cd9548a1..d62febaff53 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -741,7 +741,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
                 );
             }
             if (split == workingElems.end())
-                warn("Selector '%s' matched no installed paths", selector);
+                warn("selector '%s' matched no installed derivations", selector);
             for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) {
                 printInfo("uninstalling '%s'", removedElem->queryName());
             }

From 6d40fe573c859725cac6eb2c884ecf84965c8853 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Tue, 28 Apr 2020 11:18:54 -0600
Subject: [PATCH 2012/2196] rename to NIX_GCROOT

---
 src/nix/shell.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index fa8fe2e9bdf..fbfe24bbe30 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -279,7 +279,7 @@ struct CmdDevShell : Common, MixEnvironment
 
         setEnviron();
         // prevent garbage collection until shell exits
-        setenv("GCROOT", gcroot.data(), 1);
+        setenv("NIX_GCROOT", gcroot.data(), 1);
 
         auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 

From 22e6490311015e076a09d5608834bd0dec1d7020 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 28 Apr 2020 21:06:08 -0600
Subject: [PATCH 2013/2196] Error classname as name

---
 src/error-demo/error-demo.cc | 16 +++++++++---
 src/libstore/sqlite.cc       |  2 +-
 src/libstore/sqlite.hh       |  2 +-
 src/libutil/error.cc         | 29 +++++++++++-----------
 src/libutil/error.hh         | 47 +++++++++++++++++++++---------------
 5 files changed, 55 insertions(+), 41 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 14027278d4f..98018ca0e0c 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -4,10 +4,12 @@
 #include <iostream>
 #include <optional>
 
+using namespace nix;
+
+MakeError(DemoError, Error);
+
 int main()
 {
-    using namespace nix;
-
     makeDefaultLogger();
 
     verbosity = lvlVomit;
@@ -15,6 +17,12 @@ int main()
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
+    try {
+        throw DemoError("demo error was thrown");
+    } catch (Error &e) {
+        logger->logEI(e.info());
+    }
+
     // For completeness sake, info through vomit levels.
     // But this is maybe a heavy format for those.
     logger->logEI(
@@ -79,7 +87,7 @@ int main()
                         .prevLineOfCode = std::nullopt,
                         .errLineOfCode = "this is the problem line of code",
                         .nextLineOfCode = std::nullopt
-                    }});
+                  }});
 
     // Error with previous and next lines of code.
     logError(
@@ -93,7 +101,7 @@ int main()
                         .prevLineOfCode = std::optional("previous line of code"),
                         .errLineOfCode = "this is the problem line of code",
                         .nextLineOfCode = std::optional("next line of code"),
-                    }});
+                  }});
 
 
     return 0;
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 3407e582639..a72cd5d8860 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -196,7 +196,7 @@ SQLiteTxn::~SQLiteTxn()
     }
 }
 
-void handleSQLiteBusy(const SQLiteBusy & e)
+void handleSQLiteBusy(SQLiteBusy & e)
 {
     static std::atomic<time_t> lastWarned{0};
 
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index fd04c9b079a..ce27033c927 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -99,7 +99,7 @@ MakeError(SQLiteBusy, SQLiteError);
 
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs);
 
-void handleSQLiteBusy(const SQLiteBusy & e);
+void handleSQLiteBusy(SQLiteBusy & e);
 
 /* Convenience function for retrying a SQLite transaction when the
    database is busy. */
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 2c34325acf8..8f0e0aa1ac6 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -4,8 +4,7 @@
 #include <optional>
 #include "serialise.hh"
 
-namespace nix
-{
+namespace nix {
 
 
 const std::string nativeSystem = SYSTEM;
@@ -149,20 +148,20 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 
     // divider.
     if (einfo.name != "")
-      out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
-                 prefix,
-                 levelString,
-                 einfo.name,
-                 dashes,
-                 einfo.programName.value_or(""))
-          << std::endl;
+        out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
+                   prefix,
+                   levelString,
+                   einfo.name,
+                   dashes,
+                   einfo.programName.value_or(""))
+            << std::endl;
     else
-      out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
-                 prefix,
-                 levelString,
-                 dashes,
-                 einfo.programName.value_or(""))
-          << std::endl;
+        out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
+                   prefix,
+                   levelString,
+                   dashes,
+                   einfo.programName.value_or(""))
+            << std::endl;
 
     // filename.
     if (einfo.nixCode.has_value()) {
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index b715c3e45c8..03e43241ffc 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -24,8 +24,7 @@ namespace nix {
 using std::list;
 using std::vector;
 
-typedef enum
-{
+typedef enum {
     lvlError = 0,
     lvlWarn,
     lvlInfo,
@@ -35,8 +34,7 @@ typedef enum
     lvlVomit
 } Verbosity;
 
-struct ErrPos
-{
+struct ErrPos {
     int line;
     int column;
     string file;
@@ -57,8 +55,7 @@ struct ErrPos
     }
 };
 
-struct NixCode
-{
+struct NixCode {
     ErrPos errPos;
     std::optional<string> prevLineOfCode;
     string errLineOfCode;
@@ -67,8 +64,7 @@ struct NixCode
 
 // -------------------------------------------------
 // ErrorInfo.
-struct ErrorInfo
-{
+struct ErrorInfo {
     Verbosity level;
     string name;
     string description;
@@ -87,12 +83,20 @@ class BaseError : public std::exception
 protected:
     string prefix_; // used for location traces etc.
     ErrorInfo err;
-    string what_;
-    void initWhat()
+    std::optional<string> what_;
+    const string& calcWhat()
     {
-        std::ostringstream oss;
-        oss << err;
-        what_ = oss.str();
+        if (what_.has_value())
+            return *what_;
+        else {
+            err.name = sname();
+
+            std::ostringstream oss;
+            oss << err;
+            what_ = oss.str();
+
+            return *what_;
+        }
     }
 public:
     unsigned int status = 1; // exit status
@@ -103,31 +107,33 @@ public:
                 .hint = hintfmt(args...)
               }
         , status(status)
-    { initWhat(); }
+    {  }
 
     template<typename... Args>
     BaseError(const Args & ... args)
         : err { .level = lvlError,
                 .hint = hintfmt(args...)
               }
-    { initWhat(); }
+    {  }
 
     BaseError(ErrorInfo e)
         : err(e)
-    { initWhat(); }
+    {  }
+
+    virtual const char* sname() const { return "BaseError"; }
 
 #ifdef EXCEPTION_NEEDS_THROW_SPEC
     ~BaseError() throw () { };
-    const char * what() const throw () { return what_.c_str(); }
+    const char * what() throw () { return calcWhat().c_str(); }
 #else
-    const char * what() const noexcept { return what_.c_str(); }
+    const char * what() noexcept { return calcWhat().c_str(); }
 #endif
 
-    const string & msg() const { return what_; }
+    const string & msg() { return calcWhat(); }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
 
-    const ErrorInfo & info() const { return err; }
+    const ErrorInfo & info() { calcWhat(); return err; }
 };
 
 #define MakeError(newClass, superClass) \
@@ -135,6 +141,7 @@ public:
     {                                                   \
     public:                                             \
         using superClass::superClass;                   \
+        virtual const char* sname() const override { return #newClass; } \
     }
 
 MakeError(Error, BaseError);

From 5a34a473dd14200ffeff496f0e8e4518f7992765 Mon Sep 17 00:00:00 2001
From: Guillaume Bouchard <guillaume.bouchard@tweag.io>
Date: Wed, 29 Apr 2020 13:10:03 +0200
Subject: [PATCH 2014/2196] builtins.readFile: do not truncate content

This closes #3026 by allowing `builtins.readFile` to read a file with a
wrongly reported file size, for example, files in `/proc` may report a
file size of 0. Reading file in `/proc` is not a good enough motivation,
however I do think it just makes nix more robust by allowing more file
to be read.  Especially, I do considerer the previous behavior to be
dangerous because nix was previously reading truncated files. Examples
of file system which incorrectly report file size may be network file
system or dynamic file system (for performance reason, a dynamic file
system such as FUSE may generate the content of the file on demand).

```
nix-repl> builtins.readFile "/proc/version"
""
```

With this commit:

```
nix-repl> builtins.readFile "/proc/version"
"Linux version 5.6.7 (nixbld@localhost) (gcc version 9.3.0 (GCC)) #1-NixOS SMP Thu Apr 23 08:38:27 UTC 2020\n"
```

Here is a summary of the behavior changes:

- If the reported size is smaller, previous implementation
was silently returning a truncated file content. The new implementation
is returning the correct file content.

- If a file had a bigger reported file size, previous implementation was
failing with an exception, but the new implementation is returning the
correct file content. This change of behavior is coherent with this pull
request.

Open questions

- The behavior is unchanged for correctly reported file size, however
performances may vary because it uses the more complex sink interface.
Considering that sink is used a lot, I don't think this impacts the
performance a lot.
- `builtins.readFile` on an infinite file, such as `/dev/random` may
fill the memory.
- it does not support adding file to store, such as `${/proc/version}`.
---
 src/libutil/util.cc | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 615a7656c55..d001bc3b7cb 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -312,14 +312,7 @@ unsigned char getFileType(const Path & path)
 
 string readFile(int fd)
 {
-    struct stat st;
-    if (fstat(fd, &st) == -1)
-        throw SysError("statting file");
-
-    std::vector<unsigned char> buf(st.st_size);
-    readFull(fd, buf.data(), st.st_size);
-
-    return string((char *) buf.data(), st.st_size);
+    return drainFD(fd, true);
 }
 
 

From ca93b26db6d2d73e702ea8ecdd0a98f17ace2c7b Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Wed, 29 Apr 2020 14:39:37 +0200
Subject: [PATCH 2015/2196] Only call grantpt on MacOS systems

The commit 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 adds a `grantpt`
call on the builder pseudo terminal fd. This call is actually only
required for MacOS, but it however requires a RW access to /dev/pts
which is only RO bindmounted in the Bazel Linux sandbox. So, Nix can
not be actually run in the Bazel Linux sandbox for unneeded reasons.
---
 src/libstore/build.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 57263476573..147093fae26 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2250,10 +2250,13 @@ void DerivationGoal::startBuilder()
 
         if (chown(slaveName.c_str(), buildUser->getUID(), 0))
             throw SysError("changing owner of pseudoterminal slave");
-    } else {
+    }
+#if __APPLE__
+    else {
         if (grantpt(builderOut.readSide.get()))
             throw SysError("granting access to pseudoterminal slave");
     }
+#endif
 
     #if 0
     // Mount the pt in the sandbox so that the "tty" command works.

From e2f61263ebf41958f4f3d6f6135377024e33457c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 29 Apr 2020 10:14:32 -0600
Subject: [PATCH 2016/2196] uncrustify formatting

---
 .uncrustify.cfg              | 2989 ++++++++++++++++++++++++++++++++++
 src/error-demo/error-demo.cc |   26 +-
 src/libexpr/nixexpr.hh       |    3 +-
 src/libutil/error.cc         |   48 +-
 src/libutil/logging.cc       |   42 +-
 src/libutil/logging.hh       |    4 +-
 6 files changed, 3050 insertions(+), 62 deletions(-)
 create mode 100644 .uncrustify.cfg

diff --git a/.uncrustify.cfg b/.uncrustify.cfg
new file mode 100644
index 00000000000..23305adcfff
--- /dev/null
+++ b/.uncrustify.cfg
@@ -0,0 +1,2989 @@
+# Uncrustify-0.70.1_f
+
+#
+# General options
+#
+
+# The type of line endings.
+#
+# Default: auto
+newlines                        = auto     # lf/crlf/cr/auto
+
+# The original size of tabs in the input.
+#
+# Default: 8
+input_tab_size                  = 4        # unsigned number
+
+# The size of tabs in the output (only used if align_with_tabs=true).
+#
+# Default: 8
+output_tab_size                 = 4        # unsigned number
+
+# The ASCII value of the string escape char, usually 92 (\) or (Pawn) 94 (^).
+#
+# Default: 92
+string_escape_char              = 92       # unsigned number
+
+# Alternate string escape char (usually only used for Pawn).
+# Only works right before the quote char.
+string_escape_char2             = 0        # unsigned number
+
+# Replace tab characters found in string literals with the escape sequence \t
+# instead.
+string_replace_tab_chars        = false    # true/false
+
+# Allow interpreting '>=' and '>>=' as part of a template in code like
+# 'void f(list<list<B>>=val);'. If true, 'assert(x<0 && y>=3)' will be broken.
+# Improvements to template detection may make this option obsolete.
+tok_split_gte                   = false    # true/false
+
+# Specify the marker used in comments to disable processing of part of the
+# file.
+#
+# Default:  *INDENT-OFF*
+disable_processing_cmt          = " *INDENT-OFF*"      # string
+
+# Specify the marker used in comments to (re)enable processing in a file.
+#
+# Default:  *INDENT-ON*
+enable_processing_cmt           = " *INDENT-ON*"     # string
+
+# Enable parsing of digraphs.
+enable_digraphs                 = false    # true/false
+
+# Add or remove the UTF-8 BOM (recommend 'remove').
+utf8_bom                        = ignore   # ignore/add/remove/force
+
+# If the file contains bytes with values between 128 and 255, but is not
+# UTF-8, then output as UTF-8.
+utf8_byte                       = false    # true/false
+
+# Force the output encoding to UTF-8.
+utf8_force                      = false    # true/false
+
+# Add or remove space between 'do' and '{'.
+sp_do_brace_open                = ignore   # ignore/add/remove/force
+
+# Add or remove space between '}' and 'while'.
+sp_brace_close_while            = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'while' and '('.
+sp_while_paren_open             = ignore   # ignore/add/remove/force
+
+#
+# Spacing options
+#
+
+# Add or remove space around non-assignment symbolic operators ('+', '/', '%',
+# '<<', and so forth).
+sp_arith                        = ignore   # ignore/add/remove/force
+
+# Add or remove space around arithmetic operators '+' and '-'.
+#
+# Overrides sp_arith.
+sp_arith_additive               = ignore   # ignore/add/remove/force
+
+# Add or remove space around assignment operator '=', '+=', etc.
+sp_assign                       = ignore   # ignore/add/remove/force
+
+# Add or remove space around '=' in C++11 lambda capture specifications.
+#
+# Overrides sp_assign.
+sp_cpp_lambda_assign            = ignore   # ignore/add/remove/force
+
+# Add or remove space after the capture specification of a C++11 lambda when
+# an argument list is present, as in '[] <here> (int x){ ... }'.
+sp_cpp_lambda_square_paren      = ignore   # ignore/add/remove/force
+
+# Add or remove space after the capture specification of a C++11 lambda with
+# no argument list is present, as in '[] <here> { ... }'.
+sp_cpp_lambda_square_brace      = ignore   # ignore/add/remove/force
+
+# Add or remove space after the argument list of a C++11 lambda, as in
+# '[](int x) <here> { ... }'.
+sp_cpp_lambda_paren_brace       = ignore   # ignore/add/remove/force
+
+# Add or remove space between a lambda body and its call operator of an
+# immediately invoked lambda, as in '[]( ... ){ ... } <here> ( ... )'.
+sp_cpp_lambda_fparen            = ignore   # ignore/add/remove/force
+
+# Add or remove space around assignment operator '=' in a prototype.
+#
+# If set to ignore, use sp_assign.
+sp_assign_default               = ignore   # ignore/add/remove/force
+
+# Add or remove space before assignment operator '=', '+=', etc.
+#
+# Overrides sp_assign.
+sp_before_assign                = ignore   # ignore/add/remove/force
+
+# Add or remove space after assignment operator '=', '+=', etc.
+#
+# Overrides sp_assign.
+sp_after_assign                 = ignore   # ignore/add/remove/force
+
+# Add or remove space in 'NS_ENUM ('.
+sp_enum_paren                   = ignore   # ignore/add/remove/force
+
+# Add or remove space around assignment '=' in enum.
+sp_enum_assign                  = ignore   # ignore/add/remove/force
+
+# Add or remove space before assignment '=' in enum.
+#
+# Overrides sp_enum_assign.
+sp_enum_before_assign           = ignore   # ignore/add/remove/force
+
+# Add or remove space after assignment '=' in enum.
+#
+# Overrides sp_enum_assign.
+sp_enum_after_assign            = ignore   # ignore/add/remove/force
+
+# Add or remove space around assignment ':' in enum.
+sp_enum_colon                   = ignore   # ignore/add/remove/force
+
+# Add or remove space around preprocessor '##' concatenation operator.
+#
+# Default: add
+sp_pp_concat                    = add      # ignore/add/remove/force
+
+# Add or remove space after preprocessor '#' stringify operator.
+# Also affects the '#@' charizing operator.
+sp_pp_stringify                 = ignore   # ignore/add/remove/force
+
+# Add or remove space before preprocessor '#' stringify operator
+# as in '#define x(y) L#y'.
+sp_before_pp_stringify          = ignore   # ignore/add/remove/force
+
+# Add or remove space around boolean operators '&&' and '||'.
+sp_bool                         = ignore   # ignore/add/remove/force
+
+# Add or remove space around compare operator '<', '>', '==', etc.
+sp_compare                      = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '(' and ')'.
+sp_inside_paren                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between nested parentheses, i.e. '((' vs. ') )'.
+sp_paren_paren                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between back-to-back parentheses, i.e. ')(' vs. ') ('.
+sp_cparen_oparen                = ignore   # ignore/add/remove/force
+
+# Whether to balance spaces inside nested parentheses.
+sp_balance_nested_parens        = false    # true/false
+
+# Add or remove space between ')' and '{'.
+sp_paren_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between nested braces, i.e. '{{' vs '{ {'.
+sp_brace_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove space before pointer star '*'.
+sp_before_ptr_star              = ignore   # ignore/add/remove/force
+
+# Add or remove space before pointer star '*' that isn't followed by a
+# variable name. If set to ignore, sp_before_ptr_star is used instead.
+sp_before_unnamed_ptr_star      = ignore   # ignore/add/remove/force
+
+# Add or remove space between pointer stars '*'.
+sp_between_ptr_star             = ignore   # ignore/add/remove/force
+
+# Add or remove space after pointer star '*', if followed by a word.
+#
+# Overrides sp_type_func.
+sp_after_ptr_star               = ignore   # ignore/add/remove/force
+
+# Add or remove space after pointer caret '^', if followed by a word.
+sp_after_ptr_block_caret        = ignore   # ignore/add/remove/force
+
+# Add or remove space after pointer star '*', if followed by a qualifier.
+sp_after_ptr_star_qualifier     = ignore   # ignore/add/remove/force
+
+# Add or remove space after a pointer star '*', if followed by a function
+# prototype or function definition.
+#
+# Overrides sp_after_ptr_star and sp_type_func.
+sp_after_ptr_star_func          = ignore   # ignore/add/remove/force
+
+# Add or remove space after a pointer star '*', if followed by an open
+# parenthesis, as in 'void* (*)().
+sp_ptr_star_paren               = ignore   # ignore/add/remove/force
+
+# Add or remove space before a pointer star '*', if followed by a function
+# prototype or function definition.
+sp_before_ptr_star_func         = ignore   # ignore/add/remove/force
+
+# Add or remove space before a reference sign '&'.
+sp_before_byref                 = ignore   # ignore/add/remove/force
+
+# Add or remove space before a reference sign '&' that isn't followed by a
+# variable name. If set to ignore, sp_before_byref is used instead.
+sp_before_unnamed_byref         = ignore   # ignore/add/remove/force
+
+# Add or remove space after reference sign '&', if followed by a word.
+#
+# Overrides sp_type_func.
+sp_after_byref                  = ignore   # ignore/add/remove/force
+
+# Add or remove space after a reference sign '&', if followed by a function
+# prototype or function definition.
+#
+# Overrides sp_after_byref and sp_type_func.
+sp_after_byref_func             = ignore   # ignore/add/remove/force
+
+# Add or remove space before a reference sign '&', if followed by a function
+# prototype or function definition.
+sp_before_byref_func            = ignore   # ignore/add/remove/force
+
+# Add or remove space between type and word.
+#
+# Default: force
+sp_after_type                   = force    # ignore/add/remove/force
+
+# Add or remove space between 'decltype(...)' and word.
+sp_after_decltype               = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space before the parenthesis in the D constructs
+# 'template Foo(' and 'class Foo('.
+sp_before_template_paren        = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'template' and '<'.
+# If set to ignore, sp_before_angle is used.
+sp_template_angle               = ignore   # ignore/add/remove/force
+
+# Add or remove space before '<'.
+sp_before_angle                 = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '<' and '>'.
+sp_inside_angle                 = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '<>'.
+sp_inside_angle_empty           = ignore   # ignore/add/remove/force
+
+# Add or remove space between '>' and ':'.
+sp_angle_colon                  = ignore   # ignore/add/remove/force
+
+# Add or remove space after '>'.
+sp_after_angle                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between '>' and '(' as found in 'new List<byte>(foo);'.
+sp_angle_paren                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between '>' and '()' as found in 'new List<byte>();'.
+sp_angle_paren_empty            = ignore   # ignore/add/remove/force
+
+# Add or remove space between '>' and a word as in 'List<byte> m;' or
+# 'template <typename T> static ...'.
+sp_angle_word                   = ignore   # ignore/add/remove/force
+
+# Add or remove space between '>' and '>' in '>>' (template stuff).
+#
+# Default: add
+sp_angle_shift                  = add      # ignore/add/remove/force
+
+# (C++11) Permit removal of the space between '>>' in 'foo<bar<int> >'. Note
+# that sp_angle_shift cannot remove the space without this option.
+sp_permit_cpp11_shift           = false    # true/false
+
+# Add or remove space before '(' of control statements ('if', 'for', 'switch',
+# 'while', etc.).
+sp_before_sparen                = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '(' and ')' of control statements.
+sp_inside_sparen                = ignore   # ignore/add/remove/force
+
+# Add or remove space after '(' of control statements.
+#
+# Overrides sp_inside_sparen.
+sp_inside_sparen_open           = ignore   # ignore/add/remove/force
+
+# Add or remove space before ')' of control statements.
+#
+# Overrides sp_inside_sparen.
+sp_inside_sparen_close          = ignore   # ignore/add/remove/force
+
+# Add or remove space after ')' of control statements.
+sp_after_sparen                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and '{' of of control statements.
+sp_sparen_brace                 = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space between 'invariant' and '('.
+sp_invariant_paren              = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space after the ')' in 'invariant (C) c'.
+sp_after_invariant_paren        = ignore   # ignore/add/remove/force
+
+# Add or remove space before empty statement ';' on 'if', 'for' and 'while'.
+sp_special_semi                 = ignore   # ignore/add/remove/force
+
+# Add or remove space before ';'.
+#
+# Default: remove
+sp_before_semi                  = remove   # ignore/add/remove/force
+
+# Add or remove space before ';' in non-empty 'for' statements.
+sp_before_semi_for              = ignore   # ignore/add/remove/force
+
+# Add or remove space before a semicolon of an empty part of a for statement.
+sp_before_semi_for_empty        = ignore   # ignore/add/remove/force
+
+# Add or remove space after ';', except when followed by a comment.
+#
+# Default: add
+sp_after_semi                   = add      # ignore/add/remove/force
+
+# Add or remove space after ';' in non-empty 'for' statements.
+#
+# Default: force
+sp_after_semi_for               = force    # ignore/add/remove/force
+
+# Add or remove space after the final semicolon of an empty part of a for
+# statement, as in 'for ( ; ; <here> )'.
+sp_after_semi_for_empty         = ignore   # ignore/add/remove/force
+
+# Add or remove space before '[' (except '[]').
+sp_before_square                = ignore   # ignore/add/remove/force
+
+# Add or remove space before '[' for a variable definition.
+#
+# Default: remove
+sp_before_vardef_square         = remove   # ignore/add/remove/force
+
+# Add or remove space before '[' for asm block.
+sp_before_square_asm_block      = ignore   # ignore/add/remove/force
+
+# Add or remove space before '[]'.
+sp_before_squares               = ignore   # ignore/add/remove/force
+
+# Add or remove space before C++17 structured bindings.
+sp_cpp_before_struct_binding    = ignore   # ignore/add/remove/force
+
+# Add or remove space inside a non-empty '[' and ']'.
+sp_inside_square                = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space inside a non-empty Objective-C boxed array '@[' and
+# ']'. If set to ignore, sp_inside_square is used.
+sp_inside_square_oc_array       = ignore   # ignore/add/remove/force
+
+# Add or remove space after ',', i.e. 'a,b' vs. 'a, b'.
+sp_after_comma                  = ignore   # ignore/add/remove/force
+
+# Add or remove space before ','.
+#
+# Default: remove
+sp_before_comma                 = remove   # ignore/add/remove/force
+
+# (C#) Add or remove space between ',' and ']' in multidimensional array type
+# like 'int[,,]'.
+sp_after_mdatype_commas         = ignore   # ignore/add/remove/force
+
+# (C#) Add or remove space between '[' and ',' in multidimensional array type
+# like 'int[,,]'.
+sp_before_mdatype_commas        = ignore   # ignore/add/remove/force
+
+# (C#) Add or remove space between ',' in multidimensional array type
+# like 'int[,,]'.
+sp_between_mdatype_commas       = ignore   # ignore/add/remove/force
+
+# Add or remove space between an open parenthesis and comma,
+# i.e. '(,' vs. '( ,'.
+#
+# Default: force
+sp_paren_comma                  = force    # ignore/add/remove/force
+
+# Add or remove space before the variadic '...' when preceded by a
+# non-punctuator.
+sp_before_ellipsis              = remove   # ignore/add/remove/force
+
+# Add or remove space between a type and '...'.
+sp_type_ellipsis                = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space between a type and '?'.
+sp_type_question                = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and '...'.
+sp_paren_ellipsis               = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and a qualifier such as 'const'.
+sp_paren_qualifier              = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and 'noexcept'.
+sp_paren_noexcept               = ignore   # ignore/add/remove/force
+
+# Add or remove space after class ':'.
+sp_after_class_colon            = ignore   # ignore/add/remove/force
+
+# Add or remove space before class ':'.
+sp_before_class_colon           = ignore   # ignore/add/remove/force
+
+# Add or remove space after class constructor ':'.
+sp_after_constr_colon           = ignore   # ignore/add/remove/force
+
+# Add or remove space before class constructor ':'.
+sp_before_constr_colon          = ignore   # ignore/add/remove/force
+
+# Add or remove space before case ':'.
+#
+# Default: remove
+sp_before_case_colon            = remove   # ignore/add/remove/force
+
+# Add or remove space between 'operator' and operator sign.
+sp_after_operator               = ignore   # ignore/add/remove/force
+
+# Add or remove space between the operator symbol and the open parenthesis, as
+# in 'operator ++('.
+sp_after_operator_sym           = ignore   # ignore/add/remove/force
+
+# Overrides sp_after_operator_sym when the operator has no arguments, as in
+# 'operator *()'.
+sp_after_operator_sym_empty     = ignore   # ignore/add/remove/force
+
+# Add or remove space after C/D cast, i.e. 'cast(int)a' vs. 'cast(int) a' or
+# '(int)a' vs. '(int) a'.
+sp_after_cast                   = ignore   # ignore/add/remove/force
+
+# Add or remove spaces inside cast parentheses.
+sp_inside_paren_cast            = ignore   # ignore/add/remove/force
+
+# Add or remove space between the type and open parenthesis in a C++ cast,
+# i.e. 'int(exp)' vs. 'int (exp)'.
+sp_cpp_cast_paren               = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'sizeof' and '('.
+sp_sizeof_paren                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'sizeof' and '...'.
+sp_sizeof_ellipsis              = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'sizeof...' and '('.
+sp_sizeof_ellipsis_paren        = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'decltype' and '('.
+sp_decltype_paren               = ignore   # ignore/add/remove/force
+
+# (Pawn) Add or remove space after the tag keyword.
+sp_after_tag                    = ignore   # ignore/add/remove/force
+
+# Add or remove space inside enum '{' and '}'.
+sp_inside_braces_enum           = ignore   # ignore/add/remove/force
+
+# Add or remove space inside struct/union '{' and '}'.
+sp_inside_braces_struct         = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space inside Objective-C boxed dictionary '{' and '}'
+sp_inside_braces_oc_dict        = ignore   # ignore/add/remove/force
+
+# Add or remove space after open brace in an unnamed temporary
+# direct-list-initialization.
+sp_after_type_brace_init_lst_open = ignore   # ignore/add/remove/force
+
+# Add or remove space before close brace in an unnamed temporary
+# direct-list-initialization.
+sp_before_type_brace_init_lst_close = ignore   # ignore/add/remove/force
+
+# Add or remove space inside an unnamed temporary direct-list-initialization.
+sp_inside_type_brace_init_lst   = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '{' and '}'.
+sp_inside_braces                = ignore   # ignore/add/remove/force
+
+# Add or remove space inside '{}'.
+sp_inside_braces_empty          = ignore   # ignore/add/remove/force
+
+# Add or remove space around trailing return operator '->'.
+sp_trailing_return              = ignore   # ignore/add/remove/force
+
+# Add or remove space between return type and function name. A minimum of 1
+# is forced except for pointer return types.
+sp_type_func                    = ignore   # ignore/add/remove/force
+
+# Add or remove space between type and open brace of an unnamed temporary
+# direct-list-initialization.
+sp_type_brace_init_lst          = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '(' on function declaration.
+sp_func_proto_paren             = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '()' on function declaration
+# without parameters.
+sp_func_proto_paren_empty       = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '(' with a typedef specifier.
+sp_func_type_paren              = ignore   # ignore/add/remove/force
+
+# Add or remove space between alias name and '(' of a non-pointer function type typedef.
+sp_func_def_paren               = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '()' on function definition
+# without parameters.
+sp_func_def_paren_empty         = ignore   # ignore/add/remove/force
+
+# Add or remove space inside empty function '()'.
+# Overrides sp_after_angle unless use_sp_after_angle_always is set to true.
+sp_inside_fparens               = ignore   # ignore/add/remove/force
+
+# Add or remove space inside function '(' and ')'.
+sp_inside_fparen                = ignore   # ignore/add/remove/force
+
+# Add or remove space inside the first parentheses in a function type, as in
+# 'void (*x)(...)'.
+sp_inside_tparen                = ignore   # ignore/add/remove/force
+
+# Add or remove space between the ')' and '(' in a function type, as in
+# 'void (*x)(...)'.
+sp_after_tparen_close           = ignore   # ignore/add/remove/force
+
+# Add or remove space between ']' and '(' when part of a function call.
+sp_square_fparen                = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and '{' of function.
+sp_fparen_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and '{' of s function call in object
+# initialization.
+#
+# Overrides sp_fparen_brace.
+sp_fparen_brace_initializer     = ignore   # ignore/add/remove/force
+
+# (Java) Add or remove space between ')' and '{{' of double brace initializer.
+sp_fparen_dbrace                = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '(' on function calls.
+sp_func_call_paren              = ignore   # ignore/add/remove/force
+
+# Add or remove space between function name and '()' on function calls without
+# parameters. If set to ignore (the default), sp_func_call_paren is used.
+sp_func_call_paren_empty        = ignore   # ignore/add/remove/force
+
+# Add or remove space between the user function name and '(' on function
+# calls. You need to set a keyword to be a user function in the config file,
+# like:
+#   set func_call_user tr _ i18n
+sp_func_call_user_paren         = ignore   # ignore/add/remove/force
+
+# Add or remove space inside user function '(' and ')'.
+sp_func_call_user_inside_fparen = ignore   # ignore/add/remove/force
+
+# Add or remove space between nested parentheses with user functions,
+# i.e. '((' vs. '( ('.
+sp_func_call_user_paren_paren   = ignore   # ignore/add/remove/force
+
+# Add or remove space between a constructor/destructor and the open
+# parenthesis.
+sp_func_class_paren             = ignore   # ignore/add/remove/force
+
+# Add or remove space between a constructor without parameters or destructor
+# and '()'.
+sp_func_class_paren_empty       = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'return' and '('.
+sp_return_paren                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'return' and '{'.
+sp_return_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between '__attribute__' and '('.
+sp_attribute_paren              = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'defined' and '(' in '#if defined (FOO)'.
+sp_defined_paren                = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'throw' and '(' in 'throw (something)'.
+sp_throw_paren                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'throw' and anything other than '(' as in
+# '@throw [...];'.
+sp_after_throw                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'catch' and '(' in 'catch (something) { }'.
+# If set to ignore, sp_before_sparen is used.
+sp_catch_paren                  = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between '@catch' and '('
+# in '@catch (something) { }'. If set to ignore, sp_catch_paren is used.
+sp_oc_catch_paren               = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between class name and '('
+# in '@interface className(categoryName)<ProtocolName>:BaseClass'
+sp_oc_classname_paren           = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space between 'version' and '('
+# in 'version (something) { }'. If set to ignore, sp_before_sparen is used.
+sp_version_paren                = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space between 'scope' and '('
+# in 'scope (something) { }'. If set to ignore, sp_before_sparen is used.
+sp_scope_paren                  = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'super' and '(' in 'super (something)'.
+#
+# Default: remove
+sp_super_paren                  = remove   # ignore/add/remove/force
+
+# Add or remove space between 'this' and '(' in 'this (something)'.
+#
+# Default: remove
+sp_this_paren                   = remove   # ignore/add/remove/force
+
+# Add or remove space between a macro name and its definition.
+sp_macro                        = ignore   # ignore/add/remove/force
+
+# Add or remove space between a macro function ')' and its definition.
+sp_macro_func                   = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'else' and '{' if on the same line.
+sp_else_brace                   = ignore   # ignore/add/remove/force
+
+# Add or remove space between '}' and 'else' if on the same line.
+sp_brace_else                   = ignore   # ignore/add/remove/force
+
+# Add or remove space between '}' and the name of a typedef on the same line.
+sp_brace_typedef                = ignore   # ignore/add/remove/force
+
+# Add or remove space before the '{' of a 'catch' statement, if the '{' and
+# 'catch' are on the same line, as in 'catch (decl) <here> {'.
+sp_catch_brace                  = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space before the '{' of a '@catch' statement, if the '{'
+# and '@catch' are on the same line, as in '@catch (decl) <here> {'.
+# If set to ignore, sp_catch_brace is used.
+sp_oc_catch_brace               = ignore   # ignore/add/remove/force
+
+# Add or remove space between '}' and 'catch' if on the same line.
+sp_brace_catch                  = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between '}' and '@catch' if on the same line.
+# If set to ignore, sp_brace_catch is used.
+sp_oc_brace_catch               = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'finally' and '{' if on the same line.
+sp_finally_brace                = ignore   # ignore/add/remove/force
+
+# Add or remove space between '}' and 'finally' if on the same line.
+sp_brace_finally                = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'try' and '{' if on the same line.
+sp_try_brace                    = ignore   # ignore/add/remove/force
+
+# Add or remove space between get/set and '{' if on the same line.
+sp_getset_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove space between a variable and '{' for C++ uniform
+# initialization.
+#
+# Default: add
+sp_word_brace                   = add      # ignore/add/remove/force
+
+# Add or remove space between a variable and '{' for a namespace.
+#
+# Default: add
+sp_word_brace_ns                = add      # ignore/add/remove/force
+
+# Add or remove space before the '::' operator.
+sp_before_dc                    = ignore   # ignore/add/remove/force
+
+# Add or remove space after the '::' operator.
+sp_after_dc                     = ignore   # ignore/add/remove/force
+
+# (D) Add or remove around the D named array initializer ':' operator.
+sp_d_array_colon                = ignore   # ignore/add/remove/force
+
+# Add or remove space after the '!' (not) unary operator.
+#
+# Default: remove
+sp_not                          = remove   # ignore/add/remove/force
+
+# Add or remove space after the '~' (invert) unary operator.
+#
+# Default: remove
+sp_inv                          = remove   # ignore/add/remove/force
+
+# Add or remove space after the '&' (address-of) unary operator. This does not
+# affect the spacing after a '&' that is part of a type.
+#
+# Default: remove
+sp_addr                         = remove   # ignore/add/remove/force
+
+# Add or remove space around the '.' or '->' operators.
+#
+# Default: remove
+sp_member                       = remove   # ignore/add/remove/force
+
+# Add or remove space after the '*' (dereference) unary operator. This does
+# not affect the spacing after a '*' that is part of a type.
+#
+# Default: remove
+sp_deref                        = remove   # ignore/add/remove/force
+
+# Add or remove space after '+' or '-', as in 'x = -5' or 'y = +7'.
+#
+# Default: remove
+sp_sign                         = remove   # ignore/add/remove/force
+
+# Add or remove space between '++' and '--' the word to which it is being
+# applied, as in '(--x)' or 'y++;'.
+#
+# Default: remove
+sp_incdec                       = remove   # ignore/add/remove/force
+
+# Add or remove space before a backslash-newline at the end of a line.
+#
+# Default: add
+sp_before_nl_cont               = add      # ignore/add/remove/force
+
+# (OC) Add or remove space after the scope '+' or '-', as in '-(void) foo;'
+# or '+(int) bar;'.
+sp_after_oc_scope               = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after the colon in message specs,
+# i.e. '-(int) f:(int) x;' vs. '-(int) f: (int) x;'.
+sp_after_oc_colon               = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space before the colon in message specs,
+# i.e. '-(int) f: (int) x;' vs. '-(int) f : (int) x;'.
+sp_before_oc_colon              = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after the colon in immutable dictionary expression
+# 'NSDictionary *test = @{@"foo" :@"bar"};'.
+sp_after_oc_dict_colon          = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space before the colon in immutable dictionary expression
+# 'NSDictionary *test = @{@"foo" :@"bar"};'.
+sp_before_oc_dict_colon         = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after the colon in message specs,
+# i.e. '[object setValue:1];' vs. '[object setValue: 1];'.
+sp_after_send_oc_colon          = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space before the colon in message specs,
+# i.e. '[object setValue:1];' vs. '[object setValue :1];'.
+sp_before_send_oc_colon         = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after the (type) in message specs,
+# i.e. '-(int)f: (int) x;' vs. '-(int)f: (int)x;'.
+sp_after_oc_type                = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after the first (type) in message specs,
+# i.e. '-(int) f:(int)x;' vs. '-(int)f:(int)x;'.
+sp_after_oc_return_type         = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between '@selector' and '(',
+# i.e. '@selector(msgName)' vs. '@selector (msgName)'.
+# Also applies to '@protocol()' constructs.
+sp_after_oc_at_sel              = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between '@selector(x)' and the following word,
+# i.e. '@selector(foo) a:' vs. '@selector(foo)a:'.
+sp_after_oc_at_sel_parens       = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space inside '@selector' parentheses,
+# i.e. '@selector(foo)' vs. '@selector( foo )'.
+# Also applies to '@protocol()' constructs.
+sp_inside_oc_at_sel_parens      = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space before a block pointer caret,
+# i.e. '^int (int arg){...}' vs. ' ^int (int arg){...}'.
+sp_before_oc_block_caret        = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after a block pointer caret,
+# i.e. '^int (int arg){...}' vs. '^ int (int arg){...}'.
+sp_after_oc_block_caret         = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between the receiver and selector in a message,
+# as in '[receiver selector ...]'.
+sp_after_oc_msg_receiver        = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space after '@property'.
+sp_after_oc_property            = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove space between '@synchronized' and the open parenthesis,
+# i.e. '@synchronized(foo)' vs. '@synchronized (foo)'.
+sp_after_oc_synchronized        = ignore   # ignore/add/remove/force
+
+# Add or remove space around the ':' in 'b ? t : f'.
+sp_cond_colon                   = ignore   # ignore/add/remove/force
+
+# Add or remove space before the ':' in 'b ? t : f'.
+#
+# Overrides sp_cond_colon.
+sp_cond_colon_before            = ignore   # ignore/add/remove/force
+
+# Add or remove space after the ':' in 'b ? t : f'.
+#
+# Overrides sp_cond_colon.
+sp_cond_colon_after             = ignore   # ignore/add/remove/force
+
+# Add or remove space around the '?' in 'b ? t : f'.
+sp_cond_question                = ignore   # ignore/add/remove/force
+
+# Add or remove space before the '?' in 'b ? t : f'.
+#
+# Overrides sp_cond_question.
+sp_cond_question_before         = ignore   # ignore/add/remove/force
+
+# Add or remove space after the '?' in 'b ? t : f'.
+#
+# Overrides sp_cond_question.
+sp_cond_question_after          = ignore   # ignore/add/remove/force
+
+# In the abbreviated ternary form '(a ?: b)', add or remove space between '?'
+# and ':'.
+#
+# Overrides all other sp_cond_* options.
+sp_cond_ternary_short           = ignore   # ignore/add/remove/force
+
+# Fix the spacing between 'case' and the label. Only 'ignore' and 'force' make
+# sense here.
+sp_case_label                   = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space around the D '..' operator.
+sp_range                        = ignore   # ignore/add/remove/force
+
+# Add or remove space after ':' in a Java/C++11 range-based 'for',
+# as in 'for (Type var : expr)'.
+sp_after_for_colon              = ignore   # ignore/add/remove/force
+
+# Add or remove space before ':' in a Java/C++11 range-based 'for',
+# as in 'for (Type var : expr)'.
+sp_before_for_colon             = ignore   # ignore/add/remove/force
+
+# (D) Add or remove space between 'extern' and '(' as in 'extern (C)'.
+sp_extern_paren                 = ignore   # ignore/add/remove/force
+
+# Add or remove space after the opening of a C++ comment,
+# i.e. '// A' vs. '//A'.
+sp_cmt_cpp_start                = ignore   # ignore/add/remove/force
+
+# If true, space is added with sp_cmt_cpp_start will be added after doxygen
+# sequences like '///', '///<', '//!' and '//!<'.
+sp_cmt_cpp_doxygen              = false    # true/false
+
+# If true, space is added with sp_cmt_cpp_start will be added after Qt
+# translator or meta-data comments like '//:', '//=', and '//~'.
+sp_cmt_cpp_qttr                 = false    # true/false
+
+# Add or remove space between #else or #endif and a trailing comment.
+sp_endif_cmt                    = ignore   # ignore/add/remove/force
+
+# Add or remove space after 'new', 'delete' and 'delete[]'.
+sp_after_new                    = ignore   # ignore/add/remove/force
+
+# Add or remove space between 'new' and '(' in 'new()'.
+sp_between_new_paren            = ignore   # ignore/add/remove/force
+
+# Add or remove space between ')' and type in 'new(foo) BAR'.
+sp_after_newop_paren            = ignore   # ignore/add/remove/force
+
+# Add or remove space inside parenthesis of the new operator
+# as in 'new(foo) BAR'.
+sp_inside_newop_paren           = ignore   # ignore/add/remove/force
+
+# Add or remove space after the open parenthesis of the new operator,
+# as in 'new(foo) BAR'.
+#
+# Overrides sp_inside_newop_paren.
+sp_inside_newop_paren_open      = ignore   # ignore/add/remove/force
+
+# Add or remove space before the close parenthesis of the new operator,
+# as in 'new(foo) BAR'.
+#
+# Overrides sp_inside_newop_paren.
+sp_inside_newop_paren_close     = ignore   # ignore/add/remove/force
+
+# Add or remove space before a trailing or embedded comment.
+sp_before_tr_emb_cmt            = ignore   # ignore/add/remove/force
+
+# Number of spaces before a trailing or embedded comment.
+sp_num_before_tr_emb_cmt        = 0        # unsigned number
+
+# (Java) Add or remove space between an annotation and the open parenthesis.
+sp_annotation_paren             = ignore   # ignore/add/remove/force
+
+# If true, vbrace tokens are dropped to the previous token and skipped.
+sp_skip_vbrace_tokens           = false    # true/false
+
+# Add or remove space after 'noexcept'.
+sp_after_noexcept               = ignore   # ignore/add/remove/force
+
+# Add or remove space after '_'.
+sp_vala_after_translation       = ignore   # ignore/add/remove/force
+
+# If true, a <TAB> is inserted after #define.
+force_tab_after_define          = false    # true/false
+
+#
+# Indenting options
+#
+
+# The number of columns to indent per level. Usually 2, 3, 4, or 8.
+#
+# Default: 8
+indent_columns                  = 4        # unsigned number
+
+# The continuation indent. If non-zero, this overrides the indent of '(', '['
+# and '=' continuation indents. Negative values are OK; negative value is
+# absolute and not increased for each '(' or '[' level.
+#
+# For FreeBSD, this is set to 4.
+indent_continue                 = 0        # number
+
+# The continuation indent, only for class header line(s). If non-zero, this
+# overrides the indent of 'class' continuation indents.
+indent_continue_class_head      = 0        # unsigned number
+
+# Whether to indent empty lines (i.e. lines which contain only spaces before
+# the newline character).
+indent_single_newlines          = false    # true/false
+
+# The continuation indent for func_*_param if they are true. If non-zero, this
+# overrides the indent.
+indent_param                    = 0        # unsigned number
+
+# How to use tabs when indenting code.
+#
+# 0: Spaces only
+# 1: Indent with tabs to brace level, align with spaces (default)
+# 2: Indent and align with tabs, using spaces when not on a tabstop
+#
+# Default: 1
+indent_with_tabs                = 0        # unsigned number
+
+# Whether to indent comments that are not at a brace level with tabs on a
+# tabstop. Requires indent_with_tabs=2. If false, will use spaces.
+indent_cmt_with_tabs            = false    # true/false
+
+# Whether to indent strings broken by '\' so that they line up.
+indent_align_string             = false    # true/false
+
+# The number of spaces to indent multi-line XML strings.
+# Requires indent_align_string=true.
+indent_xml_string               = 0        # unsigned number
+
+# Spaces to indent '{' from level.
+indent_brace                    = 0        # unsigned number
+
+# Whether braces are indented to the body level.
+indent_braces                   = false    # true/false
+
+# Whether to disable indenting function braces if indent_braces=true.
+indent_braces_no_func           = false    # true/false
+
+# Whether to disable indenting class braces if indent_braces=true.
+indent_braces_no_class          = false    # true/false
+
+# Whether to disable indenting struct braces if indent_braces=true.
+indent_braces_no_struct         = false    # true/false
+
+# Whether to indent based on the size of the brace parent,
+# i.e. 'if' => 3 spaces, 'for' => 4 spaces, etc.
+indent_brace_parent             = false    # true/false
+
+# Whether to indent based on the open parenthesis instead of the open brace
+# in '({\n'.
+indent_paren_open_brace         = false    # true/false
+
+# (C#) Whether to indent the brace of a C# delegate by another level.
+indent_cs_delegate_brace        = false    # true/false
+
+# (C#) Whether to indent a C# delegate (to handle delegates with no brace) by
+# another level.
+indent_cs_delegate_body         = false    # true/false
+
+# Whether to indent the body of a 'namespace'.
+indent_namespace                = false    # true/false
+
+# Whether to indent only the first namespace, and not any nested namespaces.
+# Requires indent_namespace=true.
+indent_namespace_single_indent  = false    # true/false
+
+# The number of spaces to indent a namespace block.
+# If set to zero, use the value indent_columns
+indent_namespace_level          = 0        # unsigned number
+
+# If the body of the namespace is longer than this number, it won't be
+# indented. Requires indent_namespace=true. 0 means no limit.
+indent_namespace_limit          = 0        # unsigned number
+
+# Whether the 'extern "C"' body is indented.
+indent_extern                   = false    # true/false
+
+# Whether the 'class' body is indented.
+indent_class                    = true    # true/false
+
+# Whether to indent the stuff after a leading base class colon.
+indent_class_colon              = false    # true/false
+
+# Whether to indent based on a class colon instead of the stuff after the
+# colon. Requires indent_class_colon=true.
+indent_class_on_colon           = false    # true/false
+
+# Whether to indent the stuff after a leading class initializer colon.
+indent_constr_colon             = false    # true/false
+
+# Virtual indent from the ':' for member initializers.
+#
+# Default: 2
+indent_ctor_init_leading        = 2        # unsigned number
+
+# Additional indent for constructor initializer list.
+# Negative values decrease indent down to the first column.
+indent_ctor_init                = 0        # number
+
+# Whether to indent 'if' following 'else' as a new block under the 'else'.
+# If false, 'else\nif' is treated as 'else if' for indenting purposes.
+indent_else_if                  = false    # true/false
+
+# Amount to indent variable declarations after a open brace.
+#
+#  <0: Relative
+# >=0: Absolute
+indent_var_def_blk              = 0        # number
+
+# Whether to indent continued variable declarations instead of aligning.
+indent_var_def_cont             = false    # true/false
+
+# Whether to indent continued shift expressions ('<<' and '>>') instead of
+# aligning. Set align_left_shift=false when enabling this.
+indent_shift                    = false    # true/false
+
+# Whether to force indentation of function definitions to start in column 1.
+indent_func_def_force_col1      = false    # true/false
+
+# Whether to indent continued function call parameters one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_func_call_param          = true    # true/false
+
+# Whether to indent continued function definition parameters one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_func_def_param           = true    # true/false
+
+# for function definitions, only if indent_func_def_param is false
+# Allows to align params when appropriate and indent them when not
+# behave as if it was true if paren position is more than this value
+# if paren position is more than the option value
+indent_func_def_param_paren_pos_threshold = 0        # unsigned number
+
+# Whether to indent continued function call prototype one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_func_proto_param         = true    # true/false
+
+# Whether to indent continued function call declaration one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_func_class_param         = true    # true/false
+
+# Whether to indent continued class variable constructors one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_func_ctor_var_param      = true    # true/false
+
+# Whether to indent continued template parameter list one indent level,
+# rather than aligning parameters under the open parenthesis.
+indent_template_param           = true    # true/false
+
+# Double the indent for indent_func_xxx_param options.
+# Use both values of the options indent_columns and indent_param.
+indent_func_param_double        = false    # true/false
+
+# Indentation column for standalone 'const' qualifier on a function
+# prototype.
+indent_func_const               = 0        # unsigned number
+
+# Indentation column for standalone 'throw' qualifier on a function
+# prototype.
+indent_func_throw               = 0        # unsigned number
+
+# How to indent within a macro followed by a brace on the same line
+# This allows reducing the indent in macros that have (for example)
+# `do { ... } while (0)` blocks bracketing them.
+#
+# true:  add an indent for the brace on the same line as the macro
+# false: do not add an indent for the brace on the same line as the macro
+#
+# Default: true
+indent_macro_brace              = true     # true/false
+
+# The number of spaces to indent a continued '->' or '.'.
+# Usually set to 0, 1, or indent_columns.
+indent_member                   = 0        # unsigned number
+
+# Whether lines broken at '.' or '->' should be indented by a single indent.
+# The indent_member option will not be effective if this is set to true.
+indent_member_single            = false    # true/false
+
+# Spaces to indent single line ('//') comments on lines before code.
+indent_sing_line_comments       = 0        # unsigned number
+
+# Whether to indent trailing single line ('//') comments relative to the code
+# instead of trying to keep the same absolute column.
+indent_relative_single_line_comments = false    # true/false
+
+# Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns.
+indent_switch_case              = 0        # unsigned number
+
+# indent 'break' with 'case' from 'switch'.
+indent_switch_break_with_case   = false    # true/false
+
+# Whether to indent preprocessor statements inside of switch statements.
+#
+# Default: true
+indent_switch_pp                = true     # true/false
+
+# Spaces to shift the 'case' line, without affecting any other lines.
+# Usually 0.
+indent_case_shift               = 0        # unsigned number
+
+# Spaces to indent '{' from 'case'. By default, the brace will appear under
+# the 'c' in case. Usually set to 0 or indent_columns. Negative values are OK.
+indent_case_brace               = 0        # number
+
+# Whether to indent comments found in first column.
+indent_col1_comment             = false    # true/false
+
+# Whether to indent multi string literal in first column.
+indent_col1_multi_string_literal = false    # true/false
+
+# How to indent goto labels.
+#
+#  >0: Absolute column where 1 is the leftmost column
+# <=0: Subtract from brace indent
+#
+# Default: 1
+indent_label                    = 1        # number
+
+# How to indent access specifiers that are followed by a
+# colon.
+#
+#  >0: Absolute column where 1 is the leftmost column
+# <=0: Subtract from brace indent
+#
+# Default: 1
+indent_access_spec              = 1        # number
+
+# Whether to indent the code after an access specifier by one level.
+# If true, this option forces 'indent_access_spec=0'.
+indent_access_spec_body         = false    # true/false
+
+# If an open parenthesis is followed by a newline, whether to indent the next
+# line so that it lines up after the open parenthesis (not recommended).
+indent_paren_nl                 = false    # true/false
+
+# How to indent a close parenthesis after a newline.
+#
+# 0: Indent to body level (default)
+# 1: Align under the open parenthesis
+# 2: Indent to the brace level
+indent_paren_close              = 0        # unsigned number
+
+# Whether to indent the open parenthesis of a function definition,
+# if the parenthesis is on its own line.
+indent_paren_after_func_def     = false    # true/false
+
+# Whether to indent the open parenthesis of a function declaration,
+# if the parenthesis is on its own line.
+indent_paren_after_func_decl    = false    # true/false
+
+# Whether to indent the open parenthesis of a function call,
+# if the parenthesis is on its own line.
+indent_paren_after_func_call    = false    # true/false
+
+# Whether to indent a comma when inside a parenthesis.
+# If true, aligns under the open parenthesis.
+indent_comma_paren              = false    # true/false
+
+# Whether to indent a Boolean operator when inside a parenthesis.
+# If true, aligns under the open parenthesis.
+indent_bool_paren               = false    # true/false
+
+# Whether to indent a semicolon when inside a for parenthesis.
+# If true, aligns under the open for parenthesis.
+indent_semicolon_for_paren      = false    # true/false
+
+# Whether to align the first expression to following ones
+# if indent_bool_paren=true.
+indent_first_bool_expr          = false    # true/false
+
+# Whether to align the first expression to following ones
+# if indent_semicolon_for_paren=true.
+indent_first_for_expr           = false    # true/false
+
+# If an open square is followed by a newline, whether to indent the next line
+# so that it lines up after the open square (not recommended).
+indent_square_nl                = false    # true/false
+
+# (ESQL/C) Whether to preserve the relative indent of 'EXEC SQL' bodies.
+indent_preserve_sql             = false    # true/false
+
+# Whether to align continued statements at the '='. If false or if the '=' is
+# followed by a newline, the next line is indent one tab.
+#
+# Default: true
+indent_align_assign             = true     # true/false
+
+# Whether to align continued statements at the '('. If false or the '(' is
+# followed by a newline, the next line indent is one tab.
+#
+# Default: true
+indent_align_paren              = true     # true/false
+
+# (OC) Whether to indent Objective-C blocks at brace level instead of usual
+# rules.
+indent_oc_block                 = false    # true/false
+
+# (OC) Indent for Objective-C blocks in a message relative to the parameter
+# name.
+#
+# =0: Use indent_oc_block rules
+# >0: Use specified number of spaces to indent
+indent_oc_block_msg             = 0        # unsigned number
+
+# (OC) Minimum indent for subsequent parameters
+indent_oc_msg_colon             = 0        # unsigned number
+
+# (OC) Whether to prioritize aligning with initial colon (and stripping spaces
+# from lines, if necessary).
+#
+# Default: true
+indent_oc_msg_prioritize_first_colon = true     # true/false
+
+# (OC) Whether to indent blocks the way that Xcode does by default
+# (from the keyword if the parameter is on its own line; otherwise, from the
+# previous indentation level). Requires indent_oc_block_msg=true.
+indent_oc_block_msg_xcode_style = false    # true/false
+
+# (OC) Whether to indent blocks from where the brace is, relative to a
+# message keyword. Requires indent_oc_block_msg=true.
+indent_oc_block_msg_from_keyword = false    # true/false
+
+# (OC) Whether to indent blocks from where the brace is, relative to a message
+# colon. Requires indent_oc_block_msg=true.
+indent_oc_block_msg_from_colon  = false    # true/false
+
+# (OC) Whether to indent blocks from where the block caret is.
+# Requires indent_oc_block_msg=true.
+indent_oc_block_msg_from_caret  = false    # true/false
+
+# (OC) Whether to indent blocks from where the brace caret is.
+# Requires indent_oc_block_msg=true.
+indent_oc_block_msg_from_brace  = false    # true/false
+
+# When indenting after virtual brace open and newline add further spaces to
+# reach this minimum indent.
+indent_min_vbrace_open          = 0        # unsigned number
+
+# Whether to add further spaces after regular indent to reach next tabstop
+# when identing after virtual brace open and newline.
+indent_vbrace_open_on_tabstop   = false    # true/false
+
+# How to indent after a brace followed by another token (not a newline).
+# true:  indent all contained lines to match the token
+# false: indent all contained lines to match the brace
+#
+# Default: true
+indent_token_after_brace        = true     # true/false
+
+# Whether to indent the body of a C++11 lambda.
+indent_cpp_lambda_body          = false    # true/false
+
+# (C#) Whether to indent a 'using' block if no braces are used.
+#
+# Default: true
+indent_using_block              = true     # true/false
+
+# How to indent the continuation of ternary operator.
+#
+# 0: Off (default)
+# 1: When the `if_false` is a continuation, indent it under `if_false`
+# 2: When the `:` is a continuation, indent it under `?`
+indent_ternary_operator         = 0        # unsigned number
+
+# If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column.
+indent_off_after_return_new     = false    # true/false
+
+# If true, the tokens after return are indented with regular single indentation. By default (false) the indentation is after the return token.
+indent_single_after_return      = false    # true/false
+
+# Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they
+# have their own indentation).
+indent_ignore_asm_block         = false    # true/false
+
+#
+# Newline adding and removing options
+#
+
+# Whether to collapse empty blocks between '{' and '}'.
+nl_collapse_empty_body          = false    # true/false
+
+# Don't split one-line braced assignments, as in 'foo_t f = { 1, 2 };'.
+nl_assign_leave_one_liners      = true    # true/false
+
+# Don't split one-line braced statements inside a 'class xx { }' body.
+nl_class_leave_one_liners       = true    # true/false
+
+# Don't split one-line enums, as in 'enum foo { BAR = 15 };'
+nl_enum_leave_one_liners        = true    # true/false
+
+# Don't split one-line get or set functions.
+nl_getset_leave_one_liners      = true    # true/false
+
+# (C#) Don't split one-line property get or set functions.
+nl_cs_property_leave_one_liners = true    # true/false
+
+# Don't split one-line function definitions, as in 'int foo() { return 0; }'.
+# night modify nl_func_type_name
+nl_func_leave_one_liners        = true    # true/false
+
+# Don't split one-line C++11 lambdas, as in '[]() { return 0; }'.
+nl_cpp_lambda_leave_one_liners  = false    # true/false
+
+# Don't split one-line if/else statements, as in 'if(...) b++;'.
+nl_if_leave_one_liners          = true    # true/false
+
+# Don't split one-line while statements, as in 'while(...) b++;'.
+nl_while_leave_one_liners       = true    # true/false
+
+# Don't split one-line for statements, as in 'for(...) b++;'.
+nl_for_leave_one_liners         = true    # true/false
+
+# (OC) Don't split one-line Objective-C messages.
+nl_oc_msg_leave_one_liner       = false    # true/false
+
+# (OC) Add or remove newline between method declaration and '{'.
+nl_oc_mdef_brace                = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove newline between Objective-C block signature and '{'.
+nl_oc_block_brace               = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove blank line before '@interface' statement.
+nl_oc_before_interface          = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove blank line before '@implementation' statement.
+nl_oc_before_implementation     = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove blank line before '@end' statement.
+nl_oc_before_end                = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove newline between '@interface' and '{'.
+nl_oc_interface_brace           = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove newline between '@implementation' and '{'.
+nl_oc_implementation_brace      = ignore   # ignore/add/remove/force
+
+# Add or remove newlines at the start of the file.
+nl_start_of_file                = ignore   # ignore/add/remove/force
+
+# The minimum number of newlines at the start of the file (only used if
+# nl_start_of_file is 'add' or 'force').
+nl_start_of_file_min            = 0        # unsigned number
+
+# Add or remove newline at the end of the file.
+nl_end_of_file                  = ignore   # ignore/add/remove/force
+
+# The minimum number of newlines at the end of the file (only used if
+# nl_end_of_file is 'add' or 'force').
+nl_end_of_file_min              = 0        # unsigned number
+
+# Add or remove newline between '=' and '{'.
+nl_assign_brace                 = ignore   # ignore/add/remove/force
+
+# (D) Add or remove newline between '=' and '['.
+nl_assign_square                = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '[]' and '{'.
+nl_tsquare_brace                = ignore   # ignore/add/remove/force
+
+# (D) Add or remove newline after '= ['. Will also affect the newline before
+# the ']'.
+nl_after_square_assign          = ignore   # ignore/add/remove/force
+
+# Add or remove newline between a function call's ')' and '{', as in
+# 'list_for_each(item, &list) { }'.
+nl_fcall_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'enum' and '{'.
+nl_enum_brace                   = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'enum' and 'class'.
+nl_enum_class                   = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'enum class' and the identifier.
+nl_enum_class_identifier        = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'enum class' type and ':'.
+nl_enum_identifier_colon        = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'enum class identifier :' and type.
+nl_enum_colon_type              = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'struct and '{'.
+nl_struct_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'union' and '{'.
+nl_union_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'if' and '{'.
+nl_if_brace                     = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and 'else'.
+nl_brace_else                   = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'else if' and '{'. If set to ignore,
+# nl_if_brace is used instead.
+nl_elseif_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'else' and '{'.
+nl_else_brace                   = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'else' and 'if'.
+nl_else_if                      = ignore   # ignore/add/remove/force
+
+# Add or remove newline before 'if'/'else if' closing parenthesis.
+nl_before_if_closing_paren      = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and 'finally'.
+nl_brace_finally                = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'finally' and '{'.
+nl_finally_brace                = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'try' and '{'.
+nl_try_brace                    = ignore   # ignore/add/remove/force
+
+# Add or remove newline between get/set and '{'.
+nl_getset_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'for' and '{'.
+nl_for_brace                    = ignore   # ignore/add/remove/force
+
+# Add or remove newline before the '{' of a 'catch' statement, as in
+# 'catch (decl) <here> {'.
+nl_catch_brace                  = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove newline before the '{' of a '@catch' statement, as in
+# '@catch (decl) <here> {'. If set to ignore, nl_catch_brace is used.
+nl_oc_catch_brace               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and 'catch'.
+nl_brace_catch                  = ignore   # ignore/add/remove/force
+
+# (OC) Add or remove newline between '}' and '@catch'. If set to ignore,
+# nl_brace_catch is used.
+nl_oc_brace_catch               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and ']'.
+nl_brace_square                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and ')' in a function invocation.
+nl_brace_fparen                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'while' and '{'.
+nl_while_brace                  = ignore   # ignore/add/remove/force
+
+# (D) Add or remove newline between 'scope (x)' and '{'.
+nl_scope_brace                  = ignore   # ignore/add/remove/force
+
+# (D) Add or remove newline between 'unittest' and '{'.
+nl_unittest_brace               = ignore   # ignore/add/remove/force
+
+# (D) Add or remove newline between 'version (x)' and '{'.
+nl_version_brace                = ignore   # ignore/add/remove/force
+
+# (C#) Add or remove newline between 'using' and '{'.
+nl_using_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline between two open or close braces. Due to general
+# newline/brace handling, REMOVE may not work.
+nl_brace_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'do' and '{'.
+nl_do_brace                     = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '}' and 'while' of 'do' statement.
+nl_brace_while                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'switch' and '{'.
+nl_switch_brace                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'synchronized' and '{'.
+nl_synchronized_brace           = ignore   # ignore/add/remove/force
+
+# Add a newline between ')' and '{' if the ')' is on a different line than the
+# if/for/etc.
+#
+# Overrides nl_for_brace, nl_if_brace, nl_switch_brace, nl_while_switch and
+# nl_catch_brace.
+nl_multi_line_cond              = false    # true/false
+
+# Add a newline after '(' if an if/for/while/switch condition spans multiple
+# lines
+nl_multi_line_sparen_open       = ignore   # ignore/add/remove/force
+
+# Add a newline before ')' if an if/for/while/switch condition spans multiple
+# lines. Overrides nl_before_if_closing_paren if both are specified.
+nl_multi_line_sparen_close      = ignore   # ignore/add/remove/force
+
+# Force a newline in a define after the macro name for multi-line defines.
+nl_multi_line_define            = false    # true/false
+
+# Whether to add a newline before 'case', and a blank line before a 'case'
+# statement that follows a ';' or '}'.
+nl_before_case                  = false    # true/false
+
+# Whether to add a newline after a 'case' statement.
+nl_after_case                   = false    # true/false
+
+# Add or remove newline between a case ':' and '{'.
+#
+# Overrides nl_after_case.
+nl_case_colon_brace             = ignore   # ignore/add/remove/force
+
+# Add or remove newline between ')' and 'throw'.
+nl_before_throw                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'namespace' and '{'.
+nl_namespace_brace              = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template class.
+nl_template_class               = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template class declaration.
+#
+# Overrides nl_template_class.
+nl_template_class_decl          = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<>' of a specialized class declaration.
+#
+# Overrides nl_template_class_decl.
+nl_template_class_decl_special  = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template class definition.
+#
+# Overrides nl_template_class.
+nl_template_class_def           = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<>' of a specialized class definition.
+#
+# Overrides nl_template_class_def.
+nl_template_class_def_special   = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template function.
+nl_template_func                = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template function
+# declaration.
+#
+# Overrides nl_template_func.
+nl_template_func_decl           = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<>' of a specialized function
+# declaration.
+#
+# Overrides nl_template_func_decl.
+nl_template_func_decl_special   = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template function
+# definition.
+#
+# Overrides nl_template_func.
+nl_template_func_def            = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<>' of a specialized function
+# definition.
+#
+# Overrides nl_template_func_def.
+nl_template_func_def_special    = ignore   # ignore/add/remove/force
+
+# Add or remove newline after 'template<...>' of a template variable.
+nl_template_var                 = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'template<...>' and 'using' of a templated
+# type alias.
+nl_template_using               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'class' and '{'.
+nl_class_brace                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline before or after (depending on pos_class_comma,
+# may not be IGNORE) each',' in the base class list.
+nl_class_init_args              = ignore   # ignore/add/remove/force
+
+# Add or remove newline after each ',' in the constructor member
+# initialization. Related to nl_constr_colon, pos_constr_colon and
+# pos_constr_comma.
+nl_constr_init_args             = ignore   # ignore/add/remove/force
+
+# Add or remove newline before first element, after comma, and after last
+# element, in 'enum'.
+nl_enum_own_lines               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between return type and function name in a function
+# definition.
+# might be modified by nl_func_leave_one_liners
+nl_func_type_name               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between return type and function name inside a class
+# definition. If set to ignore, nl_func_type_name or nl_func_proto_type_name
+# is used instead.
+nl_func_type_name_class         = ignore   # ignore/add/remove/force
+
+# Add or remove newline between class specification and '::'
+# in 'void A::f() { }'. Only appears in separate member implementation (does
+# not appear with in-line implementation).
+nl_func_class_scope             = ignore   # ignore/add/remove/force
+
+# Add or remove newline between function scope and name, as in
+# 'void A :: <here> f() { }'.
+nl_func_scope_name              = ignore   # ignore/add/remove/force
+
+# Add or remove newline between return type and function name in a prototype.
+nl_func_proto_type_name         = ignore   # ignore/add/remove/force
+
+# Add or remove newline between a function name and the opening '(' in the
+# declaration.
+nl_func_paren                   = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_paren for functions with no parameters.
+nl_func_paren_empty             = ignore   # ignore/add/remove/force
+
+# Add or remove newline between a function name and the opening '(' in the
+# definition.
+nl_func_def_paren               = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_def_paren for functions with no parameters.
+nl_func_def_paren_empty         = ignore   # ignore/add/remove/force
+
+# Add or remove newline between a function name and the opening '(' in the
+# call.
+nl_func_call_paren              = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_call_paren for functions with no parameters.
+nl_func_call_paren_empty        = ignore   # ignore/add/remove/force
+
+# Add or remove newline after '(' in a function declaration.
+nl_func_decl_start              = ignore   # ignore/add/remove/force
+
+# Add or remove newline after '(' in a function definition.
+nl_func_def_start               = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_decl_start when there is only one parameter.
+nl_func_decl_start_single       = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_def_start when there is only one parameter.
+nl_func_def_start_single        = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after '(' in a function declaration if '(' and ')'
+# are in different lines. If false, nl_func_decl_start is used instead.
+nl_func_decl_start_multi_line   = false    # true/false
+
+# Whether to add a newline after '(' in a function definition if '(' and ')'
+# are in different lines. If false, nl_func_def_start is used instead.
+nl_func_def_start_multi_line    = false    # true/false
+
+# Add or remove newline after each ',' in a function declaration.
+nl_func_decl_args               = ignore   # ignore/add/remove/force
+
+# Add or remove newline after each ',' in a function definition.
+nl_func_def_args                = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after each ',' in a function declaration if '('
+# and ')' are in different lines. If false, nl_func_decl_args is used instead.
+nl_func_decl_args_multi_line    = false    # true/false
+
+# Whether to add a newline after each ',' in a function definition if '('
+# and ')' are in different lines. If false, nl_func_def_args is used instead.
+nl_func_def_args_multi_line     = false    # true/false
+
+# Add or remove newline before the ')' in a function declaration.
+nl_func_decl_end                = ignore   # ignore/add/remove/force
+
+# Add or remove newline before the ')' in a function definition.
+nl_func_def_end                 = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_decl_end when there is only one parameter.
+nl_func_decl_end_single         = ignore   # ignore/add/remove/force
+
+# Overrides nl_func_def_end when there is only one parameter.
+nl_func_def_end_single          = ignore   # ignore/add/remove/force
+
+# Whether to add a newline before ')' in a function declaration if '(' and ')'
+# are in different lines. If false, nl_func_decl_end is used instead.
+nl_func_decl_end_multi_line     = false    # true/false
+
+# Whether to add a newline before ')' in a function definition if '(' and ')'
+# are in different lines. If false, nl_func_def_end is used instead.
+nl_func_def_end_multi_line      = false    # true/false
+
+# Add or remove newline between '()' in a function declaration.
+nl_func_decl_empty              = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '()' in a function definition.
+nl_func_def_empty               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between '()' in a function call.
+nl_func_call_empty              = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after '(' in a function call,
+# has preference over nl_func_call_start_multi_line.
+nl_func_call_start              = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after '(' in a function call if '(' and ')' are in
+# different lines.
+nl_func_call_start_multi_line   = false    # true/false
+
+# Whether to add a newline after each ',' in a function call if '(' and ')'
+# are in different lines.
+nl_func_call_args_multi_line    = false    # true/false
+
+# Whether to add a newline before ')' in a function call if '(' and ')' are in
+# different lines.
+nl_func_call_end_multi_line     = false    # true/false
+
+# Whether to add a newline after '<' of a template parameter list.
+nl_template_start               = false    # true/false
+
+# Whether to add a newline after each ',' in a template parameter list.
+nl_template_args                = false    # true/false
+
+# Whether to add a newline before '>' of a template parameter list.
+nl_template_end                 = false    # true/false
+
+# (OC) Whether to put each Objective-C message parameter on a separate line.
+# See nl_oc_msg_leave_one_liner.
+nl_oc_msg_args                  = false    # true/false
+
+# Add or remove newline between function signature and '{'.
+nl_fdef_brace                   = ignore   # ignore/add/remove/force
+
+# Add or remove newline between function signature and '{',
+# if signature ends with ')'. Overrides nl_fdef_brace.
+nl_fdef_brace_cond              = ignore   # ignore/add/remove/force
+
+# Add or remove newline between C++11 lambda signature and '{'.
+nl_cpp_ldef_brace               = ignore   # ignore/add/remove/force
+
+# Add or remove newline between 'return' and the return expression.
+nl_return_expr                  = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after semicolons, except in 'for' statements.
+nl_after_semicolon              = false    # true/false
+
+# (Java) Add or remove newline between the ')' and '{{' of the double brace
+# initializer.
+nl_paren_dbrace_open            = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after the type in an unnamed temporary
+# direct-list-initialization.
+nl_type_brace_init_lst          = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after the open brace in an unnamed temporary
+# direct-list-initialization.
+nl_type_brace_init_lst_open     = ignore   # ignore/add/remove/force
+
+# Whether to add a newline before the close brace in an unnamed temporary
+# direct-list-initialization.
+nl_type_brace_init_lst_close    = ignore   # ignore/add/remove/force
+
+# Whether to add a newline after '{'. This also adds a newline before the
+# matching '}'.
+nl_after_brace_open             = false    # true/false
+
+# Whether to add a newline between the open brace and a trailing single-line
+# comment. Requires nl_after_brace_open=true.
+nl_after_brace_open_cmt         = false    # true/false
+
+# Whether to add a newline after a virtual brace open with a non-empty body.
+# These occur in un-braced if/while/do/for statement bodies.
+nl_after_vbrace_open            = false    # true/false
+
+# Whether to add a newline after a virtual brace open with an empty body.
+# These occur in un-braced if/while/do/for statement bodies.
+nl_after_vbrace_open_empty      = false    # true/false
+
+# Whether to add a newline after '}'. Does not apply if followed by a
+# necessary ';'.
+nl_after_brace_close            = false    # true/false
+
+# Whether to add a newline after a virtual brace close,
+# as in 'if (foo) a++; <here> return;'.
+nl_after_vbrace_close           = false    # true/false
+
+# Add or remove newline between the close brace and identifier,
+# as in 'struct { int a; } <here> b;'. Affects enumerations, unions and
+# structures. If set to ignore, uses nl_after_brace_close.
+nl_brace_struct_var             = ignore   # ignore/add/remove/force
+
+# Whether to alter newlines in '#define' macros.
+nl_define_macro                 = false    # true/false
+
+# Whether to alter newlines between consecutive parenthesis closes. The number
+# of closing parentheses in a line will depend on respective open parenthesis
+# lines.
+nl_squeeze_paren_close          = false    # true/false
+
+# Whether to remove blanks after '#ifxx' and '#elxx', or before '#elxx' and
+# '#endif'. Does not affect top-level #ifdefs.
+nl_squeeze_ifdef                = false    # true/false
+
+# Makes the nl_squeeze_ifdef option affect the top-level #ifdefs as well.
+nl_squeeze_ifdef_top_level      = false    # true/false
+
+# Add or remove blank line before 'if'.
+nl_before_if                    = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'if' statement. Add/Force work only if the
+# next token is not a closing brace.
+nl_after_if                     = ignore   # ignore/add/remove/force
+
+# Add or remove blank line before 'for'.
+nl_before_for                   = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'for' statement.
+nl_after_for                    = ignore   # ignore/add/remove/force
+
+# Add or remove blank line before 'while'.
+nl_before_while                 = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'while' statement.
+nl_after_while                  = ignore   # ignore/add/remove/force
+
+# Add or remove blank line before 'switch'.
+nl_before_switch                = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'switch' statement.
+nl_after_switch                 = ignore   # ignore/add/remove/force
+
+# Add or remove blank line before 'synchronized'.
+nl_before_synchronized          = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'synchronized' statement.
+nl_after_synchronized           = ignore   # ignore/add/remove/force
+
+# Add or remove blank line before 'do'.
+nl_before_do                    = ignore   # ignore/add/remove/force
+
+# Add or remove blank line after 'do/while' statement.
+nl_after_do                     = ignore   # ignore/add/remove/force
+
+# Whether to put a blank line before 'return' statements, unless after an open
+# brace.
+nl_before_return                = false    # true/false
+
+# Whether to put a blank line after 'return' statements, unless followed by a
+# close brace.
+nl_after_return                 = false    # true/false
+
+# (Java) Whether to put a blank line before a member '.' or '->' operators.
+nl_before_member                = ignore   # ignore/add/remove/force
+
+# (Java) Whether to put a blank line after a member '.' or '->' operators.
+nl_after_member                 = ignore   # ignore/add/remove/force
+
+# Whether to double-space commented-entries in 'struct'/'union'/'enum'.
+nl_ds_struct_enum_cmt           = false    # true/false
+
+# Whether to force a newline before '}' of a 'struct'/'union'/'enum'.
+# (Lower priority than eat_blanks_before_close_brace.)
+nl_ds_struct_enum_close_brace   = false    # true/false
+
+# Add or remove newline before or after (depending on pos_class_colon) a class
+# colon, as in 'class Foo <here> : <or here> public Bar'.
+nl_class_colon                  = ignore   # ignore/add/remove/force
+
+# Add or remove newline around a class constructor colon. The exact position
+# depends on nl_constr_init_args, pos_constr_colon and pos_constr_comma.
+nl_constr_colon                 = ignore   # ignore/add/remove/force
+
+# Whether to collapse a two-line namespace, like 'namespace foo\n{ decl; }'
+# into a single line. If true, prevents other brace newline rules from turning
+# such code into four lines.
+nl_namespace_two_to_one_liner   = false    # true/false
+
+# Whether to remove a newline in simple unbraced if statements, turning them
+# into one-liners, as in 'if(b)\n i++;' => 'if(b) i++;'.
+nl_create_if_one_liner          = false    # true/false
+
+# Whether to remove a newline in simple unbraced for statements, turning them
+# into one-liners, as in 'for (...)\n stmt;' => 'for (...) stmt;'.
+nl_create_for_one_liner         = false    # true/false
+
+# Whether to remove a newline in simple unbraced while statements, turning
+# them into one-liners, as in 'while (expr)\n stmt;' => 'while (expr) stmt;'.
+nl_create_while_one_liner       = false    # true/false
+
+# Whether to collapse a function definition whose body (not counting braces)
+# is only one line so that the entire definition (prototype, braces, body) is
+# a single line.
+nl_create_func_def_one_liner    = false    # true/false
+
+# Whether to collapse a function definition whose body (not counting braces)
+# is only one line so that the entire definition (prototype, braces, body) is
+# a single line.
+nl_create_list_one_liner        = false    # true/false
+
+# Whether to split one-line simple unbraced if statements into two lines by
+# adding a newline, as in 'if(b) <here> i++;'.
+nl_split_if_one_liner           = false    # true/false
+
+# Whether to split one-line simple unbraced for statements into two lines by
+# adding a newline, as in 'for (...) <here> stmt;'.
+nl_split_for_one_liner          = false    # true/false
+
+# Whether to split one-line simple unbraced while statements into two lines by
+# adding a newline, as in 'while (expr) <here> stmt;'.
+nl_split_while_one_liner        = false    # true/false
+
+#
+# Blank line options
+#
+
+# The maximum number of consecutive newlines (3 = 2 blank lines).
+nl_max                          = 0        # unsigned number
+
+# The maximum number of consecutive newlines in a function.
+nl_max_blank_in_func            = 0        # unsigned number
+
+# The number of newlines before a function prototype.
+nl_before_func_body_proto       = 0        # unsigned number
+
+# The number of newlines before a multi-line function definition.
+nl_before_func_body_def         = 0        # unsigned number
+
+# The number of newlines before a class constructor/destructor prototype.
+nl_before_func_class_proto      = 0        # unsigned number
+
+# The number of newlines before a class constructor/destructor definition.
+nl_before_func_class_def        = 0        # unsigned number
+
+# The number of newlines after a function prototype.
+nl_after_func_proto             = 0        # unsigned number
+
+# The number of newlines after a function prototype, if not followed by
+# another function prototype.
+nl_after_func_proto_group       = 0        # unsigned number
+
+# The number of newlines after a class constructor/destructor prototype.
+nl_after_func_class_proto       = 0        # unsigned number
+
+# The number of newlines after a class constructor/destructor prototype,
+# if not followed by another constructor/destructor prototype.
+nl_after_func_class_proto_group = 0        # unsigned number
+
+# Whether one-line method definitions inside a class body should be treated
+# as if they were prototypes for the purposes of adding newlines.
+#
+# Requires nl_class_leave_one_liners=true. Overrides nl_before_func_body_def
+# and nl_before_func_class_def for one-liners.
+nl_class_leave_one_liner_groups = false    # true/false
+
+# The number of newlines after '}' of a multi-line function body.
+nl_after_func_body              = 0        # unsigned number
+
+# The number of newlines after '}' of a multi-line function body in a class
+# declaration. Also affects class constructors/destructors.
+#
+# Overrides nl_after_func_body.
+nl_after_func_body_class        = 0        # unsigned number
+
+# The number of newlines after '}' of a single line function body. Also
+# affects class constructors/destructors.
+#
+# Overrides nl_after_func_body and nl_after_func_body_class.
+nl_after_func_body_one_liner    = 0        # unsigned number
+
+# The number of blank lines after a block of variable definitions at the top
+# of a function body.
+#
+# 0: No change (default).
+nl_func_var_def_blk             = 0        # unsigned number
+
+# The number of newlines before a block of typedefs. If nl_after_access_spec
+# is non-zero, that option takes precedence.
+#
+# 0: No change (default).
+nl_typedef_blk_start            = 0        # unsigned number
+
+# The number of newlines after a block of typedefs.
+#
+# 0: No change (default).
+nl_typedef_blk_end              = 0        # unsigned number
+
+# The maximum number of consecutive newlines within a block of typedefs.
+#
+# 0: No change (default).
+nl_typedef_blk_in               = 0        # unsigned number
+
+# The number of newlines before a block of variable definitions not at the top
+# of a function body. If nl_after_access_spec is non-zero, that option takes
+# precedence.
+#
+# 0: No change (default).
+nl_var_def_blk_start            = 0        # unsigned number
+
+# The number of newlines after a block of variable definitions not at the top
+# of a function body.
+#
+# 0: No change (default).
+nl_var_def_blk_end              = 0        # unsigned number
+
+# The maximum number of consecutive newlines within a block of variable
+# definitions.
+#
+# 0: No change (default).
+nl_var_def_blk_in               = 0        # unsigned number
+
+# The minimum number of newlines before a multi-line comment.
+# Doesn't apply if after a brace open or another multi-line comment.
+nl_before_block_comment         = 0        # unsigned number
+
+# The minimum number of newlines before a single-line C comment.
+# Doesn't apply if after a brace open or other single-line C comments.
+nl_before_c_comment             = 0        # unsigned number
+
+# The minimum number of newlines before a CPP comment.
+# Doesn't apply if after a brace open or other CPP comments.
+nl_before_cpp_comment           = 0        # unsigned number
+
+# Whether to force a newline after a multi-line comment.
+nl_after_multiline_comment      = false    # true/false
+
+# Whether to force a newline after a label's colon.
+nl_after_label_colon            = false    # true/false
+
+# The number of newlines after '}' or ';' of a struct/enum/union definition.
+nl_after_struct                 = 0        # unsigned number
+
+# The number of newlines before a class definition.
+nl_before_class                 = 0        # unsigned number
+
+# The number of newlines after '}' or ';' of a class definition.
+nl_after_class                  = 0        # unsigned number
+
+# The number of newlines before a namespace.
+nl_before_namespace             = 0        # unsigned number
+
+# The number of newlines after '{' of a namespace. This also adds newlines
+# before the matching '}'.
+#
+# 0: Apply eat_blanks_after_open_brace or eat_blanks_before_close_brace if
+#     applicable, otherwise no change.
+#
+# Overrides eat_blanks_after_open_brace and eat_blanks_before_close_brace.
+nl_inside_namespace             = 0        # unsigned number
+
+# The number of newlines after '}' of a namespace.
+nl_after_namespace              = 0        # unsigned number
+
+# The number of newlines before an access specifier label. This also includes
+# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count
+# if after a brace open.
+#
+# 0: No change (default).
+nl_before_access_spec           = 0        # unsigned number
+
+# The number of newlines after an access specifier label. This also includes
+# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count
+# if after a brace open.
+#
+# 0: No change (default).
+#
+# Overrides nl_typedef_blk_start and nl_var_def_blk_start.
+nl_after_access_spec            = 0        # unsigned number
+
+# The number of newlines between a function definition and the function
+# comment, as in '// comment\n <here> void foo() {...}'.
+#
+# 0: No change (default).
+nl_comment_func_def             = 0        # unsigned number
+
+# The number of newlines after a try-catch-finally block that isn't followed
+# by a brace close.
+#
+# 0: No change (default).
+nl_after_try_catch_finally      = 0        # unsigned number
+
+# (C#) The number of newlines before and after a property, indexer or event
+# declaration.
+#
+# 0: No change (default).
+nl_around_cs_property           = 0        # unsigned number
+
+# (C#) The number of newlines between the get/set/add/remove handlers.
+#
+# 0: No change (default).
+nl_between_get_set              = 0        # unsigned number
+
+# (C#) Add or remove newline between property and the '{'.
+nl_property_brace               = ignore   # ignore/add/remove/force
+
+# Whether to remove blank lines after '{'.
+eat_blanks_after_open_brace     = false    # true/false
+
+# Whether to remove blank lines before '}'.
+eat_blanks_before_close_brace   = false    # true/false
+
+# How aggressively to remove extra newlines not in preprocessor.
+#
+# 0: No change (default)
+# 1: Remove most newlines not handled by other config
+# 2: Remove all newlines and reformat completely by config
+nl_remove_extra_newlines        = 0        # unsigned number
+
+# (Java) Add or remove newline after an annotation statement. Only affects
+# annotations that are after a newline.
+nl_after_annotation             = ignore   # ignore/add/remove/force
+
+# (Java) Add or remove newline between two annotations.
+nl_between_annotation           = ignore   # ignore/add/remove/force
+
+#
+# Positioning options
+#
+
+# The position of arithmetic operators in wrapped expressions.
+pos_arith                       = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of assignment in wrapped expressions. Do not affect '='
+# followed by '{'.
+pos_assign                      = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of Boolean operators in wrapped expressions.
+pos_bool                        = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of comparison operators in wrapped expressions.
+pos_compare                     = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of conditional operators, as in the '?' and ':' of
+# 'expr ? stmt : stmt', in wrapped expressions.
+pos_conditional                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of the comma in wrapped expressions.
+pos_comma                       = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of the comma in enum entries.
+pos_enum_comma                  = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of the comma in the base class list if there is more than one
+# line. Affects nl_class_init_args.
+pos_class_comma                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of the comma in the constructor initialization list.
+# Related to nl_constr_colon, nl_constr_init_args and pos_constr_colon.
+pos_constr_comma                = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of trailing/leading class colon, between class and base class
+# list. Affects nl_class_colon.
+pos_class_colon                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+# The position of colons between constructor and member initialization.
+# Related to nl_constr_colon, nl_constr_init_args and pos_constr_comma.
+pos_constr_colon                = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
+
+#
+# Line splitting options
+#
+
+# Try to limit code width to N columns.
+code_width                      = 0        # unsigned number
+
+# Whether to fully split long 'for' statements at semi-colons.
+ls_for_split_full               = false    # true/false
+
+# Whether to fully split long function prototypes/calls at commas.
+# The option ls_code_width has priority over the option ls_func_split_full.
+ls_func_split_full              = false    # true/false
+
+# Whether to split lines as close to code_width as possible and ignore some
+# groupings.
+# The option ls_code_width has priority over the option ls_func_split_full.
+ls_code_width                   = false    # true/false
+
+#
+# Code alignment options (not left column spaces/tabs)
+#
+
+# Whether to keep non-indenting tabs.
+align_keep_tabs                 = false    # true/false
+
+# Whether to use tabs for aligning.
+align_with_tabs                 = false    # true/false
+
+# Whether to bump out to the next tab when aligning.
+align_on_tabstop                = false    # true/false
+
+# Whether to right-align numbers.
+align_number_right              = false    # true/false
+
+# Whether to keep whitespace not required for alignment.
+align_keep_extra_space          = false    # true/false
+
+# Whether to align variable definitions in prototypes and functions.
+align_func_params               = false    # true/false
+
+# The span for aligning parameter definitions in function on parameter name.
+#
+# 0: Don't align (default).
+align_func_params_span          = 0        # unsigned number
+
+# The threshold for aligning function parameter definitions.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_func_params_thresh        = 0        # number
+
+# The gap for aligning function parameter definitions.
+align_func_params_gap           = 0        # unsigned number
+
+# The span for aligning constructor value.
+#
+# 0: Don't align (default).
+align_constr_value_span         = 0        # unsigned number
+
+# The threshold for aligning constructor value.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_constr_value_thresh       = 0        # number
+
+# The gap for aligning constructor value.
+align_constr_value_gap          = 0        # unsigned number
+
+# Whether to align parameters in single-line functions that have the same
+# name. The function names must already be aligned with each other.
+align_same_func_call_params     = false    # true/false
+
+# The span for aligning function-call parameters for single line functions.
+#
+# 0: Don't align (default).
+align_same_func_call_params_span = 0        # unsigned number
+
+# The threshold for aligning function-call parameters for single line
+# functions.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_same_func_call_params_thresh = 0        # number
+
+# The span for aligning variable definitions.
+#
+# 0: Don't align (default).
+align_var_def_span              = 0        # unsigned number
+
+# How to consider (or treat) the '*' in the alignment of variable definitions.
+#
+# 0: Part of the type     'void *   foo;' (default)
+# 1: Part of the variable 'void     *foo;'
+# 2: Dangling             'void    *foo;'
+# Dangling: the '*' will not be taken into account when aligning.
+align_var_def_star_style        = 0        # unsigned number
+
+# How to consider (or treat) the '&' in the alignment of variable definitions.
+#
+# 0: Part of the type     'long &   foo;' (default)
+# 1: Part of the variable 'long     &foo;'
+# 2: Dangling             'long    &foo;'
+# Dangling: the '&' will not be taken into account when aligning.
+align_var_def_amp_style         = 0        # unsigned number
+
+# The threshold for aligning variable definitions.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_var_def_thresh            = 0        # number
+
+# The gap for aligning variable definitions.
+align_var_def_gap               = 0        # unsigned number
+
+# Whether to align the colon in struct bit fields.
+align_var_def_colon             = false    # true/false
+
+# The gap for aligning the colon in struct bit fields.
+align_var_def_colon_gap         = 0        # unsigned number
+
+# Whether to align any attribute after the variable name.
+align_var_def_attribute         = false    # true/false
+
+# Whether to align inline struct/enum/union variable definitions.
+align_var_def_inline            = false    # true/false
+
+# The span for aligning on '=' in assignments.
+#
+# 0: Don't align (default).
+align_assign_span               = 0        # unsigned number
+
+# The span for aligning on '=' in function prototype modifier.
+#
+# 0: Don't align (default).
+align_assign_func_proto_span    = 0        # unsigned number
+
+# The threshold for aligning on '=' in assignments.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_assign_thresh             = 0        # number
+
+# How to apply align_assign_span to function declaration "assignments", i.e.
+# 'virtual void foo() = 0' or '~foo() = {default|delete}'.
+#
+# 0: Align with other assignments (default)
+# 1: Align with each other, ignoring regular assignments
+# 2: Don't align
+align_assign_decl_func          = 0        # unsigned number
+
+# The span for aligning on '=' in enums.
+#
+# 0: Don't align (default).
+align_enum_equ_span             = 0        # unsigned number
+
+# The threshold for aligning on '=' in enums.
+# Use a negative number for absolute thresholds.
+#
+# 0: no limit (default).
+align_enum_equ_thresh           = 0        # number
+
+# The span for aligning class member definitions.
+#
+# 0: Don't align (default).
+align_var_class_span            = 0        # unsigned number
+
+# The threshold for aligning class member definitions.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_var_class_thresh          = 0        # number
+
+# The gap for aligning class member definitions.
+align_var_class_gap             = 0        # unsigned number
+
+# The span for aligning struct/union member definitions.
+#
+# 0: Don't align (default).
+align_var_struct_span           = 0        # unsigned number
+
+# The threshold for aligning struct/union member definitions.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_var_struct_thresh         = 0        # number
+
+# The gap for aligning struct/union member definitions.
+align_var_struct_gap            = 0        # unsigned number
+
+# The span for aligning struct initializer values.
+#
+# 0: Don't align (default).
+align_struct_init_span          = 0        # unsigned number
+
+# The span for aligning single-line typedefs.
+#
+# 0: Don't align (default).
+align_typedef_span              = 0        # unsigned number
+
+# The minimum space between the type and the synonym of a typedef.
+align_typedef_gap               = 0        # unsigned number
+
+# How to align typedef'd functions with other typedefs.
+#
+# 0: Don't mix them at all (default)
+# 1: Align the open parenthesis with the types
+# 2: Align the function type name with the other type names
+align_typedef_func              = 0        # unsigned number
+
+# How to consider (or treat) the '*' in the alignment of typedefs.
+#
+# 0: Part of the typedef type, 'typedef int * pint;' (default)
+# 1: Part of type name:        'typedef int   *pint;'
+# 2: Dangling:                 'typedef int  *pint;'
+# Dangling: the '*' will not be taken into account when aligning.
+align_typedef_star_style        = 0        # unsigned number
+
+# How to consider (or treat) the '&' in the alignment of typedefs.
+#
+# 0: Part of the typedef type, 'typedef int & intref;' (default)
+# 1: Part of type name:        'typedef int   &intref;'
+# 2: Dangling:                 'typedef int  &intref;'
+# Dangling: the '&' will not be taken into account when aligning.
+align_typedef_amp_style         = 0        # unsigned number
+
+# The span for aligning comments that end lines.
+#
+# 0: Don't align (default).
+align_right_cmt_span            = 0        # unsigned number
+
+# Minimum number of columns between preceding text and a trailing comment in
+# order for the comment to qualify for being aligned. Must be non-zero to have
+# an effect.
+align_right_cmt_gap             = 0        # unsigned number
+
+# If aligning comments, whether to mix with comments after '}' and #endif with
+# less than three spaces before the comment.
+align_right_cmt_mix             = false    # true/false
+
+# Whether to only align trailing comments that are at the same brace level.
+align_right_cmt_same_level      = false    # true/false
+
+# Minimum column at which to align trailing comments. Comments which are
+# aligned beyond this column, but which can be aligned in a lesser column,
+# may be "pulled in".
+#
+# 0: Ignore (default).
+align_right_cmt_at_col          = 0        # unsigned number
+
+# The span for aligning function prototypes.
+#
+# 0: Don't align (default).
+align_func_proto_span           = 0        # unsigned number
+
+# The threshold for aligning function prototypes.
+# Use a negative number for absolute thresholds.
+#
+# 0: No limit (default).
+align_func_proto_thresh         = 0        # number
+
+# Minimum gap between the return type and the function name.
+align_func_proto_gap            = 0        # unsigned number
+
+# Whether to align function prototypes on the 'operator' keyword instead of
+# what follows.
+align_on_operator               = false    # true/false
+
+# Whether to mix aligning prototype and variable declarations. If true,
+# align_var_def_XXX options are used instead of align_func_proto_XXX options.
+align_mix_var_proto             = false    # true/false
+
+# Whether to align single-line functions with function prototypes.
+# Uses align_func_proto_span.
+align_single_line_func          = false    # true/false
+
+# Whether to align the open brace of single-line functions.
+# Requires align_single_line_func=true. Uses align_func_proto_span.
+align_single_line_brace         = false    # true/false
+
+# Gap for align_single_line_brace.
+align_single_line_brace_gap     = 0        # unsigned number
+
+# (OC) The span for aligning Objective-C message specifications.
+#
+# 0: Don't align (default).
+align_oc_msg_spec_span          = 0        # unsigned number
+
+# Whether to align macros wrapped with a backslash and a newline. This will
+# not work right if the macro contains a multi-line comment.
+align_nl_cont                   = false    # true/false
+
+# Whether to align macro functions and variables together.
+align_pp_define_together        = false    # true/false
+
+# The span for aligning on '#define' bodies.
+#
+# =0: Don't align (default)
+# >0: Number of lines (including comments) between blocks
+align_pp_define_span            = 0        # unsigned number
+
+# The minimum space between label and value of a preprocessor define.
+align_pp_define_gap             = 0        # unsigned number
+
+# Whether to align lines that start with '<<' with previous '<<'.
+#
+# Default: true
+align_left_shift                = true     # true/false
+
+# Whether to align text after 'asm volatile ()' colons.
+align_asm_colon                 = false    # true/false
+
+# (OC) Span for aligning parameters in an Objective-C message call
+# on the ':'.
+#
+# 0: Don't align.
+align_oc_msg_colon_span         = 0        # unsigned number
+
+# (OC) Whether to always align with the first parameter, even if it is too
+# short.
+align_oc_msg_colon_first        = false    # true/false
+
+# (OC) Whether to align parameters in an Objective-C '+' or '-' declaration
+# on the ':'.
+align_oc_decl_colon             = false    # true/false
+
+#
+# Comment modification options
+#
+
+# Try to wrap comments at N columns.
+cmt_width                       = 0        # unsigned number
+
+# How to reflow comments.
+#
+# 0: No reflowing (apart from the line wrapping due to cmt_width) (default)
+# 1: No touching at all
+# 2: Full reflow
+cmt_reflow_mode                 = 0        # unsigned number
+
+# Whether to convert all tabs to spaces in comments. If false, tabs in
+# comments are left alone, unless used for indenting.
+cmt_convert_tab_to_spaces       = false    # true/false
+
+# Whether to apply changes to multi-line comments, including cmt_width,
+# keyword substitution and leading chars.
+#
+# Default: true
+cmt_indent_multi                = true     # true/false
+
+# Whether to group c-comments that look like they are in a block.
+cmt_c_group                     = false    # true/false
+
+# Whether to put an empty '/*' on the first line of the combined c-comment.
+cmt_c_nl_start                  = false    # true/false
+
+# Whether to add a newline before the closing '*/' of the combined c-comment.
+cmt_c_nl_end                    = false    # true/false
+
+# Whether to change cpp-comments into c-comments.
+cmt_cpp_to_c                    = false    # true/false
+
+# Whether to group cpp-comments that look like they are in a block. Only
+# meaningful if cmt_cpp_to_c=true.
+cmt_cpp_group                   = false    # true/false
+
+# Whether to put an empty '/*' on the first line of the combined cpp-comment
+# when converting to a c-comment.
+#
+# Requires cmt_cpp_to_c=true and cmt_cpp_group=true.
+cmt_cpp_nl_start                = false    # true/false
+
+# Whether to add a newline before the closing '*/' of the combined cpp-comment
+# when converting to a c-comment.
+#
+# Requires cmt_cpp_to_c=true and cmt_cpp_group=true.
+cmt_cpp_nl_end                  = false    # true/false
+
+# Whether to put a star on subsequent comment lines.
+cmt_star_cont                   = false    # true/false
+
+# The number of spaces to insert at the start of subsequent comment lines.
+cmt_sp_before_star_cont         = 0        # unsigned number
+
+# The number of spaces to insert after the star on subsequent comment lines.
+cmt_sp_after_star_cont          = 0        # unsigned number
+
+# For multi-line comments with a '*' lead, remove leading spaces if the first
+# and last lines of the comment are the same length.
+#
+# Default: true
+cmt_multi_check_last            = true     # true/false
+
+# For multi-line comments with a '*' lead, remove leading spaces if the first
+# and last lines of the comment are the same length AND if the length is
+# bigger as the first_len minimum.
+#
+# Default: 4
+cmt_multi_first_len_minimum     = 4        # unsigned number
+
+# Path to a file that contains text to insert at the beginning of a file if
+# the file doesn't start with a C/C++ comment. If the inserted text contains
+# '$(filename)', that will be replaced with the current file's name.
+cmt_insert_file_header          = ""         # string
+
+# Path to a file that contains text to insert at the end of a file if the
+# file doesn't end with a C/C++ comment. If the inserted text contains
+# '$(filename)', that will be replaced with the current file's name.
+cmt_insert_file_footer          = ""         # string
+
+# Path to a file that contains text to insert before a function definition if
+# the function isn't preceded by a C/C++ comment. If the inserted text
+# contains '$(function)', '$(javaparam)' or '$(fclass)', these will be
+# replaced with, respectively, the name of the function, the javadoc '@param'
+# and '@return' stuff, or the name of the class to which the member function
+# belongs.
+cmt_insert_func_header          = ""         # string
+
+# Path to a file that contains text to insert before a class if the class
+# isn't preceded by a C/C++ comment. If the inserted text contains '$(class)',
+# that will be replaced with the class name.
+cmt_insert_class_header         = ""         # string
+
+# Path to a file that contains text to insert before an Objective-C message
+# specification, if the method isn't preceded by a C/C++ comment. If the
+# inserted text contains '$(message)' or '$(javaparam)', these will be
+# replaced with, respectively, the name of the function, or the javadoc
+# '@param' and '@return' stuff.
+cmt_insert_oc_msg_header        = ""         # string
+
+# Whether a comment should be inserted if a preprocessor is encountered when
+# stepping backwards from a function name.
+#
+# Applies to cmt_insert_oc_msg_header, cmt_insert_func_header and
+# cmt_insert_class_header.
+cmt_insert_before_preproc       = false    # true/false
+
+# Whether a comment should be inserted if a function is declared inline to a
+# class definition.
+#
+# Applies to cmt_insert_func_header.
+#
+# Default: true
+cmt_insert_before_inlines       = true     # true/false
+
+# Whether a comment should be inserted if the function is a class constructor
+# or destructor.
+#
+# Applies to cmt_insert_func_header.
+cmt_insert_before_ctor_dtor     = false    # true/false
+
+#
+# Code modifying options (non-whitespace)
+#
+
+# Add or remove braces on a single-line 'do' statement.
+mod_full_brace_do               = ignore   # ignore/add/remove/force
+
+# Add or remove braces on a single-line 'for' statement.
+mod_full_brace_for              = ignore   # ignore/add/remove/force
+
+# (Pawn) Add or remove braces on a single-line function definition.
+mod_full_brace_function         = ignore   # ignore/add/remove/force
+
+# Add or remove braces on a single-line 'if' statement. Braces will not be
+# removed if the braced statement contains an 'else'.
+mod_full_brace_if               = ignore   # ignore/add/remove/force
+
+# Whether to enforce that all blocks of an 'if'/'else if'/'else' chain either
+# have, or do not have, braces. If true, braces will be added if any block
+# needs braces, and will only be removed if they can be removed from all
+# blocks.
+#
+# Overrides mod_full_brace_if.
+mod_full_brace_if_chain         = false    # true/false
+
+# Whether to add braces to all blocks of an 'if'/'else if'/'else' chain.
+# If true, mod_full_brace_if_chain will only remove braces from an 'if' that
+# does not have an 'else if' or 'else'.
+mod_full_brace_if_chain_only    = false    # true/false
+
+# Add or remove braces on single-line 'while' statement.
+mod_full_brace_while            = ignore   # ignore/add/remove/force
+
+# Add or remove braces on single-line 'using ()' statement.
+mod_full_brace_using            = ignore   # ignore/add/remove/force
+
+# Don't remove braces around statements that span N newlines
+mod_full_brace_nl               = 0        # unsigned number
+
+# Whether to prevent removal of braces from 'if'/'for'/'while'/etc. blocks
+# which span multiple lines.
+#
+# Affects:
+#   mod_full_brace_for
+#   mod_full_brace_if
+#   mod_full_brace_if_chain
+#   mod_full_brace_if_chain_only
+#   mod_full_brace_while
+#   mod_full_brace_using
+#
+# Does not affect:
+#   mod_full_brace_do
+#   mod_full_brace_function
+mod_full_brace_nl_block_rem_mlcond = false    # true/false
+
+# Add or remove unnecessary parenthesis on 'return' statement.
+mod_paren_on_return             = ignore   # ignore/add/remove/force
+
+# (Pawn) Whether to change optional semicolons to real semicolons.
+mod_pawn_semicolon              = false    # true/false
+
+# Whether to fully parenthesize Boolean expressions in 'while' and 'if'
+# statement, as in 'if (a && b > c)' => 'if (a && (b > c))'.
+mod_full_paren_if_bool          = false    # true/false
+
+# Whether to remove superfluous semicolons.
+mod_remove_extra_semicolon      = false    # true/false
+
+# If a function body exceeds the specified number of newlines and doesn't have
+# a comment after the close brace, a comment will be added.
+mod_add_long_function_closebrace_comment = 0        # unsigned number
+
+# If a namespace body exceeds the specified number of newlines and doesn't
+# have a comment after the close brace, a comment will be added.
+mod_add_long_namespace_closebrace_comment = 0        # unsigned number
+
+# If a class body exceeds the specified number of newlines and doesn't have a
+# comment after the close brace, a comment will be added.
+mod_add_long_class_closebrace_comment = 0        # unsigned number
+
+# If a switch body exceeds the specified number of newlines and doesn't have a
+# comment after the close brace, a comment will be added.
+mod_add_long_switch_closebrace_comment = 0        # unsigned number
+
+# If an #ifdef body exceeds the specified number of newlines and doesn't have
+# a comment after the #endif, a comment will be added.
+mod_add_long_ifdef_endif_comment = 0        # unsigned number
+
+# If an #ifdef or #else body exceeds the specified number of newlines and
+# doesn't have a comment after the #else, a comment will be added.
+mod_add_long_ifdef_else_comment = 0        # unsigned number
+
+# Whether to take care of the case by the mod_sort_xx options.
+mod_sort_case_sensitive         = false    # true/false
+
+# Whether to sort consecutive single-line 'import' statements.
+mod_sort_import                 = false    # true/false
+
+# (C#) Whether to sort consecutive single-line 'using' statements.
+mod_sort_using                  = false    # true/false
+
+# Whether to sort consecutive single-line '#include' statements (C/C++) and
+# '#import' statements (Objective-C). Be aware that this has the potential to
+# break your code if your includes/imports have ordering dependencies.
+mod_sort_include                = false    # true/false
+
+# Whether to move a 'break' that appears after a fully braced 'case' before
+# the close brace, as in 'case X: { ... } break;' => 'case X: { ... break; }'.
+mod_move_case_break             = false    # true/false
+
+# Add or remove braces around a fully braced case statement. Will only remove
+# braces if there are no variable declarations in the block.
+mod_case_brace                  = ignore   # ignore/add/remove/force
+
+# Whether to remove a void 'return;' that appears as the last statement in a
+# function.
+mod_remove_empty_return         = false    # true/false
+
+# Add or remove the comma after the last value of an enumeration.
+mod_enum_last_comma             = ignore   # ignore/add/remove/force
+
+# (OC) Whether to organize the properties. If true, properties will be
+# rearranged according to the mod_sort_oc_property_*_weight factors.
+mod_sort_oc_properties          = false    # true/false
+
+# (OC) Weight of a class property modifier.
+mod_sort_oc_property_class_weight = 0        # number
+
+# (OC) Weight of 'atomic' and 'nonatomic'.
+mod_sort_oc_property_thread_safe_weight = 0        # number
+
+# (OC) Weight of 'readwrite' when organizing properties.
+mod_sort_oc_property_readwrite_weight = 0        # number
+
+# (OC) Weight of a reference type specifier ('retain', 'copy', 'assign',
+# 'weak', 'strong') when organizing properties.
+mod_sort_oc_property_reference_weight = 0        # number
+
+# (OC) Weight of getter type ('getter=') when organizing properties.
+mod_sort_oc_property_getter_weight = 0        # number
+
+# (OC) Weight of setter type ('setter=') when organizing properties.
+mod_sort_oc_property_setter_weight = 0        # number
+
+# (OC) Weight of nullability type ('nullable', 'nonnull', 'null_unspecified',
+# 'null_resettable') when organizing properties.
+mod_sort_oc_property_nullability_weight = 0        # number
+
+#
+# Preprocessor options
+#
+
+# Add or remove indentation of preprocessor directives inside #if blocks
+# at brace level 0 (file-level).
+pp_indent                       = ignore   # ignore/add/remove/force
+
+# Whether to indent #if/#else/#endif at the brace level. If false, these are
+# indented from column 1.
+pp_indent_at_level              = false    # true/false
+
+# Specifies the number of columns to indent preprocessors per level
+# at brace level 0 (file-level). If pp_indent_at_level=false, also specifies
+# the number of columns to indent preprocessors per level
+# at brace level > 0 (function-level).
+#
+# Default: 1
+pp_indent_count                 = 1        # unsigned number
+
+# Add or remove space after # based on pp_level of #if blocks.
+pp_space                        = ignore   # ignore/add/remove/force
+
+# Sets the number of spaces per level added with pp_space.
+pp_space_count                  = 0        # unsigned number
+
+# The indent for '#region' and '#endregion' in C# and '#pragma region' in
+# C/C++. Negative values decrease indent down to the first column.
+pp_indent_region                = 0        # number
+
+# Whether to indent the code between #region and #endregion.
+pp_region_indent_code           = false    # true/false
+
+# If pp_indent_at_level=true, sets the indent for #if, #else and #endif when
+# not at file-level. Negative values decrease indent down to the first column.
+#
+# =0: Indent preprocessors using output_tab_size
+# >0: Column at which all preprocessors will be indented
+pp_indent_if                    = 0        # number
+
+# Whether to indent the code between #if, #else and #endif.
+pp_if_indent_code               = false    # true/false
+
+# Whether to indent '#define' at the brace level. If false, these are
+# indented from column 1.
+pp_define_at_level              = false    # true/false
+
+# Whether to ignore the '#define' body while formatting.
+pp_ignore_define_body           = false    # true/false
+
+# Whether to indent case statements between #if, #else, and #endif.
+# Only applies to the indent of the preprocesser that the case statements
+# directly inside of.
+#
+# Default: true
+pp_indent_case                  = true     # true/false
+
+# Whether to indent whole function definitions between #if, #else, and #endif.
+# Only applies to the indent of the preprocesser that the function definition
+# is directly inside of.
+#
+# Default: true
+pp_indent_func_def              = true     # true/false
+
+# Whether to indent extern C blocks between #if, #else, and #endif.
+# Only applies to the indent of the preprocesser that the extern block is
+# directly inside of.
+#
+# Default: true
+pp_indent_extern                = true     # true/false
+
+# Whether to indent braces directly inside #if, #else, and #endif.
+# Only applies to the indent of the preprocesser that the braces are directly
+# inside of.
+#
+# Default: true
+pp_indent_brace                 = true     # true/false
+
+#
+# Sort includes options
+#
+
+# The regex for include category with priority 0.
+include_category_0              = ""         # string
+
+# The regex for include category with priority 1.
+include_category_1              = ""         # string
+
+# The regex for include category with priority 2.
+include_category_2              = ""         # string
+
+#
+# Use or Do not Use options
+#
+
+# true:  indent_func_call_param will be used (default)
+# false: indent_func_call_param will NOT be used
+#
+# Default: true
+use_indent_func_call_param      = true     # true/false
+
+# The value of the indentation for a continuation line is calculated
+# differently if the statement is:
+# - a declaration: your case with QString fileName ...
+# - an assignment: your case with pSettings = new QSettings( ...
+#
+# At the second case the indentation value might be used twice:
+# - at the assignment
+# - at the function call (if present)
+#
+# To prevent the double use of the indentation value, use this option with the
+# value 'true'.
+#
+# true:  indent_continue will be used only once
+# false: indent_continue will be used every time (default)
+use_indent_continue_only_once   = false    # true/false
+
+# The value might be used twice:
+# - at the assignment
+# - at the opening brace
+#
+# To prevent the double use of the indentation value, use this option with the
+# value 'true'.
+#
+# true:  indentation will be used only once
+# false: indentation will be used every time (default)
+indent_cpp_lambda_only_once     = false    # true/false
+
+# Whether sp_after_angle takes precedence over sp_inside_fparen. This was the
+# historic behavior, but is probably not the desired behavior, so this is off
+# by default.
+use_sp_after_angle_always       = false    # true/false
+
+# Whether to apply special formatting for Qt SIGNAL/SLOT macros. Essentially,
+# this tries to format these so that they match Qt's normalized form (i.e. the
+# result of QMetaObject::normalizedSignature), which can slightly improve the
+# performance of the QObject::connect call, rather than how they would
+# otherwise be formatted.
+#
+# See options_for_QT.cpp for details.
+#
+# Default: true
+use_options_overriding_for_qt_macros = true     # true/false
+
+#
+# Warn levels - 1: error, 2: warning (default), 3: note
+#
+
+# (C#) Warning is given if doing tab-to-\t replacement and we have found one
+# in a C# verbatim string literal.
+#
+# Default: 2
+warn_level_tabs_found_in_verbatim_string_literals = 2        # unsigned number
+
+# Meaning of the settings:
+#   Ignore - do not do any changes
+#   Add    - makes sure there is 1 or more space/brace/newline/etc
+#   Force  - makes sure there is exactly 1 space/brace/newline/etc,
+#            behaves like Add in some contexts
+#   Remove - removes space/brace/newline/etc
+#
+#
+# - Token(s) can be treated as specific type(s) with the 'set' option:
+#     `set tokenType tokenString [tokenString...]`
+#
+#     Example:
+#       `set BOOL __AND__ __OR__`
+#
+#     tokenTypes are defined in src/token_enum.h, use them without the
+#     'CT_' prefix: 'CT_BOOL' => 'BOOL'
+#
+#
+# - Token(s) can be treated as type(s) with the 'type' option.
+#     `type tokenString [tokenString...]`
+#
+#     Example:
+#       `type int c_uint_8 Rectangle`
+#
+#     This can also be achieved with `set TYPE int c_uint_8 Rectangle`
+#
+#
+# To embed whitespace in tokenStrings use the '\' escape character, or quote
+# the tokenStrings. These quotes are supported: "'`
+#
+#
+# - Support for the auto detection of languages through the file ending can be
+#   added using the 'file_ext' command.
+#     `file_ext langType langString [langString..]`
+#
+#     Example:
+#       `file_ext CPP .ch .cxx .cpp.in`
+#
+#     langTypes are defined in uncrusify_types.h in the lang_flag_e enum, use
+#     them without the 'LANG_' prefix: 'LANG_CPP' => 'CPP'
+#
+#
+# - Custom macro-based indentation can be set up using 'macro-open',
+#   'macro-else' and 'macro-close'.
+#     `(macro-open | macro-else | macro-close) tokenString`
+#
+#     Example:
+#       `macro-open  BEGIN_TEMPLATE_MESSAGE_MAP`
+#       `macro-open  BEGIN_MESSAGE_MAP`
+#       `macro-close END_MESSAGE_MAP`
+#
+#
+# option(s) with 'not default' value: 0
+#
+
+file_ext CPP .ch .cxx .cpp.in .cc .hh
+
diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 98018ca0e0c..437a761c4d1 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -29,38 +29,38 @@ int main()
         ErrorInfo { .level = lvlInfo,
                     .name = "Info name",
                     .description = "Info description",
-                  });
+        });
 
     logger->logEI(
         ErrorInfo { .level = lvlTalkative,
                     .name = "Talkative name",
                     .description = "Talkative description",
-                  });
+        });
 
     logger->logEI(
         ErrorInfo { .level = lvlChatty,
                     .name = "Chatty name",
                     .description = "Chatty description",
-                  });
+        });
 
     logger->logEI(
         ErrorInfo { .level = lvlDebug,
                     .name = "Debug name",
                     .description = "Debug description",
-                  });
+        });
 
     logger->logEI(
         ErrorInfo { .level = lvlVomit,
                     .name = "Vomit name",
                     .description = "Vomit description",
-                  });
+        });
 
 
     // Error in a program; no hint and no nix code.
     logError(
         ErrorInfo { .name = "name",
                     .description = "error description",
-                  });
+        });
 
     // Warning with name, description, and hint.
     // The hintfmt function makes all the substituted text yellow.
@@ -68,7 +68,7 @@ int main()
         ErrorInfo { .name = "name",
                     .description = "error description",
                     .hint = hintfmt("there was a %1%", "warning"),
-                  });
+        });
 
 
     // Warning with nix file, line number, column, and the lines of
@@ -80,28 +80,28 @@ int main()
         ErrorInfo { .name = "warning name",
                     .description = "warning description",
                     .hint = hintfmt("this hint has %1% templated %2%!!",
-                                    "yellow",
-                                    "values"),
+                        "yellow",
+                        "values"),
                     .nixCode = NixCode {
                         .errPos = Pos(problem_file, 40, 13),
                         .prevLineOfCode = std::nullopt,
                         .errLineOfCode = "this is the problem line of code",
                         .nextLineOfCode = std::nullopt
-                  }});
+                    }});
 
     // Error with previous and next lines of code.
     logError(
         ErrorInfo { .name = "error name",
                     .description = "error description",
                     .hint = hintfmt("this hint has %1% templated %2%!!",
-                                    "yellow",
-                                    "values"),
+                        "yellow",
+                        "values"),
                     .nixCode = NixCode {
                         .errPos = Pos(problem_file, 40, 13),
                         .prevLineOfCode = std::optional("previous line of code"),
                         .errLineOfCode = "this is the problem line of code",
                         .nextLineOfCode = std::optional("next line of code"),
-                  }});
+                    }});
 
 
     return 0;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 79e3f90e50d..a8bae0a619e 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -234,7 +234,8 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); };
+            throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos);
+    };
     void setName(Symbol & name);
     string showNamePos() const;
     COMMON_METHODS
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 8f0e0aa1ac6..91fa1ccd822 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -45,17 +45,17 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
     // previous line of code.
     if (nixCode.prevLineOfCode.has_value()) {
         std::cout << fmt("%1% %|2$5d|| %3%",
-                         prefix,
-                         (nixCode.errPos.line - 1),
-                         *nixCode.prevLineOfCode)
+            prefix,
+            (nixCode.errPos.line - 1),
+            *nixCode.prevLineOfCode)
                   << std::endl;
     }
 
     // line of code containing the error.%2$+5d%
     std::cout << fmt("%1% %|2$5d|| %3%",
-                     prefix,
-                     (nixCode.errPos.line),
-                     nixCode.errLineOfCode)
+        prefix,
+        (nixCode.errPos.line),
+        nixCode.errLineOfCode)
               << std::endl;
 
     // error arrows for the column range.
@@ -69,17 +69,17 @@ void printCodeLines(const string &prefix, const NixCode &nixCode)
         std::string arrows("^");
 
         std::cout << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-                         prefix,
-                         spaces,
-                         arrows) << std::endl;
+            prefix,
+            spaces,
+            arrows) << std::endl;
     }
 
     // next line of code.
     if (nixCode.nextLineOfCode.has_value()) {
         std::cout << fmt("%1% %|2$5d|| %3%",
-                         prefix,
-                         (nixCode.errPos.line + 1),
-                         *nixCode.nextLineOfCode)
+            prefix,
+            (nixCode.errPos.line + 1),
+            *nixCode.nextLineOfCode)
                   << std::endl;
     }
 }
@@ -149,18 +149,18 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     // divider.
     if (einfo.name != "")
         out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
-                   prefix,
-                   levelString,
-                   einfo.name,
-                   dashes,
-                   einfo.programName.value_or(""))
+            prefix,
+            levelString,
+            einfo.name,
+            dashes,
+            einfo.programName.value_or(""))
             << std::endl;
     else
         out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
-                   prefix,
-                   levelString,
-                   dashes,
-                   einfo.programName.value_or(""))
+            prefix,
+            levelString,
+            dashes,
+            einfo.programName.value_or(""))
             << std::endl;
 
     // filename.
@@ -171,9 +171,9 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
                            : "";
 
             out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
-                       prefix,
-                       einfo.nixCode->errPos.file,
-                       eline) << std::endl;
+                prefix,
+                einfo.nixCode->errPos.file,
+                eline) << std::endl;
             out << prefix << std::endl;
         } else {
             out << fmt("%1%from command line argument", prefix) << std::endl;
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 4a8b9864020..e1750eb2a5f 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -60,7 +60,7 @@ class SimpleLogger : public Logger
 
     void logEI(const ErrorInfo & ei) override
     {
-        std::stringstream oss; 
+        std::stringstream oss;
         oss << ei;
 
         log(ei.level, oss.str());
@@ -68,7 +68,7 @@ class SimpleLogger : public Logger
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent)
-        override
+    override
     {
         if (lvl <= verbosity && !s.empty())
             log(lvl, s + "...");
@@ -111,8 +111,7 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
-struct JSONLogger : Logger
-{
+struct JSONLogger : Logger {
     Logger & prevLogger;
 
     JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
@@ -155,28 +154,26 @@ struct JSONLogger : Logger
         json["level"] = ei.level;
         json["msg"] = oss.str();
 
-        // Extra things that COULD go into json.  Useful?  
+        // Extra things that COULD go into json.  Useful?
         // TODO: decide if useful.
         // TODO: make a json obj that goes into json["msg"]?
         json["name"] = ei.name;
         json["description"] = ei.description;
-        if (ei.hint.has_value())
-        {
-          json["hint"] = ei.hint->str();
+        if (ei.hint.has_value()) {
+            json["hint"] = ei.hint->str();
         }
-        if (ei.nixCode.has_value())
-        {
-          if (ei.nixCode->errPos.line != 0)
-            json["line"] = ei.nixCode->errPos.line;
-          if (ei.nixCode->errPos.column != 0)
-            json["column"] = ei.nixCode->errPos.column;
-          if (ei.nixCode->errPos.file != "")
-            json["file"] = ei.nixCode->errPos.file;
-          if (ei.nixCode->prevLineOfCode.has_value())
-            json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode;
-          json["errLineOfCode"] = ei.nixCode->errLineOfCode;
-          if (ei.nixCode->nextLineOfCode.has_value())
-            json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode;
+        if (ei.nixCode.has_value()) {
+            if (ei.nixCode->errPos.line != 0)
+                json["line"] = ei.nixCode->errPos.line;
+            if (ei.nixCode->errPos.column != 0)
+                json["column"] = ei.nixCode->errPos.column;
+            if (ei.nixCode->errPos.file != "")
+                json["file"] = ei.nixCode->errPos.file;
+            if (ei.nixCode->prevLineOfCode.has_value())
+                json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode;
+            json["errLineOfCode"] = ei.nixCode->errLineOfCode;
+            if (ei.nixCode->nextLineOfCode.has_value())
+                json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode;
         }
 
         write(json);
@@ -278,7 +275,8 @@ bool handleJSONLogMessage(const std::string & msg,
     return true;
 }
 
-Activity::~Activity() {
+Activity::~Activity()
+{
     try {
         logger.stopActivity(id);
     } catch (...) {
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 89fd9841977..0c4980b835a 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -62,12 +62,12 @@ public:
 
     virtual void logEI(const ErrorInfo &ei) = 0;
 
-    void logEI(Verbosity lvl, ErrorInfo ei) 
+    void logEI(Verbosity lvl, ErrorInfo ei)
     {
         ei.level = lvl;
         logEI(ei);
     }
-    
+
     virtual void warn(const std::string & msg);
 
     virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

From 7afcb5af988eb0ce73c9916b809f8528dfb14c0f Mon Sep 17 00:00:00 2001
From: Guillaume Bouchard <guillaume.bouchard@tweag.io>
Date: Wed, 29 Apr 2020 18:42:19 +0200
Subject: [PATCH 2017/2196] Remove the `drain` argument from `readFile`

Now it is always `drain` (see previous commit).
---
 src/libstore/gc.cc       | 4 ++--
 src/libutil/serialise.hh | 3 +++
 src/libutil/util.cc      | 4 ++--
 src/libutil/util.hh      | 2 +-
 src/nix/ls.cc            | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 6bab1e37c90..3cd35c9452e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -419,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
                 try {
                     auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
                     for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
@@ -427,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                     }
 
                     auto envFile = fmt("/proc/%s/environ", ent->d_name);
-                    auto envString = readFile(envFile, true);
+                    auto envString = readFile(envFile);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         unchecked[i->str()].emplace(envFile);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5780c93a630..a04118512fc 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -148,6 +148,9 @@ struct StringSink : Sink
 {
     ref<std::string> s;
     StringSink() : s(make_ref<std::string>()) { };
+    explicit StringSink(const size_t reservedSize) : s(make_ref<std::string>()) {
+      s->reserve(reservedSize);
+    };
     StringSink(ref<std::string> s) : s(s) { };
     void operator () (const unsigned char * data, size_t len) override;
 };
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index d001bc3b7cb..bb4af747bd5 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -316,12 +316,12 @@ string readFile(int fd)
 }
 
 
-string readFile(const Path & path, bool drain)
+string readFile(const Path & path)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd)
         throw SysError(format("opening file '%1%'") % path);
-    return drain ? drainFD(fd.get()) : readFile(fd.get());
+    return readFile(fd.get());
 }
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index d7ee62bcca4..32ef9a79a93 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -103,7 +103,7 @@ unsigned char getFileType(const Path & path);
 
 /* Read the contents of a file into a string. */
 string readFile(int fd);
-string readFile(const Path & path, bool drain = false);
+string readFile(const Path & path);
 void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 8590199d73b..ee308a6ecbf 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -136,7 +136,7 @@ struct CmdLsNar : Command, MixLs
 
     void run() override
     {
-        list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true))));
+        list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
     }
 };
 

From 2e5be2a7495ac0b204454c74664e590a38d039d3 Mon Sep 17 00:00:00 2001
From: Guillaume Bouchard <guillaume.bouchard@tweag.io>
Date: Wed, 29 Apr 2020 18:44:01 +0200
Subject: [PATCH 2018/2196] StringSink pre allocate

When used with `readFile`, we have a pretty good heuristic of the file
size, so `reserve` this in the `string`. This will save some allocation
/ copy when the string is growing.
---
 src/libutil/util.cc | 10 +++++++---
 src/libutil/util.hh |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index bb4af747bd5..71db92d772a 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -312,7 +312,11 @@ unsigned char getFileType(const Path & path)
 
 string readFile(int fd)
 {
-    return drainFD(fd, true);
+    struct stat st;
+    if (fstat(fd, &st) == -1)
+        throw SysError("statting file");
+
+    return drainFD(fd, true, st.st_size);
 }
 
 
@@ -658,9 +662,9 @@ void writeFull(int fd, const string & s, bool allowInterrupts)
 }
 
 
-string drainFD(int fd, bool block)
+string drainFD(int fd, bool block, const size_t reserveSize)
 {
-    StringSink sink;
+    StringSink sink(reserveSize);
     drainFD(fd, sink, block);
     return std::move(*sink.s);
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 32ef9a79a93..1b263abccbb 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -162,7 +162,7 @@ MakeError(EndOfFile, Error);
 
 
 /* Read a file descriptor until EOF occurs. */
-string drainFD(int fd, bool block = true);
+string drainFD(int fd, bool block = true, const size_t reserveSize=0);
 
 void drainFD(int fd, Sink & sink, bool block = true);
 

From 2d0f766a778ac790be1ab1918db7c319ac7e5792 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 29 Apr 2020 11:52:35 -0600
Subject: [PATCH 2019/2196] more style tweaks

---
 .uncrustify.cfg | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.uncrustify.cfg b/.uncrustify.cfg
index 23305adcfff..b28935dcc10 100644
--- a/.uncrustify.cfg
+++ b/.uncrustify.cfg
@@ -97,7 +97,7 @@ sp_cpp_lambda_square_paren      = ignore   # ignore/add/remove/force
 
 # Add or remove space after the capture specification of a C++11 lambda with
 # no argument list is present, as in '[] <here> { ... }'.
-sp_cpp_lambda_square_brace      = ignore   # ignore/add/remove/force
+sp_cpp_lambda_square_brace      = remove  # ignore/add/remove/force
 
 # Add or remove space after the argument list of a C++11 lambda, as in
 # '[](int x) <here> { ... }'.
@@ -279,11 +279,11 @@ sp_angle_word                   = ignore   # ignore/add/remove/force
 # Add or remove space between '>' and '>' in '>>' (template stuff).
 #
 # Default: add
-sp_angle_shift                  = add      # ignore/add/remove/force
+sp_angle_shift                  = ignore      # ignore/add/remove/force
 
 # (C++11) Permit removal of the space between '>>' in 'foo<bar<int> >'. Note
 # that sp_angle_shift cannot remove the space without this option.
-sp_permit_cpp11_shift           = false    # true/false
+sp_permit_cpp11_shift           = true    # true/false
 
 # Add or remove space before '(' of control statements ('if', 'for', 'switch',
 # 'while', etc.).
@@ -1157,7 +1157,7 @@ indent_label                    = 1        # number
 # <=0: Subtract from brace indent
 #
 # Default: 1
-indent_access_spec              = 1        # number
+indent_access_spec              = -4        # number
 
 # Whether to indent the code after an access specifier by one level.
 # If true, this option forces 'indent_access_spec=0'.
@@ -1333,7 +1333,7 @@ nl_cs_property_leave_one_liners = true    # true/false
 nl_func_leave_one_liners        = true    # true/false
 
 # Don't split one-line C++11 lambdas, as in '[]() { return 0; }'.
-nl_cpp_lambda_leave_one_liners  = false    # true/false
+nl_cpp_lambda_leave_one_liners  = true    # true/false
 
 # Don't split one-line if/else statements, as in 'if(...) b++;'.
 nl_if_leave_one_liners          = true    # true/false

From d1229859c2612587bfd04111bae6584c0c9df051 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Wed, 29 Apr 2020 22:48:29 +0200
Subject: [PATCH 2020/2196] Fix displaying error-position in
 `builtins.fetch{Tree,Tarball}`

Without dereferencing this pointer, you'd get an error like this:

```
error: unsupported argument 'abc' to 'fetchTarball', at 0x13627e8
```
---
 src/libexpr/primops/fetchTree.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 43c58485ae4..c5a0d988683 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -108,7 +108,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to '%s', at %s",
-                    attr.name, who, attr.pos);
+                    attr.name, who, *attr.pos);
         }
 
         if (!url)

From 39ff80d031c5c9c1831ba5ea597c0de0181bfe34 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 29 Apr 2020 18:57:05 -0600
Subject: [PATCH 2021/2196] errorinfo constructor test

---
 src/error-demo/error-demo.cc       | 10 ++++++++++
 src/libstore/binary-cache-store.cc |  2 +-
 src/libstore/remote-store.cc       |  2 +-
 src/libutil/error.hh               |  7 ++++---
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 437a761c4d1..2c4f2da6ef2 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -23,6 +23,16 @@ int main()
         logger->logEI(e.info());
     }
 
+
+    // ErrorInfo constructor
+    try {
+        auto e = Error("generic error");
+        throw DemoError(e.info());
+    } catch (Error &e) {
+        logger->logEI(e.info());
+    }
+
+
     // For completeness sake, info through vomit levels.
     // But this is maybe a heavy format for those.
     logger->logEI(
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4b7385c6bd3..97e34a75d70 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -284,7 +284,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
     try {
         getFile(info->url, *decompressor);
     } catch (NoSuchBinaryCacheFile & e) {
-        throw SubstituteGone(e.what());
+        throw SubstituteGone(e.info());
     }
 
     decompressor->finish();
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 0f8126aee23..cc336e460e3 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -365,7 +365,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
             } catch (Error & e) {
                 // Ugly backwards compatibility hack.
                 if (e.msg().find("is not valid") != std::string::npos)
-                    throw InvalidPath(e.what());
+                    throw InvalidPath(e.info());
                 throw;
             }
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 03e43241ffc..48e6311bd60 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -83,6 +83,7 @@ class BaseError : public std::exception
 protected:
     string prefix_; // used for location traces etc.
     ErrorInfo err;
+
     std::optional<string> what_;
     const string& calcWhat()
     {
@@ -107,18 +108,18 @@ public:
                 .hint = hintfmt(args...)
               }
         , status(status)
-    {  }
+    { }
 
     template<typename... Args>
     BaseError(const Args & ... args)
         : err { .level = lvlError,
                 .hint = hintfmt(args...)
               }
-    {  }
+    { }
 
     BaseError(ErrorInfo e)
         : err(e)
-    {  }
+    { }
 
     virtual const char* sname() const { return "BaseError"; }
 

From 2fcfc6c2c6ff144131d669ebfb2caa63c8b41417 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Apr 2020 13:05:29 +0200
Subject: [PATCH 2022/2196] nix dev-shell: Refactor script for getting the
 environment

---
 src/nix/get-env.sh |  8 ++++++++
 src/nix/local.mk   |  2 ++
 src/nix/shell.cc   | 18 ++++++++----------
 3 files changed, 18 insertions(+), 10 deletions(-)
 create mode 100644 src/nix/get-env.sh

diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
new file mode 100644
index 00000000000..782f97b7559
--- /dev/null
+++ b/src/nix/get-env.sh
@@ -0,0 +1,8 @@
+set -e
+export IN_NIX_SHELL=impure
+export dontAddDisableDepTrack=1
+if [[ -n $stdenv ]]; then
+    source $stdenv/setup
+fi
+export > $out
+set >> $out
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 033675e893c..15cef55bbef 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -27,3 +27,5 @@ $(foreach name, \
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
+
+src/nix/shell.cc: src/nix/get-env.sh.gen.hh
diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index fbfe24bbe30..c8bf35a13ac 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -83,6 +83,10 @@ BuildEnvironment readEnvironment(const Path & path)
     return res;
 }
 
+const static std::string getEnvSh =
+    #include "get-env.sh.gen.hh"
+    ;
+
 /* Given an existing derivation, return the shell environment as
    initialised by stdenv's setup script. We do this by building a
    modified derivation with the same dependencies and nearly the same
@@ -94,16 +98,9 @@ StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
     if (builder != "bash")
         throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");
 
-    drv.args = {
-        "-c",
-        "set -e; "
-        "export IN_NIX_SHELL=impure; "
-        "export dontAddDisableDepTrack=1; "
-        "if [[ -n $stdenv ]]; then "
-        "  source $stdenv/setup; "
-        "fi; "
-        "export > $out; "
-        "set >> $out "};
+    auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
+
+    drv.args = {store->printStorePath(getEnvShPath)};
 
     /* Remove derivation checks. */
     drv.env.erase("allowedReferences");
@@ -120,6 +117,7 @@ StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
         drv.env.erase(output.first);
     drv.env["out"] = "";
     drv.env["outputs"] = "out";
+    drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = hashDerivationModulo(*store, drv, true);
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
     drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", ""));

From efe6c186eabeb85830be7bef9aea4dd7eb2357e7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Apr 2020 14:39:26 +0200
Subject: [PATCH 2023/2196] nix dev-shell: Support structured attrs

Tested against https://github.com/NixOS/nixpkgs/pull/72074.

Fixes #3540.
---
 src/nix/get-env.sh |  1 +
 src/nix/shell.cc   | 45 ++++++++++++++++++++++++++++++++-------------
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 782f97b7559..a25ec43a9dc 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -1,4 +1,5 @@
 set -e
+if [ -e .attrs.sh ]; then source .attrs.sh; fi
 export IN_NIX_SHELL=impure
 export dontAddDisableDepTrack=1
 if [[ -n $stdenv ]]; then
diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index c8bf35a13ac..f0ffceaa7f9 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -13,7 +13,8 @@ using namespace nix;
 
 struct Var
 {
-    bool exported;
+    bool exported = true;
+    bool associative = false;
     std::string value; // quoted string or array
 };
 
@@ -48,11 +49,17 @@ BuildEnvironment readEnvironment(const Path & path)
     static std::string quotedStringRegex =
         R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
 
-    static std::string arrayRegex =
-        R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re";
+    static std::string indexedArrayRegex =
+        R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
 
     static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n");
+        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
+
+    /* Note: we distinguish between an indexed and associative array
+       using the space before the closing parenthesis. Will
+       undoubtedly regret this some day. */
+    static std::regex assocArrayRegex(
+        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
 
     static std::regex functionRegex(
         "^" + varNameRegex + " \\(\\) *\n");
@@ -68,7 +75,12 @@ BuildEnvironment readEnvironment(const Path & path)
 
         else if (std::regex_search(pos, file.cend(), match, varRegex)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }});
+            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
+            pos = match[0].second;
+            res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
@@ -92,8 +104,10 @@ const static std::string getEnvSh =
    modified derivation with the same dependencies and nearly the same
    initial environment variables, that just writes the resulting
    environment to a file and exits. */
-StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
+StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 {
+    auto drv = store->derivationFromPath(drvPath);
+
     auto builder = baseNameOf(drv.builder);
     if (builder != "bash")
         throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");
@@ -108,11 +122,12 @@ StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
     drv.env.erase("disallowedReferences");
     drv.env.erase("disallowedRequisites");
 
-    // FIXME: handle structured attrs
-
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
-    auto drvName = drv.env["name"] + "-env";
+    auto drvName = std::string(drvPath.name());
+    assert(hasSuffix(drvName, ".drv"));
+    drvName.resize(drvName.size() - 4);
+    drvName += "-env";
     for (auto & output : drv.outputs)
         drv.env.erase(output.first);
     drv.env["out"] = "";
@@ -161,9 +176,13 @@ struct Common : InstallableCommand, MixProfile
 
         for (auto & i : buildEnvironment.env) {
             if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
-                out << fmt("%s=%s\n", i.first, i.second.value);
-                if (i.second.exported)
-                    out << fmt("export %s\n", i.first);
+                if (i.second.associative)
+                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
+                else {
+                    out << fmt("%s=%s\n", i.first, i.second.value);
+                    if (i.second.exported)
+                        out << fmt("export %s\n", i.first);
+                }
             }
         }
 
@@ -194,7 +213,7 @@ struct Common : InstallableCommand, MixProfile
 
             auto & drvPath = *drvs.begin();
 
-            return getDerivationEnvironment(store, store->derivationFromPath(drvPath));
+            return getDerivationEnvironment(store, drvPath);
         }
     }
 

From 0135fd6ec4dcd15771008b867f28047e5c2bb605 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Apr 2020 14:46:51 +0200
Subject: [PATCH 2024/2196] nix dev-shell: Unset shellHook

This avoids inheriting the caller's shellHook, which can happen when
running a dev-shell inside a dev-shell.
---
 src/nix/shell.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index f0ffceaa7f9..9c45a935d16 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -172,6 +172,8 @@ struct Common : InstallableCommand, MixProfile
 
     void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
     {
+        out << "unset shellHook\n";
+
         out << "nix_saved_PATH=\"$PATH\"\n";
 
         for (auto & i : buildEnvironment.env) {

From a15f918cbac51dee2e82b4cb0535e7ae5f4ff587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Thu, 30 Apr 2020 14:57:40 +0200
Subject: [PATCH 2025/2196] Update issue templates

---
 .github/ISSUE_TEMPLATE/bug_report.md      | 26 +++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 00000000000..5c9e1d9ccb8
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,26 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Steps To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**`nix-env --version` output: **
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000000..392ed30c66c
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: improvement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

From 625868b33dab37c0750081843be8091e22808376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Thu, 30 Apr 2020 15:01:23 +0200
Subject: [PATCH 2026/2196] Update issue templates

---
 .github/ISSUE_TEMPLATE.md            | 27 ---------------------------
 .github/ISSUE_TEMPLATE/bug_report.md | 10 ++++++++--
 2 files changed, 8 insertions(+), 29 deletions(-)
 delete mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
deleted file mode 100644
index 3372b1f03f7..00000000000
--- a/.github/ISSUE_TEMPLATE.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<!--
-
-# Filing a Nix issue
-
-*WAIT* Are you sure you're filing your issue in the right repository?
-
-We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
-
-This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
-
-If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
-
-Examples of _Nix_ issues:
-
-- Nix segfaults when I run `nix-build -A blahblah`
-- The Nix language needs a new builtin: `builtins.foobar`
-- Regression in the behavior of `nix-env` in Nix 2.0
-
-Examples of _nixpkgs_ issues:
-
-- glibc is b0rked on aarch64
-- chromium in NixOS doesn't support U2F but google-chrome does!
-- The OpenJDK package on macOS is missing a key component
-
-Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
-
--->
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 5c9e1d9ccb8..e6d346bc1cf 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -8,19 +8,25 @@ assignees: ''
 ---
 
 **Describe the bug**
+
 A clear and concise description of what the bug is.
 
+If you have a problem with a specific package or NixOS,
+you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. 
+
 **Steps To Reproduce**
-Steps to reproduce the behavior:
+
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
 **Expected behavior**
+
 A clear and concise description of what you expected to happen.
 
-**`nix-env --version` output: **
+**`nix-env --version` output**
 
 **Additional context**
+
 Add any other context about the problem here.

From 171b4ce85c491563004ad6b74377a8cb70a7d06c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 30 Apr 2020 09:57:01 -0600
Subject: [PATCH 2027/2196] typo

---
 src/nix/shell.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/shell.cc b/src/nix/shell.cc
index 71e640667fc..e0546ef78d8 100644
--- a/src/nix/shell.cc
+++ b/src/nix/shell.cc
@@ -219,7 +219,7 @@ struct CmdDevShell : Common, MixEnvironment
         mkFlag()
             .longName("command")
             .shortName('c')
-            .description("command and arguments to be executed insted of an interactive shell")
+            .description("command and arguments to be executed instead of an interactive shell")
             .labels({"command", "args"})
             .arity(ArityAny)
             .handler([&](std::vector<std::string> ss) {

From 533343628dcd4cacc8ff74d429372bfbd860a683 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Thu, 30 Apr 2020 22:01:28 +0200
Subject: [PATCH 2028/2196] Revamp README.md

---
 README.md | 57 +++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 9c0c8788742..9aaca6ed32c 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,55 @@
+# Nix
+
 [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
 [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
 
-Nix, the purely functional package manager
-------------------------------------------
+Nix is a powerful package manager for Linux and other Unix systems that makes package
+management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
+for more details.
+
+## Installation
+
+On Linux and macOS the easiest way to Install Nix is to run the following shell command
+(as a user other than root):
+
+```
+$ curl -L https://nixos.org/nix/install | sh
+```
+
+Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
+
+## Building And Developing
+
+### Building Nix
+
+You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix):
+
+```
+$ nix-build ./release.nix -A build.aarch64-linux
+$ nix-build ./release.nix -A build.x86_64-darwin
+$ nix-build ./release.nix -A build.i686-linux
+$ nix-build ./release.nix -A build.x86_64-linux
+```
+
+### Development Environment
 
-Nix is a new take on package management that is fairly unique. Because of its
-purity aspects, a lot of issues found in traditional package managers don't
-appear with Nix.
+You can use the provided `shell.nix` to easily bootstrap working development environment:
 
-To find out more about the tool, usage and installation instructions, please
-read the manual, which is available on the Nix website at
-<https://nixos.org/nix/manual>.
+```
+$ nix-shell
+$ ./bootstrap.sh
+$ ./configure
+$ make
+```
 
-## Contributing
+## Additional Resources
 
-Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
-of the manual. It helps you to get started with building Nix from source.
+- [Nix manual](https://nixos.org/nix/manual)
+- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix)
+- [Nix - A One Pager](https://github.com/tazjin/nix-1p)
+- [NixOS Discourse](https://discourse.nixos.org/)
+- #nix / #nixos on irc.freenode.net
 
 ## License
 
-Nix is released under the LGPL v2.1
+Nix is released under the [LGPL v2.1](./COPYING)

From f5d3215c8740feec8053a697eca04c12eb71191b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 30 Apr 2020 16:31:47 -0600
Subject: [PATCH 2029/2196] logError

---
 src/error-demo/error-demo.cc | 2 --
 src/nix/verify.cc            | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 2c4f2da6ef2..5142584160e 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -65,7 +65,6 @@ int main()
                     .description = "Vomit description",
         });
 
-
     // Error in a program; no hint and no nix code.
     logError(
         ErrorInfo { .name = "name",
@@ -80,7 +79,6 @@ int main()
                     .hint = hintfmt("there was a %1%", "warning"),
         });
 
-
     // Warning with nix file, line number, column, and the lines of
     // code where a warning occurred.
     SymbolTable testTable;
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 9b065880366..6e043dc2d5c 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -137,7 +137,7 @@ struct CmdVerify : StorePathsCommand
                                 doSigs(info2->sigs);
                             } catch (InvalidPath &) {
                             } catch (Error & e) {
-                                printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
+                                logError(e.info());
                             }
                         }
 
@@ -156,7 +156,7 @@ struct CmdVerify : StorePathsCommand
                 done++;
 
             } catch (Error & e) {
-                printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
+                logError(e.info());
                 failed++;
             }
 

From a3030e3c3186f980716c475155c387bf18041a3f Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 30 Apr 2020 17:56:26 -0600
Subject: [PATCH 2030/2196] fix error calls

---
 src/nix-channel/nix-channel.cc | 30 ++++++++++--------------------
 src/nix-store/nix-store.cc     |  2 +-
 2 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index c403e37804d..589a061c74c 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -37,8 +37,7 @@ static void writeChannels()
 {
     auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)};
     if (!channelsFD)
-        throw Error("");
-        // throw SysError("opening '%1%' for writing", channelsList);
+        throw SysError("opening '%1%' for writing", channelsList);
     for (const auto & channel : channels)
         writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n");
 }
@@ -47,11 +46,9 @@ static void writeChannels()
 static void addChannel(const string & url, const string & name)
 {
     if (!regex_search(url, std::regex("^(file|http|https)://")))
-        throw Error("");
-        // throw Error("invalid channel URL '%1%'", url);
+        throw Error("invalid channel URL '%1%'", url);
     if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$")))
-        throw Error("");
-        // throw Error("invalid channel identifier '%1%'", name);
+        throw Error("invalid channel identifier '%1%'", name);
     readChannels();
     channels[name] = url;
     writeChannels();
@@ -143,11 +140,9 @@ static void update(const StringSet & channelNames)
         if (S_ISLNK(st.st_mode))
             // old-skool ~/.nix-defexpr
             if (unlink(nixDefExpr.c_str()) == -1)
-                throw Error("");
-        //         throw SysError("unlinking %1%", nixDefExpr);
+                throw SysError("unlinking %1%", nixDefExpr);
     } else if (errno != ENOENT) {
-        throw Error("");
-        // throw SysError("getting status of %1%", nixDefExpr);
+        throw SysError("getting status of %1%", nixDefExpr);
     }
     createDirs(nixDefExpr);
     auto channelLink = nixDefExpr + "/channels";
@@ -199,12 +194,10 @@ static int _main(int argc, char ** argv)
 
         switch (cmd) {
             case cNone:
-                throw Error("");
-        //         throw UsageError("no command specified");
+                throw UsageError("no command specified");
             case cAdd:
                 if (args.size() < 1 || args.size() > 2)
-                    throw Error("");
-        //             throw UsageError("'--add' requires one or two arguments");
+                    throw UsageError("'--add' requires one or two arguments");
                 {
                 auto url = args[0];
                 std::string name;
@@ -220,14 +213,12 @@ static int _main(int argc, char ** argv)
                 break;
             case cRemove:
                 if (args.size() != 1)
-                    throw Error("");
-        //             throw UsageError("'--remove' requires one argument");
+                    throw UsageError("'--remove' requires one argument");
                 removeChannel(args[0]);
                 break;
             case cList:
                 if (!args.empty())
-                    throw Error("");
-        //             throw UsageError("'--list' expects no arguments");
+                    throw UsageError("'--list' expects no arguments");
                 readChannels();
                 for (const auto & channel : channels)
                     std::cout << channel.first << ' ' << channel.second << '\n';
@@ -237,8 +228,7 @@ static int _main(int argc, char ** argv)
                 break;
             case cRollback:
                 if (args.size() > 1)
-                    throw Error("");
-        //             throw UsageError("'--rollback' has at most one argument");
+                    throw UsageError("'--rollback' has at most one argument");
                 Strings envArgs{"--profile", profile};
                 if (args.size() == 1) {
                     envArgs.push_back("--switch-generation");
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e4dd9bc96a8..1e8a88a7b92 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -906,7 +906,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (!writeAllowed) throw Error("building paths is not allowed");
 
-                auto drvPath = store->parseStorePath(readString(in)); // inonal ony
+                auto drvPath = store->parseStorePath(readString(in)); // informational only
                 BasicDerivation drv;
                 readDerivation(in, *store, drv);
 

From 536512d273f1d5821f65e9df77fe8f47b823ce6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Fri, 1 May 2020 11:17:38 +0200
Subject: [PATCH 2031/2196] Update README.md

Co-authored-by: worldofpeace <worldofpeace@protonmail.ch>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9aaca6ed32c..454b0ffa001 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux
 
 ### Development Environment
 
-You can use the provided `shell.nix` to easily bootstrap working development environment:
+You can use the provided `shell.nix` to easily bootstrap a working development environment:
 
 ```
 $ nix-shell

From 30616d8e86f7ef635de561905eab0022000547b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Fri, 1 May 2020 11:19:18 +0200
Subject: [PATCH 2032/2196] Update README.md

Co-authored-by: worldofpeace <worldofpeace@protonmail.ch>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 454b0ffa001..0f2f9bd1a2d 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,7 @@ $ make
 ## Additional Resources
 
 - [Nix manual](https://nixos.org/nix/manual)
-- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix)
+- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
 - [Nix - A One Pager](https://github.com/tazjin/nix-1p)
 - [NixOS Discourse](https://discourse.nixos.org/)
 - #nix / #nixos on irc.freenode.net

From 2a434fc62bdb4391620d8030aec6b8d4a9f8ba95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Fri, 1 May 2020 11:20:39 +0200
Subject: [PATCH 2033/2196] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0f2f9bd1a2d..44687a15037 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ $ make
 - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
 - [Nix - A One Pager](https://github.com/tazjin/nix-1p)
 - [NixOS Discourse](https://discourse.nixos.org/)
-- #nix / #nixos on irc.freenode.net
+- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
 
 ## License
 

From 404a94ab6914405948640bedb92d1f9ddd502a8a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 May 2020 11:47:26 +0200
Subject: [PATCH 2034/2196] Tweak README.md

---
 README.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 44687a15037..a1588284dcf 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Information on additional installation methods is available on the [Nix download
 
 ### Building Nix
 
-You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix):
+You can build Nix using one of the targets provided by [release.nix](./release.nix):
 
 ```
 $ nix-build ./release.nix -A build.aarch64-linux
@@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux
 
 ### Development Environment
 
-You can use the provided `shell.nix` to easily bootstrap a working development environment:
+You can use the provided `shell.nix` to get a working development environment:
 
 ```
 $ nix-shell
@@ -46,10 +46,9 @@ $ make
 
 - [Nix manual](https://nixos.org/nix/manual)
 - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
-- [Nix - A One Pager](https://github.com/tazjin/nix-1p)
 - [NixOS Discourse](https://discourse.nixos.org/)
 - [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
 
 ## License
 
-Nix is released under the [LGPL v2.1](./COPYING)
+Nix is released under the [LGPL v2.1](./COPYING).

From 5eebc4ad1d3965096b57c574d361d9274a7daafb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 20 Mar 2020 13:39:39 +0100
Subject: [PATCH 2035/2196] Fix coverage job

(cherry picked from commit bc5d4843a916dc4f99516e187a63108e8006f81f)
---
 release.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/release.nix b/release.nix
index 65b035957db..f5729cee318 100644
--- a/release.nix
+++ b/release.nix
@@ -241,6 +241,11 @@ let
 
         src = nix;
 
+        preConfigure =
+          ''
+            ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor
+          '';
+
         enableParallelBuilding = true;
 
         buildInputs = buildDeps ++ propagatedDeps;

From d6b4047c2fd8c197bf7a9f0f9a2f1cfb214e2e89 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Wed, 29 Apr 2020 14:02:37 -0600
Subject: [PATCH 2036/2196] rename nix run to nix shell and nix app to nix run

(cherry picked from commit 5d8504b9789ffebabe8226227c4061dd48354177)
---
 src/nix/{shell.cc => dev-shell.cc} |  0
 src/nix/run.cc                     | 14 +++++++-------
 2 files changed, 7 insertions(+), 7 deletions(-)
 rename src/nix/{shell.cc => dev-shell.cc} (100%)

diff --git a/src/nix/shell.cc b/src/nix/dev-shell.cc
similarity index 100%
rename from src/nix/shell.cc
rename to src/nix/dev-shell.cc
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 5334531fd99..ebfec36d914 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -57,11 +57,11 @@ struct RunCommon : virtual Command
     }
 };
 
-struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
+struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
 {
     std::vector<std::string> command = { getEnv("SHELL").value_or("bash") };
 
-    CmdRun()
+    CmdShell()
     {
         mkFlag()
             .longName("command")
@@ -85,19 +85,19 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
         return {
             Example{
                 "To start a shell providing GNU Hello from NixOS 17.03:",
-                "nix run -f channel:nixos-17.03 hello"
+                "nix shell -f channel:nixos-17.03 hello"
             },
             Example{
                 "To start a shell providing youtube-dl from your 'nixpkgs' channel:",
-                "nix run nixpkgs.youtube-dl"
+                "nix shell nixpkgs.youtube-dl"
             },
             Example{
                 "To run GNU Hello:",
-                "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"
+                "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'"
             },
             Example{
                 "To run GNU Hello in a chroot store:",
-                "nix run --store ~/my-nix nixpkgs.hello -c hello"
+                "nix shell --store ~/my-nix nixpkgs.hello -c hello"
             },
         };
     }
@@ -141,7 +141,7 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
     }
 };
 
-static auto r1 = registerCommand<CmdRun>("run");
+static auto r1 = registerCommand<CmdShell>("shell");
 
 void chrootHelper(int argc, char * * argv)
 {

From 2852a486f864c385696fb7ee28836b183976ca63 Mon Sep 17 00:00:00 2001
From: Matthew Kenigsberg <matthewkenigsberg@gmail.com>
Date: Wed, 29 Apr 2020 15:37:21 -0600
Subject: [PATCH 2037/2196] rename run to shell in tests

(cherry picked from commit f459ca547fda4b0ca477d895413e86b69a580ab0)
---
 tests/local.mk                     |  2 +-
 tests/run.sh                       | 28 ----------------------------
 tests/{run.nix => shell-hello.nix} |  0
 tests/shell.sh                     | 28 ++++++++++++++++++++++++++++
 4 files changed, 29 insertions(+), 29 deletions(-)
 delete mode 100644 tests/run.sh
 rename tests/{run.nix => shell-hello.nix} (100%)
 create mode 100644 tests/shell.sh

diff --git a/tests/local.mk b/tests/local.mk
index c5602773d75..56e5640ca33 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -21,7 +21,7 @@ nix_tests = \
   fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh \
+  shell.sh \
   brotli.sh \
   pure-eval.sh \
   check.sh \
diff --git a/tests/run.sh b/tests/run.sh
deleted file mode 100644
index d1dbfd6bd4a..00000000000
--- a/tests/run.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-source common.sh
-
-clearStore
-clearCache
-
-nix run -f run.nix hello -c hello | grep 'Hello World'
-nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS'
-
-if ! canUseSandbox; then exit; fi
-
-chmod -R u+w $TEST_ROOT/store0 || true
-rm -rf $TEST_ROOT/store0
-
-clearStore
-
-path=$(nix eval --raw -f run.nix hello)
-
-# Note: we need the sandbox paths to ensure that the shell is
-# visible in the sandbox.
-nix run --sandbox-build-dir /build-tmp \
-    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
-    --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
-
-path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
-
-[[ $path/bin/hello = $path2 ]]
-
-[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]
diff --git a/tests/run.nix b/tests/shell-hello.nix
similarity index 100%
rename from tests/run.nix
rename to tests/shell-hello.nix
diff --git a/tests/shell.sh b/tests/shell.sh
new file mode 100644
index 00000000000..7a9ee8ab099
--- /dev/null
+++ b/tests/shell.sh
@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
+nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
+
+if ! canUseSandbox; then exit; fi
+
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
+
+clearStore
+
+path=$(nix eval --raw -f shell-hello.nix hello)
+
+# Note: we need the sandbox paths to ensure that the shell is
+# visible in the sandbox.
+nix shell --sandbox-build-dir /build-tmp \
+    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
+    --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World'
+
+path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello')
+
+[[ $path/bin/hello = $path2 ]]
+
+[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]

From c089c52d5f1cff888552f485775b74226dcbe618 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 May 2020 12:42:39 +0200
Subject: [PATCH 2038/2196] Fix build

---
 src/nix/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/local.mk b/src/nix/local.mk
index 15cef55bbef..8c0eed19ef2 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
 
-src/nix/shell.cc: src/nix/get-env.sh.gen.hh
+src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh

From 4b99c09f5ccd385d2bf0c82a8c9a4ae1658abbe8 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 1 May 2020 14:32:06 -0600
Subject: [PATCH 2039/2196] convert some errors

---
 src/libstore/builtins/buildenv.cc | 12 +++++++--
 src/libstore/download.cc          |  7 ++++-
 src/libstore/gc.cc                | 13 ++++++---
 src/libstore/local-store.cc       | 44 +++++++++++++++++++++++--------
 src/libstore/optimise-store.cc    | 18 ++++++++++---
 src/libstore/remote-store.cc      |  9 +++++++
 src/libstore/sqlite.cc            |  5 +++-
 7 files changed, 86 insertions(+), 22 deletions(-)

diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 00076909494..64085fcc9b9 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -22,7 +22,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            printError("warning: not including '%s' in the user environment because it's not a directory", srcDir);
+            logWarning(
+                ErrorInfo { 
+                    .name = "Create Links - Directory",
+                    .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
+            });
             return;
         }
         throw;
@@ -41,7 +45,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                printError("warning: skipping dangling symlink '%s'", dstFile);
+                logWarning(
+                    ErrorInfo { 
+                        .name = "Create Links - Skipping Symlink",
+                        .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
+                });
                 continue;
             }
             throw;
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 7d0cb449ba5..60c6a80f332 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -600,7 +600,12 @@ struct CurlDownloader : public Downloader
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            printError("unexpected error in download thread: %s", e.what());
+            logError(
+                ErrorInfo { 
+                    .name = "download",
+                    .hint = hintfmt("unexpected error in download thread: %s", 
+                                    e.what())
+                });
         }
 
         {
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 27bd3f3e946..4f6f78a9966 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -41,7 +41,8 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
         throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printError(format("waiting for the big garbage collector lock..."));
+        // TODO: info?
+        printError("waiting for the big garbage collector lock...");
         lockFile(fdGCLock.get(), lockType, true);
     }
 
@@ -129,10 +130,14 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     if (settings.checkRootReachability) {
         auto roots = findRoots(false);
         if (roots[storePath.clone()].count(gcRoot) == 0)
-            printError(
-                "warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+
+            logWarning(
+                ErrorInfo { 
+                    .name = "GC Root",
+                    .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
                 "therefore, '%2%' might be removed by the garbage collector",
-                gcRoot, printStorePath(storePath));
+                gcRoot, printStorePath(storePath))
+            });
     }
 
     /* Grab the global GC root, causing us to block while a GC is in
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index db5a3b53cca..1f16a22b117 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -87,8 +87,13 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist")
-                % settings.buildUsersGroup);
+            logError(
+                ErrorInfo { 
+                    .name = "'build-users-group' not found",
+                    .hint = hintfmt(
+                        "warning: the group '%1%' specified in 'build-users-group' does not exist",
+                        settings.buildUsersGroup)
+            });
         else {
             struct stat st;
             if (stat(realStoreDir.c_str(), &st))
@@ -876,7 +881,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
             } catch (SubstituterDisabled &) {
             } catch (Error & e) {
                 if (settings.tryFallback)
-                    printError(e.what());
+                    logError(e.info());
                 else
                     throw;
             }
@@ -1237,9 +1242,13 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                printError(
+                logError(
+                    ErrorInfo { 
+                        .name = "Invalid Hash",
+                        .hint = hintfmt(
                     "link '%s' was modified! expected hash '%s', got '%s'",
-                    linkPath, link.name, hash);
+                    linkPath, link.name, hash)
+                });
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
                         printError("removed link '%s'", linkPath);
@@ -1272,8 +1281,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(), current.first.to_string());
+                    logError(
+                        ErrorInfo { 
+                            .name = "Invalid Hash - Path Modified",
+                            .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
+                            printStorePath(i), info->narHash.to_string(), current.first.to_string())
+                    });
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
@@ -1304,7 +1317,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* It's possible that the path got GC'ed, so ignore
                    errors on invalid paths. */
                 if (isValidPath(i))
-                    printError("error: %s", e.msg());
+                    logError(e.info());
                 else
                     warn(e.msg());
                 errors = true;
@@ -1324,7 +1337,11 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        printError("path '%s' is not in the Nix store", pathS);
+        logError(
+            ErrorInfo { 
+                .name = "Nix path not found",
+                .hint = hintfmt("path '%s' is not in the Nix store", pathS)
+        });
         return;
     }
 
@@ -1347,12 +1364,17 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            printError("path '%s' disappeared, but it still has valid referrers!", pathS);
+            // TODO log as warning if repair successful??
+            logError(
+                ErrorInfo { 
+                    .name = "Missing path with referrers",
+                    .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
+            });
             if (repair)
                 try {
                     repairPath(path);
                 } catch (Error & e) {
-                    warn(e.msg());
+                    logWarning(e.info());
                     errors = true;
                 }
             else errors = true;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index f9cf2bb6b87..9f611218349 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -130,7 +130,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        printError("skipping suspicious writable file '%1%'", path);
+        logWarning(
+            ErrorInfo { 
+                .name = "Suspicious File",
+                .hint = hintfmt("skipping suspicious writable file '%1%'", path)
+        });
         return;
     }
 
@@ -194,7 +198,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        printError("removing corrupted link '%1%'", linkPath);
+        logWarning(
+            ErrorInfo { 
+                .name = "Corrupted Link",
+                .hint = hintfmt("removing corrupted link '%1%'", linkPath)
+        });
         unlink(linkPath.c_str());
         goto retry;
     }
@@ -229,7 +237,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            printError("unable to unlink '%1%'", tempLink);
+            logError(
+                ErrorInfo { 
+                    .name = "Unlink error",
+                    .hint = hintfmt("unable to unlink '%1%'", tempLink)
+            });
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cc336e460e3..7c50c106597 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -780,6 +780,15 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         }
 
         else if (msg == STDERR_NEXT)
+            // TODO: is this really an ErrorInfo error?  Seems like we're forwarding the 
+            // stderr output of the remote to current stderr/log 
+            // ErrorInfo gets lost in this scenario.
+            // An alternative might be a logger on the remote that forwards ErrorInfo and etc.
+            // logError(
+            //     ErrorInfo { 
+            //         // .name = "Remote Store" TODO reasonable name.
+            //         .hint = hintfmt(chomp(readString(from)))
+            // });
             printError(chomp(readString(from)));
 
         else if (msg == STDERR_START_ACTIVITY) {
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index a72cd5d8860..082c5400513 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -204,7 +204,10 @@ void handleSQLiteBusy(SQLiteBusy & e)
 
     if (now > lastWarned + 10) {
         lastWarned = now;
-        printError("warning: %s", e.what());
+        logWarning(
+            ErrorInfo { .name = "sqlite busy",
+                        .hint = hintfmt(e.what())
+            });
     }
 
     /* Sleep for a while since retrying the transaction right away

From 2aeb874e832ae4c326f1115e55fdcaf7c2f4e916 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 1 May 2020 23:32:01 +0200
Subject: [PATCH 2040/2196] Improve help-message for nix-repl

* Remove obsolete `printHelp` function
* Add an example to demonstrate how to list all available commands
  within the REPL
---
 src/nix/repl.cc | 44 ++++++++++----------------------------------
 1 file changed, 10 insertions(+), 34 deletions(-)

diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 27727bd2550..ea8ff1553b5 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -82,40 +82,6 @@ struct NixRepl : gc
 };
 
 
-void printHelp()
-{
-    std::cout
-         << "Usage: nix-repl [--help] [--version] [-I path] paths...\n"
-         << "\n"
-         << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n"
-         << "\n"
-         << "Options:\n"
-         << "    --help\n"
-         << "        Prints out a summary of the command syntax and exits.\n"
-         << "\n"
-         << "    --version\n"
-         << "        Prints out the Nix version number on standard output and exits.\n"
-         << "\n"
-         << "    -I path\n"
-         << "        Add a path to the Nix expression search path. This option may be given\n"
-         << "        multiple times. See the NIX_PATH environment variable for information on\n"
-         << "        the semantics of the Nix search path. Paths added through -I take\n"
-         << "        precedence over NIX_PATH.\n"
-         << "\n"
-         << "    paths...\n"
-         << "        A list of paths to files containing Nix expressions which nix-repl will\n"
-         << "        load and add to its scope.\n"
-         << "\n"
-         << "        A path surrounded in < and > will be looked up in the Nix expression search\n"
-         << "        path, as in the Nix language itself.\n"
-         << "\n"
-         << "        If an element of paths starts with http:// or https://, it is interpreted\n"
-         << "        as the URL of a tarball that will be downloaded and unpacked to a temporary\n"
-         << "        location. The tarball must include a single top-level directory containing\n"
-         << "        at least a file named default.nix.\n";
-}
-
-
 string removeWhitespace(string s)
 {
     s = chomp(s);
@@ -809,6 +775,16 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return "start an interactive environment for evaluating Nix expressions";
     }
 
+    Examples examples() override
+    {
+        return {
+          Example{
+            "Display all special commands within the REPL:",
+              "nix repl\n  nix-repl> :?"
+          }
+        };
+    }
+
     void run(ref<Store> store) override
     {
         auto repl = std::make_unique<NixRepl>(searchPath, openStore());

From f16e24f95e7d7fa3a5c3b2c8d43d171fa5b9cf85 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 2 May 2020 10:58:33 +0200
Subject: [PATCH 2041/2196] remote-store: don't log raw stderr by default

For remote stores the log messages are already forwarded as structured
STDERR_RESULT messages so the old format is duplicate information.  But
still included with -vvv since it could be useful for debugging
problems.

    $ nix build -L /nix/store/nl71b2niws857ffiaggyrkjwgx9jjzc0-foo.drv --store ssh-ng://localhost
    Hello World!
    foo> Hello World!
    [1/0/1 built] building foo

Fixes #3556
---
 src/libstore/remote-store.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8c55da2686e..07ef79382d0 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -779,8 +779,10 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
             return std::make_exception_ptr(Error(status, error));
         }
 
-        else if (msg == STDERR_NEXT)
-            printError(chomp(readString(from)));
+        else if (msg == STDERR_NEXT) {
+            string s = chomp(readString(from));
+            printMsg(lvlVomit, "stderr %s", s);
+        }
 
         else if (msg == STDERR_START_ACTIVITY) {
             auto act = readNum<ActivityId>(from);

From 4769eea5e2229c01c3b5d050891b071a45a024d5 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 2 May 2020 23:26:12 +0200
Subject: [PATCH 2042/2196] logging: handle build log lines in simple logger

The raw stderr output isn't logged anymore so the build logs need to be
printed by the default logger in order for the old commands like
nix-build to still show build output.
---
 src/libutil/logging.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 3cc4ef8f15b..777650de5cd 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -63,6 +63,16 @@ class SimpleLogger : public Logger
         writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
     }
 
+    void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
+    {
+        if (type == resBuildLogLine || type == resPostBuildLogLine) {
+            assert(0 < fields.size());
+            assert(fields[0].type == Logger::Field::tString);
+            auto lastLine = fields[0].s;
+            log(lvlInfo, lastLine);
+        }
+    }
+
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent)
         override

From ab6f0b9641ad6e9cef72f73d23e31138a97a225b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 3 May 2020 08:01:25 -0600
Subject: [PATCH 2043/2196] convert some printError calls to logError

---
 src/build-remote/build-remote.cc              |  8 +-
 src/libexpr/parser.y                          | 15 ++-
 src/libmain/shared.cc                         | 10 +-
 src/libstore/build.cc                         | 97 ++++++++++++++-----
 src/libstore/pathlocks.cc                     |  3 +-
 src/libstore/s3-binary-cache-store.cc         |  4 +-
 src/libstore/store-api.cc                     |  2 +-
 src/libutil/affinity.cc                       |  4 +-
 src/libutil/error.hh                          |  6 ++
 src/libutil/logging.hh                        |  1 +
 src/libutil/util.cc                           | 18 ++--
 src/nix-build/nix-build.cc                    |  2 +
 src/nix-daemon/nix-daemon.cc                  |  3 +-
 src/nix-env/nix-env.cc                        | 23 ++++-
 src/nix-env/user-env.cc                       |  2 +-
 src/nix-store/nix-store.cc                    | 15 ++-
 src/nix/upgrade-nix.cc                        |  8 +-
 src/nix/verify.cc                             | 20 +++-
 .../resolve-system-dependencies.cc            | 24 ++++-
 19 files changed, 195 insertions(+), 70 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 00340b7872a..0d6859596fa 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -200,9 +200,13 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    printError("cannot build on '%s': %s%s",
+                    logError(
+                        ErrorInfo { 
+                            .name = "remote build",
+                            .hint = hintfmt("cannot build on '%s': %s%s",
                         bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg));
+                        (msg.empty() ? "" : ": " + msg))
+                    });
                     bestMachine->enabled = false;
                     continue;
                 }
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 3767532d5ef..3ed9a7a4fce 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -689,8 +689,12 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             request.unpack = true;
             res = { true, getDownloader()->downloadCached(store, request).path };
         } catch (DownloadError & e) {
-            // TODO: change to warn()?
-            printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second);
+            logWarning(
+                ErrorInfo { 
+                    .name = "Download Error",
+                    .hint = hintfmt("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
+
             res = { false, "" };
         }
     } else {
@@ -698,8 +702,11 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            // TODO: change to warn()?
-            printError("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second);
+            logWarning(
+                ErrorInfo { 
+                    .name = "Search path not found",
+                    .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     }
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 8551eb0484a..5773d90cc0a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -300,21 +300,21 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (UsageError & e) {
         // TODO: switch to logError
         // logError(e.info());
-        printError(
+        _printError(
             format("%1%\nTry '%2% --help' for more information.")
             % e.what() % programName);
         return 1;
     } catch (BaseError & e) {
         // logError(e.info());
-        printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+        _printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
         if (e.prefix() != "" && !settings.showTrace)
-            printError("(use '--show-trace' to show detailed location information)");
+            _printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
-        printError(error + "out of memory");
+        _printError(error + "out of memory");
         return 1;
     } catch (std::exception & e) {
-        printError(error + e.what());
+        _printError(error + e.what());
         return 1;
     }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2301adb16d1..bacbd5808f6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -493,7 +493,9 @@ void handleDiffHook(
             if (diffRes.second != "")
                 printError(chomp(diffRes.second));
         } catch (Error & error) {
-            printError("diff hook execution failed: %s", error.what());
+            // logError(error.info())
+            // TODO append message onto errorinfo...
+            _printError("diff hook execution failed: %s", error.what());
         }
     }
 }
@@ -1144,7 +1146,11 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath));
+        logError(
+            ErrorInfo { 
+                .name = "missing derivation during build",
+                .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))
+        });
         done(BuildResult::MiscFailure);
         return;
     }
@@ -1295,8 +1301,14 @@ void DerivationGoal::repairClosure()
     /* Check each path (slow!). */
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
-        printError("found corrupted or missing path '%s' in the output closure of '%s'",
-            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
+        logError(
+            ErrorInfo { 
+                .name = "Corrupt path in closure",
+                .hint = hintfmt(
+                    "found corrupted or missing path '%s' in the output closure of '%s'",
+                    worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
+        });
+
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
             addWaitee(worker.makeSubstitutionGoal(i, Repair));
@@ -1330,8 +1342,13 @@ void DerivationGoal::inputsRealised()
     if (nrFailed != 0) {
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        printError("cannot build derivation '%s': %s dependencies couldn't be built",
-            worker.store.printStorePath(drvPath), nrFailed);
+        logError(
+            ErrorInfo { 
+                .name = "Dependencies could not be built",
+                .hint = hintfmt(
+                    "cannot build derivation '%s': %s dependencies couldn't be built",
+                    worker.store.printStorePath(drvPath), nrFailed)
+        });
         done(BuildResult::DependencyFailed);
         return;
     }
@@ -1489,7 +1506,7 @@ void DerivationGoal::tryToBuild()
         startBuilder();
 
     } catch (BuildError & e) {
-        printError(e.msg());
+        logError(e.info());
         outputLocks.unlock();
         buildUser.reset();
         worker.permanentFailure = true;
@@ -1709,7 +1726,7 @@ void DerivationGoal::buildDone()
         outputLocks.unlock();
 
     } catch (BuildError & e) {
-        printError(e.msg());
+        logError(e.info());
 
         outputLocks.unlock();
 
@@ -1788,8 +1805,13 @@ HookReply DerivationGoal::tryBuildHook()
 
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
-            printError("build hook died unexpectedly: %s",
-                chomp(drainFD(worker.hook->fromHook.readSide.get())));
+            logError(
+                ErrorInfo { 
+                    .name = "Build hook died",
+                    .hint = hintfmt(
+                        "build hook died unexpectedly: %s",
+                        chomp(drainFD(worker.hook->fromHook.readSide.get())))
+            });
             worker.hook = 0;
             return rpDecline;
         } else
@@ -3783,10 +3805,10 @@ void DerivationGoal::registerOutputs()
                 result.isNonDeterministic = true;
                 Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
                 bool prevExists = keepPreviousRound && pathExists(prev);
-                auto msg = prevExists
-                    ? fmt("output '%s' of '%s' differs from '%s' from previous round",
+                hintformat hint = prevExists
+                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
                         worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : fmt("output '%s' of '%s' differs from previous round",
+                    : hintfmt("output '%s' of '%s' differs from previous round",
                         worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
 
                 handleDiffHook(
@@ -3796,9 +3818,15 @@ void DerivationGoal::registerOutputs()
                     worker.store.printStorePath(drvPath), tmpDir);
 
                 if (settings.enforceDeterminism)
-                    throw NotDeterministic(msg);
+                    throw NotDeterministic(hint);
+
+                logError(
+                    ErrorInfo { 
+                        .name = "Output determinism error",
+                        .hint = hint
+                });
+
 
-                printError(msg);
                 curRound = nrRounds; // we know enough, bail out early
             }
     }
@@ -4060,9 +4088,13 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            printError(
-                format("%1% killed after writing more than %2% bytes of log output")
-                % getName() % settings.maxLogSize);
+            logError(
+                ErrorInfo { 
+                    .name = "Max log size exceeded",
+                    .hint = hintfmt(
+                        "%1% killed after writing more than %2% bytes of log output",
+                        getName(), settings.maxLogSize)
+            });
             killChild();
             done(BuildResult::LogLimitExceeded);
             return;
@@ -4352,7 +4384,7 @@ void SubstitutionGoal::tryNext()
         throw;
     } catch (Error & e) {
         if (settings.tryFallback) {
-            printError(e.what());
+            logError(e.info());
             tryNext();
             return;
         }
@@ -4864,9 +4896,13 @@ void Worker::waitForInput()
             j->respectTimeouts &&
             after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
-            printError(
-                format("%1% timed out after %2% seconds of silence")
-                % goal->getName() % settings.maxSilentTime);
+            logError(
+                ErrorInfo { 
+                    .name = "Silent build timeout",
+                    .hint = hintfmt(
+                        "%1% timed out after %2% seconds of silence",
+                        goal->getName(), settings.maxSilentTime)
+            });
             goal->timedOut();
         }
 
@@ -4875,9 +4911,13 @@ void Worker::waitForInput()
             j->respectTimeouts &&
             after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
-            printError(
-                format("%1% timed out after %2% seconds")
-                % goal->getName() % settings.buildTimeout);
+            logError(
+                ErrorInfo { 
+                    .name = "Build timeout",
+                    .hint = hintfmt(
+                        "%1% timed out after %2% seconds",
+                        goal->getName(), settings.buildTimeout)
+            });
             goal->timedOut();
         }
     }
@@ -4939,7 +4979,12 @@ bool Worker::pathContentsGood(const StorePath & path)
         res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache.insert_or_assign(path.clone(), res);
-    if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path));
+    if (!res) 
+        logError(
+            ErrorInfo { 
+                .name = "Corrupted path",
+                .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
+        });
     return res;
 }
 
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 52d430ffd68..926f4ea1e44 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -160,7 +160,8 @@ void PathLocks::unlock()
 
         if (close(i.first) == -1)
             printError(
-                format("error (ignored): cannot close lock file on '%1%'") % i.second);
+                "error (ignored): cannot close lock file on '%1%'",
+                i.second);
 
         debug(format("lock released on '%1%'") % i.second);
     }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 0326821f66b..3175eb69ba9 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -111,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",
-                error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
+                error.GetExceptionName(),
+                error.GetMessage(),
+                CalculateDelayBeforeNextRetry(error, attemptedRetries));
         return retry;
     }
 };
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a04d5013ee5..542e5c552d8 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -961,7 +961,7 @@ std::list<ref<Store>> getDefaultSubstituters()
             try {
                 stores.push_back(openStore(uri));
             } catch (Error & e) {
-                printError("warning: %s", e.what());
+                logWarning(e.info());
             }
         };
 
diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc
index 98f8287ada6..e3cf33b58ca 100644
--- a/src/libutil/affinity.cc
+++ b/src/libutil/affinity.cc
@@ -25,7 +25,7 @@ void setAffinityTo(int cpu)
     CPU_ZERO(&newAffinity);
     CPU_SET(cpu, &newAffinity);
     if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
-        printError(format("failed to lock thread to CPU %1%") % cpu);
+        printError("failed to lock thread to CPU %1%", cpu);
 #endif
 }
 
@@ -47,7 +47,7 @@ void restoreAffinity()
 #if __linux__
     if (!didSaveAffinity) return;
     if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
-        printError("failed to restore affinity %1%");
+        _printError("failed to restore affinity");
 #endif
 }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 48e6311bd60..86cff56091d 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -117,6 +117,12 @@ public:
               }
     { }
 
+    BaseError(hintformat hint)
+        : err { .level = lvlError,
+                .hint = hint
+              }
+    { }
+
     BaseError(ErrorInfo e)
         : err(e)
     { }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 0c4980b835a..5f03cdf41cc 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -149,6 +149,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
         } \
     } while (0)
 
+#define _printError(args...) printMsg(lvlError, args)
 #define printError(args...) printMsg(lvlError, args)
 #define printInfo(args...) printMsg(lvlInfo, args)
 #define printTalkative(args...) printMsg(lvlTalkative, args)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 747a9e991ec..68dc1b738d0 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1417,15 +1417,15 @@ string base64Decode(const string & s)
 }
 
 
-void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc)
-{
-    try {
-        failure(exc);
-    } catch (std::exception & e) {
-        printError("uncaught exception: %s", e.what());
-        abort();
-    }
-}
+// void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc)
+// {
+//     try {
+//         failure(exc);
+//     } catch (std::exception & e) {
+//         printError("uncaught exception: %s", e.what());
+//         abort();
+//     }
+// }
 
 
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 0a058a31b49..401c8d3404e 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -368,6 +368,8 @@ static void _main(int argc, char * * argv)
                 shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
+                // TODO: append error msg
+                logError(e.info());
                 printError("warning: %s; will use bash from your environment", e.what());
                 shell = "bash";
             }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 5f78ab46409..3cd2c3a6b31 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -247,7 +247,8 @@ static void daemonLoop(char * * argv)
         } catch (Interrupted & e) {
             return;
         } catch (Error & e) {
-            printError("error processing connection: %1%", e.msg());
+            // TODO append error message
+            _printError("error processing connection: %1%", e.msg());
         }
     }
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index dde8875f1c8..76008d00cc3 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -123,7 +123,11 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             if (!attrs.insert(attrName).second) {
-                printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2);
+                logError(
+                    ErrorInfo { 
+                        .name = "Name collision",
+                        .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
+                });
                 continue;
             }
             /* Load the expression on demand. */
@@ -860,7 +864,12 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
+                logError(
+                    ErrorInfo { 
+                        .name = "Invalid meta attribute",
+                        .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
+                            i.queryName(), j)
+                });
                 placeholder.write(nullptr);
             } else {
                 PathSet context;
@@ -1110,8 +1119,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             XMLAttrs attrs2;
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
-                            if (!v)
-                                printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
+                            if (!v) 
+                                logError(
+                                    ErrorInfo { 
+                                        .name = "Invalid meta attribute",
+                                        .hint = hintfmt(
+                                            "derivation '%s' has invalid meta attribute '%s'",
+                                            i.queryName(), j)
+                                });
                             else {
                                 if (v->type == tString) {
                                     attrs2["type"] = "string";
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index f852916d896..2484b97593f 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -146,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         Path lockTokenCur = optimisticLockProfile(profile);
         if (lockToken != lockTokenCur) {
-            printError(format("profile '%1%' changed while we were busy; restarting") % profile);
+            printError("profile '%1%' changed while we were busy; restarting", profile);
             return false;
         }
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 1e8a88a7b92..2e2276b4a80 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -726,9 +726,15 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
-            printError(
-                "path '%s' was modified! expected hash '%s', got '%s'",
-                store->printStorePath(path), info->narHash.to_string(), current.first.to_string());
+            logError(
+                ErrorInfo { 
+                    .name = "Hash match error",
+                    .hint = hintfmt(
+                        "path '%s' was modified! expected hash '%s', got '%s'",
+                        store->printStorePath(path),
+                        info->narHash.to_string(),
+                        current.first.to_string())
+            });
             status = 1;
         }
     }
@@ -835,7 +841,8 @@ static void opServe(Strings opFlags, Strings opArgs)
                             for (auto & p : willSubstitute) subs.emplace_back(p.clone());
                             store->buildPaths(subs);
                         } catch (Error & e) {
-                            printError("warning: %1%", e.msg());
+                            // logWarning(e.info())  TODO:  
+                            _printError("warning: %1%", e.msg());
                         }
                 }
 
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index c05c29517b2..8f4e529bc54 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -64,7 +64,13 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         if (dryRun) {
             stopProgressBar();
-            printError("would upgrade to version %s", version);
+            // TODO change to info?
+            logWarning(
+                ErrorInfo { 
+                    .name = "Version update",
+                    .hint = hintfmt("would upgrade to version %s", version)
+            });
+            // printError("would upgrade to version %s", version);
             return;
         }
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 6e043dc2d5c..f532172392f 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -97,9 +97,15 @@ struct CmdVerify : StorePathsCommand
                     if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
-                        printError(
-                            "path '%s' was modified! expected hash '%s', got '%s'",
-                            store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string());
+                        logError(
+                            ErrorInfo { 
+                                .name = "Hash error - path modified",
+                                .hint = hintfmt(
+                                    "path '%s' was modified! expected hash '%s', got '%s'",
+                                    store->printStorePath(info->path),
+                                    info->narHash.to_string(),
+                                    hash.first.to_string())
+                            });
                     }
 
                 }
@@ -148,7 +154,13 @@ struct CmdVerify : StorePathsCommand
                     if (!good) {
                         untrusted++;
                         act2.result(resUntrustedPath, store->printStorePath(info->path));
-                        printError("path '%s' is untrusted", store->printStorePath(info->path));
+                        logError(
+                            ErrorInfo { 
+                                .name = "Untrusted path",
+                                .hint = hintfmt("path '%s' is untrusted",
+                                    store->printStorePath(info->path))
+                        });
+
                     }
 
                 }
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 8f0c99c84b2..dcea72529e0 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -39,12 +39,20 @@ std::set<std::string> runResolver(const Path & filename)
         throw SysError("statting '%s'", filename);
 
     if (!S_ISREG(st.st_mode)) {
-        printError("file '%s' is not a regular file", filename);
+        logError(
+            ErrorInfo { 
+                .name = "Regular MACH file",
+                .hint = hintfmt("file '%s' is not a regular file", filename)
+        });
         return {};
     }
 
     if (st.st_size < sizeof(mach_header_64)) {
-        printError("file '%s' is too short for a MACH binary", filename);
+        logError(
+            ErrorInfo { 
+                .name = "File too short",
+                .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
+        });
         return {};
     }
 
@@ -66,13 +74,21 @@ std::set<std::string> runResolver(const Path & filename)
             }
         }
         if (mach64_offset == 0) {
-            printError(format("Could not find any mach64 blobs in file '%1%', continuing...") % filename);
+            logError(
+                ErrorInfo { 
+                    .name = "No mach64 blobs",
+                    .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
+            });
             return {};
         }
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
-        printError(format("Object file has unknown magic number '%1%', skipping it...") % magic);
+        logError(
+            ErrorInfo { 
+                .name = "Magic number",
+                .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
+        });
         return {};
     }
 

From c05f0e3093d4fa4ea9b07b038099ff9555258717 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 12:28:28 -0600
Subject: [PATCH 2044/2196] closer but still lambda indent problems

---
 .uncrustify.cfg | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.uncrustify.cfg b/.uncrustify.cfg
index b28935dcc10..6bf91ec3c68 100644
--- a/.uncrustify.cfg
+++ b/.uncrustify.cfg
@@ -1041,14 +1041,14 @@ indent_else_if                  = false    # true/false
 indent_var_def_blk              = 0        # number
 
 # Whether to indent continued variable declarations instead of aligning.
-indent_var_def_cont             = false    # true/false
+indent_var_def_cont             = true    # true/false
 
 # Whether to indent continued shift expressions ('<<' and '>>') instead of
 # aligning. Set align_left_shift=false when enabling this.
 indent_shift                    = false    # true/false
 
 # Whether to force indentation of function definitions to start in column 1.
-indent_func_def_force_col1      = false    # true/false
+indent_func_def_force_col1      = true    # true/false
 
 # Whether to indent continued function call parameters one indent level,
 # rather than aligning parameters under the open parenthesis.
@@ -1118,7 +1118,7 @@ indent_sing_line_comments       = 0        # unsigned number
 indent_relative_single_line_comments = false    # true/false
 
 # Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns.
-indent_switch_case              = 0        # unsigned number
+indent_switch_case              = indent_columns       # unsigned number
 
 # indent 'break' with 'case' from 'switch'.
 indent_switch_break_with_case   = false    # true/false
@@ -1217,13 +1217,13 @@ indent_preserve_sql             = false    # true/false
 # followed by a newline, the next line is indent one tab.
 #
 # Default: true
-indent_align_assign             = true     # true/false
+indent_align_assign             = false     # true/false
 
 # Whether to align continued statements at the '('. If false or the '(' is
 # followed by a newline, the next line indent is one tab.
 #
 # Default: true
-indent_align_paren              = true     # true/false
+indent_align_paren              = true    # true/false,
 
 # (OC) Whether to indent Objective-C blocks at brace level instead of usual
 # rules.
@@ -1299,7 +1299,8 @@ indent_ternary_operator         = 0        # unsigned number
 # If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column.
 indent_off_after_return_new     = false    # true/false
 
-# If true, the tokens after return are indented with regular single indentation. By default (false) the indentation is after the return token.
+# If true, the tokens after return are indented with regular single indentation. 
+# By default (false) the indentation is after the return token.
 indent_single_after_return      = false    # true/false
 
 # Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they
@@ -2902,7 +2903,7 @@ use_indent_continue_only_once   = false    # true/false
 #
 # true:  indentation will be used only once
 # false: indentation will be used every time (default)
-indent_cpp_lambda_only_once     = false    # true/false
+indent_cpp_lambda_only_once     = true    # true/false
 
 # Whether sp_after_angle takes precedence over sp_inside_fparen. This was the
 # historic behavior, but is probably not the desired behavior, so this is off

From 9c5ece44a7f32784dd5e2ea0faf110054f9233a8 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 13:46:15 -0600
Subject: [PATCH 2045/2196] separate msgs instead of appending to what()

---
 src/libmain/shared.cc | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 5773d90cc0a..db1e0ba1d43 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -298,23 +298,21 @@ int handleExceptions(const string & programName, std::function<void()> fun)
     } catch (Exit & e) {
         return e.status;
     } catch (UsageError & e) {
-        // TODO: switch to logError
-        // logError(e.info());
-        _printError(
-            format("%1%\nTry '%2% --help' for more information.")
-            % e.what() % programName);
+        logError(e.info());
+        printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        // logError(e.info());
-        _printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+        if (settings.showTrace && e.prefix() != "")
+            printError(e.prefix());
+        logError(e.info());
         if (e.prefix() != "" && !settings.showTrace)
-            _printError("(use '--show-trace' to show detailed location information)");
+            printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {
-        _printError(error + "out of memory");
+        printError(error + "out of memory");
         return 1;
     } catch (std::exception & e) {
-        _printError(error + e.what());
+        printError(error + e.what());
         return 1;
     }
 

From e9f10beed1a30fe80b315a9de4c4c2ab11527db8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 May 2020 22:40:02 +0200
Subject: [PATCH 2046/2196] precompiled-headers.h: Don't include our own
 headers

---
 precompiled-headers.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/precompiled-headers.h b/precompiled-headers.h
index e0d885b230c..079aa496e27 100644
--- a/precompiled-headers.h
+++ b/precompiled-headers.h
@@ -56,6 +56,3 @@
 #include <sys/wait.h>
 #include <termios.h>
 #include <unistd.h>
-
-#include "util.hh"
-#include "args.hh"

From a721a0b1140bf489d645f5d85737acafc1c57c65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 4 May 2020 22:40:19 +0200
Subject: [PATCH 2047/2196] Flag: Use designated initializers

---
 src/libexpr/common-eval-args.cc |  35 ++++----
 src/libmain/common-args.cc      |  68 ++++++++--------
 src/libmain/shared.cc           |  59 +++++++-------
 src/libstore/globals.cc         |  33 ++++----
 src/libutil/args.cc             |  44 +++++-----
 src/libutil/args.hh             | 139 +++++++++++++++++---------------
 src/libutil/config.cc           |  35 ++++----
 src/nix/add-to-store.cc         |  13 +--
 src/nix/build.cc                |  22 ++---
 src/nix/command.cc              |  77 +++++++++---------
 src/nix/copy.cc                 |  47 ++++++-----
 src/nix/dev-shell.cc            |  16 ++--
 src/nix/hash.cc                 |   8 +-
 src/nix/installables.cc         |  13 +--
 src/nix/main.cc                 |  68 +++++++++-------
 src/nix/run.cc                  |  16 ++--
 src/nix/search.cc               |  22 ++---
 src/nix/show-derivation.cc      |  11 +--
 src/nix/sigs.cc                 |  27 ++++---
 src/nix/upgrade-nix.cc          |  26 +++---
 src/nix/verify.cc               |  14 ++--
 src/nix/why-depends.cc          |  11 +--
 22 files changed, 424 insertions(+), 380 deletions(-)

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 10c9c16bbf7..44baadd53b6 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -10,24 +10,27 @@ namespace nix {
 
 MixEvalArgs::MixEvalArgs()
 {
-    mkFlag()
-        .longName("arg")
-        .description("argument to be passed to Nix functions")
-        .labels({"name", "expr"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; });
+    addFlag({
+        .longName = "arg",
+        .description = "argument to be passed to Nix functions",
+        .labels = {"name", "expr"},
+        .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
+    });
 
-    mkFlag()
-        .longName("argstr")
-        .description("string-valued argument to be passed to Nix functions")
-        .labels({"name", "string"})
-        .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; });
+    addFlag({
+        .longName = "argstr",
+        .description = "string-valued argument to be passed to Nix functions",
+        .labels = {"name", "string"},
+        .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
+    });
 
-    mkFlag()
-        .shortName('I')
-        .longName("include")
-        .description("add a path to the list of locations used to look up <...> file names")
-        .label("path")
-        .handler([&](std::string s) { searchPath.push_back(s); });
+    addFlag({
+        .longName = "include",
+        .shortName = 'I',
+        .description = "add a path to the list of locations used to look up <...> file names",
+        .labels = {"path"},
+        .handler = {[&](std::string s) { searchPath.push_back(s); }}
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 9e1d7cee60e..51e199ea5ce 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -6,43 +6,47 @@ namespace nix {
 MixCommonArgs::MixCommonArgs(const string & programName)
     : programName(programName)
 {
-    mkFlag()
-        .longName("verbose")
-        .shortName('v')
-        .description("increase verbosity level")
-        .handler([]() { verbosity = (Verbosity) (verbosity + 1); });
-
-    mkFlag()
-        .longName("quiet")
-        .description("decrease verbosity level")
-        .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; });
-
-    mkFlag()
-        .longName("debug")
-        .description("enable debug output")
-        .handler([]() { verbosity = lvlDebug; });
-
-    mkFlag()
-        .longName("option")
-        .labels({"name", "value"})
-        .description("set a Nix configuration option (overriding nix.conf)")
-        .arity(2)
-        .handler([](std::vector<std::string> ss) {
+    addFlag({
+        .longName = "verbose",
+        .shortName = 'v',
+        .description = "increase verbosity level",
+        .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
+    });
+
+    addFlag({
+        .longName = "quiet",
+        .description = "decrease verbosity level",
+        .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
+    });
+
+    addFlag({
+        .longName = "debug",
+        .description = "enable debug output",
+        .handler = {[]() { verbosity = lvlDebug; }},
+    });
+
+    addFlag({
+        .longName = "option",
+        .description = "set a Nix configuration option (overriding nix.conf)",
+        .labels = {"name", "value"},
+        .handler = {[](std::string name, std::string value) {
             try {
-                globalConfig.set(ss[0], ss[1]);
+                globalConfig.set(name, value);
             } catch (UsageError & e) {
                 warn(e.what());
             }
-        });
-
-    mkFlag()
-        .longName("max-jobs")
-        .shortName('j')
-        .label("jobs")
-        .description("maximum number of parallel builds")
-        .handler([=](std::string s) {
+        }},
+    });
+
+    addFlag({
+        .longName = "max-jobs",
+        .shortName = 'j',
+        .description = "maximum number of parallel builds",
+        .labels = Strings{"jobs"},
+        .handler = {[=](std::string s) {
             settings.set("max-jobs", s);
-        });
+        }}
+    });
 
     std::string cat = "config";
     globalConfig.convertToArgs(*this, cat);
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 8586257bfbc..70d1f0186c2 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -165,28 +165,32 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
     : MixCommonArgs(programName), parseArg(parseArg)
 {
-    mkFlag()
-        .longName("no-build-output")
-        .shortName('Q')
-        .description("do not show build output")
-        .set(&settings.verboseBuild, false);
-
-    mkFlag()
-        .longName("keep-failed")
-        .shortName('K')
-        .description("keep temporary directories of failed builds")
-        .set(&(bool&) settings.keepFailed, true);
-
-    mkFlag()
-        .longName("keep-going")
-        .shortName('k')
-        .description("keep going after a build fails")
-        .set(&(bool&) settings.keepGoing, true);
-
-    mkFlag()
-        .longName("fallback")
-        .description("build from source if substitution fails")
-        .set(&(bool&) settings.tryFallback, true);
+    addFlag({
+        .longName = "no-build-output",
+        .shortName = 'Q',
+        .description = "do not show build output",
+        .handler = {&settings.verboseBuild, false},
+    });
+
+    addFlag({
+        .longName = "keep-failed",
+        .shortName ='K',
+        .description = "keep temporary directories of failed builds",
+        .handler = {&(bool&) settings.keepFailed, true},
+    });
+
+    addFlag({
+        .longName = "keep-going",
+        .shortName ='k',
+        .description = "keep going after a build fails",
+        .handler = {&(bool&) settings.keepGoing, true},
+    });
+
+    addFlag({
+        .longName = "fallback",
+        .description = "build from source if substitution fails",
+        .handler = {&(bool&) settings.tryFallback, true},
+    });
 
     auto intSettingAlias = [&](char shortName, const std::string & longName,
         const std::string & description, const std::string & dest) {
@@ -205,11 +209,12 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
         &gcWarning, false);
 
-    mkFlag()
-        .longName("store")
-        .label("store-uri")
-        .description("URI of the Nix store to use")
-        .dest(&(std::string&) settings.storeUri);
+    addFlag({
+        .longName = "store",
+        .description = "URI of the Nix store to use",
+        .labels = {"store-uri"},
+        .handler = {&(std::string&) settings.storeUri},
+    });
 }
 
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index d6ea0318ec0..a0a2d850ec1 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -167,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
 
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description("Enable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smEnabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description("Disable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smDisabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("relaxed-" + name)
-        .description("Enable sandboxing, but allow builds to disable it.")
-        .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = "Enable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smEnabled); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = "Disable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smDisabled); }}
+    });
+    args.addFlag({
+        .longName = "relaxed-" + name,
+        .description = "Enable sandboxing, but allow builds to disable it.",
+        .category = category,
+        .handler = {[=]() { override(smRelaxed); }}
+    });
 }
 
 void MaxBuildJobsSetting::set(const std::string & str)
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ba15ea57109..91fc2f581c6 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -3,16 +3,14 @@
 
 namespace nix {
 
-Args::FlagMaker Args::mkFlag()
-{
-    return FlagMaker(*this);
-}
-
-Args::FlagMaker::~FlagMaker()
+void Args::addFlag(Flag && flag_)
 {
+    auto flag = std::make_shared<Flag>(std::move(flag_));
+    if (flag->handler.arity != ArityAny)
+        assert(flag->handler.arity == flag->labels.size());
     assert(flag->longName != "");
-    args.longFlags[flag->longName] = flag;
-    if (flag->shortName) args.shortFlags[flag->shortName] = flag;
+    longFlags[flag->longName] = flag;
+    if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
 void Args::parseCmdline(const Strings & _cmdline)
@@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
         std::vector<std::string> args;
-        for (size_t n = 0 ; n < flag.arity; ++n) {
+        for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.arity == ArityAny) break;
-                throw UsageError(format("flag '%1%' requires %2% argument(s)")
-                    % name % flag.arity);
+                if (flag.handler.arity == ArityAny) break;
+                throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             args.push_back(*pos++);
         }
-        flag.handler(std::move(args));
+        flag.handler.fun(std::move(args));
         return true;
     };
 
@@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
-Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
+Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
 {
-    arity(1);
-    label("type");
-    description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
-    handler([ht](std::string s) {
-        *ht = parseHashType(s);
-        if (*ht == htUnknown)
-            throw UsageError("unknown hash type '%1%'", s);
-    });
-    return *this;
+    return Flag {
+        .longName = std::move(longName),
+        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+        .labels = {"hash-algo"},
+        .handler = {[ht](std::string s) {
+            *ht = parseHashType(s);
+            if (*ht == htUnknown)
+                throw UsageError("unknown hash type '%1%'", s);
+        }}
+    };
 }
 
 Strings argvToStrings(int argc, char * * argv)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 967efbe1c7e..9b5e316a5b6 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -32,13 +32,59 @@ protected:
     struct Flag
     {
         typedef std::shared_ptr<Flag> ptr;
+
+        struct Handler
+        {
+            std::function<void(std::vector<std::string>)> fun;
+            size_t arity;
+
+            Handler() {}
+
+            Handler(std::function<void(std::vector<std::string>)> && fun)
+                : fun(std::move(fun))
+                , arity(ArityAny)
+            { }
+
+            Handler(std::function<void()> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
+                , arity(0)
+            { }
+
+            Handler(std::function<void(std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]));
+                  })
+                , arity(1)
+            { }
+
+            Handler(std::function<void(std::string, std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]), std::move(ss[1]));
+                  })
+                , arity(2)
+            { }
+
+            template<class T>
+            Handler(T * dest)
+                : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+                , arity(1)
+            { }
+
+            template<class T>
+            Handler(T * dest, const T & val)
+                : fun([=](std::vector<std::string> ss) { *dest = val; })
+                , arity(0)
+            { }
+        };
+
         std::string longName;
         char shortName = 0;
         std::string description;
-        Strings labels;
-        size_t arity = 0;
-        std::function<void(std::vector<std::string>)> handler;
         std::string category;
+        Strings labels;
+        Handler handler;
+
+        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
     };
 
     std::map<std::string, Flag::ptr> longFlags;
@@ -65,49 +111,7 @@ protected:
 
 public:
 
-    class FlagMaker
-    {
-        Args & args;
-        Flag::ptr flag;
-        friend class Args;
-        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }
-    public:
-        ~FlagMaker();
-        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }
-        FlagMaker & shortName(char s) { flag->shortName = s; return *this; }
-        FlagMaker & description(const std::string & s) { flag->description = s; return *this; }
-        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }
-        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }
-        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }
-        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }
-        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }
-        FlagMaker & handler(std::function<void(std::string)> handler) {
-            flag->arity = 1;
-            flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
-            return *this;
-        }
-        FlagMaker & category(const std::string & s) { flag->category = s; return *this; }
-
-        template<class T>
-        FlagMaker & dest(T * dest)
-        {
-            flag->arity = 1;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
-            return *this;
-        }
-
-        template<class T>
-        FlagMaker & set(T * dest, const T & val)
-        {
-            flag->arity = 0;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
-            return *this;
-        }
-
-        FlagMaker & mkHashTypeFlag(HashType * ht);
-    };
-
-    FlagMaker mkFlag();
+    void addFlag(Flag && flag);
 
     /* Helper functions for constructing flags / positional
        arguments. */
@@ -116,13 +120,13 @@ public:
         const std::string & label, const std::string & description,
         std::function<void(std::string)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({label})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) { fun(ss[0]); });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {label},
+            .handler = {[=](std::string s) { fun(s); }}
+        });
     }
 
     void mkFlag(char shortName, const std::string & name,
@@ -135,11 +139,12 @@ public:
     void mkFlag(char shortName, const std::string & longName, const std::string & description,
         T * dest, const T & value)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .description(description)
-            .handler([=](std::vector<std::string> ss) { *dest = value; });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .handler = {[=]() { *dest = value; }}
+        });
     }
 
     template<class I>
@@ -155,18 +160,18 @@ public:
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void(I)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({"N"})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) {
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {"N"},
+            .handler = {[=](std::string s) {
                 I n;
-                if (!string2Int(ss[0], n))
+                if (!string2Int(s, n))
                     throw UsageError("flag '--%s' requires a integer argument", longName);
                 fun(n);
-            });
+            }}
+        });
     }
 
     /* Expect a string argument. */
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 7551d97d1a7..f03e444ecc3 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -177,12 +177,13 @@ void BaseSetting<T>::toJSON(JSONPlaceholder & out)
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .arity(1)
-        .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .labels = {"value"},
+        .handler = {[=](std::string s) { overriden = true; set(s); }},
+    });
 }
 
 template<> void BaseSetting<std::string>::set(const std::string & str)
@@ -227,16 +228,18 @@ template<> std::string BaseSetting<bool>::to_string() const
 
 template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(true); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(false); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(true); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(false); }}
+    });
 }
 
 template<> void BaseSetting<Strings>::set(const std::string & str)
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index ed02227db17..00da01f7e26 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -14,12 +14,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     {
         expectArg("path", &path);
 
-        mkFlag()
-            .longName("name")
-            .shortName('n')
-            .description("name component of the store path")
-            .labels({"name"})
-            .dest(&namePart);
+        addFlag({
+            .longName = "name",
+            .shortName = 'n',
+            .description = "name component of the store path",
+            .labels = {"name"},
+            .handler = {&namePart},
+        });
     }
 
     std::string description() override
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 0b07628364c..850e09ce843 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -11,17 +11,19 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 
     CmdBuild()
     {
-        mkFlag()
-            .longName("out-link")
-            .shortName('o')
-            .description("path of the symlink to the build result")
-            .labels({"path"})
-            .dest(&outLink);
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "path of the symlink to the build result",
+            .labels = {"path"},
+            .handler = {&outLink},
+        });
 
-        mkFlag()
-            .longName("no-link")
-            .description("do not create a symlink to the build result")
-            .set(&outLink, Path(""));
+        addFlag({
+            .longName = "no-link",
+            .description = "do not create a symlink to the build result",
+            .handler = {&outLink, Path("")},
+        });
     }
 
     std::string description() override
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 99b24d2a2e3..71b02771918 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -35,16 +35,18 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
-        mkFlag()
-            .longName("no-recursive")
-            .description("apply operation to specified paths only")
-            .set(&this->recursive, false);
+        addFlag({
+            .longName = "no-recursive",
+            .description = "apply operation to specified paths only",
+            .handler = {&this->recursive, false},
+        });
     else
-        mkFlag()
-            .longName("recursive")
-            .shortName('r')
-            .description("apply operation to closure of the specified paths")
-            .set(&this->recursive, true);
+        addFlag({
+            .longName = "recursive",
+            .shortName = 'r',
+            .description = "apply operation to closure of the specified paths",
+            .handler = {&this->recursive, true},
+        });
 
     mkFlag(0, "all", "apply operation to the entire store", &all);
 }
@@ -101,11 +103,12 @@ Strings editorFor(const Pos & pos)
 
 MixProfile::MixProfile()
 {
-    mkFlag()
-        .longName("profile")
-        .description("profile to update")
-        .labels({"path"})
-        .dest(&profile);
+    addFlag({
+        .longName = "profile",
+        .description = "profile to update",
+        .labels = {"path"},
+        .handler = {&profile},
+    });
 }
 
 void MixProfile::updateProfile(const StorePath & storePath)
@@ -145,28 +148,30 @@ MixDefaultProfile::MixDefaultProfile()
     profile = getDefaultProfile();
 }
 
-MixEnvironment::MixEnvironment() : ignoreEnvironment(false) {
-    mkFlag()
-        .longName("ignore-environment")
-        .shortName('i')
-        .description("clear the entire environment (except those specified with --keep)")
-        .set(&ignoreEnvironment, true);
-
-    mkFlag()
-        .longName("keep")
-        .shortName('k')
-        .description("keep specified environment variable")
-        .arity(1)
-        .labels({"name"})
-        .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
-
-    mkFlag()
-        .longName("unset")
-        .shortName('u')
-        .description("unset specified environment variable")
-        .arity(1)
-        .labels({"name"})
-        .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
+MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
+{
+    addFlag({
+        .longName = "ignore-environment",
+        .shortName = 'i',
+        .description = "clear the entire environment (except those specified with --keep)",
+        .handler = {&ignoreEnvironment, true},
+    });
+
+    addFlag({
+        .longName = "keep",
+        .shortName = 'k',
+        .description = "keep specified environment variable",
+        .labels = {"name"},
+        .handler = {[&](std::string s) { keep.insert(s); }},
+    });
+
+    addFlag({
+        .longName = "unset",
+        .shortName = 'u',
+        .description = "unset specified environment variable",
+        .labels = {"name"},
+        .handler = {[&](std::string s) { unset.insert(s); }},
+    });
 }
 
 void MixEnvironment::setEnviron() {
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 85c777d38ab..77673a1c2d4 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -19,27 +19,32 @@ struct CmdCopy : StorePathsCommand
     CmdCopy()
         : StorePathsCommand(true)
     {
-        mkFlag()
-            .longName("from")
-            .labels({"store-uri"})
-            .description("URI of the source Nix store")
-            .dest(&srcUri);
-        mkFlag()
-            .longName("to")
-            .labels({"store-uri"})
-            .description("URI of the destination Nix store")
-            .dest(&dstUri);
-
-        mkFlag()
-            .longName("no-check-sigs")
-            .description("do not require that paths are signed by trusted keys")
-            .set(&checkSigs, NoCheckSigs);
-
-        mkFlag()
-            .longName("substitute-on-destination")
-            .shortName('s')
-            .description("whether to try substitutes on the destination store (only supported by SSH)")
-            .set(&substitute, Substitute);
+        addFlag({
+            .longName = "from",
+            .description = "URI of the source Nix store",
+            .labels = {"store-uri"},
+            .handler = {&srcUri},
+        });
+
+        addFlag({
+            .longName = "to",
+            .description = "URI of the destination Nix store",
+            .labels = {"store-uri"},
+            .handler = {&dstUri},
+        });
+
+        addFlag({
+            .longName = "no-check-sigs",
+            .description = "do not require that paths are signed by trusted keys",
+            .handler = {&checkSigs, NoCheckSigs},
+        });
+
+        addFlag({
+            .longName = "substitute-on-destination",
+            .shortName = 's',
+            .description = "whether to try substitutes on the destination store (only supported by SSH)",
+            .handler = {&substitute, Substitute},
+        });
     }
 
     std::string description() override
diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc
index 9c45a935d16..2bdf598393c 100644
--- a/src/nix/dev-shell.cc
+++ b/src/nix/dev-shell.cc
@@ -237,16 +237,16 @@ struct CmdDevShell : Common, MixEnvironment
 
     CmdDevShell()
     {
-        mkFlag()
-            .longName("command")
-            .shortName('c')
-            .description("command and arguments to be executed insted of an interactive shell")
-            .labels({"command", "args"})
-            .arity(ArityAny)
-            .handler([&](std::vector<std::string> ss) {
+        addFlag({
+            .longName = "command",
+            .shortName = 'c',
+            .description = "command and arguments to be executed insted of an interactive shell",
+            .labels = {"command", "args"},
+            .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
-            });
+            }}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 01628cf6cb5..9b9509d3ab6 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -23,9 +23,7 @@ struct CmdHash : Command
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base16);
-        mkFlag()
-            .longName("type")
-            .mkHashTypeFlag(&ht);
+        addFlag(Flag::mkHashTypeFlag("type", &ht));
         #if 0
         mkFlag()
             .longName("modulo")
@@ -76,9 +74,7 @@ struct CmdToBase : Command
 
     CmdToBase(Base base) : base(base)
     {
-        mkFlag()
-            .longName("type")
-            .mkHashTypeFlag(&ht);
+        addFlag(Flag::mkHashTypeFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 1d70ad3d554..937d692063b 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -15,12 +15,13 @@ namespace nix {
 
 SourceExprCommand::SourceExprCommand()
 {
-    mkFlag()
-        .shortName('f')
-        .longName("file")
-        .label("file")
-        .description("evaluate FILE rather than the default")
-        .dest(&file);
+    addFlag({
+        .longName = "file",
+        .shortName = 'f',
+        .description = "evaluate FILE rather than the default",
+        .labels = {"file"},
+        .handler = {&file}
+    });
 }
 
 Value * SourceExprCommand::getSourceExpr(EvalState & state)
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2c64c74767c..57b8bed9fd9 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -59,15 +59,16 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
-        mkFlag()
-            .longName("help")
-            .description("show usage information")
-            .handler([&]() { showHelpAndExit(); });
-
-        mkFlag()
-            .longName("help-config")
-            .description("show configuration options")
-            .handler([&]() {
+        addFlag({
+            .longName = "help",
+            .description = "show usage information",
+            .handler = {[&]() { showHelpAndExit(); }},
+        });
+
+        addFlag({
+            .longName = "help-config",
+            .description = "show configuration options",
+            .handler = {[&]() {
                 std::cout << "The following configuration options are available:\n\n";
                 Table2 tbl;
                 std::map<std::string, Config::SettingInfo> settings;
@@ -76,28 +77,33 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
                     tbl.emplace_back(s.first, s.second.description);
                 printTable(std::cout, tbl);
                 throw Exit();
-            });
-
-        mkFlag()
-            .longName("print-build-logs")
-            .shortName('L')
-            .description("print full build logs on stderr")
-            .set(&printBuildLogs, true);
-
-        mkFlag()
-            .longName("version")
-            .description("show version information")
-            .handler([&]() { printVersion(programName); });
-
-        mkFlag()
-            .longName("no-net")
-            .description("disable substituters and consider all previously downloaded files up-to-date")
-            .handler([&]() { useNet = false; });
-
-        mkFlag()
-            .longName("refresh")
-            .description("consider all previously downloaded files out-of-date")
-            .handler([&]() { refresh = true; });
+            }},
+        });
+
+        addFlag({
+            .longName = "print-build-logs",
+            .shortName = 'L',
+            .description = "print full build logs on stderr",
+            .handler = {&printBuildLogs, true},
+        });
+
+        addFlag({
+            .longName = "version",
+            .description = "show version information",
+            .handler = {[&]() { printVersion(programName); }},
+        });
+
+        addFlag({
+            .longName = "no-net",
+            .description = "disable substituters and consider all previously downloaded files up-to-date",
+            .handler = {[&]() { useNet = false; }},
+        });
+
+        addFlag({
+            .longName = "refresh",
+            .description = "consider all previously downloaded files out-of-date",
+            .handler = {[&]() { refresh = true; }},
+        });
     }
 
     void printFlags(std::ostream & out) override
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ebfec36d914..b888281a59c 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -63,16 +63,16 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
 
     CmdShell()
     {
-        mkFlag()
-            .longName("command")
-            .shortName('c')
-            .description("command and arguments to be executed; defaults to '$SHELL'")
-            .labels({"command", "args"})
-            .arity(ArityAny)
-            .handler([&](std::vector<std::string> ss) {
+        addFlag({
+            .longName = "command",
+            .shortName = 'c',
+            .description = "command and arguments to be executed; defaults to '$SHELL'",
+            .labels = {"command", "args"},
+            .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
-            });
+            }}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 76927454307..fcad6be84d5 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -40,16 +40,18 @@ struct CmdSearch : SourceExprCommand, MixJSON
     {
         expectArgs("regex", &res);
 
-        mkFlag()
-            .longName("update-cache")
-            .shortName('u')
-            .description("update the package search cache")
-            .handler([&]() { writeCache = true; useCache = false; });
-
-        mkFlag()
-            .longName("no-cache")
-            .description("do not use or update the package search cache")
-            .handler([&]() { writeCache = false; useCache = false; });
+        addFlag({
+            .longName = "update-cache",
+            .shortName = 'u',
+            .description = "update the package search cache",
+            .handler = {[&]() { writeCache = true; useCache = false; }}
+        });
+
+        addFlag({
+            .longName = "no-cache",
+            .description = "do not use or update the package search cache",
+            .handler = {[&]() { writeCache = false; useCache = false; }}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 0ede7b46839..b6f24599fe1 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -15,11 +15,12 @@ struct CmdShowDerivation : InstallablesCommand
 
     CmdShowDerivation()
     {
-        mkFlag()
-            .longName("recursive")
-            .shortName('r')
-            .description("include the dependencies of the specified derivations")
-            .set(&recursive, true);
+        addFlag({
+            .longName = "recursive",
+            .shortName = 'r',
+            .description = "include the dependencies of the specified derivations",
+            .handler = {&recursive, true}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 5f07448e09a..a91465c2a67 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand
 
     CmdCopySigs()
     {
-        mkFlag()
-            .longName("substituter")
-            .shortName('s')
-            .labels({"store-uri"})
-            .description("use signatures from specified store")
-            .arity(1)
-            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
+        addFlag({
+            .longName = "substituter",
+            .shortName = 's',
+            .description = "use signatures from specified store",
+            .labels = {"store-uri"},
+            .handler = {[&](std::string s) { substituterUris.push_back(s); }},
+        });
     }
 
     std::string description() override
@@ -98,12 +98,13 @@ struct CmdSignPaths : StorePathsCommand
 
     CmdSignPaths()
     {
-        mkFlag()
-            .shortName('k')
-            .longName("key-file")
-            .label("file")
-            .description("file containing the secret signing key")
-            .dest(&secretKeyFile);
+        addFlag({
+            .longName = "key-file",
+            .shortName = 'k',
+            .description = "file containing the secret signing key",
+            .labels = {"file"},
+            .handler = {&secretKeyFile}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 4fcc6a7210b..32efcc3a7ac 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -16,18 +16,20 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
     CmdUpgradeNix()
     {
-        mkFlag()
-            .longName("profile")
-            .shortName('p')
-            .labels({"profile-dir"})
-            .description("the Nix profile to upgrade")
-            .dest(&profileDir);
-
-        mkFlag()
-            .longName("nix-store-paths-url")
-            .labels({"url"})
-            .description("URL of the file that contains the store paths of the latest Nix release")
-            .dest(&storePathsUrl);
+        addFlag({
+            .longName = "profile",
+            .shortName = 'p',
+            .description = "the Nix profile to upgrade",
+            .labels = {"profile-dir"},
+            .handler = {&profileDir}
+        });
+
+        addFlag({
+            .longName = "nix-store-paths-url",
+            .description = "URL of the file that contains the store paths of the latest Nix release",
+            .labels = {"url"},
+            .handler = {&storePathsUrl}
+        });
     }
 
     std::string description() override
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 9b065880366..08a36ac5025 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -20,13 +20,13 @@ struct CmdVerify : StorePathsCommand
     {
         mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
         mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
-        mkFlag()
-            .longName("substituter")
-            .shortName('s')
-            .labels({"store-uri"})
-            .description("use signatures from specified store")
-            .arity(1)
-            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
+        addFlag({
+            .longName = "substituter",
+            .shortName = 's',
+            .description = "use signatures from specified store",
+            .labels = {"store-uri"},
+            .handler = {[&](std::string s) { substituterUris.push_back(s); }}
+        });
         mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
     }
 
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index f9acc7f13ea..36a3ee863cc 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -37,11 +37,12 @@ struct CmdWhyDepends : SourceExprCommand
         expectArg("package", &_package);
         expectArg("dependency", &_dependency);
 
-        mkFlag()
-            .longName("all")
-            .shortName('a')
-            .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path")
-            .set(&all, true);
+        addFlag({
+            .longName = "all",
+            .shortName = 'a',
+            .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path",
+            .handler = {&all, true},
+        });
     }
 
     std::string description() override

From afaa541013549c254f9c54e0865dc254973eea96 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 14:44:00 -0600
Subject: [PATCH 2048/2196] affinity operator<<

---
 src/libutil/affinity.cc | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc
index e3cf33b58ca..ac2295e4ab7 100644
--- a/src/libutil/affinity.cc
+++ b/src/libutil/affinity.cc
@@ -12,6 +12,17 @@ namespace nix {
 #if __linux__
 static bool didSaveAffinity = false;
 static cpu_set_t savedAffinity;
+
+std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset)
+{
+    auto count = CPU_COUNT(&cset);
+    for (int i=0; i < count; ++i)
+    {
+        os << (CPU_ISSET(i,&cset) ? "1" : "0");
+    }
+
+    return os;
+}
 #endif
 
 
@@ -47,7 +58,11 @@ void restoreAffinity()
 #if __linux__
     if (!didSaveAffinity) return;
     if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
-        _printError("failed to restore affinity");
+    {
+        std::ostringstream oss;
+        oss << savedAffinity;
+        printError("failed to restore CPU affinity %1%", oss.str());
+    }
 #endif
 }
 

From 8c8f2b74ec3fe37f8661c8411749965b6e8d44d2 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 14:44:42 -0600
Subject: [PATCH 2049/2196] log as warning

---
 src/nix-store/nix-store.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 2e2276b4a80..57063d42fd5 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -841,8 +841,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                             for (auto & p : willSubstitute) subs.emplace_back(p.clone());
                             store->buildPaths(subs);
                         } catch (Error & e) {
-                            // logWarning(e.info())  TODO:  
-                            _printError("warning: %1%", e.msg());
+                            logWarning(e.info());
                         }
                 }
 

From e2fc575c6122b40f5d660ae04f269e118354c101 Mon Sep 17 00:00:00 2001
From: Jude Taylor <jude.taylor@dfinity.org>
Date: Mon, 4 May 2020 14:42:06 -0700
Subject: [PATCH 2050/2196] nix auto-gc: use fragment size

---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 6bab1e37c90..d210defe57e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -889,7 +889,7 @@ void LocalStore::autoGC(bool sync)
         if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
-        return (uint64_t) st.f_bavail * st.f_bsize;
+        return (uint64_t) st.f_bavail * st.f_frsize;
     };
 
     std::shared_future<void> future;

From f30de61578edd9c19744256985185045e7baab84 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 16:19:20 -0600
Subject: [PATCH 2051/2196] add normaltxt, yellowify->yellowtxt

---
 src/libutil/fmt.hh | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index d182adc3a14..bfacfeb4de0 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -73,18 +73,31 @@ inline std::string fmt(const std::string & fs, const Args & ... args)
 // are always in yellow.
 
 template <class T>
-struct yellowify
+struct yellowtxt
 {
-    yellowify(T &s) : value(s) {}
+    yellowtxt(T &s) : value(s) {}
     T &value;
 };
 
 template <class T>
-std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
+std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y)
 {
     return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
 }
 
+template <class T>
+struct normaltxt
+{
+    normaltxt(T &s) : value(s) {}
+    T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y)
+{
+    return out << ANSI_NORMAL << y.value;
+}
+
 class hintformat
 {
 public:
@@ -100,7 +113,7 @@ public:
     template<class T>
     hintformat& operator%(const T &value)
     {
-        fmt % yellowify(value);
+        fmt % yellowtxt(value);
         return *this;
     }
 

From 7ffb5efdbc943851d2ee9d0573dca3e96b9bd742 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 4 May 2020 16:19:57 -0600
Subject: [PATCH 2052/2196] appending to hints; remove _printError

---
 src/error-demo/error-demo.cc |  7 +++++--
 src/libstore/build.cc        | 12 ++++++++----
 src/libutil/logging.hh       |  1 -
 src/nix-daemon/nix-daemon.cc |  7 +++++--
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 5142584160e..82b03d71b92 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -26,10 +26,13 @@ int main()
 
     // ErrorInfo constructor
     try {
-        auto e = Error("generic error");
+        auto e = Error("some error");
         throw DemoError(e.info());
     } catch (Error &e) {
-        logger->logEI(e.info());
+        ErrorInfo ei = e.info();
+        string prevhint = (e.info().hint.has_value() ? e.info().hint->str() : "");
+        ei.hint = std::optional(hintfmt("previous hint was: %s", normaltxt(prevhint)));
+        logger->logEI(ei);
     }
 
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index bacbd5808f6..f8cc1ce369e 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -488,14 +488,18 @@ void handleDiffHook(
 
             auto diffRes = runProgram(diffHookOptions);
             if (!statusOk(diffRes.first))
-                throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first));
+                throw ExecError(diffRes.first,
+                    "diff-hook program '%1%' %2%",
+                    diffHook,
+                    statusToString(diffRes.first));
 
             if (diffRes.second != "")
                 printError(chomp(diffRes.second));
         } catch (Error & error) {
-            // logError(error.info())
-            // TODO append message onto errorinfo...
-            _printError("diff hook execution failed: %s", error.what());
+            ErrorInfo ei = error.info();
+            string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : "");
+            ei.hint = std::optional(hintfmt("diff hook execution failed: %s", prevhint));
+            logError(ei);
         }
     }
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5f03cdf41cc..0c4980b835a 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -149,7 +149,6 @@ extern Verbosity verbosity; /* suppress msgs > this */
         } \
     } while (0)
 
-#define _printError(args...) printMsg(lvlError, args)
 #define printError(args...) printMsg(lvlError, args)
 #define printInfo(args...) printMsg(lvlInfo, args)
 #define printTalkative(args...) printMsg(lvlTalkative, args)
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 3cd2c3a6b31..5336db1364f 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -246,9 +246,12 @@ static void daemonLoop(char * * argv)
 
         } catch (Interrupted & e) {
             return;
-        } catch (Error & e) {
+        } catch (Error & error) {
             // TODO append error message
-            _printError("error processing connection: %1%", e.msg());
+            ErrorInfo ei = error.info();
+            string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : "");
+            ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint));
+            logError(ei);
         }
     }
 }

From 04967dee9d08befe4661e6fa5da4a00da0538a13 Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Tue, 5 May 2020 13:04:36 +0300
Subject: [PATCH 2053/2196] Wait for build users when none are available

---
 src/libstore/build.cc | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 147093fae26..1b7e6d75e94 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -507,6 +507,9 @@ class UserLock
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
+    bool findFreeUser();
+
+
     string user;
     uid_t uid;
     gid_t gid;
@@ -526,10 +529,19 @@ class UserLock
 
 };
 
-
 UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
+    createDirs(settings.nixStateDir + "/userpool");
+
+    if (findFreeUser()) return;
+
+    printError("waiting for build users");
+
+    do std::this_thread::sleep_for(std::chrono::seconds(2)); while (! findFreeUser());
+}
+
+bool UserLock::findFreeUser() {
 
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -559,7 +571,6 @@ UserLock::UserLock()
             throw Error(format("the user '%1%' in the group '%2%' does not exist")
                 % i % settings.buildUsersGroup);
 
-        createDirs(settings.nixStateDir + "/userpool");
 
         fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
 
@@ -590,16 +601,12 @@ UserLock::UserLock()
             supplementaryGIDs.resize(ngroups);
 #endif
 
-            return;
+            return true;
         }
     }
-
-    throw Error(format("all build users are currently in use; "
-        "consider creating additional users and adding them to the '%1%' group")
-        % settings.buildUsersGroup);
+    return false;
 }
 
-
 void UserLock::kill()
 {
     killUser(uid);

From f132d82a796c91fcb741c127f37c963622b4cae4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 May 2020 15:18:23 +0200
Subject: [PATCH 2054/2196] nix --help: Group commands

---
 src/libutil/ansicolor.hh            | 24 ++++++------
 src/libutil/args.cc                 | 57 ++++++++++++++++-------------
 src/libutil/args.hh                 | 18 +++++----
 src/nix/add-to-store.cc             |  2 +
 src/nix/cat.cc                      |  8 +++-
 src/nix/command.hh                  |  4 ++
 src/nix/copy.cc                     |  2 +
 src/nix/dev-shell.cc                |  2 +
 src/nix/doctor.cc                   |  2 +
 src/nix/dump-path.cc                |  2 +
 src/nix/edit.cc                     |  2 +
 src/nix/eval.cc                     |  2 +
 src/nix/hash.cc                     |  4 ++
 src/nix/log.cc                      |  2 +
 src/nix/ls.cc                       |  8 +++-
 src/nix/main.cc                     | 18 ++++++---
 src/nix/make-content-addressable.cc |  3 ++
 src/nix/optimise-store.cc           |  2 +
 src/nix/path-info.cc                |  2 +
 src/nix/ping-store.cc               |  2 +
 src/nix/show-config.cc              |  2 +
 src/nix/show-derivation.cc          |  2 +
 src/nix/sigs.cc                     |  4 ++
 src/nix/upgrade-nix.cc              |  2 +
 src/nix/verify.cc                   |  2 +
 src/nix/why-depends.cc              |  2 +
 26 files changed, 125 insertions(+), 55 deletions(-)

diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
index 390bd4d17c3..8ae07b0924e 100644
--- a/src/libutil/ansicolor.hh
+++ b/src/libutil/ansicolor.hh
@@ -1,13 +1,15 @@
-#pragma once 
+#pragma once
+
+namespace nix {
+
+/* Some ANSI escape sequences. */
+#define ANSI_NORMAL "\e[0m"
+#define ANSI_BOLD "\e[1m"
+#define ANSI_FAINT "\e[2m"
+#define ANSI_ITALIC "\e[3m"
+#define ANSI_RED "\e[31;1m"
+#define ANSI_GREEN "\e[32;1m"
+#define ANSI_YELLOW "\e[33;1m"
+#define ANSI_BLUE "\e[34;1m"
 
-namespace nix 
-{
-  /* Some ANSI escape sequences. */
-  #define ANSI_NORMAL "\e[0m"
-  #define ANSI_BOLD "\e[1m"
-  #define ANSI_FAINT "\e[2m"
-  #define ANSI_RED "\e[31;1m"
-  #define ANSI_GREEN "\e[32;1m"
-  #define ANSI_YELLOW "\e[33;1m"
-  #define ANSI_BLUE "\e[34;1m"
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 91fc2f581c6..f829415d106 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -59,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline)
 
 void Args::printHelp(const string & programName, std::ostream & out)
 {
-    std::cout << "Usage: " << programName << " <FLAGS>...";
+    std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
     for (auto & exp : expectedArgs) {
         std::cout << renderLabels({exp.label});
         // FIXME: handle arity > 1
@@ -70,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out)
 
     auto s = description();
     if (s != "")
-        std::cout << "\nSummary: " << s << ".\n";
+        std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
 
     if (longFlags.size()) {
         std::cout << "\n";
-        std::cout << "Flags:\n";
+        std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
         printFlags(out);
     }
 }
@@ -181,7 +181,7 @@ std::string renderLabels(const Strings & labels)
     std::string res;
     for (auto label : labels) {
         for (auto & c : label) c = std::toupper(c);
-        res += " <" + label + ">";
+        res += " " ANSI_ITALIC + label + ANSI_NORMAL;
     }
     return res;
 }
@@ -190,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table)
 {
     size_t max = 0;
     for (auto & row : table)
-        max = std::max(max, row.first.size());
+        max = std::max(max, filterANSIEscapes(row.first, true).size());
     for (auto & row : table) {
         out << "  " << row.first
-            << std::string(max - row.first.size() + 2, ' ')
+            << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
             << row.second << "\n";
     }
 }
@@ -204,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out)
 
     auto exs = examples();
     if (!exs.empty()) {
-        out << "\n";
-        out << "Examples:\n";
+        out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
         for (auto & ex : exs)
             out << "\n"
                 << "  " << ex.description << "\n" // FIXME: wrap
@@ -221,49 +220,55 @@ MultiCommand::MultiCommand(const Commands & commands)
         auto i = commands.find(ss[0]);
         if (i == commands.end())
             throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = i->second();
-        command->_name = ss[0];
+        command = {ss[0], i->second()};
     }});
+
+    categories[Command::catDefault] = "Available commands";
 }
 
 void MultiCommand::printHelp(const string & programName, std::ostream & out)
 {
     if (command) {
-        command->printHelp(programName + " " + command->name(), out);
+        command->second->printHelp(programName + " " + command->first, out);
         return;
     }
 
-    out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n";
+    out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
 
-    out << "\n";
-    out << "Common flags:\n";
+    out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
     printFlags(out);
 
-    out << "\n";
-    out << "Available commands:\n";
+    std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
 
-    Table2 table;
-    for (auto & i : commands) {
-        auto command = i.second();
-        command->_name = i.first;
-        auto descr = command->description();
-        if (!descr.empty())
-            table.push_back(std::make_pair(command->name(), descr));
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        commandsByCategory[command->category()].insert_or_assign(name, command);
+    }
+
+    for (auto & [category, commands] : commandsByCategory) {
+        out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
+
+        Table2 table;
+        for (auto & [name, command] : commands) {
+            auto descr = command->description();
+            if (!descr.empty())
+                table.push_back(std::make_pair(name, descr));
+        }
+        printTable(out, table);
     }
-    printTable(out, table);
 }
 
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     if (Args::processFlag(pos, end)) return true;
-    if (command && command->processFlag(pos, end)) return true;
+    if (command && command->second->processFlag(pos, end)) return true;
     return false;
 }
 
 bool MultiCommand::processArgs(const Strings & args, bool finish)
 {
     if (command)
-        return command->processArgs(args, finish);
+        return command->second->processArgs(args, finish);
     else
         return Args::processArgs(args, finish);
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 9b5e316a5b6..1932e6a8ae4 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -197,17 +197,10 @@ public:
    run() method. */
 struct Command : virtual Args
 {
-private:
-    std::string _name;
-
     friend class MultiCommand;
 
-public:
-
     virtual ~Command() { }
 
-    std::string name() { return _name; }
-
     virtual void prepare() { };
     virtual void run() = 0;
 
@@ -221,6 +214,12 @@ public:
 
     virtual Examples examples() { return Examples(); }
 
+    typedef int Category;
+
+    static constexpr Category catDefault = 0;
+
+    virtual Category category() { return catDefault; }
+
     void printHelp(const string & programName, std::ostream & out) override;
 };
 
@@ -233,7 +232,10 @@ class MultiCommand : virtual Args
 public:
     Commands commands;
 
-    std::shared_ptr<Command> command;
+    std::map<Command::Category, std::string> categories;
+
+    // Selected command, if any.
+    std::optional<std::pair<std::string, ref<Command>>> command;
 
     MultiCommand(const Commands & commands);
 
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 00da01f7e26..1d298903bf0 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -34,6 +34,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         if (!namePart) namePart = baseNameOf(path);
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 851f90abd1b..fd91f20362b 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -30,9 +30,11 @@ struct CmdCatStore : StoreCommand, MixCat
 
     std::string description() override
     {
-        return "print the contents of a store file on stdout";
+        return "print the contents of a file in the Nix store on stdout";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         cat(store->getFSAccessor());
@@ -51,9 +53,11 @@ struct CmdCatNar : StoreCommand, MixCat
 
     std::string description() override
     {
-        return "print the contents of a file inside a NAR file";
+        return "print the contents of a file inside a NAR file on stdout";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         cat(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
diff --git a/src/nix/command.hh b/src/nix/command.hh
index bf43d950f6a..959d5f19dbc 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -10,6 +10,10 @@ namespace nix {
 
 extern std::string programPath;
 
+static constexpr Command::Category catSecondary = 100;
+static constexpr Command::Category catUtility = 101;
+static constexpr Command::Category catNixInstallation = 102;
+
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 77673a1c2d4..c7c38709d19 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -80,6 +80,8 @@ struct CmdCopy : StorePathsCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     ref<Store> createStore() override
     {
         return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc
index 2bdf598393c..d300f6a230f 100644
--- a/src/nix/dev-shell.cc
+++ b/src/nix/dev-shell.cc
@@ -328,6 +328,8 @@ struct CmdPrintDevEnv : Common
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         auto buildEnvironment = getBuildEnvironment(store).first;
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 0aa634d6e0f..cc36354f17f 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -43,6 +43,8 @@ struct CmdDoctor : StoreCommand
         return "check your system for potential problems and print a PASS or FAIL for each check.";
     }
 
+    Category category() override { return catNixInstallation; }
+
     void run(ref<Store> store) override
     {
         logger->log("Running checks against store uri: " + store->getUri());
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index bb741b57202..e1de71bf830 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -20,6 +20,8 @@ struct CmdDumpPath : StorePathCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, const StorePath & storePath) override
     {
         FdSink sink(STDOUT_FILENO);
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 1683eada0b8..067d3a9738b 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -25,6 +25,8 @@ struct CmdEdit : InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         auto state = getEvalState();
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 86a1e8b681f..26e98ac2a0f 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -45,6 +45,8 @@ struct CmdEval : MixJSON, InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         if (raw && json)
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 9b9509d3ab6..366314227c8 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -41,6 +41,8 @@ struct CmdHash : Command
             : "print cryptographic hash of the NAR serialisation of a path";
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         for (auto path : paths) {
@@ -87,6 +89,8 @@ struct CmdToBase : Command
             "SRI");
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         for (auto s : args)
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 795991cb783..3fe22f6c27a 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -31,6 +31,8 @@ struct CmdLog : InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         settings.readOnlyMode = true;
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 8590199d73b..6ae4df27ebe 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -100,9 +100,11 @@ struct CmdLsStore : StoreCommand, MixLs
 
     std::string description() override
     {
-        return "show information about a store path";
+        return "show information about a path in the Nix store";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         list(store->getFSAccessor());
@@ -131,9 +133,11 @@ struct CmdLsNar : Command, MixLs
 
     std::string description() override
     {
-        return "show information about the contents of a NAR file";
+        return "show information about a path inside a NAR file";
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true))));
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 57b8bed9fd9..5cf09c4f056 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -59,6 +59,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
+        categories.clear();
+        categories[Command::catDefault] = "Main commands";
+        categories[catSecondary] = "Infrequently used commands";
+        categories[catUtility] = "Utility/scripting commands";
+        categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation";
+
         addFlag({
             .longName = "help",
             .description = "show usage information",
@@ -111,8 +117,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         Args::printFlags(out);
         std::cout <<
             "\n"
-            "In addition, most configuration settings can be overriden using '--<name> <value>'.\n"
-            "Boolean settings can be overriden using '--<name>' or '--no-<name>'. See 'nix\n"
+            "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n"
+            "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n"
             "--help-config' for a list of configuration settings.\n";
     }
 
@@ -121,10 +127,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         MultiCommand::printHelp(programName, out);
 
 #if 0
-        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
+        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n";
 #endif
 
-        std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
+        std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n";
     }
 
     void showHelpAndExit()
@@ -191,8 +197,8 @@ void mainWrapped(int argc, char * * argv)
     if (args.refresh)
         settings.tarballTtl = 0;
 
-    args.command->prepare();
-    args.command->run();
+    args.command->second->prepare();
+    args.command->second->run();
 }
 
 }
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index f9c7fef3f01..8803461f48f 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -31,6 +31,9 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             },
         };
     }
+
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         auto paths = store->topoSortPaths(storePathsToSet(storePaths));
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index fed012b0454..b4595187929 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -23,6 +23,8 @@ struct CmdOptimiseStore : StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         store->optimiseStore();
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 45ec297d2fb..88d7fffd445 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -29,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         return "query information about store paths";
     }
 
+    Category category() override { return catSecondary; }
+
     Examples examples() override
     {
         return {
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 3a2e542a38e..127397a292c 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -21,6 +21,8 @@ struct CmdPingStore : StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         store->connect();
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 6104b10bcd1..4fd8886de08 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -13,6 +13,8 @@ struct CmdShowConfig : Command, MixJSON
         return "show the Nix configuration";
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         if (json) {
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index b6f24599fe1..22c569f3cdd 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -42,6 +42,8 @@ struct CmdShowDerivation : InstallablesCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         auto drvPaths = toDerivations(store, installables, true);
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index a91465c2a67..6c9b9a79286 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -27,6 +27,8 @@ struct CmdCopySigs : StorePathsCommand
         return "copy path signatures from substituters (like binary caches)";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (substituterUris.empty())
@@ -112,6 +114,8 @@ struct CmdSignPaths : StorePathsCommand
         return "sign the specified paths";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (secretKeyFile.empty())
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 32efcc3a7ac..678780f333f 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -51,6 +51,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         };
     }
 
+    Category category() override { return catNixInstallation; }
+
     void run(ref<Store> store) override
     {
         evalSettings.pureEval = true;
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 08a36ac5025..cf1fa6a990c 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -49,6 +49,8 @@ struct CmdVerify : StorePathsCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         std::vector<ref<Store>> substituters;
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 36a3ee863cc..6057beedb64 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -68,6 +68,8 @@ struct CmdWhyDepends : SourceExprCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         auto package = parseInstallable(*this, store, _package, false);

From 909b4a882057a92ae8b40b8afdab3c4ac30da915 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 May 2020 15:27:47 +0200
Subject: [PATCH 2055/2196] nix doctor: Consistency

---
 src/nix/doctor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index cc36354f17f..82e92cdd0a9 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -40,7 +40,7 @@ struct CmdDoctor : StoreCommand
 
     std::string description() override
     {
-        return "check your system for potential problems and print a PASS or FAIL for each check.";
+        return "check your system for potential problems and print a PASS or FAIL for each check";
     }
 
     Category category() override { return catNixInstallation; }

From fd4911269f9a9ce0f9647c5fb88658074f654eff Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 May 2020 10:54:01 +0200
Subject: [PATCH 2056/2196] Revert "Merge pull request #3558 from
 LnL7/ssh-ng-stderr"

This reverts commit 3ebfbecdd187002569257f7cb183bf9e0b39af1e, reversing
changes made to c089c52d5f1cff888552f485775b74226dcbe618.

https://github.com/NixOS/nix/pull/3558
---
 src/libstore/remote-store.cc |  6 ++----
 src/libutil/logging.cc       | 10 ----------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 07ef79382d0..8c55da2686e 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -779,10 +779,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
             return std::make_exception_ptr(Error(status, error));
         }
 
-        else if (msg == STDERR_NEXT) {
-            string s = chomp(readString(from));
-            printMsg(lvlVomit, "stderr %s", s);
-        }
+        else if (msg == STDERR_NEXT)
+            printError(chomp(readString(from)));
 
         else if (msg == STDERR_START_ACTIVITY) {
             auto act = readNum<ActivityId>(from);
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 777650de5cd..3cc4ef8f15b 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -63,16 +63,6 @@ class SimpleLogger : public Logger
         writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n");
     }
 
-    void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
-    {
-        if (type == resBuildLogLine || type == resPostBuildLogLine) {
-            assert(0 < fields.size());
-            assert(fields[0].type == Logger::Field::tString);
-            auto lastLine = fields[0].s;
-            log(lvlInfo, lastLine);
-        }
-    }
-
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
         const std::string & s, const Fields & fields, ActivityId parent)
         override

From 9be46859a90d5a0771e079615d324644a08efb41 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Wed, 6 May 2020 11:21:12 +0200
Subject: [PATCH 2057/2196] libstore/build.cc: more explicit about form of
 output

Be more explicit about why we expect a regular file as output
when outputHashMode=flat for a fixed output derivation.
---
 src/libstore/build.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 57263476573..e0c7324c744 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3678,7 +3678,8 @@ void DerivationGoal::registerOutputs()
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
-                        format("output path '%1%' should be a non-executable regular file") % path);
+                        format("output path '%1%' should be a non-executable regular file "
+                               "since recursive hashing is not enabled (outputHashMode=flat).") % path);
             }
 
             /* Check the hash. In hash mode, move the path produced by

From 85c1932c9425c15d8d16da4777ed27667874a744 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 5 May 2020 12:09:46 +0100
Subject: [PATCH 2058/2196] nix/search: no error for empty search results if
 json is enabled

- result list will be always empty if --json is passed
- for scripts an empty search result is not really an error,
  we rather want to distinguish between evaluation errors and empty results
---
 src/nix/search.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/search.cc b/src/nix/search.cc
index fcad6be84d5..ba72c1e799f 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -265,7 +265,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
 
-        if (results.size() == 0)
+        if (!json && results.size() == 0)
             throw Error("no results for the given search term(s)!");
 
         RunPager pager;

From 58ed1e6d6842688ac2d32355265e0d89049f7e36 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Tue, 5 May 2020 17:56:46 +0200
Subject: [PATCH 2059/2196] WIP: add unit tests for libutil

This is a proof on concept to evaluate writing unit tests for Nix using
google test (https://github.com/google/googletest).

In order to execute tests:

$ make unit-tests
$ ./unit-tests

The Makefile rules for `unit-tests` is a complete hack.
---
 Makefile                      |   3 +-
 release-common.nix            |   1 +
 tests/unit-tests/local.mk     |   4 +
 tests/unit-tests/test-util.cc | 594 ++++++++++++++++++++++++++++++++++
 4 files changed, 601 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit-tests/local.mk
 create mode 100644 tests/unit-tests/test-util.cc

diff --git a/Makefile b/Makefile
index e3057c36ca2..06de35a2709 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,8 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk
+  tests/plugins/local.mk \
+  tests/unit-tests/local.mk
 
 -include Makefile.config
 
diff --git a/release-common.nix b/release-common.nix
index b6c8f3d818d..7e7de005d2f 100644
--- a/release-common.nix
+++ b/release-common.nix
@@ -55,6 +55,7 @@ rec {
       # Tests
       git
       mercurial
+      gmock
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk
new file mode 100644
index 00000000000..ed11261c1a0
--- /dev/null
+++ b/tests/unit-tests/local.mk
@@ -0,0 +1,4 @@
+LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec
+unit-tests:
+	echo $(nix_LDFLAGS)
+	$(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc
diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc
new file mode 100644
index 00000000000..05d8a0039d9
--- /dev/null
+++ b/tests/unit-tests/test-util.cc
@@ -0,0 +1,594 @@
+#include "../../src/libutil/util.hh"
+#include "../../src/libutil/types.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * absPath
+     * --------------------------------------------------------------------------*/
+
+    TEST(absPath, doesntChangeRoot) {
+        auto p = absPath("/");
+
+        ASSERT_STREQ(p.c_str(), "/");
+    }
+
+    TEST(absPath, turnsEmptyPathIntoCWD) {
+        char cwd[PATH_MAX+1];
+        auto p = absPath("");
+
+        ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX));
+    }
+
+    TEST(absPath, usesOptionalBasePathWhenGiven) {
+        char _cwd[PATH_MAX+1];
+        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
+
+        auto p = absPath("", cwd);
+
+        ASSERT_STREQ(p.c_str(), cwd);
+    }
+
+    TEST(absPath, isIdempotent) {
+        char _cwd[PATH_MAX+1];
+        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
+        auto p1 = absPath(cwd);
+        auto p2 = absPath(p1);
+
+        ASSERT_STREQ(p1.c_str(), p2.c_str());
+    }
+
+
+    TEST(absPath, pathIsCanonicalised) {
+        auto path = "/some/path/with/trailing/dot/.";
+        auto p1 = absPath(path);
+        auto p2 = absPath(p1);
+
+        ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot");
+        ASSERT_STREQ(p1.c_str(), p2.c_str());
+    }
+
+    /* ----------------------------------------------------------------------------
+     * canonPath
+     * --------------------------------------------------------------------------*/
+
+    TEST(canonPath, removesTrailingSlashes) {
+        auto path = "/this/is/a/path//";
+        auto p = canonPath(path);
+
+        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+    }
+
+    TEST(canonPath, removesDots) {
+        auto path = "/this/./is/a/path/./";
+        auto p = canonPath(path);
+
+        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+    }
+
+    TEST(canonPath, removesDots2) {
+        auto path = "/this/a/../is/a////path/foo/..";
+        auto p = canonPath(path);
+
+        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+    }
+
+    TEST(canonPath, requiresAbsolutePath) {
+        ASSERT_ANY_THROW(canonPath("."));
+        ASSERT_ANY_THROW(canonPath(".."));
+        ASSERT_ANY_THROW(canonPath("../"));
+        ASSERT_DEATH({ canonPath(""); }, "path != \"\"");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * dirOf
+     * --------------------------------------------------------------------------*/
+
+    // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo")
+    // should both return "" but it actually returns "/" in both cases
+    TEST(dirOf, DISABLED_returnsEmptyStringForRoot) {
+        auto p = dirOf("/");
+
+        ASSERT_STREQ(p.c_str(), "");
+    }
+
+    TEST(dirOf, returnsFirstPathComponent) {
+        auto p1 = dirOf("/dir/");
+        ASSERT_STREQ(p1.c_str(), "/dir");
+        auto p2 = dirOf("/dir");
+        ASSERT_STREQ(p2.c_str(), "/");
+        auto p3 = dirOf("/dir/..");
+        ASSERT_STREQ(p3.c_str(), "/dir");
+        auto p4 = dirOf("/dir/../");
+        ASSERT_STREQ(p4.c_str(), "/dir/..");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * baseNameOf
+     * --------------------------------------------------------------------------*/
+
+    TEST(baseNameOf, emptyPath) {
+        auto p1 = baseNameOf("");
+        ASSERT_STREQ(std::string(p1).c_str(), "");
+    }
+
+    TEST(baseNameOf, pathOnRoot) {
+        auto p1 = baseNameOf("/dir");
+        ASSERT_STREQ(std::string(p1).c_str(), "dir");
+    }
+
+    TEST(baseNameOf, relativePath) {
+        auto p1 = baseNameOf("dir/foo");
+        ASSERT_STREQ(std::string(p1).c_str(), "foo");
+    }
+
+    TEST(baseNameOf, pathWithTrailingSlashRoot) {
+        auto p1 = baseNameOf("/");
+        ASSERT_STREQ(std::string(p1).c_str(), "");
+    }
+
+    // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return
+    // "" but it actually returns "dir"
+    TEST(baseNameOf, DISABLED_trailingSlash) {
+        auto p1 = baseNameOf("/dir/");
+        ASSERT_STREQ(std::string(p1).c_str(), "");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * isInDir
+     * --------------------------------------------------------------------------*/
+
+    TEST(isInDir, trivialCase) {
+        auto p1 = isInDir("/foo/bar", "/foo");
+        ASSERT_EQ(p1, true);
+    }
+
+    TEST(isInDir, notInDir) {
+        auto p1 = isInDir("/zes/foo/bar", "/foo");
+        ASSERT_EQ(p1, false);
+    }
+
+    // XXX: hm, bug or feature? :) Looking at the implementation
+    // this might be problematic.
+    TEST(isInDir, emptyDir) {
+        auto p1 = isInDir("/zes/foo/bar", "");
+        ASSERT_EQ(p1, true);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * isDirOrInDir
+     * --------------------------------------------------------------------------*/
+
+    TEST(isDirOrInDir, trueForSameDirectory) {
+        ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true);
+        ASSERT_EQ(isDirOrInDir("/", "/"), true);
+    }
+
+    TEST(isDirOrInDir, trueForEmptyPaths) {
+        ASSERT_EQ(isDirOrInDir("", ""), true);
+    }
+
+    TEST(isDirOrInDir, falseForDisjunctPaths) {
+        ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false);
+    }
+
+    TEST(isDirOrInDir, relativePaths) {
+        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true);
+    }
+
+    // XXX: while it is possible to use "." or ".." in the
+    // first argument this doesn't seem to work in the second.
+    TEST(isDirOrInDir, DISABLED_shouldWork) {
+        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true);
+
+    }
+
+    /* ----------------------------------------------------------------------------
+     * pathExists
+     * --------------------------------------------------------------------------*/
+
+    TEST(pathExists, rootExists) {
+        ASSERT_TRUE(pathExists("/"));
+    }
+
+    TEST(pathExists, cwdExists) {
+        ASSERT_TRUE(pathExists("."));
+    }
+
+    TEST(pathExists, bogusPathDoesNotExist) {
+        ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * concatStringsSep
+     * --------------------------------------------------------------------------*/
+
+    TEST(concatStringsSep, buildCommaSeparatedString) {
+        Strings strings;
+        strings.push_back("this");
+        strings.push_back("is");
+        strings.push_back("great");
+
+        ASSERT_EQ(concatStringsSep(",", strings), "this,is,great");
+    }
+
+    TEST(concatStringsSep, buildStringWithEmptySeparator) {
+        Strings strings;
+        strings.push_back("this");
+        strings.push_back("is");
+        strings.push_back("great");
+
+        ASSERT_EQ(concatStringsSep("", strings), "thisisgreat");
+    }
+
+    TEST(concatStringsSep, buildSingleString) {
+        Strings strings;
+        strings.push_back("this");
+
+        ASSERT_EQ(concatStringsSep(",", strings), "this");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * hasPrefix
+     * --------------------------------------------------------------------------*/
+
+    TEST(hasPrefix, emptyStringHasNoPrefix) {
+        ASSERT_FALSE(hasPrefix("", "foo"));
+    }
+
+    TEST(hasPrefix, emptyStringIsAlwaysPrefix) {
+        ASSERT_TRUE(hasPrefix("foo", ""));
+        ASSERT_TRUE(hasPrefix("jshjkfhsadf", ""));
+    }
+
+    TEST(hasPrefix, trivialCase) {
+        ASSERT_TRUE(hasPrefix("foobar", "foo"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * hasSuffix
+     * --------------------------------------------------------------------------*/
+
+    TEST(hasSuffix, emptyStringHasNoSuffix) {
+        ASSERT_FALSE(hasSuffix("", "foo"));
+    }
+
+    TEST(hasSuffix, trivialCase) {
+        ASSERT_TRUE(hasSuffix("foo", "foo"));
+        ASSERT_TRUE(hasSuffix("foobar", "bar"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * base64Encode
+     * --------------------------------------------------------------------------*/
+
+    TEST(base64Encode, emptyString) {
+        ASSERT_STREQ(base64Encode("").c_str(), "");
+    }
+
+    TEST(base64Encode, encodesAString) {
+        ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
+    }
+
+    TEST(base64Encode, encodeAndDecode) {
+        auto s = "quod erat demonstrandum";
+        auto encoded = base64Encode(s);
+        auto decoded = base64Decode(encoded);
+
+        ASSERT_STREQ(decoded.c_str(), s);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * base64Decode
+     * --------------------------------------------------------------------------*/
+
+    TEST(base64Decode, emptyString) {
+        ASSERT_STREQ(base64Decode("").c_str(), "");
+    }
+
+    TEST(base64Decode, decodeAString) {
+        ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * toLower
+     * --------------------------------------------------------------------------*/
+
+    TEST(toLower, emptyString) {
+        ASSERT_STREQ(toLower("").c_str(), "");
+    }
+
+    TEST(toLower, nonLetters) {
+        auto s = "!@(*$#)(@#=\\234_";
+        ASSERT_STREQ(toLower(s).c_str(), s);
+    }
+
+    // XXX: std::tolower() doesn't cover this. This probably doesn't matter
+    // since the context is paths and store paths specifically where such
+    // characters are not allowed.
+    TEST(toLower, DISABLED_umlauts) {
+        auto s = "ÄÖÜ";
+        ASSERT_STREQ(toLower(s).c_str(), "äöü");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * string2Float
+     * --------------------------------------------------------------------------*/
+
+    TEST(string2Float, emptyString) {
+        double n;
+        ASSERT_EQ(string2Float("", n), false);
+    }
+
+    TEST(string2Float, trivialConversions) {
+        double n;
+        ASSERT_EQ(string2Float("1.0", n), true);
+        ASSERT_EQ(n, 1.0);
+
+        ASSERT_EQ(string2Float("0.0", n), true);
+        ASSERT_EQ(n, 0.0);
+
+        ASSERT_EQ(string2Float("-100.25", n), true);
+        ASSERT_EQ(n, (-100.25));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * string2Int
+     * --------------------------------------------------------------------------*/
+
+    TEST(string2Int, emptyString) {
+        double n;
+        ASSERT_EQ(string2Int("", n), false);
+    }
+
+    TEST(string2Int, trivialConversions) {
+        double n;
+        ASSERT_EQ(string2Int("1", n), true);
+        ASSERT_EQ(n, 1);
+
+        ASSERT_EQ(string2Int("0", n), true);
+        ASSERT_EQ(n, 0);
+
+        ASSERT_EQ(string2Int("-100", n), true);
+        ASSERT_EQ(n, (-100));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * statusOk
+     * --------------------------------------------------------------------------*/
+
+    TEST(statusOk, zeroIsOk) {
+        ASSERT_EQ(statusOk(0), true);
+        ASSERT_EQ(statusOk(1), false);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * replaceInSet : XXX: this function isn't called anywhere!
+     * --------------------------------------------------------------------------*/
+
+
+    /* ----------------------------------------------------------------------------
+     * rewriteStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(rewriteStrings, emptyString) {
+        StringMap rewrites;
+        rewrites["this"] = "that";
+
+        ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), "");
+    }
+
+    TEST(rewriteStrings, emptyRewrites) {
+        StringMap rewrites;
+
+        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that");
+    }
+
+    TEST(rewriteStrings, successfulRewrite) {
+        StringMap rewrites;
+        rewrites["this"] = "that";
+
+        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that");
+    }
+
+    TEST(rewriteStrings, doesntOccur) {
+        StringMap rewrites;
+        rewrites["foo"] = "bar";
+
+        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * replaceStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(replaceStrings, emptyString) {
+        ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), "");
+        ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that");
+    }
+
+    TEST(replaceStrings, successfulReplace) {
+        ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that");
+    }
+
+    TEST(replaceStrings, doesntOccur) {
+        ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * trim
+     * --------------------------------------------------------------------------*/
+
+    TEST(trim, emptyString) {
+        ASSERT_STREQ(trim("").c_str(), "");
+    }
+
+    TEST(trim, removesWhitespace) {
+        ASSERT_STREQ(trim("foo").c_str(), "foo");
+        ASSERT_STREQ(trim("     foo ").c_str(), "foo");
+        ASSERT_STREQ(trim("     foo bar baz").c_str(), "foo bar baz");
+        ASSERT_STREQ(trim("     \t foo bar baz\n").c_str(), "foo bar baz");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * chomp
+     * --------------------------------------------------------------------------*/
+
+    TEST(chomp, emptyString) {
+        ASSERT_STREQ(chomp("").c_str(), "");
+    }
+
+    TEST(chomp, removesWhitespace) {
+        ASSERT_STREQ(chomp("foo").c_str(), "foo");
+        ASSERT_STREQ(chomp("foo ").c_str(), "foo");
+        ASSERT_STREQ(chomp(" foo ").c_str(), " foo");
+        ASSERT_STREQ(chomp(" foo bar baz  ").c_str(), " foo bar baz");
+        ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * quoteStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(quoteStrings, empty) {
+        Strings s = { };
+        Strings expected = { };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    TEST(quoteStrings, emptyStrings) {
+        Strings s = { "", "", "" };
+        Strings expected = { "''", "''", "''" };
+        ASSERT_EQ(quoteStrings(s), expected);
+
+    }
+
+    TEST(quoteStrings, trivialQuote) {
+        Strings s = { "foo", "bar", "baz" };
+        Strings expected = { "'foo'", "'bar'", "'baz'" };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    TEST(quoteStrings, quotedStrings) {
+        Strings s = { "'foo'", "'bar'", "'baz'" };
+        Strings expected = { "''foo''", "''bar''", "''baz''" };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * tokenizeString
+     * --------------------------------------------------------------------------*/
+
+    TEST(tokenizeString, empty) {
+        auto s = "";
+        Strings expected = {  };
+
+        ASSERT_EQ(tokenizeString<Strings>(""), expected);
+    }
+
+    TEST(tokenizeString, tokenizeSpacesWithDefaults) {
+        auto s = "foo bar baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsWithDefaults) {
+        auto s = "foo\tbar\tbaz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) {
+        auto s = "foo\t bar\t baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) {
+        auto s = "foo\t\n bar\t\n baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) {
+        auto s = "foo\t\n\r bar\t\n\r baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+
+        auto s2 = "foo \t\n\r bar \t\n\r baz";
+        Strings expected2 = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s2), expected2);
+    }
+
+    TEST(tokenizeString, tokenizeWithCustomSep) {
+        auto s = "foo\n,bar\n,baz\n";
+        Strings expected = { "foo\n", "bar\n", "baz\n" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s, ","), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * get
+     * --------------------------------------------------------------------------*/
+
+    TEST(get, emptyContainer) {
+        StringMap s = { };
+        auto expected = std::nullopt;
+
+        ASSERT_EQ(get(s, "one"), expected);
+    }
+
+    TEST(get, getFromContainer) {
+        StringMap s;
+        s["one"] = "yi";
+        s["two"] = "er";
+        auto expected = "yi";
+
+        ASSERT_EQ(get(s, "one"), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * filterANSIEscapes
+     * --------------------------------------------------------------------------*/
+
+    TEST(filterANSIEscapes, emptyString) {
+        auto s = "";
+        auto expected = "";
+
+        ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected);
+    }
+
+    TEST(filterANSIEscapes, doesntChangePrintableChars) {
+        auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f";
+
+        ASSERT_STREQ(filterANSIEscapes(s).c_str(), s);
+    }
+
+    TEST(filterANSIEscapes, filtersColorCodes) {
+        auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m";
+
+        ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" );
+        ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " );
+        ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A  " );
+        ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A  B" );
+        ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A  B  C" );
+    }
+
+    TEST(filterANSIEscapes, expandsTabs) {
+        auto s = "foo\tbar\tbaz";
+
+        ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo     bar     baz" );
+    }
+
+}

From e76ad2e48a71aa9804311f4dfb34374a5295b3b6 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 6 May 2020 14:07:20 -0600
Subject: [PATCH 2060/2196] implement SysError errno handling

---
 src/error-demo/error-demo.cc |  8 ++++++++
 src/libutil/error.cc         |  8 --------
 src/libutil/error.hh         | 13 +++++++------
 src/libutil/fmt.hh           |  8 ++++----
 src/libutil/util.cc          |  3 ++-
 5 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 82b03d71b92..b1b313e3229 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -35,6 +35,14 @@ int main()
         logger->logEI(ei);
     }
 
+    // SysError; picks up errno
+    try {
+        auto x = readFile(-1);
+    }
+    catch (Error &e) {
+        std::cout << "error: " << e.sname() << std::endl;
+        logError(e.info());
+    }
 
     // For completeness sake, info through vomit levels.
     // But this is maybe a heavy format for those.
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 91fa1ccd822..d4305ddd8b3 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -16,14 +16,6 @@ BaseError & BaseError::addPrefix(const FormatOrString & fs)
     return *this;
 }
 
-
-std::string SysError::addErrno(const std::string & s)
-{
-    errNo = errno;
-    return s + ": " + strerror(errNo);
-}
-
-
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
 std::ostream& operator<<(std::ostream &os, const hintformat &hf)
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 86cff56091d..2155ad34486 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -160,13 +160,14 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-        : Error(args...)  // TODO addErrNo for hintfmt
-          // : Error(addErrno(hintfmt(args...)))
-    { }
-
-private:
+      :Error("")
+    {
+        errNo = errno;
+        auto hf = hintfmt(args...);
+        err.hint = hintfmt("%1% : %2%", normaltxt(hf.str()), strerror(errNo));
+    }
 
-    std::string addErrno(const std::string & s);
+    virtual const char* sname() const override { return "SysError"; }
 };
 
 }
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index bfacfeb4de0..12ab9c40718 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -75,8 +75,8 @@ inline std::string fmt(const std::string & fs, const Args & ... args)
 template <class T>
 struct yellowtxt
 {
-    yellowtxt(T &s) : value(s) {}
-    T &value;
+    yellowtxt(const T &s) : value(s) {}
+    const T &value;
 };
 
 template <class T>
@@ -88,8 +88,8 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y)
 template <class T>
 struct normaltxt
 {
-    normaltxt(T &s) : value(s) {}
-    T &value;
+    normaltxt(const T &s) : value(s) {}
+    const T &value;
 };
 
 template <class T>
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 68dc1b738d0..e8f22ab71cf 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -312,7 +312,8 @@ string readFile(const Path & path, bool drain)
 void readFile(const Path & path, Sink & sink)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError("opening file '%s'", path);
+    if (!fd)  
+        throw SysError("opening file '%s'", path);
     drainFD(fd.get(), sink);
 }
 

From e3901638b50d91ddc9c356adc35ded00a00fe94b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 6 May 2020 15:01:13 -0600
Subject: [PATCH 2061/2196] todo removal

---
 src/error-demo/error-demo.cc | 1 -
 src/libstore/gc.cc           | 1 -
 src/nix-daemon/nix-daemon.cc | 1 -
 src/nix/upgrade-nix.cc       | 2 --
 4 files changed, 5 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index b1b313e3229..0d3dcf61603 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -23,7 +23,6 @@ int main()
         logger->logEI(e.info());
     }
 
-
     // ErrorInfo constructor
     try {
         auto e = Error("some error");
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 4f6f78a9966..629c4360c93 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -41,7 +41,6 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
         throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
-        // TODO: info?
         printError("waiting for the big garbage collector lock...");
         lockFile(fdGCLock.get(), lockType, true);
     }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 5336db1364f..731ab22e5cb 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -247,7 +247,6 @@ static void daemonLoop(char * * argv)
         } catch (Interrupted & e) {
             return;
         } catch (Error & error) {
-            // TODO append error message
             ErrorInfo ei = error.info();
             string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : "");
             ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint));
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 8f4e529bc54..831a83fd15f 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -64,13 +64,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         if (dryRun) {
             stopProgressBar();
-            // TODO change to info?
             logWarning(
                 ErrorInfo { 
                     .name = "Version update",
                     .hint = hintfmt("would upgrade to version %s", version)
             });
-            // printError("would upgrade to version %s", version);
             return;
         }
 

From 479e8bf00b680343231d2b618e130141079a2097 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 May 2020 16:08:15 +0200
Subject: [PATCH 2062/2196] Manual: Fix typo

---
 doc/manual/advanced-topics/post-build-hook.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
index 08a7a772f43..57ee99e84d6 100644
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 
 <para>Now, copy the path back from the cache:</para>
 <screen>
-$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
+$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
 copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
 warning: you did not specify '--add-root'; the result might be removed by the garbage collector
 /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

From 41caaaad365b8e5260069c20458a2043254a6989 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 May 2020 16:37:33 +0200
Subject: [PATCH 2063/2196] Manual: Typo

---
 doc/manual/advanced-topics/post-build-hook.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml
index 57ee99e84d6..acfe9e3cca1 100644
--- a/doc/manual/advanced-topics/post-build-hook.xml
+++ b/doc/manual/advanced-topics/post-build-hook.xml
@@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
 </programlisting>
 
-<para>we will restart the Nix daemon a later step.</para>
+<para>We will restart the Nix daemon in a later step.</para>
 </section>
 
 <section>

From 987b3d6469f03bff59614bd3b9097df858d07263 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Thu, 7 May 2020 18:10:07 +0200
Subject: [PATCH 2064/2196] Use ASSERT_EQ instead of ASSERT_STREQ

No need to use `c_str()` in combination with `ASSERT_STREQ`.
It's possible to just use ASSERT_EQ on std::string
---
 tests/unit-tests/test-util.cc | 108 +++++++++++++++++-----------------
 1 file changed, 54 insertions(+), 54 deletions(-)

diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc
index 05d8a0039d9..62ca8f11ccf 100644
--- a/tests/unit-tests/test-util.cc
+++ b/tests/unit-tests/test-util.cc
@@ -12,14 +12,14 @@ namespace nix {
     TEST(absPath, doesntChangeRoot) {
         auto p = absPath("/");
 
-        ASSERT_STREQ(p.c_str(), "/");
+        ASSERT_EQ(p, "/");
     }
 
     TEST(absPath, turnsEmptyPathIntoCWD) {
         char cwd[PATH_MAX+1];
         auto p = absPath("");
 
-        ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX));
+        ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX));
     }
 
     TEST(absPath, usesOptionalBasePathWhenGiven) {
@@ -28,7 +28,7 @@ namespace nix {
 
         auto p = absPath("", cwd);
 
-        ASSERT_STREQ(p.c_str(), cwd);
+        ASSERT_EQ(p, cwd);
     }
 
     TEST(absPath, isIdempotent) {
@@ -37,7 +37,7 @@ namespace nix {
         auto p1 = absPath(cwd);
         auto p2 = absPath(p1);
 
-        ASSERT_STREQ(p1.c_str(), p2.c_str());
+        ASSERT_EQ(p1, p2);
     }
 
 
@@ -46,8 +46,8 @@ namespace nix {
         auto p1 = absPath(path);
         auto p2 = absPath(p1);
 
-        ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot");
-        ASSERT_STREQ(p1.c_str(), p2.c_str());
+        ASSERT_EQ(p1, "/some/path/with/trailing/dot");
+        ASSERT_EQ(p1, p2);
     }
 
     /* ----------------------------------------------------------------------------
@@ -58,21 +58,21 @@ namespace nix {
         auto path = "/this/is/a/path//";
         auto p = canonPath(path);
 
-        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+        ASSERT_EQ(p, "/this/is/a/path");
     }
 
     TEST(canonPath, removesDots) {
         auto path = "/this/./is/a/path/./";
         auto p = canonPath(path);
 
-        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+        ASSERT_EQ(p, "/this/is/a/path");
     }
 
     TEST(canonPath, removesDots2) {
         auto path = "/this/a/../is/a////path/foo/..";
         auto p = canonPath(path);
 
-        ASSERT_STREQ(p.c_str(), "/this/is/a/path");
+        ASSERT_EQ(p, "/this/is/a/path");
     }
 
     TEST(canonPath, requiresAbsolutePath) {
@@ -91,18 +91,18 @@ namespace nix {
     TEST(dirOf, DISABLED_returnsEmptyStringForRoot) {
         auto p = dirOf("/");
 
-        ASSERT_STREQ(p.c_str(), "");
+        ASSERT_EQ(p, "");
     }
 
     TEST(dirOf, returnsFirstPathComponent) {
         auto p1 = dirOf("/dir/");
-        ASSERT_STREQ(p1.c_str(), "/dir");
+        ASSERT_EQ(p1, "/dir");
         auto p2 = dirOf("/dir");
-        ASSERT_STREQ(p2.c_str(), "/");
+        ASSERT_EQ(p2, "/");
         auto p3 = dirOf("/dir/..");
-        ASSERT_STREQ(p3.c_str(), "/dir");
+        ASSERT_EQ(p3, "/dir");
         auto p4 = dirOf("/dir/../");
-        ASSERT_STREQ(p4.c_str(), "/dir/..");
+        ASSERT_EQ(p4, "/dir/..");
     }
 
     /* ----------------------------------------------------------------------------
@@ -111,29 +111,29 @@ namespace nix {
 
     TEST(baseNameOf, emptyPath) {
         auto p1 = baseNameOf("");
-        ASSERT_STREQ(std::string(p1).c_str(), "");
+        ASSERT_EQ(std::string(p1), "");
     }
 
     TEST(baseNameOf, pathOnRoot) {
         auto p1 = baseNameOf("/dir");
-        ASSERT_STREQ(std::string(p1).c_str(), "dir");
+        ASSERT_EQ(std::string(p1), "dir");
     }
 
     TEST(baseNameOf, relativePath) {
         auto p1 = baseNameOf("dir/foo");
-        ASSERT_STREQ(std::string(p1).c_str(), "foo");
+        ASSERT_EQ(std::string(p1), "foo");
     }
 
     TEST(baseNameOf, pathWithTrailingSlashRoot) {
         auto p1 = baseNameOf("/");
-        ASSERT_STREQ(std::string(p1).c_str(), "");
+        ASSERT_EQ(std::string(p1), "");
     }
 
     // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return
     // "" but it actually returns "dir"
     TEST(baseNameOf, DISABLED_trailingSlash) {
         auto p1 = baseNameOf("/dir/");
-        ASSERT_STREQ(std::string(p1).c_str(), "");
+        ASSERT_EQ(std::string(p1), "");
     }
 
     /* ----------------------------------------------------------------------------
@@ -265,11 +265,11 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(base64Encode, emptyString) {
-        ASSERT_STREQ(base64Encode("").c_str(), "");
+        ASSERT_EQ(base64Encode(""), "");
     }
 
     TEST(base64Encode, encodesAString) {
-        ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
+        ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
     }
 
     TEST(base64Encode, encodeAndDecode) {
@@ -277,7 +277,7 @@ namespace nix {
         auto encoded = base64Encode(s);
         auto decoded = base64Decode(encoded);
 
-        ASSERT_STREQ(decoded.c_str(), s);
+        ASSERT_EQ(decoded, s);
     }
 
     /* ----------------------------------------------------------------------------
@@ -285,11 +285,11 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(base64Decode, emptyString) {
-        ASSERT_STREQ(base64Decode("").c_str(), "");
+        ASSERT_EQ(base64Decode(""), "");
     }
 
     TEST(base64Decode, decodeAString) {
-        ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum");
+        ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
     }
 
     /* ----------------------------------------------------------------------------
@@ -297,12 +297,12 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(toLower, emptyString) {
-        ASSERT_STREQ(toLower("").c_str(), "");
+        ASSERT_EQ(toLower(""), "");
     }
 
     TEST(toLower, nonLetters) {
         auto s = "!@(*$#)(@#=\\234_";
-        ASSERT_STREQ(toLower(s).c_str(), s);
+        ASSERT_EQ(toLower(s), s);
     }
 
     // XXX: std::tolower() doesn't cover this. This probably doesn't matter
@@ -310,7 +310,7 @@ namespace nix {
     // characters are not allowed.
     TEST(toLower, DISABLED_umlauts) {
         auto s = "ÄÖÜ";
-        ASSERT_STREQ(toLower(s).c_str(), "äöü");
+        ASSERT_EQ(toLower(s), "äöü");
     }
 
     /* ----------------------------------------------------------------------------
@@ -377,27 +377,27 @@ namespace nix {
         StringMap rewrites;
         rewrites["this"] = "that";
 
-        ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), "");
+        ASSERT_EQ(rewriteStrings("", rewrites), "");
     }
 
     TEST(rewriteStrings, emptyRewrites) {
         StringMap rewrites;
 
-        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that");
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
     }
 
     TEST(rewriteStrings, successfulRewrite) {
         StringMap rewrites;
         rewrites["this"] = "that";
 
-        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that");
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that");
     }
 
     TEST(rewriteStrings, doesntOccur) {
         StringMap rewrites;
         rewrites["foo"] = "bar";
 
-        ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that");
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
     }
 
     /* ----------------------------------------------------------------------------
@@ -405,16 +405,16 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(replaceStrings, emptyString) {
-        ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), "");
-        ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that");
+        ASSERT_EQ(replaceStrings("", "this", "that"), "");
+        ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that");
     }
 
     TEST(replaceStrings, successfulReplace) {
-        ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that");
+        ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that");
     }
 
     TEST(replaceStrings, doesntOccur) {
-        ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that");
+        ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that");
     }
 
     /* ----------------------------------------------------------------------------
@@ -422,14 +422,14 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(trim, emptyString) {
-        ASSERT_STREQ(trim("").c_str(), "");
+        ASSERT_EQ(trim(""), "");
     }
 
     TEST(trim, removesWhitespace) {
-        ASSERT_STREQ(trim("foo").c_str(), "foo");
-        ASSERT_STREQ(trim("     foo ").c_str(), "foo");
-        ASSERT_STREQ(trim("     foo bar baz").c_str(), "foo bar baz");
-        ASSERT_STREQ(trim("     \t foo bar baz\n").c_str(), "foo bar baz");
+        ASSERT_EQ(trim("foo"), "foo");
+        ASSERT_EQ(trim("     foo "), "foo");
+        ASSERT_EQ(trim("     foo bar baz"), "foo bar baz");
+        ASSERT_EQ(trim("     \t foo bar baz\n"), "foo bar baz");
     }
 
     /* ----------------------------------------------------------------------------
@@ -437,15 +437,15 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(chomp, emptyString) {
-        ASSERT_STREQ(chomp("").c_str(), "");
+        ASSERT_EQ(chomp(""), "");
     }
 
     TEST(chomp, removesWhitespace) {
-        ASSERT_STREQ(chomp("foo").c_str(), "foo");
-        ASSERT_STREQ(chomp("foo ").c_str(), "foo");
-        ASSERT_STREQ(chomp(" foo ").c_str(), " foo");
-        ASSERT_STREQ(chomp(" foo bar baz  ").c_str(), " foo bar baz");
-        ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz");
+        ASSERT_EQ(chomp("foo"), "foo");
+        ASSERT_EQ(chomp("foo "), "foo");
+        ASSERT_EQ(chomp(" foo "), " foo");
+        ASSERT_EQ(chomp(" foo bar baz  "), " foo bar baz");
+        ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz");
     }
 
     /* ----------------------------------------------------------------------------
@@ -566,29 +566,29 @@ namespace nix {
         auto s = "";
         auto expected = "";
 
-        ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected);
+        ASSERT_EQ(filterANSIEscapes(s), expected);
     }
 
     TEST(filterANSIEscapes, doesntChangePrintableChars) {
         auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f";
 
-        ASSERT_STREQ(filterANSIEscapes(s).c_str(), s);
+        ASSERT_EQ(filterANSIEscapes(s), s);
     }
 
     TEST(filterANSIEscapes, filtersColorCodes) {
         auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m";
 
-        ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" );
-        ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " );
-        ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A  " );
-        ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A  B" );
-        ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A  B  C" );
+        ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" );
+        ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " );
+        ASSERT_EQ(filterANSIEscapes(s, true, 4), " A  " );
+        ASSERT_EQ(filterANSIEscapes(s, true, 5), " A  B" );
+        ASSERT_EQ(filterANSIEscapes(s, true, 8), " A  B  C" );
     }
 
     TEST(filterANSIEscapes, expandsTabs) {
         auto s = "foo\tbar\tbaz";
 
-        ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo     bar     baz" );
+        ASSERT_EQ(filterANSIEscapes(s, true), "foo     bar     baz" );
     }
 
 }

From 1f3602a2c9c3bfeea7cc8ee4d478c77dec349206 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Thu, 7 May 2020 18:15:13 +0200
Subject: [PATCH 2065/2196] Remove replaceInSet

The function isn't being used anywhere so it seems safe to remove
---
 src/libutil/util.hh           | 11 -----------
 tests/unit-tests/test-util.cc |  4 ----
 2 files changed, 15 deletions(-)

diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 1b263abccbb..8770add64a4 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -389,17 +389,6 @@ string replaceStrings(const std::string & s,
 std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
 
 
-/* If a set contains 'from', remove it and insert 'to'. */
-template<typename T>
-void replaceInSet(std::set<T> & set, const T & from, const T & to)
-{
-    auto i = set.find(from);
-    if (i == set.end()) return;
-    set.erase(i);
-    set.insert(to);
-}
-
-
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);
diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc
index 62ca8f11ccf..e0738614e79 100644
--- a/tests/unit-tests/test-util.cc
+++ b/tests/unit-tests/test-util.cc
@@ -364,10 +364,6 @@ namespace nix {
         ASSERT_EQ(statusOk(1), false);
     }
 
-    /* ----------------------------------------------------------------------------
-     * replaceInSet : XXX: this function isn't called anywhere!
-     * --------------------------------------------------------------------------*/
-
 
     /* ----------------------------------------------------------------------------
      * rewriteStrings

From 73d0b5d807edee05910cff4a30549393d5375d89 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Thu, 7 May 2020 19:29:10 +0200
Subject: [PATCH 2066/2196] Drop unnecessary std::string

---
 tests/unit-tests/test-util.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc
index e0738614e79..a0fa7c10989 100644
--- a/tests/unit-tests/test-util.cc
+++ b/tests/unit-tests/test-util.cc
@@ -111,29 +111,29 @@ namespace nix {
 
     TEST(baseNameOf, emptyPath) {
         auto p1 = baseNameOf("");
-        ASSERT_EQ(std::string(p1), "");
+        ASSERT_EQ(p1, "");
     }
 
     TEST(baseNameOf, pathOnRoot) {
         auto p1 = baseNameOf("/dir");
-        ASSERT_EQ(std::string(p1), "dir");
+        ASSERT_EQ(p1, "dir");
     }
 
     TEST(baseNameOf, relativePath) {
         auto p1 = baseNameOf("dir/foo");
-        ASSERT_EQ(std::string(p1), "foo");
+        ASSERT_EQ(p1, "foo");
     }
 
     TEST(baseNameOf, pathWithTrailingSlashRoot) {
         auto p1 = baseNameOf("/");
-        ASSERT_EQ(std::string(p1), "");
+        ASSERT_EQ(p1, "");
     }
 
     // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return
     // "" but it actually returns "dir"
     TEST(baseNameOf, DISABLED_trailingSlash) {
         auto p1 = baseNameOf("/dir/");
-        ASSERT_EQ(std::string(p1), "");
+        ASSERT_EQ(p1, "");
     }
 
     /* ----------------------------------------------------------------------------

From 1b801cec407454b904466153dfc56cec6b433f4b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 7 May 2020 16:43:36 -0600
Subject: [PATCH 2067/2196] pretending to be const

---
 src/error-demo/error-demo.cc | 16 ++++++++++++++++
 src/libutil/error.hh         | 12 ++++++------
 src/nix-build/nix-build.cc   |  2 +-
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 0d3dcf61603..89ba5a78d24 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -43,6 +43,22 @@ int main()
         logError(e.info());
     }
 
+    // current exception
+    try {
+        throw DemoError("DemoError handled as a %1%", "std::exception");
+    }
+    catch (...) {
+        const std::exception_ptr &eptr = std::current_exception();
+        try
+        {
+            std::rethrow_exception(eptr);
+        }
+        catch (std::exception& e)
+        {
+            std::cerr << e.what() << std::endl;
+        }
+    }
+
     // For completeness sake, info through vomit levels.
     // But this is maybe a heavy format for those.
     logger->logEI(
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 2155ad34486..19a806cc1d2 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -82,10 +82,10 @@ class BaseError : public std::exception
 {
 protected:
     string prefix_; // used for location traces etc.
-    ErrorInfo err;
+    mutable ErrorInfo err;
 
-    std::optional<string> what_;
-    const string& calcWhat()
+    mutable std::optional<string> what_;
+    const string& calcWhat() const
     {
         if (what_.has_value())
             return *what_;
@@ -131,12 +131,12 @@ public:
 
 #ifdef EXCEPTION_NEEDS_THROW_SPEC
     ~BaseError() throw () { };
-    const char * what() throw () { return calcWhat().c_str(); }
+    const char * what() const throw () { return calcWhat().c_str(); }
 #else
-    const char * what() noexcept { return calcWhat().c_str(); }
+    const char * what() const noexcept override { return calcWhat().c_str(); }
 #endif
 
-    const string & msg() { return calcWhat(); }
+    const string & msg() const { return calcWhat(); }
     const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 401c8d3404e..3c2f4f00ce2 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -368,7 +368,7 @@ static void _main(int argc, char * * argv)
                 shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
-                // TODO: append error msg
+                // TODO: append error msg; warn()?
                 logError(e.info());
                 printError("warning: %s; will use bash from your environment", e.what());
                 shell = "bash";

From 14073fb76be0f46cb9335edf747fcfa1a5275b7b Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Fri, 8 May 2020 12:22:39 +0300
Subject: [PATCH 2068/2196] Don't block while waiting for build users

---
 src/libstore/build.cc | 54 +++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1b7e6d75e94..00101f55325 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -507,9 +507,6 @@ class UserLock
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
-    bool findFreeUser();
-
-
     string user;
     uid_t uid;
     gid_t gid;
@@ -525,20 +522,19 @@ class UserLock
     uid_t getGID() { assert(gid); return gid; }
     std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
+    bool findFreeUser();
+
     bool enabled() { return uid != 0; }
 
 };
 
+
 UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
     createDirs(settings.nixStateDir + "/userpool");
-
-    if (findFreeUser()) return;
-
-    printError("waiting for build users");
-
-    do std::this_thread::sleep_for(std::chrono::seconds(2)); while (! findFreeUser());
+    /* Mark that user is not enabled by default */
+    uid = 0;
 }
 
 bool UserLock::findFreeUser() {
@@ -1398,6 +1394,30 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
+    /* If `build-users-group' is not empty, then we have to build as
+       one of the members of that group. */
+    if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
+        if (!buildUser) buildUser = std::make_unique<UserLock>();
+
+        if (!buildUser->enabled()) {
+            if (!buildUser->findFreeUser()) {
+                debug("waiting for build users");
+                worker.waitForAWhile(shared_from_this());
+                return;
+            }
+
+            /* Make sure that no other processes are executing under this
+               uid. */
+            buildUser->kill();
+        }
+#else
+        /* Don't know how to block the creation of setuid/setgid
+           binaries on this platform. */
+        throw Error("build users are not supported on this platform for security reasons");
+#endif
+    }
+
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -1950,22 +1970,6 @@ void DerivationGoal::startBuilder()
         #endif
     }
 
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        buildUser = std::make_unique<UserLock>();
-
-        /* Make sure that no other processes are executing under this
-           uid. */
-        buildUser->kill();
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);

From 772e5db828c9924c803b4d126bca72d7e123614b Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Fri, 8 May 2020 12:29:00 +0300
Subject: [PATCH 2069/2196] Mention build users in the 'waiting for' message

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 00101f55325..a8c1cd565b7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4830,7 +4830,7 @@ void Worker::waitForInput()
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
         if (lastWokenUp == steady_time_point::min())
-            printError("waiting for locks or build slots...");
+            printError("waiting for locks, build slots or build users...");
         if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
         timeout = std::max(1L,
             (long) std::chrono::duration_cast<std::chrono::seconds>(

From 7cc7cef9502dc388706561cbdf275e070520e65d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 May 2020 11:34:09 +0200
Subject: [PATCH 2070/2196] Move unit tests to sr/libutil/tests, use mk make
 rules

---
 Makefile                                              |  4 ++--
 src/libutil/tests/local.mk                            | 11 +++++++++++
 .../test-util.cc => src/libutil/tests/tests.cc        |  4 ++--
 tests/unit-tests/local.mk                             |  4 ----
 4 files changed, 15 insertions(+), 8 deletions(-)
 create mode 100644 src/libutil/tests/local.mk
 rename tests/unit-tests/test-util.cc => src/libutil/tests/tests.cc (99%)
 delete mode 100644 tests/unit-tests/local.mk

diff --git a/Makefile b/Makefile
index 06de35a2709..0ba011e2ad0 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ makefiles = \
   local.mk \
   nix-rust/local.mk \
   src/libutil/local.mk \
+  src/libutil/tests/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
@@ -16,8 +17,7 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk \
-  tests/unit-tests/local.mk
+  tests/plugins/local.mk
 
 -include Makefile.config
 
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
new file mode 100644
index 00000000000..9c2d2a7bbc8
--- /dev/null
+++ b/src/libutil/tests/local.mk
@@ -0,0 +1,11 @@
+programs += libutil-tests
+
+libutil-tests_DIR := $(d)
+
+libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libutil-tests_CXXFLAGS += -I src/libutil
+
+libutil-tests_LIBS = libutil
+
+libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main)
diff --git a/tests/unit-tests/test-util.cc b/src/libutil/tests/tests.cc
similarity index 99%
rename from tests/unit-tests/test-util.cc
rename to src/libutil/tests/tests.cc
index a0fa7c10989..fb593b5e654 100644
--- a/tests/unit-tests/test-util.cc
+++ b/src/libutil/tests/tests.cc
@@ -1,5 +1,5 @@
-#include "../../src/libutil/util.hh"
-#include "../../src/libutil/types.hh"
+#include "util.hh"
+#include "types.hh"
 
 #include <gtest/gtest.h>
 
diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk
deleted file mode 100644
index ed11261c1a0..00000000000
--- a/tests/unit-tests/local.mk
+++ /dev/null
@@ -1,4 +0,0 @@
-LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec
-unit-tests:
-	echo $(nix_LDFLAGS)
-	$(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc

From 72b9d971bc4ff5b5dcb9349d03526a5ef3e4300e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 May 2020 11:35:57 +0200
Subject: [PATCH 2071/2196] Fix warning

---
 src/libutil/tests/tests.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index fb593b5e654..0fb4411e834 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -481,8 +481,7 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(tokenizeString, empty) {
-        auto s = "";
-        Strings expected = {  };
+        Strings expected = { };
 
         ASSERT_EQ(tokenizeString<Strings>(""), expected);
     }

From 7898cdb75a721cc21867ccdbcc523e95f2db0354 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 May 2020 11:49:40 +0200
Subject: [PATCH 2072/2196] make check: Run unit tests

---
 mk/programs.mk             | 6 ++++++
 mk/tracing.mk              | 1 +
 nix-rust/local.mk          | 2 +-
 src/libutil/tests/local.mk | 2 ++
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/mk/programs.mk b/mk/programs.mk
index d93df446808..2d0c988d4cd 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -76,4 +76,10 @@ define build-program
   programs-list += $$($(1)_PATH)
   clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
+
+  # Phony target to run this program (typically as a dependency of 'check').
+  .PHONY: $(1)_RUN
+  $(1)_RUN: $$($(1)_PATH)
+	$(trace-test) $$($(1)_PATH)
+
 endef
diff --git a/mk/tracing.mk b/mk/tracing.mk
index 13912d3c782..54c77ab6075 100644
--- a/mk/tracing.mk
+++ b/mk/tracing.mk
@@ -11,6 +11,7 @@ ifeq ($(V), 0)
   trace-javac   = @echo "  JAVAC " $@;
   trace-jar     = @echo "  JAR   " $@;
   trace-mkdir   = @echo "  MKDIR " $@;
+  trace-test    = @echo "  TEST  " $@;
 
   suppress  = @
 
diff --git a/nix-rust/local.mk b/nix-rust/local.mk
index 1e006e500c8..e4bfde31b8b 100644
--- a/nix-rust/local.mk
+++ b/nix-rust/local.mk
@@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
 check: rust-tests
 
 rust-tests:
-	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
 endif
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index 9c2d2a7bbc8..a188d5093a4 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -1,3 +1,5 @@
+check: libutil-tests_RUN
+
 programs += libutil-tests
 
 libutil-tests_DIR := $(d)

From ca657525b88e788b72ce53ecc6d140a573476644 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 May 2020 12:03:27 +0200
Subject: [PATCH 2073/2196] Don't install unit tests

---
 mk/programs.mk             | 20 ++++++++++++--------
 src/libutil/tests/local.mk |  2 ++
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/mk/programs.mk b/mk/programs.mk
index 2d0c988d4cd..3fa9685c39d 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -35,24 +35,28 @@ define build-program
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
-  $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
+  ifdef $(1)_INSTALL_DIR
 
-  install: $(DESTDIR)$$($(1)_INSTALL_PATH)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
 
-  ifeq ($(BUILD_SHARED_LIBS), 1)
+    $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
-    _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+    install: $(DESTDIR)$$($(1)_INSTALL_PATH)
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+    ifeq ($(BUILD_SHARED_LIBS), 1)
+
+      _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
+
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
-  else
+    else
 
-    $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
+      $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
 	install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
 
+    endif
   endif
 
   # Propagate CFLAGS and CXXFLAGS to the individual object files.
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index a188d5093a4..5f4b3edc5d1 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -4,6 +4,8 @@ programs += libutil-tests
 
 libutil-tests_DIR := $(d)
 
+libutil-tests_INSTALL_DIR :=
+
 libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil-tests_CXXFLAGS += -I src/libutil

From 5b8883faac0cff928a4359b4260b5be09a311d4b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 May 2020 12:09:00 +0200
Subject: [PATCH 2074/2196] configure: Look for gtest

---
 Makefile.config.in         | 19 ++++++++++---------
 configure.ac               |  4 ++++
 src/libutil/tests/local.mk |  2 +-
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index e7a12089a86..b632444e8ab 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -1,36 +1,38 @@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-LDFLAGS = @LDFLAGS@
+EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
-HAVE_SODIUM = @HAVE_SODIUM@
+GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-BOOST_LDFLAGS = @BOOST_LDFLAGS@
+HAVE_SODIUM = @HAVE_SODIUM@
+LDFLAGS = @LDFLAGS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
+LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
-LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
-LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
-EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
-lsof = @lsof@
 datadir = @datadir@
 datarootdir = @datarootdir@
+doc_generate = @doc_generate@
 docdir = @docdir@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
+lsof = @lsof@
 mandir = @mandir@
 pkglibdir = $(libdir)/$(PACKAGE_NAME)
 prefix = @prefix@
@@ -38,6 +40,5 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@
diff --git a/configure.ac b/configure.ac
index b868343d251..c3007b4b6b4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -266,6 +266,10 @@ if test "$gc" = yes; then
 fi
 
 
+# Look for gtest.
+PKG_CHECK_MODULES([GTEST], [gtest_main])
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index 5f4b3edc5d1..a297edb6478 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -12,4 +12,4 @@ libutil-tests_CXXFLAGS += -I src/libutil
 
 libutil-tests_LIBS = libutil
 
-libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main)
+libutil-tests_LDFLAGS := $(GTEST_LIBS)

From e3df9c2a6e867d0e038cb26af40501511607e007 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Fri, 8 May 2020 15:03:44 +0200
Subject: [PATCH 2075/2196] Enable `dirOf` test

Adjusted the doc comment for `dirOf` to reflect the implementation
behavior.
---
 src/libutil/tests/tests.cc | 6 ++----
 src/libutil/util.hh        | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 0fb4411e834..1b016430da8 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -86,12 +86,10 @@ namespace nix {
      * dirOf
      * --------------------------------------------------------------------------*/
 
-    // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo")
-    // should both return "" but it actually returns "/" in both cases
-    TEST(dirOf, DISABLED_returnsEmptyStringForRoot) {
+    TEST(dirOf, returnsEmptyStringForRoot) {
         auto p = dirOf("/");
 
-        ASSERT_EQ(p, "");
+        ASSERT_EQ(p, "/");
     }
 
     TEST(dirOf, returnsFirstPathComponent) {
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 8770add64a4..38a0f7a5cc6 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -58,8 +58,8 @@ Path canonPath(const Path & path, bool resolveSymlinks = false);
 
 /* Return the directory part of the given canonical path, i.e.,
    everything before the final `/'.  If the path is the root or an
-   immediate child thereof (e.g., `/foo'), this means an empty string
-   is returned. */
+   immediate child thereof (e.g., `/foo'), this means `/'
+   is returned.*/
 Path dirOf(const Path & path);
 
 /* Return the base name of the given canonical path, i.e., everything

From 2191141274cfefcb9ffbafbbfcdc58414fb28f1a Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Fri, 8 May 2020 15:07:40 +0200
Subject: [PATCH 2076/2196] Enable `baseNameOf` test

Add note about removal of trailing slashes in the doc comment of
baseNameOf and enabled the test.
---
 src/libutil/tests/tests.cc | 6 ++----
 src/libutil/util.hh        | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 1b016430da8..d46f6a5a4bc 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -127,11 +127,9 @@ namespace nix {
         ASSERT_EQ(p1, "");
     }
 
-    // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return
-    // "" but it actually returns "dir"
-    TEST(baseNameOf, DISABLED_trailingSlash) {
+    TEST(baseNameOf, trailingSlash) {
         auto p1 = baseNameOf("/dir/");
-        ASSERT_EQ(p1, "");
+        ASSERT_EQ(p1, "dir");
     }
 
     /* ----------------------------------------------------------------------------
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 38a0f7a5cc6..a63ee05b377 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -63,7 +63,7 @@ Path canonPath(const Path & path, bool resolveSymlinks = false);
 Path dirOf(const Path & path);
 
 /* Return the base name of the given canonical path, i.e., everything
-   following the final `/'. */
+   following the final `/' (trailing slashes are removed). */
 std::string_view baseNameOf(std::string_view path);
 
 /* Check whether 'path' is a descendant of 'dir'. */

From 181a47d8845e06edc9653e99de589376a1a96bb6 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Fri, 8 May 2020 15:13:55 +0200
Subject: [PATCH 2077/2196] Enable toLower umlauts test

Update comment and enable the test
---
 src/libutil/tests/tests.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index d46f6a5a4bc..5abfe2c9c96 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -301,12 +301,12 @@ namespace nix {
         ASSERT_EQ(toLower(s), s);
     }
 
-    // XXX: std::tolower() doesn't cover this. This probably doesn't matter
-    // since the context is paths and store paths specifically where such
-    // characters are not allowed.
-    TEST(toLower, DISABLED_umlauts) {
+    // std::tolower() doesn't handle unicode characters. In the context of
+    // store paths this isn't relevant but doesn't hurt to record this behavior
+    // here.
+    TEST(toLower, umlauts) {
         auto s = "ÄÖÜ";
-        ASSERT_EQ(toLower(s), "äöü");
+        ASSERT_EQ(toLower(s), "ÄÖÜ");
     }
 
     /* ----------------------------------------------------------------------------

From 55eb71714854b262b5e1079ff250a13cc0bbf644 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Fri, 8 May 2020 18:18:28 -0600
Subject: [PATCH 2078/2196] add pos to errorinfo, remove from hints

---
 src/error-demo/error-demo.cc          |  17 +-
 src/libexpr/attr-set.hh               |   7 +-
 src/libexpr/eval-inline.hh            |  18 +-
 src/libexpr/eval.cc                   |  76 ++++--
 src/libexpr/parser.y                  |  17 +-
 src/libexpr/primops.cc                | 331 +++++++++++++++++++++-----
 src/libexpr/primops/context.cc        |  20 +-
 src/libexpr/primops/fetchGit.cc       |  12 +-
 src/libexpr/primops/fetchMercurial.cc |  13 +-
 src/libutil/error.cc                  | 170 ++++++-------
 src/libutil/error.hh                  |  11 +-
 src/libutil/logging.cc                |   3 +-
 tests/misc.sh                         |   4 +-
 13 files changed, 506 insertions(+), 193 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 89ba5a78d24..41293427c9b 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -126,17 +126,28 @@ int main()
     // Error with previous and next lines of code.
     logError(
         ErrorInfo { .name = "error name",
-                    .description = "error description",
+                    .description = "error with code lines",
                     .hint = hintfmt("this hint has %1% templated %2%!!",
                         "yellow",
                         "values"),
                     .nixCode = NixCode {
                         .errPos = Pos(problem_file, 40, 13),
-                        .prevLineOfCode = std::optional("previous line of code"),
+                        .prevLineOfCode = "previous line of code",
                         .errLineOfCode = "this is the problem line of code",
-                        .nextLineOfCode = std::optional("next line of code"),
+                        .nextLineOfCode = "next line of code",
                     }});
 
 
+    // Error without lines of code.
+    logError(
+        ErrorInfo { .name = "error name",
+                    .description = "error without any code lines.",
+                    .hint = hintfmt("this hint has %1% templated %2%!!",
+                        "yellow",
+                        "values"),
+                    .nixCode = NixCode {
+                        .errPos = Pos(problem_file, 40, 13)
+                    }});
+
     return 0;
 }
diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 118c7bd5d1f..f5651891f04 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -76,7 +76,12 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error("attribute '%s' missing, at %s", name, pos);
+            throw Error(
+                ErrorInfo { 
+                    .hint = hintfmt("attribute '%s' missing", name),
+                    .nixCode = NixCode { .errPos = pos }
+                });
+
         return *a;
     }
 
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 4d82ccf0976..d03633cc7fa 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -9,7 +9,11 @@ namespace nix {
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
 {
-    throw EvalError(s, pos);
+    throw EvalError(
+        ErrorInfo { 
+            .hint = hintfmt(s),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
@@ -20,7 +24,11 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
 {
-    throw TypeError(s, showType(v), pos);
+    throw TypeError(
+        ErrorInfo { 
+            .hint = hintfmt(s, showType(v)),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 
@@ -43,7 +51,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered, at %1%", pos);
+        throwEvalError("infinite recursion encountered", pos);
 }
 
 
@@ -59,7 +67,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a set was expected", v, pos);
 }
 
 
@@ -75,7 +83,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a list was expected", v, pos);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 516c25b022b..a19b85be458 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -498,7 +498,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
 {
-    throw EvalError(s, s2, pos);
+    throw EvalError(
+        ErrorInfo { 
+            .hint = hintfmt(s, s2),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
@@ -508,7 +512,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
 {
-    throw EvalError(s, s2, s3, pos);
+    throw EvalError(
+        ErrorInfo { 
+            .hint = hintfmt(s, s2, s3),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
@@ -518,7 +526,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, co
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
 {
-    throw TypeError(s, pos);
+    throw TypeError(
+        ErrorInfo { 
+            .hint = hintfmt(s),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
@@ -528,17 +540,29 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
 {
-    throw TypeError(s, fun.showNamePos(), s2, pos);
+    throw TypeError(
+        ErrorInfo { 
+            .hint = hintfmt(s, fun.showNamePos(), s2),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError(s, s1, pos);
+    throw AssertionError(
+        ErrorInfo { 
+            .hint = hintfmt(s, s1),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
 {
-    throw UndefinedVarError(s, s1, pos);
+    throw UndefinedVarError(
+        ErrorInfo { 
+            .hint = hintfmt(s, s1),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@@ -804,7 +828,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a Boolean was expected", v, pos);
     return v.boolean;
 }
 
@@ -1006,7 +1030,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute '%1%' missing, at %2%", name, pos);
+                    throwEvalError("attribute '%1%' missing", name, pos);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1132,7 +1156,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos);
+        throwTypeError("attempt to call something which is not a function but %1%", fun, pos);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1160,7 +1184,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%",
+                if (!i.def) throwTypeError("%1% called without required argument '%2%'",
                     lambda, i.name, pos);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
@@ -1176,7 +1200,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos);
+                    throwTypeError("%1% called with unexpected argument '%2%'", lambda, i.name, pos);
             abort(); // can't happen
         }
     }
@@ -1417,14 +1441,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos);
+                throwEvalError("cannot add %1% to an integer", showType(vTmp), pos);
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos);
+                throwEvalError("cannot add %1% to a float", showType(vTmp), pos);
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1435,7 +1459,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
+            throwEvalError("a string that refers to a store path cannot be appended to a path", pos);
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1484,7 +1508,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError("value is %1% while an integer was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while an integer was expected", v, pos);
     return v.integer;
 }
 
@@ -1495,7 +1519,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError("value is %1% while a float was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a float was expected", v, pos);
     return v.fpoint;
 }
 
@@ -1504,7 +1528,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a Boolean was expected", v, pos);
     return v.boolean;
 }
 
@@ -1519,7 +1543,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError("value is %1% while a function was expected, at %2%", v, pos);
+        throwTypeError("value is %1% while a function was expected", v, pos);
 }
 
 
@@ -1528,7 +1552,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError("value is %1% while a string was expected, at %2%", v, pos);
+            throwTypeError("value is %1% while a string was expected", v, pos);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1557,7 +1581,7 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%",
+            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0], pos);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
@@ -1614,7 +1638,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos);
+        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string", pos);
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1645,7 +1669,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError("cannot coerce %1% to a string, at %2%", v, pos);
+    throwTypeError("cannot coerce %1% to a string", v, pos);
 }
 
 
@@ -1676,7 +1700,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos);
+        throwEvalError("string '%1%' doesn't represent an absolute path", path, pos);
     return path;
 }
 
@@ -1883,7 +1907,11 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError("cannot coerce %1% to a string, at %2%", showType(), pos);
+    throw TypeError(
+        ErrorInfo { 
+            .hint = hintfmt("cannot coerce %1% to a string", showType()),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 3ed9a7a4fce..a3ad54a3d29 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -404,7 +404,12 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+          throw ParseError(
+              ErrorInfo { 
+                  .hint = hintfmt("URL literals are disabled"),
+                  .nixCode = NixCode { .errPos = CUR_POS }
+              });
+
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -669,10 +674,12 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    string f = 
-        "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)"
-        + string(pos ? ", at %2%" : "");
-    throw ThrownError(f, path, pos);
+
+    throw ThrownError(
+        ErrorInfo { 
+            .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path),
+            .nixCode = NixCode { .errPos = pos }
+        });
 }
 
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 13eb1ba5878..771136af9b2 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -94,8 +94,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%",
-                        path, e.path, pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid",
+                        path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
+
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -171,8 +176,13 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%"
-           , path, e.path, pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt(
+                    "cannot import '%1%', since path '%2%' is not valid",
+                   path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
     }
 
     path = state.checkSourcePath(path);
@@ -207,7 +217,11 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto elems = args[0]->listElems();
     auto count = args[0]->listSize();
     if (count == 0) {
-        throw EvalError("at least one argument to 'exec' required, at %1%", pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("at least one argument to 'exec' required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     }
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
@@ -218,9 +232,12 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError("cannot execute '%1%', since path '%2%' is not valid, at %3%"
-           , program, e.path, pos);
-    }
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid"
+                     , program, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });}
 
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
@@ -371,7 +388,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError("attribute 'startSet' required, at %1%", pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("attribute 'startSet' required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -382,7 +403,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError("attribute 'operator' required, at %1%", pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("attribute 'operator' required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     state.forceValue(*op->value);
 
     /* Construct the closure by applying the operator to element of
@@ -401,7 +426,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError("attribute 'key' required, at %1%", pos);
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("attribute 'key' required"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
         state.forceValue(*key->value);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -533,7 +562,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError("required attribute 'name' missing, at %1%", pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("required attribute 'name' missing"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
@@ -576,25 +609,45 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         auto handleHashMode = [&](const std::string & s) {
             if (s == "recursive") outputHashRecursive = true;
             else if (s == "flat") outputHashRecursive = false;
-            else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
+            else 
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+                        .nixCode = NixCode { .errPos = posDrvName }
+                    });
         };
 
         auto handleOutputs = [&](const Strings & ss) {
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError("duplicate derivation output '%1%', at %2%", j, posDrvName);
+                    throw EvalError(
+                        ErrorInfo { 
+                            .hint = hintfmt("duplicate derivation output '%1%'", j),
+                            .nixCode = NixCode { .errPos = posDrvName }
+                        });
+
+
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError("invalid derivation output name 'drv', at %1%", posDrvName);
+                    throw EvalError(
+                        ErrorInfo { 
+                            .hint = hintfmt("invalid derivation output name 'drv'" ),
+                            .nixCode = NixCode { .errPos = posDrvName }
+                        });
+
                 outputs.insert(j);
             }
             if (outputs.empty())
-                throw EvalError("derivation cannot have an empty set of outputs, at %1%", posDrvName);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("derivation cannot have an empty set of outputs"),
+                        .nixCode = NixCode { .errPos = posDrvName }
+                    });
         };
 
         try {
@@ -706,18 +759,37 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError("required attribute 'builder' missing, at %1%", posDrvName);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("required attribute 'builder' missing"),
+                .nixCode = NixCode { .errPos = posDrvName }
+            });
+
     if (drv.platform == "")
-        throw EvalError("required attribute 'system' missing, at %1%", posDrvName);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("required attribute 'system' missing"),
+                .nixCode = NixCode { .errPos = posDrvName }
+            });
+
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
-        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+                .nixCode = NixCode { .errPos = posDrvName }
+            });
+
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
-            throw Error("multiple outputs are not supported in fixed-output derivations, at %1%", posDrvName);
+            throw Error(
+                ErrorInfo { 
+                    .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+                    .nixCode = NixCode { .errPos = posDrvName }
+                });
 
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
@@ -818,7 +890,11 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError("path '%1%' is not in the Nix store, at %2%", path, pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("path '%1%' is not in the Nix store", path),
+                .nixCode = NixCode { .errPos = pos }
+            });
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
         state.store->ensurePath(state.store->parseStorePath(path2));
@@ -835,8 +911,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
         throw EvalError(
-                "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%", 
-                path, e.path, pos);
+            ErrorInfo {
+                .hint = hintfmt(
+                    "cannot check the existence of '%1%', since path '%2%' is not valid", 
+                    path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
     }
 
     try {
@@ -879,9 +959,13 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%"
-           , path, e.path, pos);
-    }
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid"
+                     , path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
+ }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
         throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
@@ -908,7 +992,11 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError("attribute 'path' missing, at %1%", pos);
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("attribute 'path' missing"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -916,8 +1004,12 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError("cannot find '%1%', since path '%2%' is not valid, at %3%",
-                        path, e.path, pos);
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid",
+                        path, e.path),
+                    .nixCode = NixCode { .errPos = pos }
+                });
         }
 
         searchPath.emplace_back(prefix, path);
@@ -934,7 +1026,11 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error("unknown hash type '%1%', at %2%", type, pos);
+      throw Error(
+          ErrorInfo {
+              .hint = hintfmt("unknown hash type '%1%'", type),
+              .nixCode = NixCode { .errPos = pos }
+          });
 
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
@@ -950,8 +1046,12 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%",
-                        path, e.path, pos);
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid",
+                        path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
     }
 
     DirEntries entries = readDirectory(state.checkSourcePath(path));
@@ -1021,7 +1121,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%", name, pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt(
+                        "in 'toFile': the file '%1%' cannot refer to derivation outputs",  \
+                        name),
+                    .nixCode = NixCode { .errPos = pos }
+                });
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1089,11 +1195,21 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError("first argument in call to 'filterSource' is not a function but %1%, at %2%", showType(*args[0]), pos);
+        throw TypeError(
+            ErrorInfo { 
+                .hint = hintfmt(
+                    "first argument in call to 'filterSource' is not a function but %1%", 
+                    showType(*args[0])),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
 }
@@ -1113,7 +1229,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
-                throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, *attr.pos);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("string '%1%' cannot refer to other paths", 
+                            path),
+                        .nixCode = NixCode { .errPos = *attr.pos }
+                    });
+
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
@@ -1124,10 +1246,19 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else if (n == "sha256")
             expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
-            throw EvalError("unsupported argument '%1%' to 'addPath', at %2%", attr.name, *attr.pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("unsupported argument '%1%' to 'addPath'",
+                        attr.name),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
     }
     if (path.empty())
-        throw EvalError("'path' required, at %1%", pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("'path' required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     if (name.empty())
         name = baseNameOf(path);
 
@@ -1185,7 +1316,12 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError("attribute '%1%' missing, at %2%", attr, pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("attribute '%1%' missing", attr),
+                .nixCode = NixCode { .errPos = pos }
+            });
+
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value);
@@ -1265,14 +1401,23 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError("'name' attribute missing in a call to 'listToAttrs', at %1%", pos);
+            throw TypeError(
+                ErrorInfo { 
+                    .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), 
+                    .nixCode = NixCode { .errPos = pos }
+                });
+
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError("'value' attribute missing in a call to 'listToAttrs', at %1%", pos);
+                throw TypeError(
+                    ErrorInfo { 
+                        .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), 
+                        .nixCode = NixCode { .errPos = pos }
+                    });
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
@@ -1346,7 +1491,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0]);
     if (args[0]->type != tLambda)
-        throw TypeError("'functionArgs' requires a function, at %1%", pos);
+        throw TypeError(
+            ErrorInfo { 
+                .hint = hintfmt("'functionArgs' requires a function"), 
+                .nixCode = NixCode { .errPos = pos }
+            });
+
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1396,7 +1546,12 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
 {
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
-        throw Error("list index %1% is out of bounds, at %2%", n, pos);
+        throw Error(
+            ErrorInfo { 
+                .hint = hintfmt("list index %1% is out of bounds", n), 
+                .nixCode = NixCode { .errPos = pos }
+            });
+
     state.forceValue(*list.listElems()[n]);
     v = *list.listElems()[n];
 }
@@ -1423,7 +1578,12 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error("'tail' called on an empty list, at %1%", pos);
+        throw Error(
+            ErrorInfo { 
+                .hint = hintfmt("'tail' called on an empty list"), 
+                .nixCode = NixCode { .errPos = pos }
+            });
+
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
         v.listElems()[n] = args[0]->listElems()[n + 1];
@@ -1564,7 +1724,12 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
-        throw EvalError("cannot create list of size %1%, at %2%", len, pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("cannot create list of size %1%", len), 
+                .nixCode = NixCode { .errPos = pos }
+            });
+
 
     state.mkList(v, len);
 
@@ -1722,7 +1887,12 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     state.forceValue(*args[1], pos);
 
     NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) throw EvalError("division by zero, at %1%", pos);
+    if (f2 == 0) 
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("division by zero"), 
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
@@ -1731,7 +1901,12 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         NixInt i2 = state.forceInt(*args[1], pos);
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError("overflow in integer division, at %1%", pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("overflow in integer division"), 
+                    .nixCode = NixCode { .errPos = pos }
+                });
+
         mkInt(v, i1 / i2);
     }
 }
@@ -1787,7 +1962,12 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) throw EvalError("negative start position in 'substring', at %1%", pos);
+    if (start < 0) 
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("negative start position in 'substring'"), 
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1807,7 +1987,11 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error("unknown hash type '%1%', at %2%", type, pos);
+        throw Error(
+            ErrorInfo { 
+                .hint = hintfmt("unknown hash type '%1%'", type), 
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
@@ -1849,10 +2033,18 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), 
+                    .nixCode = NixCode { .errPos = pos }
+                });
         } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
+          throw EvalError(
+              ErrorInfo { 
+                  .hint = hintfmt("invalid regular expression '%s'", re), 
+                  .nixCode = NixCode { .errPos = pos }
+              });
         }
     }
 }
@@ -1917,10 +2109,18 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
           // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos);
-        } else {
-          throw EvalError("invalid regular expression '%s', at %s", re, pos);
-        }
+          throw EvalError(
+              ErrorInfo { 
+                  .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), 
+                  .nixCode = NixCode { .errPos = pos }
+              });
+         } else {
+          throw EvalError(
+              ErrorInfo { 
+                  .hint = hintfmt("invalid regular expression '%s'", re), 
+                  .nixCode = NixCode { .errPos = pos }
+              });
+         }
     }
 }
 
@@ -1950,7 +2150,11 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%", pos);
+        throw EvalError(
+            ErrorInfo { 
+                .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), 
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     vector<string> from;
     from.reserve(args[0]->listSize());
@@ -2072,11 +2276,20 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else if (n == "name")
                 request.name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%1%' to '%2%', at %3%", attr.name, who, attr.pos);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("unsupported argument '%1%' to '%2%'", attr.name, who), 
+                        .nixCode = NixCode { .errPos = pos }
+                    });
+
         }
 
         if (request.uri.empty())
-            throw EvalError("'url' argument required, at %1%", pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("'url' argument required"), 
+                    .nixCode = NixCode { .errPos = pos }
+                });
 
     } else
         request.uri = state.forceStringNoCtx(*args[0], pos);
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 94fa0158c80..768936453bc 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -146,7 +146,12 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                    .nixCode = NixCode { .errPos = *i.pos }
+                });
+
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -160,7 +165,12 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError(
+                        ErrorInfo { 
+                            .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                            .nixCode = NixCode { .errPos = *i.pos }
+                        });
+
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +180,11 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                        .nixCode = NixCode { .errPos = *i.pos }
+                    });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 480cc1adcbf..68ffed51376 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -217,11 +217,19 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                        .nixCode = NixCode { .errPos = *attr.pos }
+                    });
         }
 
         if (url.empty())
-            throw EvalError("'url' argument required, at %1%", pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("'url' argument required"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 76029f5489c..ce99928e247 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -187,11 +187,20 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
+                throw EvalError(
+                    ErrorInfo { 
+                        .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                        .nixCode = NixCode { .errPos = *attr.pos }
+                    });
+
         }
 
         if (url.empty())
-            throw EvalError("'url' argument required, at %1%", pos);
+            throw EvalError(
+                ErrorInfo { 
+                    .hint = hintfmt("'url' argument required"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index d4305ddd8b3..ce5dde6c39b 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -1,4 +1,4 @@
-#include "types.hh"
+#include "error.hh"
 
 #include <iostream>
 #include <optional>
@@ -25,54 +25,60 @@ std::ostream& operator<<(std::ostream &os, const hintformat &hf)
 
 string showErrPos(const ErrPos &errPos)
 {
-    if (errPos.column > 0) {
-        return fmt("(%1%:%2%)", errPos.line, errPos.column);
-    } else {
-        return fmt("(%1%)", errPos.line);
-    };
+    if (errPos.line > 0) {
+        if (errPos.column > 0) {
+            return fmt("(%1%:%2%)", errPos.line, errPos.column);
+        } else {
+            return fmt("(%1%)", errPos.line);
+        }
+    }
+    else {
+        return "";
+    }
 }
 
-void printCodeLines(const string &prefix, const NixCode &nixCode)
+void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
 {
     // previous line of code.
     if (nixCode.prevLineOfCode.has_value()) {
-        std::cout << fmt("%1% %|2$5d|| %3%",
+        out << fmt("%1% %|2$5d|| %3%",
             prefix,
             (nixCode.errPos.line - 1),
             *nixCode.prevLineOfCode)
-                  << std::endl;
+            << std::endl;
     }
 
-    // line of code containing the error.%2$+5d%
-    std::cout << fmt("%1% %|2$5d|| %3%",
-        prefix,
-        (nixCode.errPos.line),
-        nixCode.errLineOfCode)
-              << std::endl;
-
-    // error arrows for the column range.
-    if (nixCode.errPos.column > 0) {
-        int start = nixCode.errPos.column;
-        std::string spaces;
-        for (int i = 0; i < start; ++i) {
-            spaces.append(" ");
-        }
+    if (nixCode.errLineOfCode.has_value()) {
+        // line of code containing the error.
+        out << fmt("%1% %|2$5d|| %3%",
+            prefix,
+            (nixCode.errPos.line),
+            *nixCode.errLineOfCode)
+            << std::endl;
+        // error arrows for the column range.
+        if (nixCode.errPos.column > 0) {
+            int start = nixCode.errPos.column;
+            std::string spaces;
+            for (int i = 0; i < start; ++i) {
+                spaces.append(" ");
+            }
 
-        std::string arrows("^");
+            std::string arrows("^");
 
-        std::cout << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-            prefix,
-            spaces,
-            arrows) << std::endl;
+            out << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                prefix,
+                spaces,
+                arrows) << std::endl;
+        }
     }
 
     // next line of code.
     if (nixCode.nextLineOfCode.has_value()) {
-        std::cout << fmt("%1% %|2$5d|| %3%",
+        out << fmt("%1% %|2$5d|| %3%",
             prefix,
             (nixCode.errPos.line + 1),
             *nixCode.nextLineOfCode)
-                  << std::endl;
+            << std::endl;
     }
 }
 
@@ -83,52 +89,52 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 
     string levelString;
     switch (einfo.level) {
-    case Verbosity::lvlError: {
-        levelString = ANSI_RED;
-        levelString += "error:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlWarn: {
-        levelString = ANSI_YELLOW;
-        levelString += "warning:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlInfo: {
-        levelString = ANSI_GREEN;
-        levelString += "info:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlTalkative: {
-        levelString = ANSI_GREEN;
-        levelString += "talk:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlChatty: {
-        levelString = ANSI_GREEN;
-        levelString += "chat:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlVomit: {
-        levelString = ANSI_GREEN;
-        levelString += "vomit:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    case Verbosity::lvlDebug: {
-        levelString = ANSI_YELLOW;
-        levelString += "debug:";
-        levelString += ANSI_NORMAL;
-        break;
-    }
-    default: {
-        levelString = fmt("invalid error level: %1%", einfo.level);
-        break;
-    }
+        case Verbosity::lvlError: {
+            levelString = ANSI_RED;
+            levelString += "error:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlWarn: {
+            levelString = ANSI_YELLOW;
+            levelString += "warning:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlInfo: {
+            levelString = ANSI_GREEN;
+            levelString += "info:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlTalkative: {
+            levelString = ANSI_GREEN;
+            levelString += "talk:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlChatty: {
+            levelString = ANSI_GREEN;
+            levelString += "chat:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlVomit: {
+            levelString = ANSI_GREEN;
+            levelString += "vomit:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        case Verbosity::lvlDebug: {
+            levelString = ANSI_YELLOW;
+            levelString += "debug:";
+            levelString += ANSI_NORMAL;
+            break;
+        }
+        default: {
+            levelString = fmt("invalid error level: %1%", einfo.level);
+            break;
+        }
     }
 
     int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
@@ -158,14 +164,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     // filename.
     if (einfo.nixCode.has_value()) {
         if (einfo.nixCode->errPos.file != "") {
-            string eline = einfo.nixCode->errLineOfCode != ""
-                           ? string(" ") + showErrPos(einfo.nixCode->errPos)
-                           : "";
-
-            out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
+            out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
                 prefix,
                 einfo.nixCode->errPos.file,
-                eline) << std::endl;
+                showErrPos(einfo.nixCode->errPos)) << std::endl;
             out << prefix << std::endl;
         } else {
             out << fmt("%1%from command line argument", prefix) << std::endl;
@@ -180,8 +182,8 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     }
 
     // lines of code.
-    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") {
-        printCodeLines(prefix, *einfo.nixCode);
+    if (einfo.nixCode.has_value()) {
+        printCodeLines(out, prefix, *einfo.nixCode);
         out << prefix << std::endl;
     }
 
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 19a806cc1d2..7e2cca2f97e 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -35,10 +35,15 @@ typedef enum {
 } Verbosity;
 
 struct ErrPos {
-    int line;
-    int column;
+    int line = 0;
+    int column = 0;
     string file;
 
+    operator bool() const
+    {
+        return line != 0;
+    }
+
     template <class P>
     ErrPos& operator=(const P &pos)
     {
@@ -58,7 +63,7 @@ struct ErrPos {
 struct NixCode {
     ErrPos errPos;
     std::optional<string> prevLineOfCode;
-    string errLineOfCode;
+    std::optional<string> errLineOfCode;
     std::optional<string> nextLineOfCode;
 };
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index e1750eb2a5f..4244d12219b 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -171,7 +171,8 @@ struct JSONLogger : Logger {
                 json["file"] = ei.nixCode->errPos.file;
             if (ei.nixCode->prevLineOfCode.has_value())
                 json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode;
-            json["errLineOfCode"] = ei.nixCode->errLineOfCode;
+            if (ei.nixCode->errLineOfCode.has_value())
+                json["errLineOfCode"] = *ei.nixCode->errLineOfCode;
             if (ei.nixCode->nextLineOfCode.has_value())
                 json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode;
         }
diff --git a/tests/misc.sh b/tests/misc.sh
index eda0164167f..fd4908e25dc 100644
--- a/tests/misc.sh
+++ b/tests/misc.sh
@@ -16,4 +16,6 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"
+eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+echo $eval_res | grep "(string) (1:15)"
+echo $eval_res | grep "infinite recursion encountered"

From 52cffafd2462c03466db15909dde250f07677d35 Mon Sep 17 00:00:00 2001
From: Dani <dmedinag@users.noreply.github.com>
Date: Sat, 9 May 2020 13:48:31 +0200
Subject: [PATCH 2079/2196] Fix typo

---
 doc/manual/release-notes/rl-0.8.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml
index 784b26c6b7d..825798fa9b0 100644
--- a/doc/manual/release-notes/rl-0.8.xml
+++ b/doc/manual/release-notes/rl-0.8.xml
@@ -8,7 +8,7 @@
 
 <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
 As a result, <command>nix-pull</command> manifests and channels built
-for Nix 0.7 and below will now work anymore.  However, the Nix
+for Nix 0.7 and below will not work anymore.  However, the Nix
 expression language has not changed, so you can still build from
 source.  Also, existing user environments continue to work.  Nix 0.8
 will automatically upgrade the database schema of previous

From 446649e5403f9c41601eee6438eaeed5212b9190 Mon Sep 17 00:00:00 2001
From: Shao Cheng <astrohavoc@gmail.com>
Date: Sat, 9 May 2020 15:59:39 +0200
Subject: [PATCH 2080/2196] Update "Upgrading Nix" documentation

This PR proposes two changes to the "Upgrading Nix" documentation:

* Besides updating `nixpkgs.nix`, we also update `nixpkgs.cacert`, so that the certificates are up-to-date as well.
* Add the instructions for multi-user mode on Linux.
---
 doc/manual/installation/upgrading.xml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml
index 30670d7fec9..592f63895c5 100644
--- a/doc/manual/installation/upgrading.xml
+++ b/doc/manual/installation/upgrading.xml
@@ -17,6 +17,11 @@
 
   <para>
     Single-user installations of Nix should run this:
-    <command>nix-channel --update; nix-env -iA nixpkgs.nix</command>
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
+  </para>
+      
+  <para>
+    Multi-user Nix users on Linux should run this with sudo:
+    <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
   </para>
 </chapter>

From 146f9c114ff347d372b5ab6c5fc00fdd3f2895da Mon Sep 17 00:00:00 2001
From: Benjamin Hipple <bhipple@protonmail.com>
Date: Sat, 9 May 2020 10:58:43 -0400
Subject: [PATCH 2081/2196] doc: consistently refer to 'fixed-output' with a
 dash

General cleanup that makes it easier to search for the term.
---
 doc/manual/command-ref/nix-store.xml | 2 +-
 src/libstore/build.cc                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml
index 1ddb5408d5d..d71f9d8e47d 100644
--- a/doc/manual/command-ref/nix-store.xml
+++ b/doc/manual/command-ref/nix-store.xml
@@ -936,7 +936,7 @@ $ nix-store --add ./foo.c
 
 <para>The operation <option>--add-fixed</option> adds the specified paths to
 the Nix store.  Unlike <option>--add</option> paths are registered using the
-specified hashing algorithm, resulting in the same output path as a fixed output
+specified hashing algorithm, resulting in the same output path as a fixed-output
 derivation.  This can be used for sources that are not available from a public
 url or broke since the download expression was written.
 </para>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 147093fae26..6258bcb3347 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3155,7 +3155,7 @@ void DerivationGoal::runChild()
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
-                // potential impurities introduced in fixed outputs.
+                // potential impurities introduced in fixed-outputs.
                 writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
 
                 ss.push_back("/etc/services");

From 1d8144e36b18d3467f1b29b80800ed3eb9b876c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Mon, 11 May 2020 18:14:23 +0200
Subject: [PATCH 2082/2196] Update src/libstore/build.cc

---
 src/libstore/build.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index e0c7324c744..f9b70191068 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3679,7 +3679,7 @@ void DerivationGoal::registerOutputs()
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
                         format("output path '%1%' should be a non-executable regular file "
-                               "since recursive hashing is not enabled (outputHashMode=flat).") % path);
+                               "since recursive hashing is not enabled (outputHashMode=flat)") % path);
             }
 
             /* Check the hash. In hash mode, move the path produced by

From 958e81987b5b86fba06090078a556f51037aa23c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 11 May 2020 13:02:16 -0600
Subject: [PATCH 2083/2196] switch from printError warnings to logWarnings

---
 src/libstore/build.cc      | 14 +++++++++++---
 src/libstore/store-api.cc  |  6 +++++-
 src/libutil/error.cc       |  2 +-
 src/libutil/serialise.cc   |  5 ++++-
 src/nix-build/nix-build.cc |  9 ++++++---
 src/nix-store/nix-store.cc |  5 ++++-
 6 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f8cc1ce369e..a7d6e53b099 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3647,7 +3647,11 @@ void DerivationGoal::registerOutputs()
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
         if (!outputRewrites.empty()) {
-            printError("warning: rewriting hashes in '%1%'; cross fingers", path);
+            logWarning(
+                ErrorInfo {
+                    .name = "Rewriting hashes", 
+                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
+                    });
 
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
@@ -4414,8 +4418,12 @@ void SubstitutionGoal::tryNext()
         && !sub->isTrusted
         && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
     {
-        printError("warning: substituter '%s' does not have a valid signature for path '%s'",
-            sub->getUri(), worker.store.printStorePath(storePath));
+        logWarning(
+            ErrorInfo {
+                .name = "Invalid path signature", 
+                .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), worker.store.printStorePath(storePath))
+            });
         tryNext();
         return;
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 542e5c552d8..d1281d130c0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -769,7 +769,11 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
     auto warn = [&]() {
-        printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
+        logWarning(
+            ErrorInfo{
+                .name = "Path not content-addressed", 
+                .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
+            });
     };
 
     if (hasPrefix(ca, "text:")) {
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index ce5dde6c39b..e4f45a2d3c4 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -161,7 +161,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
             einfo.programName.value_or(""))
             << std::endl;
 
-    // filename.
+    // filename, line, column.
     if (einfo.nixCode.has_value()) {
         if (einfo.nixCode->errPos.file != "") {
             out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 8201549fd7d..35f7ee917cd 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -52,7 +52,10 @@ size_t threshold = 256 * 1024 * 1024;
 
 static void warnLargeDump()
 {
-    printError("warning: dumping very large path (> 256 MiB); this may run out of memory");
+    logWarning(ErrorInfo {
+        .name = "Large path",
+        .description = "dumping very large path (> 256 MiB); this may run out of memory"
+        });
 }
 
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 3c2f4f00ce2..03200d050ed 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -368,9 +368,12 @@ static void _main(int argc, char * * argv)
                 shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
-                // TODO: append error msg; warn()?
-                logError(e.info());
-                printError("warning: %s; will use bash from your environment", e.what());
+                logWarning(
+                    ErrorInfo {
+                        .name = "bashInteractive",
+                        .hint = hintfmt("%s; will use bash from your environment", 
+                            (e.info().hint ? e.info().hint->str() : ""))
+                    });
                 shell = "bash";
             }
         }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 57063d42fd5..9b5cceccf58 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -704,7 +704,10 @@ static void opVerify(Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", i);
 
     if (store->verifyStore(checkContents, repair)) {
-        printError("warning: not all errors were fixed");
+        logWarning(ErrorInfo { 
+            .name = "Store consistency",
+            .description = "not all errors were fixed"
+            });
         throw Exit(1);
     }
 }

From 536bbf53e12c80f52c2679aec734d895b0058f5b Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 11 May 2020 13:58:38 -0600
Subject: [PATCH 2084/2196] comments and cleanup

---
 src/error-demo/error-demo.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 41293427c9b..8f484431c3b 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -10,27 +10,27 @@ MakeError(DemoError, Error);
 
 int main()
 {
-    makeDefaultLogger();
-
     verbosity = lvlVomit;
 
     // In each program where errors occur, this has to be set.
     ErrorInfo::programName = std::optional("error-demo");
 
+    // 'DemoError' appears as error name.
     try {
         throw DemoError("demo error was thrown");
     } catch (Error &e) {
         logger->logEI(e.info());
     }
 
-    // ErrorInfo constructor
+    // appending to the hint from the previous error
     try {
-        auto e = Error("some error");
+        auto e = Error("initial error");
         throw DemoError(e.info());
     } catch (Error &e) {
         ErrorInfo ei = e.info();
-        string prevhint = (e.info().hint.has_value() ? e.info().hint->str() : "");
-        ei.hint = std::optional(hintfmt("previous hint was: %s", normaltxt(prevhint)));
+        // using normaltxt to avoid the default yellow highlighting.
+        ei.hint = hintfmt("%s; subsequent error message.", 
+            normaltxt(e.info().hint ? e.info().hint->str() : ""));
         logger->logEI(ei);
     }
 
@@ -38,8 +38,8 @@ int main()
     try {
         auto x = readFile(-1);
     }
-    catch (Error &e) {
-        std::cout << "error: " << e.sname() << std::endl;
+    catch (SysError &e) {
+        std::cout << "errno was: " << e.errNo << std::endl;
         logError(e.info());
     }
 
@@ -59,7 +59,7 @@ int main()
         }
     }
 
-    // For completeness sake, info through vomit levels.
+    // For completeness sake, show 'info' through 'vomit' levels.
     // But this is maybe a heavy format for those.
     logger->logEI(
         ErrorInfo { .level = lvlInfo,

From b93c1bf3d67716231d2ff7ee4154b8c80a251b10 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 11 May 2020 15:52:15 -0600
Subject: [PATCH 2085/2196] fixes to merged code

---
 src/libexpr/nixexpr.hh       |  1 +
 src/libexpr/primops.cc       |  2 +-
 src/libmain/stack.cc         |  2 +-
 src/libstore/build.cc        | 20 ++++++++++----------
 src/libstore/filetransfer.hh |  3 ++-
 src/libstore/local-store.cc  |  1 +
 src/libstore/sqlite.hh       |  2 +-
 src/libutil/args.cc          |  7 -------
 src/libutil/error.hh         |  4 +---
 src/libutil/logging.hh       |  1 +
 src/libutil/types.hh         |  1 +
 src/libutil/url.hh           |  2 +-
 src/libutil/util.cc          | 16 ++++++++--------
 13 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 137fe7198e8..a185d578573 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -2,6 +2,7 @@
 
 #include "value.hh"
 #include "symbol-table.hh"
+#include "error.hh"
 
 #include <map>
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 10e8c6b4255..657ce300148 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1124,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
                 ErrorInfo { 
                     .hint = hintfmt(
                         "in 'toFile': the file named '%1%' must not contain a reference "
-                        "to a derivation but contains (%2%)"
+                        "to a derivation but contains (%2%)",
                         name,
                         path),
                     .nixCode = NixCode { .errPos = pos }
diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index e6224de7d28..b0a4a4c5dbe 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -1,4 +1,4 @@
-#include "types.hh"
+#include "error.hh"
 
 #include <cstring>
 #include <cstddef>
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d6f3553d96d..0359b10ac5d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -547,7 +547,7 @@ UserLock::UserLock()
     /* Copy the result of getgrnam. */
     Strings users;
     for (char * * p = gr->gr_mem; *p; ++p) {
-        debug(format("found build user '%1%'") % *p);
+        debug("found build user '%1%'", *p);
         users.push_back(*p);
     }
 
@@ -558,7 +558,7 @@ UserLock::UserLock()
     /* Find a user account that isn't currently in use for another
        build. */
     for (auto & i : users) {
-        debug(format("trying user '%1%'") % i);
+        debug("trying user '%1%'", i);
 
         struct passwd * pw = getpwnam(i.c_str());
         if (!pw)
@@ -1794,7 +1794,7 @@ HookReply DerivationGoal::tryBuildHook()
             }
         }
 
-        debug(format("hook reply is '%1%'") % reply);
+        debug("hook reply is '%1%'", reply);
 
         if (reply == "decline")
             return rpDecline;
@@ -2255,7 +2255,7 @@ void DerivationGoal::startBuilder()
         startDaemon();
 
     /* Run the builder. */
-    printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder);
+    printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
 
     /* Create the log file. */
     Path logFile = openLogFile();
@@ -3195,7 +3195,7 @@ void DerivationGoal::runChild()
                filesystem that we want in the chroot
                environment. */
             auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
-                debug(format("bind mounting '%1%' to '%2%'") % source % target);
+                debug("bind mounting '%1%' to '%2%'", source, target);
                 struct stat st;
                 if (stat(source.c_str(), &st) == -1) {
                     if (optional && errno == ENOENT)
@@ -3572,7 +3572,7 @@ static void moveCheckToStore(const Path & src, const Path & dst)
        directory's parent link ".."). */
     struct stat st;
     if (lstat(src.c_str(), &st) == -1) {
-        throw SysError(format("getting attributes of path '%1%'") % src);
+        throw SysError("getting attributes of path '%1%'", src);
     }
 
     bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
@@ -3581,7 +3581,7 @@ static void moveCheckToStore(const Path & src, const Path & dst)
         chmod_(src, st.st_mode | S_IWUSR);
 
     if (rename(src.c_str(), dst.c_str()))
-        throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
+        throw SysError("renaming '%1%' to '%2%'", src, dst);
 
     if (changePerm)
         chmod_(dst, st.st_mode);
@@ -4911,15 +4911,15 @@ void Worker::waitForInput()
                 // FIXME: is there a cleaner way to handle pt close
                 // than EIO? Is this even standard?
                 if (rd == 0 || (rd == -1 && errno == EIO)) {
-                    debug(format("%1%: got EOF") % goal->getName());
+                    debug("%1%: got EOF", goal->getName());
                     goal->handleEOF(k);
                     j->fds.erase(k);
                 } else if (rd == -1) {
                     if (errno != EINTR)
                         throw SysError("%s: read failed", goal->getName());
                 } else {
-                    printMsg(lvlVomit, format("%1%: read %2% bytes")
-                        % goal->getName() % rd);
+                    printMsg(lvlVomit, "%1%: read %2% bytes",
+                        goal->getName(), rd);
                     string data((char *) buffer.data(), rd);
                     j->lastOutput = after;
                     goal->handleChildOutput(k, data);
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 517b1a7d3d9..11dca2fe0f0 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -103,8 +103,9 @@ class FileTransferError : public Error
 {
 public:
     FileTransfer::Error error;
+    template<typename... Args>
     FileTransferError(FileTransfer::Error error, const Args & ... args)
-        : Error(fs), error(error)
+        : Error(args...), error(error)
     { }
 };
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1f16a22b117..eb754854309 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -6,6 +6,7 @@
 #include "derivations.hh"
 #include "nar-info.hh"
 #include "references.hh"
+#include "error.hh"
 
 #include <iostream>
 #include <algorithm>
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index ce27033c927..ccfac48d8eb 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -3,7 +3,7 @@
 #include <functional>
 #include <string>
 
-#include "types.hh"
+#include "error.hh"
 
 struct sqlite3;
 struct sqlite3_stmt;
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 423b99c96df..10d6e89bb7c 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -101,15 +101,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         std::vector<std::string> args;
         for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-<<<<<<< HEAD
-                if (flag.arity == ArityAny) break;
-                throw UsageError("flag '%1%' requires %2% argument(s)",
-                    name,
-                    flag.arity);
-=======
                 if (flag.handler.arity == ArityAny) break;
                 throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
->>>>>>> master
             }
             args.push_back(*pos++);
         }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 7e2cca2f97e..19c6e35a1c0 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -2,6 +2,7 @@
 
 
 #include "ref.hh"
+#include "types.hh"
 
 #include <list>
 #include <memory>
@@ -21,9 +22,6 @@
 
 namespace nix {
 
-using std::list;
-using std::vector;
-
 typedef enum {
     lvlError = 0,
     lvlWarn,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index bb7d356c8e9..39692a29113 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "types.hh"
+#include "error.hh"
 
 namespace nix {
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index a831b99245b..4d783b56434 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -13,6 +13,7 @@ namespace nix {
 using std::list;
 using std::set;
 using std::vector;
+using std::string;
 
 typedef list<std::string> Strings;
 typedef set<std::string> StringSet;
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 1503023a2fc..a184eadce07 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "types.hh"
+#include "error.hh"
 
 #include <regex>
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 8207ff70117..ac7c2967b4a 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -276,7 +276,7 @@ DirEntries readDirectory(DIR *dir, const Path & path)
 DirEntries readDirectory(const Path & path)
 {
     AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+    if (!dir) throw SysError("opening directory '%1%'", path);
 
     return readDirectory(dir.get(), path);
 }
@@ -306,7 +306,7 @@ string readFile(const Path & path)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd)
-        throw SysError(format("opening file '%1%'") % path);
+        throw SysError("opening file '%1%'", path);
     return readFile(fd.get());
 }
 
@@ -394,15 +394,15 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
         const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
         if ((st.st_mode & PERM_MASK) != PERM_MASK) {
             if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
-                throw SysError(format("chmod '%1%'") % path);
+                throw SysError("chmod '%1%'", path);
         }
 
         int fd = openat(parentfd, path.c_str(), O_RDONLY);
         if (!fd)
-            throw SysError(format("opening directory '%1%'") % path);
+            throw SysError("opening directory '%1%'", path);
         AutoCloseDir dir(fdopendir(fd));
         if (!dir)
-            throw SysError(format("opening directory '%1%'") % path);
+            throw SysError("opening directory '%1%'", path);
         for (auto & i : readDirectory(dir.get(), path))
             _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
     }
@@ -426,7 +426,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
         // for backwards compatibility.
         if (errno == ENOENT) return;
 
-        throw SysError(format("opening directory '%1%'") % path);
+        throw SysError("opening directory '%1%'", path);
     }
 
     _deletePath(dirfd.get(), path, bytesFreed);
@@ -845,7 +845,7 @@ int Pid::kill()
 {
     assert(pid != -1);
 
-    debug(format("killing process %1%") % pid);
+    debug("killing process %1%", pid);
 
     /* Send the requested signal to the child.  If it has its own
        process group, send the signal to every process in the child
@@ -903,7 +903,7 @@ pid_t Pid::release()
 
 void killUser(uid_t uid)
 {
-    debug(format("killing all processes running under uid '%1%'") % uid);
+    debug("killing all processes running under uid '%1%'", uid);
 
     assert(uid != 0); /* just to be safe... */
 

From 631642c5b40032a087b8e2e131b2ebcac00913d3 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 11 May 2020 16:58:08 -0600
Subject: [PATCH 2086/2196] new format for pos

---
 src/libexpr/parser.y | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 82d5753ab0e..6d50d74bab8 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -64,15 +64,23 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError("attribute '%1%' at %2% already defined at %3%",
-        showAttrPath(attrPath), pos, prevPos);
+    throw ParseError(
+        ErrorInfo {
+            .hint = hintfmt("attribute '%1%' already defined at %2%",
+                showAttrPath(attrPath), prevPos),
+            .nixCode = NixCode { .errPos = pos },
+        });
 }
 
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError("attribute '%1%' at %2% already defined at %3%",
-        attr, pos, prevPos);
+    throw ParseError(
+        ErrorInfo {
+            .hint = hintfmt("attribute '%1%' already defined at %2%",
+                attr, prevPos),
+            .nixCode = NixCode { .errPos = pos },
+        });
 }
 
 
@@ -140,8 +148,12 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError("duplicate formal function argument '%1%' at %2%",
-            formal.name, pos);
+        throw ParseError(
+            ErrorInfo {
+                .hint = hintfmt("duplicate formal function argument '%1%'",
+                    formal.name),
+                .nixCode = NixCode { .errPos = pos },
+            });
     formals->formals.push_front(formal);
 }
 
@@ -327,7 +339,11 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError("dynamic attributes not allowed in let at %1%", CUR_POS);
+        throw ParseError(
+            ErrorInfo {
+                .hint = hintfmt("dynamic attributes not allowed in let"),
+                .nixCode = NixCode { .errPos = CUR_POS },
+            });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
@@ -479,8 +495,11 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError("dynamic attributes not allowed in inherit at %1%",
-              makeCurPos(@2, data));
+          throw ParseError(
+              ErrorInfo {
+                  .hint = hintfmt("dynamic attributes not allowed in inherit"),
+                  .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+              });
     }
   | { $$ = new AttrPath; }
   ;

From 7c3138844ca9ab92e2f231a7d17d7aee420ae76e Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Mon, 11 May 2020 17:34:57 -0600
Subject: [PATCH 2087/2196] more pos reporting

---
 src/libexpr/nixexpr.cc | 8 ++++++--
 src/libexpr/parser.y   | 8 +++++---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index a9955f6df7c..91a50830542 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -267,8 +267,12 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw UndefinedVarError("undefined variable '%1%' at %2%", name, pos);
-
+    if (withLevel == -1) 
+        throw UndefinedVarError(
+            ErrorInfo {
+                .hint = hintfmt("undefined variable '%1%'", name),
+                .nixCode = NixCode { .errPos = pos }
+            });
     fromWith = true;
     this->level = withLevel;
 }
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 6d50d74bab8..3ae7bbafd45 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -31,7 +31,7 @@ namespace nix {
         Expr * result;
         Path basePath;
         Symbol path;
-        string error;
+        ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
@@ -261,8 +261,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at %2%")
-        % error % makeCurPos(*loc, data)).str();
+    data->error = ErrorInfo { 
+        .hint = hintfmt(error),
+        .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
+        };
 }
 
 

From 46be11b762d6048746728e6e546dd4f0fac581fc Mon Sep 17 00:00:00 2001
From: Pavol Rusnak <pavol@rusnak.io>
Date: Tue, 12 May 2020 12:13:40 +0200
Subject: [PATCH 2088/2196] Introduce NIX_INSTALLER_NO_CHANNEL_ADD which skips
 nix-channel --add

---
 scripts/install-nix-from-closure.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index e00708f6cdb..21915b37d1d 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -130,13 +130,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
-if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-    $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
-fi
-if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    if ! $nix/bin/nix-channel --update nixpkgs; then
-        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
-        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
+        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    fi
+    if [ -z "$_NIX_INSTALLER_TEST" ]; then
+        if ! $nix/bin/nix-channel --update nixpkgs; then
+            echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+            echo "To try again later, run \"nix-channel --update nixpkgs\"."
+        fi
     fi
 fi
 

From 5722f9690c39d136ffbf452f3b85df09e8127712 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 12 May 2020 13:49:55 +0200
Subject: [PATCH 2089/2196] tests/binary-cache.sh: Improve incomplete closure
 test

Issue #3373.
---
 tests/binary-cache.sh          | 18 ++++++++++++++++--
 tests/build-hook.nix           |  4 ++--
 tests/build-remote.sh          |  4 ++--
 tests/config.nix.in            |  2 +-
 tests/dependencies.builder1.sh |  2 --
 tests/dependencies.builder2.sh |  2 --
 tests/dependencies.nix         | 15 ++++++++++++---
 tests/dependencies.sh          |  2 +-
 tests/export-graph.sh          |  2 +-
 tests/gc-concurrent.nix        |  6 +++---
 tests/nix-channel.sh           |  8 ++++----
 11 files changed, 42 insertions(+), 23 deletions(-)
 delete mode 100644 tests/dependencies.builder1.sh
 delete mode 100644 tests/dependencies.builder2.sh

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index a3c3c78472f..17b63d978e7 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -105,10 +105,24 @@ mv $cacheDir/nar2 $cacheDir/nar
 # incomplete closure.
 clearStore
 
-rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
+rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
-grep -q "copying path" $TEST_ROOT/log
+grep -q "copying path.*input-0" $TEST_ROOT/log
+grep -q "copying path.*input-2" $TEST_ROOT/log
+grep -q "copying path.*top" $TEST_ROOT/log
+
+
+# Idem, but without cached .narinfo.
+clearStore
+clearCacheCache
+
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+grep -q "don't know how to build" $TEST_ROOT/log
+grep -q "building.*input-1" $TEST_ROOT/log
+grep -q "building.*input-2" $TEST_ROOT/log
+grep -q "copying path.*input-0" $TEST_ROOT/log
+grep -q "copying path.*top" $TEST_ROOT/log
 
 
 if [ -n "$HAVE_SODIUM" ]; then
diff --git a/tests/build-hook.nix b/tests/build-hook.nix
index 8bff0fe7903..8c5ca8cd3c7 100644
--- a/tests/build-hook.nix
+++ b/tests/build-hook.nix
@@ -4,13 +4,13 @@ let
 
   input1 = mkDerivation {
     name = "build-hook-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
     requiredSystemFeatures = ["foo"];
   };
 
   input2 = mkDerivation {
     name = "build-hook-input-2";
-    builder = ./dependencies.builder2.sh;
+    buildCommand = "mkdir $out; echo BAR > $out/bar";
   };
 
 in
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index ddd68f327a1..a550f4460de 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -20,5 +20,5 @@ cat $outPath/foobar | grep FOOBAR
 
 # Ensure that input1 was built on store1 due to the required feature.
 p=$(readlink -f $outPath/input-2)
-(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh)
-nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh
+(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh)
+nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh
diff --git a/tests/config.nix.in b/tests/config.nix.in
index 0ec2eba6ba6..a57a8c5962e 100644
--- a/tests/config.nix.in
+++ b/tests/config.nix.in
@@ -11,7 +11,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
diff --git a/tests/dependencies.builder1.sh b/tests/dependencies.builder1.sh
deleted file mode 100644
index 4b006a17d70..00000000000
--- a/tests/dependencies.builder1.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-mkdir $out
-echo FOO > $out/foo
diff --git a/tests/dependencies.builder2.sh b/tests/dependencies.builder2.sh
deleted file mode 100644
index 4f886fdb3a1..00000000000
--- a/tests/dependencies.builder2.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-mkdir $out
-echo BAR > $out/bar
diff --git a/tests/dependencies.nix b/tests/dependencies.nix
index eca4b2964cf..e320d81c92e 100644
--- a/tests/dependencies.nix
+++ b/tests/dependencies.nix
@@ -2,18 +2,27 @@ with import ./config.nix;
 
 let {
 
+  input0 = mkDerivation {
+    name = "dependencies-input-0";
+    buildCommand = "mkdir $out; echo foo > $out/bar";
+  };
+
   input1 = mkDerivation {
     name = "dependencies-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
   };
 
   input2 = mkDerivation {
     name = "dependencies-input-2";
-    builder = "${./dependencies.builder2.sh}";
+    buildCommand = ''
+      mkdir $out
+      echo BAR > $out/bar
+      echo ${input0} > $out/input0
+    '';
   };
 
   body = mkDerivation {
-    name = "dependencies";
+    name = "dependencies-top";
     builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
     input1 = input1 + "/.";
     input2 = "${input2}/.";
diff --git a/tests/dependencies.sh b/tests/dependencies.sh
index 8d0fdc10f0b..092950aa7ed 100644
--- a/tests/dependencies.sh
+++ b/tests/dependencies.sh
@@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix)
 
 echo "derivation is $drvPath"
 
-nix-store -q --tree "$drvPath" | grep '───.*builder1.sh'
+nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh'
 
 # Test Graphviz graph generation.
 nix-store -q --graph "$drvPath" > $TEST_ROOT/graph
diff --git a/tests/export-graph.sh b/tests/export-graph.sh
index a6fd6905442..a1449b34ef2 100644
--- a/tests/export-graph.sh
+++ b/tests/export-graph.sh
@@ -11,7 +11,7 @@ checkRef() {
 
 outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result)
 
-test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references"
+test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references"
 
 checkRef input-2
 for i in $(cat $outPath); do checkRef $i; done
diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix
index c0595cc471b..21671ea2c13 100644
--- a/tests/gc-concurrent.nix
+++ b/tests/gc-concurrent.nix
@@ -4,12 +4,12 @@ rec {
 
   input1 = mkDerivation {
     name = "dependencies-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
   };
 
   input2 = mkDerivation {
     name = "dependencies-input-2";
-    builder = ./dependencies.builder2.sh;
+    buildCommand = "mkdir $out; echo BAR > $out/bar";
   };
 
   test1 = mkDerivation {
@@ -23,5 +23,5 @@ rec {
     builder = ./gc-concurrent2.builder.sh;
     inherit input1 input2;
   };
-  
+
 }
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 93f837befce..49c68981acf 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -32,10 +32,10 @@ if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 
 # Do an install.
-nix-env -i dependencies
+nix-env -i dependencies-top
 [ -e $TEST_HOME/.nix-profile/foobar ]
 
 clearProfiles
@@ -51,9 +51,9 @@ if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 
 # Do an install.
-nix-env -i dependencies
+nix-env -i dependencies-top
 [ -e $TEST_HOME/.nix-profile/foobar ]
 

From 268ecf5b3f3efd4e2a1fd78d088bdc69c27b6c7e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 May 2020 16:46:25 +0200
Subject: [PATCH 2090/2196] nix: Don't require
 --experimental-features=nix-command for some subcommands

---
 src/nix/main.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5cf09c4f056..ef301580ab9 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -167,12 +167,15 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
-    settings.requireExperimentalFeature("nix-command");
-
     initPlugins();
 
     if (!args.command) args.showHelpAndExit();
 
+    if (args.command->first != "repl"
+        && args.command->first != "doctor"
+        && args.command->first != "upgrade-nix")
+        settings.requireExperimentalFeature("nix-command");
+
     Finally f([]() { stopProgressBar(); });
 
     startProgressBar(args.printBuildLogs);

From ebc024df2287085d48ed6194aa756fd70c07f76c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 May 2020 17:25:27 +0200
Subject: [PATCH 2091/2196] Show hint how to enable experimental features

---
 src/libstore/globals.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index a0a2d850ec1..bee94cbd88c 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -130,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%s' is disabled", name);
+        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
 }
 
 bool Settings::isWSL1()

From ec870b9c853ad86fb1ccb482ca87802f1b155a2c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 10:52:26 -0600
Subject: [PATCH 2092/2196] new pos format for more errors

---
 src/libexpr/eval.cc              | 11 ++++++++---
 src/libexpr/nixexpr.hh           |  6 +++++-
 src/libexpr/primops/fetchTree.cc | 23 +++++++++++++++++------
 src/libexpr/primops/fromTOML.cc  |  6 +++++-
 src/libstore/local-store.cc      |  1 -
 5 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 65071206f25..7af665118a1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -529,7 +529,12 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
 {
-    throw EvalError(s, sym, p1, p2);
+    // p1 is where the error occurred; p2 is a position mentioned in the message.
+    throw EvalError(
+        ErrorInfo { 
+            .hint = hintfmt(s, sym, p2),
+            .nixCode = NixCode { .errPos = p1 }
+        });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
@@ -638,7 +643,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos);
+            throwUndefinedVarError("undefined variable '%1%'", var.name, var.pos);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -950,7 +955,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos);
+            throwEvalError("dynamic attribute '%1%' already defined at %2%", nameSym, i.pos, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index a185d578573..47d0e85ec7f 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -236,7 +236,11 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos);
+            throw ParseError(
+                ErrorInfo {
+                    .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                    .nixCode = NixCode { .errPos = pos }
+                });
     };
     void setName(Symbol & name);
     string showNamePos() const;
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index c5a0d988683..af3b61d6a5c 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -66,7 +66,11 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         }
 
         if (!attrs.count("type"))
-            throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
+            throw Error(
+                ErrorInfo {
+                    .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                    .nixCode = NixCode { .errPos = pos }
+                    });
 
         input = fetchers::inputFromAttrs(attrs);
     } else
@@ -107,13 +111,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError("unsupported argument '%s' to '%s', at %s",
-                    attr.name, who, *attr.pos);
-        }
+                throw EvalError(
+                    ErrorInfo {
+                        .hint = hintfmt("unsupported argument '%s' to '%s'",
+                            attr.name, who),
+                        .nixCode = NixCode { .errPos = *attr.pos }
+                    });
+            }
 
         if (!url)
-            throw EvalError("'url' argument required, at %s", pos);
-
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("'url' argument required"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index c43324dbb51..94806940130 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -81,7 +81,11 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError("while parsing a TOML string at %s: %s", pos, e.what());
+        throw EvalError(
+            ErrorInfo {
+                .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+                .nixCode = NixCode { .errPos = pos }
+            });
     }
 }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index eb754854309..1f16a22b117 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -6,7 +6,6 @@
 #include "derivations.hh"
 #include "nar-info.hh"
 #include "references.hh"
-#include "error.hh"
 
 #include <iostream>
 #include <algorithm>

From 9e12b2f5b86963bc6d3a4ee37cc759357fb62263 Mon Sep 17 00:00:00 2001
From: Pavol Rusnak <pavol@rusnak.io>
Date: Tue, 12 May 2020 18:58:13 +0200
Subject: [PATCH 2093/2196] Expose installer configuration environment
 variables via command line flags

---
 scripts/install-nix-from-closure.sh | 61 +++++++++++++++++------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 21915b37d1d..e06530ddf3d 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -40,30 +40,43 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-# Trivially handle the --daemon / --no-daemon options
-if [ "x${1:-}" = "x--no-daemon" ]; then
-    INSTALL_MODE=no-daemon
-elif [ "x${1:-}" = "x--daemon" ]; then
-    INSTALL_MODE=daemon
-elif [ "x${1:-}" != "x" ]; then
-    (
-        echo "Nix Installer [--daemon|--no-daemon]"
-
-        echo "Choose installation method."
-        echo ""
-        echo " --daemon:    Installs and configures a background daemon that manages the store,"
-        echo "              providing multi-user support and better isolation for local builds."
-        echo "              Both for security and reproducibility, this method is recommended if"
-        echo "              supported on your platform."
-        echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
-        echo ""
-        echo " --no-daemon: Simple, single-user installation that does not require root and is"
-        echo "              trivial to uninstall."
-        echo "              (default)"
-        echo ""
-    ) >&2
-    exit
-fi
+
+# handle the command line flags
+while [ "x${1:-}" != "x" ]; do
+    if [ "x${1:-}" = "x--no-daemon" ]; then
+        INSTALL_MODE=no-daemon
+    elif [ "x${1:-}" = "x--daemon" ]; then
+        INSTALL_MODE=daemon
+    elif [ "x${1:-}" = "x--no-channel-add" ]; then
+        NIX_INSTALLER_NO_CHANNEL_ADD=1
+    elif [ "x${1:-}" = "x--no-modify-profile" ]; then
+        NIX_INSTALLER_NO_MODIFY_PROFILE=1
+    elif [ "x${1:-}" != "x" ]; then
+        (
+            echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]"
+
+            echo "Choose installation method."
+            echo ""
+            echo " --daemon:            Installs and configures a background daemon that manages the store,"
+            echo "                      providing multi-user support and better isolation for local builds."
+            echo "                      Both for security and reproducibility, this method is recommended if"
+            echo "                      supported on your platform."
+            echo "                      See https://nixos.org/nix/manual/#sect-multi-user-installation"
+            echo ""
+            echo " --no-daemon:         Simple, single-user installation that does not require root and is"
+            echo "                      trivial to uninstall."
+            echo "                      (default)"
+            echo ""
+            echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
+            echo ""
+            echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
+            echo "                      is installed by default."
+            echo ""
+        ) >&2
+        exit
+    fi
+    shift
+done
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
     printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'

From 2a19bf86197083e17d9b35596c21e3c1b3f4a170 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 11:27:37 -0600
Subject: [PATCH 2094/2196] move pos to the first arg, to indicate its not used
 in a fmt template

---
 src/libexpr/eval-inline.hh | 10 +++----
 src/libexpr/eval.cc        | 58 +++++++++++++++++++-------------------
 2 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index e6b8386651d..eee49e02e3e 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -7,7 +7,7 @@
 
 namespace nix {
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
     throw EvalError(
         ErrorInfo { 
@@ -22,7 +22,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 }
 
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
     throw TypeError(
         ErrorInfo { 
@@ -51,7 +51,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.type == tBlackhole)
-        throwEvalError("infinite recursion encountered", pos);
+        throwEvalError(pos, "infinite recursion encountered");
 }
 
 
@@ -67,7 +67,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while a set was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 
 
@@ -83,7 +83,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (!v.isList())
-        throwTypeError("value is %1% while a list was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a list was expected", v);
 }
 
 /* Note: Various places expect the allocated memory to be zeroed. */
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7af665118a1..e4bc6dc8359 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -504,7 +504,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
     throw EvalError(s, s2);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
     throw EvalError(
         ErrorInfo { 
@@ -518,7 +518,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
     throw EvalError(s, s2, s3);
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
     throw EvalError(
         ErrorInfo { 
@@ -527,7 +527,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
         });
 }
 
-LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2))
+LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
 {
     // p1 is where the error occurred; p2 is a position mentioned in the message.
     throw EvalError(
@@ -537,7 +537,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, co
         });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
     throw TypeError(
         ErrorInfo { 
@@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
     throw TypeError(s, s1);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos))
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
     throw TypeError(
         ErrorInfo { 
@@ -560,7 +560,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun
         });
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
     throw AssertionError(
         ErrorInfo { 
@@ -569,7 +569,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1
         });
 }
 
-LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
+LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
     throw UndefinedVarError(
         ErrorInfo { 
@@ -643,7 +643,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
             return j->value;
         }
         if (!env->prevWith)
-            throwUndefinedVarError("undefined variable '%1%'", var.name, var.pos);
+            throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
         for (size_t l = env->prevWith; l; --l, env = env->up) ;
     }
 }
@@ -841,7 +841,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
     Value v;
     e->eval(*this, env, v);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -955,7 +955,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Symbol nameSym = state.symbols.create(nameVal.string.s);
         Bindings::iterator j = v.attrs->find(nameSym);
         if (j != v.attrs->end())
-            throwEvalError("dynamic attribute '%1%' already defined at %2%", nameSym, i.pos, *j->pos);
+            throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
@@ -1043,7 +1043,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos);
                 if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError("attribute '%1%' missing", name, pos);
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -1169,7 +1169,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
     }
 
     if (fun.type != tLambda)
-        throwTypeError("attempt to call something which is not a function but %1%", fun, pos);
+        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
 
@@ -1197,8 +1197,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         for (auto & i : lambda.formals->formals) {
             Bindings::iterator j = arg.attrs->find(i.name);
             if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError("%1% called without required argument '%2%'",
-                    lambda, i.name, pos);
+                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                    lambda, i.name);
                 env2.values[displ++] = i.def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -1213,7 +1213,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
                user. */
             for (auto & i : *arg.attrs)
                 if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError("%1% called with unexpected argument '%2%'", lambda, i.name, pos);
+                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
             abort(); // can't happen
         }
     }
@@ -1302,7 +1302,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos)) {
         std::ostringstream out;
         cond->show(out);
-        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
+        throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
     }
     body->eval(state, env, v);
 }
@@ -1454,14 +1454,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to an integer", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == tFloat) {
             if (vTmp.type == tInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type == tFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError("cannot add %1% to a float", showType(vTmp), pos);
+                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
             s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
     }
@@ -1472,7 +1472,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         mkFloat(v, nf);
     else if (firstType == tPath) {
         if (!context.empty())
-            throwEvalError("a string that refers to a store path cannot be appended to a path", pos);
+            throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
         mkPath(v, path.c_str());
     } else
@@ -1521,7 +1521,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tInt)
-        throwTypeError("value is %1% while an integer was expected", v, pos);
+        throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
 
@@ -1532,7 +1532,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
     if (v.type == tInt)
         return v.integer;
     else if (v.type != tFloat)
-        throwTypeError("value is %1% while a float was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
 
@@ -1541,7 +1541,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tBool)
-        throwTypeError("value is %1% while a Boolean was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
 
@@ -1556,7 +1556,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
     if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
-        throwTypeError("value is %1% while a function was expected", v, pos);
+        throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
 
@@ -1565,7 +1565,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
     forceValue(v, pos);
     if (v.type != tString) {
         if (pos)
-            throwTypeError("value is %1% while a string was expected", v, pos);
+            throwTypeError(pos, "value is %1% while a string was expected", v);
         else
             throwTypeError("value is %1% while a string was expected", v);
     }
@@ -1594,8 +1594,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
-                v.string.s, v.string.context[0], pos);
+            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", 
+                v.string.s, v.string.context[0]);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
@@ -1651,7 +1651,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
             return *maybeString;
         }
         auto i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string", pos);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
@@ -1682,7 +1682,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         }
     }
 
-    throwTypeError("cannot coerce %1% to a string", v, pos);
+    throwTypeError(pos, "cannot coerce %1% to a string", v);
 }
 
 
@@ -1713,7 +1713,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
 {
     string path = coerceToString(pos, v, context, false, false);
     if (path == "" || path[0] != '/')
-        throwEvalError("string '%1%' doesn't represent an absolute path", path, pos);
+        throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
     return path;
 }
 

From 19cffc29c995af8fb155f2764664f0b079879b4f Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 12:09:12 -0600
Subject: [PATCH 2095/2196] remove unused extra json fields

---
 src/libstore/local-store.cc  |  1 -
 src/libstore/remote-store.cc |  9 ---------
 src/libutil/logging.cc       | 24 ------------------------
 3 files changed, 34 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1f16a22b117..63a825af067 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1364,7 +1364,6 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            // TODO log as warning if repair successful??
             logError(
                 ErrorInfo { 
                     .name = "Missing path with referrers",
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 7c50c106597..cc336e460e3 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -780,15 +780,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         }
 
         else if (msg == STDERR_NEXT)
-            // TODO: is this really an ErrorInfo error?  Seems like we're forwarding the 
-            // stderr output of the remote to current stderr/log 
-            // ErrorInfo gets lost in this scenario.
-            // An alternative might be a logger on the remote that forwards ErrorInfo and etc.
-            // logError(
-            //     ErrorInfo { 
-            //         // .name = "Remote Store" TODO reasonable name.
-            //         .hint = hintfmt(chomp(readString(from)))
-            // });
             printError(chomp(readString(from)));
 
         else if (msg == STDERR_START_ACTIVITY) {
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 7c75456ee86..736ce604e01 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -150,7 +150,6 @@ struct JSONLogger : Logger {
 
     void logEI(const ErrorInfo & ei) override
     {
-        // add fields like Pos info and etc?
         std::ostringstream oss;
         oss << ei;
 
@@ -159,29 +158,6 @@ struct JSONLogger : Logger {
         json["level"] = ei.level;
         json["msg"] = oss.str();
 
-        // Extra things that COULD go into json.  Useful?
-        // TODO: decide if useful.
-        // TODO: make a json obj that goes into json["msg"]?
-        json["name"] = ei.name;
-        json["description"] = ei.description;
-        if (ei.hint.has_value()) {
-            json["hint"] = ei.hint->str();
-        }
-        if (ei.nixCode.has_value()) {
-            if (ei.nixCode->errPos.line != 0)
-                json["line"] = ei.nixCode->errPos.line;
-            if (ei.nixCode->errPos.column != 0)
-                json["column"] = ei.nixCode->errPos.column;
-            if (ei.nixCode->errPos.file != "")
-                json["file"] = ei.nixCode->errPos.file;
-            if (ei.nixCode->prevLineOfCode.has_value())
-                json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode;
-            if (ei.nixCode->errLineOfCode.has_value())
-                json["errLineOfCode"] = *ei.nixCode->errLineOfCode;
-            if (ei.nixCode->nextLineOfCode.has_value())
-                json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode;
-        }
-
         write(json);
     }
 

From d608793e4feb79fe615e3071c7ce561dd2dea0a2 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 12:09:57 -0600
Subject: [PATCH 2096/2196] remove uncrustify cfg

---
 .uncrustify.cfg | 2990 -----------------------------------------------
 1 file changed, 2990 deletions(-)
 delete mode 100644 .uncrustify.cfg

diff --git a/.uncrustify.cfg b/.uncrustify.cfg
deleted file mode 100644
index 6bf91ec3c68..00000000000
--- a/.uncrustify.cfg
+++ /dev/null
@@ -1,2990 +0,0 @@
-# Uncrustify-0.70.1_f
-
-#
-# General options
-#
-
-# The type of line endings.
-#
-# Default: auto
-newlines                        = auto     # lf/crlf/cr/auto
-
-# The original size of tabs in the input.
-#
-# Default: 8
-input_tab_size                  = 4        # unsigned number
-
-# The size of tabs in the output (only used if align_with_tabs=true).
-#
-# Default: 8
-output_tab_size                 = 4        # unsigned number
-
-# The ASCII value of the string escape char, usually 92 (\) or (Pawn) 94 (^).
-#
-# Default: 92
-string_escape_char              = 92       # unsigned number
-
-# Alternate string escape char (usually only used for Pawn).
-# Only works right before the quote char.
-string_escape_char2             = 0        # unsigned number
-
-# Replace tab characters found in string literals with the escape sequence \t
-# instead.
-string_replace_tab_chars        = false    # true/false
-
-# Allow interpreting '>=' and '>>=' as part of a template in code like
-# 'void f(list<list<B>>=val);'. If true, 'assert(x<0 && y>=3)' will be broken.
-# Improvements to template detection may make this option obsolete.
-tok_split_gte                   = false    # true/false
-
-# Specify the marker used in comments to disable processing of part of the
-# file.
-#
-# Default:  *INDENT-OFF*
-disable_processing_cmt          = " *INDENT-OFF*"      # string
-
-# Specify the marker used in comments to (re)enable processing in a file.
-#
-# Default:  *INDENT-ON*
-enable_processing_cmt           = " *INDENT-ON*"     # string
-
-# Enable parsing of digraphs.
-enable_digraphs                 = false    # true/false
-
-# Add or remove the UTF-8 BOM (recommend 'remove').
-utf8_bom                        = ignore   # ignore/add/remove/force
-
-# If the file contains bytes with values between 128 and 255, but is not
-# UTF-8, then output as UTF-8.
-utf8_byte                       = false    # true/false
-
-# Force the output encoding to UTF-8.
-utf8_force                      = false    # true/false
-
-# Add or remove space between 'do' and '{'.
-sp_do_brace_open                = ignore   # ignore/add/remove/force
-
-# Add or remove space between '}' and 'while'.
-sp_brace_close_while            = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'while' and '('.
-sp_while_paren_open             = ignore   # ignore/add/remove/force
-
-#
-# Spacing options
-#
-
-# Add or remove space around non-assignment symbolic operators ('+', '/', '%',
-# '<<', and so forth).
-sp_arith                        = ignore   # ignore/add/remove/force
-
-# Add or remove space around arithmetic operators '+' and '-'.
-#
-# Overrides sp_arith.
-sp_arith_additive               = ignore   # ignore/add/remove/force
-
-# Add or remove space around assignment operator '=', '+=', etc.
-sp_assign                       = ignore   # ignore/add/remove/force
-
-# Add or remove space around '=' in C++11 lambda capture specifications.
-#
-# Overrides sp_assign.
-sp_cpp_lambda_assign            = ignore   # ignore/add/remove/force
-
-# Add or remove space after the capture specification of a C++11 lambda when
-# an argument list is present, as in '[] <here> (int x){ ... }'.
-sp_cpp_lambda_square_paren      = ignore   # ignore/add/remove/force
-
-# Add or remove space after the capture specification of a C++11 lambda with
-# no argument list is present, as in '[] <here> { ... }'.
-sp_cpp_lambda_square_brace      = remove  # ignore/add/remove/force
-
-# Add or remove space after the argument list of a C++11 lambda, as in
-# '[](int x) <here> { ... }'.
-sp_cpp_lambda_paren_brace       = ignore   # ignore/add/remove/force
-
-# Add or remove space between a lambda body and its call operator of an
-# immediately invoked lambda, as in '[]( ... ){ ... } <here> ( ... )'.
-sp_cpp_lambda_fparen            = ignore   # ignore/add/remove/force
-
-# Add or remove space around assignment operator '=' in a prototype.
-#
-# If set to ignore, use sp_assign.
-sp_assign_default               = ignore   # ignore/add/remove/force
-
-# Add or remove space before assignment operator '=', '+=', etc.
-#
-# Overrides sp_assign.
-sp_before_assign                = ignore   # ignore/add/remove/force
-
-# Add or remove space after assignment operator '=', '+=', etc.
-#
-# Overrides sp_assign.
-sp_after_assign                 = ignore   # ignore/add/remove/force
-
-# Add or remove space in 'NS_ENUM ('.
-sp_enum_paren                   = ignore   # ignore/add/remove/force
-
-# Add or remove space around assignment '=' in enum.
-sp_enum_assign                  = ignore   # ignore/add/remove/force
-
-# Add or remove space before assignment '=' in enum.
-#
-# Overrides sp_enum_assign.
-sp_enum_before_assign           = ignore   # ignore/add/remove/force
-
-# Add or remove space after assignment '=' in enum.
-#
-# Overrides sp_enum_assign.
-sp_enum_after_assign            = ignore   # ignore/add/remove/force
-
-# Add or remove space around assignment ':' in enum.
-sp_enum_colon                   = ignore   # ignore/add/remove/force
-
-# Add or remove space around preprocessor '##' concatenation operator.
-#
-# Default: add
-sp_pp_concat                    = add      # ignore/add/remove/force
-
-# Add or remove space after preprocessor '#' stringify operator.
-# Also affects the '#@' charizing operator.
-sp_pp_stringify                 = ignore   # ignore/add/remove/force
-
-# Add or remove space before preprocessor '#' stringify operator
-# as in '#define x(y) L#y'.
-sp_before_pp_stringify          = ignore   # ignore/add/remove/force
-
-# Add or remove space around boolean operators '&&' and '||'.
-sp_bool                         = ignore   # ignore/add/remove/force
-
-# Add or remove space around compare operator '<', '>', '==', etc.
-sp_compare                      = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '(' and ')'.
-sp_inside_paren                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between nested parentheses, i.e. '((' vs. ') )'.
-sp_paren_paren                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between back-to-back parentheses, i.e. ')(' vs. ') ('.
-sp_cparen_oparen                = ignore   # ignore/add/remove/force
-
-# Whether to balance spaces inside nested parentheses.
-sp_balance_nested_parens        = false    # true/false
-
-# Add or remove space between ')' and '{'.
-sp_paren_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between nested braces, i.e. '{{' vs '{ {'.
-sp_brace_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove space before pointer star '*'.
-sp_before_ptr_star              = ignore   # ignore/add/remove/force
-
-# Add or remove space before pointer star '*' that isn't followed by a
-# variable name. If set to ignore, sp_before_ptr_star is used instead.
-sp_before_unnamed_ptr_star      = ignore   # ignore/add/remove/force
-
-# Add or remove space between pointer stars '*'.
-sp_between_ptr_star             = ignore   # ignore/add/remove/force
-
-# Add or remove space after pointer star '*', if followed by a word.
-#
-# Overrides sp_type_func.
-sp_after_ptr_star               = ignore   # ignore/add/remove/force
-
-# Add or remove space after pointer caret '^', if followed by a word.
-sp_after_ptr_block_caret        = ignore   # ignore/add/remove/force
-
-# Add or remove space after pointer star '*', if followed by a qualifier.
-sp_after_ptr_star_qualifier     = ignore   # ignore/add/remove/force
-
-# Add or remove space after a pointer star '*', if followed by a function
-# prototype or function definition.
-#
-# Overrides sp_after_ptr_star and sp_type_func.
-sp_after_ptr_star_func          = ignore   # ignore/add/remove/force
-
-# Add or remove space after a pointer star '*', if followed by an open
-# parenthesis, as in 'void* (*)().
-sp_ptr_star_paren               = ignore   # ignore/add/remove/force
-
-# Add or remove space before a pointer star '*', if followed by a function
-# prototype or function definition.
-sp_before_ptr_star_func         = ignore   # ignore/add/remove/force
-
-# Add or remove space before a reference sign '&'.
-sp_before_byref                 = ignore   # ignore/add/remove/force
-
-# Add or remove space before a reference sign '&' that isn't followed by a
-# variable name. If set to ignore, sp_before_byref is used instead.
-sp_before_unnamed_byref         = ignore   # ignore/add/remove/force
-
-# Add or remove space after reference sign '&', if followed by a word.
-#
-# Overrides sp_type_func.
-sp_after_byref                  = ignore   # ignore/add/remove/force
-
-# Add or remove space after a reference sign '&', if followed by a function
-# prototype or function definition.
-#
-# Overrides sp_after_byref and sp_type_func.
-sp_after_byref_func             = ignore   # ignore/add/remove/force
-
-# Add or remove space before a reference sign '&', if followed by a function
-# prototype or function definition.
-sp_before_byref_func            = ignore   # ignore/add/remove/force
-
-# Add or remove space between type and word.
-#
-# Default: force
-sp_after_type                   = force    # ignore/add/remove/force
-
-# Add or remove space between 'decltype(...)' and word.
-sp_after_decltype               = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space before the parenthesis in the D constructs
-# 'template Foo(' and 'class Foo('.
-sp_before_template_paren        = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'template' and '<'.
-# If set to ignore, sp_before_angle is used.
-sp_template_angle               = ignore   # ignore/add/remove/force
-
-# Add or remove space before '<'.
-sp_before_angle                 = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '<' and '>'.
-sp_inside_angle                 = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '<>'.
-sp_inside_angle_empty           = ignore   # ignore/add/remove/force
-
-# Add or remove space between '>' and ':'.
-sp_angle_colon                  = ignore   # ignore/add/remove/force
-
-# Add or remove space after '>'.
-sp_after_angle                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between '>' and '(' as found in 'new List<byte>(foo);'.
-sp_angle_paren                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between '>' and '()' as found in 'new List<byte>();'.
-sp_angle_paren_empty            = ignore   # ignore/add/remove/force
-
-# Add or remove space between '>' and a word as in 'List<byte> m;' or
-# 'template <typename T> static ...'.
-sp_angle_word                   = ignore   # ignore/add/remove/force
-
-# Add or remove space between '>' and '>' in '>>' (template stuff).
-#
-# Default: add
-sp_angle_shift                  = ignore      # ignore/add/remove/force
-
-# (C++11) Permit removal of the space between '>>' in 'foo<bar<int> >'. Note
-# that sp_angle_shift cannot remove the space without this option.
-sp_permit_cpp11_shift           = true    # true/false
-
-# Add or remove space before '(' of control statements ('if', 'for', 'switch',
-# 'while', etc.).
-sp_before_sparen                = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '(' and ')' of control statements.
-sp_inside_sparen                = ignore   # ignore/add/remove/force
-
-# Add or remove space after '(' of control statements.
-#
-# Overrides sp_inside_sparen.
-sp_inside_sparen_open           = ignore   # ignore/add/remove/force
-
-# Add or remove space before ')' of control statements.
-#
-# Overrides sp_inside_sparen.
-sp_inside_sparen_close          = ignore   # ignore/add/remove/force
-
-# Add or remove space after ')' of control statements.
-sp_after_sparen                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and '{' of of control statements.
-sp_sparen_brace                 = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space between 'invariant' and '('.
-sp_invariant_paren              = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space after the ')' in 'invariant (C) c'.
-sp_after_invariant_paren        = ignore   # ignore/add/remove/force
-
-# Add or remove space before empty statement ';' on 'if', 'for' and 'while'.
-sp_special_semi                 = ignore   # ignore/add/remove/force
-
-# Add or remove space before ';'.
-#
-# Default: remove
-sp_before_semi                  = remove   # ignore/add/remove/force
-
-# Add or remove space before ';' in non-empty 'for' statements.
-sp_before_semi_for              = ignore   # ignore/add/remove/force
-
-# Add or remove space before a semicolon of an empty part of a for statement.
-sp_before_semi_for_empty        = ignore   # ignore/add/remove/force
-
-# Add or remove space after ';', except when followed by a comment.
-#
-# Default: add
-sp_after_semi                   = add      # ignore/add/remove/force
-
-# Add or remove space after ';' in non-empty 'for' statements.
-#
-# Default: force
-sp_after_semi_for               = force    # ignore/add/remove/force
-
-# Add or remove space after the final semicolon of an empty part of a for
-# statement, as in 'for ( ; ; <here> )'.
-sp_after_semi_for_empty         = ignore   # ignore/add/remove/force
-
-# Add or remove space before '[' (except '[]').
-sp_before_square                = ignore   # ignore/add/remove/force
-
-# Add or remove space before '[' for a variable definition.
-#
-# Default: remove
-sp_before_vardef_square         = remove   # ignore/add/remove/force
-
-# Add or remove space before '[' for asm block.
-sp_before_square_asm_block      = ignore   # ignore/add/remove/force
-
-# Add or remove space before '[]'.
-sp_before_squares               = ignore   # ignore/add/remove/force
-
-# Add or remove space before C++17 structured bindings.
-sp_cpp_before_struct_binding    = ignore   # ignore/add/remove/force
-
-# Add or remove space inside a non-empty '[' and ']'.
-sp_inside_square                = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space inside a non-empty Objective-C boxed array '@[' and
-# ']'. If set to ignore, sp_inside_square is used.
-sp_inside_square_oc_array       = ignore   # ignore/add/remove/force
-
-# Add or remove space after ',', i.e. 'a,b' vs. 'a, b'.
-sp_after_comma                  = ignore   # ignore/add/remove/force
-
-# Add or remove space before ','.
-#
-# Default: remove
-sp_before_comma                 = remove   # ignore/add/remove/force
-
-# (C#) Add or remove space between ',' and ']' in multidimensional array type
-# like 'int[,,]'.
-sp_after_mdatype_commas         = ignore   # ignore/add/remove/force
-
-# (C#) Add or remove space between '[' and ',' in multidimensional array type
-# like 'int[,,]'.
-sp_before_mdatype_commas        = ignore   # ignore/add/remove/force
-
-# (C#) Add or remove space between ',' in multidimensional array type
-# like 'int[,,]'.
-sp_between_mdatype_commas       = ignore   # ignore/add/remove/force
-
-# Add or remove space between an open parenthesis and comma,
-# i.e. '(,' vs. '( ,'.
-#
-# Default: force
-sp_paren_comma                  = force    # ignore/add/remove/force
-
-# Add or remove space before the variadic '...' when preceded by a
-# non-punctuator.
-sp_before_ellipsis              = remove   # ignore/add/remove/force
-
-# Add or remove space between a type and '...'.
-sp_type_ellipsis                = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space between a type and '?'.
-sp_type_question                = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and '...'.
-sp_paren_ellipsis               = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and a qualifier such as 'const'.
-sp_paren_qualifier              = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and 'noexcept'.
-sp_paren_noexcept               = ignore   # ignore/add/remove/force
-
-# Add or remove space after class ':'.
-sp_after_class_colon            = ignore   # ignore/add/remove/force
-
-# Add or remove space before class ':'.
-sp_before_class_colon           = ignore   # ignore/add/remove/force
-
-# Add or remove space after class constructor ':'.
-sp_after_constr_colon           = ignore   # ignore/add/remove/force
-
-# Add or remove space before class constructor ':'.
-sp_before_constr_colon          = ignore   # ignore/add/remove/force
-
-# Add or remove space before case ':'.
-#
-# Default: remove
-sp_before_case_colon            = remove   # ignore/add/remove/force
-
-# Add or remove space between 'operator' and operator sign.
-sp_after_operator               = ignore   # ignore/add/remove/force
-
-# Add or remove space between the operator symbol and the open parenthesis, as
-# in 'operator ++('.
-sp_after_operator_sym           = ignore   # ignore/add/remove/force
-
-# Overrides sp_after_operator_sym when the operator has no arguments, as in
-# 'operator *()'.
-sp_after_operator_sym_empty     = ignore   # ignore/add/remove/force
-
-# Add or remove space after C/D cast, i.e. 'cast(int)a' vs. 'cast(int) a' or
-# '(int)a' vs. '(int) a'.
-sp_after_cast                   = ignore   # ignore/add/remove/force
-
-# Add or remove spaces inside cast parentheses.
-sp_inside_paren_cast            = ignore   # ignore/add/remove/force
-
-# Add or remove space between the type and open parenthesis in a C++ cast,
-# i.e. 'int(exp)' vs. 'int (exp)'.
-sp_cpp_cast_paren               = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'sizeof' and '('.
-sp_sizeof_paren                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'sizeof' and '...'.
-sp_sizeof_ellipsis              = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'sizeof...' and '('.
-sp_sizeof_ellipsis_paren        = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'decltype' and '('.
-sp_decltype_paren               = ignore   # ignore/add/remove/force
-
-# (Pawn) Add or remove space after the tag keyword.
-sp_after_tag                    = ignore   # ignore/add/remove/force
-
-# Add or remove space inside enum '{' and '}'.
-sp_inside_braces_enum           = ignore   # ignore/add/remove/force
-
-# Add or remove space inside struct/union '{' and '}'.
-sp_inside_braces_struct         = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space inside Objective-C boxed dictionary '{' and '}'
-sp_inside_braces_oc_dict        = ignore   # ignore/add/remove/force
-
-# Add or remove space after open brace in an unnamed temporary
-# direct-list-initialization.
-sp_after_type_brace_init_lst_open = ignore   # ignore/add/remove/force
-
-# Add or remove space before close brace in an unnamed temporary
-# direct-list-initialization.
-sp_before_type_brace_init_lst_close = ignore   # ignore/add/remove/force
-
-# Add or remove space inside an unnamed temporary direct-list-initialization.
-sp_inside_type_brace_init_lst   = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '{' and '}'.
-sp_inside_braces                = ignore   # ignore/add/remove/force
-
-# Add or remove space inside '{}'.
-sp_inside_braces_empty          = ignore   # ignore/add/remove/force
-
-# Add or remove space around trailing return operator '->'.
-sp_trailing_return              = ignore   # ignore/add/remove/force
-
-# Add or remove space between return type and function name. A minimum of 1
-# is forced except for pointer return types.
-sp_type_func                    = ignore   # ignore/add/remove/force
-
-# Add or remove space between type and open brace of an unnamed temporary
-# direct-list-initialization.
-sp_type_brace_init_lst          = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '(' on function declaration.
-sp_func_proto_paren             = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '()' on function declaration
-# without parameters.
-sp_func_proto_paren_empty       = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '(' with a typedef specifier.
-sp_func_type_paren              = ignore   # ignore/add/remove/force
-
-# Add or remove space between alias name and '(' of a non-pointer function type typedef.
-sp_func_def_paren               = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '()' on function definition
-# without parameters.
-sp_func_def_paren_empty         = ignore   # ignore/add/remove/force
-
-# Add or remove space inside empty function '()'.
-# Overrides sp_after_angle unless use_sp_after_angle_always is set to true.
-sp_inside_fparens               = ignore   # ignore/add/remove/force
-
-# Add or remove space inside function '(' and ')'.
-sp_inside_fparen                = ignore   # ignore/add/remove/force
-
-# Add or remove space inside the first parentheses in a function type, as in
-# 'void (*x)(...)'.
-sp_inside_tparen                = ignore   # ignore/add/remove/force
-
-# Add or remove space between the ')' and '(' in a function type, as in
-# 'void (*x)(...)'.
-sp_after_tparen_close           = ignore   # ignore/add/remove/force
-
-# Add or remove space between ']' and '(' when part of a function call.
-sp_square_fparen                = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and '{' of function.
-sp_fparen_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and '{' of s function call in object
-# initialization.
-#
-# Overrides sp_fparen_brace.
-sp_fparen_brace_initializer     = ignore   # ignore/add/remove/force
-
-# (Java) Add or remove space between ')' and '{{' of double brace initializer.
-sp_fparen_dbrace                = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '(' on function calls.
-sp_func_call_paren              = ignore   # ignore/add/remove/force
-
-# Add or remove space between function name and '()' on function calls without
-# parameters. If set to ignore (the default), sp_func_call_paren is used.
-sp_func_call_paren_empty        = ignore   # ignore/add/remove/force
-
-# Add or remove space between the user function name and '(' on function
-# calls. You need to set a keyword to be a user function in the config file,
-# like:
-#   set func_call_user tr _ i18n
-sp_func_call_user_paren         = ignore   # ignore/add/remove/force
-
-# Add or remove space inside user function '(' and ')'.
-sp_func_call_user_inside_fparen = ignore   # ignore/add/remove/force
-
-# Add or remove space between nested parentheses with user functions,
-# i.e. '((' vs. '( ('.
-sp_func_call_user_paren_paren   = ignore   # ignore/add/remove/force
-
-# Add or remove space between a constructor/destructor and the open
-# parenthesis.
-sp_func_class_paren             = ignore   # ignore/add/remove/force
-
-# Add or remove space between a constructor without parameters or destructor
-# and '()'.
-sp_func_class_paren_empty       = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'return' and '('.
-sp_return_paren                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'return' and '{'.
-sp_return_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between '__attribute__' and '('.
-sp_attribute_paren              = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'defined' and '(' in '#if defined (FOO)'.
-sp_defined_paren                = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'throw' and '(' in 'throw (something)'.
-sp_throw_paren                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'throw' and anything other than '(' as in
-# '@throw [...];'.
-sp_after_throw                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'catch' and '(' in 'catch (something) { }'.
-# If set to ignore, sp_before_sparen is used.
-sp_catch_paren                  = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between '@catch' and '('
-# in '@catch (something) { }'. If set to ignore, sp_catch_paren is used.
-sp_oc_catch_paren               = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between class name and '('
-# in '@interface className(categoryName)<ProtocolName>:BaseClass'
-sp_oc_classname_paren           = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space between 'version' and '('
-# in 'version (something) { }'. If set to ignore, sp_before_sparen is used.
-sp_version_paren                = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space between 'scope' and '('
-# in 'scope (something) { }'. If set to ignore, sp_before_sparen is used.
-sp_scope_paren                  = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'super' and '(' in 'super (something)'.
-#
-# Default: remove
-sp_super_paren                  = remove   # ignore/add/remove/force
-
-# Add or remove space between 'this' and '(' in 'this (something)'.
-#
-# Default: remove
-sp_this_paren                   = remove   # ignore/add/remove/force
-
-# Add or remove space between a macro name and its definition.
-sp_macro                        = ignore   # ignore/add/remove/force
-
-# Add or remove space between a macro function ')' and its definition.
-sp_macro_func                   = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'else' and '{' if on the same line.
-sp_else_brace                   = ignore   # ignore/add/remove/force
-
-# Add or remove space between '}' and 'else' if on the same line.
-sp_brace_else                   = ignore   # ignore/add/remove/force
-
-# Add or remove space between '}' and the name of a typedef on the same line.
-sp_brace_typedef                = ignore   # ignore/add/remove/force
-
-# Add or remove space before the '{' of a 'catch' statement, if the '{' and
-# 'catch' are on the same line, as in 'catch (decl) <here> {'.
-sp_catch_brace                  = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space before the '{' of a '@catch' statement, if the '{'
-# and '@catch' are on the same line, as in '@catch (decl) <here> {'.
-# If set to ignore, sp_catch_brace is used.
-sp_oc_catch_brace               = ignore   # ignore/add/remove/force
-
-# Add or remove space between '}' and 'catch' if on the same line.
-sp_brace_catch                  = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between '}' and '@catch' if on the same line.
-# If set to ignore, sp_brace_catch is used.
-sp_oc_brace_catch               = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'finally' and '{' if on the same line.
-sp_finally_brace                = ignore   # ignore/add/remove/force
-
-# Add or remove space between '}' and 'finally' if on the same line.
-sp_brace_finally                = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'try' and '{' if on the same line.
-sp_try_brace                    = ignore   # ignore/add/remove/force
-
-# Add or remove space between get/set and '{' if on the same line.
-sp_getset_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove space between a variable and '{' for C++ uniform
-# initialization.
-#
-# Default: add
-sp_word_brace                   = add      # ignore/add/remove/force
-
-# Add or remove space between a variable and '{' for a namespace.
-#
-# Default: add
-sp_word_brace_ns                = add      # ignore/add/remove/force
-
-# Add or remove space before the '::' operator.
-sp_before_dc                    = ignore   # ignore/add/remove/force
-
-# Add or remove space after the '::' operator.
-sp_after_dc                     = ignore   # ignore/add/remove/force
-
-# (D) Add or remove around the D named array initializer ':' operator.
-sp_d_array_colon                = ignore   # ignore/add/remove/force
-
-# Add or remove space after the '!' (not) unary operator.
-#
-# Default: remove
-sp_not                          = remove   # ignore/add/remove/force
-
-# Add or remove space after the '~' (invert) unary operator.
-#
-# Default: remove
-sp_inv                          = remove   # ignore/add/remove/force
-
-# Add or remove space after the '&' (address-of) unary operator. This does not
-# affect the spacing after a '&' that is part of a type.
-#
-# Default: remove
-sp_addr                         = remove   # ignore/add/remove/force
-
-# Add or remove space around the '.' or '->' operators.
-#
-# Default: remove
-sp_member                       = remove   # ignore/add/remove/force
-
-# Add or remove space after the '*' (dereference) unary operator. This does
-# not affect the spacing after a '*' that is part of a type.
-#
-# Default: remove
-sp_deref                        = remove   # ignore/add/remove/force
-
-# Add or remove space after '+' or '-', as in 'x = -5' or 'y = +7'.
-#
-# Default: remove
-sp_sign                         = remove   # ignore/add/remove/force
-
-# Add or remove space between '++' and '--' the word to which it is being
-# applied, as in '(--x)' or 'y++;'.
-#
-# Default: remove
-sp_incdec                       = remove   # ignore/add/remove/force
-
-# Add or remove space before a backslash-newline at the end of a line.
-#
-# Default: add
-sp_before_nl_cont               = add      # ignore/add/remove/force
-
-# (OC) Add or remove space after the scope '+' or '-', as in '-(void) foo;'
-# or '+(int) bar;'.
-sp_after_oc_scope               = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after the colon in message specs,
-# i.e. '-(int) f:(int) x;' vs. '-(int) f: (int) x;'.
-sp_after_oc_colon               = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space before the colon in message specs,
-# i.e. '-(int) f: (int) x;' vs. '-(int) f : (int) x;'.
-sp_before_oc_colon              = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after the colon in immutable dictionary expression
-# 'NSDictionary *test = @{@"foo" :@"bar"};'.
-sp_after_oc_dict_colon          = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space before the colon in immutable dictionary expression
-# 'NSDictionary *test = @{@"foo" :@"bar"};'.
-sp_before_oc_dict_colon         = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after the colon in message specs,
-# i.e. '[object setValue:1];' vs. '[object setValue: 1];'.
-sp_after_send_oc_colon          = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space before the colon in message specs,
-# i.e. '[object setValue:1];' vs. '[object setValue :1];'.
-sp_before_send_oc_colon         = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after the (type) in message specs,
-# i.e. '-(int)f: (int) x;' vs. '-(int)f: (int)x;'.
-sp_after_oc_type                = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after the first (type) in message specs,
-# i.e. '-(int) f:(int)x;' vs. '-(int)f:(int)x;'.
-sp_after_oc_return_type         = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between '@selector' and '(',
-# i.e. '@selector(msgName)' vs. '@selector (msgName)'.
-# Also applies to '@protocol()' constructs.
-sp_after_oc_at_sel              = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between '@selector(x)' and the following word,
-# i.e. '@selector(foo) a:' vs. '@selector(foo)a:'.
-sp_after_oc_at_sel_parens       = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space inside '@selector' parentheses,
-# i.e. '@selector(foo)' vs. '@selector( foo )'.
-# Also applies to '@protocol()' constructs.
-sp_inside_oc_at_sel_parens      = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space before a block pointer caret,
-# i.e. '^int (int arg){...}' vs. ' ^int (int arg){...}'.
-sp_before_oc_block_caret        = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after a block pointer caret,
-# i.e. '^int (int arg){...}' vs. '^ int (int arg){...}'.
-sp_after_oc_block_caret         = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between the receiver and selector in a message,
-# as in '[receiver selector ...]'.
-sp_after_oc_msg_receiver        = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space after '@property'.
-sp_after_oc_property            = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove space between '@synchronized' and the open parenthesis,
-# i.e. '@synchronized(foo)' vs. '@synchronized (foo)'.
-sp_after_oc_synchronized        = ignore   # ignore/add/remove/force
-
-# Add or remove space around the ':' in 'b ? t : f'.
-sp_cond_colon                   = ignore   # ignore/add/remove/force
-
-# Add or remove space before the ':' in 'b ? t : f'.
-#
-# Overrides sp_cond_colon.
-sp_cond_colon_before            = ignore   # ignore/add/remove/force
-
-# Add or remove space after the ':' in 'b ? t : f'.
-#
-# Overrides sp_cond_colon.
-sp_cond_colon_after             = ignore   # ignore/add/remove/force
-
-# Add or remove space around the '?' in 'b ? t : f'.
-sp_cond_question                = ignore   # ignore/add/remove/force
-
-# Add or remove space before the '?' in 'b ? t : f'.
-#
-# Overrides sp_cond_question.
-sp_cond_question_before         = ignore   # ignore/add/remove/force
-
-# Add or remove space after the '?' in 'b ? t : f'.
-#
-# Overrides sp_cond_question.
-sp_cond_question_after          = ignore   # ignore/add/remove/force
-
-# In the abbreviated ternary form '(a ?: b)', add or remove space between '?'
-# and ':'.
-#
-# Overrides all other sp_cond_* options.
-sp_cond_ternary_short           = ignore   # ignore/add/remove/force
-
-# Fix the spacing between 'case' and the label. Only 'ignore' and 'force' make
-# sense here.
-sp_case_label                   = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space around the D '..' operator.
-sp_range                        = ignore   # ignore/add/remove/force
-
-# Add or remove space after ':' in a Java/C++11 range-based 'for',
-# as in 'for (Type var : expr)'.
-sp_after_for_colon              = ignore   # ignore/add/remove/force
-
-# Add or remove space before ':' in a Java/C++11 range-based 'for',
-# as in 'for (Type var : expr)'.
-sp_before_for_colon             = ignore   # ignore/add/remove/force
-
-# (D) Add or remove space between 'extern' and '(' as in 'extern (C)'.
-sp_extern_paren                 = ignore   # ignore/add/remove/force
-
-# Add or remove space after the opening of a C++ comment,
-# i.e. '// A' vs. '//A'.
-sp_cmt_cpp_start                = ignore   # ignore/add/remove/force
-
-# If true, space is added with sp_cmt_cpp_start will be added after doxygen
-# sequences like '///', '///<', '//!' and '//!<'.
-sp_cmt_cpp_doxygen              = false    # true/false
-
-# If true, space is added with sp_cmt_cpp_start will be added after Qt
-# translator or meta-data comments like '//:', '//=', and '//~'.
-sp_cmt_cpp_qttr                 = false    # true/false
-
-# Add or remove space between #else or #endif and a trailing comment.
-sp_endif_cmt                    = ignore   # ignore/add/remove/force
-
-# Add or remove space after 'new', 'delete' and 'delete[]'.
-sp_after_new                    = ignore   # ignore/add/remove/force
-
-# Add or remove space between 'new' and '(' in 'new()'.
-sp_between_new_paren            = ignore   # ignore/add/remove/force
-
-# Add or remove space between ')' and type in 'new(foo) BAR'.
-sp_after_newop_paren            = ignore   # ignore/add/remove/force
-
-# Add or remove space inside parenthesis of the new operator
-# as in 'new(foo) BAR'.
-sp_inside_newop_paren           = ignore   # ignore/add/remove/force
-
-# Add or remove space after the open parenthesis of the new operator,
-# as in 'new(foo) BAR'.
-#
-# Overrides sp_inside_newop_paren.
-sp_inside_newop_paren_open      = ignore   # ignore/add/remove/force
-
-# Add or remove space before the close parenthesis of the new operator,
-# as in 'new(foo) BAR'.
-#
-# Overrides sp_inside_newop_paren.
-sp_inside_newop_paren_close     = ignore   # ignore/add/remove/force
-
-# Add or remove space before a trailing or embedded comment.
-sp_before_tr_emb_cmt            = ignore   # ignore/add/remove/force
-
-# Number of spaces before a trailing or embedded comment.
-sp_num_before_tr_emb_cmt        = 0        # unsigned number
-
-# (Java) Add or remove space between an annotation and the open parenthesis.
-sp_annotation_paren             = ignore   # ignore/add/remove/force
-
-# If true, vbrace tokens are dropped to the previous token and skipped.
-sp_skip_vbrace_tokens           = false    # true/false
-
-# Add or remove space after 'noexcept'.
-sp_after_noexcept               = ignore   # ignore/add/remove/force
-
-# Add or remove space after '_'.
-sp_vala_after_translation       = ignore   # ignore/add/remove/force
-
-# If true, a <TAB> is inserted after #define.
-force_tab_after_define          = false    # true/false
-
-#
-# Indenting options
-#
-
-# The number of columns to indent per level. Usually 2, 3, 4, or 8.
-#
-# Default: 8
-indent_columns                  = 4        # unsigned number
-
-# The continuation indent. If non-zero, this overrides the indent of '(', '['
-# and '=' continuation indents. Negative values are OK; negative value is
-# absolute and not increased for each '(' or '[' level.
-#
-# For FreeBSD, this is set to 4.
-indent_continue                 = 0        # number
-
-# The continuation indent, only for class header line(s). If non-zero, this
-# overrides the indent of 'class' continuation indents.
-indent_continue_class_head      = 0        # unsigned number
-
-# Whether to indent empty lines (i.e. lines which contain only spaces before
-# the newline character).
-indent_single_newlines          = false    # true/false
-
-# The continuation indent for func_*_param if they are true. If non-zero, this
-# overrides the indent.
-indent_param                    = 0        # unsigned number
-
-# How to use tabs when indenting code.
-#
-# 0: Spaces only
-# 1: Indent with tabs to brace level, align with spaces (default)
-# 2: Indent and align with tabs, using spaces when not on a tabstop
-#
-# Default: 1
-indent_with_tabs                = 0        # unsigned number
-
-# Whether to indent comments that are not at a brace level with tabs on a
-# tabstop. Requires indent_with_tabs=2. If false, will use spaces.
-indent_cmt_with_tabs            = false    # true/false
-
-# Whether to indent strings broken by '\' so that they line up.
-indent_align_string             = false    # true/false
-
-# The number of spaces to indent multi-line XML strings.
-# Requires indent_align_string=true.
-indent_xml_string               = 0        # unsigned number
-
-# Spaces to indent '{' from level.
-indent_brace                    = 0        # unsigned number
-
-# Whether braces are indented to the body level.
-indent_braces                   = false    # true/false
-
-# Whether to disable indenting function braces if indent_braces=true.
-indent_braces_no_func           = false    # true/false
-
-# Whether to disable indenting class braces if indent_braces=true.
-indent_braces_no_class          = false    # true/false
-
-# Whether to disable indenting struct braces if indent_braces=true.
-indent_braces_no_struct         = false    # true/false
-
-# Whether to indent based on the size of the brace parent,
-# i.e. 'if' => 3 spaces, 'for' => 4 spaces, etc.
-indent_brace_parent             = false    # true/false
-
-# Whether to indent based on the open parenthesis instead of the open brace
-# in '({\n'.
-indent_paren_open_brace         = false    # true/false
-
-# (C#) Whether to indent the brace of a C# delegate by another level.
-indent_cs_delegate_brace        = false    # true/false
-
-# (C#) Whether to indent a C# delegate (to handle delegates with no brace) by
-# another level.
-indent_cs_delegate_body         = false    # true/false
-
-# Whether to indent the body of a 'namespace'.
-indent_namespace                = false    # true/false
-
-# Whether to indent only the first namespace, and not any nested namespaces.
-# Requires indent_namespace=true.
-indent_namespace_single_indent  = false    # true/false
-
-# The number of spaces to indent a namespace block.
-# If set to zero, use the value indent_columns
-indent_namespace_level          = 0        # unsigned number
-
-# If the body of the namespace is longer than this number, it won't be
-# indented. Requires indent_namespace=true. 0 means no limit.
-indent_namespace_limit          = 0        # unsigned number
-
-# Whether the 'extern "C"' body is indented.
-indent_extern                   = false    # true/false
-
-# Whether the 'class' body is indented.
-indent_class                    = true    # true/false
-
-# Whether to indent the stuff after a leading base class colon.
-indent_class_colon              = false    # true/false
-
-# Whether to indent based on a class colon instead of the stuff after the
-# colon. Requires indent_class_colon=true.
-indent_class_on_colon           = false    # true/false
-
-# Whether to indent the stuff after a leading class initializer colon.
-indent_constr_colon             = false    # true/false
-
-# Virtual indent from the ':' for member initializers.
-#
-# Default: 2
-indent_ctor_init_leading        = 2        # unsigned number
-
-# Additional indent for constructor initializer list.
-# Negative values decrease indent down to the first column.
-indent_ctor_init                = 0        # number
-
-# Whether to indent 'if' following 'else' as a new block under the 'else'.
-# If false, 'else\nif' is treated as 'else if' for indenting purposes.
-indent_else_if                  = false    # true/false
-
-# Amount to indent variable declarations after a open brace.
-#
-#  <0: Relative
-# >=0: Absolute
-indent_var_def_blk              = 0        # number
-
-# Whether to indent continued variable declarations instead of aligning.
-indent_var_def_cont             = true    # true/false
-
-# Whether to indent continued shift expressions ('<<' and '>>') instead of
-# aligning. Set align_left_shift=false when enabling this.
-indent_shift                    = false    # true/false
-
-# Whether to force indentation of function definitions to start in column 1.
-indent_func_def_force_col1      = true    # true/false
-
-# Whether to indent continued function call parameters one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_func_call_param          = true    # true/false
-
-# Whether to indent continued function definition parameters one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_func_def_param           = true    # true/false
-
-# for function definitions, only if indent_func_def_param is false
-# Allows to align params when appropriate and indent them when not
-# behave as if it was true if paren position is more than this value
-# if paren position is more than the option value
-indent_func_def_param_paren_pos_threshold = 0        # unsigned number
-
-# Whether to indent continued function call prototype one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_func_proto_param         = true    # true/false
-
-# Whether to indent continued function call declaration one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_func_class_param         = true    # true/false
-
-# Whether to indent continued class variable constructors one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_func_ctor_var_param      = true    # true/false
-
-# Whether to indent continued template parameter list one indent level,
-# rather than aligning parameters under the open parenthesis.
-indent_template_param           = true    # true/false
-
-# Double the indent for indent_func_xxx_param options.
-# Use both values of the options indent_columns and indent_param.
-indent_func_param_double        = false    # true/false
-
-# Indentation column for standalone 'const' qualifier on a function
-# prototype.
-indent_func_const               = 0        # unsigned number
-
-# Indentation column for standalone 'throw' qualifier on a function
-# prototype.
-indent_func_throw               = 0        # unsigned number
-
-# How to indent within a macro followed by a brace on the same line
-# This allows reducing the indent in macros that have (for example)
-# `do { ... } while (0)` blocks bracketing them.
-#
-# true:  add an indent for the brace on the same line as the macro
-# false: do not add an indent for the brace on the same line as the macro
-#
-# Default: true
-indent_macro_brace              = true     # true/false
-
-# The number of spaces to indent a continued '->' or '.'.
-# Usually set to 0, 1, or indent_columns.
-indent_member                   = 0        # unsigned number
-
-# Whether lines broken at '.' or '->' should be indented by a single indent.
-# The indent_member option will not be effective if this is set to true.
-indent_member_single            = false    # true/false
-
-# Spaces to indent single line ('//') comments on lines before code.
-indent_sing_line_comments       = 0        # unsigned number
-
-# Whether to indent trailing single line ('//') comments relative to the code
-# instead of trying to keep the same absolute column.
-indent_relative_single_line_comments = false    # true/false
-
-# Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns.
-indent_switch_case              = indent_columns       # unsigned number
-
-# indent 'break' with 'case' from 'switch'.
-indent_switch_break_with_case   = false    # true/false
-
-# Whether to indent preprocessor statements inside of switch statements.
-#
-# Default: true
-indent_switch_pp                = true     # true/false
-
-# Spaces to shift the 'case' line, without affecting any other lines.
-# Usually 0.
-indent_case_shift               = 0        # unsigned number
-
-# Spaces to indent '{' from 'case'. By default, the brace will appear under
-# the 'c' in case. Usually set to 0 or indent_columns. Negative values are OK.
-indent_case_brace               = 0        # number
-
-# Whether to indent comments found in first column.
-indent_col1_comment             = false    # true/false
-
-# Whether to indent multi string literal in first column.
-indent_col1_multi_string_literal = false    # true/false
-
-# How to indent goto labels.
-#
-#  >0: Absolute column where 1 is the leftmost column
-# <=0: Subtract from brace indent
-#
-# Default: 1
-indent_label                    = 1        # number
-
-# How to indent access specifiers that are followed by a
-# colon.
-#
-#  >0: Absolute column where 1 is the leftmost column
-# <=0: Subtract from brace indent
-#
-# Default: 1
-indent_access_spec              = -4        # number
-
-# Whether to indent the code after an access specifier by one level.
-# If true, this option forces 'indent_access_spec=0'.
-indent_access_spec_body         = false    # true/false
-
-# If an open parenthesis is followed by a newline, whether to indent the next
-# line so that it lines up after the open parenthesis (not recommended).
-indent_paren_nl                 = false    # true/false
-
-# How to indent a close parenthesis after a newline.
-#
-# 0: Indent to body level (default)
-# 1: Align under the open parenthesis
-# 2: Indent to the brace level
-indent_paren_close              = 0        # unsigned number
-
-# Whether to indent the open parenthesis of a function definition,
-# if the parenthesis is on its own line.
-indent_paren_after_func_def     = false    # true/false
-
-# Whether to indent the open parenthesis of a function declaration,
-# if the parenthesis is on its own line.
-indent_paren_after_func_decl    = false    # true/false
-
-# Whether to indent the open parenthesis of a function call,
-# if the parenthesis is on its own line.
-indent_paren_after_func_call    = false    # true/false
-
-# Whether to indent a comma when inside a parenthesis.
-# If true, aligns under the open parenthesis.
-indent_comma_paren              = false    # true/false
-
-# Whether to indent a Boolean operator when inside a parenthesis.
-# If true, aligns under the open parenthesis.
-indent_bool_paren               = false    # true/false
-
-# Whether to indent a semicolon when inside a for parenthesis.
-# If true, aligns under the open for parenthesis.
-indent_semicolon_for_paren      = false    # true/false
-
-# Whether to align the first expression to following ones
-# if indent_bool_paren=true.
-indent_first_bool_expr          = false    # true/false
-
-# Whether to align the first expression to following ones
-# if indent_semicolon_for_paren=true.
-indent_first_for_expr           = false    # true/false
-
-# If an open square is followed by a newline, whether to indent the next line
-# so that it lines up after the open square (not recommended).
-indent_square_nl                = false    # true/false
-
-# (ESQL/C) Whether to preserve the relative indent of 'EXEC SQL' bodies.
-indent_preserve_sql             = false    # true/false
-
-# Whether to align continued statements at the '='. If false or if the '=' is
-# followed by a newline, the next line is indent one tab.
-#
-# Default: true
-indent_align_assign             = false     # true/false
-
-# Whether to align continued statements at the '('. If false or the '(' is
-# followed by a newline, the next line indent is one tab.
-#
-# Default: true
-indent_align_paren              = true    # true/false,
-
-# (OC) Whether to indent Objective-C blocks at brace level instead of usual
-# rules.
-indent_oc_block                 = false    # true/false
-
-# (OC) Indent for Objective-C blocks in a message relative to the parameter
-# name.
-#
-# =0: Use indent_oc_block rules
-# >0: Use specified number of spaces to indent
-indent_oc_block_msg             = 0        # unsigned number
-
-# (OC) Minimum indent for subsequent parameters
-indent_oc_msg_colon             = 0        # unsigned number
-
-# (OC) Whether to prioritize aligning with initial colon (and stripping spaces
-# from lines, if necessary).
-#
-# Default: true
-indent_oc_msg_prioritize_first_colon = true     # true/false
-
-# (OC) Whether to indent blocks the way that Xcode does by default
-# (from the keyword if the parameter is on its own line; otherwise, from the
-# previous indentation level). Requires indent_oc_block_msg=true.
-indent_oc_block_msg_xcode_style = false    # true/false
-
-# (OC) Whether to indent blocks from where the brace is, relative to a
-# message keyword. Requires indent_oc_block_msg=true.
-indent_oc_block_msg_from_keyword = false    # true/false
-
-# (OC) Whether to indent blocks from where the brace is, relative to a message
-# colon. Requires indent_oc_block_msg=true.
-indent_oc_block_msg_from_colon  = false    # true/false
-
-# (OC) Whether to indent blocks from where the block caret is.
-# Requires indent_oc_block_msg=true.
-indent_oc_block_msg_from_caret  = false    # true/false
-
-# (OC) Whether to indent blocks from where the brace caret is.
-# Requires indent_oc_block_msg=true.
-indent_oc_block_msg_from_brace  = false    # true/false
-
-# When indenting after virtual brace open and newline add further spaces to
-# reach this minimum indent.
-indent_min_vbrace_open          = 0        # unsigned number
-
-# Whether to add further spaces after regular indent to reach next tabstop
-# when identing after virtual brace open and newline.
-indent_vbrace_open_on_tabstop   = false    # true/false
-
-# How to indent after a brace followed by another token (not a newline).
-# true:  indent all contained lines to match the token
-# false: indent all contained lines to match the brace
-#
-# Default: true
-indent_token_after_brace        = true     # true/false
-
-# Whether to indent the body of a C++11 lambda.
-indent_cpp_lambda_body          = false    # true/false
-
-# (C#) Whether to indent a 'using' block if no braces are used.
-#
-# Default: true
-indent_using_block              = true     # true/false
-
-# How to indent the continuation of ternary operator.
-#
-# 0: Off (default)
-# 1: When the `if_false` is a continuation, indent it under `if_false`
-# 2: When the `:` is a continuation, indent it under `?`
-indent_ternary_operator         = 0        # unsigned number
-
-# If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column.
-indent_off_after_return_new     = false    # true/false
-
-# If true, the tokens after return are indented with regular single indentation. 
-# By default (false) the indentation is after the return token.
-indent_single_after_return      = false    # true/false
-
-# Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they
-# have their own indentation).
-indent_ignore_asm_block         = false    # true/false
-
-#
-# Newline adding and removing options
-#
-
-# Whether to collapse empty blocks between '{' and '}'.
-nl_collapse_empty_body          = false    # true/false
-
-# Don't split one-line braced assignments, as in 'foo_t f = { 1, 2 };'.
-nl_assign_leave_one_liners      = true    # true/false
-
-# Don't split one-line braced statements inside a 'class xx { }' body.
-nl_class_leave_one_liners       = true    # true/false
-
-# Don't split one-line enums, as in 'enum foo { BAR = 15 };'
-nl_enum_leave_one_liners        = true    # true/false
-
-# Don't split one-line get or set functions.
-nl_getset_leave_one_liners      = true    # true/false
-
-# (C#) Don't split one-line property get or set functions.
-nl_cs_property_leave_one_liners = true    # true/false
-
-# Don't split one-line function definitions, as in 'int foo() { return 0; }'.
-# night modify nl_func_type_name
-nl_func_leave_one_liners        = true    # true/false
-
-# Don't split one-line C++11 lambdas, as in '[]() { return 0; }'.
-nl_cpp_lambda_leave_one_liners  = true    # true/false
-
-# Don't split one-line if/else statements, as in 'if(...) b++;'.
-nl_if_leave_one_liners          = true    # true/false
-
-# Don't split one-line while statements, as in 'while(...) b++;'.
-nl_while_leave_one_liners       = true    # true/false
-
-# Don't split one-line for statements, as in 'for(...) b++;'.
-nl_for_leave_one_liners         = true    # true/false
-
-# (OC) Don't split one-line Objective-C messages.
-nl_oc_msg_leave_one_liner       = false    # true/false
-
-# (OC) Add or remove newline between method declaration and '{'.
-nl_oc_mdef_brace                = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove newline between Objective-C block signature and '{'.
-nl_oc_block_brace               = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove blank line before '@interface' statement.
-nl_oc_before_interface          = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove blank line before '@implementation' statement.
-nl_oc_before_implementation     = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove blank line before '@end' statement.
-nl_oc_before_end                = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove newline between '@interface' and '{'.
-nl_oc_interface_brace           = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove newline between '@implementation' and '{'.
-nl_oc_implementation_brace      = ignore   # ignore/add/remove/force
-
-# Add or remove newlines at the start of the file.
-nl_start_of_file                = ignore   # ignore/add/remove/force
-
-# The minimum number of newlines at the start of the file (only used if
-# nl_start_of_file is 'add' or 'force').
-nl_start_of_file_min            = 0        # unsigned number
-
-# Add or remove newline at the end of the file.
-nl_end_of_file                  = ignore   # ignore/add/remove/force
-
-# The minimum number of newlines at the end of the file (only used if
-# nl_end_of_file is 'add' or 'force').
-nl_end_of_file_min              = 0        # unsigned number
-
-# Add or remove newline between '=' and '{'.
-nl_assign_brace                 = ignore   # ignore/add/remove/force
-
-# (D) Add or remove newline between '=' and '['.
-nl_assign_square                = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '[]' and '{'.
-nl_tsquare_brace                = ignore   # ignore/add/remove/force
-
-# (D) Add or remove newline after '= ['. Will also affect the newline before
-# the ']'.
-nl_after_square_assign          = ignore   # ignore/add/remove/force
-
-# Add or remove newline between a function call's ')' and '{', as in
-# 'list_for_each(item, &list) { }'.
-nl_fcall_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'enum' and '{'.
-nl_enum_brace                   = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'enum' and 'class'.
-nl_enum_class                   = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'enum class' and the identifier.
-nl_enum_class_identifier        = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'enum class' type and ':'.
-nl_enum_identifier_colon        = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'enum class identifier :' and type.
-nl_enum_colon_type              = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'struct and '{'.
-nl_struct_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'union' and '{'.
-nl_union_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'if' and '{'.
-nl_if_brace                     = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and 'else'.
-nl_brace_else                   = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'else if' and '{'. If set to ignore,
-# nl_if_brace is used instead.
-nl_elseif_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'else' and '{'.
-nl_else_brace                   = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'else' and 'if'.
-nl_else_if                      = ignore   # ignore/add/remove/force
-
-# Add or remove newline before 'if'/'else if' closing parenthesis.
-nl_before_if_closing_paren      = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and 'finally'.
-nl_brace_finally                = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'finally' and '{'.
-nl_finally_brace                = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'try' and '{'.
-nl_try_brace                    = ignore   # ignore/add/remove/force
-
-# Add or remove newline between get/set and '{'.
-nl_getset_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'for' and '{'.
-nl_for_brace                    = ignore   # ignore/add/remove/force
-
-# Add or remove newline before the '{' of a 'catch' statement, as in
-# 'catch (decl) <here> {'.
-nl_catch_brace                  = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove newline before the '{' of a '@catch' statement, as in
-# '@catch (decl) <here> {'. If set to ignore, nl_catch_brace is used.
-nl_oc_catch_brace               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and 'catch'.
-nl_brace_catch                  = ignore   # ignore/add/remove/force
-
-# (OC) Add or remove newline between '}' and '@catch'. If set to ignore,
-# nl_brace_catch is used.
-nl_oc_brace_catch               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and ']'.
-nl_brace_square                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and ')' in a function invocation.
-nl_brace_fparen                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'while' and '{'.
-nl_while_brace                  = ignore   # ignore/add/remove/force
-
-# (D) Add or remove newline between 'scope (x)' and '{'.
-nl_scope_brace                  = ignore   # ignore/add/remove/force
-
-# (D) Add or remove newline between 'unittest' and '{'.
-nl_unittest_brace               = ignore   # ignore/add/remove/force
-
-# (D) Add or remove newline between 'version (x)' and '{'.
-nl_version_brace                = ignore   # ignore/add/remove/force
-
-# (C#) Add or remove newline between 'using' and '{'.
-nl_using_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline between two open or close braces. Due to general
-# newline/brace handling, REMOVE may not work.
-nl_brace_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'do' and '{'.
-nl_do_brace                     = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '}' and 'while' of 'do' statement.
-nl_brace_while                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'switch' and '{'.
-nl_switch_brace                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'synchronized' and '{'.
-nl_synchronized_brace           = ignore   # ignore/add/remove/force
-
-# Add a newline between ')' and '{' if the ')' is on a different line than the
-# if/for/etc.
-#
-# Overrides nl_for_brace, nl_if_brace, nl_switch_brace, nl_while_switch and
-# nl_catch_brace.
-nl_multi_line_cond              = false    # true/false
-
-# Add a newline after '(' if an if/for/while/switch condition spans multiple
-# lines
-nl_multi_line_sparen_open       = ignore   # ignore/add/remove/force
-
-# Add a newline before ')' if an if/for/while/switch condition spans multiple
-# lines. Overrides nl_before_if_closing_paren if both are specified.
-nl_multi_line_sparen_close      = ignore   # ignore/add/remove/force
-
-# Force a newline in a define after the macro name for multi-line defines.
-nl_multi_line_define            = false    # true/false
-
-# Whether to add a newline before 'case', and a blank line before a 'case'
-# statement that follows a ';' or '}'.
-nl_before_case                  = false    # true/false
-
-# Whether to add a newline after a 'case' statement.
-nl_after_case                   = false    # true/false
-
-# Add or remove newline between a case ':' and '{'.
-#
-# Overrides nl_after_case.
-nl_case_colon_brace             = ignore   # ignore/add/remove/force
-
-# Add or remove newline between ')' and 'throw'.
-nl_before_throw                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'namespace' and '{'.
-nl_namespace_brace              = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template class.
-nl_template_class               = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template class declaration.
-#
-# Overrides nl_template_class.
-nl_template_class_decl          = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<>' of a specialized class declaration.
-#
-# Overrides nl_template_class_decl.
-nl_template_class_decl_special  = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template class definition.
-#
-# Overrides nl_template_class.
-nl_template_class_def           = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<>' of a specialized class definition.
-#
-# Overrides nl_template_class_def.
-nl_template_class_def_special   = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template function.
-nl_template_func                = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template function
-# declaration.
-#
-# Overrides nl_template_func.
-nl_template_func_decl           = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<>' of a specialized function
-# declaration.
-#
-# Overrides nl_template_func_decl.
-nl_template_func_decl_special   = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template function
-# definition.
-#
-# Overrides nl_template_func.
-nl_template_func_def            = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<>' of a specialized function
-# definition.
-#
-# Overrides nl_template_func_def.
-nl_template_func_def_special    = ignore   # ignore/add/remove/force
-
-# Add or remove newline after 'template<...>' of a template variable.
-nl_template_var                 = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'template<...>' and 'using' of a templated
-# type alias.
-nl_template_using               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'class' and '{'.
-nl_class_brace                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline before or after (depending on pos_class_comma,
-# may not be IGNORE) each',' in the base class list.
-nl_class_init_args              = ignore   # ignore/add/remove/force
-
-# Add or remove newline after each ',' in the constructor member
-# initialization. Related to nl_constr_colon, pos_constr_colon and
-# pos_constr_comma.
-nl_constr_init_args             = ignore   # ignore/add/remove/force
-
-# Add or remove newline before first element, after comma, and after last
-# element, in 'enum'.
-nl_enum_own_lines               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between return type and function name in a function
-# definition.
-# might be modified by nl_func_leave_one_liners
-nl_func_type_name               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between return type and function name inside a class
-# definition. If set to ignore, nl_func_type_name or nl_func_proto_type_name
-# is used instead.
-nl_func_type_name_class         = ignore   # ignore/add/remove/force
-
-# Add or remove newline between class specification and '::'
-# in 'void A::f() { }'. Only appears in separate member implementation (does
-# not appear with in-line implementation).
-nl_func_class_scope             = ignore   # ignore/add/remove/force
-
-# Add or remove newline between function scope and name, as in
-# 'void A :: <here> f() { }'.
-nl_func_scope_name              = ignore   # ignore/add/remove/force
-
-# Add or remove newline between return type and function name in a prototype.
-nl_func_proto_type_name         = ignore   # ignore/add/remove/force
-
-# Add or remove newline between a function name and the opening '(' in the
-# declaration.
-nl_func_paren                   = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_paren for functions with no parameters.
-nl_func_paren_empty             = ignore   # ignore/add/remove/force
-
-# Add or remove newline between a function name and the opening '(' in the
-# definition.
-nl_func_def_paren               = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_def_paren for functions with no parameters.
-nl_func_def_paren_empty         = ignore   # ignore/add/remove/force
-
-# Add or remove newline between a function name and the opening '(' in the
-# call.
-nl_func_call_paren              = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_call_paren for functions with no parameters.
-nl_func_call_paren_empty        = ignore   # ignore/add/remove/force
-
-# Add or remove newline after '(' in a function declaration.
-nl_func_decl_start              = ignore   # ignore/add/remove/force
-
-# Add or remove newline after '(' in a function definition.
-nl_func_def_start               = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_decl_start when there is only one parameter.
-nl_func_decl_start_single       = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_def_start when there is only one parameter.
-nl_func_def_start_single        = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after '(' in a function declaration if '(' and ')'
-# are in different lines. If false, nl_func_decl_start is used instead.
-nl_func_decl_start_multi_line   = false    # true/false
-
-# Whether to add a newline after '(' in a function definition if '(' and ')'
-# are in different lines. If false, nl_func_def_start is used instead.
-nl_func_def_start_multi_line    = false    # true/false
-
-# Add or remove newline after each ',' in a function declaration.
-nl_func_decl_args               = ignore   # ignore/add/remove/force
-
-# Add or remove newline after each ',' in a function definition.
-nl_func_def_args                = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after each ',' in a function declaration if '('
-# and ')' are in different lines. If false, nl_func_decl_args is used instead.
-nl_func_decl_args_multi_line    = false    # true/false
-
-# Whether to add a newline after each ',' in a function definition if '('
-# and ')' are in different lines. If false, nl_func_def_args is used instead.
-nl_func_def_args_multi_line     = false    # true/false
-
-# Add or remove newline before the ')' in a function declaration.
-nl_func_decl_end                = ignore   # ignore/add/remove/force
-
-# Add or remove newline before the ')' in a function definition.
-nl_func_def_end                 = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_decl_end when there is only one parameter.
-nl_func_decl_end_single         = ignore   # ignore/add/remove/force
-
-# Overrides nl_func_def_end when there is only one parameter.
-nl_func_def_end_single          = ignore   # ignore/add/remove/force
-
-# Whether to add a newline before ')' in a function declaration if '(' and ')'
-# are in different lines. If false, nl_func_decl_end is used instead.
-nl_func_decl_end_multi_line     = false    # true/false
-
-# Whether to add a newline before ')' in a function definition if '(' and ')'
-# are in different lines. If false, nl_func_def_end is used instead.
-nl_func_def_end_multi_line      = false    # true/false
-
-# Add or remove newline between '()' in a function declaration.
-nl_func_decl_empty              = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '()' in a function definition.
-nl_func_def_empty               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between '()' in a function call.
-nl_func_call_empty              = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after '(' in a function call,
-# has preference over nl_func_call_start_multi_line.
-nl_func_call_start              = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after '(' in a function call if '(' and ')' are in
-# different lines.
-nl_func_call_start_multi_line   = false    # true/false
-
-# Whether to add a newline after each ',' in a function call if '(' and ')'
-# are in different lines.
-nl_func_call_args_multi_line    = false    # true/false
-
-# Whether to add a newline before ')' in a function call if '(' and ')' are in
-# different lines.
-nl_func_call_end_multi_line     = false    # true/false
-
-# Whether to add a newline after '<' of a template parameter list.
-nl_template_start               = false    # true/false
-
-# Whether to add a newline after each ',' in a template parameter list.
-nl_template_args                = false    # true/false
-
-# Whether to add a newline before '>' of a template parameter list.
-nl_template_end                 = false    # true/false
-
-# (OC) Whether to put each Objective-C message parameter on a separate line.
-# See nl_oc_msg_leave_one_liner.
-nl_oc_msg_args                  = false    # true/false
-
-# Add or remove newline between function signature and '{'.
-nl_fdef_brace                   = ignore   # ignore/add/remove/force
-
-# Add or remove newline between function signature and '{',
-# if signature ends with ')'. Overrides nl_fdef_brace.
-nl_fdef_brace_cond              = ignore   # ignore/add/remove/force
-
-# Add or remove newline between C++11 lambda signature and '{'.
-nl_cpp_ldef_brace               = ignore   # ignore/add/remove/force
-
-# Add or remove newline between 'return' and the return expression.
-nl_return_expr                  = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after semicolons, except in 'for' statements.
-nl_after_semicolon              = false    # true/false
-
-# (Java) Add or remove newline between the ')' and '{{' of the double brace
-# initializer.
-nl_paren_dbrace_open            = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after the type in an unnamed temporary
-# direct-list-initialization.
-nl_type_brace_init_lst          = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after the open brace in an unnamed temporary
-# direct-list-initialization.
-nl_type_brace_init_lst_open     = ignore   # ignore/add/remove/force
-
-# Whether to add a newline before the close brace in an unnamed temporary
-# direct-list-initialization.
-nl_type_brace_init_lst_close    = ignore   # ignore/add/remove/force
-
-# Whether to add a newline after '{'. This also adds a newline before the
-# matching '}'.
-nl_after_brace_open             = false    # true/false
-
-# Whether to add a newline between the open brace and a trailing single-line
-# comment. Requires nl_after_brace_open=true.
-nl_after_brace_open_cmt         = false    # true/false
-
-# Whether to add a newline after a virtual brace open with a non-empty body.
-# These occur in un-braced if/while/do/for statement bodies.
-nl_after_vbrace_open            = false    # true/false
-
-# Whether to add a newline after a virtual brace open with an empty body.
-# These occur in un-braced if/while/do/for statement bodies.
-nl_after_vbrace_open_empty      = false    # true/false
-
-# Whether to add a newline after '}'. Does not apply if followed by a
-# necessary ';'.
-nl_after_brace_close            = false    # true/false
-
-# Whether to add a newline after a virtual brace close,
-# as in 'if (foo) a++; <here> return;'.
-nl_after_vbrace_close           = false    # true/false
-
-# Add or remove newline between the close brace and identifier,
-# as in 'struct { int a; } <here> b;'. Affects enumerations, unions and
-# structures. If set to ignore, uses nl_after_brace_close.
-nl_brace_struct_var             = ignore   # ignore/add/remove/force
-
-# Whether to alter newlines in '#define' macros.
-nl_define_macro                 = false    # true/false
-
-# Whether to alter newlines between consecutive parenthesis closes. The number
-# of closing parentheses in a line will depend on respective open parenthesis
-# lines.
-nl_squeeze_paren_close          = false    # true/false
-
-# Whether to remove blanks after '#ifxx' and '#elxx', or before '#elxx' and
-# '#endif'. Does not affect top-level #ifdefs.
-nl_squeeze_ifdef                = false    # true/false
-
-# Makes the nl_squeeze_ifdef option affect the top-level #ifdefs as well.
-nl_squeeze_ifdef_top_level      = false    # true/false
-
-# Add or remove blank line before 'if'.
-nl_before_if                    = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'if' statement. Add/Force work only if the
-# next token is not a closing brace.
-nl_after_if                     = ignore   # ignore/add/remove/force
-
-# Add or remove blank line before 'for'.
-nl_before_for                   = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'for' statement.
-nl_after_for                    = ignore   # ignore/add/remove/force
-
-# Add or remove blank line before 'while'.
-nl_before_while                 = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'while' statement.
-nl_after_while                  = ignore   # ignore/add/remove/force
-
-# Add or remove blank line before 'switch'.
-nl_before_switch                = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'switch' statement.
-nl_after_switch                 = ignore   # ignore/add/remove/force
-
-# Add or remove blank line before 'synchronized'.
-nl_before_synchronized          = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'synchronized' statement.
-nl_after_synchronized           = ignore   # ignore/add/remove/force
-
-# Add or remove blank line before 'do'.
-nl_before_do                    = ignore   # ignore/add/remove/force
-
-# Add or remove blank line after 'do/while' statement.
-nl_after_do                     = ignore   # ignore/add/remove/force
-
-# Whether to put a blank line before 'return' statements, unless after an open
-# brace.
-nl_before_return                = false    # true/false
-
-# Whether to put a blank line after 'return' statements, unless followed by a
-# close brace.
-nl_after_return                 = false    # true/false
-
-# (Java) Whether to put a blank line before a member '.' or '->' operators.
-nl_before_member                = ignore   # ignore/add/remove/force
-
-# (Java) Whether to put a blank line after a member '.' or '->' operators.
-nl_after_member                 = ignore   # ignore/add/remove/force
-
-# Whether to double-space commented-entries in 'struct'/'union'/'enum'.
-nl_ds_struct_enum_cmt           = false    # true/false
-
-# Whether to force a newline before '}' of a 'struct'/'union'/'enum'.
-# (Lower priority than eat_blanks_before_close_brace.)
-nl_ds_struct_enum_close_brace   = false    # true/false
-
-# Add or remove newline before or after (depending on pos_class_colon) a class
-# colon, as in 'class Foo <here> : <or here> public Bar'.
-nl_class_colon                  = ignore   # ignore/add/remove/force
-
-# Add or remove newline around a class constructor colon. The exact position
-# depends on nl_constr_init_args, pos_constr_colon and pos_constr_comma.
-nl_constr_colon                 = ignore   # ignore/add/remove/force
-
-# Whether to collapse a two-line namespace, like 'namespace foo\n{ decl; }'
-# into a single line. If true, prevents other brace newline rules from turning
-# such code into four lines.
-nl_namespace_two_to_one_liner   = false    # true/false
-
-# Whether to remove a newline in simple unbraced if statements, turning them
-# into one-liners, as in 'if(b)\n i++;' => 'if(b) i++;'.
-nl_create_if_one_liner          = false    # true/false
-
-# Whether to remove a newline in simple unbraced for statements, turning them
-# into one-liners, as in 'for (...)\n stmt;' => 'for (...) stmt;'.
-nl_create_for_one_liner         = false    # true/false
-
-# Whether to remove a newline in simple unbraced while statements, turning
-# them into one-liners, as in 'while (expr)\n stmt;' => 'while (expr) stmt;'.
-nl_create_while_one_liner       = false    # true/false
-
-# Whether to collapse a function definition whose body (not counting braces)
-# is only one line so that the entire definition (prototype, braces, body) is
-# a single line.
-nl_create_func_def_one_liner    = false    # true/false
-
-# Whether to collapse a function definition whose body (not counting braces)
-# is only one line so that the entire definition (prototype, braces, body) is
-# a single line.
-nl_create_list_one_liner        = false    # true/false
-
-# Whether to split one-line simple unbraced if statements into two lines by
-# adding a newline, as in 'if(b) <here> i++;'.
-nl_split_if_one_liner           = false    # true/false
-
-# Whether to split one-line simple unbraced for statements into two lines by
-# adding a newline, as in 'for (...) <here> stmt;'.
-nl_split_for_one_liner          = false    # true/false
-
-# Whether to split one-line simple unbraced while statements into two lines by
-# adding a newline, as in 'while (expr) <here> stmt;'.
-nl_split_while_one_liner        = false    # true/false
-
-#
-# Blank line options
-#
-
-# The maximum number of consecutive newlines (3 = 2 blank lines).
-nl_max                          = 0        # unsigned number
-
-# The maximum number of consecutive newlines in a function.
-nl_max_blank_in_func            = 0        # unsigned number
-
-# The number of newlines before a function prototype.
-nl_before_func_body_proto       = 0        # unsigned number
-
-# The number of newlines before a multi-line function definition.
-nl_before_func_body_def         = 0        # unsigned number
-
-# The number of newlines before a class constructor/destructor prototype.
-nl_before_func_class_proto      = 0        # unsigned number
-
-# The number of newlines before a class constructor/destructor definition.
-nl_before_func_class_def        = 0        # unsigned number
-
-# The number of newlines after a function prototype.
-nl_after_func_proto             = 0        # unsigned number
-
-# The number of newlines after a function prototype, if not followed by
-# another function prototype.
-nl_after_func_proto_group       = 0        # unsigned number
-
-# The number of newlines after a class constructor/destructor prototype.
-nl_after_func_class_proto       = 0        # unsigned number
-
-# The number of newlines after a class constructor/destructor prototype,
-# if not followed by another constructor/destructor prototype.
-nl_after_func_class_proto_group = 0        # unsigned number
-
-# Whether one-line method definitions inside a class body should be treated
-# as if they were prototypes for the purposes of adding newlines.
-#
-# Requires nl_class_leave_one_liners=true. Overrides nl_before_func_body_def
-# and nl_before_func_class_def for one-liners.
-nl_class_leave_one_liner_groups = false    # true/false
-
-# The number of newlines after '}' of a multi-line function body.
-nl_after_func_body              = 0        # unsigned number
-
-# The number of newlines after '}' of a multi-line function body in a class
-# declaration. Also affects class constructors/destructors.
-#
-# Overrides nl_after_func_body.
-nl_after_func_body_class        = 0        # unsigned number
-
-# The number of newlines after '}' of a single line function body. Also
-# affects class constructors/destructors.
-#
-# Overrides nl_after_func_body and nl_after_func_body_class.
-nl_after_func_body_one_liner    = 0        # unsigned number
-
-# The number of blank lines after a block of variable definitions at the top
-# of a function body.
-#
-# 0: No change (default).
-nl_func_var_def_blk             = 0        # unsigned number
-
-# The number of newlines before a block of typedefs. If nl_after_access_spec
-# is non-zero, that option takes precedence.
-#
-# 0: No change (default).
-nl_typedef_blk_start            = 0        # unsigned number
-
-# The number of newlines after a block of typedefs.
-#
-# 0: No change (default).
-nl_typedef_blk_end              = 0        # unsigned number
-
-# The maximum number of consecutive newlines within a block of typedefs.
-#
-# 0: No change (default).
-nl_typedef_blk_in               = 0        # unsigned number
-
-# The number of newlines before a block of variable definitions not at the top
-# of a function body. If nl_after_access_spec is non-zero, that option takes
-# precedence.
-#
-# 0: No change (default).
-nl_var_def_blk_start            = 0        # unsigned number
-
-# The number of newlines after a block of variable definitions not at the top
-# of a function body.
-#
-# 0: No change (default).
-nl_var_def_blk_end              = 0        # unsigned number
-
-# The maximum number of consecutive newlines within a block of variable
-# definitions.
-#
-# 0: No change (default).
-nl_var_def_blk_in               = 0        # unsigned number
-
-# The minimum number of newlines before a multi-line comment.
-# Doesn't apply if after a brace open or another multi-line comment.
-nl_before_block_comment         = 0        # unsigned number
-
-# The minimum number of newlines before a single-line C comment.
-# Doesn't apply if after a brace open or other single-line C comments.
-nl_before_c_comment             = 0        # unsigned number
-
-# The minimum number of newlines before a CPP comment.
-# Doesn't apply if after a brace open or other CPP comments.
-nl_before_cpp_comment           = 0        # unsigned number
-
-# Whether to force a newline after a multi-line comment.
-nl_after_multiline_comment      = false    # true/false
-
-# Whether to force a newline after a label's colon.
-nl_after_label_colon            = false    # true/false
-
-# The number of newlines after '}' or ';' of a struct/enum/union definition.
-nl_after_struct                 = 0        # unsigned number
-
-# The number of newlines before a class definition.
-nl_before_class                 = 0        # unsigned number
-
-# The number of newlines after '}' or ';' of a class definition.
-nl_after_class                  = 0        # unsigned number
-
-# The number of newlines before a namespace.
-nl_before_namespace             = 0        # unsigned number
-
-# The number of newlines after '{' of a namespace. This also adds newlines
-# before the matching '}'.
-#
-# 0: Apply eat_blanks_after_open_brace or eat_blanks_before_close_brace if
-#     applicable, otherwise no change.
-#
-# Overrides eat_blanks_after_open_brace and eat_blanks_before_close_brace.
-nl_inside_namespace             = 0        # unsigned number
-
-# The number of newlines after '}' of a namespace.
-nl_after_namespace              = 0        # unsigned number
-
-# The number of newlines before an access specifier label. This also includes
-# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count
-# if after a brace open.
-#
-# 0: No change (default).
-nl_before_access_spec           = 0        # unsigned number
-
-# The number of newlines after an access specifier label. This also includes
-# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count
-# if after a brace open.
-#
-# 0: No change (default).
-#
-# Overrides nl_typedef_blk_start and nl_var_def_blk_start.
-nl_after_access_spec            = 0        # unsigned number
-
-# The number of newlines between a function definition and the function
-# comment, as in '// comment\n <here> void foo() {...}'.
-#
-# 0: No change (default).
-nl_comment_func_def             = 0        # unsigned number
-
-# The number of newlines after a try-catch-finally block that isn't followed
-# by a brace close.
-#
-# 0: No change (default).
-nl_after_try_catch_finally      = 0        # unsigned number
-
-# (C#) The number of newlines before and after a property, indexer or event
-# declaration.
-#
-# 0: No change (default).
-nl_around_cs_property           = 0        # unsigned number
-
-# (C#) The number of newlines between the get/set/add/remove handlers.
-#
-# 0: No change (default).
-nl_between_get_set              = 0        # unsigned number
-
-# (C#) Add or remove newline between property and the '{'.
-nl_property_brace               = ignore   # ignore/add/remove/force
-
-# Whether to remove blank lines after '{'.
-eat_blanks_after_open_brace     = false    # true/false
-
-# Whether to remove blank lines before '}'.
-eat_blanks_before_close_brace   = false    # true/false
-
-# How aggressively to remove extra newlines not in preprocessor.
-#
-# 0: No change (default)
-# 1: Remove most newlines not handled by other config
-# 2: Remove all newlines and reformat completely by config
-nl_remove_extra_newlines        = 0        # unsigned number
-
-# (Java) Add or remove newline after an annotation statement. Only affects
-# annotations that are after a newline.
-nl_after_annotation             = ignore   # ignore/add/remove/force
-
-# (Java) Add or remove newline between two annotations.
-nl_between_annotation           = ignore   # ignore/add/remove/force
-
-#
-# Positioning options
-#
-
-# The position of arithmetic operators in wrapped expressions.
-pos_arith                       = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of assignment in wrapped expressions. Do not affect '='
-# followed by '{'.
-pos_assign                      = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of Boolean operators in wrapped expressions.
-pos_bool                        = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of comparison operators in wrapped expressions.
-pos_compare                     = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of conditional operators, as in the '?' and ':' of
-# 'expr ? stmt : stmt', in wrapped expressions.
-pos_conditional                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of the comma in wrapped expressions.
-pos_comma                       = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of the comma in enum entries.
-pos_enum_comma                  = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of the comma in the base class list if there is more than one
-# line. Affects nl_class_init_args.
-pos_class_comma                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of the comma in the constructor initialization list.
-# Related to nl_constr_colon, nl_constr_init_args and pos_constr_colon.
-pos_constr_comma                = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of trailing/leading class colon, between class and base class
-# list. Affects nl_class_colon.
-pos_class_colon                 = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-# The position of colons between constructor and member initialization.
-# Related to nl_constr_colon, nl_constr_init_args and pos_constr_comma.
-pos_constr_colon                = ignore   # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force
-
-#
-# Line splitting options
-#
-
-# Try to limit code width to N columns.
-code_width                      = 0        # unsigned number
-
-# Whether to fully split long 'for' statements at semi-colons.
-ls_for_split_full               = false    # true/false
-
-# Whether to fully split long function prototypes/calls at commas.
-# The option ls_code_width has priority over the option ls_func_split_full.
-ls_func_split_full              = false    # true/false
-
-# Whether to split lines as close to code_width as possible and ignore some
-# groupings.
-# The option ls_code_width has priority over the option ls_func_split_full.
-ls_code_width                   = false    # true/false
-
-#
-# Code alignment options (not left column spaces/tabs)
-#
-
-# Whether to keep non-indenting tabs.
-align_keep_tabs                 = false    # true/false
-
-# Whether to use tabs for aligning.
-align_with_tabs                 = false    # true/false
-
-# Whether to bump out to the next tab when aligning.
-align_on_tabstop                = false    # true/false
-
-# Whether to right-align numbers.
-align_number_right              = false    # true/false
-
-# Whether to keep whitespace not required for alignment.
-align_keep_extra_space          = false    # true/false
-
-# Whether to align variable definitions in prototypes and functions.
-align_func_params               = false    # true/false
-
-# The span for aligning parameter definitions in function on parameter name.
-#
-# 0: Don't align (default).
-align_func_params_span          = 0        # unsigned number
-
-# The threshold for aligning function parameter definitions.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_func_params_thresh        = 0        # number
-
-# The gap for aligning function parameter definitions.
-align_func_params_gap           = 0        # unsigned number
-
-# The span for aligning constructor value.
-#
-# 0: Don't align (default).
-align_constr_value_span         = 0        # unsigned number
-
-# The threshold for aligning constructor value.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_constr_value_thresh       = 0        # number
-
-# The gap for aligning constructor value.
-align_constr_value_gap          = 0        # unsigned number
-
-# Whether to align parameters in single-line functions that have the same
-# name. The function names must already be aligned with each other.
-align_same_func_call_params     = false    # true/false
-
-# The span for aligning function-call parameters for single line functions.
-#
-# 0: Don't align (default).
-align_same_func_call_params_span = 0        # unsigned number
-
-# The threshold for aligning function-call parameters for single line
-# functions.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_same_func_call_params_thresh = 0        # number
-
-# The span for aligning variable definitions.
-#
-# 0: Don't align (default).
-align_var_def_span              = 0        # unsigned number
-
-# How to consider (or treat) the '*' in the alignment of variable definitions.
-#
-# 0: Part of the type     'void *   foo;' (default)
-# 1: Part of the variable 'void     *foo;'
-# 2: Dangling             'void    *foo;'
-# Dangling: the '*' will not be taken into account when aligning.
-align_var_def_star_style        = 0        # unsigned number
-
-# How to consider (or treat) the '&' in the alignment of variable definitions.
-#
-# 0: Part of the type     'long &   foo;' (default)
-# 1: Part of the variable 'long     &foo;'
-# 2: Dangling             'long    &foo;'
-# Dangling: the '&' will not be taken into account when aligning.
-align_var_def_amp_style         = 0        # unsigned number
-
-# The threshold for aligning variable definitions.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_var_def_thresh            = 0        # number
-
-# The gap for aligning variable definitions.
-align_var_def_gap               = 0        # unsigned number
-
-# Whether to align the colon in struct bit fields.
-align_var_def_colon             = false    # true/false
-
-# The gap for aligning the colon in struct bit fields.
-align_var_def_colon_gap         = 0        # unsigned number
-
-# Whether to align any attribute after the variable name.
-align_var_def_attribute         = false    # true/false
-
-# Whether to align inline struct/enum/union variable definitions.
-align_var_def_inline            = false    # true/false
-
-# The span for aligning on '=' in assignments.
-#
-# 0: Don't align (default).
-align_assign_span               = 0        # unsigned number
-
-# The span for aligning on '=' in function prototype modifier.
-#
-# 0: Don't align (default).
-align_assign_func_proto_span    = 0        # unsigned number
-
-# The threshold for aligning on '=' in assignments.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_assign_thresh             = 0        # number
-
-# How to apply align_assign_span to function declaration "assignments", i.e.
-# 'virtual void foo() = 0' or '~foo() = {default|delete}'.
-#
-# 0: Align with other assignments (default)
-# 1: Align with each other, ignoring regular assignments
-# 2: Don't align
-align_assign_decl_func          = 0        # unsigned number
-
-# The span for aligning on '=' in enums.
-#
-# 0: Don't align (default).
-align_enum_equ_span             = 0        # unsigned number
-
-# The threshold for aligning on '=' in enums.
-# Use a negative number for absolute thresholds.
-#
-# 0: no limit (default).
-align_enum_equ_thresh           = 0        # number
-
-# The span for aligning class member definitions.
-#
-# 0: Don't align (default).
-align_var_class_span            = 0        # unsigned number
-
-# The threshold for aligning class member definitions.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_var_class_thresh          = 0        # number
-
-# The gap for aligning class member definitions.
-align_var_class_gap             = 0        # unsigned number
-
-# The span for aligning struct/union member definitions.
-#
-# 0: Don't align (default).
-align_var_struct_span           = 0        # unsigned number
-
-# The threshold for aligning struct/union member definitions.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_var_struct_thresh         = 0        # number
-
-# The gap for aligning struct/union member definitions.
-align_var_struct_gap            = 0        # unsigned number
-
-# The span for aligning struct initializer values.
-#
-# 0: Don't align (default).
-align_struct_init_span          = 0        # unsigned number
-
-# The span for aligning single-line typedefs.
-#
-# 0: Don't align (default).
-align_typedef_span              = 0        # unsigned number
-
-# The minimum space between the type and the synonym of a typedef.
-align_typedef_gap               = 0        # unsigned number
-
-# How to align typedef'd functions with other typedefs.
-#
-# 0: Don't mix them at all (default)
-# 1: Align the open parenthesis with the types
-# 2: Align the function type name with the other type names
-align_typedef_func              = 0        # unsigned number
-
-# How to consider (or treat) the '*' in the alignment of typedefs.
-#
-# 0: Part of the typedef type, 'typedef int * pint;' (default)
-# 1: Part of type name:        'typedef int   *pint;'
-# 2: Dangling:                 'typedef int  *pint;'
-# Dangling: the '*' will not be taken into account when aligning.
-align_typedef_star_style        = 0        # unsigned number
-
-# How to consider (or treat) the '&' in the alignment of typedefs.
-#
-# 0: Part of the typedef type, 'typedef int & intref;' (default)
-# 1: Part of type name:        'typedef int   &intref;'
-# 2: Dangling:                 'typedef int  &intref;'
-# Dangling: the '&' will not be taken into account when aligning.
-align_typedef_amp_style         = 0        # unsigned number
-
-# The span for aligning comments that end lines.
-#
-# 0: Don't align (default).
-align_right_cmt_span            = 0        # unsigned number
-
-# Minimum number of columns between preceding text and a trailing comment in
-# order for the comment to qualify for being aligned. Must be non-zero to have
-# an effect.
-align_right_cmt_gap             = 0        # unsigned number
-
-# If aligning comments, whether to mix with comments after '}' and #endif with
-# less than three spaces before the comment.
-align_right_cmt_mix             = false    # true/false
-
-# Whether to only align trailing comments that are at the same brace level.
-align_right_cmt_same_level      = false    # true/false
-
-# Minimum column at which to align trailing comments. Comments which are
-# aligned beyond this column, but which can be aligned in a lesser column,
-# may be "pulled in".
-#
-# 0: Ignore (default).
-align_right_cmt_at_col          = 0        # unsigned number
-
-# The span for aligning function prototypes.
-#
-# 0: Don't align (default).
-align_func_proto_span           = 0        # unsigned number
-
-# The threshold for aligning function prototypes.
-# Use a negative number for absolute thresholds.
-#
-# 0: No limit (default).
-align_func_proto_thresh         = 0        # number
-
-# Minimum gap between the return type and the function name.
-align_func_proto_gap            = 0        # unsigned number
-
-# Whether to align function prototypes on the 'operator' keyword instead of
-# what follows.
-align_on_operator               = false    # true/false
-
-# Whether to mix aligning prototype and variable declarations. If true,
-# align_var_def_XXX options are used instead of align_func_proto_XXX options.
-align_mix_var_proto             = false    # true/false
-
-# Whether to align single-line functions with function prototypes.
-# Uses align_func_proto_span.
-align_single_line_func          = false    # true/false
-
-# Whether to align the open brace of single-line functions.
-# Requires align_single_line_func=true. Uses align_func_proto_span.
-align_single_line_brace         = false    # true/false
-
-# Gap for align_single_line_brace.
-align_single_line_brace_gap     = 0        # unsigned number
-
-# (OC) The span for aligning Objective-C message specifications.
-#
-# 0: Don't align (default).
-align_oc_msg_spec_span          = 0        # unsigned number
-
-# Whether to align macros wrapped with a backslash and a newline. This will
-# not work right if the macro contains a multi-line comment.
-align_nl_cont                   = false    # true/false
-
-# Whether to align macro functions and variables together.
-align_pp_define_together        = false    # true/false
-
-# The span for aligning on '#define' bodies.
-#
-# =0: Don't align (default)
-# >0: Number of lines (including comments) between blocks
-align_pp_define_span            = 0        # unsigned number
-
-# The minimum space between label and value of a preprocessor define.
-align_pp_define_gap             = 0        # unsigned number
-
-# Whether to align lines that start with '<<' with previous '<<'.
-#
-# Default: true
-align_left_shift                = true     # true/false
-
-# Whether to align text after 'asm volatile ()' colons.
-align_asm_colon                 = false    # true/false
-
-# (OC) Span for aligning parameters in an Objective-C message call
-# on the ':'.
-#
-# 0: Don't align.
-align_oc_msg_colon_span         = 0        # unsigned number
-
-# (OC) Whether to always align with the first parameter, even if it is too
-# short.
-align_oc_msg_colon_first        = false    # true/false
-
-# (OC) Whether to align parameters in an Objective-C '+' or '-' declaration
-# on the ':'.
-align_oc_decl_colon             = false    # true/false
-
-#
-# Comment modification options
-#
-
-# Try to wrap comments at N columns.
-cmt_width                       = 0        # unsigned number
-
-# How to reflow comments.
-#
-# 0: No reflowing (apart from the line wrapping due to cmt_width) (default)
-# 1: No touching at all
-# 2: Full reflow
-cmt_reflow_mode                 = 0        # unsigned number
-
-# Whether to convert all tabs to spaces in comments. If false, tabs in
-# comments are left alone, unless used for indenting.
-cmt_convert_tab_to_spaces       = false    # true/false
-
-# Whether to apply changes to multi-line comments, including cmt_width,
-# keyword substitution and leading chars.
-#
-# Default: true
-cmt_indent_multi                = true     # true/false
-
-# Whether to group c-comments that look like they are in a block.
-cmt_c_group                     = false    # true/false
-
-# Whether to put an empty '/*' on the first line of the combined c-comment.
-cmt_c_nl_start                  = false    # true/false
-
-# Whether to add a newline before the closing '*/' of the combined c-comment.
-cmt_c_nl_end                    = false    # true/false
-
-# Whether to change cpp-comments into c-comments.
-cmt_cpp_to_c                    = false    # true/false
-
-# Whether to group cpp-comments that look like they are in a block. Only
-# meaningful if cmt_cpp_to_c=true.
-cmt_cpp_group                   = false    # true/false
-
-# Whether to put an empty '/*' on the first line of the combined cpp-comment
-# when converting to a c-comment.
-#
-# Requires cmt_cpp_to_c=true and cmt_cpp_group=true.
-cmt_cpp_nl_start                = false    # true/false
-
-# Whether to add a newline before the closing '*/' of the combined cpp-comment
-# when converting to a c-comment.
-#
-# Requires cmt_cpp_to_c=true and cmt_cpp_group=true.
-cmt_cpp_nl_end                  = false    # true/false
-
-# Whether to put a star on subsequent comment lines.
-cmt_star_cont                   = false    # true/false
-
-# The number of spaces to insert at the start of subsequent comment lines.
-cmt_sp_before_star_cont         = 0        # unsigned number
-
-# The number of spaces to insert after the star on subsequent comment lines.
-cmt_sp_after_star_cont          = 0        # unsigned number
-
-# For multi-line comments with a '*' lead, remove leading spaces if the first
-# and last lines of the comment are the same length.
-#
-# Default: true
-cmt_multi_check_last            = true     # true/false
-
-# For multi-line comments with a '*' lead, remove leading spaces if the first
-# and last lines of the comment are the same length AND if the length is
-# bigger as the first_len minimum.
-#
-# Default: 4
-cmt_multi_first_len_minimum     = 4        # unsigned number
-
-# Path to a file that contains text to insert at the beginning of a file if
-# the file doesn't start with a C/C++ comment. If the inserted text contains
-# '$(filename)', that will be replaced with the current file's name.
-cmt_insert_file_header          = ""         # string
-
-# Path to a file that contains text to insert at the end of a file if the
-# file doesn't end with a C/C++ comment. If the inserted text contains
-# '$(filename)', that will be replaced with the current file's name.
-cmt_insert_file_footer          = ""         # string
-
-# Path to a file that contains text to insert before a function definition if
-# the function isn't preceded by a C/C++ comment. If the inserted text
-# contains '$(function)', '$(javaparam)' or '$(fclass)', these will be
-# replaced with, respectively, the name of the function, the javadoc '@param'
-# and '@return' stuff, or the name of the class to which the member function
-# belongs.
-cmt_insert_func_header          = ""         # string
-
-# Path to a file that contains text to insert before a class if the class
-# isn't preceded by a C/C++ comment. If the inserted text contains '$(class)',
-# that will be replaced with the class name.
-cmt_insert_class_header         = ""         # string
-
-# Path to a file that contains text to insert before an Objective-C message
-# specification, if the method isn't preceded by a C/C++ comment. If the
-# inserted text contains '$(message)' or '$(javaparam)', these will be
-# replaced with, respectively, the name of the function, or the javadoc
-# '@param' and '@return' stuff.
-cmt_insert_oc_msg_header        = ""         # string
-
-# Whether a comment should be inserted if a preprocessor is encountered when
-# stepping backwards from a function name.
-#
-# Applies to cmt_insert_oc_msg_header, cmt_insert_func_header and
-# cmt_insert_class_header.
-cmt_insert_before_preproc       = false    # true/false
-
-# Whether a comment should be inserted if a function is declared inline to a
-# class definition.
-#
-# Applies to cmt_insert_func_header.
-#
-# Default: true
-cmt_insert_before_inlines       = true     # true/false
-
-# Whether a comment should be inserted if the function is a class constructor
-# or destructor.
-#
-# Applies to cmt_insert_func_header.
-cmt_insert_before_ctor_dtor     = false    # true/false
-
-#
-# Code modifying options (non-whitespace)
-#
-
-# Add or remove braces on a single-line 'do' statement.
-mod_full_brace_do               = ignore   # ignore/add/remove/force
-
-# Add or remove braces on a single-line 'for' statement.
-mod_full_brace_for              = ignore   # ignore/add/remove/force
-
-# (Pawn) Add or remove braces on a single-line function definition.
-mod_full_brace_function         = ignore   # ignore/add/remove/force
-
-# Add or remove braces on a single-line 'if' statement. Braces will not be
-# removed if the braced statement contains an 'else'.
-mod_full_brace_if               = ignore   # ignore/add/remove/force
-
-# Whether to enforce that all blocks of an 'if'/'else if'/'else' chain either
-# have, or do not have, braces. If true, braces will be added if any block
-# needs braces, and will only be removed if they can be removed from all
-# blocks.
-#
-# Overrides mod_full_brace_if.
-mod_full_brace_if_chain         = false    # true/false
-
-# Whether to add braces to all blocks of an 'if'/'else if'/'else' chain.
-# If true, mod_full_brace_if_chain will only remove braces from an 'if' that
-# does not have an 'else if' or 'else'.
-mod_full_brace_if_chain_only    = false    # true/false
-
-# Add or remove braces on single-line 'while' statement.
-mod_full_brace_while            = ignore   # ignore/add/remove/force
-
-# Add or remove braces on single-line 'using ()' statement.
-mod_full_brace_using            = ignore   # ignore/add/remove/force
-
-# Don't remove braces around statements that span N newlines
-mod_full_brace_nl               = 0        # unsigned number
-
-# Whether to prevent removal of braces from 'if'/'for'/'while'/etc. blocks
-# which span multiple lines.
-#
-# Affects:
-#   mod_full_brace_for
-#   mod_full_brace_if
-#   mod_full_brace_if_chain
-#   mod_full_brace_if_chain_only
-#   mod_full_brace_while
-#   mod_full_brace_using
-#
-# Does not affect:
-#   mod_full_brace_do
-#   mod_full_brace_function
-mod_full_brace_nl_block_rem_mlcond = false    # true/false
-
-# Add or remove unnecessary parenthesis on 'return' statement.
-mod_paren_on_return             = ignore   # ignore/add/remove/force
-
-# (Pawn) Whether to change optional semicolons to real semicolons.
-mod_pawn_semicolon              = false    # true/false
-
-# Whether to fully parenthesize Boolean expressions in 'while' and 'if'
-# statement, as in 'if (a && b > c)' => 'if (a && (b > c))'.
-mod_full_paren_if_bool          = false    # true/false
-
-# Whether to remove superfluous semicolons.
-mod_remove_extra_semicolon      = false    # true/false
-
-# If a function body exceeds the specified number of newlines and doesn't have
-# a comment after the close brace, a comment will be added.
-mod_add_long_function_closebrace_comment = 0        # unsigned number
-
-# If a namespace body exceeds the specified number of newlines and doesn't
-# have a comment after the close brace, a comment will be added.
-mod_add_long_namespace_closebrace_comment = 0        # unsigned number
-
-# If a class body exceeds the specified number of newlines and doesn't have a
-# comment after the close brace, a comment will be added.
-mod_add_long_class_closebrace_comment = 0        # unsigned number
-
-# If a switch body exceeds the specified number of newlines and doesn't have a
-# comment after the close brace, a comment will be added.
-mod_add_long_switch_closebrace_comment = 0        # unsigned number
-
-# If an #ifdef body exceeds the specified number of newlines and doesn't have
-# a comment after the #endif, a comment will be added.
-mod_add_long_ifdef_endif_comment = 0        # unsigned number
-
-# If an #ifdef or #else body exceeds the specified number of newlines and
-# doesn't have a comment after the #else, a comment will be added.
-mod_add_long_ifdef_else_comment = 0        # unsigned number
-
-# Whether to take care of the case by the mod_sort_xx options.
-mod_sort_case_sensitive         = false    # true/false
-
-# Whether to sort consecutive single-line 'import' statements.
-mod_sort_import                 = false    # true/false
-
-# (C#) Whether to sort consecutive single-line 'using' statements.
-mod_sort_using                  = false    # true/false
-
-# Whether to sort consecutive single-line '#include' statements (C/C++) and
-# '#import' statements (Objective-C). Be aware that this has the potential to
-# break your code if your includes/imports have ordering dependencies.
-mod_sort_include                = false    # true/false
-
-# Whether to move a 'break' that appears after a fully braced 'case' before
-# the close brace, as in 'case X: { ... } break;' => 'case X: { ... break; }'.
-mod_move_case_break             = false    # true/false
-
-# Add or remove braces around a fully braced case statement. Will only remove
-# braces if there are no variable declarations in the block.
-mod_case_brace                  = ignore   # ignore/add/remove/force
-
-# Whether to remove a void 'return;' that appears as the last statement in a
-# function.
-mod_remove_empty_return         = false    # true/false
-
-# Add or remove the comma after the last value of an enumeration.
-mod_enum_last_comma             = ignore   # ignore/add/remove/force
-
-# (OC) Whether to organize the properties. If true, properties will be
-# rearranged according to the mod_sort_oc_property_*_weight factors.
-mod_sort_oc_properties          = false    # true/false
-
-# (OC) Weight of a class property modifier.
-mod_sort_oc_property_class_weight = 0        # number
-
-# (OC) Weight of 'atomic' and 'nonatomic'.
-mod_sort_oc_property_thread_safe_weight = 0        # number
-
-# (OC) Weight of 'readwrite' when organizing properties.
-mod_sort_oc_property_readwrite_weight = 0        # number
-
-# (OC) Weight of a reference type specifier ('retain', 'copy', 'assign',
-# 'weak', 'strong') when organizing properties.
-mod_sort_oc_property_reference_weight = 0        # number
-
-# (OC) Weight of getter type ('getter=') when organizing properties.
-mod_sort_oc_property_getter_weight = 0        # number
-
-# (OC) Weight of setter type ('setter=') when organizing properties.
-mod_sort_oc_property_setter_weight = 0        # number
-
-# (OC) Weight of nullability type ('nullable', 'nonnull', 'null_unspecified',
-# 'null_resettable') when organizing properties.
-mod_sort_oc_property_nullability_weight = 0        # number
-
-#
-# Preprocessor options
-#
-
-# Add or remove indentation of preprocessor directives inside #if blocks
-# at brace level 0 (file-level).
-pp_indent                       = ignore   # ignore/add/remove/force
-
-# Whether to indent #if/#else/#endif at the brace level. If false, these are
-# indented from column 1.
-pp_indent_at_level              = false    # true/false
-
-# Specifies the number of columns to indent preprocessors per level
-# at brace level 0 (file-level). If pp_indent_at_level=false, also specifies
-# the number of columns to indent preprocessors per level
-# at brace level > 0 (function-level).
-#
-# Default: 1
-pp_indent_count                 = 1        # unsigned number
-
-# Add or remove space after # based on pp_level of #if blocks.
-pp_space                        = ignore   # ignore/add/remove/force
-
-# Sets the number of spaces per level added with pp_space.
-pp_space_count                  = 0        # unsigned number
-
-# The indent for '#region' and '#endregion' in C# and '#pragma region' in
-# C/C++. Negative values decrease indent down to the first column.
-pp_indent_region                = 0        # number
-
-# Whether to indent the code between #region and #endregion.
-pp_region_indent_code           = false    # true/false
-
-# If pp_indent_at_level=true, sets the indent for #if, #else and #endif when
-# not at file-level. Negative values decrease indent down to the first column.
-#
-# =0: Indent preprocessors using output_tab_size
-# >0: Column at which all preprocessors will be indented
-pp_indent_if                    = 0        # number
-
-# Whether to indent the code between #if, #else and #endif.
-pp_if_indent_code               = false    # true/false
-
-# Whether to indent '#define' at the brace level. If false, these are
-# indented from column 1.
-pp_define_at_level              = false    # true/false
-
-# Whether to ignore the '#define' body while formatting.
-pp_ignore_define_body           = false    # true/false
-
-# Whether to indent case statements between #if, #else, and #endif.
-# Only applies to the indent of the preprocesser that the case statements
-# directly inside of.
-#
-# Default: true
-pp_indent_case                  = true     # true/false
-
-# Whether to indent whole function definitions between #if, #else, and #endif.
-# Only applies to the indent of the preprocesser that the function definition
-# is directly inside of.
-#
-# Default: true
-pp_indent_func_def              = true     # true/false
-
-# Whether to indent extern C blocks between #if, #else, and #endif.
-# Only applies to the indent of the preprocesser that the extern block is
-# directly inside of.
-#
-# Default: true
-pp_indent_extern                = true     # true/false
-
-# Whether to indent braces directly inside #if, #else, and #endif.
-# Only applies to the indent of the preprocesser that the braces are directly
-# inside of.
-#
-# Default: true
-pp_indent_brace                 = true     # true/false
-
-#
-# Sort includes options
-#
-
-# The regex for include category with priority 0.
-include_category_0              = ""         # string
-
-# The regex for include category with priority 1.
-include_category_1              = ""         # string
-
-# The regex for include category with priority 2.
-include_category_2              = ""         # string
-
-#
-# Use or Do not Use options
-#
-
-# true:  indent_func_call_param will be used (default)
-# false: indent_func_call_param will NOT be used
-#
-# Default: true
-use_indent_func_call_param      = true     # true/false
-
-# The value of the indentation for a continuation line is calculated
-# differently if the statement is:
-# - a declaration: your case with QString fileName ...
-# - an assignment: your case with pSettings = new QSettings( ...
-#
-# At the second case the indentation value might be used twice:
-# - at the assignment
-# - at the function call (if present)
-#
-# To prevent the double use of the indentation value, use this option with the
-# value 'true'.
-#
-# true:  indent_continue will be used only once
-# false: indent_continue will be used every time (default)
-use_indent_continue_only_once   = false    # true/false
-
-# The value might be used twice:
-# - at the assignment
-# - at the opening brace
-#
-# To prevent the double use of the indentation value, use this option with the
-# value 'true'.
-#
-# true:  indentation will be used only once
-# false: indentation will be used every time (default)
-indent_cpp_lambda_only_once     = true    # true/false
-
-# Whether sp_after_angle takes precedence over sp_inside_fparen. This was the
-# historic behavior, but is probably not the desired behavior, so this is off
-# by default.
-use_sp_after_angle_always       = false    # true/false
-
-# Whether to apply special formatting for Qt SIGNAL/SLOT macros. Essentially,
-# this tries to format these so that they match Qt's normalized form (i.e. the
-# result of QMetaObject::normalizedSignature), which can slightly improve the
-# performance of the QObject::connect call, rather than how they would
-# otherwise be formatted.
-#
-# See options_for_QT.cpp for details.
-#
-# Default: true
-use_options_overriding_for_qt_macros = true     # true/false
-
-#
-# Warn levels - 1: error, 2: warning (default), 3: note
-#
-
-# (C#) Warning is given if doing tab-to-\t replacement and we have found one
-# in a C# verbatim string literal.
-#
-# Default: 2
-warn_level_tabs_found_in_verbatim_string_literals = 2        # unsigned number
-
-# Meaning of the settings:
-#   Ignore - do not do any changes
-#   Add    - makes sure there is 1 or more space/brace/newline/etc
-#   Force  - makes sure there is exactly 1 space/brace/newline/etc,
-#            behaves like Add in some contexts
-#   Remove - removes space/brace/newline/etc
-#
-#
-# - Token(s) can be treated as specific type(s) with the 'set' option:
-#     `set tokenType tokenString [tokenString...]`
-#
-#     Example:
-#       `set BOOL __AND__ __OR__`
-#
-#     tokenTypes are defined in src/token_enum.h, use them without the
-#     'CT_' prefix: 'CT_BOOL' => 'BOOL'
-#
-#
-# - Token(s) can be treated as type(s) with the 'type' option.
-#     `type tokenString [tokenString...]`
-#
-#     Example:
-#       `type int c_uint_8 Rectangle`
-#
-#     This can also be achieved with `set TYPE int c_uint_8 Rectangle`
-#
-#
-# To embed whitespace in tokenStrings use the '\' escape character, or quote
-# the tokenStrings. These quotes are supported: "'`
-#
-#
-# - Support for the auto detection of languages through the file ending can be
-#   added using the 'file_ext' command.
-#     `file_ext langType langString [langString..]`
-#
-#     Example:
-#       `file_ext CPP .ch .cxx .cpp.in`
-#
-#     langTypes are defined in uncrusify_types.h in the lang_flag_e enum, use
-#     them without the 'LANG_' prefix: 'LANG_CPP' => 'CPP'
-#
-#
-# - Custom macro-based indentation can be set up using 'macro-open',
-#   'macro-else' and 'macro-close'.
-#     `(macro-open | macro-else | macro-close) tokenString`
-#
-#     Example:
-#       `macro-open  BEGIN_TEMPLATE_MESSAGE_MAP`
-#       `macro-open  BEGIN_MESSAGE_MAP`
-#       `macro-close END_MESSAGE_MAP`
-#
-#
-# option(s) with 'not default' value: 0
-#
-
-file_ext CPP .ch .cxx .cpp.in .cc .hh
-

From 72ecccee5781ec0b57b2860f677b8a64db43cda7 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 12:19:34 -0600
Subject: [PATCH 2097/2196] convert to logWarning format

---
 src/libexpr/parser.y | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 3ae7bbafd45..49fb8ad3745 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -717,7 +717,11 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, store->toRealPath(fetchers::downloadTarball(
                         store, resolveUri(elem.second), "source", false).storePath) };
         } catch (FileTransferError & e) {
-            printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second);
+            logWarning(
+                ErrorInfo { 
+                    .name = "Entry download",
+                    .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            });
             res = { false, "" };
         }
     } else {
@@ -727,7 +731,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         else {
             logWarning(
                 ErrorInfo { 
-                    .name = "Search path not found",
+                    .name = "Entry not found",
                     .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };

From 960d4362ed2215f87a596e0a994e39aa28680db1 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 13:54:18 -0600
Subject: [PATCH 2098/2196] hint only

---
 src/error-demo/error-demo.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 8f484431c3b..82e9f05801c 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -149,5 +149,13 @@ int main()
                         .errPos = Pos(problem_file, 40, 13)
                     }});
 
+    // Error with only hint and name..
+    logError(
+        ErrorInfo { .name = "error name",
+                    .hint = hintfmt("hint %1%", "only"),
+                    .nixCode = NixCode {
+                        .errPos = Pos(problem_file, 40, 13)
+                    }});
+
     return 0;
 }

From ecbb8e9c0a4374b26defde2cd10932d633affed3 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 12 May 2020 14:41:30 -0600
Subject: [PATCH 2099/2196] no blank line if no LOC

---
 Makefile                     | 3 ++-
 src/error-demo/error-demo.cc | 1 +
 src/libutil/error.cc         | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 0ba011e2ad0..05c4bdc8a01 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,8 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk
+  tests/plugins/local.mk \
+  src/error-demo/local.mk
 
 -include Makefile.config
 
diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 82e9f05801c..0216092e360 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -1,5 +1,6 @@
 #include "logging.hh"
 #include "nixexpr.hh"
+#include "util.hh"
 
 #include <iostream>
 #include <optional>
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index e4f45a2d3c4..0765a2945f7 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -182,7 +182,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
     }
 
     // lines of code.
-    if (einfo.nixCode.has_value()) {
+    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
         printCodeLines(out, prefix, *einfo.nixCode);
         out << prefix << std::endl;
     }

From bfca5fc395bdaf5823413e1a1679f1b0b6db29dd Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 13 May 2020 09:52:36 -0600
Subject: [PATCH 2100/2196] change status messages to info level

---
 src/libstore/build.cc               |  2 +-
 src/libstore/gc.cc                  | 12 ++++++------
 src/libstore/local-store.cc         | 18 +++++++++---------
 src/libutil/logging.cc              |  6 +++++-
 src/libutil/util.cc                 | 13 +------------
 src/nix-env/user-env.cc             |  2 +-
 src/nix/make-content-addressable.cc |  2 +-
 src/nix/upgrade-nix.cc              |  2 +-
 8 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0359b10ac5d..961110d58a3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -4862,7 +4862,7 @@ void Worker::waitForInput()
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
         if (lastWokenUp == steady_time_point::min())
-            printError("waiting for locks or build slots...");
+            printInfo("waiting for locks or build slots...");
         if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
         timeout = std::max(1L,
             (long) std::chrono::duration_cast<std::chrono::seconds>(
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index e9942325707..e529416b7f4 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -41,7 +41,7 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType)
         throw SysError("opening global GC lock '%1%'", fnGCLock);
 
     if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printError("waiting for the big garbage collector lock...");
+        printInfo("waiting for the big garbage collector lock...");
         lockFile(fdGCLock.get(), lockType, true);
     }
 
@@ -231,7 +231,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(fd->get(), ltWrite, false)) {
-            printError("removing stale temporary roots file '%1%'", path);
+            printInfo("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
             writeFull(fd->get(), "d");
             continue;
@@ -751,7 +751,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    printError("finding garbage collector roots...");
+    printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
@@ -803,9 +803,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     } else if (options.maxFreed > 0) {
 
         if (state.shouldDelete)
-            printError("deleting garbage...");
+            printInfo("deleting garbage...");
         else
-            printError("determining live/dead paths...");
+            printInfo("determining live/dead paths...");
 
         try {
 
@@ -872,7 +872,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
-        printError("deleting unused links...");
+        printInfo("deleting unused links...");
         removeUnusedLinks(state);
     }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 63a825af067..59d5510328d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -152,7 +152,7 @@ LocalStore::LocalStore(const Params & params)
     globalLock = openLockFile(globalLockPath.c_str(), true);
 
     if (!lockFile(globalLock.get(), ltRead, false)) {
-        printError("waiting for the big Nix store lock...");
+        printInfo("waiting for the big Nix store lock...");
         lockFile(globalLock.get(), ltRead, true);
     }
 
@@ -183,7 +183,7 @@ LocalStore::LocalStore(const Params & params)
                 "please upgrade Nix to version 1.11 first.");
 
         if (!lockFile(globalLock.get(), ltWrite, false)) {
-            printError("waiting for exclusive access to the Nix store...");
+            printInfo("waiting for exclusive access to the Nix store...");
             lockFile(globalLock.get(), ltWrite, true);
         }
 
@@ -261,7 +261,7 @@ LocalStore::~LocalStore()
     }
 
     if (future.valid()) {
-        printError("waiting for auto-GC to finish on exit...");
+        printInfo("waiting for auto-GC to finish on exit...");
         future.get();
     }
 
@@ -1210,7 +1210,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path)
 
 bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 {
-    printError(format("reading the Nix store..."));
+    printInfo(format("reading the Nix store..."));
 
     bool errors = false;
 
@@ -1251,7 +1251,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 });
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
-                        printError("removed link '%s'", linkPath);
+                        printInfo("removed link '%s'", linkPath);
                     else
                         throw SysError("removing corrupt link '%s'", linkPath);
                 } else {
@@ -1294,14 +1294,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                     /* Fill in missing hashes. */
                     if (info->narHash == nullHash) {
-                        printError("fixing missing hash on '%s'", printStorePath(i));
+                        printInfo("fixing missing hash on '%s'", printStorePath(i));
                         info->narHash = current.first;
                         update = true;
                     }
 
                     /* Fill in missing narSize fields (from old stores). */
                     if (info->narSize == 0) {
-                        printError("updating size field on '%s' to %s", printStorePath(i), current.second);
+                        printInfo("updating size field on '%s' to %s", printStorePath(i), current.second);
                         info->narSize = current.second;
                         update = true;
                     }
@@ -1360,7 +1360,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             }
 
         if (canInvalidate) {
-            printError("path '%s' disappeared, removing from database...", pathS);
+            printInfo("path '%s' disappeared, removing from database...", pathS);
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
@@ -1431,7 +1431,7 @@ static void makeMutable(const Path & path)
 void LocalStore::upgradeStore7()
 {
     if (getuid() != 0) return;
-    printError("removing immutable bits from the Nix store (this may take a while)...");
+    printInfo("removing immutable bits from the Nix store (this may take a while)...");
     makeMutable(realStoreDir);
 }
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 736ce604e01..071a7ec7eac 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -251,7 +251,11 @@ bool handleJSONLogMessage(const std::string & msg,
         }
 
     } catch (std::exception & e) {
-        printError("bad log message from builder: %s", e.what());
+        logError(
+            ErrorInfo { 
+                .name = "Json log message",
+                .hint = hintfmt("bad log message from builder: %s", e.what())
+        });
     }
 
     return true;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index ac7c2967b4a..dcf89ff6922 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -857,7 +857,7 @@ int Pid::kill()
 #if __FreeBSD__ || __APPLE__
         if (errno != EPERM || ::kill(pid, 0) != 0)
 #endif
-            printError((SysError("killing process %d", pid).msg()));
+            logError(SysError("killing process %d", pid).info());
     }
 
     return wait();
@@ -1447,17 +1447,6 @@ string base64Decode(const string & s)
 }
 
 
-// void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc)
-// {
-//     try {
-//         failure(exc);
-//     } catch (std::exception & e) {
-//         printError("uncaught exception: %s", e.what());
-//         abort();
-//     }
-// }
-
-
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 2484b97593f..f804b77a04e 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -146,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         Path lockTokenCur = optimisticLockProfile(profile);
         if (lockToken != lockTokenCur) {
-            printError("profile '%1%' changed while we were busy; restarting", profile);
+            printInfo("profile '%1%' changed while we were busy; restarting", profile);
             return false;
         }
 
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 8803461f48f..0da4fa4b42d 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -85,7 +85,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             info.ca = makeFixedOutputCA(true, info.narHash);
 
             if (!json)
-                printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
+                printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
 
             auto source = sinkToSource([&](Sink & nextSink) {
                 RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink);
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 9e7ebcd9c6a..fdf94e5a37d 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -98,7 +98,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
                 {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"});
         }
 
-        printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
+        printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
     }
 
     /* Return the profile in which Nix is installed. */

From c79d4addab7ea476f551bc8727e4d1f27adcef31 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 13 May 2020 10:02:18 -0600
Subject: [PATCH 2101/2196] consistent capitalization

---
 src/build-remote/build-remote.cc  | 2 +-
 src/libstore/builtins/buildenv.cc | 4 ++--
 src/libstore/filetransfer.cc      | 2 +-
 src/libstore/gc.cc                | 2 +-
 src/libstore/local-store.cc       | 4 ++--
 src/libstore/optimise-store.cc    | 4 ++--
 src/libstore/sqlite.cc            | 2 +-
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 0d6859596fa..0caa97fa4dd 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -202,7 +202,7 @@ static int _main(int argc, char * * argv)
                     auto msg = chomp(drainFD(5, false));
                     logError(
                         ErrorInfo { 
-                            .name = "remote build",
+                            .name = "Remote build",
                             .hint = hintfmt("cannot build on '%s': %s%s",
                         bestMachine->storeUri, e.what(),
                         (msg.empty() ? "" : ": " + msg))
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 64085fcc9b9..6c493ed770b 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -24,7 +24,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         if (e.errNo == ENOTDIR) {
             logWarning(
                 ErrorInfo { 
-                    .name = "Create Links - Directory",
+                    .name = "Create links - directory",
                     .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
             });
             return;
@@ -47,7 +47,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
                 logWarning(
                     ErrorInfo { 
-                        .name = "Create Links - Skipping Symlink",
+                        .name = "Create links - skipping symlink",
                         .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
                 });
                 continue;
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 3285eddedef..89588d66269 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -601,7 +601,7 @@ struct curlFileTransfer : public FileTransfer
         } catch (std::exception & e) {
             logError(
                 ErrorInfo { 
-                    .name = "download",
+                    .name = "File transfer",
                     .hint = hintfmt("unexpected error in download thread: %s", 
                                     e.what())
                 });
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index e529416b7f4..d53d458e9ed 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -132,7 +132,7 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
 
             logWarning(
                 ErrorInfo { 
-                    .name = "GC Root",
+                    .name = "GC root",
                     .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
                 "therefore, '%2%' might be removed by the garbage collector",
                 gcRoot, printStorePath(storePath))
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 59d5510328d..194eefc7827 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1244,7 +1244,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             if (hash != link.name) {
                 logError(
                     ErrorInfo { 
-                        .name = "Invalid Hash",
+                        .name = "Invalid hash",
                         .hint = hintfmt(
                     "link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash)
@@ -1283,7 +1283,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     logError(
                         ErrorInfo { 
-                            .name = "Invalid Hash - Path Modified",
+                            .name = "Invalid hash - path modified",
                             .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
                             printStorePath(i), info->narHash.to_string(), current.first.to_string())
                     });
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 9f611218349..c9ecca5a871 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -132,7 +132,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
         logWarning(
             ErrorInfo { 
-                .name = "Suspicious File",
+                .name = "Suspicious file",
                 .hint = hintfmt("skipping suspicious writable file '%1%'", path)
         });
         return;
@@ -200,7 +200,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (st.st_size != stLink.st_size) {
         logWarning(
             ErrorInfo { 
-                .name = "Corrupted Link",
+                .name = "Corrupted link",
                 .hint = hintfmt("removing corrupted link '%1%'", linkPath)
         });
         unlink(linkPath.c_str());
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 082c5400513..34ef5e57b50 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -205,7 +205,7 @@ void handleSQLiteBusy(SQLiteBusy & e)
     if (now > lastWarned + 10) {
         lastWarned = now;
         logWarning(
-            ErrorInfo { .name = "sqlite busy",
+            ErrorInfo { .name = "Sqlite busy",
                         .hint = hintfmt(e.what())
             });
     }

From d44bac1d9232374bf823ed24f5a5ae693f89fb7d Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 13 May 2020 12:39:45 -0600
Subject: [PATCH 2102/2196] remove error-demo from Makefile again

---
 Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 05c4bdc8a01..0ba011e2ad0 100644
--- a/Makefile
+++ b/Makefile
@@ -17,8 +17,7 @@ makefiles = \
   misc/upstart/local.mk \
   doc/manual/local.mk \
   tests/local.mk \
-  tests/plugins/local.mk \
-  src/error-demo/local.mk
+  tests/plugins/local.mk
 
 -include Makefile.config
 

From ef9dd9f9bc18abc9761812e30a26008c99a26166 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 13 May 2020 15:56:39 -0600
Subject: [PATCH 2103/2196] formatting and a few minor changes

---
 src/libexpr/parser.y                  |  2 -
 src/libexpr/primops.cc                | 59 ++++++++++++---------------
 src/libexpr/primops/context.cc        |  2 -
 src/libexpr/primops/fetchMercurial.cc |  1 -
 src/libstore/build.cc                 | 40 ++++++++----------
 src/libstore/gc.cc                    |  1 -
 src/libstore/local-store.cc           |  2 +-
 src/libstore/sqlite.cc                |  2 +-
 src/libstore/sqlite.hh                |  2 +-
 src/libutil/types.hh                  |  8 ++--
 src/nix-daemon/nix-daemon.cc          |  4 +-
 src/nix-env/nix-env.cc                |  6 +--
 src/nix-store/nix-store.cc            |  2 +-
 13 files changed, 54 insertions(+), 77 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 49fb8ad3745..0417a3c217f 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -427,7 +427,6 @@ expr_simple
                   .hint = hintfmt("URL literals are disabled"),
                   .nixCode = NixCode { .errPos = CUR_POS }
               });
-
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -696,7 +695,6 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-
     throw ThrownError(
         ErrorInfo { 
             .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path),
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 657ce300148..d21c4981321 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -99,7 +99,6 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
                         path, e.path),
                 .nixCode = NixCode { .errPos = pos }
             });
-
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -179,7 +178,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
             ErrorInfo {
                 .hint = hintfmt(
                     "cannot import '%1%', since path '%2%' is not valid",
-                   path, e.path),
+                    path, e.path),
                 .nixCode = NixCode { .errPos = pos }
             });
     }
@@ -199,8 +198,8 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
         if (message)
             throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message);
         else
-            throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected"
-                   , sym, path);
+            throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected",
+                sym, path);
     }
 
     (func)(state, v);
@@ -233,8 +232,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     } catch (InvalidPathError & e) {
         throw EvalError(
             ErrorInfo {
-                .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid"
-                     , program, e.path),
+                .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+                    program, e.path),
                 .nixCode = NixCode { .errPos = pos }
             });}
 
@@ -625,8 +624,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                             .hint = hintfmt("duplicate derivation output '%1%'", j),
                             .nixCode = NixCode { .errPos = posDrvName }
                         });
-
-
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
@@ -638,7 +635,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                             .hint = hintfmt("invalid derivation output name 'drv'" ),
                             .nixCode = NixCode { .errPos = posDrvName }
                         });
-
                 outputs.insert(j);
             }
             if (outputs.empty())
@@ -1236,7 +1232,6 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
                             path),
                         .nixCode = NixCode { .errPos = *attr.pos }
                     });
-
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
@@ -1322,7 +1317,6 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
                 .hint = hintfmt("attribute '%1%' missing", attr),
                 .nixCode = NixCode { .errPos = pos }
             });
-
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
@@ -1407,7 +1401,6 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
                     .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), 
                     .nixCode = NixCode { .errPos = pos }
                 });
-
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
@@ -1419,7 +1412,6 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
                         .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), 
                         .nixCode = NixCode { .errPos = pos }
                     });
-
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -1498,7 +1490,6 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
                 .nixCode = NixCode { .errPos = pos }
             });
 
-
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
         return;
@@ -2038,16 +2029,16 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
         if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), 
+                ErrorInfo {
+                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
                     .nixCode = NixCode { .errPos = pos }
                 });
         } else {
-          throw EvalError(
-              ErrorInfo { 
-                  .hint = hintfmt("invalid regular expression '%s'", re), 
-                  .nixCode = NixCode { .errPos = pos }
-              });
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("invalid regular expression '%s'", re),
+                    .nixCode = NixCode { .errPos = pos }
+                });
         }
     }
 }
@@ -2111,19 +2102,19 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
 
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
-          // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-          throw EvalError(
-              ErrorInfo { 
-                  .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), 
-                  .nixCode = NixCode { .errPos = pos }
-              });
-         } else {
-          throw EvalError(
-              ErrorInfo { 
-                  .hint = hintfmt("invalid regular expression '%s'", re), 
-                  .nixCode = NixCode { .errPos = pos }
-              });
-         }
+            // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                    .nixCode = NixCode { .errPos = pos }
+                });
+        } else {
+            throw EvalError(
+                ErrorInfo {
+                    .hint = hintfmt("invalid regular expression '%s'", re),
+                    .nixCode = NixCode { .errPos = pos }
+                });
+        }
     }
 }
 
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 768936453bc..7f895fc0159 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -151,7 +151,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                     .hint = hintfmt("Context key '%s' is not a store path", i.name),
                     .nixCode = NixCode { .errPos = *i.pos }
                 });
-
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -170,7 +169,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                             .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
                             .nixCode = NixCode { .errPos = *i.pos }
                         });
-
                 }
                 context.insert("=" + string(i.name));
             }
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 1e8b9d4a02a..bb008ba6b36 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -43,7 +43,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
                         .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
                         .nixCode = NixCode { .errPos = *attr.pos }
                     });
-
         }
 
         if (url.empty())
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 961110d58a3..64bc9c4b854 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -497,8 +497,8 @@ void handleDiffHook(
                 printError(chomp(diffRes.second));
         } catch (Error & error) {
             ErrorInfo ei = error.info();
-            string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : "");
-            ei.hint = std::optional(hintfmt("diff hook execution failed: %s", prevhint));
+            ei.hint = hintfmt("diff hook execution failed: %s",
+                (error.info().hint.has_value() ? error.info().hint->str() : ""));
             logError(ei);
         }
     }
@@ -1151,7 +1151,7 @@ void DerivationGoal::loadDerivation()
 
     if (nrFailed != 0) {
         logError(
-            ErrorInfo { 
+            ErrorInfo {
                 .name = "missing derivation during build",
                 .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))
         });
@@ -1306,13 +1306,12 @@ void DerivationGoal::repairClosure()
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
         logError(
-            ErrorInfo { 
+            ErrorInfo {
                 .name = "Corrupt path in closure",
                 .hint = hintfmt(
                     "found corrupted or missing path '%s' in the output closure of '%s'",
                     worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
         });
-
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
             addWaitee(worker.makeSubstitutionGoal(i, Repair));
@@ -1347,7 +1346,7 @@ void DerivationGoal::inputsRealised()
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
         logError(
-            ErrorInfo { 
+            ErrorInfo {
                 .name = "Dependencies could not be built",
                 .hint = hintfmt(
                     "cannot build derivation '%s': %s dependencies couldn't be built",
@@ -1811,7 +1810,7 @@ HookReply DerivationGoal::tryBuildHook()
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
             logError(
-                ErrorInfo { 
+                ErrorInfo {
                     .name = "Build hook died",
                     .hint = hintfmt(
                         "build hook died unexpectedly: %s",
@@ -3676,7 +3675,7 @@ void DerivationGoal::registerOutputs()
         if (!outputRewrites.empty()) {
             logWarning(
                 ErrorInfo {
-                    .name = "Rewriting hashes", 
+                    .name = "Rewriting hashes",
                     .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
                     });
 
@@ -3856,7 +3855,7 @@ void DerivationGoal::registerOutputs()
                     throw NotDeterministic(hint);
 
                 logError(
-                    ErrorInfo { 
+                    ErrorInfo {
                         .name = "Output determinism error",
                         .hint = hint
                 });
@@ -4124,7 +4123,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
             logError(
-                ErrorInfo { 
+                ErrorInfo {
                     .name = "Max log size exceeded",
                     .hint = hintfmt(
                         "%1% killed after writing more than %2% bytes of log output",
@@ -4447,9 +4446,9 @@ void SubstitutionGoal::tryNext()
     {
         logWarning(
             ErrorInfo {
-                .name = "Invalid path signature", 
+                .name = "Invalid path signature",
                 .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-            sub->getUri(), worker.store.printStorePath(storePath))
+                    sub->getUri(), worker.store.printStorePath(storePath))
             });
         tryNext();
         return;
@@ -4809,7 +4808,7 @@ void Worker::run(const Goals & _topGoals)
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
         else {
-            if (awake.empty() && 0 == settings.maxBuildJobs) 
+            if (awake.empty() && 0 == settings.maxBuildJobs)
                 throw Error("unable to start any build; either increase '--max-jobs' "
                             "or enable remote builds");
             assert(!awake.empty());
@@ -4933,7 +4932,7 @@ void Worker::waitForInput()
             after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
             logError(
-                ErrorInfo { 
+                ErrorInfo {
                     .name = "Silent build timeout",
                     .hint = hintfmt(
                         "%1% timed out after %2% seconds of silence",
@@ -4948,7 +4947,7 @@ void Worker::waitForInput()
             after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
             logError(
-                ErrorInfo { 
+                ErrorInfo {
                     .name = "Build timeout",
                     .hint = hintfmt(
                         "%1% timed out after %2% seconds",
@@ -4969,9 +4968,6 @@ void Worker::waitForInput()
 }
 
 
-
-
-
 unsigned int Worker::exitStatus()
 {
     /*
@@ -5015,9 +5011,9 @@ bool Worker::pathContentsGood(const StorePath & path)
         res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache.insert_or_assign(path.clone(), res);
-    if (!res) 
+    if (!res)
         logError(
-            ErrorInfo { 
+            ErrorInfo {
                 .name = "Corrupted path",
                 .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
         });
@@ -5136,8 +5132,4 @@ void LocalStore::repairPath(const StorePath & path)
 }
 
 
-
 }
-
-
-
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index d53d458e9ed..92f9328c3b8 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -129,7 +129,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     if (settings.checkRootReachability) {
         auto roots = findRoots(false);
         if (roots[storePath.clone()].count(gcRoot) == 0)
-
             logWarning(
                 ErrorInfo { 
                     .name = "GC root",
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 194eefc7827..6f88a3e322c 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -504,7 +504,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
             chown(path.c_str(), geteuid(), getegid()) == -1)
 #endif
             throw SysError("changing owner of '%1%' to %2%",
-                  path, geteuid());
+                path, geteuid());
     }
 
     if (S_ISDIR(st.st_mode)) {
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 34ef5e57b50..16cdb6619b4 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -196,7 +196,7 @@ SQLiteTxn::~SQLiteTxn()
     }
 }
 
-void handleSQLiteBusy(SQLiteBusy & e)
+void handleSQLiteBusy(const SQLiteBusy & e)
 {
     static std::atomic<time_t> lastWarned{0};
 
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index ccfac48d8eb..dd81ab05135 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -99,7 +99,7 @@ MakeError(SQLiteBusy, SQLiteError);
 
 [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs);
 
-void handleSQLiteBusy(SQLiteBusy & e);
+void handleSQLiteBusy(const SQLiteBusy & e);
 
 /* Convenience function for retrying a SQLite transaction when the
    database is busy. */
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 4d783b56434..66072e8f765 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -15,13 +15,13 @@ using std::set;
 using std::vector;
 using std::string;
 
-typedef list<std::string> Strings;
-typedef set<std::string> StringSet;
-typedef std::map<std::string, std::string> StringMap;
+typedef list<string> Strings;
+typedef set<string> StringSet;
+typedef std::map<string, string> StringMap;
 
 /* Paths are just strings. */
 
-typedef std::string Path;
+typedef string Path;
 typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 731ab22e5cb..582c78d14ed 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -248,8 +248,8 @@ static void daemonLoop(char * * argv)
             return;
         } catch (Error & error) {
             ErrorInfo ei = error.info();
-            string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : "");
-            ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint));
+            ei.hint = std::optional(hintfmt("error processing connection: %1%",
+                (error.info().hint.has_value() ? error.info().hint->str() : "")));
             logError(ei);
         }
     }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 0f99f5fe15c..bbce5dcdafe 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -69,7 +69,7 @@ typedef void (* Operation) (Globals & globals,
 static string needArg(Strings::iterator & i,
     Strings & args, const string & arg)
 {
-    if (i == args.end()) throw UsageError( "'%1%' requires an argument", arg);
+    if (i == args.end()) throw UsageError("'%1%' requires an argument", arg);
     return *i++;
 }
 
@@ -151,7 +151,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
 {
     struct stat st;
     if (stat(path.c_str(), &st) == -1)
-        throw SysError("getting inon about '%1%'", path);
+        throw SysError("getting information about '%1%'", path);
 
     if (isNixExpr(path, st))
         state.evalFile(path, v);
@@ -1233,7 +1233,7 @@ static void switchGeneration(Globals & globals, int dstGen)
     if (!dst) {
         if (dstGen == prevGen)
             throw Error("no generation older than the current (%1%) exists",
-                        curGen);
+                curGen);
         else
             throw Error("generation %1% does not exist", dstGen);
     }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 9b5cceccf58..77deea5a7c2 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -731,7 +731,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         if (current.first != info->narHash) {
             logError(
                 ErrorInfo { 
-                    .name = "Hash match error",
+                    .name = "Hash mismatch",
                     .hint = hintfmt(
                         "path '%s' was modified! expected hash '%s', got '%s'",
                         store->printStorePath(path),

From 183dd28266e0292116da6cfbc3e2643577adbfbe Mon Sep 17 00:00:00 2001
From: Alexander Bantyev <balsoft@balsoft.ru>
Date: Thu, 14 May 2020 17:00:54 +0300
Subject: [PATCH 2104/2196] Don't lock a user while doing remote builds

---
 src/libstore/build.cc | 88 ++++++++++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index a8c1cd565b7..f7fc23b358c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -507,6 +507,7 @@ class UserLock
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
+    bool isEnabled;
     string user;
     uid_t uid;
     gid_t gid;
@@ -524,7 +525,7 @@ class UserLock
 
     bool findFreeUser();
 
-    bool enabled() { return uid != 0; }
+    bool enabled() { return isEnabled; }
 
 };
 
@@ -535,9 +536,11 @@ UserLock::UserLock()
     createDirs(settings.nixStateDir + "/userpool");
     /* Mark that user is not enabled by default */
     uid = 0;
+    isEnabled = false;
 }
 
 bool UserLock::findFreeUser() {
+    if (enabled()) return true;
 
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -597,6 +600,7 @@ bool UserLock::findFreeUser() {
             supplementaryGIDs.resize(ngroups);
 #endif
 
+            isEnabled = true;
             return true;
         }
     }
@@ -931,6 +935,7 @@ class DerivationGoal : public Goal
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
+    void tryLocalBuild();
     void buildDone();
 
     /* Is the build hook willing to perform the build? */
@@ -1002,6 +1007,8 @@ class DerivationGoal : public Goal
         Goal::amDone(result);
     }
 
+    void started();
+
     void done(BuildResult::Status status, const string & msg = "");
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
@@ -1389,35 +1396,24 @@ void DerivationGoal::inputsRealised()
     result = BuildResult();
 }
 
+void DerivationGoal::started() {
+    auto msg = fmt(
+        buildMode == bmRepair ? "repairing outputs of '%s'" :
+        buildMode == bmCheck ? "checking outputs of '%s'" :
+        nrRounds > 1 ? "building '%s' (round %d/%d)" :
+        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+    fmt("building '%s'", worker.store.printStorePath(drvPath));
+    if (hook) msg += fmt(" on '%s'", machineName);
+    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
+        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
+    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+    worker.updateProgress();
+}
 
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
-
-        if (!buildUser->enabled()) {
-            if (!buildUser->findFreeUser()) {
-                debug("waiting for build users");
-                worker.waitForAWhile(shared_from_this());
-                return;
-            }
-
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -1464,20 +1460,6 @@ void DerivationGoal::tryToBuild()
        supported for local builds. */
     bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
 
-    auto started = [&]() {
-        auto msg = fmt(
-            buildMode == bmRepair ? "repairing outputs of '%s'" :
-            buildMode == bmCheck ? "checking outputs of '%s'" :
-            nrRounds > 1 ? "building '%s' (round %d/%d)" :
-            "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-        fmt("building '%s'", worker.store.printStorePath(drvPath));
-        if (hook) msg += fmt(" on '%s'", machineName);
-        act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-            Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-        mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-        worker.updateProgress();
-    };
-
     /* Is the build hook willing to accept this job? */
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -1510,6 +1492,34 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
+    state = &DerivationGoal::tryLocalBuild;
+    worker.wakeUp(shared_from_this());
+}
+
+void DerivationGoal::tryLocalBuild() {
+
+    /* If `build-users-group' is not empty, then we have to build as
+       one of the members of that group. */
+    if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
+        if (!buildUser) buildUser = std::make_unique<UserLock>();
+
+        if (buildUser->findFreeUser()) {
+            /* Make sure that no other processes are executing under this
+               uid. */
+            buildUser->kill();
+        } else {
+            debug("waiting for build users");
+            worker.waitForAWhile(shared_from_this());
+            return;
+        }
+#else
+        /* Don't know how to block the creation of setuid/setgid
+           binaries on this platform. */
+        throw Error("build users are not supported on this platform for security reasons");
+#endif
+    }
+
     try {
 
         /* Okay, we have to build. */

From 4daccb279c547b90b0c17c8a64825f04abdf0f7e Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 14 May 2020 10:28:17 -0600
Subject: [PATCH 2105/2196] formatting

---
 src/libexpr/primops.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d21c4981321..9b9cbfc2e68 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -767,7 +767,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .nixCode = NixCode { .errPos = posDrvName }
             });
 
-
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
         throw EvalError(
@@ -776,7 +775,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .nixCode = NixCode { .errPos = posDrvName }
             });
 
-
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")

From 19694aa213961daa5cbe2263bfaca53dc068a40c Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 14 May 2020 12:28:18 -0600
Subject: [PATCH 2106/2196] fix compile errors

---
 src/libutil/error.cc | 16 ++++++++++++++++
 src/libutil/error.hh | 16 ++--------------
 src/libutil/types.hh |  2 +-
 3 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 0765a2945f7..59628d875a5 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -3,6 +3,7 @@
 #include <iostream>
 #include <optional>
 #include "serialise.hh"
+#include <sstream>
 
 namespace nix {
 
@@ -16,6 +17,21 @@ BaseError & BaseError::addPrefix(const FormatOrString & fs)
     return *this;
 }
 
+const string& BaseError::calcWhat() const
+{
+    if (what_.has_value())
+        return *what_;
+    else {
+        err.name = sname();
+
+        std::ostringstream oss;
+        oss << err;
+        what_ = oss.str();
+
+        return *what_;
+    }
+}
+
 std::optional<string> ErrorInfo::programName = std::nullopt;
 
 std::ostream& operator<<(std::ostream &os, const hintformat &hf)
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 19c6e35a1c0..b374c2780fd 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -88,20 +88,8 @@ protected:
     mutable ErrorInfo err;
 
     mutable std::optional<string> what_;
-    const string& calcWhat() const
-    {
-        if (what_.has_value())
-            return *what_;
-        else {
-            err.name = sname();
-
-            std::ostringstream oss;
-            oss << err;
-            what_ = oss.str();
-
-            return *what_;
-        }
-    }
+    const string& calcWhat() const;
+    
 public:
     unsigned int status = 1; // exit status
 
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 66072e8f765..89ae108f9cb 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -6,7 +6,7 @@
 #include <list>
 #include <set>
 #include <map>
-
+#include <vector>
 
 namespace nix {
 

From 546b179d0a89e9f27a02e92004da0f8f08e5041a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Fri, 15 May 2020 10:06:14 +0200
Subject: [PATCH 2107/2196] actions: use latest OS

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 87997414d77..7feefc85588 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,7 +6,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos]
+        os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2

From e223eeac09c6468db80ee0497c84960b4ec73e02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sat, 16 May 2020 08:40:55 +0100
Subject: [PATCH 2108/2196] Remove -j option from simple-build-testing

By default Nix/NixOS already set a reasonable default `max-jobs = auto`
so we don't need to mention it in this tutorial.
The option is still documented in other parts of the documentation
if users ever stumble over this.

Fixes https://github.com/NixOS/nix/issues/2531
---
 doc/manual/expressions/simple-building-testing.xml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml
index 7326a3e76ad..ce0a1636dc9 100644
--- a/doc/manual/expressions/simple-building-testing.xml
+++ b/doc/manual/expressions/simple-building-testing.xml
@@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
 So it is always safe to run multiple instances of Nix in parallel
 (which isn’t the case with, say, <command>make</command>).</para>
 
-<para>If you have a system with multiple CPUs, you may want to have
-Nix build different derivations in parallel (insofar as possible).
-Just pass the option <link linkend='opt-max-jobs'><option>-j
-<replaceable>N</replaceable></option></link>, where
-<replaceable>N</replaceable> is the maximum number of jobs to be run
-in parallel, or set.  Typically this should be the number of
-CPUs.</para>
-
 </section>

From 5ef64f05e6e493d5fe69c87127328f68500b9e50 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 18 May 2020 15:50:29 +0200
Subject: [PATCH 2109/2196] Cleanup

---
 src/libstore/build.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4f2f38d63c9..a2b16f95cc8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -507,10 +507,10 @@ class UserLock
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
-    bool isEnabled;
+    bool isEnabled = false;
     string user;
-    uid_t uid;
-    gid_t gid;
+    uid_t uid = 0;
+    gid_t gid = 0;
     std::vector<gid_t> supplementaryGIDs;
 
 public:
@@ -534,9 +534,6 @@ UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
     createDirs(settings.nixStateDir + "/userpool");
-    /* Mark that user is not enabled by default */
-    uid = 0;
-    isEnabled = false;
 }
 
 bool UserLock::findFreeUser() {

From a73a820a5d73e0c595ff807a752f76ef1184ca2d Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Wed, 20 May 2020 16:27:53 +0200
Subject: [PATCH 2110/2196] Add unit testes for url.cc

This adds tests for

- parseURL
- percentDecode
- decodeQuery
---
 src/libutil/tests/tests.cc |   6 +-
 src/libutil/tests/url.cc   | 266 +++++++++++++++++++++++++++++++++++++
 2 files changed, 271 insertions(+), 1 deletion(-)
 create mode 100644 src/libutil/tests/url.cc

diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 5abfe2c9c96..8e77ccbe17a 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -5,6 +5,8 @@
 
 namespace nix {
 
+/* ----------- tests for util.hh ------------------------------------------------*/
+
     /* ----------------------------------------------------------------------------
      * absPath
      * --------------------------------------------------------------------------*/
@@ -15,6 +17,9 @@ namespace nix {
         ASSERT_EQ(p, "/");
     }
 
+
+
+
     TEST(absPath, turnsEmptyPathIntoCWD) {
         char cwd[PATH_MAX+1];
         auto p = absPath("");
@@ -581,5 +586,4 @@ namespace nix {
 
         ASSERT_EQ(filterANSIEscapes(s, true), "foo     bar     baz" );
     }
-
 }
diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc
new file mode 100644
index 00000000000..80646ad3e32
--- /dev/null
+++ b/src/libutil/tests/url.cc
@@ -0,0 +1,266 @@
+#include "url.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+/* ----------- tests for url.hh --------------------------------------------------*/
+
+    string print_map(std::map<string, string> m) {
+        std::map<string, string>::iterator it;
+        string s = "{ ";
+        for (it = m.begin(); it != m.end(); ++it) {
+            s += "{ ";
+            s += it->first;
+            s += " = ";
+            s += it->second;
+            s += " } ";
+        }
+        s += "}";
+        return s;
+    }
+
+
+    std::ostream& operator<<(std::ostream& os, const ParsedURL& p) {
+        return os << "\n"
+                  << "url:       " << p.url << "\n"
+                  << "base:      " << p.base << "\n"
+                  << "scheme:    " << p.scheme << "\n"
+                  << "authority: " << p.authority.value() << "\n"
+                  << "path:      " << p.path << "\n"
+                  << "query:     " << print_map(p.query) << "\n"
+                  << "fragment:  " << p.fragment << "\n";
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrl) {
+        auto s = "http://www.example.org/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://www.example.org/file.tar.gz",
+            .base = "http://www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpsUrl) {
+        auto s = "https://www.example.org/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "https://www.example.org/file.tar.gz",
+            .base = "https://www.example.org/file.tar.gz",
+            .scheme = "https",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) {
+        auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "https://www.example.org/file.tar.gz",
+            .base = "https://www.example.org/file.tar.gz",
+            .scheme = "https",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
+            .fragment = "hello",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) {
+        auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://www.example.org/file.tar.gz",
+            .base = "http://www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "field", "value" } },
+            .fragment = "?foo=bar#",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+
+    TEST(parseURL, parseIPv4Address) {
+        auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://127.0.0.1:8080/file.tar.gz",
+            .base = "https://127.0.0.1:8080/file.tar.gz",
+            .scheme = "http",
+            .authority = "127.0.0.1:8080",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
+            .fragment = "hello",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parseIPv6Address) {
+        auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .scheme = "http",
+            .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .path = "",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+
+    }
+
+    TEST(parseURL, parseEmptyQueryParams) {
+        auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&";
+        auto parsed = parseURL(s);
+        ASSERT_EQ(parsed.query, (StringMap) { });
+    }
+
+    TEST(parseURL, parseUserPassword) {
+        auto s = "http://user:pass@www.example.org:8080/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://user:pass@www.example.org/file.tar.gz",
+            .base = "http://user:pass@www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "user:pass@www.example.org:8080",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parseFileURLWithQueryAndFragment) {
+        auto s = "file:///none/of/your/business";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "",
+            .base = "",
+            .scheme = "file",
+            .authority = "",
+            .path = "/none/of/your/business",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+
+    }
+
+    TEST(parseURL, parsedUrlsIsEqualToItself) {
+        auto s = "http://www.example.org/file.tar.gz";
+        auto url = parseURL(s);
+
+        ASSERT_TRUE(url == url);
+    }
+
+    TEST(parseURL, parseFTPUrl) {
+        auto s = "ftp://ftp.nixos.org/downloads/nixos.iso";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "ftp://ftp.nixos.org/downloads/nixos.iso",
+            .base = "ftp://ftp.nixos.org/downloads/nixos.iso",
+            .scheme = "ftp",
+            .authority = "ftp.nixos.org",
+            .path = "/downloads/nixos.iso",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesAnythingInUriFormat) {
+        auto s = "whatever://github.com/NixOS/nixpkgs.git";
+        auto parsed = parseURL(s);
+    }
+
+    TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) {
+        auto s = "whatever:github.com/NixOS/nixpkgs.git";
+        auto parsed = parseURL(s);
+    }
+
+    TEST(parseURL, emptyStringIsInvalidURL) {
+        ASSERT_THROW(parseURL(""), Error);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * decodeQuery
+     * --------------------------------------------------------------------------*/
+
+    TEST(decodeQuery, emptyStringYieldsEmptyMap) {
+        auto d = decodeQuery("");
+        ASSERT_EQ(d, (StringMap) { });
+    }
+
+    TEST(decodeQuery, simpleDecode) {
+        auto d = decodeQuery("yi=one&er=two");
+        ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } }));
+    }
+
+    TEST(decodeQuery, decodeUrlEncodedArgs) {
+        auto d = decodeQuery("arg=%3D%3D%40%3D%3D");
+        ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } }));
+    }
+
+    TEST(decodeQuery, decodeArgWithEmptyValue) {
+        auto d = decodeQuery("arg=");
+        ASSERT_EQ(d, ((StringMap) { { "arg", ""} }));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * percentDecode
+     * --------------------------------------------------------------------------*/
+
+    TEST(percentDecode, decodesUrlEncodedString) {
+        string s = "==@==";
+        string d = percentDecode("%3D%3D%40%3D%3D");
+        ASSERT_EQ(d, s);
+    }
+
+    TEST(percentDecode, multipleDecodesAreIdempotent) {
+        string once = percentDecode("%3D%3D%40%3D%3D");
+        string twice = percentDecode(once);
+
+        ASSERT_EQ(once, twice);
+    }
+
+    TEST(percentDecode, trailingPercent) {
+        string s = "==@==%";
+        string d = percentDecode("%3D%3D%40%3D%3D%25");
+
+        ASSERT_EQ(d, s);
+    }
+
+}

From c8cb558849da8ef88f15cc2a70c570f1f5013a30 Mon Sep 17 00:00:00 2001
From: Krzysztof Gogolewski <krzysztof.gogolewski@tweag.io>
Date: Thu, 21 May 2020 19:29:13 +0200
Subject: [PATCH 2111/2196] documentation: avoid unquoted URLs

---
 doc/manual/command-ref/conf-file.xml           | 2 +-
 doc/manual/expressions/advanced-attributes.xml | 4 ++--
 doc/manual/expressions/builtins.xml            | 4 ++--
 doc/manual/expressions/expression-syntax.xml   | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1820598e51d..1fa74a14334 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -386,7 +386,7 @@ false</literal>.</para>
 
 <programlisting>
 builtins.fetchurl {
-  url = https://example.org/foo-1.2.3.tar.xz;
+  url = "https://example.org/foo-1.2.3.tar.xz";
   sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
 }
 </programlisting>
diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml
index 372d03de705..5759ff50e73 100644
--- a/doc/manual/expressions/advanced-attributes.xml
+++ b/doc/manual/expressions/advanced-attributes.xml
@@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 <programlisting>
 fetchurl {
-  url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
@@ -189,7 +189,7 @@ fetchurl {
 
 <programlisting>
 fetchurl {
-  url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+  url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
   sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
 }
 </programlisting>
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index f71a8f3be1e..6709c408146 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz;
+  url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz";
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 
@@ -1406,7 +1406,7 @@ stdenv.mkDerivation {
   ";
 
   src = fetchurl {
-    url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl;
diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml
index 42b9dca3623..a3de207135e 100644
--- a/doc/manual/expressions/expression-syntax.xml
+++ b/doc/manual/expressions/expression-syntax.xml
@@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
   name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
   builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
   src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
-    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
+    url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
     sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
   };
   inherit perl; <co xml:id='ex-hello-nix-co-6' />

From 0726ad5825f60e543d9cf535c62673685adbf5c8 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 15 Dec 2019 16:43:43 +0100
Subject: [PATCH 2112/2196] install: configure and bootstrap synthetic.conf on
 darwin

Starting macOS 10.15 /nix can't be creasted directly anymore due to the
readonly filesystem, but synthetic.conf was introduced to enable
creating mountpoints or symlinks for special usecases like package
managers.
---
 release.nix                         |  12 +++-
 scripts/create-darwin-volume.sh     | 107 ++++++++++++++++++++++++++++
 scripts/install-nix-from-closure.sh |  95 +++++++++++++++---------
 3 files changed, 177 insertions(+), 37 deletions(-)
 create mode 100755 scripts/create-darwin-volume.sh

diff --git a/release.nix b/release.nix
index f5729cee318..2cc4bb4c096 100644
--- a/release.nix
+++ b/release.nix
@@ -177,10 +177,10 @@ let
         }
         ''
           cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
           substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
-
           substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
             --subst-var-by nix ${toplevel} \
             --subst-var-by cacert ${cacert}
@@ -195,6 +195,7 @@ let
             # SC1090: Don't worry about not being able to find
             #         $nix/etc/profile.d/nix.sh
             shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
             shellcheck $TMPDIR/install-darwin-multi-user.sh
             shellcheck $TMPDIR/install-systemd-multi-user.sh
 
@@ -210,6 +211,7 @@ let
           fi
 
           chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
           chmod +x $TMPDIR/install-darwin-multi-user.sh
           chmod +x $TMPDIR/install-systemd-multi-user.sh
           chmod +x $TMPDIR/install-multi-user
@@ -222,11 +224,15 @@ let
             --absolute-names \
             --hard-dereference \
             --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
             --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
             --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
             $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user $TMPDIR/reginfo \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
             $(cat ${installerClosureInfo}/store-paths)
         '');
 
diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
new file mode 100755
index 00000000000..f3e0d46f10d
--- /dev/null
+++ b/scripts/create-darwin-volume.sh
@@ -0,0 +1,107 @@
+#!/bin/sh
+set -e
+
+root_disks() {
+    diskutil list -plist /
+}
+
+apfs_volumes_for() {
+    disk=$1
+    diskutil apfs list -plist "$disk"
+}
+
+disk_identifier() {
+    xpath "/plist/dict/key[text()='WholeDisks']/following-sibling::array[1]/string/text()" 2>/dev/null
+}
+
+volume_get() {
+    key=$1 i=$2
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
+}
+
+find_nix_volume() {
+    disk=$1
+    i=1
+    volumes=$(apfs_volumes_for "$disk")
+    while true; do
+        name=$(echo "$volumes" | volume_get "Name" "$i")
+        if [ -z "$name" ]; then
+            break
+        fi
+        case "$name" in
+            [Nn]ix*)
+                echo "$name"
+                break
+                ;;
+        esac
+        i=$((i+1))
+    done
+}
+
+test_fstab() {
+    grep -q "/nix" /etc/fstab 2>/dev/null
+}
+
+test_synthetic_conf() {
+    grep -q "^nix" /etc/synthetic.conf 2>/dev/null
+}
+
+test_nix() {
+    test -d "/nix"
+}
+
+main() {
+    (
+        echo ""
+        echo "     ------------------------------------------------------------------ "
+        echo "    | This installer will create a volume for the nix store and        |"
+        echo "    | configure it to mount at /nix.  Follow these steps to uninstall. |"
+        echo "     ------------------------------------------------------------------ "
+        echo ""
+        echo "  1. Remove the entry from fstab using 'sudo vifs'"
+        echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
+        echo "  3. Delete /etc/synthetic.conf"
+        echo ""
+    ) >&2
+
+    if [ -L "/nix" ]; then
+        echo "error: /nix is a symlink, please remove it or edit synthetic.conf (requires reboot)" >&2
+        echo "  /nix -> $(readlink "/nix")" >&2
+        exit 2
+    fi
+
+    if ! test_synthetic_conf; then
+        echo "Configuring /etc/synthetic.conf..." >&2
+        echo nix | sudo tee /etc/synthetic.conf
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+    fi
+
+    if ! test_nix; then
+        echo "Creating mountpoint for /nix..." >&2
+        sudo mkdir /nix
+    fi
+
+    disk=$(root_disks | disk_identifier)
+    volume=$(find_nix_volume "$disk")
+    if [ -z "$volume" ]; then
+        echo "Creating a Nix Store volume..." >&2
+        sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
+        volume="Nix Store"
+    else
+        echo "Using existing '$volume' volume" >&2
+    fi
+
+    if ! test_fstab; then
+        echo "Configuring /etc/fstab..." >&2
+        label=$(echo "$volume" | sed 's/ /\\040/g')
+        printf "\$a\nLABEL=%s /nix apfs rw\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+    fi
+
+    echo "The following options can be enabled to disable spotlight indexing" >&2
+    echo "of the volume, which might be desirable." >&2
+    echo "" >&2
+    echo "   $ mdutil -i off /nix" >&2
+    echo "" >&2
+}
+
+main "$@"
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index e06530ddf3d..34be7ee6a6f 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -40,44 +40,62 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
 fi
 
 INSTALL_MODE=no-daemon
-
+CREATE_DARWIN_VOLUME=0
 # handle the command line flags
-while [ "x${1:-}" != "x" ]; do
-    if [ "x${1:-}" = "x--no-daemon" ]; then
-        INSTALL_MODE=no-daemon
-    elif [ "x${1:-}" = "x--daemon" ]; then
-        INSTALL_MODE=daemon
-    elif [ "x${1:-}" = "x--no-channel-add" ]; then
-        NIX_INSTALLER_NO_CHANNEL_ADD=1
-    elif [ "x${1:-}" = "x--no-modify-profile" ]; then
-        NIX_INSTALLER_NO_MODIFY_PROFILE=1
-    elif [ "x${1:-}" != "x" ]; then
-        (
-            echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]"
-
-            echo "Choose installation method."
-            echo ""
-            echo " --daemon:            Installs and configures a background daemon that manages the store,"
-            echo "                      providing multi-user support and better isolation for local builds."
-            echo "                      Both for security and reproducibility, this method is recommended if"
-            echo "                      supported on your platform."
-            echo "                      See https://nixos.org/nix/manual/#sect-multi-user-installation"
-            echo ""
-            echo " --no-daemon:         Simple, single-user installation that does not require root and is"
-            echo "                      trivial to uninstall."
-            echo "                      (default)"
-            echo ""
-            echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
-            echo ""
-            echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
-            echo "                      is installed by default."
-            echo ""
-        ) >&2
-        exit
-    fi
+while [ $# -gt 0 ]; do
+    case $1 in
+        --daemon)
+            INSTALL_MODE=daemon;;
+        --no-daemon)
+            INSTALL_MODE=no-daemon;;
+        --no-channel-add)
+            NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+        --no-modify-profile)
+            NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
+        --create-volume)
+            CREATE_DARWIN_VOLUME=1;;
+        *)
+            (
+                echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]"
+
+                echo "Choose installation method."
+                echo ""
+                echo " --daemon:    Installs and configures a background daemon that manages the store,"
+                echo "              providing multi-user support and better isolation for local builds."
+                echo "              Both for security and reproducibility, this method is recommended if"
+                echo "              supported on your platform."
+                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo ""
+                echo " --no-daemon: Simple, single-user installation that does not require root and is"
+                echo "              trivial to uninstall."
+                echo "              (default)"
+                echo ""
+                echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
+                echo ""
+                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
+                echo "                      is installed by default."
+                echo ""
+            ) >&2
+
+            if [ "$(uname -s)" = "Darwin" ]; then
+                (
+                    echo " --create-volume: Create an APFS volume for the store and create the /nix"
+                    echo "                  mountpoint for it using synthetic.conf."
+                    echo "                  (required on macOS >=10.15)"
+                    echo "                  See https://nixos.org/nix/manual/#sect-darwin-apfs-volume"
+                    echo ""
+                ) >&2
+            fi
+            exit;;
+    esac
     shift
 done
 
+if [ "$(uname -s)" = "Darwin" ] && [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
+    printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
+    "$self/create-darwin-volume.sh"
+fi
+
 if [ "$INSTALL_MODE" = "daemon" ]; then
     printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
     exec "$self/install-multi-user"
@@ -95,6 +113,15 @@ if ! [ -e $dest ]; then
     echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2
     if ! sudo sh -c "$cmd"; then
         echo "$0: please manually run '$cmd' as root to create $dest" >&2
+        if [ "$(uname -s)" = "Darwin" ]; then
+            (
+                echo ""
+                echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
+                echo "Use --create-volume or run the preparation steps manually."
+                echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume."
+                echo ""
+            ) >&2
+        fi
         exit 1
     fi
 fi

From 10202628b911980f05fc2c9460995af30f1bcbf3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sun, 15 Dec 2019 21:11:14 +0100
Subject: [PATCH 2113/2196] install: also configure ~/.zshenv

The default login shell for users on macOS 10.15 changed from bash to
zsh.  So while generally nonstandard we need to configure it to make nix
function out of the box on macOS.
---
 scripts/install-nix-from-closure.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 34be7ee6a6f..88275d1f008 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -197,6 +197,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
+    for i in .zshenv .zshrc; do
+        fn="$HOME/$i"
+        if [ -w "$fn" ]; then
+            if ! grep -q "$p" "$fn"; then
+                echo "modifying $fn..." >&2
+                echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+            fi
+            added=1
+            break
+        fi
+    done
 fi
 
 if [ -z "$added" ]; then

From 083bb3bbfcdccebd06bde81a66f158d51ed6e455 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 13 Feb 2020 12:57:35 +0100
Subject: [PATCH 2114/2196] install: show macOS 10.15 message with --daemon

---
 scripts/create-darwin-volume.sh     |  8 ++++----
 scripts/install-nix-from-closure.sh | 28 ++++++++++++++++------------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index f3e0d46f10d..d6fb44f43ed 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -1,8 +1,8 @@
 #!/bin/sh
 set -e
 
-root_disks() {
-    diskutil list -plist /
+root_disk() {
+    diskutil info -plist /
 }
 
 apfs_volumes_for() {
@@ -11,7 +11,7 @@ apfs_volumes_for() {
 }
 
 disk_identifier() {
-    xpath "/plist/dict/key[text()='WholeDisks']/following-sibling::array[1]/string/text()" 2>/dev/null
+    xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
 }
 
 volume_get() {
@@ -81,7 +81,7 @@ main() {
         sudo mkdir /nix
     fi
 
-    disk=$(root_disks | disk_identifier)
+    disk=$(root_disk | disk_identifier)
     volume=$(find_nix_volume "$disk")
     if [ -z "$volume" ]; then
         echo "Creating a Nix Store volume..." >&2
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 88275d1f008..7d32bf92ef0 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -91,9 +91,22 @@ while [ $# -gt 0 ]; do
     shift
 done
 
-if [ "$(uname -s)" = "Darwin" ] && [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
-    printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
-    "$self/create-darwin-volume.sh"
+if [ "$(uname -s)" = "Darwin" ]; then
+    if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
+        printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
+        "$self/create-darwin-volume.sh"
+    fi
+
+    info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
+    if ! [ -e $dest ] && [ -n "$info" ]; then
+        (
+            echo ""
+            echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
+            echo "Use --create-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume."
+            echo ""
+        ) >&2
+    fi
 fi
 
 if [ "$INSTALL_MODE" = "daemon" ]; then
@@ -113,15 +126,6 @@ if ! [ -e $dest ]; then
     echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2
     if ! sudo sh -c "$cmd"; then
         echo "$0: please manually run '$cmd' as root to create $dest" >&2
-        if [ "$(uname -s)" = "Darwin" ]; then
-            (
-                echo ""
-                echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-                echo "Use --create-volume or run the preparation steps manually."
-                echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume."
-                echo ""
-            ) >&2
-        fi
         exit 1
     fi
 fi

From ee89b7797d4ec1db6dad9df5fb3bb8cc2f05de12 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Fri, 17 Jan 2020 23:27:29 +0100
Subject: [PATCH 2115/2196] manual: add apfs volume section

---
 doc/manual/installation/installing-binary.xml | 103 ++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 3f57f47b5bf..86cbce6bf50 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -136,6 +136,109 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
+<section xml:id="sect-apfs-volume-installation">
+  <title>APFS Volume Installation</title>
+
+  <para>
+    The root filesystem is read-only as of macOS 10.15 Catalina, all writable
+    paths to a separate data volume.  This means creating or writing to <filename>/nix</filename>
+    is not allowed.  While changing the default prefix would be possible, it's
+    a very intrusive change that has side effects we want to avoid for now.
+  </para>
+
+  <para>
+    For common writable locations <literal>firmlinks</literal> where introduced,
+    described by Apple as a "bi-directional wormhole" between two filesystems.
+    Essentially a bind mount for APFS volumes.  However this is (currently) not
+    user configurable and only available for paths like <filename>/Users</filename>.
+  </para>
+
+  <para>
+    For special cases like NFS mount points or package manager roots <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
+    provides a mechanism for some limited, user-controlled file-creation at <filename>/</filename>.
+    This only applies on a reboot, but <command>apfs.util</command> can be used
+    to trigger the creation (not deletion) of new entries.
+  </para>
+
+<screen>
+alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+</screen>
+
+  <itemizedlist>
+    <listitem>
+      <para>
+        The simplest solution is creating a symlink with <filename>/etc/synthetic.conf</filename>
+        to the data volume. (not recommended)
+      </para>
+
+<screen>
+nix    /System/Volumes/Data/nix
+</screen>
+
+<screen>
+alice$ ls -l /
+lrwxr-xr-x   1 root  wheel    25 Jan  1  2019 nix -> /System/Volumes/Data/nix
+</screen>
+
+      <para>
+        However builds that detect or resolve this symlink will leak the canonical
+        location or even fail in certain cases, making this approach undesirable.
+      </para>
+    </listitem>
+
+    <listitem>
+      <para>
+        An empty directory can also be created using <filename>/etc/synthetic.conf</filename>,
+        this won't be writable but can be used as a mount point.  And with
+        <literal>APFS</literal> it's relatively easy to create an separate
+        volume for nix instead.
+      </para>
+
+<screen>
+nix
+</screen>
+
+<screen>
+alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
+alice$ mount
+/dev/disk1s6 on /nix (apfs, local, journaled)
+</screen>
+
+      <para>
+        This does make the installation more complicated, requiring both
+        <filename>/etc/synthetic.conf</filename> as well as <filename>/etc/fstab</filename>
+      </para>
+
+<screen>
+#
+# Warning - this file should only be modified with vifs(8)
+#
+# Failure to do so is unsupported and may be destructive.
+#
+LABEL=Nix\040Store /nix apfs rw
+</screen>
+
+      <para>
+        On macOS volumes are also mounted quite late, launchd services or other
+        things that start during login will start before our volume is mounted.
+        For these cases eg. <command>wait4path</command> must be used for
+        things that depend on <filename>/nix</filename>.
+      </para>
+
+      <para>
+        This new volume also won't be encrypted by default, and enabling is
+        only possible interactively?
+      </para>
+
+<screen>
+diskutil apfs enableFileVault /nix -user disk
+</screen>
+
+    </listitem>
+  </itemizedlist>
+
+</section>
+
 <section xml:id="sect-nix-install-pinned-version-url">
   <title>Installing a pinned Nix version from a URL</title>
 

From caface1980344347f2a50701ec133e98e6094c8c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 5 Mar 2020 23:12:52 +0100
Subject: [PATCH 2116/2196] install: hide the store volume on darwin

---
 scripts/create-darwin-volume.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index d6fb44f43ed..a1a67aceaa5 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -94,7 +94,7 @@ main() {
     if ! test_fstab; then
         echo "Configuring /etc/fstab..." >&2
         label=$(echo "$volume" | sed 's/ /\\040/g')
-        printf "\$a\nLABEL=%s /nix apfs rw\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
+        printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
     fi
 
     echo "The following options can be enabled to disable spotlight indexing" >&2

From 04f597c3f4d0ac8b8a677c798642329a5a6768e3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 7 Mar 2020 12:01:40 +0100
Subject: [PATCH 2117/2196] install: improve output and error handling

---
 doc/manual/installation/installing-binary.xml |  3 ++-
 scripts/create-darwin-volume.sh               | 17 ++++++++++++++---
 scripts/install-nix-from-closure.sh           |  7 ++++---
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 86cbce6bf50..7f4bba3a0a3 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -199,6 +199,7 @@ nix
 </screen>
 
 <screen>
+alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
 alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
 alice$ mount
 /dev/disk1s6 on /nix (apfs, local, journaled)
@@ -231,7 +232,7 @@ LABEL=Nix\040Store /nix apfs rw
       </para>
 
 <screen>
-diskutil apfs enableFileVault /nix -user disk
+alice$ diskutil apfs enableFileVault /nix -user disk
 </screen>
 
     </listitem>
diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index a1a67aceaa5..e2b49c9a042 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -73,12 +73,22 @@ main() {
     if ! test_synthetic_conf; then
         echo "Configuring /etc/synthetic.conf..." >&2
         echo nix | sudo tee /etc/synthetic.conf
-        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
+        if ! test_synthetic_conf; then
+            echo "error: failed to configure synthetic.conf" >&2
+            exit 1
+        fi
     fi
 
     if ! test_nix; then
         echo "Creating mountpoint for /nix..." >&2
-        sudo mkdir /nix
+        /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
+        if ! test_nix; then
+            sudo mkdir -p /nix 2>/dev/null || true
+        fi
+        if ! test_nix; then
+            echo "error: failed to bootstrap /nix, a reboot might be required" >&2
+            exit 1
+        fi
     fi
 
     disk=$(root_disk | disk_identifier)
@@ -97,10 +107,11 @@ main() {
         printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
     fi
 
+    echo "" >&2
     echo "The following options can be enabled to disable spotlight indexing" >&2
     echo "of the volume, which might be desirable." >&2
     echo "" >&2
-    echo "   $ mdutil -i off /nix" >&2
+    echo "   $ sudo mdutil -i off /nix" >&2
     echo "" >&2
 }
 
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 7d32bf92ef0..2f291ed4c4f 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -82,7 +82,7 @@ while [ $# -gt 0 ]; do
                     echo " --create-volume: Create an APFS volume for the store and create the /nix"
                     echo "                  mountpoint for it using synthetic.conf."
                     echo "                  (required on macOS >=10.15)"
-                    echo "                  See https://nixos.org/nix/manual/#sect-darwin-apfs-volume"
+                    echo "                  See https://nixos.org/nix/manual/#sect-apfs-volume-installation"
                     echo ""
                 ) >&2
             fi
@@ -102,10 +102,11 @@ if [ "$(uname -s)" = "Darwin" ]; then
         (
             echo ""
             echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-            echo "Use --create-volume or run the preparation steps manually."
-            echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume."
+            echo "Use sh <(curl https://nixos.org/nix/install) --create-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation"
             echo ""
         ) >&2
+        exit 1
     fi
 fi
 

From bc24c09968bb35fce599151f86d123cb5984f727 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 18 Mar 2020 20:34:46 +0100
Subject: [PATCH 2118/2196] install: make synthetic.conf and fstab checks
 stricter

---
 scripts/create-darwin-volume.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index e2b49c9a042..ea41334440c 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -39,11 +39,15 @@ find_nix_volume() {
 }
 
 test_fstab() {
-    grep -q "/nix" /etc/fstab 2>/dev/null
+    grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
+}
+
+test_nix_symlink() {
+    [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
 }
 
 test_synthetic_conf() {
-    grep -q "^nix" /etc/synthetic.conf 2>/dev/null
+    grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
 }
 
 test_nix() {
@@ -60,12 +64,12 @@ main() {
         echo ""
         echo "  1. Remove the entry from fstab using 'sudo vifs'"
         echo "  2. Destroy the data volume using 'diskutil apfs deleteVolume'"
-        echo "  3. Delete /etc/synthetic.conf"
+        echo "  3. Remove the 'nix' line from /etc/synthetic.conf or the file"
         echo ""
     ) >&2
 
-    if [ -L "/nix" ]; then
-        echo "error: /nix is a symlink, please remove it or edit synthetic.conf (requires reboot)" >&2
+    if test_nix_symlink; then
+        echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
         echo "  /nix -> $(readlink "/nix")" >&2
         exit 2
     fi

From 33865752960c9a2ff28eb9024f20d2103918685c Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 26 Mar 2020 20:14:08 +0100
Subject: [PATCH 2119/2196] manual: clarify volume creation section

---
 doc/manual/installation/installing-binary.xml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 7f4bba3a0a3..c11ba9cce01 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -141,13 +141,14 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
   <para>
     The root filesystem is read-only as of macOS 10.15 Catalina, all writable
-    paths to a separate data volume.  This means creating or writing to <filename>/nix</filename>
-    is not allowed.  While changing the default prefix would be possible, it's
-    a very intrusive change that has side effects we want to avoid for now.
+    paths were moved to a separate data volume.  This means creating or writing
+    to <filename>/nix</filename> is not allowed.  While changing the default prefix
+    would be possible, it's a very intrusive change that has side effects we want to
+    avoid for now.
   </para>
 
   <para>
-    For common writable locations <literal>firmlinks</literal> where introduced,
+    For common writable locations <literal>firmlinks</literal> were introduced,
     described by Apple as a "bi-directional wormhole" between two filesystems.
     Essentially a bind mount for APFS volumes.  However this is (currently) not
     user configurable and only available for paths like <filename>/Users</filename>.
@@ -156,8 +157,10 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     For special cases like NFS mount points or package manager roots <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
     provides a mechanism for some limited, user-controlled file-creation at <filename>/</filename>.
-    This only applies on a reboot, but <command>apfs.util</command> can be used
-    to trigger the creation (not deletion) of new entries.
+    This only applies at boot time, however <command>apfs.util</command> can be used
+    to trigger the creation (not deletion) of new entries without a reboot.
+    It would be ideal if this could create firmlinks, however a symlink or mountpoint
+    are the only options.
   </para>
 
 <screen>

From 477d7c2d07e146c91950401b8b9d9380ce6787e5 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 26 Mar 2020 21:51:13 +0100
Subject: [PATCH 2120/2196] installer: refuse apfs volume creation when
 FileVault is enabled

---
 doc/manual/installation/installing-binary.xml |  6 +++--
 scripts/create-darwin-volume.sh               | 22 +++++++++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index c11ba9cce01..4982486629e 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -230,8 +230,10 @@ LABEL=Nix\040Store /nix apfs rw
       </para>
 
       <para>
-        This new volume also won't be encrypted by default, and enabling is
-        only possible interactively?
+        This new volume also won't be encrypted by default, and enabling it
+        requires extra setup.  For machines with a <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>
+        all data is already entrypted at rest, older hardware won't even when
+        FileVault is enabled for the rest of the system.
       </para>
 
 <screen>
diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index ea41334440c..a0da85f4388 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -14,7 +14,12 @@ disk_identifier() {
     xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
 }
 
-volume_get() {
+volume_list_true() {
+    key=$1 t=$2
+    xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
+}
+
+volume_get_string() {
     key=$1 i=$2
     xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
 }
@@ -24,7 +29,7 @@ find_nix_volume() {
     i=1
     volumes=$(apfs_volumes_for "$disk")
     while true; do
-        name=$(echo "$volumes" | volume_get "Name" "$i")
+        name=$(echo "$volumes" | volume_get_string "Name" "$i")
         if [ -z "$name" ]; then
             break
         fi
@@ -54,6 +59,12 @@ test_nix() {
     test -d "/nix"
 }
 
+test_filevault() {
+    disk=$1
+    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true || return
+    ! sudo xartutil --list >/dev/null 2>/dev/null
+}
+
 main() {
     (
         echo ""
@@ -99,6 +110,13 @@ main() {
     volume=$(find_nix_volume "$disk")
     if [ -z "$volume" ]; then
         echo "Creating a Nix Store volume..." >&2
+
+        if test_filevault "$disk"; then
+            echo "error: FileVault detected, refusing to create unencrypted volume" >&2
+            echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" >&2
+            exit 1
+        fi
+
         sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
         volume="Nix Store"
     else

From 2b0a81d92d28994374465c44c79f020d5e044700 Mon Sep 17 00:00:00 2001
From: "Travis A. Everett" <travis.a.everett@gmail.com>
Date: Thu, 14 May 2020 21:59:10 -0500
Subject: [PATCH 2121/2196] focus on golden-path covering most scenarios

This should handle installation scenarios we can handle with
anything resembling confidence. Goal is approximating the existing
setup--not enforcing a best-practice...

Approaches (+ installer-handled, - manual) and configs each covers:

+ no change needed; /nix OK on boot volume:
  All pre-Catalina (regardless of T2 or FileVault use)

+ create new unencrypted volume:
  Catalina, pre-T2, no FileVault

+ create new encrypted-at-rest volume:
  Catalina, pre-T2, FileVault
  Catalina, T2, no FileVault

- require user to pre-create encrypted volume
  Catalina, T2, FileVault
---
 doc/manual/installation/installing-binary.xml | 362 +++++++++++++-----
 scripts/create-darwin-volume.sh               |  77 +++-
 scripts/install-nix-from-closure.sh           |  19 +-
 3 files changed, 337 insertions(+), 121 deletions(-)

diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml
index 4982486629e..8d548f0ea0b 100644
--- a/doc/manual/installation/installing-binary.xml
+++ b/doc/manual/installation/installing-binary.xml
@@ -6,16 +6,30 @@
 
 <title>Installing a Binary Distribution</title>
 
-<para>If you are using Linux or macOS, the easiest way to install Nix
-is to run the following command:
+<para>
+  If you are using Linux or macOS versions up to 10.14 (Mojave), the
+  easiest way to install Nix is to run the following command:
+</para>
 
 <screen>
   $ sh &lt;(curl https://nixos.org/nix/install)
 </screen>
 
-As of Nix 2.1.0, the Nix installer will always default to creating a
-single-user installation, however opting in to the multi-user
-installation is highly recommended.
+<para>
+  If you're using macOS 10.15 (Catalina) or newer, consult
+  <link linkend="sect-macos-installation">the macOS installation instructions</link>
+  before installing.
+</para>
+
+<para>
+  As of Nix 2.1.0, the Nix installer will always default to creating a
+  single-user installation, however opting in to the multi-user
+  installation is highly recommended.
+  <!-- TODO: this explains *neither* why the default version is
+  single-user, nor why we'd recommend multi-user over the default.
+  True prospective users don't have much basis for evaluating this.
+  What's it to me? Who should pick which? Why? What if I pick wrong?
+  -->
 </para>
 
 <section xml:id="sect-single-user-installation">
@@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
 root.  The script will invoke <command>sudo</command> to create
 <filename>/nix</filename> if it doesn’t already exist.  If you don’t
 have <command>sudo</command>, you should manually create
-<command>/nix</command> first as root, e.g.:
+<filename>/nix</filename> first as root, e.g.:
 
 <screen>
 $ mkdir /nix
@@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst
 <filename>.bash_profile</filename>, <filename>.bash_login</filename>
 and <filename>.profile</filename> to source
 <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
-the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment
+the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
 variable before executing the install script to disable this
 behaviour.
 </para>
@@ -81,12 +95,10 @@ $ rm -rf /nix
   <para>
     You can instruct the installer to perform a multi-user
     installation on your system:
-
-    <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --daemon
-</screen>
   </para>
 
+  <screen>sh &lt;(curl https://nixos.org/nix/install) --daemon</screen>
+
   <para>
     The multi-user installation of Nix will create build users between
     the user IDs 30001 and 30032, and a group with the group ID 30000.
@@ -136,82 +148,253 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 
 </section>
 
-<section xml:id="sect-apfs-volume-installation">
-  <title>APFS Volume Installation</title>
+<section xml:id="sect-macos-installation">
+  <title>macOS Installation</title>
 
   <para>
-    The root filesystem is read-only as of macOS 10.15 Catalina, all writable
-    paths were moved to a separate data volume.  This means creating or writing
-    to <filename>/nix</filename> is not allowed.  While changing the default prefix
-    would be possible, it's a very intrusive change that has side effects we want to
-    avoid for now.
+    Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
+    This means <filename>/nix</filename> can no longer live on your system
+    volume, and that you'll need a workaround to install Nix.
   </para>
 
   <para>
-    For common writable locations <literal>firmlinks</literal> were introduced,
-    described by Apple as a "bi-directional wormhole" between two filesystems.
-    Essentially a bind mount for APFS volumes.  However this is (currently) not
-    user configurable and only available for paths like <filename>/Users</filename>.
+    The recommended approach, which creates an unencrypted APFS volume
+    for your Nix store and a "synthetic" empty directory to mount it
+    over at <filename>/nix</filename>, is least likely to impair Nix
+    or your system.
   </para>
 
+  <note><para>
+    With all separate-volume approaches, it's possible something on
+    your system (particularly daemons/services and restored apps) may
+    need access to your Nix store before the volume is mounted. Adding
+    additional encryption makes this more likely.
+  </para></note>
+
   <para>
-    For special cases like NFS mount points or package manager roots <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
-    provides a mechanism for some limited, user-controlled file-creation at <filename>/</filename>.
-    This only applies at boot time, however <command>apfs.util</command> can be used
-    to trigger the creation (not deletion) of new entries without a reboot.
-    It would be ideal if this could create firmlinks, however a symlink or mountpoint
-    are the only options.
+    If you're using a recent Mac with a
+    <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
+    your drive will still be encrypted at rest (in which case "unencrypted"
+    is a bit of a misnomer). To use this approach, just install Nix with:
   </para>
 
-<screen>
-alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
-</screen>
-
-  <itemizedlist>
-    <listitem>
-      <para>
-        The simplest solution is creating a symlink with <filename>/etc/synthetic.conf</filename>
-        to the data volume. (not recommended)
-      </para>
-
-<screen>
-nix    /System/Volumes/Data/nix
-</screen>
-
-<screen>
-alice$ ls -l /
-lrwxr-xr-x   1 root  wheel    25 Jan  1  2019 nix -> /System/Volumes/Data/nix
-</screen>
-
-      <para>
-        However builds that detect or resolve this symlink will leak the canonical
-        location or even fail in certain cases, making this approach undesirable.
-      </para>
-    </listitem>
+  <screen>$ sh &lt;(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
 
-    <listitem>
-      <para>
-        An empty directory can also be created using <filename>/etc/synthetic.conf</filename>,
-        this won't be writable but can be used as a mount point.  And with
-        <literal>APFS</literal> it's relatively easy to create an separate
-        volume for nix instead.
-      </para>
-
-<screen>
-nix
-</screen>
-
-<screen>
-alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
-alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
-alice$ mount
-/dev/disk1s6 on /nix (apfs, local, journaled)
-</screen>
+  <para>
+    If you don't like the sound of this, you'll want to weigh the
+    other approaches and tradeoffs detailed in this section.
+  </para>
 
-      <para>
-        This does make the installation more complicated, requiring both
-        <filename>/etc/synthetic.conf</filename> as well as <filename>/etc/fstab</filename>
-      </para>
+  <note>
+    <title>Eventual solutions?</title>
+    <para>
+      All of the known workarounds have drawbacks, but we hope
+      better solutions will be available in the future. Some that
+      we have our eye on are:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          A true firmlink would enable the Nix store to live on the
+          primary data volume without the build problems caused by
+          the symlink approach. End users cannot currently
+          create true firmlinks.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          If the Nix store volume shared FileVault encryption
+          with the primary data volume (probably by using the same
+          volume group and role), FileVault encryption could be
+          easily supported by the installer without requiring
+          manual setup by each user.
+        </para>
+      </listitem>
+    </orderedlist>
+  </note>
+
+  <section xml:id="sect-macos-installation-change-store-prefix">
+    <title>Change the Nix store path prefix</title>
+    <para>
+      Changing the default prefix for the Nix store is a simple
+      approach which enables you to leave it on your root volume,
+      where it can take full advantage of FileVault encryption if
+      enabled. Unfortunately, this approach also opts your device out
+      of some benefits that are enabled by using the same prefix
+      across systems:
+
+      <itemizedlist>
+        <listitem>
+          <para>
+            Your system won't be able to take advantage of the binary
+            cache (unless someone is able to stand up and support
+            duplicate caching infrastructure), which means you'll
+            spend more time waiting for builds.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            It's harder to build and deploy packages to Linux systems.
+          </para>
+        </listitem>
+        <!-- TODO: may be more here -->
+      </itemizedlist>
+
+      <!-- TODO: Yes, but how?! -->
+
+      It would also possible (and often requested) to just apply this
+      change ecosystem-wide, but it's an intrusive process that has
+      side effects we want to avoid for now.
+      <!-- magnificent hand-wavy gesture -->
+    </para>
+    <para>
+    </para>
+  </section>
+
+  <section xml:id="sect-macos-installation-encrypted-volume">
+    <title>Use a separate encrypted volume</title>
+    <para>
+      If you like, you can also add encryption to the recommended
+      approach taken by the installer. You can do this by pre-creating
+      an encrypted volume before you run the installer--or you can
+      run the installer and encrypt the volume it creates later.
+      <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
+    </para>
+    <para>
+      In either case, adding encryption to a second volume isn't quite
+      as simple as enabling FileVault for your boot volume. Before you
+      dive in, there are a few things to weigh:
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          The additional volume won't be encrypted with your existing
+          FileVault key, so you'll need another mechanism to decrypt
+          the volume.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can store the password in Keychain to automatically
+          decrypt the volume on boot--but it'll have to wait on Keychain
+          and may not mount before your GUI apps restore. If any of
+          your launchd agents or apps depend on Nix-installed software
+          (for example, if you use a Nix-installed login shell), the
+          restore may fail or break.
+        </para>
+        <para>
+          On a case-by-case basis, you may be able to work around this
+          problem by using <command>wait4path</command> to block
+          execution until your executable is available.
+        </para>
+        <para>
+          It's also possible to decrypt and mount the volume earlier
+          with a login hook--but this mechanism appears to be
+          deprecated and its future is unclear.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          You can hard-code the password in the clear, so that your
+          store volume can be decrypted before Keychain is available.
+        </para>
+      </listitem>
+    </orderedlist>
+    <para>
+      If you are comfortable navigating these tradeoffs, you can encrypt the volume with
+      something along the lines of:
+      <!-- TODO:
+      I don't know if this also needs from-scratch instructions?
+      can we just recommend use-the-installer-and-then-encrypt?
+      -->
+    </para>
+    <!--
+    TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
+
+    It may be more clear to use encryptVolume, here? FileVault seems
+    heavily associated with the boot-volume behavior; I worry
+    a little that it can mislead here, especially as it gets
+    copied around minus doc context...?
+    -->
+    <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
+
+    <!-- TODO: and then go into detail on the mount/decrypt approaches? -->
+  </section>
+
+  <section xml:id="sect-macos-installation-symlink">
+    <!--
+    Maybe a good razor is: if we'd hate having to support someone who
+    installed Nix this way, it shouldn't even be detailed?
+    -->
+    <title>Symlink the Nix store to a custom location</title>
+    <para>
+      Another simple approach is using <filename>/etc/synthetic.conf</filename>
+      to symlink the Nix store to the data volume. This option also
+      enables your store to share any configured FileVault encryption.
+      Unfortunately, builds that resolve the symlink may leak the
+      canonical path or even fail.
+    </para>
+    <para>
+      Because of these downsides, we can't recommend this approach.
+    </para>
+    <!-- Leaving out instructions for this one. -->
+  </section>
+
+  <section xml:id="sect-macos-installation-recommended-notes">
+    <title>Notes on the recommended approach</title>
+    <para>
+      This section goes into a little more detail on the recommended
+      approach. You don't need to understand it to run the installer,
+      but it can serve as a helpful reference if you run into trouble.
+    </para>
+    <orderedlist>
+      <listitem>
+        <para>
+          In order to compose user-writable locations into the new
+          read-only system root, Apple introduced a new concept called
+          <literal>firmlinks</literal>, which it describes as a
+          "bi-directional wormhole" between two filesystems. You can
+          see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
+          Unfortunately, firmlinks aren't (currently?) user-configurable.
+        </para>
+
+        <para>
+          For special cases like NFS mount points or package manager roots,
+          <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
+          supports limited user-controlled file-creation (of symlinks,
+          and synthetic empty directories) at <filename>/</filename>.
+          To create a synthetic empty directory for mounting at <filename>/nix</filename>,
+          add the following line to <filename>/etc/synthetic.conf</filename>
+          (create it if necessary):
+        </para>
+
+        <screen>nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          This configuration is applied at boot time, but you can use
+          <command>apfs.util</command> to trigger creation (not deletion)
+          of new entries without a reboot:
+        </para>
+
+        <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Create the new APFS volume with diskutil:
+        </para>
+
+        <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
+      </listitem>
+
+      <listitem>
+        <para>
+          Using <command>vifs</command>, add the new mount to
+          <filename>/etc/fstab</filename>. If it doesn't already have
+          other entries, it should look something like:
+        </para>
 
 <screen>
 #
@@ -219,29 +402,16 @@ alice$ mount
 #
 # Failure to do so is unsupported and may be destructive.
 #
-LABEL=Nix\040Store /nix apfs rw
-</screen>
-
-      <para>
-        On macOS volumes are also mounted quite late, launchd services or other
-        things that start during login will start before our volume is mounted.
-        For these cases eg. <command>wait4path</command> must be used for
-        things that depend on <filename>/nix</filename>.
-      </para>
-
-      <para>
-        This new volume also won't be encrypted by default, and enabling it
-        requires extra setup.  For machines with a <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>
-        all data is already entrypted at rest, older hardware won't even when
-        FileVault is enabled for the rest of the system.
-      </para>
-
-<screen>
-alice$ diskutil apfs enableFileVault /nix -user disk
+LABEL=Nix\040Store /nix apfs rw,nobrowse
 </screen>
 
-    </listitem>
-  </itemizedlist>
+        <para>
+          The nobrowse setting will keep Spotlight from indexing this
+          volume, and keep it from showing up on your desktop.
+        </para>
+      </listitem>
+    </orderedlist>
+  </section>
 
 </section>
 
diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index a0da85f4388..47cc3e913ea 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -59,10 +59,45 @@ test_nix() {
     test -d "/nix"
 }
 
-test_filevault() {
+test_t2_chip_present(){
+    # Use xartutil to see if system has a t2 chip.
+    #
+    # This isn't well-documented on its own; until it is,
+    # let's keep track of knowledge/assumptions.
+    #
+    # Warnings:
+    # - Don't search "xart" if porn will cause you trouble :)
+    # - Other xartutil flags do dangerous things. Don't run them
+    #   naively. If you must, search "xartutil" first.
+    #
+    # Assumptions:
+    # - the "xART session seeds recovery utility"
+    #   appears to interact with xartstorageremoted
+    # - `sudo xartutil --list` lists xART sessions
+    #   and their seeds and exits 0 if successful. If
+    #   not, it exits 1 and prints an error such as:
+    #   xartutil: ERROR: No supported link to the SEP present
+    # - xART sessions/seeds are present when a T2 chip is
+    #   (and not, otherwise)
+    # - the presence of a T2 chip means a newly-created
+    #   volume on the primary drive will be
+    #   encrypted at rest
+    # - all together: `sudo xartutil --list`
+    #   should exit 0 if a new Nix Store volume will
+    #   be encrypted at rest, and exit 1 if not.
+    sudo xartutil --list >/dev/null 2>/dev/null
+}
+
+test_filevault_in_use() {
     disk=$1
-    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true || return
-    ! sudo xartutil --list >/dev/null 2>/dev/null
+    #      list vols on disk | get value of Filevault key | value is true
+    apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
+}
+
+# use after error msg for conditions we don't understand
+suggest_report_error(){
+    # ex "error: something sad happened :(" >&2
+    echo "       please report this @ https://github.com/nixos/nix/issues" >&2
 }
 
 main() {
@@ -89,7 +124,8 @@ main() {
         echo "Configuring /etc/synthetic.conf..." >&2
         echo nix | sudo tee /etc/synthetic.conf
         if ! test_synthetic_conf; then
-            echo "error: failed to configure synthetic.conf" >&2
+            echo "error: failed to configure synthetic.conf;" >&2
+            suggest_report_error
             exit 1
         fi
     fi
@@ -101,7 +137,8 @@ main() {
             sudo mkdir -p /nix 2>/dev/null || true
         fi
         if ! test_nix; then
-            echo "error: failed to bootstrap /nix, a reboot might be required" >&2
+            echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
+            suggest_report_error
             exit 1
         fi
     fi
@@ -111,10 +148,25 @@ main() {
     if [ -z "$volume" ]; then
         echo "Creating a Nix Store volume..." >&2
 
-        if test_filevault "$disk"; then
-            echo "error: FileVault detected, refusing to create unencrypted volume" >&2
-            echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" >&2
-            exit 1
+        if test_filevault_in_use "$disk"; then
+            # TODO: Not sure if it's in-scope now, but `diskutil apfs list`
+            # shows both filevault and encrypted at rest status, and it
+            # may be the more semantic way to test for this? It'll show
+            # `FileVault:                 No (Encrypted at rest)`
+            # `FileVault:                 No`
+            # `FileVault:                 Yes (Unlocked)`
+            # and so on.
+            if test_t2_chip_present; then
+                echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
+                echo "         is only encrypted at rest." >&2
+                echo "         See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+            else
+                echo "error: refusing to create Nix store volume because the boot volume is" >&2
+                echo "       FileVault encrypted, but encryption-at-rest is not available." >&2
+                echo "       Manually create a volume for the store and re-run this script." >&2
+                echo "       See https://nixos.org/nix/manual/#sect-macos-installation" >&2
+                exit 1
+            fi
         fi
 
         sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
@@ -128,13 +180,6 @@ main() {
         label=$(echo "$volume" | sed 's/ /\\040/g')
         printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
     fi
-
-    echo "" >&2
-    echo "The following options can be enabled to disable spotlight indexing" >&2
-    echo "of the volume, which might be desirable." >&2
-    echo "" >&2
-    echo "   $ sudo mdutil -i off /nix" >&2
-    echo "" >&2
 }
 
 main "$@"
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 2f291ed4c4f..72aa5abf5c2 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -52,7 +52,7 @@ while [ $# -gt 0 ]; do
             NIX_INSTALLER_NO_CHANNEL_ADD=1;;
         --no-modify-profile)
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
-        --create-volume)
+        --darwin-use-unencrypted-nix-store-volume)
             CREATE_DARWIN_VOLUME=1;;
         *)
             (
@@ -77,12 +77,13 @@ while [ $# -gt 0 ]; do
                 echo ""
             ) >&2
 
-            if [ "$(uname -s)" = "Darwin" ]; then
+            # darwin and Catalina+
+            if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
                 (
-                    echo " --create-volume: Create an APFS volume for the store and create the /nix"
-                    echo "                  mountpoint for it using synthetic.conf."
-                    echo "                  (required on macOS >=10.15)"
-                    echo "                  See https://nixos.org/nix/manual/#sect-apfs-volume-installation"
+                    echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
+                    echo "              store and mount it at /nix. This is the recommended way to create"
+                    echo "              /nix with a read-only / on macOS >=10.15."
+                    echo "              See: https://nixos.org/nix/manual/#sect-macos-installation"
                     echo ""
                 ) >&2
             fi
@@ -98,12 +99,12 @@ if [ "$(uname -s)" = "Darwin" ]; then
     fi
 
     info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
-    if ! [ -e $dest ] && [ -n "$info" ]; then
+    if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
         (
             echo ""
             echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-            echo "Use sh <(curl https://nixos.org/nix/install) --create-volume or run the preparation steps manually."
-            echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation"
+            echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
+            echo "See https://nixos.org/nix/manual/#sect-macos-installation"
             echo ""
         ) >&2
         exit 1

From d3df1889a1f49b114a8d78270ea8a6d0523f5e35 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Thu, 21 May 2020 20:03:09 +0200
Subject: [PATCH 2122/2196] installer: don't clobber synthetic.conf

---
 scripts/create-darwin-volume.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index 47cc3e913ea..5214d16529b 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -122,7 +122,7 @@ main() {
 
     if ! test_synthetic_conf; then
         echo "Configuring /etc/synthetic.conf..." >&2
-        echo nix | sudo tee /etc/synthetic.conf
+        echo nix | sudo tee -a /etc/synthetic.conf
         if ! test_synthetic_conf; then
             echo "error: failed to configure synthetic.conf;" >&2
             suggest_report_error

From 909d8cb2934869c945ac1cc20dfb71df513042eb Mon Sep 17 00:00:00 2001
From: Suraj Barkale <suraj@barkale.com>
Date: Fri, 22 May 2020 11:05:25 +1000
Subject: [PATCH 2123/2196] Use /etc/zshenv instead of /etc/zshrc for profile

As noted in https://github.com/NixOS/nix/issues/3456 the `/etc/zshenv` file provides a better place for sourcing the nix environment.
---
 scripts/install-multi-user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a0f1deb987f..83819273344 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -28,7 +28,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 

From 2a7ea2eb6c54c82d5e858ea6ae9de929face5e55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@enlambda.com>
Date: Sat, 23 May 2020 11:12:05 +0200
Subject: [PATCH 2124/2196] scripts/create-darwin-volume.sh: remove unused
 variable

---
 scripts/create-darwin-volume.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh
index 5214d16529b..dac30d72d73 100755
--- a/scripts/create-darwin-volume.sh
+++ b/scripts/create-darwin-volume.sh
@@ -15,7 +15,7 @@ disk_identifier() {
 }
 
 volume_list_true() {
-    key=$1 t=$2
+    key=$1
     xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
 }
 

From 6f6bdd63a0f6dae4ab91422645f548837029dee9 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Sat, 23 May 2020 12:10:12 +0200
Subject: [PATCH 2125/2196] fix hydra build products

Since the binary tarball was replaced none of the hydra builds include
the manual.  The dist phase isn't enabled by default the manual build
products where not written.
---
 release.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/release.nix b/release.nix
index 2cc4bb4c096..2a320e1c3bc 100644
--- a/release.nix
+++ b/release.nix
@@ -115,17 +115,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        postInstall = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
+
         doCheck = true;
 
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
 
         separateDebugInfo = true;
-
-        preDist = ''
-          mkdir -p $doc/nix-support
-          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-        '';
       });
 
 

From e2af11ce071fc109ad3e517bf15a87024d0a9ae5 Mon Sep 17 00:00:00 2001
From: Joachim Breitner <mail@joachim-breitner.de>
Date: Sat, 23 May 2020 15:26:59 +0200
Subject: [PATCH 2126/2196] Manpages: Do not refer to nixpkgs-channels

Unless I am misinformed, using the `nixpkgs` repository directly is now
preferred?
---
 doc/manual/command-ref/env-common.xml | 2 +-
 doc/manual/command-ref/nix-env.xml    | 7 ++-----
 doc/manual/command-ref/nix-shell.xml  | 6 +++---
 doc/manual/expressions/builtins.xml   | 4 ++--
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml
index 0217de7b2ab..8466cc46366 100644
--- a/doc/manual/command-ref/env-common.xml
+++ b/doc/manual/command-ref/env-common.xml
@@ -53,7 +53,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <envar>NIX_PATH</envar> to
 
     <screen>
-nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen>
+nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
 
     tells Nix to download the latest revision in the Nixpkgs/NixOS
     15.09 channel.</para>
diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml
index 9c03ccce11c..2b95b68191e 100644
--- a/doc/manual/command-ref/nix-env.xml
+++ b/doc/manual/command-ref/nix-env.xml
@@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
 14.12 channel:
 
 <screen>
-$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
 </screen>
 
-(The GitHub repository <literal>nixpkgs-channels</literal> is updated
-automatically from the main <literal>nixpkgs</literal> repository
-after certain tests have succeeded and binaries have been built and
-uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
+</para>
 
 </refsection>
 
diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml
index 766482460c8..2fef323c53d 100644
--- a/doc/manual/command-ref/nix-shell.xml
+++ b/doc/manual/command-ref/nix-shell.xml
@@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
 containing the Pan package from a specific revision of Nixpkgs:
 
 <screen>
-$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
 
 [nix-shell:~]$ pan --version
 Pan 0.139
@@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
 <programlisting><![CDATA[
 #! /usr/bin/env nix-shell
 #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
 
 import Network.HTTP
 import Text.HTML.TagSoup
@@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
 revision of Nixpkgs:
 
 <programlisting>
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
 </programlisting>
 
 </para>
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml
index 6709c408146..a18c5801a62 100644
--- a/doc/manual/expressions/builtins.xml
+++ b/doc/manual/expressions/builtins.xml
@@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
     particular version of Nixpkgs, e.g.
 
 <programlisting>
-with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
+with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
 
 stdenv.mkDerivation { … }
 </programlisting>
@@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
 
 <programlisting>
 with import (fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz";
+  url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
   sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
 }) {};
 

From ecc5c90dfc0e9877aef89bf0192203cd1a8f2b21 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Mon, 25 May 2020 11:50:41 +0200
Subject: [PATCH 2127/2196] Add unit tests for hashing functions

---
 src/libutil/tests/hash.cc | 80 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 src/libutil/tests/hash.cc

diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc
new file mode 100644
index 00000000000..7cb4398178a
--- /dev/null
+++ b/src/libutil/tests/hash.cc
@@ -0,0 +1,80 @@
+#include "hash.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * hashString
+     * --------------------------------------------------------------------------*/
+
+    TEST(hashString, testKnownMD5Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc1321
+        auto s1 = "";
+        auto hash = hashString(HashType::htMD5, s1);
+        ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e");
+    }
+
+    TEST(hashString, testKnownMD5Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc1321
+        auto s2 = "abc";
+        auto hash = hashString(HashType::htMD5, s2);
+        ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72");
+    }
+
+    TEST(hashString, testKnownSHA1Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc3174
+        auto s = "abc";
+        auto hash = hashString(HashType::htSHA1, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
+    }
+
+    TEST(hashString, testKnownSHA1Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc3174
+        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+        auto hash = hashString(HashType::htSHA1, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
+    }
+
+    TEST(hashString, testKnownSHA256Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abc";
+
+        auto hash = hashString(HashType::htSHA256, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),
+                "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
+    }
+
+    TEST(hashString, testKnownSHA256Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+        auto hash = hashString(HashType::htSHA256, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),
+                "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
+    }
+
+    TEST(hashString, testKnownSHA512Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abc";
+        auto hash = hashString(HashType::htSHA512, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),
+                "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9"
+                "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
+                "454d4423643ce80e2a9ac94fa54ca49f");
+    }
+
+    TEST(hashString, testKnownSHA512Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
+
+        auto hash = hashString(HashType::htSHA512, s);
+        ASSERT_EQ(hash.to_string(Base::Base16),
+                "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1"
+                "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
+                "c7d329eeb6dd26545e96e55b874be909");
+    }
+
+    TEST(hashString, hashingWithUnknownAlgoExits) {
+        auto s = "unknown";
+        ASSERT_DEATH(hashString(HashType::htUnknown, s), "");
+    }
+}

From c284700867ce09abee1f891a402271a42d207b4d Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Mon, 25 May 2020 11:57:45 +0200
Subject: [PATCH 2128/2196] Add unit tests for "json.hh"

---
 src/libutil/tests/json.cc | 193 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 193 insertions(+)
 create mode 100644 src/libutil/tests/json.cc

diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc
new file mode 100644
index 00000000000..dea73f53a91
--- /dev/null
+++ b/src/libutil/tests/json.cc
@@ -0,0 +1,193 @@
+#include "json.hh"
+#include <gtest/gtest.h>
+#include <sstream>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * toJSON
+     * --------------------------------------------------------------------------*/
+
+    TEST(toJSON, quotesCharPtr) {
+        const char* input = "test";
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "\"test\"");
+    }
+
+    TEST(toJSON, quotesStdString) {
+        std::string input = "test";
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "\"test\"");
+    }
+
+    TEST(toJSON, convertsNullptrtoNull) {
+        auto input = nullptr;
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "null");
+    }
+
+    TEST(toJSON, convertsNullToNull) {
+        const char* input = 0;
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "null");
+    }
+
+
+    TEST(toJSON, convertsFloat) {
+        auto input = 1.024f;
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "1.024");
+    }
+
+    TEST(toJSON, convertsDouble) {
+        const double input = 1.024;
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "1.024");
+    }
+
+    TEST(toJSON, convertsBool) {
+        auto input = false;
+        std::stringstream out;
+        toJSON(out, input);
+
+        ASSERT_EQ(out.str(), "false");
+    }
+
+    TEST(toJSON, quotesTab) {
+        std::stringstream out;
+        toJSON(out, "\t");
+
+        ASSERT_EQ(out.str(), "\"\\t\"");
+    }
+
+    TEST(toJSON, quotesNewline) {
+        std::stringstream out;
+        toJSON(out, "\n");
+
+        ASSERT_EQ(out.str(), "\"\\n\"");
+    }
+
+    TEST(toJSON, quotesCreturn) {
+        std::stringstream out;
+        toJSON(out, "\r");
+
+        ASSERT_EQ(out.str(), "\"\\r\"");
+    }
+
+    TEST(toJSON, quotesCreturnNewLine) {
+        std::stringstream out;
+        toJSON(out, "\r\n");
+
+        ASSERT_EQ(out.str(), "\"\\r\\n\"");
+    }
+
+    TEST(toJSON, quotesDoublequotes) {
+        std::stringstream out;
+        toJSON(out, "\"");
+
+        ASSERT_EQ(out.str(), "\"\\\"\"");
+    }
+
+    TEST(toJSON, substringEscape) {
+        std::stringstream out;
+        const char *s = "foo\t";
+        toJSON(out, s+3, s + strlen(s));
+
+        ASSERT_EQ(out.str(), "\"\\t\"");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * JSONObject
+     * --------------------------------------------------------------------------*/
+
+    TEST(JSONObject, emptyObject) {
+        std::stringstream out;
+        {
+            JSONObject t(out);
+        }
+        ASSERT_EQ(out.str(), "{}");
+    }
+
+    TEST(JSONObject, objectWithList) {
+        std::stringstream out;
+        {
+            JSONObject t(out);
+            auto l = t.list("list");
+            l.elem("element");
+        }
+        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
+    }
+
+    TEST(JSONObject, objectWithListIndent) {
+        std::stringstream out;
+        {
+            JSONObject t(out, true);
+            auto l = t.list("list");
+            l.elem("element");
+        }
+        ASSERT_EQ(out.str(),
+R"#({
+  "list": [
+    "element"
+  ]
+})#");
+    }
+
+    TEST(JSONObject, objectWithPlaceholderAndList) {
+        std::stringstream out;
+        {
+            JSONObject t(out);
+            auto l = t.placeholder("list");
+            l.list().elem("element");
+        }
+
+        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
+    }
+
+    TEST(JSONObject, objectWithPlaceholderAndObject) {
+        std::stringstream out;
+        {
+            JSONObject t(out);
+            auto l = t.placeholder("object");
+            l.object().attr("key", "value");
+        }
+
+        ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * JSONList
+     * --------------------------------------------------------------------------*/
+
+    TEST(JSONList, empty) {
+        std::stringstream out;
+        {
+            JSONList l(out);
+        }
+        ASSERT_EQ(out.str(), R"#([])#");
+    }
+
+    TEST(JSONList, withElements) {
+        std::stringstream out;
+        {
+            JSONList l(out);
+            l.elem("one");
+            l.object();
+            l.placeholder().write("three");
+        }
+        ASSERT_EQ(out.str(), R"#(["one",{},"three"])#");
+    }
+}
+

From 90b0c630a0e9e66f69f1d24b538982c20b5486b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Wed, 26 Feb 2020 10:50:40 +0100
Subject: [PATCH 2129/2196] install-multi-user: allow overriding user count

---
 scripts/install-multi-user.sh       | 4 +++-
 scripts/install-nix-from-closure.sh | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a0f1deb987f..991cf19983c 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -20,7 +20,9 @@ readonly GREEN='\033[32m'
 readonly GREEN_UL='\033[4;32m'
 readonly RED='\033[31m'
 
-readonly NIX_USER_COUNT="32"
+# installer allows overriding build user count to speed up installation
+# as creating each user takes non-trivial amount of time on macos
+readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="30000"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 readonly NIX_FIRST_BUILD_UID="30001"
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 72aa5abf5c2..6ea23a44ad6 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -50,6 +50,9 @@ while [ $# -gt 0 ]; do
             INSTALL_MODE=no-daemon;;
         --no-channel-add)
             NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+        --daemon-user-count)
+            NIX_USER_COUNT=$2
+            shift;;
         --no-modify-profile)
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
         --darwin-use-unencrypted-nix-store-volume)

From 573ff8dfcaacc4495e7f7880d86f70145a074578 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Mon, 25 May 2020 17:31:46 +0200
Subject: [PATCH 2130/2196] Allow passing extra nix.conf to installer

---
 scripts/install-multi-user.sh       | 1 +
 scripts/install-nix-from-closure.sh | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a0f1deb987f..b0cb5194394 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -646,6 +646,7 @@ setup_default_profile() {
 
 place_nix_configuration() {
     cat <<EOF > "$SCRATCH/nix.conf"
+$NIX_EXTRA_CONF
 build-users-group = $NIX_BUILD_GROUP_NAME
 EOF
     _sudo "to place the default nix daemon configuration (part 2)" \
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 72aa5abf5c2..3e0312c7803 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -41,6 +41,7 @@ fi
 
 INSTALL_MODE=no-daemon
 CREATE_DARWIN_VOLUME=0
+NIX_EXTRA_CONF=
 # handle the command line flags
 while [ $# -gt 0 ]; do
     case $1 in
@@ -54,6 +55,9 @@ while [ $# -gt 0 ]; do
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
         --darwin-use-unencrypted-nix-store-volume)
             CREATE_DARWIN_VOLUME=1;;
+        --nix-extra-conf-file)
+            NIX_EXTRA_CONF=$(cat $2)
+            shift;;
         *)
             (
                 echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]"

From 4b388e84318722f970326967e1648675655740fe Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Mon, 25 May 2020 18:34:55 +0200
Subject: [PATCH 2131/2196] Add unit tests for xml-writer

---
 src/libutil/tests/xml-writer.cc | 105 ++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 src/libutil/tests/xml-writer.cc

diff --git a/src/libutil/tests/xml-writer.cc b/src/libutil/tests/xml-writer.cc
new file mode 100644
index 00000000000..adcde25c9f1
--- /dev/null
+++ b/src/libutil/tests/xml-writer.cc
@@ -0,0 +1,105 @@
+#include "xml-writer.hh"
+#include <gtest/gtest.h>
+#include <sstream>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * XMLWriter
+     * --------------------------------------------------------------------------*/
+
+    TEST(XMLWriter, emptyObject) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n");
+    }
+
+    TEST(XMLWriter, objectWithEmptyElement) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            t.openElement("foobar");
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>");
+    }
+
+    TEST(XMLWriter, objectWithElementWithAttrs) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            XMLAttrs attrs = {
+                { "foo", "bar" }
+            };
+            t.openElement("foobar", attrs);
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\"></foobar>");
+    }
+
+    TEST(XMLWriter, objectWithElementWithEmptyAttrs) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            XMLAttrs attrs = {};
+            t.openElement("foobar", attrs);
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>");
+    }
+
+    TEST(XMLWriter, objectWithElementWithAttrsEscaping) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            XMLAttrs attrs = {
+                { "<key>", "<value>" }
+            };
+            t.openElement("foobar", attrs);
+        }
+
+        // XXX: While "<value>" is escaped, "<key>" isn't which I think is a bug.
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar <key>=\"&lt;value&gt;\"></foobar>");
+    }
+
+    TEST(XMLWriter, objectWithElementWithAttrsIndented) {
+        std::stringstream out;
+        {
+            XMLWriter t(true, out);
+            XMLAttrs attrs = {
+                { "foo", "bar" }
+            };
+            t.openElement("foobar", attrs);
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\">\n</foobar>\n");
+    }
+
+    TEST(XMLWriter, writeEmptyElement) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            t.writeEmptyElement("foobar");
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar />");
+    }
+
+    TEST(XMLWriter, writeEmptyElementWithAttributes) {
+        std::stringstream out;
+        {
+            XMLWriter t(false, out);
+            XMLAttrs attrs = {
+                { "foo", "bar" }
+            };
+            t.writeEmptyElement("foobar", attrs);
+
+        }
+
+        ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\" />");
+    }
+
+}

From 1a5ac894e929cefceec22c0f1ca248ee6be445ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 26 May 2020 15:49:26 +0200
Subject: [PATCH 2132/2196] Fix installer script bugs

- --no-channel-add didn't have effect on multi-user installation
- some new flags didn't work at all
- document all installer flags
---
 scripts/install-multi-user.sh       | 24 ++++++++++++++----------
 scripts/install-nix-from-closure.sh | 13 ++++++++-----
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 74cc6a5b082..ab41e024215 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -29,6 +29,7 @@ readonly NIX_FIRST_BUILD_UID="30001"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
+readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
 readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
@@ -452,9 +453,11 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
-    _sudo "to set up the default system channel (part 1)" \
-          install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+        echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
+        _sudo "to set up the default system channel (part 1)" \
+            install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
+    fi
 }
 
 welcome_to_nix() {
@@ -636,13 +639,14 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
-    # otherwise it will be lost in environments where sudo doesn't pass
-    # all the environment variables by default.
-    _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
-          || channel_update_failed=1
-
+    if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+        # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
+        # otherwise it will be lost in environments where sudo doesn't pass
+        # all the environment variables by default.
+        _sudo "to update the default channel in the default profile" \
+            HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+            || channel_update_failed=1
+    fi
 }
 
 
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 635aaa16d23..826ca8b8c1a 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -41,7 +41,6 @@ fi
 
 INSTALL_MODE=no-daemon
 CREATE_DARWIN_VOLUME=0
-NIX_EXTRA_CONF=
 # handle the command line flags
 while [ $# -gt 0 ]; do
     case $1 in
@@ -50,20 +49,20 @@ while [ $# -gt 0 ]; do
         --no-daemon)
             INSTALL_MODE=no-daemon;;
         --no-channel-add)
-            NIX_INSTALLER_NO_CHANNEL_ADD=1;;
+            export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
         --daemon-user-count)
-            NIX_USER_COUNT=$2
+            export NIX_USER_COUNT=$2
             shift;;
         --no-modify-profile)
             NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
         --darwin-use-unencrypted-nix-store-volume)
             CREATE_DARWIN_VOLUME=1;;
         --nix-extra-conf-file)
-            NIX_EXTRA_CONF=$(cat $2)
+            export NIX_EXTRA_CONF="$(cat $2)"
             shift;;
         *)
             (
-                echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]"
+                echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
 
                 echo "Choose installation method."
                 echo ""
@@ -82,6 +81,10 @@ while [ $# -gt 0 ]; do
                 echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
                 echo "                      is installed by default."
                 echo ""
+                echo " --daemon-user-count: Number of build users to create. Defaults to 32."
+                echo ""
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo ""
             ) >&2
 
             # darwin and Catalina+

From 3d3c219d917525b0a131c4332dd65eadfc818f49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domen=20Ko=C5=BEar?= <domen@dev.si>
Date: Tue, 26 May 2020 16:23:03 +0200
Subject: [PATCH 2133/2196] installer: fix unused variable

---
 scripts/install-multi-user.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index ab41e024215..d04b4bbbf68 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -453,7 +453,7 @@ create_directories() {
 }
 
 place_channel_configuration() {
-    if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
         echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
         _sudo "to set up the default system channel (part 1)" \
             install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
@@ -639,7 +639,7 @@ setup_default_profile() {
         export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
     fi
 
-    if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
+    if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
         # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
         # otherwise it will be lost in environments where sudo doesn't pass
         # all the environment variables by default.

From b90241ceb13500238e428f8943d221f24bf5322b Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Tue, 26 May 2020 11:32:41 -0400
Subject: [PATCH 2134/2196] Change remaining bools with FileIngestionMethod

---
 src/libfetchers/git.cc       | 4 ++--
 src/libfetchers/mercurial.cc | 2 +-
 src/libfetchers/tarball.cc   | 6 +++---
 src/libfetchers/tree-info.cc | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7c18cf67fd0..17cc602285c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -195,7 +195,7 @@ struct GitInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 auto tree = Tree {
                     .actualPath = store->printStorePath(storePath),
@@ -347,7 +347,7 @@ struct GitInput : Input
             unpackTarfile(*source, tmpDir);
         }
 
-        auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
+        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
         auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
 
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 1d65715714e..2e0d4bf4d83 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -114,7 +114,7 @@ struct MercurialInput : Input
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
+                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {Tree {
                     .actualPath = store->printStorePath(storePath),
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 695525b3108..bf2b2a5ff59 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -67,10 +67,10 @@ DownloadFileResult downloadFile(
         StringSink sink;
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
         info.narHash = hashString(htSHA256, *sink.s);
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(false, hash);
+        info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
         store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
@@ -141,7 +141,7 @@ Tree downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc
index 5788e94a197..b2d8cfc8d03 100644
--- a/src/libfetchers/tree-info.cc
+++ b/src/libfetchers/tree-info.cc
@@ -8,7 +8,7 @@ namespace nix::fetchers {
 StorePath TreeInfo::computeStorePath(Store & store) const
 {
     assert(narHash);
-    return store.makeFixedOutputPath(true, narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
 }
 
 }

From 93129cf1ddf7f91ddec11ba7a6a364ab54f345c5 Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Tue, 26 May 2020 17:49:31 +0200
Subject: [PATCH 2135/2196] Add unit tests for config.cc

---
 src/libutil/tests/config.cc | 204 ++++++++++++++++++++++++++++++++++++
 1 file changed, 204 insertions(+)
 create mode 100644 src/libutil/tests/config.cc

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
new file mode 100644
index 00000000000..b0be24d5e75
--- /dev/null
+++ b/src/libutil/tests/config.cc
@@ -0,0 +1,204 @@
+#include "json.hh"
+#include "config.hh"
+#include "args.hh"
+
+#include <sstream>
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * Config
+     * --------------------------------------------------------------------------*/
+
+    TEST(Config, setUndefinedSetting) {
+        Config config;
+        ASSERT_EQ(config.set("undefined-key", "value"), false);
+    }
+
+    TEST(Config, setDefinedSetting) {
+        Config config;
+        std::string value;
+        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
+        ASSERT_EQ(config.set("name-of-the-setting", "value"), true);
+    }
+
+    TEST(Config, getDefinedSetting) {
+        Config config;
+        std::string value;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
+
+        config.getSettings(settings, /* overridenOnly = */ false);
+        const auto iter = settings.find("name-of-the-setting");
+        ASSERT_NE(iter, settings.end());
+        ASSERT_EQ(iter->second.value, "");
+        ASSERT_EQ(iter->second.description, "description");
+    }
+
+    TEST(Config, getDefinedOverridenSettingNotSet) {
+        Config config;
+        std::string value;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
+
+        config.getSettings(settings, /* overridenOnly = */ true);
+        const auto e = settings.find("name-of-the-setting");
+        ASSERT_EQ(e, settings.end());
+    }
+
+    TEST(Config, getDefinedSettingSet1) {
+        Config config;
+        std::string value;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, value, "name-of-the-setting", "description"};
+
+        setting.assign("value");
+
+        config.getSettings(settings, /* overridenOnly = */ false);
+        const auto iter = settings.find("name-of-the-setting");
+        ASSERT_NE(iter, settings.end());
+        ASSERT_EQ(iter->second.value, "value");
+        ASSERT_EQ(iter->second.description, "description");
+    }
+
+    TEST(Config, getDefinedSettingSet2) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+
+        ASSERT_TRUE(config.set("name-of-the-setting", "value"));
+
+        config.getSettings(settings, /* overridenOnly = */ false);
+        const auto e = settings.find("name-of-the-setting");
+        ASSERT_NE(e, settings.end());
+        ASSERT_EQ(e->second.value, "value");
+        ASSERT_EQ(e->second.description, "description");
+    }
+
+    TEST(Config, addSetting) {
+        class TestSetting : public AbstractSetting {
+            public:
+            TestSetting() : AbstractSetting("test", "test", {}) {}
+            void set(const std::string & value) {}
+            std::string to_string() const { return {}; }
+        };
+
+        Config config;
+        TestSetting setting;
+
+        ASSERT_FALSE(config.set("test", "value"));
+        config.addSetting(&setting);
+        ASSERT_TRUE(config.set("test", "value"));
+    }
+
+    TEST(Config, withInitialValue) {
+        const StringMap initials = {
+            { "key", "value" },
+        };
+        Config config(initials);
+
+        {
+            std::map<std::string, Config::SettingInfo> settings;
+            config.getSettings(settings, /* overridenOnly = */ false);
+            ASSERT_EQ(settings.find("key"), settings.end());
+        }
+
+        Setting<std::string> setting{&config, "default-value", "key", "description"};
+
+        {
+            std::map<std::string, Config::SettingInfo> settings;
+            config.getSettings(settings, /* overridenOnly = */ false);
+            ASSERT_EQ(settings["key"].value, "value");
+        }
+    }
+
+    TEST(Config, resetOverriden) {
+        Config config;
+        config.resetOverriden();
+    }
+
+    TEST(Config, resetOverridenWithSetting) {
+        Config config;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+
+        {
+            std::map<std::string, Config::SettingInfo> settings;
+
+            setting.set("foo");
+            ASSERT_EQ(setting.get(), "foo");
+            config.getSettings(settings, /* overridenOnly = */ true);
+            ASSERT_TRUE(settings.empty());
+        }
+
+        {
+            std::map<std::string, Config::SettingInfo> settings;
+
+            setting.override("bar");
+            ASSERT_TRUE(setting.overriden);
+            ASSERT_EQ(setting.get(), "bar");
+            config.getSettings(settings, /* overridenOnly = */ true);
+            ASSERT_FALSE(settings.empty());
+        }
+
+        {
+            std::map<std::string, Config::SettingInfo> settings;
+
+            config.resetOverriden();
+            ASSERT_FALSE(setting.overriden);
+            config.getSettings(settings, /* overridenOnly = */ true);
+            ASSERT_TRUE(settings.empty());
+        }
+    }
+
+    TEST(Config, toJSONOnEmptyConfig) {
+        std::stringstream out;
+        { // Scoped to force the destructor of JSONObject to write the final `}`
+            JSONObject obj(out);
+            Config config;
+            config.toJSON(obj);
+        }
+
+        ASSERT_EQ(out.str(), "{}");
+    }
+
+    TEST(Config, toJSONOnNonEmptyConfig) {
+        std::stringstream out;
+        { // Scoped to force the destructor of JSONObject to write the final `}`
+            JSONObject obj(out);
+
+            Config config;
+            std::map<std::string, Config::SettingInfo> settings;
+            Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+            setting.assign("value");
+
+            config.toJSON(obj);
+        }
+        ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#");
+    }
+
+    TEST(Config, setSettingAlias) {
+        Config config;
+        Setting<std::string> setting{&config, "", "some-int", "best number", { "another-int" }};
+        ASSERT_TRUE(config.set("some-int", "1"));
+        ASSERT_EQ(setting.get(), "1");
+        ASSERT_TRUE(config.set("another-int", "2"));
+        ASSERT_EQ(setting.get(), "2");
+        ASSERT_TRUE(config.set("some-int", "3"));
+        ASSERT_EQ(setting.get(), "3");
+    }
+
+    /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything
+     * useful (these days).  Whenever we add a new setting to Config the
+     * unknown settings are always considered.  In which case is this function
+     * actually useful? Is there some way to register a Setting without calling
+     * addSetting? */
+    TEST(Config, DISABLED_reapplyUnknownSettings) {
+        Config config;
+        ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue"));
+        Setting<std::string> setting{&config, "default", "name-of-the-setting", "description"};
+        ASSERT_EQ(setting.get(), "default");
+        config.reapplyUnknownSettings();
+        ASSERT_EQ(setting.get(), "unknownvalue");
+    }
+}

From e1b8c64c04b40694418de87e3e0d118cf0677bfc Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Wed, 27 May 2020 14:39:02 +0200
Subject: [PATCH 2136/2196] config.cc: extract parts of applyConfigFile into
 applyConfig

This moves the actual parsing of configuration contents into applyConfig
which applyConfigFile is then going to call. By changing this we can now
test the configuration file parsing without actually create a file on
disk.
---
 src/libutil/config.cc | 95 ++++++++++++++++++++++---------------------
 src/libutil/config.hh |  1 +
 2 files changed, 50 insertions(+), 46 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index f03e444ecc3..8fc700a2bfa 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -65,60 +65,63 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool override
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void AbstractConfig::applyConfigFile(const Path & path)
-{
-    try {
-        string contents = readFile(path);
-
-        unsigned int pos = 0;
-
-        while (pos < contents.size()) {
-            string line;
-            while (pos < contents.size() && contents[pos] != '\n')
-                line += contents[pos++];
-            pos++;
-
-            string::size_type hash = line.find('#');
-            if (hash != string::npos)
-                line = string(line, 0, hash);
-
-            vector<string> tokens = tokenizeString<vector<string> >(line);
-            if (tokens.empty()) continue;
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    unsigned int pos = 0;
+
+    while (pos < contents.size()) {
+        string line;
+        while (pos < contents.size() && contents[pos] != '\n')
+            line += contents[pos++];
+        pos++;
+
+        string::size_type hash = line.find('#');
+        if (hash != string::npos)
+            line = string(line, 0, hash);
+
+        vector<string> tokens = tokenizeString<vector<string> >(line);
+        if (tokens.empty()) continue;
+
+        if (tokens.size() < 2)
+            throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+
+        auto include = false;
+        auto ignoreMissing = false;
+        if (tokens[0] == "include")
+            include = true;
+        else if (tokens[0] == "!include") {
+            include = true;
+            ignoreMissing = true;
+        }
 
-            if (tokens.size() < 2)
+        if (include) {
+            if (tokens.size() != 2)
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
-
-            auto include = false;
-            auto ignoreMissing = false;
-            if (tokens[0] == "include")
-                include = true;
-            else if (tokens[0] == "!include") {
-                include = true;
-                ignoreMissing = true;
+            auto p = absPath(tokens[1], dirOf(path));
+            if (pathExists(p)) {
+                applyConfigFile(p);
+            } else if (!ignoreMissing) {
+                throw Error("file '%1%' included from '%2%' not found", p, path);
             }
+            continue;
+        }
 
-            if (include) {
-                if (tokens.size() != 2)
-                    throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
-                auto p = absPath(tokens[1], dirOf(path));
-                if (pathExists(p)) {
-                    applyConfigFile(p);
-                } else if (!ignoreMissing) {
-                    throw Error("file '%1%' included from '%2%' not found", p, path);
-                }
-                continue;
-            }
+        if (tokens[1] != "=")
+            throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
 
-            if (tokens[1] != "=")
-                throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
+        string name = tokens[0];
 
-            string name = tokens[0];
+        vector<string>::iterator i = tokens.begin();
+        advance(i, 2);
 
-            vector<string>::iterator i = tokens.begin();
-            advance(i, 2);
+        set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
+    };
+}
 
-            set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
-        };
+void AbstractConfig::applyConfigFile(const Path & path)
+{
+    try {
+        string contents = readFile(path);
+        applyConfig(contents, path);
     } catch (SysError &) { }
 }
 
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 7ea78fdaf5c..b04cba88b38 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -33,6 +33,7 @@ public:
 
     virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
 
+    void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
     void applyConfigFile(const Path & path);
 
     virtual void resetOverriden() = 0;

From 9df3d8ccd7a3130faccb73933d0c196a81d605eb Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Wed, 27 May 2020 14:41:01 +0200
Subject: [PATCH 2137/2196] tests/config.cc: add tests for Config::applyConfig

---
 src/libutil/tests/config.cc | 60 +++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index b0be24d5e75..74c59fd315b 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -201,4 +201,64 @@ namespace nix {
         config.reapplyUnknownSettings();
         ASSERT_EQ(setting.get(), "unknownvalue");
     }
+
+    TEST(Config, applyConfigEmpty) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        config.applyConfig("");
+        config.getSettings(settings);
+        ASSERT_TRUE(settings.empty());
+    }
+
+    TEST(Config, applyConfigEmptyWithComment) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        config.applyConfig("# just a comment");
+        config.getSettings(settings);
+        ASSERT_TRUE(settings.empty());
+    }
+
+    TEST(Config, applyConfigAssignment) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+        config.applyConfig(
+            "name-of-the-setting = value-from-file #useful comment\n"
+            "# name-of-the-setting = foo\n"
+        );
+        config.getSettings(settings);
+        ASSERT_FALSE(settings.empty());
+        ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file");
+    }
+
+    TEST(Config, applyConfigWithReassignedSetting) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+        config.applyConfig(
+            "name-of-the-setting = first-value\n"
+            "name-of-the-setting = second-value\n"
+        );
+        config.getSettings(settings);
+        ASSERT_FALSE(settings.empty());
+        ASSERT_EQ(settings["name-of-the-setting"].value, "second-value");
+    }
+
+    TEST(Config, applyConfigFailsOnMissingIncludes) {
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+
+        ASSERT_THROW(config.applyConfig(
+            "name-of-the-setting = value-from-file\n"
+            "# name-of-the-setting = foo\n"
+            "include /nix/store/does/not/exist.nix"
+        ), Error);
+    }
+
+    TEST(Config, applyConfigInvalidThrows) {
+        Config config;
+        ASSERT_THROW(config.applyConfig("value == key"), UsageError);
+        ASSERT_THROW(config.applyConfig("value "), UsageError);
+    }
 }

From fc137d2f007c32f27a49e1a00d5114c7d7663419 Mon Sep 17 00:00:00 2001
From: Andreas Rammhold <andreas@rammhold.de>
Date: Wed, 27 May 2020 16:37:24 +0200
Subject: [PATCH 2138/2196] config.hh: Add documentation

Provides some general overview on the mechanics of Config/Setting and
comments for the public methods of Config.
---
 src/libutil/config.hh | 69 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index b04cba88b38..5c7a70a2edf 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -7,6 +7,38 @@
 
 namespace nix {
 
+/**
+ * The Config class provides Nix runtime configurations.
+ *
+ * What is a Configuration?
+ *   A collection of uniquely named Settings.
+ *
+ * What is a Setting?
+ *   Each property that you can set in a configuration corresponds to a
+ *   `Setting`. A setting records value and description of a property
+ *   with a default and optional aliases.
+ *
+ * A valid configuration consists of settings that are registered to a
+ * `Config` object instance:
+ *
+ *   Config config;
+ *   Setting<std::string> systemSetting{&config, "x86_64-linux", "system", "the current system"};
+ *
+ * The above creates a `Config` object and registers a setting called "system"
+ * via the variable `systemSetting` with it. The setting defaults to the string
+ * "x86_64-linux", it's description is "the current system". All of the
+ * registered settings can then be accessed as shown below:
+ *
+ *   std::map<std::string, Config::SettingInfo> settings;
+ *   config.getSettings(settings);
+ *   config["system"].description == "the current system"
+ *   config["system"].value == "x86_64-linux"
+ *
+ *
+ * The above retrieves all currently known settings from the `Config` object
+ * and adds them to the `settings` map.
+ */
+
 class Args;
 class AbstractSetting;
 class JSONPlaceholder;
@@ -23,6 +55,10 @@ protected:
 
 public:
 
+    /**
+     * Sets the value referenced by `name` to `value`. Returns true if the
+     * setting is known, false otherwise.
+     */
     virtual bool set(const std::string & name, const std::string & value) = 0;
 
     struct SettingInfo
@@ -31,19 +67,52 @@ public:
         std::string description;
     };
 
+    /**
+     * Adds the currently known settings to the given result map `res`.
+     * - res: map to store settings in
+     * - overridenOnly: when set to true only overridden settings will be added to `res`
+     */
     virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
 
+    /**
+     * Parses the configuration in `contents` and applies it
+     * - contents: configuration contents to be parsed and applied
+     * - path: location of the configuration file
+     */
     void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
+
+    /**
+     * Applies a nix configuration file
+     * - path: the location of the config file to apply
+     */
     void applyConfigFile(const Path & path);
 
+    /**
+     * Resets the `overridden` flag of all Settings
+     */
     virtual void resetOverriden() = 0;
 
+    /**
+     * Outputs all settings to JSON
+     * - out: JSONObject to write the configuration to
+     */
     virtual void toJSON(JSONObject & out) = 0;
 
+    /**
+     * Converts settings to `Args` to be used on the command line interface
+     * - args: args to write to
+     * - category: category of the settings
+     */
     virtual void convertToArgs(Args & args, const std::string & category) = 0;
 
+    /**
+     * Logs a warning for each unregistered setting
+     */
     void warnUnknownSettings();
 
+    /**
+     * Re-applies all previously attempted changes to unknown settings
+     */
     void reapplyUnknownSettings();
 };
 

From 7873fd175d9e4e5bdb281e005b18c388bcfe1dd8 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 27 May 2020 13:11:45 -0500
Subject: [PATCH 2139/2196] =?UTF-8?q?Don=E2=80=99t=20use=20FileIngestionMe?=
 =?UTF-8?q?thod=20for=20StorePathsCommand?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is a different recursive than used in makeFixedOutputPath.
---
 src/nix/command.cc | 10 +++++-----
 src/nix/command.hh |  4 ++--
 src/nix/copy.cc    |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/nix/command.cc b/src/nix/command.cc
index ea0ade88e04..71b02771918 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -31,21 +31,21 @@ void StoreCommand::run()
     run(getStore());
 }
 
-StorePathsCommand::StorePathsCommand(FileIngestionMethod recursive)
+StorePathsCommand::StorePathsCommand(bool recursive)
     : recursive(recursive)
 {
-    if (recursive == FileIngestionMethod::Recursive)
+    if (recursive)
         addFlag({
             .longName = "no-recursive",
             .description = "apply operation to specified paths only",
-            .handler = {&this->recursive, FileIngestionMethod::Flat},
+            .handler = {&this->recursive, false},
         });
     else
         addFlag({
             .longName = "recursive",
             .shortName = 'r',
             .description = "apply operation to closure of the specified paths",
-            .handler = {&this->recursive, FileIngestionMethod::Recursive},
+            .handler = {&this->recursive, true},
         });
 
     mkFlag(0, "all", "apply operation to the entire store", &all);
@@ -66,7 +66,7 @@ void StorePathsCommand::run(ref<Store> store)
         for (auto & p : toStorePaths(store, realiseMode, installables))
             storePaths.push_back(p.clone());
 
-        if (recursive == FileIngestionMethod::Recursive) {
+        if (recursive) {
             StorePathSet closure;
             store->computeFSClosure(storePathsToSet(storePaths), closure, false, false);
             storePaths.clear();
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 09c621b5b10..959d5f19dbc 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -92,7 +92,7 @@ struct StorePathsCommand : public InstallablesCommand
 {
 private:
 
-    FileIngestionMethod recursive = FileIngestionMethod::Flat;
+    bool recursive = false;
     bool all = false;
 
 protected:
@@ -101,7 +101,7 @@ protected:
 
 public:
 
-    StorePathsCommand(FileIngestionMethod recursive = FileIngestionMethod::Flat);
+    StorePathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 60aa3f14bd3..c7c38709d19 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -17,7 +17,7 @@ struct CmdCopy : StorePathsCommand
     SubstituteFlag substitute = NoSubstitute;
 
     CmdCopy()
-        : StorePathsCommand(FileIngestionMethod::Recursive)
+        : StorePathsCommand(true)
     {
         addFlag({
             .longName = "from",

From c66441a646c2bbfab05fa4a4fd05c2ae897dda8b Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 27 May 2020 13:04:20 -0500
Subject: [PATCH 2140/2196] =?UTF-8?q?Rename=20some=20variables=20named=20?=
 =?UTF-8?q?=E2=80=9Crecursive=E2=80=9D=20to=20=E2=80=9Cmethod=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is much less confusing since recursive is no longer a boolean.
---
 src/libexpr/primops.cc             | 14 +++++++-------
 src/libstore/binary-cache-store.cc |  6 +++---
 src/libstore/binary-cache-store.hh |  2 +-
 src/libstore/build.cc              | 18 +++++++++---------
 src/libstore/local-store.cc        | 22 +++++++++++-----------
 src/libstore/store-api.cc          |  6 +++---
 6 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index fcbb75ceb08..d458ab2721f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1042,7 +1042,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
 
 static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v)
+    Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
 {
     const auto path = evalSettings.pureEval && expectedHash ?
         path_ :
@@ -1073,12 +1073,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
     std::optional<StorePath> expectedStorePath;
     if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
+        expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
     Path dstPath;
     if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
         dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
         if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
             throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
@@ -1108,7 +1108,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Path path;
     string name;
     Value * filterFun = nullptr;
-    auto recursive = FileIngestionMethod::Recursive;
+    auto method = FileIngestionMethod::Recursive;
     Hash expectedHash;
 
     for (auto & attr : *args[0]->attrs) {
@@ -1124,7 +1124,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             state.forceValue(*attr.value, pos);
             filterFun = attr.value;
         } else if (n == "recursive")
-            recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
+            method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
         else if (n == "sha256")
             expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
@@ -1135,7 +1135,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
 }
 
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index a61af4a001c..f13736c58c0 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
 
@@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (recursive == FileIngestionMethod::Recursive) {
+    if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
     } else {
@@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
+    ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
     addToStore(info, sink.s, repair, CheckSigs, nullptr);
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 1a1ea636c7c..4ff5609ed88 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -79,7 +79,7 @@ public:
         std::shared_ptr<FSAccessor> accessor) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod recursive, HashType hashAlgo,
+        FileIngestionMethod method, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 272c4f1ae10..f5c132a8397 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
     { throw Error("addToStore"); }
 
@@ -2747,9 +2747,9 @@ struct RestrictedStore : public LocalFSStore
     }
 
     StorePath addToStoreFromDump(const string & dump, const string & name,
-        FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
     {
-        auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
         goal.addDependency(path);
         return path;
     }
@@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs()
 
         if (fixedOutput) {
 
-            FileIngestionMethod recursive; Hash h;
-            i.second.parseHashInfo(recursive, h);
+            FileIngestionMethod outputHashMode; Hash h;
+            i.second.parseHashInfo(outputHashMode, h);
 
-            if (recursive == FileIngestionMethod::Flat) {
+            if (outputHashMode == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
@@ -3705,11 +3705,11 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
-            Hash h2 = recursive == FileIngestionMethod::Recursive
+            Hash h2 = outputHashMode == FileIngestionMethod::Recursive
                 ? hashPath(h.type, actualPath).first
                 : hashFile(h.type, actualPath);
 
-            auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
+            auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
 
             if (h != h2) {
 
@@ -3738,7 +3738,7 @@ void DerivationGoal::registerOutputs()
             else
                 assert(worker.store.parseStorePath(path) == dest);
 
-            ca = makeFixedOutputCA(recursive, h2);
+            ca = makeFixedOutputCA(outputHashMode, h2);
         }
 
         /* Get rid of all weird permissions.  This also checks that
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 746f81beb78..80851b59106 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        FileIngestionMethod recursive; Hash h;
-        out->second.parseHashInfo(recursive, h);
+        FileIngestionMethod method; Hash h;
+        out->second.parseHashInfo(method, h);
 
-        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
+        check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
     }
 
     else {
@@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
-    FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {
     Hash h = hashString(hashAlgo, dump);
 
-    auto dstPath = makeFixedOutputPath(recursive, h, name);
+    auto dstPath = makeFixedOutputPath(method, h, name);
 
     addTempRoot(dstPath);
 
@@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (recursive == FileIngestionMethod::Recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
             } else
@@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
                above (if called with recursive == true and hashAlgo ==
                sha256); otherwise, compute it here. */
             HashResult hash;
-            if (recursive == FileIngestionMethod::Recursive) {
+            if (method == FileIngestionMethod::Recursive) {
                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
                 hash.second = dump.size();
             } else
@@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
             ValidPathInfo info(dstPath.clone());
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ca = makeFixedOutputCA(recursive, h);
+            info.ca = makeFixedOutputCA(method, h);
             registerValidPath(info);
         }
 
@@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
 
 StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     Path srcPath(absPath(_srcPath));
 
@@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
        method for very large paths, but `copyPath' is mainly used for
        small files. */
     StringSink sink;
-    if (recursive == FileIngestionMethod::Recursive)
+    if (method == FileIngestionMethod::Recursive)
         dumpPath(srcPath, sink, filter);
     else
         sink.s = make_ref<std::string>(readFile(srcPath));
 
-    return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair);
+    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
 }
 
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 86cb20a26b8..d479b86cb11 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -204,12 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
 
 
 std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
-    const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const
+    const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
 {
-    Hash h = recursive == FileIngestionMethod::Recursive
+    Hash h = method == FileIngestionMethod::Recursive
         ? hashPath(hashAlgo, srcPath, filter).first
         : hashFile(hashAlgo, srcPath);
-    return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
+    return std::make_pair(makeFixedOutputPath(method, h, name), h);
 }
 
 

From 4e6d7cb55a884ac19990b487c2020ea257e0f9c3 Mon Sep 17 00:00:00 2001
From: Daiderd Jordan <daiderd@gmail.com>
Date: Wed, 27 May 2020 20:58:42 +0200
Subject: [PATCH 2141/2196] installer: don't require xz on darwin

On macOS the system tar has builtin support for lzma while xz isn't
available as a separate binary.  There's no builtin package manager
there available either so having to install lzma (without nix) would be
rather painful.
---
 scripts/install.in | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/install.in b/scripts/install.in
index 6709f00d4a8..1d26c4ff0eb 100644
--- a/scripts/install.in
+++ b/scripts/install.in
@@ -36,7 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
-require_util xz "unpack the binary tarball"
+if [ "$(uname -s)" != "Darwin" ]; then
+    require_util xz "unpack the binary tarball"
+fi
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

From 0f96f45061e574c9cf217e3dd5634dfdbf210235 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 27 May 2020 23:50:11 -0400
Subject: [PATCH 2142/2196] Use `FileIngestionMethod` for `nix hash`

There was an enum there that matched in perfectly.
---
 src/nix/hash.cc | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 366314227c8..0a7c343d0dd 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -9,15 +9,14 @@ using namespace nix;
 
 struct CmdHash : Command
 {
-    enum Mode { mFile, mPath };
-    Mode mode;
+    FileIngestionMethod mode;
     Base base = SRI;
     bool truncate = false;
     HashType ht = htSHA256;
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
-    CmdHash(Mode mode) : mode(mode)
+    CmdHash(FileIngestionMethod mode) : mode(mode)
     {
         mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base64);
@@ -36,9 +35,14 @@ struct CmdHash : Command
 
     std::string description() override
     {
-        return mode == mFile
-            ? "print cryptographic hash of a regular file"
-            : "print cryptographic hash of the NAR serialisation of a path";
+        const char* d;
+        switch (mode) {
+        case FileIngestionMethod::Flat:
+            d = "print cryptographic hash of a regular file";
+        case FileIngestionMethod::Recursive:
+            d = "print cryptographic hash of the NAR serialisation of a path";
+        };
+        return d;
     }
 
     Category category() override { return catUtility; }
@@ -53,10 +57,14 @@ struct CmdHash : Command
             else
                 hashSink = std::make_unique<HashSink>(ht);
 
-            if (mode == mFile)
+            switch (mode) {
+            case FileIngestionMethod::Flat:
                 readFile(path, *hashSink);
-            else
+                break;
+            case FileIngestionMethod::Recursive:
                 dumpPath(path, *hashSink);
+                break;
+            }
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
@@ -65,8 +73,8 @@ struct CmdHash : Command
     }
 };
 
-static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); });
-static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); });
+static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
+static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
 
 struct CmdToBase : Command
 {
@@ -137,7 +145,7 @@ static int compatNixHash(int argc, char * * argv)
     });
 
     if (op == opHash) {
-        CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath);
+        CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
         cmd.ht = ht;
         cmd.base = base32 ? Base32 : Base16;
         cmd.truncate = truncate;

From fac0c2d54a6b04175b40009506f2720d2594ed4e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 29 May 2020 16:19:48 -0400
Subject: [PATCH 2143/2196] Remove addToStore variant as requested by `FIXME`

The idea is it's always more flexible to consumer a `Source` than a
plain string, and it might even reduce memory consumption.

I also looked at `addToStoreFromDump` with its `// FIXME: remove?`, but
the worked needed for that is far more up for interpretation, so I
punted for now.
---
 src/libfetchers/tarball.cc         |  3 ++-
 src/libstore/binary-cache-store.cc | 11 ++++++++---
 src/libstore/binary-cache-store.hh |  2 +-
 src/libstore/export-import.cc      |  5 ++++-
 src/libstore/store-api.cc          | 15 ---------------
 src/libstore/store-api.hh          |  7 +------
 src/nix/add-to-store.cc            |  6 ++++--
 7 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index bf2b2a5ff59..b6e57379bd0 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -71,7 +71,8 @@ DownloadFileResult downloadFile(
         info.narHash = hashString(htSHA256, *sink.s);
         info.narSize = sink.s->size();
         info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
-        store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
+        auto source = StringSource { *sink.s };
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f13736c58c0..357962007a4 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
+    // FIXME: See if we can use the original source to reduce memory usage.
+    auto nar = make_ref<std::string>(narSource.drain());
+
     if (!repair && isValidPath(info.path)) return;
 
     /* Verify that all references are valid. This may do some .narinfo
@@ -347,7 +350,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
 
     ValidPathInfo info(makeFixedOutputPath(method, h, name));
 
-    addToStore(info, sink.s, repair, CheckSigs, nullptr);
+    auto source = StringSource { *sink.s };
+    addToStore(info, source, repair, CheckSigs, nullptr);
 
     return std::move(info.path);
 }
@@ -361,7 +365,8 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, sink.s, repair, CheckSigs, nullptr);
+        auto source = StringSource { *sink.s };
+        addToStore(info, source, repair, CheckSigs, nullptr);
     }
 
     return std::move(info.path);
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 4ff5609ed88..52ef8aa7ac2 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -74,7 +74,7 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+    void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 4692d1a7be3..f0d01a24043 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -1,3 +1,4 @@
+#include "serialise.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
@@ -100,7 +101,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
+        // Can't use underlying source, which would have been exhausted
+        auto source = StringSource { *tee.source.data };
+        addToStore(info, source, NoRepair, checkSigs, accessor);
 
         res.push_back(info.path.clone());
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index d479b86cb11..095363d0c9b 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -840,21 +840,6 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 }
 
 
-void Store::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs,
-    std::shared_ptr<FSAccessor> accessor)
-{
-    addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor);
-}
-
-void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-    RepairFlag repair, CheckSigsFlag checkSigs,
-    std::shared_ptr<FSAccessor> accessor)
-{
-    StringSource source(*nar);
-    addToStore(info, source, repair, checkSigs, accessor);
-}
-
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3def209a636..b1e25fc7d66 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -450,12 +450,7 @@ public:
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0);
-
-    // FIXME: remove
-    virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
-        RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
-        std::shared_ptr<FSAccessor> accessor = 0);
+        std::shared_ptr<FSAccessor> accessor = 0) = 0;
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 4b4ba81cb22..f43f774c1c8 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -50,8 +50,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         info.narSize = sink.s->size();
         info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
 
-        if (!dryRun)
-            store->addToStore(info, sink.s);
+        if (!dryRun) {
+            auto source = StringSource { *sink.s };
+            store->addToStore(info, source);
+        }
 
         logger->stdout("%s", store->printStorePath(info.path));
     }

From 77007d4eabcf7090d1d3fbbdf84a67fb2262cf78 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Sat, 30 May 2020 12:29:35 +0200
Subject: [PATCH 2144/2196] Improve ref validity checking in fetchGit

The previous regex was too strict and did not match what git was allowing. It
could lead to `fetchGit` not accepting valid branch names, even though they
exist in a repository (for example, branch names containing `/`, which are
pretty standard, like `release/1.0` branches).

The new regex defines what a branch name should **NOT** contain. It takes the
definitions from `refs.c` in https://github.com/git/git and `git help
check-ref-format` pages.

This change also introduces a test for ref name validity checking, which
compares the result from Nix with the result of `git check-ref-format --branch`.
---
 src/libfetchers/git.cc |   2 +-
 src/libutil/url.cc     |   1 +
 src/libutil/url.hh     |   6 +++
 tests/fetchGitRefs.sh  | 111 +++++++++++++++++++++++++++++++++++++++++
 tests/local.mk         |   1 +
 5 files changed, 120 insertions(+), 1 deletion(-)
 create mode 100644 tests/fetchGitRefs.sh

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 17cc602285c..e069057eb7a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -418,7 +418,7 @@ struct GitInputScheme : InputScheme
 
         auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
+            if (std::regex_search(*ref, badGitRefRegex))
                 throw BadURL("invalid Git branch/tag name '%s'", *ref);
             input->ref = *ref;
         }
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 5d5328e5de6..88c09eef9d7 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -4,6 +4,7 @@
 namespace nix {
 
 std::regex refRegex(refRegexS, std::regex::ECMAScript);
+std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript);
 std::regex revRegex(revRegexS, std::regex::ECMAScript);
 std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
 
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 1503023a2fc..4a0d4071ba6 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -49,6 +49,12 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
 const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
 extern std::regex refRegex;
 
+// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
+// This is because of the definition of a ref in refs.c in https://github.com/git/git
+// See tests/fetchGitRefs.sh for the full definition
+const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
+extern std::regex badGitRefRegex;
+
 // A Git revision (a SHA-1 commit hash).
 const static std::string revRegexS = "[0-9a-fA-F]{40}";
 extern std::regex revRegex;
diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh
new file mode 100644
index 00000000000..93993ae9098
--- /dev/null
+++ b/tests/fetchGitRefs.sh
@@ -0,0 +1,111 @@
+source common.sh
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git tests"
+    exit 99
+fi
+
+clearStore
+
+repo="$TEST_ROOT/git"
+
+rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
+
+git init "$repo"
+git -C "$repo" config user.email "foobar@example.com"
+git -C "$repo" config user.name "Foobar"
+
+echo utrecht > "$repo"/hello
+git -C "$repo" add hello
+git -C "$repo" commit -m 'Bla1'
+
+path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
+
+# Test various combinations of ref names
+# (taken from the git project)
+
+# git help check-ref-format
+#       Git imposes the following rules on how references are named:
+#
+#        1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot .  or end with the sequence .lock.
+#        2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived.
+#        3. They cannot have two consecutive dots ..  anywhere.
+#        4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere.
+#        5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule.
+#        6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule)
+#        7. They cannot end with a dot ..
+#        8. They cannot contain a sequence @{.
+#        9. They cannot be the single character @.
+#       10. They cannot contain a \.
+
+valid_ref() {
+    { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
+    git check-ref-format --branch "$1" >/dev/null
+    git -C "$repo" branch "$1" master >/dev/null
+    path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
+    [[ $path1 = $path ]]
+    git -C "$repo" branch -D "$1" >/dev/null
+}
+
+invalid_ref() {
+    { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
+    # special case for a sole @:
+    # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel
+    if [ "$1" = "@" ]; then
+        (! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1)
+    else
+        (! git check-ref-format --branch "$1" >/dev/null 2>&1)
+    fi
+    nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'error: invalid Git branch/tag name' >/dev/null
+}
+
+
+valid_ref 'foox'
+valid_ref '1337'
+valid_ref 'foo.baz'
+valid_ref 'foo/bar/baz'
+valid_ref 'foo./bar'
+valid_ref 'heads/foo@bar'
+valid_ref "$(printf 'heads/fu\303\237')"
+valid_ref 'foo-bar-baz'
+valid_ref '$1'
+valid_ref 'foo.locke'
+
+invalid_ref 'refs///heads/foo'
+invalid_ref 'heads/foo/'
+invalid_ref '///heads/foo'
+invalid_ref '.foo'
+invalid_ref './foo'
+invalid_ref './foo/bar'
+invalid_ref 'foo/./bar'
+invalid_ref 'foo/bar/.'
+invalid_ref 'foo bar'
+invalid_ref 'foo?bar'
+invalid_ref 'foo^bar'
+invalid_ref 'foo~bar'
+invalid_ref 'foo:bar'
+invalid_ref 'foo[bar'
+invalid_ref 'foo/bar/.'
+invalid_ref '.refs/foo'
+invalid_ref 'refs/heads/foo.'
+invalid_ref 'heads/foo..bar'
+invalid_ref 'heads/foo?bar'
+invalid_ref 'heads/foo.lock'
+invalid_ref 'heads///foo.lock'
+invalid_ref 'foo.lock/bar'
+invalid_ref 'foo.lock///bar'
+invalid_ref 'heads/v@{ation'
+invalid_ref 'heads/foo\.ar' # should fail due to \
+invalid_ref 'heads/foo\bar' # should fail due to \
+invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB
+invalid_ref "$(printf 'heads/foo\177')"
+invalid_ref '@'
+
+invalid_ref 'foo/*'
+invalid_ref '*/foo'
+invalid_ref 'foo/*/bar'
+invalid_ref '*'
+invalid_ref 'foo/*/*'
+invalid_ref '*/foo/*'
+invalid_ref '/foo'
+invalid_ref ''
diff --git a/tests/local.mk b/tests/local.mk
index 56e5640ca33..536661af88e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -18,6 +18,7 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
+  fetchGitRefs.sh \
   fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \

From fb38459d6e58508245553380cccc03c0dbaa1542 Mon Sep 17 00:00:00 2001
From: Nikola Knezevic <nikola@knezevic.ch>
Date: Sat, 30 May 2020 12:33:38 +0200
Subject: [PATCH 2145/2196] Ensure we restrict refspec interpretation while
 fetching

As `git fetch` may chose to interpret refspec to it's liking, ensure that we
only pass refs that begin with `refs/` as is, otherwise, prepend them with
`refs/heads`. Otherwise, branches named `heads/foo` (I know it's bad, but it's
allowed), would be fetched as `foo`, instead of `heads/foo`.
---
 src/libfetchers/git.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index e069057eb7a..75ce5ee8b46 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -282,7 +282,10 @@ struct GitInput : Input
                 // FIXME: git stderr messes up our progress indicator, so
                 // we're using --quiet for now. Should process its stderr.
                 try {
-                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
+                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
+                        ? *input->ref
+                        : "refs/heads/" + *input->ref;
+                    runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);

From eca1ff7a9f784926c52533c81cb5403a2cb804d3 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Sun, 31 May 2020 01:05:05 +0200
Subject: [PATCH 2146/2196] Add tests for lru-cache.hh

---
 src/libutil/tests/lru-cache.cc | 130 +++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100644 src/libutil/tests/lru-cache.cc

diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc
new file mode 100644
index 00000000000..598977a6dd0
--- /dev/null
+++ b/src/libutil/tests/lru-cache.cc
@@ -0,0 +1,130 @@
+#include "lru-cache.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * size
+     * --------------------------------------------------------------------------*/
+
+    TEST(LRUCache, sizeOfEmptyCacheIsZero) {
+        LRUCache<std::string, std::string> c(10);
+        ASSERT_EQ(c.size(), 0);
+    }
+
+    TEST(LRUCache, sizeOfSingleElementCacheIsOne) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("foo", "bar");
+        ASSERT_EQ(c.size(), 1);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * upsert / get
+     * --------------------------------------------------------------------------*/
+
+    TEST(LRUCache, getFromEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        auto val = c.get("x");
+        ASSERT_EQ(val.has_value(), false);
+    }
+
+    TEST(LRUCache, getExistingValue) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("foo", "bar");
+        auto val = c.get("foo");
+        ASSERT_EQ(val, "bar");
+    }
+
+    TEST(LRUCache, getNonExistingValueFromNonEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("foo", "bar");
+        auto val = c.get("another");
+        ASSERT_EQ(val.has_value(), false);
+    }
+
+    TEST(LRUCache, upsertOnZeroCapacityCache) {
+        LRUCache<std::string, std::string> c(0);
+        c.upsert("foo", "bar");
+        auto val = c.get("foo");
+        ASSERT_EQ(val.has_value(), false);
+    }
+
+    TEST(LRUCache, updateExistingValue) {
+        LRUCache<std::string, std::string> c(1);
+        c.upsert("foo", "bar");
+
+        auto val = c.get("foo");
+        ASSERT_EQ(val.value_or("error"), "bar");
+        ASSERT_EQ(c.size(), 1);
+
+        c.upsert("foo", "changed");
+        val = c.get("foo");
+        ASSERT_EQ(val.value_or("error"), "changed");
+        ASSERT_EQ(c.size(), 1);
+    }
+
+    TEST(LRUCache, overwriteOldestWhenCapacityIsReached) {
+        LRUCache<std::string, std::string> c(3);
+        c.upsert("one", "eins");
+        c.upsert("two", "zwei");
+        c.upsert("three", "drei");
+
+        ASSERT_EQ(c.size(), 3);
+        ASSERT_EQ(c.get("one").value_or("error"), "eins");
+
+        // exceed capacity
+        c.upsert("another", "whatever");
+
+        ASSERT_EQ(c.size(), 3);
+        // Retrieving "one" makes it the most recent element thus
+        // two will be the oldest one and thus replaced.
+        ASSERT_EQ(c.get("two").has_value(), false);
+        ASSERT_EQ(c.get("another").value(), "whatever");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * clear
+     * --------------------------------------------------------------------------*/
+
+    TEST(LRUCache, clearEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.clear();
+        ASSERT_EQ(c.size(), 0);
+    }
+
+    TEST(LRUCache, clearNonEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("one", "eins");
+        c.upsert("two", "zwei");
+        c.upsert("three", "drei");
+        ASSERT_EQ(c.size(), 3);
+        c.clear();
+        ASSERT_EQ(c.size(), 0);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * erase
+     * --------------------------------------------------------------------------*/
+
+    TEST(LRUCache, eraseFromEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.erase("foo");
+        ASSERT_EQ(c.size(), 0);
+    }
+
+    TEST(LRUCache, eraseMissingFromNonEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("one", "eins");
+        c.erase("foo");
+        ASSERT_EQ(c.size(), 1);
+        ASSERT_EQ(c.get("one").value_or("error"), "eins");
+    }
+
+    TEST(LRUCache, eraseFromNonEmptyCache) {
+        LRUCache<std::string, std::string> c(10);
+        c.upsert("one", "eins");
+        c.erase("one");
+        ASSERT_EQ(c.size(), 0);
+        ASSERT_EQ(c.get("one").value_or("empty"), "empty");
+    }
+}

From e9fee8e6a7ff8540a04620e7d75a987cfe89ab00 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 Jun 2020 12:06:59 +0200
Subject: [PATCH 2147/2196] src/libutil/tests/lru-cache.cc: Check erase()

Co-authored-by: James Lee <jbit@jbit.net>
---
 src/libutil/tests/lru-cache.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc
index 598977a6dd0..091d3d5ede1 100644
--- a/src/libutil/tests/lru-cache.cc
+++ b/src/libutil/tests/lru-cache.cc
@@ -108,14 +108,14 @@ namespace nix {
 
     TEST(LRUCache, eraseFromEmptyCache) {
         LRUCache<std::string, std::string> c(10);
-        c.erase("foo");
+        ASSERT_EQ(c.erase("foo"), false);
         ASSERT_EQ(c.size(), 0);
     }
 
     TEST(LRUCache, eraseMissingFromNonEmptyCache) {
         LRUCache<std::string, std::string> c(10);
         c.upsert("one", "eins");
-        c.erase("foo");
+        ASSERT_EQ(c.erase("foo"), false);
         ASSERT_EQ(c.size(), 1);
         ASSERT_EQ(c.get("one").value_or("error"), "eins");
     }
@@ -123,7 +123,7 @@ namespace nix {
     TEST(LRUCache, eraseFromNonEmptyCache) {
         LRUCache<std::string, std::string> c(10);
         c.upsert("one", "eins");
-        c.erase("one");
+        ASSERT_EQ(c.erase("one"), true);
         ASSERT_EQ(c.size(), 0);
         ASSERT_EQ(c.get("one").value_or("empty"), "empty");
     }

From d82d230b4015c50e698e3be1ec7a3e12277a3251 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 2 Jun 2020 08:22:24 -0600
Subject: [PATCH 2148/2196] elide the 'ErrorInfo' in logError and logWarning
 calls

---
 src/build-remote/build-remote.cc              |  11 +-
 src/error-demo/error-demo.cc                  | 111 +++++++++---------
 src/libstore/build.cc                         | 107 ++++++++---------
 src/libstore/filetransfer.cc                  |  11 +-
 src/libstore/local-store.cc                   |  45 ++++---
 src/libstore/optimise-store.cc                |  21 ++--
 src/libutil/logging.cc                        |   7 +-
 src/nix-env/nix-env.cc                        |  27 ++---
 src/nix-store/nix-store.cc                    |  17 ++-
 src/nix/verify.cc                             |  26 ++--
 .../resolve-system-dependencies.cc            |  28 ++---
 11 files changed, 187 insertions(+), 224 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 0caa97fa4dd..2f0b8c82589 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -200,12 +200,11 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    logError(
-                        ErrorInfo { 
-                            .name = "Remote build",
-                            .hint = hintfmt("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
-                        (msg.empty() ? "" : ": " + msg))
+                    logError({ 
+                        .name = "Remote build",
+                        .hint = hintfmt("cannot build on '%s': %s%s",
+                            bestMachine->storeUri, e.what(),
+                            (msg.empty() ? "" : ": " + msg))
                     });
                     bestMachine->enabled = false;
                     continue;
diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
index 0216092e360..f1e17b3de41 100644
--- a/src/error-demo/error-demo.cc
+++ b/src/error-demo/error-demo.cc
@@ -62,101 +62,96 @@ int main()
 
     // For completeness sake, show 'info' through 'vomit' levels.
     // But this is maybe a heavy format for those.
-    logger->logEI(
-        ErrorInfo { .level = lvlInfo,
+    logger->logEI({ .level = lvlInfo,
                     .name = "Info name",
                     .description = "Info description",
         });
 
-    logger->logEI(
-        ErrorInfo { .level = lvlTalkative,
+    logger->logEI({ .level = lvlTalkative,
                     .name = "Talkative name",
                     .description = "Talkative description",
         });
 
-    logger->logEI(
-        ErrorInfo { .level = lvlChatty,
+    logger->logEI({ .level = lvlChatty,
                     .name = "Chatty name",
                     .description = "Chatty description",
         });
 
-    logger->logEI(
-        ErrorInfo { .level = lvlDebug,
+    logger->logEI({ .level = lvlDebug,
                     .name = "Debug name",
                     .description = "Debug description",
         });
 
-    logger->logEI(
-        ErrorInfo { .level = lvlVomit,
+    logger->logEI({ .level = lvlVomit,
                     .name = "Vomit name",
                     .description = "Vomit description",
         });
 
     // Error in a program; no hint and no nix code.
-    logError(
-        ErrorInfo { .name = "name",
-                    .description = "error description",
-        });
+    logError({ 
+        .name = "name",
+        .description = "error description",
+    });
 
     // Warning with name, description, and hint.
     // The hintfmt function makes all the substituted text yellow.
-    logWarning(
-        ErrorInfo { .name = "name",
-                    .description = "error description",
-                    .hint = hintfmt("there was a %1%", "warning"),
-        });
+    logWarning({ 
+        .name = "name",
+        .description = "error description",
+        .hint = hintfmt("there was a %1%", "warning"),
+    });
 
     // Warning with nix file, line number, column, and the lines of
     // code where a warning occurred.
     SymbolTable testTable;
     auto problem_file = testTable.create("myfile.nix");
 
-    logWarning(
-        ErrorInfo { .name = "warning name",
-                    .description = "warning description",
-                    .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
-                        "values"),
-                    .nixCode = NixCode {
-                        .errPos = Pos(problem_file, 40, 13),
-                        .prevLineOfCode = std::nullopt,
-                        .errLineOfCode = "this is the problem line of code",
-                        .nextLineOfCode = std::nullopt
-                    }});
+    logWarning({ 
+        .name = "warning name",
+        .description = "warning description",
+        .hint = hintfmt("this hint has %1% templated %2%!!",
+            "yellow",
+            "values"),
+        .nixCode = NixCode {
+            .errPos = Pos(problem_file, 40, 13),
+            .prevLineOfCode = std::nullopt,
+            .errLineOfCode = "this is the problem line of code",
+            .nextLineOfCode = std::nullopt
+    }});
 
     // Error with previous and next lines of code.
-    logError(
-        ErrorInfo { .name = "error name",
-                    .description = "error with code lines",
-                    .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
-                        "values"),
-                    .nixCode = NixCode {
-                        .errPos = Pos(problem_file, 40, 13),
-                        .prevLineOfCode = "previous line of code",
-                        .errLineOfCode = "this is the problem line of code",
-                        .nextLineOfCode = "next line of code",
-                    }});
+    logError({ 
+        .name = "error name",
+        .description = "error with code lines",
+        .hint = hintfmt("this hint has %1% templated %2%!!",
+            "yellow",
+            "values"),
+        .nixCode = NixCode {
+            .errPos = Pos(problem_file, 40, 13),
+            .prevLineOfCode = "previous line of code",
+            .errLineOfCode = "this is the problem line of code",
+            .nextLineOfCode = "next line of code",
+    }});
 
 
     // Error without lines of code.
-    logError(
-        ErrorInfo { .name = "error name",
-                    .description = "error without any code lines.",
-                    .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
-                        "values"),
-                    .nixCode = NixCode {
-                        .errPos = Pos(problem_file, 40, 13)
-                    }});
+    logError({ 
+        .name = "error name",
+        .description = "error without any code lines.",
+        .hint = hintfmt("this hint has %1% templated %2%!!",
+            "yellow",
+            "values"),
+        .nixCode = NixCode {
+            .errPos = Pos(problem_file, 40, 13)
+    }});
 
     // Error with only hint and name..
-    logError(
-        ErrorInfo { .name = "error name",
-                    .hint = hintfmt("hint %1%", "only"),
-                    .nixCode = NixCode {
-                        .errPos = Pos(problem_file, 40, 13)
-                    }});
+    logError({ 
+        .name = "error name",
+        .hint = hintfmt("hint %1%", "only"),
+        .nixCode = NixCode {
+            .errPos = Pos(problem_file, 40, 13)
+    }});
 
     return 0;
 }
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4630432c943..04e8d2ebeb5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1158,10 +1158,9 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        logError(
-            ErrorInfo {
-                .name = "missing derivation during build",
-                .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))
+        logError({
+            .name = "missing derivation during build",
+            .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))
         });
         done(BuildResult::MiscFailure);
         return;
@@ -1313,12 +1312,11 @@ void DerivationGoal::repairClosure()
     /* Check each path (slow!). */
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
-        logError(
-            ErrorInfo {
-                .name = "Corrupt path in closure",
-                .hint = hintfmt(
-                    "found corrupted or missing path '%s' in the output closure of '%s'",
-                    worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
+        logError({
+            .name = "Corrupt path in closure",
+            .hint = hintfmt(
+                "found corrupted or missing path '%s' in the output closure of '%s'",
+                worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
         });
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
@@ -1353,12 +1351,11 @@ void DerivationGoal::inputsRealised()
     if (nrFailed != 0) {
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        logError(
-            ErrorInfo {
-                .name = "Dependencies could not be built",
-                .hint = hintfmt(
-                    "cannot build derivation '%s': %s dependencies couldn't be built",
-                    worker.store.printStorePath(drvPath), nrFailed)
+        logError({
+            .name = "Dependencies could not be built",
+            .hint = hintfmt(
+                "cannot build derivation '%s': %s dependencies couldn't be built",
+                worker.store.printStorePath(drvPath), nrFailed)
         });
         done(BuildResult::DependencyFailed);
         return;
@@ -1844,12 +1841,11 @@ HookReply DerivationGoal::tryBuildHook()
 
     } catch (SysError & e) {
         if (e.errNo == EPIPE) {
-            logError(
-                ErrorInfo {
-                    .name = "Build hook died",
-                    .hint = hintfmt(
-                        "build hook died unexpectedly: %s",
-                        chomp(drainFD(worker.hook->fromHook.readSide.get())))
+            logError({
+                .name = "Build hook died",
+                .hint = hintfmt(
+                    "build hook died unexpectedly: %s",
+                    chomp(drainFD(worker.hook->fromHook.readSide.get())))
             });
             worker.hook = 0;
             return rpDecline;
@@ -3692,11 +3688,10 @@ void DerivationGoal::registerOutputs()
         /* Apply hash rewriting if necessary. */
         bool rewritten = false;
         if (!outputRewrites.empty()) {
-            logWarning(
-                ErrorInfo {
-                    .name = "Rewriting hashes",
-                    .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
-                    });
+            logWarning({
+                .name = "Rewriting hashes",
+                .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
+            });
 
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
@@ -3875,10 +3870,9 @@ void DerivationGoal::registerOutputs()
                 if (settings.enforceDeterminism)
                     throw NotDeterministic(hint);
 
-                logError(
-                    ErrorInfo {
-                        .name = "Output determinism error",
-                        .hint = hint
+                logError({
+                    .name = "Output determinism error",
+                    .hint = hint
                 });
 
 
@@ -4145,12 +4139,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            logError(
-                ErrorInfo {
-                    .name = "Max log size exceeded",
-                    .hint = hintfmt(
-                        "%1% killed after writing more than %2% bytes of log output",
-                        getName(), settings.maxLogSize)
+            logError({
+                .name = "Max log size exceeded",
+                .hint = hintfmt(
+                    "%1% killed after writing more than %2% bytes of log output",
+                    getName(), settings.maxLogSize)
             });
             killChild();
             done(BuildResult::LogLimitExceeded);
@@ -4467,12 +4460,11 @@ void SubstitutionGoal::tryNext()
         && !sub->isTrusted
         && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
     {
-        logWarning(
-            ErrorInfo {
-                .name = "Invalid path signature",
-                .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
-                    sub->getUri(), worker.store.printStorePath(storePath))
-            });
+        logWarning({
+            .name = "Invalid path signature",
+            .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
+                sub->getUri(), worker.store.printStorePath(storePath))
+        });
         tryNext();
         return;
     }
@@ -4954,12 +4946,11 @@ void Worker::waitForInput()
             j->respectTimeouts &&
             after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
-            logError(
-                ErrorInfo {
-                    .name = "Silent build timeout",
-                    .hint = hintfmt(
-                        "%1% timed out after %2% seconds of silence",
-                        goal->getName(), settings.maxSilentTime)
+            logError({
+                .name = "Silent build timeout",
+                .hint = hintfmt(
+                    "%1% timed out after %2% seconds of silence",
+                    goal->getName(), settings.maxSilentTime)
             });
             goal->timedOut();
         }
@@ -4969,12 +4960,11 @@ void Worker::waitForInput()
             j->respectTimeouts &&
             after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
-            logError(
-                ErrorInfo {
-                    .name = "Build timeout",
-                    .hint = hintfmt(
-                        "%1% timed out after %2% seconds",
-                        goal->getName(), settings.buildTimeout)
+            logError({
+                .name = "Build timeout",
+                .hint = hintfmt(
+                    "%1% timed out after %2% seconds",
+                    goal->getName(), settings.buildTimeout)
             });
             goal->timedOut();
         }
@@ -5035,10 +5025,9 @@ bool Worker::pathContentsGood(const StorePath & path)
     }
     pathContentsGoodCache.insert_or_assign(path.clone(), res);
     if (!res)
-        logError(
-            ErrorInfo {
-                .name = "Corrupted path",
-                .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
+        logError({
+            .name = "Corrupted path",
+            .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
         });
     return res;
 }
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 89588d66269..6ca3393abf7 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -599,12 +599,11 @@ struct curlFileTransfer : public FileTransfer
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            logError(
-                ErrorInfo { 
-                    .name = "File transfer",
-                    .hint = hintfmt("unexpected error in download thread: %s", 
-                                    e.what())
-                });
+            logError({ 
+                .name = "File transfer",
+                .hint = hintfmt("unexpected error in download thread: %s", 
+                                e.what())
+            });
         }
 
         {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index abcfe5271e3..53fb4ce6830 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -87,12 +87,11 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            logError(
-                ErrorInfo { 
-                    .name = "'build-users-group' not found",
-                    .hint = hintfmt(
-                        "warning: the group '%1%' specified in 'build-users-group' does not exist",
-                        settings.buildUsersGroup)
+            logError({ 
+                .name = "'build-users-group' not found",
+                .hint = hintfmt(
+                    "warning: the group '%1%' specified in 'build-users-group' does not exist",
+                    settings.buildUsersGroup)
             });
         else {
             struct stat st;
@@ -1242,12 +1241,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                logError(
-                    ErrorInfo { 
-                        .name = "Invalid hash",
-                        .hint = hintfmt(
-                    "link '%s' was modified! expected hash '%s', got '%s'",
-                    linkPath, link.name, hash)
+                logError({ 
+                    .name = "Invalid hash",
+                    .hint = hintfmt(
+                        "link '%s' was modified! expected hash '%s', got '%s'",
+                        linkPath, link.name, hash)
                 });
                 if (repair) {
                     if (unlink(linkPath.c_str()) == 0)
@@ -1281,11 +1279,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    logError(
-                        ErrorInfo { 
-                            .name = "Invalid hash - path modified",
-                            .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
-                            printStorePath(i), info->narHash.to_string(), current.first.to_string())
+                    logError({ 
+                        .name = "Invalid hash - path modified",
+                        .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
+                        printStorePath(i), info->narHash.to_string(), current.first.to_string())
                     });
                     if (repair) repairPath(i); else errors = true;
                 } else {
@@ -1337,10 +1334,9 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        logError(
-            ErrorInfo { 
-                .name = "Nix path not found",
-                .hint = hintfmt("path '%s' is not in the Nix store", pathS)
+        logError({ 
+            .name = "Nix path not found",
+            .hint = hintfmt("path '%s' is not in the Nix store", pathS)
         });
         return;
     }
@@ -1364,10 +1360,9 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            logError(
-                ErrorInfo { 
-                    .name = "Missing path with referrers",
-                    .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
+            logError({ 
+                .name = "Missing path with referrers",
+                .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
             });
             if (repair)
                 try {
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index c9ecca5a871..cc0507be253 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -130,10 +130,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        logWarning(
-            ErrorInfo { 
-                .name = "Suspicious file",
-                .hint = hintfmt("skipping suspicious writable file '%1%'", path)
+        logWarning({ 
+            .name = "Suspicious file",
+            .hint = hintfmt("skipping suspicious writable file '%1%'", path)
         });
         return;
     }
@@ -198,10 +197,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        logWarning(
-            ErrorInfo { 
-                .name = "Corrupted link",
-                .hint = hintfmt("removing corrupted link '%1%'", linkPath)
+        logWarning({ 
+            .name = "Corrupted link",
+            .hint = hintfmt("removing corrupted link '%1%'", linkPath)
         });
         unlink(linkPath.c_str());
         goto retry;
@@ -237,10 +235,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            logError(
-                ErrorInfo { 
-                    .name = "Unlink error",
-                    .hint = hintfmt("unable to unlink '%1%'", tempLink)
+            logError({ 
+                .name = "Unlink error",
+                .hint = hintfmt("unable to unlink '%1%'", tempLink)
             });
         if (errno == EMLINK) {
             /* Some filesystems generate too many links on the rename,
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 071a7ec7eac..41378b0db95 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -251,10 +251,9 @@ bool handleJSONLogMessage(const std::string & msg,
         }
 
     } catch (std::exception & e) {
-        logError(
-            ErrorInfo { 
-                .name = "Json log message",
-                .hint = hintfmt("bad log message from builder: %s", e.what())
+        logError({ 
+            .name = "Json log message",
+            .hint = hintfmt("bad log message from builder: %s", e.what())
         });
     }
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index bbce5dcdafe..8231a07a438 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -123,10 +123,9 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             if (!attrs.insert(attrName).second) {
-                logError(
-                    ErrorInfo { 
-                        .name = "Name collision",
-                        .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
+                logError({ 
+                    .name = "Name collision",
+                    .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
                 });
                 continue;
             }
@@ -875,11 +874,10 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                logError(
-                    ErrorInfo { 
-                        .name = "Invalid meta attribute",
-                        .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
-                            i.queryName(), j)
+                logError({ 
+                    .name = "Invalid meta attribute",
+                    .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
+                        i.queryName(), j)
                 });
                 placeholder.write(nullptr);
             } else {
@@ -1131,12 +1129,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
                             if (!v) 
-                                logError(
-                                    ErrorInfo { 
-                                        .name = "Invalid meta attribute",
-                                        .hint = hintfmt(
-                                            "derivation '%s' has invalid meta attribute '%s'",
-                                            i.queryName(), j)
+                                logError({ 
+                                    .name = "Invalid meta attribute",
+                                    .hint = hintfmt(
+                                        "derivation '%s' has invalid meta attribute '%s'",
+                                        i.queryName(), j)
                                 });
                             else {
                                 if (v->type == tString) {
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 6b83093b1ce..5e42736fcdc 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -704,7 +704,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", i);
 
     if (store->verifyStore(checkContents, repair)) {
-        logWarning(ErrorInfo { 
+        logWarning({ 
             .name = "Store consistency",
             .description = "not all errors were fixed"
             });
@@ -729,14 +729,13 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
-            logError(
-                ErrorInfo { 
-                    .name = "Hash mismatch",
-                    .hint = hintfmt(
-                        "path '%s' was modified! expected hash '%s', got '%s'",
-                        store->printStorePath(path),
-                        info->narHash.to_string(),
-                        current.first.to_string())
+            logError({ 
+                .name = "Hash mismatch",
+                .hint = hintfmt(
+                    "path '%s' was modified! expected hash '%s', got '%s'",
+                    store->printStorePath(path),
+                    info->narHash.to_string(),
+                    current.first.to_string())
             });
             status = 1;
         }
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 8c845bfc2cd..8ecd9a8f3b7 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -99,15 +99,14 @@ struct CmdVerify : StorePathsCommand
                     if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
-                        logError(
-                            ErrorInfo { 
-                                .name = "Hash error - path modified",
-                                .hint = hintfmt(
-                                    "path '%s' was modified! expected hash '%s', got '%s'",
-                                    store->printStorePath(info->path),
-                                    info->narHash.to_string(),
-                                    hash.first.to_string())
-                            });
+                        logError({ 
+                            .name = "Hash error - path modified",
+                            .hint = hintfmt(
+                                "path '%s' was modified! expected hash '%s', got '%s'",
+                                store->printStorePath(info->path),
+                                info->narHash.to_string(),
+                                hash.first.to_string())
+                        });
                     }
 
                 }
@@ -156,11 +155,10 @@ struct CmdVerify : StorePathsCommand
                     if (!good) {
                         untrusted++;
                         act2.result(resUntrustedPath, store->printStorePath(info->path));
-                        logError(
-                            ErrorInfo { 
-                                .name = "Untrusted path",
-                                .hint = hintfmt("path '%s' is untrusted",
-                                    store->printStorePath(info->path))
+                        logError({ 
+                            .name = "Untrusted path",
+                            .hint = hintfmt("path '%s' is untrusted",
+                                store->printStorePath(info->path))
                         });
 
                     }
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index dcea72529e0..82feacb3dbb 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -39,19 +39,17 @@ std::set<std::string> runResolver(const Path & filename)
         throw SysError("statting '%s'", filename);
 
     if (!S_ISREG(st.st_mode)) {
-        logError(
-            ErrorInfo { 
-                .name = "Regular MACH file",
-                .hint = hintfmt("file '%s' is not a regular file", filename)
+        logError({ 
+            .name = "Regular MACH file",
+            .hint = hintfmt("file '%s' is not a regular file", filename)
         });
         return {};
     }
 
     if (st.st_size < sizeof(mach_header_64)) {
-        logError(
-            ErrorInfo { 
-                .name = "File too short",
-                .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
+        logError({ 
+            .name = "File too short",
+            .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
         });
         return {};
     }
@@ -74,20 +72,18 @@ std::set<std::string> runResolver(const Path & filename)
             }
         }
         if (mach64_offset == 0) {
-            logError(
-                ErrorInfo { 
-                    .name = "No mach64 blobs",
-                    .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
+            logError({ 
+                .name = "No mach64 blobs",
+                .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
             });
             return {};
         }
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
-        logError(
-            ErrorInfo { 
-                .name = "Magic number",
-                .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
+        logError({ 
+            .name = "Magic number",
+            .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
         });
         return {};
     }

From 156d4f8bc892baa4e25814d5f79a332994c750d1 Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Tue, 2 Jun 2020 08:45:37 -0600
Subject: [PATCH 2149/2196] remove extra space in SysErrors

---
 src/libutil/error.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index b374c2780fd..0ca0c8b1555 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -155,7 +155,7 @@ public:
     {
         errNo = errno;
         auto hf = hintfmt(args...);
-        err.hint = hintfmt("%1% : %2%", normaltxt(hf.str()), strerror(errNo));
+        err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }
 
     virtual const char* sname() const override { return "SysError"; }

From 01572c2198de49071827f0be9f5db202bac21703 Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Wed, 3 Jun 2020 06:15:22 -0400
Subject: [PATCH 2150/2196] Missing `#include <cassert>` in `lru-cache.hh`
 (#3654)

This was a latent bug that just appeared because of the tests that were
added. Remember to wait for CI! :)
---
 src/libutil/lru-cache.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
index 8b83f842c32..6ef4a3e067d 100644
--- a/src/libutil/lru-cache.hh
+++ b/src/libutil/lru-cache.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cassert>
 #include <map>
 #include <list>
 #include <optional>

From 6ee03b8444e1838b9985a9fd4f0f46947e958b3b Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Wed, 3 Jun 2020 12:38:23 +0200
Subject: [PATCH 2151/2196] libutils/hash: remove default encoding

This will make it easier to reason about the hash encoding and switch to
SRI everywhere where possible.
---
 perl/lib/Nix/Store.xs               |  4 ++--
 src/libexpr/primops/fetchTree.cc    |  4 ++--
 src/libfetchers/fetchers.cc         |  4 ++--
 src/libfetchers/tarball.cc          |  6 +++---
 src/libstore/build.cc               |  2 +-
 src/libstore/export-import.cc       |  2 +-
 src/libstore/local-store.cc         | 10 +++++-----
 src/libstore/nar-info-disk-cache.cc |  4 ++--
 src/libstore/nar-info.cc            |  4 ++--
 src/libstore/optimise-store.cc      |  2 +-
 src/libstore/store-api.cc           | 12 ++++++------
 src/libutil/hash.hh                 |  2 +-
 src/libutil/tests/hash.cc           | 16 ++++++++--------
 src/nix-store/nix-store.cc          |  6 +++---
 src/nix/verify.cc                   |  2 +-
 15 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 890310b3e80..945ed49c754 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16);
+            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index c5a0d988683..b9040879879 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -23,7 +23,7 @@ void emitTreeAttrs(
 
     assert(tree.info.narHash);
     mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI));
+        tree.info.narHash.to_string(SRI, true));
 
     if (input->getRev()) {
         mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
@@ -140,7 +140,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             : hashFile(htSHA256, path);
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-                *url, expectedHash->to_string(), hash.to_string());
+                *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
     if (state.allowedPaths)
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 94ac30e3800..ca555114da1 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -47,7 +47,7 @@ Attrs Input::toAttrs() const
 {
     auto attrs = toAttrsInternal();
     if (narHash)
-        attrs.emplace("narHash", narHash->to_string(SRI));
+        attrs.emplace("narHash", narHash->to_string(SRI, true));
     attrs.emplace("type", type());
     return attrs;
 }
@@ -67,7 +67,7 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store)
 
     if (narHash && narHash != input->narHash)
         throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-            to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
+            to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
 
     return {std::move(tree), input};
 }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b6e57379bd0..1b0eba18a08 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -196,9 +196,9 @@ struct TarballInput : Input
         // NAR hashes are preferred over file hashes since tar/zip files
         // don't have a canonical representation.
         if (narHash)
-            url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
+            url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
         else if (hash)
-            url2.query.insert_or_assign("hash", hash->to_string(SRI));
+            url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
         return url2;
     }
 
@@ -207,7 +207,7 @@ struct TarballInput : Input
         Attrs attrs;
         attrs.emplace("url", url.to_string());
         if (hash)
-            attrs.emplace("hash", hash->to_string(SRI));
+            attrs.emplace("hash", hash->to_string(SRI, true));
         return attrs;
     }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f5c132a8397..9582a900758 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3718,7 +3718,7 @@ void DerivationGoal::registerOutputs()
                 worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
+                        worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
 
                 Path actualDest = worker.store.Store::toRealPath(dest);
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index f0d01a24043..54471d4a3b1 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -57,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     Hash hash = hashAndWriteSink.currentHash();
     if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash.to_string(), hash.to_string());
+            printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
     hashAndWriteSink
         << exportMagic
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 80851b59106..1c3795eb1c0 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -580,7 +580,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     state.stmtRegisterValidPath.use()
         (printStorePath(info.path))
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.registrationTime == 0 ? time(0) : info.registrationTime)
         (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
         (info.narSize, info.narSize != 0)
@@ -680,7 +680,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 {
     state.stmtUpdatePathInfo.use()
         (info.narSize, info.narSize != 0)
-        (info.narHash.to_string(Base16))
+        (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (info.ca, !info.ca.empty())
@@ -1022,7 +1022,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
-                    printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
+                    printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  wanted: %s\n  got:   %s",
@@ -1155,7 +1155,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
             info.narHash = narHash;
             info.narSize = sink.s->size();
             info.references = cloneStorePathSet(references);
-            info.ca = "text:" + hash.to_string();
+            info.ca = "text:" + hash.to_string(Base32, true);
             registerValidPath(info);
         }
 
@@ -1273,7 +1273,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(), current.first.to_string());
+                        printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true));
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 442541330a2..e8cf1d17724 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -230,9 +230,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
                     (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
-                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                    (info->narHash.to_string())
+                    (info->narHash.to_string(Base32, true))
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 1375094b5bf..6b16be08a13 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const
     assert(compression != "");
     res += "Compression: " + compression + "\n";
     assert(fileHash.type == htSHA256);
-    res += "FileHash: " + fileHash.to_string(Base32) + "\n";
+    res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.type == htSHA256);
-    res += "NarHash: " + narHash.to_string(Base32) + "\n";
+    res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 8ac382e9dbf..3f4b72b9c92 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -150,7 +150,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(htSHA256, path).first;
-    debug(format("'%1%' has hash '%2%'") % path % hash.to_string());
+    debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true));
 
     /* Check if this is a known hash. */
     Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 095363d0c9b..0645fca8416 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -142,7 +142,7 @@ StorePath Store::makeStorePath(const string & type,
     const Hash & hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name);
+    string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
     auto h = compressHash(hashString(htSHA256, s), 20);
     return StorePath::make(h.hash, name);
 }
@@ -186,7 +186,7 @@ StorePath Store::makeFixedOutputPath(
             hashString(htSHA256,
                 "fixed:out:"
                 + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
-                + hash.to_string(Base16) + ":"),
+                + hash.to_string(Base16, true) + ":"),
             name);
     }
 }
@@ -461,7 +461,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash.to_string(hashBase))
+                .attr("narHash", info->narHash.to_string(hashBase, true))
                 .attr("narSize", info->narSize);
 
             {
@@ -504,7 +504,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
                     if (!narInfo->url.empty())
                         jsonPath.attr("url", narInfo->url);
                     if (narInfo->fileHash)
-                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string());
+                        jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
                     if (narInfo->fileSize)
                         jsonPath.attr("downloadSize", narInfo->fileSize);
                     if (showClosureSize)
@@ -760,7 +760,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
             store.printStorePath(path));
     return
         "1;" + store.printStorePath(path) + ";"
-        + narHash.to_string(Base32) + ";"
+        + narHash.to_string(Base32, true) + ";"
         + std::to_string(narSize) + ";"
         + concatStringsSep(",", store.printStorePathSet(references));
 }
@@ -836,7 +836,7 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
 {
     return "fixed:"
         + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
-        + hash.to_string();
+        + hash.to_string(Base32, true);
 }
 
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index ea9fca3e74f..4ff13f6b23a 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -79,7 +79,7 @@ struct Hash
     /* Return a string representation of the hash, in base-16, base-32
        or base-64. By default, this is prefixed by the hash type
        (e.g. "sha256:"). */
-    std::string to_string(Base base = Base32, bool includeType = true) const;
+    std::string to_string(Base base, bool includeType) const;
 
     std::string gitRev() const
     {
diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc
index 7cb4398178a..5334b046e5d 100644
--- a/src/libutil/tests/hash.cc
+++ b/src/libutil/tests/hash.cc
@@ -11,28 +11,28 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";
         auto hash = hashString(HashType::htMD5, s1);
-        ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e");
+        ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e");
     }
 
     TEST(hashString, testKnownMD5Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s2 = "abc";
         auto hash = hashString(HashType::htMD5, s2);
-        ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72");
+        ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72");
     }
 
     TEST(hashString, testKnownSHA1Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abc";
         auto hash = hashString(HashType::htSHA1, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
+        ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
     }
 
     TEST(hashString, testKnownSHA1Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
         auto hash = hashString(HashType::htSHA1, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
+        ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
     }
 
     TEST(hashString, testKnownSHA256Hashes1) {
@@ -40,7 +40,7 @@ namespace nix {
         auto s = "abc";
 
         auto hash = hashString(HashType::htSHA256, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),
+        ASSERT_EQ(hash.to_string(Base::Base16, true),
                 "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
     }
 
@@ -48,7 +48,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
         auto hash = hashString(HashType::htSHA256, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),
+        ASSERT_EQ(hash.to_string(Base::Base16, true),
                 "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
     }
 
@@ -56,7 +56,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abc";
         auto hash = hashString(HashType::htSHA512, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),
+        ASSERT_EQ(hash.to_string(Base::Base16, true),
                 "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9"
                 "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
                 "454d4423643ce80e2a9ac94fa54ca49f");
@@ -67,7 +67,7 @@ namespace nix {
         auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
 
         auto hash = hashString(HashType::htSHA512, s);
-        ASSERT_EQ(hash.to_string(Base::Base16),
+        ASSERT_EQ(hash.to_string(Base::Base16, true),
                 "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1"
                 "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
                 "c7d329eeb6dd26545e96e55b874be909");
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 3a3060ad82d..9491a0c26a1 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -373,7 +373,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
                         assert(info->narHash.type == htSHA256);
-                        cout << fmt("%s\n", info->narHash.to_string(Base32));
+                        cout << fmt("%s\n", info->narHash.to_string(Base32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
                 }
@@ -728,7 +728,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         if (current.first != info->narHash) {
             printError(
                 "path '%s' was modified! expected hash '%s', got '%s'",
-                store->printStorePath(path), info->narHash.to_string(), current.first.to_string());
+                store->printStorePath(path), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true));
             status = 1;
         }
     }
@@ -856,7 +856,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs;
+                            out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs;
                     } catch (InvalidPath &) {
                     }
                 }
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index cf1fa6a990c..287dad10123 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -101,7 +101,7 @@ struct CmdVerify : StorePathsCommand
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
                         printError(
                             "path '%s' was modified! expected hash '%s', got '%s'",
-                            store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string());
+                            store->printStorePath(info->path), info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true));
                     }
 
                 }

From f97576c5d91d5278155a67e2b8982d2233d4335a Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 3 Jun 2020 14:47:00 -0600
Subject: [PATCH 2152/2196] newline-as-prefix; no final newline in output.

---
 src/libutil/error.cc | 73 +++++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 34 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 59628d875a5..539a7543fb4 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -57,20 +57,20 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 {
     // previous line of code.
     if (nixCode.prevLineOfCode.has_value()) {
-        out << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (nixCode.errPos.line - 1),
-            *nixCode.prevLineOfCode)
-            << std::endl;
+        out << std::endl 
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line - 1),
+                *nixCode.prevLineOfCode);
     }
 
     if (nixCode.errLineOfCode.has_value()) {
         // line of code containing the error.
-        out << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (nixCode.errPos.line),
-            *nixCode.errLineOfCode)
-            << std::endl;
+        out << std::endl
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line),
+                *nixCode.errLineOfCode);
         // error arrows for the column range.
         if (nixCode.errPos.column > 0) {
             int start = nixCode.errPos.column;
@@ -81,20 +81,21 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 
             std::string arrows("^");
 
-            out << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-                prefix,
-                spaces,
-                arrows) << std::endl;
+            out << std::endl
+                << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                    prefix,
+                    spaces,
+                    arrows);
         }
     }
 
     // next line of code.
     if (nixCode.nextLineOfCode.has_value()) {
-        out << fmt("%1% %|2$5d|| %3%",
-            prefix,
-            (nixCode.errPos.line + 1),
-            *nixCode.nextLineOfCode)
-            << std::endl;
+        out << std::endl
+            << fmt("%1% %|2$5d|| %3%",
+                prefix,
+                (nixCode.errPos.line + 1),
+                *nixCode.nextLineOfCode);
     }
 }
 
@@ -167,46 +168,50 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
             levelString,
             einfo.name,
             dashes,
-            einfo.programName.value_or(""))
-            << std::endl;
+            einfo.programName.value_or(""));
     else
         out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
             prefix,
             levelString,
             dashes,
-            einfo.programName.value_or(""))
-            << std::endl;
+            einfo.programName.value_or(""));
 
-    // filename, line, column.
+    bool nl = false;  // intersperse newline between sections.
     if (einfo.nixCode.has_value()) {
         if (einfo.nixCode->errPos.file != "") {
-            out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
+            // filename, line, column.
+            out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
                 prefix,
                 einfo.nixCode->errPos.file,
-                showErrPos(einfo.nixCode->errPos)) << std::endl;
-            out << prefix << std::endl;
+                showErrPos(einfo.nixCode->errPos));
         } else {
-            out << fmt("%1%from command line argument", prefix) << std::endl;
-            out << prefix << std::endl;
+            out << std::endl << fmt("%1%from command line argument", prefix);
         }
+        nl = true;
     }
 
     // description
     if (einfo.description != "") {
-        out << prefix << einfo.description << std::endl;
-        out << prefix << std::endl;
+        if (nl)
+            out << std::endl << prefix;
+        out << std::endl << prefix << einfo.description;
+        nl = true;
     }
 
     // lines of code.
     if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
+        if (nl)
+            out << std::endl << prefix;
         printCodeLines(out, prefix, *einfo.nixCode);
-        out << prefix << std::endl;
+        nl = true;
     }
 
     // hint
     if (einfo.hint.has_value()) {
-        out << prefix << *einfo.hint << std::endl;
-        out << prefix << std::endl;
+        if (nl)
+            out << std::endl << prefix;
+        out << std::endl << prefix << *einfo.hint;
+        nl = true;
     }
 
     return out;

From 721943e1d44b63d1a4055309fad39e1f4c0f9f0d Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Wed, 3 Jun 2020 17:32:57 -0600
Subject: [PATCH 2153/2196] update error grep

---
 tests/fetchGitRefs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh
index 93993ae9098..23934698e3e 100644
--- a/tests/fetchGitRefs.sh
+++ b/tests/fetchGitRefs.sh
@@ -56,7 +56,7 @@ invalid_ref() {
     else
         (! git check-ref-format --branch "$1" >/dev/null 2>&1)
     fi
-    nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'error: invalid Git branch/tag name' >/dev/null
+    nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null
 }
 
 

From 61e3d598b67290ca66f34fcac7b7373f506240da Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Jun 2020 10:57:40 +0200
Subject: [PATCH 2154/2196] Rename 'nix dev-shell' to 'nix develop'

Fixes #3648.
---
 src/nix/{dev-shell.cc => develop.cc} | 14 +++++++-------
 src/nix/local.mk                     |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)
 rename src/nix/{dev-shell.cc => develop.cc} (96%)

diff --git a/src/nix/dev-shell.cc b/src/nix/develop.cc
similarity index 96%
rename from src/nix/dev-shell.cc
rename to src/nix/develop.cc
index d300f6a230f..3045d7dc3a5 100644
--- a/src/nix/dev-shell.cc
+++ b/src/nix/develop.cc
@@ -110,7 +110,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
     auto builder = baseNameOf(drv.builder);
     if (builder != "bash")
-        throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");
+        throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
 
     auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
 
@@ -231,11 +231,11 @@ struct Common : InstallableCommand, MixProfile
     }
 };
 
-struct CmdDevShell : Common, MixEnvironment
+struct CmdDevelop : Common, MixEnvironment
 {
     std::vector<std::string> command;
 
-    CmdDevShell()
+    CmdDevelop()
     {
         addFlag({
             .longName = "command",
@@ -259,15 +259,15 @@ struct CmdDevShell : Common, MixEnvironment
         return {
             Example{
                 "To get the build environment of GNU hello:",
-                "nix dev-shell nixpkgs.hello"
+                "nix develop nixpkgs.hello"
             },
             Example{
                 "To store the build environment in a profile:",
-                "nix dev-shell --profile /tmp/my-shell nixpkgs.hello"
+                "nix develop --profile /tmp/my-shell nixpkgs.hello"
             },
             Example{
                 "To use a build environment previously recorded in a profile:",
-                "nix dev-shell /tmp/my-shell"
+                "nix develop /tmp/my-shell"
             },
         };
     }
@@ -341,4 +341,4 @@ struct CmdPrintDevEnv : Common
 };
 
 static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
-static auto r2 = registerCommand<CmdDevShell>("dev-shell");
+static auto r2 = registerCommand<CmdDevelop>("develop");
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 8c0eed19ef2..43b7754e34b 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
 
-src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh
+src/nix/develop.cc: src/nix/get-env.sh.gen.hh

From 0f44b60e6dc999697bf9f2a4b3652a0551016598 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 4 Jun 2020 11:14:19 +0200
Subject: [PATCH 2155/2196] Make 'nix dev-shell' a deprecated alias for 'nix
 develop'

---
 src/libutil/args.cc | 11 ++++++++---
 src/libutil/args.hh |  2 ++
 src/libutil/util.hh |  3 +--
 src/nix/main.cc     |  2 ++
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index f829415d106..afeaf4cea5d 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -217,10 +217,15 @@ MultiCommand::MultiCommand(const Commands & commands)
 {
     expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
         assert(!command);
-        auto i = commands.find(ss[0]);
+        auto cmd = ss[0];
+        if (auto alias = get(deprecatedAliases, cmd)) {
+            warn("'%s' is a deprecated alias for '%s'", cmd, *alias);
+            cmd = *alias;
+        }
+        auto i = commands.find(cmd);
         if (i == commands.end())
-            throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = {ss[0], i->second()};
+            throw UsageError("'%s' is not a recognised command", cmd);
+        command = {cmd, i->second()};
     }});
 
     categories[Command::catDefault] = "Available commands";
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 1932e6a8ae4..154d1e6aadb 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -234,6 +234,8 @@ public:
 
     std::map<Command::Category, std::string> categories;
 
+    std::map<std::string, std::string> deprecatedAliases;
+
     // Selected command, if any.
     std::optional<std::pair<std::string, ref<Command>>> command;
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a63ee05b377..4b117f9bc05 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -459,8 +459,7 @@ string base64Encode(const string & s);
 string base64Decode(const string & s);
 
 
-/* Get a value for the specified key from an associate container, or a
-   default value if the key doesn't exist. */
+/* Get a value for the specified key from an associate container. */
 template <class T>
 std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
 {
diff --git a/src/nix/main.cc b/src/nix/main.cc
index ef301580ab9..1120ba5efa4 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -110,6 +110,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .description = "consider all previously downloaded files out-of-date",
             .handler = {[&]() { refresh = true; }},
         });
+
+        deprecatedAliases.insert({"dev-shell", "develop"});
     }
 
     void printFlags(std::ostream & out) override

From 2299ef705c2d69add371ee4ccca04de9f2628853 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Wed, 3 Jun 2020 15:18:46 -0500
Subject: [PATCH 2156/2196] Add error message when FileIngestionMethod is out
 of bounds
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

bool coerces anything >0 to true, but in the future we may have other
file ingestion methods. This shows a better error message when the
“recursive” byte isn’t 1.
---
 src/libstore/daemon.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index f1afdff699e..5cff170dd93 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -358,8 +358,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         std::string s, baseName;
         FileIngestionMethod method;
         {
-            bool fixed, recursive;
+            bool fixed; uint8_t recursive;
             from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
+                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
             method = FileIngestionMethod { recursive };
             /* Compatibility hack. */
             if (!fixed) {

From 94427ffee3493077206f960e3cda9bbb659583be Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Thu, 4 Jun 2020 11:53:19 -0600
Subject: [PATCH 2157/2196] add some comments

---
 src/libutil/error.cc   |  6 +++++-
 src/libutil/error.hh   | 21 +++++++++++++++++++--
 src/libutil/logging.hh | 33 ++++++++++++++++++---------------
 3 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 539a7543fb4..1fcb8111c5e 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -10,13 +10,16 @@ namespace nix {
 
 const std::string nativeSystem = SYSTEM;
 
-
+// addPrefix is used for show-trace.  Strings added with addPrefix
+// will print ahead of the error itself.
 BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
     prefix_ = fs.s + prefix_;
     return *this;
 }
 
+// c++ std::exception descendants must have a 'const char* what()' function.
+// This stringifies the error and caches it for use by what(), or similarly by msg().
 const string& BaseError::calcWhat() const
 {
     if (what_.has_value())
@@ -53,6 +56,7 @@ string showErrPos(const ErrPos &errPos)
     }
 }
 
+// if nixCode contains lines of code, print them to the ostream, indicating the error column.
 void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
 {
     // previous line of code.
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 0ca0c8b1555..8a48fa105b1 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -22,6 +22,23 @@
 
 namespace nix {
 
+/* 
+
+This file defines two main structs/classes used in nix error handling.
+
+ErrorInfo provides a standard payload of error information, with conversion to string
+happening in the logger rather than at the call site.
+
+BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+an ErrorInfo.
+
+ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+logError or logWarning macros.
+
+See the error-demo.cc program for usage examples.
+
+*/
+
 typedef enum {
     lvlError = 0,
     lvlWarn,
@@ -32,6 +49,7 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
+// ErrPos indicates the location of an error in a nix file.
 struct ErrPos {
     int line = 0;
     int column = 0;
@@ -42,6 +60,7 @@ struct ErrPos {
         return line != 0;
     }
 
+    // convert from the Pos struct, found in libexpr.
     template <class P>
     ErrPos& operator=(const P &pos)
     {
@@ -65,8 +84,6 @@ struct NixCode {
     std::optional<string> nextLineOfCode;
 };
 
-// -------------------------------------------------
-// ErrorInfo.
 struct ErrorInfo {
     Verbosity level;
     string name;
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 39692a29113..eeb7233e917 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -150,9 +150,23 @@ bool handleJSONLogMessage(const std::string & msg,
 
 extern Verbosity verbosity; /* suppress msgs > this */
 
-/* Print a message if the current log level is at least the specified
-   level. Note that this has to be implemented as a macro to ensure
-   that the arguments are evaluated lazily. */
+/* Print a message with the standard ErrorInfo format.
+   In general, use these 'log' macros for reporting problems that may require user
+   intervention or that need more explanation.  Use the 'print' macros for more
+   lightweight status messages. */
+#define logErrorInfo(level, errorInfo...) \
+    do { \
+        if (level <= nix::verbosity) { \
+            logger->logEI(level, errorInfo); \
+        } \
+    } while (0)
+
+#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo)
+#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo)
+
+/* Print a string message if the current log level is at least the specified
+   level. Note that this has to be implemented as a macro to ensure that the
+   arguments are evaluated lazily. */
 #define printMsg(level, args...) \
     do { \
         if (level <= nix::verbosity) { \
@@ -166,18 +180,7 @@ extern Verbosity verbosity; /* suppress msgs > this */
 #define debug(args...) printMsg(lvlDebug, args)
 #define vomit(args...) printMsg(lvlVomit, args)
 
-#define logErrorInfo(level, errorInfo...) \
-    do { \
-        if (level <= nix::verbosity) { \
-            logger->logEI(level, errorInfo); \
-        } \
-    } while (0)
-
-#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo)
-#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo)
-
-
-
+/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */
 template<typename... Args>
 inline void warn(const std::string & fs, const Args & ... args)
 {

From efc5e45e95734cdade04ad6231c5185a298357d0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 4 Jun 2020 21:05:41 +0000
Subject: [PATCH 2158/2196] Add `src/libutil/tests/libutil-tests` to
 `.gitignore`

I gather this comes from the new unit tests.
---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index ad568412313..9830265701d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,9 @@ perl/Makefile.config
 # /src/libstore/
 *.gen.*
 
+# /src/libutil/
+/src/libutil/tests/libutil-tests
+
 /src/nix/nix
 
 # /src/nix-env/

From 39e84c35d0b8bf5b49f45736febd9f506e3ee4ae Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jun 2020 10:45:05 +0200
Subject: [PATCH 2159/2196] Fix log-prefix of `nix build -L`

Alternative fix to #3661. The cause was that 'name' is a
std::string_view into a temporary which could get overwritten.
---
 src/nix/progress-bar.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 8e7ba95a341..c677010989e 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -154,7 +154,7 @@ class ProgressBar : public Logger
         state->activitiesByType[type].its.emplace(act, i);
 
         if (type == actBuild) {
-            auto name = storePathToName(getS(fields, 0));
+            std::string name(storePathToName(getS(fields, 0)));
             if (hasSuffix(name, ".drv"))
                 name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);

From fb4724220430bee79391a345fed217e3d36858bf Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyfox@gentoo.org>
Date: Thu, 31 Aug 2017 21:41:36 +0100
Subject: [PATCH 2160/2196] src/libmain/stack.cc: fix 'ucontext' usage on
 glibc-2.26

Build fails as:

$ make
  CXX    src/libmain/stack.o
src/libmain/stack.cc: In function 'void nix::sigsegvHandler(int, siginfo_t*, void*)':
src/libmain/stack.cc:21:21: error: 'ucontext' was not declared in this scope
     sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
                     ^~~~~~~~
src/libmain/stack.cc:21:21: note: suggested alternative: 'ucontext_t'
     sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
                     ^~~~~~~~
                     ucontext_t

It's caused by upstream rename:
https://sourceware.org/git/?p=glibc.git;a=commitdiff;h=251287734e89a52da3db682a8241eb6bccc050c9

which basically changes
    typedef struct ucontext {} ucontext_t;
to
    typedef struct ucontext_t {} ucontext_t;

The change uses ucontext_t.

Signed-off-by: Sergei Trofimovich <slyfox@gentoo.org>
---
 src/libmain/stack.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc
index abf59dc4baa..b9eb2fa18e5 100644
--- a/src/libmain/stack.cc
+++ b/src/libmain/stack.cc
@@ -20,9 +20,9 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
     bool haveSP = true;
     char * sp = 0;
 #if defined(__x86_64__) && defined(REG_RSP)
-    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
+    sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP];
 #elif defined(REG_ESP)
-    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP];
+    sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP];
 #else
     haveSP = false;
 #endif

From 952e72c804ce79ffeee8f58261d4cbdeb9d56ac6 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Sat, 6 Jun 2020 10:22:32 +0200
Subject: [PATCH 2161/2196] Add tests for logging.hh

---
 src/libutil/tests/local.mk   |   2 +-
 src/libutil/tests/logging.cc | 251 +++++++++++++++++++++++++++++++++++
 2 files changed, 252 insertions(+), 1 deletion(-)
 create mode 100644 src/libutil/tests/logging.cc

diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
index a297edb6478..815e185600e 100644
--- a/src/libutil/tests/local.mk
+++ b/src/libutil/tests/local.mk
@@ -8,7 +8,7 @@ libutil-tests_INSTALL_DIR :=
 
 libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil-tests_CXXFLAGS += -I src/libutil
+libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr
 
 libutil-tests_LIBS = libutil
 
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
new file mode 100644
index 00000000000..ac015a16d5e
--- /dev/null
+++ b/src/libutil/tests/logging.cc
@@ -0,0 +1,251 @@
+#include "logging.hh"
+#include "nixexpr.hh"
+#include "util.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * logEI
+     * --------------------------------------------------------------------------*/
+
+    TEST(logEI, catpuresBasicProperties) {
+
+        MakeError(TestError, Error);
+        ErrorInfo::programName = std::optional("error-unit-test");
+
+        try {
+            throw TestError("an error for testing purposes");
+        } catch (Error &e) {
+            testing::internal::CaptureStderr();
+            logger->logEI(e.info());
+            auto str = testing::internal::GetCapturedStderr();
+
+            ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\nan error for testing purposes\n");
+        }
+    }
+
+    TEST(logEI, appendingHintsToPreviousError) {
+
+        MakeError(TestError, Error);
+        ErrorInfo::programName = std::optional("error-unit-test");
+
+        try {
+            auto e = Error("initial error");
+            throw TestError(e.info());
+        } catch (Error &e) {
+            ErrorInfo ei = e.info();
+            ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : ""));
+
+            testing::internal::CaptureStderr();
+            logger->logEI(ei);
+            auto str = testing::internal::GetCapturedStderr();
+
+            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n");
+        }
+
+    }
+
+    TEST(logEI, picksUpSysErrorExitCode) {
+
+        MakeError(TestError, Error);
+        ErrorInfo::programName = std::optional("error-unit-test");
+
+        try {
+            auto x = readFile(-1);
+        }
+        catch (SysError &e) {
+            testing::internal::CaptureStderr();
+            logError(e.info());
+            auto str = testing::internal::GetCapturedStderr();
+
+            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError ------------------------------------- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n");
+
+        }
+    }
+
+    TEST(logEI, loggingErrorOnInfoLevel) {
+        testing::internal::CaptureStderr();
+
+        logger->logEI({ .level = lvlInfo,
+                .name = "Info name",
+                .description = "Info description",
+                });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name ------------------------------------- error-unit-test\x1B[0m\nInfo description\n");
+    }
+
+    TEST(logEI, loggingErrorOnTalkativeLevel) {
+        testing::internal::CaptureStderr();
+
+        logger->logEI({ .level = lvlTalkative,
+                        .name = "Talkative name",
+                        .description = "Talkative description",
+                        });
+
+        auto str = testing::internal::GetCapturedStderr();
+        // XXX: why is this the empty string?
+        ASSERT_STREQ(str.c_str(), "");
+    }
+
+    TEST(logEI, loggingErrorOnChattyLevel) {
+        testing::internal::CaptureStderr();
+
+        logger->logEI({ .level = lvlChatty,
+                        .name = "Chatty name",
+                        .description = "Talkative description",
+                        });
+
+        auto str = testing::internal::GetCapturedStderr();
+        // XXX: why is this the empty string?
+        ASSERT_STREQ(str.c_str(), "");
+    }
+
+    TEST(logEI, loggingErrorOnDebugLevel) {
+        testing::internal::CaptureStderr();
+
+        logger->logEI({ .level = lvlDebug,
+                        .name = "Debug name",
+                        .description = "Debug description",
+                        });
+
+        auto str = testing::internal::GetCapturedStderr();
+        // XXX: why is this the empty string?
+        ASSERT_STREQ(str.c_str(), "");
+    }
+
+    TEST(logEI, loggingErrorOnVomitLevel) {
+        testing::internal::CaptureStderr();
+
+        logger->logEI({ .level = lvlVomit,
+                        .name = "Vomit name",
+                        .description = "Vomit description",
+                        });
+
+        auto str = testing::internal::GetCapturedStderr();
+        // XXX: why is this the empty string?
+        ASSERT_STREQ(str.c_str(), "");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * logError
+     * --------------------------------------------------------------------------*/
+
+
+    TEST(logError, logErrorWithoutHintOrCode) {
+        testing::internal::CaptureStderr();
+
+        logError({
+                .name = "name",
+                .description = "error description",
+                });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name ----------------------------------------- error-unit-test\x1B[0m\nerror description\n");
+    }
+
+    TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("myfile.nix");
+
+        testing::internal::CaptureStderr();
+
+        logError({
+                .name = "error name",
+                .description = "error with code lines",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                        "yellow",
+                        "values"),
+                .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = "previous line of code",
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = "next line of code",
+                }});
+
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n    39| previous line of code\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n    41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+    }
+
+    TEST(logError, logErrorWithoutLinesOfCode) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("myfile.nix");
+        testing::internal::CaptureStderr();
+
+        logError({
+                .name = "error name",
+                .description = "error without any code lines.",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                        "yellow",
+                        "values"),
+                .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13)
+                }});
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+    }
+
+    TEST(logError, logErrorWithOnlyHintAndName) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("myfile.nix");
+        testing::internal::CaptureStderr();
+
+        logError({
+                .name = "error name",
+                .hint = hintfmt("hint %1%", "only"),
+                .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13)
+                }});
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
+
+    }
+
+    /* ----------------------------------------------------------------------------
+     * logWarning
+     * --------------------------------------------------------------------------*/
+
+    TEST(logWarning, logWarningWithNameDescriptionAndHint) {
+        testing::internal::CaptureStderr();
+
+        logWarning({
+            .name = "name",
+            .description = "error description",
+            .hint = hintfmt("there was a %1%", "warning"),
+        });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --------------------------------------- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
+    }
+
+    TEST(logWarning, logWarningWithFileLineNumAndCode) {
+
+        SymbolTable testTable;
+        auto problem_file = testTable.create("myfile.nix");
+
+        testing::internal::CaptureStderr();
+
+        logWarning({
+                .name = "warning name",
+                .description = "warning description",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                        "yellow",
+                        "values"),
+                .nixCode = NixCode {
+                .errPos = Pos(problem_file, 40, 13),
+                .prevLineOfCode = std::nullopt,
+                .errLineOfCode = "this is the problem line of code",
+                .nextLineOfCode = std::nullopt
+                }});
+
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name ------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+    }
+
+}

From e60747b5fbe8b9c646ffe07ef8d28d949302efa6 Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Sat, 6 Jun 2020 10:23:12 +0200
Subject: [PATCH 2162/2196] Remove error-demo/error-demo.cc

The logging.hh superseeds the demo
---
 src/error-demo/error-demo.cc | 157 -----------------------------------
 src/error-demo/local.mk      |  12 ---
 2 files changed, 169 deletions(-)
 delete mode 100644 src/error-demo/error-demo.cc
 delete mode 100644 src/error-demo/local.mk

diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc
deleted file mode 100644
index f1e17b3de41..00000000000
--- a/src/error-demo/error-demo.cc
+++ /dev/null
@@ -1,157 +0,0 @@
-#include "logging.hh"
-#include "nixexpr.hh"
-#include "util.hh"
-
-#include <iostream>
-#include <optional>
-
-using namespace nix;
-
-MakeError(DemoError, Error);
-
-int main()
-{
-    verbosity = lvlVomit;
-
-    // In each program where errors occur, this has to be set.
-    ErrorInfo::programName = std::optional("error-demo");
-
-    // 'DemoError' appears as error name.
-    try {
-        throw DemoError("demo error was thrown");
-    } catch (Error &e) {
-        logger->logEI(e.info());
-    }
-
-    // appending to the hint from the previous error
-    try {
-        auto e = Error("initial error");
-        throw DemoError(e.info());
-    } catch (Error &e) {
-        ErrorInfo ei = e.info();
-        // using normaltxt to avoid the default yellow highlighting.
-        ei.hint = hintfmt("%s; subsequent error message.", 
-            normaltxt(e.info().hint ? e.info().hint->str() : ""));
-        logger->logEI(ei);
-    }
-
-    // SysError; picks up errno
-    try {
-        auto x = readFile(-1);
-    }
-    catch (SysError &e) {
-        std::cout << "errno was: " << e.errNo << std::endl;
-        logError(e.info());
-    }
-
-    // current exception
-    try {
-        throw DemoError("DemoError handled as a %1%", "std::exception");
-    }
-    catch (...) {
-        const std::exception_ptr &eptr = std::current_exception();
-        try
-        {
-            std::rethrow_exception(eptr);
-        }
-        catch (std::exception& e)
-        {
-            std::cerr << e.what() << std::endl;
-        }
-    }
-
-    // For completeness sake, show 'info' through 'vomit' levels.
-    // But this is maybe a heavy format for those.
-    logger->logEI({ .level = lvlInfo,
-                    .name = "Info name",
-                    .description = "Info description",
-        });
-
-    logger->logEI({ .level = lvlTalkative,
-                    .name = "Talkative name",
-                    .description = "Talkative description",
-        });
-
-    logger->logEI({ .level = lvlChatty,
-                    .name = "Chatty name",
-                    .description = "Chatty description",
-        });
-
-    logger->logEI({ .level = lvlDebug,
-                    .name = "Debug name",
-                    .description = "Debug description",
-        });
-
-    logger->logEI({ .level = lvlVomit,
-                    .name = "Vomit name",
-                    .description = "Vomit description",
-        });
-
-    // Error in a program; no hint and no nix code.
-    logError({ 
-        .name = "name",
-        .description = "error description",
-    });
-
-    // Warning with name, description, and hint.
-    // The hintfmt function makes all the substituted text yellow.
-    logWarning({ 
-        .name = "name",
-        .description = "error description",
-        .hint = hintfmt("there was a %1%", "warning"),
-    });
-
-    // Warning with nix file, line number, column, and the lines of
-    // code where a warning occurred.
-    SymbolTable testTable;
-    auto problem_file = testTable.create("myfile.nix");
-
-    logWarning({ 
-        .name = "warning name",
-        .description = "warning description",
-        .hint = hintfmt("this hint has %1% templated %2%!!",
-            "yellow",
-            "values"),
-        .nixCode = NixCode {
-            .errPos = Pos(problem_file, 40, 13),
-            .prevLineOfCode = std::nullopt,
-            .errLineOfCode = "this is the problem line of code",
-            .nextLineOfCode = std::nullopt
-    }});
-
-    // Error with previous and next lines of code.
-    logError({ 
-        .name = "error name",
-        .description = "error with code lines",
-        .hint = hintfmt("this hint has %1% templated %2%!!",
-            "yellow",
-            "values"),
-        .nixCode = NixCode {
-            .errPos = Pos(problem_file, 40, 13),
-            .prevLineOfCode = "previous line of code",
-            .errLineOfCode = "this is the problem line of code",
-            .nextLineOfCode = "next line of code",
-    }});
-
-
-    // Error without lines of code.
-    logError({ 
-        .name = "error name",
-        .description = "error without any code lines.",
-        .hint = hintfmt("this hint has %1% templated %2%!!",
-            "yellow",
-            "values"),
-        .nixCode = NixCode {
-            .errPos = Pos(problem_file, 40, 13)
-    }});
-
-    // Error with only hint and name..
-    logError({ 
-        .name = "error name",
-        .hint = hintfmt("hint %1%", "only"),
-        .nixCode = NixCode {
-            .errPos = Pos(problem_file, 40, 13)
-    }});
-
-    return 0;
-}
diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk
deleted file mode 100644
index 2c528490a2a..00000000000
--- a/src/error-demo/local.mk
+++ /dev/null
@@ -1,12 +0,0 @@
-programs += error-demo
-
-error-demo_DIR := $(d)
-
-error-demo_SOURCES := \
-  $(wildcard $(d)/*.cc) \
-
-error-demo_CXXFLAGS += -I src/libutil -I src/libexpr
-
-error-demo_LIBS = libutil libexpr
-
-error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

From 94c347577ecea5dcd10a31ebfadf94db6ca5ab0d Mon Sep 17 00:00:00 2001
From: Ben Burdette <bburdette@gmail.com>
Date: Sun, 7 Jun 2020 07:24:49 -0600
Subject: [PATCH 2163/2196] set verbosity levels

---
 src/libutil/tests/logging.cc | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index ac015a16d5e..fbdc91253b5 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -78,6 +78,8 @@ namespace nix {
     }
 
     TEST(logEI, loggingErrorOnTalkativeLevel) {
+        verbosity = lvlTalkative;
+
         testing::internal::CaptureStderr();
 
         logger->logEI({ .level = lvlTalkative,
@@ -86,11 +88,12 @@ namespace nix {
                         });
 
         auto str = testing::internal::GetCapturedStderr();
-        // XXX: why is this the empty string?
-        ASSERT_STREQ(str.c_str(), "");
+        ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name -------------------------------- error-unit-test\x1B[0m\nTalkative description\n");
     }
 
     TEST(logEI, loggingErrorOnChattyLevel) {
+        verbosity = lvlChatty;
+
         testing::internal::CaptureStderr();
 
         logger->logEI({ .level = lvlChatty,
@@ -99,11 +102,12 @@ namespace nix {
                         });
 
         auto str = testing::internal::GetCapturedStderr();
-        // XXX: why is this the empty string?
-        ASSERT_STREQ(str.c_str(), "");
+        ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name ----------------------------------- error-unit-test\x1B[0m\nTalkative description\n");
     }
 
     TEST(logEI, loggingErrorOnDebugLevel) {
+        verbosity = lvlDebug;
+
         testing::internal::CaptureStderr();
 
         logger->logEI({ .level = lvlDebug,
@@ -112,11 +116,12 @@ namespace nix {
                         });
 
         auto str = testing::internal::GetCapturedStderr();
-        // XXX: why is this the empty string?
-        ASSERT_STREQ(str.c_str(), "");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name ----------------------------------- error-unit-test\x1B[0m\nDebug description\n");
     }
 
     TEST(logEI, loggingErrorOnVomitLevel) {
+        verbosity = lvlVomit;
+
         testing::internal::CaptureStderr();
 
         logger->logEI({ .level = lvlVomit,
@@ -125,8 +130,7 @@ namespace nix {
                         });
 
         auto str = testing::internal::GetCapturedStderr();
-        // XXX: why is this the empty string?
-        ASSERT_STREQ(str.c_str(), "");
+        ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name ----------------------------------- error-unit-test\x1B[0m\nVomit description\n");
     }
 
     /* ----------------------------------------------------------------------------

From 170e86dff5724264e0d3d25b9af1bd42df6aec74 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 5 Jun 2020 17:01:02 +0200
Subject: [PATCH 2164/2196] Make the logger customisable

Add a new `--log-format` cli argument to change the format of the logs.
The possible values are
- raw (the default one for old-style commands)
- bar (the default one for new-style commands)
- bar-with-logs (equivalent to `--print-build-logs`)
- internal-json (the internal machine-readable json format)
---
 src/libmain/common-args.cc |  9 ++++++++
 src/libmain/loggers.cc     | 47 ++++++++++++++++++++++++++++++++++++++
 src/libmain/loggers.hh     | 19 +++++++++++++++
 src/libutil/logging.cc     |  4 ++--
 src/libutil/logging.hh     |  4 +++-
 src/libutil/util.cc        |  2 +-
 src/nix-build/nix-build.cc |  4 ++++
 src/nix-env/nix-env.cc     |  2 ++
 src/nix-store/nix-store.cc |  2 ++
 src/nix/main.cc            | 11 +++++----
 src/nix/progress-bar.cc    | 14 ++++++++----
 src/nix/progress-bar.hh    |  2 ++
 12 files changed, 107 insertions(+), 13 deletions(-)
 create mode 100644 src/libmain/loggers.cc
 create mode 100644 src/libmain/loggers.hh

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 51e199ea5ce..a0cccceb4f7 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -1,5 +1,6 @@
 #include "common-args.hh"
 #include "globals.hh"
+#include "loggers.hh"
 
 namespace nix {
 
@@ -38,6 +39,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
         }},
     });
 
+    addFlag({
+        .longName = "log-format",
+        .description = "Format of the logs. One of \"raw\", \"internal-json\", \"bar\" "
+                        "or \"bar-with-logs\".",
+        .labels = {"format"},
+        .handler = {[](std::string format) { setLogFormat(format); }},
+    });
+
     addFlag({
         .longName = "max-jobs",
         .shortName = 'j',
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
new file mode 100644
index 00000000000..d3d5b104bd5
--- /dev/null
+++ b/src/libmain/loggers.cc
@@ -0,0 +1,47 @@
+#include "loggers.hh"
+#include "../nix/progress-bar.hh"
+
+namespace nix {
+
+LogFormat defaultLogFormat = LogFormat::raw;
+
+LogFormat parseLogFormat(const string &logFormatStr) {
+    if (logFormatStr == "raw")
+        return LogFormat::raw;
+    else if (logFormatStr == "internal-json")
+        return LogFormat::internalJson;
+    else if (logFormatStr == "bar")
+        return LogFormat::bar;
+    else if (logFormatStr == "bar-with-logs")
+        return LogFormat::barWithLogs;
+    throw Error(format("option 'log-format' has an invalid value '%s'") %
+                logFormatStr);
+}
+
+Logger *makeDefaultLogger() {
+    switch (defaultLogFormat) {
+    case LogFormat::raw:
+        return makeSimpleLogger();
+    case LogFormat::internalJson:
+        return makeJSONLogger(*makeSimpleLogger());
+    case LogFormat::bar:
+        return makeProgressBar();
+    case LogFormat::barWithLogs:
+        return makeProgressBar(true);
+    }
+}
+
+void setLogFormat(const string &logFormatStr) {
+    setLogFormat(parseLogFormat(logFormatStr));
+}
+
+void setLogFormat(const LogFormat &logFormat) {
+    defaultLogFormat = logFormat;
+    createDefaultLogger();
+}
+
+void createDefaultLogger() {
+    logger = makeDefaultLogger();
+}
+
+}
diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh
new file mode 100644
index 00000000000..f50cbb682fe
--- /dev/null
+++ b/src/libmain/loggers.hh
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "types.hh"
+
+namespace nix {
+
+enum class LogFormat {
+  raw,
+  internalJson,
+  bar,
+  barWithLogs,
+};
+
+void setLogFormat(const string &logFormatStr);
+void setLogFormat(const LogFormat &logFormat);
+
+void createDefaultLogger();
+
+}
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 3cc4ef8f15b..6aec16e58d4 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -18,7 +18,7 @@ void setCurActivity(const ActivityId activityId)
     curActivity = activityId;
 }
 
-Logger * logger = makeDefaultLogger();
+Logger * logger = makeSimpleLogger();
 
 void Logger::warn(const std::string & msg)
 {
@@ -94,7 +94,7 @@ void writeToStderr(const string & s)
     }
 }
 
-Logger * makeDefaultLogger()
+Logger * makeSimpleLogger()
 {
     return new SimpleLogger();
 }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 18c24d50877..e319790fccd 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -63,6 +63,8 @@ public:
 
     virtual ~Logger() { }
 
+    virtual void stop() { };
+
     virtual void log(Verbosity lvl, const FormatOrString & fs) = 0;
 
     void log(const FormatOrString & fs)
@@ -141,7 +143,7 @@ struct PushActivity
 
 extern Logger * logger;
 
-Logger * makeDefaultLogger();
+Logger * makeSimpleLogger();
 
 Logger * makeJSONLogger(Logger & prevLogger);
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 71db92d772a..e0a99152b56 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -989,7 +989,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 {
     auto wrapper = [&]() {
         if (!options.allowVfork)
-            logger = makeDefaultLogger();
+            logger = makeSimpleLogger();
         try {
 #if __linux__
             if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 0a058a31b49..8649de5e9e8 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -472,6 +472,8 @@ static void _main(int argc, char * * argv)
 
         restoreSignals();
 
+        logger->stop();
+
         execvp(shell->c_str(), argPtrs.data());
 
         throw SysError("executing shell '%s'", *shell);
@@ -521,6 +523,8 @@ static void _main(int argc, char * * argv)
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
 
+        logger->stop();
+
         for (auto & path : outPaths)
             std::cout << path << '\n';
     }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index d62febaff53..f7b04eb2bfa 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -1446,6 +1446,8 @@ static int _main(int argc, char * * argv)
 
         globals.state->printStats();
 
+        logger->stop();
+
         return 0;
     }
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 3a3060ad82d..708591b1414 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -1098,6 +1098,8 @@ static int _main(int argc, char * * argv)
 
         op(opFlags, opArgs);
 
+        logger->stop();
+
         return 0;
     }
 }
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 1120ba5efa4..203901168b9 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -10,6 +10,7 @@
 #include "progress-bar.hh"
 #include "filetransfer.hh"
 #include "finally.hh"
+#include "loggers.hh"
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -90,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName = "print-build-logs",
             .shortName = 'L',
             .description = "print full build logs on stderr",
-            .handler = {&printBuildLogs, true},
+            .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
         });
 
         addFlag({
@@ -165,6 +166,10 @@ void mainWrapped(int argc, char * * argv)
     verbosity = lvlWarn;
     settings.verboseBuild = false;
 
+    setLogFormat("bar");
+
+    Finally f([] { logger->stop(); });
+
     NixArgs args;
 
     args.parseCmdline(argvToStrings(argc, argv));
@@ -178,10 +183,6 @@ void mainWrapped(int argc, char * * argv)
         && args.command->first != "upgrade-nix")
         settings.requireExperimentalFeature("nix-command");
 
-    Finally f([]() { stopProgressBar(); });
-
-    startProgressBar(args.printBuildLogs);
-
     if (args.useNet && !haveInternet()) {
         warn("you don't have Internet access; disabling some network-dependent features");
         args.useNet = false;
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index c677010989e..828541bfebd 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -106,7 +106,7 @@ class ProgressBar : public Logger
         updateThread.join();
     }
 
-    void stop()
+    void stop() override
     {
         auto state(state_.lock());
         if (!state->active) return;
@@ -457,11 +457,17 @@ class ProgressBar : public Logger
     }
 };
 
-void startProgressBar(bool printBuildLogs)
+Logger *makeProgressBar(bool printBuildLogs)
 {
-    logger = new ProgressBar(
+    return new ProgressBar(
         printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
+        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+    );
+}
+
+void startProgressBar(bool printBuildLogs)
+{
+    logger = makeProgressBar(printBuildLogs);
 }
 
 void stopProgressBar()
diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh
index 4d61175c24e..60d0a2076cc 100644
--- a/src/nix/progress-bar.hh
+++ b/src/nix/progress-bar.hh
@@ -4,6 +4,8 @@
 
 namespace nix {
 
+Logger* makeProgressBar(bool printBuildLogs = false);
+
 void startProgressBar(bool printBuildLogs = false);
 
 void stopProgressBar();

From 2c4de6af1033367168320f43b0f04062bdac9234 Mon Sep 17 00:00:00 2001
From: zimbatm <zimbatm@zimbatm.com>
Date: Thu, 26 Sep 2019 16:55:51 +0000
Subject: [PATCH 2165/2196] add documentation

---
 doc/manual/command-ref/opt-common-syn.xml |  6 ++++-
 doc/manual/command-ref/opt-common.xml     | 31 +++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml
index b610b54b962..2660e3bb1cc 100644
--- a/doc/manual/command-ref/opt-common-syn.xml
+++ b/doc/manual/command-ref/opt-common-syn.xml
@@ -1,5 +1,5 @@
 <nop xmlns="http://docbook.org/ns/docbook">
-  
+
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'>
@@ -11,6 +11,10 @@
 <arg>
   <arg choice='plain'><option>--quiet</option></arg>
 </arg>
+<arg>
+  <option>--log-format</option>
+  <replaceable>format</replaceable>
+</arg>
 <arg>
   <group choice='plain'>
     <arg choice='plain'><option>--no-build-output</option></arg>
diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml
index 0383bfaedf1..a68eef1d0e7 100644
--- a/doc/manual/command-ref/opt-common.xml
+++ b/doc/manual/command-ref/opt-common.xml
@@ -92,6 +92,37 @@
 </varlistentry>
 
 
+<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
+
+  <listitem>
+
+  <para>This option can be used to change the output of the log format, with
+  <replaceable>format</replaceable> being one of:</para>
+
+  <variablelist>
+
+    <varlistentry><term>raw</term>
+    <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>internal-json</term>
+    <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar</term>
+    <listitem><para>Only display a progress bar during the builds.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>bar-with-logs</term>
+    <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
+    </varlistentry>
+
+  </variablelist>
+
+  </listitem>
+
+</varlistentry>
+
 <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
 
   <listitem><para>By default, output written by builders to standard

From 4983401440e1c46d6c576bc36ac86169bd296f9f Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Fri, 5 Jun 2020 18:20:11 +0200
Subject: [PATCH 2166/2196] Unify the printing of the logs between
 bar-with-logs and raw

Make the printing of the build logs systematically go through the
logger, and replicate the behavior of `no-build-output` by having two
different loggers (one that prints the build logs and one that doesn't)
---
 src/libmain/loggers.cc  |  6 +++++-
 src/libmain/loggers.hh  |  1 +
 src/libmain/shared.cc   |  3 ++-
 src/libstore/build.cc   | 17 ++++-------------
 src/libutil/logging.cc  | 30 ++++++++++++++++++++++++++----
 src/libutil/logging.hh  |  5 ++++-
 src/nix/progress-bar.cc |  4 ++++
 7 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index d3d5b104bd5..b810969314c 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -8,6 +8,8 @@ LogFormat defaultLogFormat = LogFormat::raw;
 LogFormat parseLogFormat(const string &logFormatStr) {
     if (logFormatStr == "raw")
         return LogFormat::raw;
+    else if (logFormatStr == "raw-with-logs")
+        return LogFormat::rawWithLogs;
     else if (logFormatStr == "internal-json")
         return LogFormat::internalJson;
     else if (logFormatStr == "bar")
@@ -21,7 +23,9 @@ LogFormat parseLogFormat(const string &logFormatStr) {
 Logger *makeDefaultLogger() {
     switch (defaultLogFormat) {
     case LogFormat::raw:
-        return makeSimpleLogger();
+        return makeSimpleLogger(false);
+    case LogFormat::rawWithLogs:
+        return makeSimpleLogger(true);
     case LogFormat::internalJson:
         return makeJSONLogger(*makeSimpleLogger());
     case LogFormat::bar:
diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh
index f50cbb682fe..f9ba5ee5e2d 100644
--- a/src/libmain/loggers.hh
+++ b/src/libmain/loggers.hh
@@ -6,6 +6,7 @@ namespace nix {
 
 enum class LogFormat {
   raw,
+  rawWithLogs,
   internalJson,
   bar,
   barWithLogs,
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 70d1f0186c2..3bbb5cf9380 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "loggers.hh"
 
 #include <algorithm>
 #include <cctype>
@@ -169,7 +170,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
         .longName = "no-build-output",
         .shortName = 'Q',
         .description = "do not show build output",
-        .handler = {&settings.verboseBuild, false},
+        .handler = {[&]() {setLogFormat(LogFormat::raw); }},
     });
 
     addFlag({
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index f5c132a8397..2d022093cab 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1642,7 +1642,7 @@ void DerivationGoal::buildDone()
                 worker.store.printStorePath(drvPath),
                 statusToString(status));
 
-            if (!settings.verboseBuild && !logTail.empty()) {
+            if (!logger->isVerbose() && !logTail.empty()) {
                 msg += (format("; last %d log lines:") % logTail.size()).str();
                 for (auto & line : logTail)
                     msg += "\n  " + line;
@@ -1691,11 +1691,7 @@ void DerivationGoal::buildDone()
                 }
 
                 void flushLine() {
-                    if (settings.verboseBuild) {
-                        printError("post-build-hook: " + currentLine);
-                    } else {
-                        act.result(resPostBuildLogLine, currentLine);
-                    }
+                    act.result(resPostBuildLogLine, currentLine);
                     currentLine.clear();
                 }
 
@@ -4155,13 +4151,8 @@ void DerivationGoal::flushLine()
         ;
 
     else {
-        if (settings.verboseBuild &&
-            (settings.printRepeatedBuilds || curRound == 1))
-            printError(currentLogLine);
-        else {
-            logTail.push_back(currentLogLine);
-            if (logTail.size() > settings.logLines) logTail.pop_front();
-        }
+        logTail.push_back(currentLogLine);
+        if (logTail.size() > settings.logLines) logTail.pop_front();
 
         act->result(resBuildLogLine, currentLogLine);
     }
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 6aec16e58d4..15cbc1589ea 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -18,7 +18,7 @@ void setCurActivity(const ActivityId activityId)
     curActivity = activityId;
 }
 
-Logger * logger = makeSimpleLogger();
+Logger * logger = makeSimpleLogger(true);
 
 void Logger::warn(const std::string & msg)
 {
@@ -35,13 +35,19 @@ class SimpleLogger : public Logger
 public:
 
     bool systemd, tty;
+    bool printBuildLogs;
 
-    SimpleLogger()
+    SimpleLogger(bool printBuildLogs)
+        : printBuildLogs(printBuildLogs)
     {
         systemd = getEnv("IN_SYSTEMD") == "1";
         tty = isatty(STDERR_FILENO);
     }
 
+    bool isVerbose() override {
+        return printBuildLogs;
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         if (lvl > verbosity) return;
@@ -70,6 +76,18 @@ class SimpleLogger : public Logger
         if (lvl <= verbosity && !s.empty())
             log(lvl, s + "...");
     }
+
+    void result(ActivityId act, ResultType type, const Fields & fields) override
+    {
+        if (type == resBuildLogLine && printBuildLogs) {
+            auto lastLine = fields[0].s;
+            printError(lastLine);
+        }
+        else if (type == resPostBuildLogLine && printBuildLogs) {
+            auto lastLine = fields[0].s;
+            printError("post-build-hook: " + lastLine);
+        }
+    }
 };
 
 Verbosity verbosity = lvlInfo;
@@ -94,9 +112,9 @@ void writeToStderr(const string & s)
     }
 }
 
-Logger * makeSimpleLogger()
+Logger * makeSimpleLogger(bool printBuildLogs)
 {
-    return new SimpleLogger();
+    return new SimpleLogger(printBuildLogs);
 }
 
 std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
@@ -114,6 +132,10 @@ struct JSONLogger : Logger
 
     JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
 
+    bool isVerbose() override {
+        return true;
+    }
+
     void addFields(nlohmann::json & json, const Fields & fields)
     {
         if (fields.empty()) return;
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index e319790fccd..e3d91e01fb4 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -65,6 +65,9 @@ public:
 
     virtual void stop() { };
 
+    // Whether the logger prints the whole build log
+    virtual bool isVerbose() { return false; }
+
     virtual void log(Verbosity lvl, const FormatOrString & fs) = 0;
 
     void log(const FormatOrString & fs)
@@ -143,7 +146,7 @@ struct PushActivity
 
 extern Logger * logger;
 
-Logger * makeSimpleLogger();
+Logger * makeSimpleLogger(bool printBuildLogs = true);
 
 Logger * makeJSONLogger(Logger & prevLogger);
 
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index 828541bfebd..c9ba8971486 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -119,6 +119,10 @@ class ProgressBar : public Logger
         quitCV.notify_one();
     }
 
+    bool isVerbose() override {
+        return printBuildLogs;
+    }
+
     void log(Verbosity lvl, const FormatOrString & fs) override
     {
         auto state(state_.lock());

From f6ac888d3e1897f37a8accb9c46ebdf4fc2ca204 Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Jun 2020 10:01:14 +0200
Subject: [PATCH 2167/2196] Actually test nix-env with a remote store

The `remote-store` test loads the `user-env` one to test nix-env when
using the daemon, but actually does it incorrectly because every test
starts (in `common.sh`) by resetting the value of `NIX_REMOTE`, meaning
that the `user-env` test will never use the daemon.

Fix this by setting `NIX_REMOTE_` before sourcing `user-env.sh` in the
`remote-store` test, so that `NIX_REMOTE` is correctly set inside the
test
---
 tests/remote-store.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index 77437658ead..4cc73465aea 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -4,7 +4,7 @@ clearStore
 
 startDaemon
 
-storeCleared=1 $SHELL ./user-envs.sh
+storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
 
 nix-store --dump-db > $TEST_ROOT/d1
 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2

From c27f92698bbbb99df3c4403db6590276318940a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 Jun 2020 13:24:01 +0200
Subject: [PATCH 2168/2196] Style fixes

---
 src/libmain/common-args.cc | 4 ++--
 src/libmain/loggers.cc     | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index a0cccceb4f7..051668e53a2 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -41,8 +41,8 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "Format of the logs. One of \"raw\", \"internal-json\", \"bar\" "
-                        "or \"bar-with-logs\".",
+        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
+                        "or \"bar-with-logs\"",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index b810969314c..ee3836382fa 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -16,8 +16,7 @@ LogFormat parseLogFormat(const string &logFormatStr) {
         return LogFormat::bar;
     else if (logFormatStr == "bar-with-logs")
         return LogFormat::barWithLogs;
-    throw Error(format("option 'log-format' has an invalid value '%s'") %
-                logFormatStr);
+    throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
 }
 
 Logger *makeDefaultLogger() {

From 801112de1af8e519e29e1e1c678f97de45a8c49d Mon Sep 17 00:00:00 2001
From: regnat <rg@regnat.ovh>
Date: Mon, 8 Jun 2020 17:16:52 +0200
Subject: [PATCH 2169/2196] Move progress-bar.cc to libmain

Needed so that we can include it as a logger in loggers.cc without
adding a dependency on nix

This also requires moving names.hh to libutil to prevent a circular
dependency between libmain and libexpr
---
 src/libmain/loggers.cc                   | 2 +-
 src/{nix => libmain}/progress-bar.cc     | 0
 src/{nix => libmain}/progress-bar.hh     | 0
 src/{libexpr => libutil}/names.cc        | 0
 src/{libexpr => libutil}/names.hh        | 0
 src/nix-prefetch-url/nix-prefetch-url.cc | 2 +-
 6 files changed, 2 insertions(+), 2 deletions(-)
 rename src/{nix => libmain}/progress-bar.cc (100%)
 rename src/{nix => libmain}/progress-bar.hh (100%)
 rename src/{libexpr => libutil}/names.cc (100%)
 rename src/{libexpr => libutil}/names.hh (100%)

diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index ee3836382fa..41f0b35fb3c 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -1,5 +1,5 @@
 #include "loggers.hh"
-#include "../nix/progress-bar.hh"
+#include "progress-bar.hh"
 
 namespace nix {
 
diff --git a/src/nix/progress-bar.cc b/src/libmain/progress-bar.cc
similarity index 100%
rename from src/nix/progress-bar.cc
rename to src/libmain/progress-bar.cc
diff --git a/src/nix/progress-bar.hh b/src/libmain/progress-bar.hh
similarity index 100%
rename from src/nix/progress-bar.hh
rename to src/libmain/progress-bar.hh
diff --git a/src/libexpr/names.cc b/src/libutil/names.cc
similarity index 100%
rename from src/libexpr/names.cc
rename to src/libutil/names.cc
diff --git a/src/libexpr/names.hh b/src/libutil/names.hh
similarity index 100%
rename from src/libexpr/names.hh
rename to src/libutil/names.hh
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 5a686c8cd67..b645bdc1b0a 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -8,7 +8,7 @@
 #include "attr-path.hh"
 #include "finally.hh"
 #include "../nix/legacy.hh"
-#include "../nix/progress-bar.hh"
+#include "progress-bar.hh"
 #include "tarfile.hh"
 
 #include <iostream>

From 62b6cb9f91f5727fa7a69e905150da9b8b477c05 Mon Sep 17 00:00:00 2001
From: Carlo Nucera <carlo.nucera@protonmail.com>
Date: Mon, 8 Jun 2020 11:56:21 -0400
Subject: [PATCH 2170/2196] Update the ipfs binary store code

---
 src/libstore/ipfs-binary-cache-store.cc | 38 +++++++++++++------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc
index 05de85b4cbc..d6e4309ae7b 100644
--- a/src/libstore/ipfs-binary-cache-store.cc
+++ b/src/libstore/ipfs-binary-cache-store.cc
@@ -44,10 +44,10 @@ class IPFSBinaryCacheStore : public BinaryCacheStore
       const Params & params, const Path & _cacheUri)
             : BinaryCacheStore(params)
             , cacheUri(_cacheUri)
-            , ipfsAPIHost(get(params, "host", "127.0.0.1"))
-            , ipfsAPIPort(std::stoi(get(params, "port", "5001")))
-            , useIpfsGateway(get(params, "use_gateway", "0") == "1")
-            , ipfsGatewayURL(get(params, "gateway", "https://ipfs.io"))
+            , ipfsAPIHost(get(params, "host").value_or("127.0.0.1"))
+            , ipfsAPIPort(std::stoi(get(params, "port").value_or("5001")))
+            , useIpfsGateway(get(params, "use_gateway").value_or("0") == "1")
+            , ipfsGatewayURL(get(params, "gateway").value_or("https://ipfs.io"))
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
@@ -66,13 +66,16 @@ class IPFSBinaryCacheStore : public BinaryCacheStore
 
     void init() override
     {
-        if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
+        if (auto cacheInfo = diskCache->cacheExists(getUri())) {
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
+            priority.setDefault(fmt("%d", cacheInfo->priority));
+        } else {
             try {
               BinaryCacheStore::init();
             } catch (UploadToIPFS &) {
               throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
             }
-            diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
+            diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
         }
     }
 
@@ -108,8 +111,7 @@ class IPFSBinaryCacheStore : public BinaryCacheStore
     }
 
     void getFile(const std::string & path,
-        std::function<void(std::shared_ptr<std::string>)> success,
-        std::function<void(std::exception_ptr exc)> failure) override
+        Callback<std::shared_ptr<std::string>> callback) noexcept override
     {
         /*
          * TODO: Try local mount first, best to share code with
@@ -120,21 +122,21 @@ class IPFSBinaryCacheStore : public BinaryCacheStore
         //request.showProgress = DownloadRequest::no;
         request.tries = 8;
 
+        auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
         getDownloader()->enqueueDownload(request,
-            [success](const DownloadResult & result) {
-                success(result.data);
-            },
-            [success, failure](std::exception_ptr exc) {
+            {[callbackPtr](std::future<DownloadResult> result){
                 try {
-                    std::rethrow_exception(exc);
+                    (*callbackPtr)(result.get().data);
                 } catch (DownloadError & e) {
-                    if (e.error == Downloader::NotFound)
-                        return success(0);
-                    failure(exc);
+                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                        return (*callbackPtr)(std::shared_ptr<std::string>());
+                    callbackPtr->rethrow();
                 } catch (...) {
-                    failure(exc);
+                    callbackPtr->rethrow();
                 }
-            });
+            }}
+        );
     }
 
 };

From 762273f1fd48b2b2f2bbedca65abfd4c07b5af05 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 9 Jun 2020 01:23:37 -0500
Subject: [PATCH 2171/2196] Allow empty hash in derivations

follow up of https://github.com/NixOS/nix/pull/3544

This allows hash="" so that it can be used for debugging purposes. For
instance, this gives you an error message like:

  warning: found empty hash, assuming you wanted 'sha256:0000000000000000000000000000000000000000000000000000'
  hash mismatch in fixed-output derivation '/nix/store/asx6qw1r1xk6iak6y6jph4n58h4hdmbm-nix':
    wanted: sha256:0000000000000000000000000000000000000000000000000000
    got:    sha256:0fpfhipl9v1mfzw2ffmxiyyzqwlkvww22bh9wcy4qrfslb4jm429
---
 src/libexpr/primops.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d458ab2721f..de04fd2beb4 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -719,7 +719,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);
 
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
-        Hash h(*outputHash, ht);
+
+        Hash h;
+        if (outputHash->empty()) {
+            h = Hash(ht);
+            printError("warning: found empty hash, assuming you wanted '%s'", h.to_string());
+        } else
+            h = Hash(*outputHash, ht);
 
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);

From 19aa892f2064d437d32a3c12758d6a623b7ae8e5 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 9 Jun 2020 11:10:54 -0500
Subject: [PATCH 2172/2196] Support empty hash in fetchers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fetchTarball, fetchTree, and fetchGit all have *optional* hash attrs.
This means that we need to be careful with what we allow to avoid
accidentally making these defaults. When ‘hash = ""’ we assume the
empty hash is wanted.
---
 src/libexpr/primops.cc           | 11 ++++++++---
 src/libexpr/primops/fetchTree.cc | 11 ++++++++---
 src/libfetchers/fetchers.cc      | 11 ++++++++---
 src/libfetchers/tarball.cc       | 11 ++++++++---
 4 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index de04fd2beb4..df3d4a45917 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1131,9 +1131,14 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             filterFun = attr.value;
         } else if (n == "recursive")
             method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
-        else if (n == "sha256")
-            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-        else
+        else if (n == "sha256") {
+            auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos);
+            if (hashStr == "") {
+                expectedHash = Hash(htSHA256);
+                printError("warning: found empty hash, assuming you wanted '%s'", expectedHash.to_string());
+            } else
+                expectedHash = Hash(hashStr, htSHA256);
+        } else
             throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
     }
     if (path.empty())
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index c5a0d988683..745f65adfcb 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -102,9 +102,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             string n(attr.name);
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256")
-                expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
-            else if (n == "name")
+            else if (n == "sha256") {
+                auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos);
+                if (hashStr == "") {
+                    expectedHash = Hash(htSHA256);
+                    printError("warning: found empty hash, assuming you wanted '%s'", expectedHash->to_string());
+                } else
+                    expectedHash = Hash(hashStr, htSHA256);
+            } else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to '%s', at %s",
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 94ac30e3800..988a8dc6923 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -34,9 +34,14 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
     for (auto & inputScheme : *inputSchemes) {
         auto res = inputScheme->inputFromAttrs(attrs2);
         if (res) {
-            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
-                // FIXME: require SRI hash.
-                res->narHash = Hash(*narHash);
+            if (auto narHash = maybeGetStrAttr(attrs, "narHash")) {
+                if (narHash->empty()) {
+                    res->narHash = Hash(htUnknown);
+                    printError("warning: found empty hash, assuming you wanted '%s'", res->narHash->to_string());
+                } else
+                    // FIXME: require SRI hash.
+                    res->narHash = Hash(*narHash);
+            }
             return res;
         }
     }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index b6e57379bd0..e4dafec0b48 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -263,9 +263,14 @@ struct TarballInputScheme : InputScheme
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            // FIXME: require SRI hash.
-            input->hash = Hash(*hash);
+        if (auto hash = maybeGetStrAttr(attrs, "hash")) {
+            if (hash->empty()) {
+                input->hash = Hash(htUnknown);
+                printError("warning: found empty hash, assuming you wanted '%s'", input->hash->to_string());
+            } else
+                // FIXME: require SRI hash.
+                input->hash = Hash(*hash);
+        }
 
         return input;
     }

From b2c8061b44b01a39d953d75db30e39bd8f300021 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 9 Jun 2020 16:53:53 -0500
Subject: [PATCH 2173/2196] Disable extra-platforms = i686-linux on wsl1
 (#3676)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

WSL1 doesn’t support i686-linux emulation, see https://github.com/microsoft/wsl/issues/2468
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index da95fd3ae91..2fbcafff87b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -271,7 +271,7 @@ public:
         "listed in 'trusted-public-keys'."};
 
     Setting<StringSet> extraPlatforms{this,
-        std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{},
+        std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
         "Additional platforms that can be built on the local system. "
         "These may be supported natively (e.g. armv7 on some aarch64 CPUs "

From 7eca8a16eaf74bc15a816e24005a65e5480d2a79 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 9 Jun 2020 23:35:38 -0500
Subject: [PATCH 2174/2196] Prelink static libraries into an object file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This combines the *.o into a big .o producing one translation unit.
This preserve our unused static initializers, as specified in the C++
standard:

  If no variable or function is odr-used from a given translation
  unit, the non-local variables defined in that translation unit may
  never be initialized (this models the behavior of an on-demand
  dynamic library).

Note that this is very similar to how the --whole-archive flag works.
One advantage of this is that users of the final .a library don’t have
to worry about specifying --whole-archive, or that we have unused
static initializers at all!
---
 mk/libraries.mk | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mk/libraries.mk b/mk/libraries.mk
index 307e29b9d05..e6ef2e3ec7d 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -125,7 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ar) $(AR) crs $$@ $$?
+	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

From 4750d98bbd7c233e993b1ed66dd5573e6bdf543c Mon Sep 17 00:00:00 2001
From: Tobias Pflug <tobias.pflug@gmail.com>
Date: Mon, 8 Jun 2020 17:39:46 +0200
Subject: [PATCH 2175/2196] Add tests for pool.hh

---
 src/libutil/tests/pool.cc | 127 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 src/libutil/tests/pool.cc

diff --git a/src/libutil/tests/pool.cc b/src/libutil/tests/pool.cc
new file mode 100644
index 00000000000..127e42dda2b
--- /dev/null
+++ b/src/libutil/tests/pool.cc
@@ -0,0 +1,127 @@
+#include "pool.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    struct TestResource
+    {
+
+        TestResource() {
+            static int counter = 0;
+            num = counter++;
+        }
+
+        int dummyValue = 1;
+        bool good = true;
+        int num;
+    };
+
+    /* ----------------------------------------------------------------------------
+     * Pool
+     * --------------------------------------------------------------------------*/
+
+    TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) {
+        auto isGood = [](const ref<TestResource> & r) { return r->good; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+
+        ASSERT_EQ(pool.count(), 0);
+        ASSERT_EQ(pool.capacity(), 1);
+    }
+
+    TEST(Pool, freshPoolCanGetAResource) {
+        auto isGood = [](const ref<TestResource> & r) { return r->good; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+        ASSERT_EQ(pool.count(), 0);
+
+        TestResource r = *(pool.get());
+
+        ASSERT_EQ(pool.count(), 1);
+        ASSERT_EQ(pool.capacity(), 1);
+        ASSERT_EQ(r.dummyValue, 1);
+        ASSERT_EQ(r.good, true);
+    }
+
+    TEST(Pool, capacityCanBeIncremented) {
+        auto isGood = [](const ref<TestResource> & r) { return r->good; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+        ASSERT_EQ(pool.capacity(), 1);
+        pool.incCapacity();
+        ASSERT_EQ(pool.capacity(), 2);
+    }
+
+    TEST(Pool, capacityCanBeDecremented) {
+        auto isGood = [](const ref<TestResource> & r) { return r->good; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+        ASSERT_EQ(pool.capacity(), 1);
+        pool.decCapacity();
+        ASSERT_EQ(pool.capacity(), 0);
+    }
+
+    TEST(Pool, flushBadDropsOutOfScopeResources) {
+        auto isGood = [](const ref<TestResource> & r) { return false; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+
+        {
+            auto _r = pool.get();
+            ASSERT_EQ(pool.count(), 1);
+        }
+
+        pool.flushBad();
+        ASSERT_EQ(pool.count(), 0);
+    }
+
+    // Test that the resources we allocate are being reused when they are still good.
+    TEST(Pool, reuseResource) {
+        auto isGood = [](const ref<TestResource> & r) { return true; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+
+        // Compare the instance counter between the two handles. We expect them to be equal
+        // as the pool should hand out the same (still) good one again.
+        int counter = -1;
+        {
+            Pool<TestResource>::Handle h = pool.get();
+            counter = h->num;
+        } // the first handle goes out of scope
+
+        { // the second handle should contain the same resource (with the same counter value)
+            Pool<TestResource>::Handle h = pool.get();
+            ASSERT_EQ(h->num, counter);
+        }
+    }
+
+    // Test that the resources we allocate are being thrown away when they are no longer good.
+    TEST(Pool, badResourceIsNotReused) {
+        auto isGood = [](const ref<TestResource> & r) { return false; };
+        auto createResource = []() { return make_ref<TestResource>(); };
+
+        Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood);
+
+        // Compare the instance counter between the two handles. We expect them
+        // to *not* be equal as the pool should hand out a new instance after
+        // the first one was returned.
+        int counter = -1;
+        {
+            Pool<TestResource>::Handle h = pool.get();
+            counter = h->num;
+        } // the first handle goes out of scope
+
+        {
+          // the second handle should contain a different resource (with a
+          //different counter value)
+            Pool<TestResource>::Handle h = pool.get();
+            ASSERT_NE(h->num, counter);
+        }
+    }
+}

From 8bd892117a324c22803366b872bfe7ed783b34eb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 Jun 2020 15:39:08 +0200
Subject: [PATCH 2176/2196] Style fixes

---
 src/libmain/loggers.cc      | 8 ++++----
 src/libmain/loggers.hh      | 4 ++--
 src/libmain/progress-bar.cc | 2 +-
 src/libmain/progress-bar.hh | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index 41f0b35fb3c..350f90266cd 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -5,7 +5,7 @@ namespace nix {
 
 LogFormat defaultLogFormat = LogFormat::raw;
 
-LogFormat parseLogFormat(const string &logFormatStr) {
+LogFormat parseLogFormat(const std::string & logFormatStr) {
     if (logFormatStr == "raw")
         return LogFormat::raw;
     else if (logFormatStr == "raw-with-logs")
@@ -19,7 +19,7 @@ LogFormat parseLogFormat(const string &logFormatStr) {
     throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
 }
 
-Logger *makeDefaultLogger() {
+Logger * makeDefaultLogger() {
     switch (defaultLogFormat) {
     case LogFormat::raw:
         return makeSimpleLogger(false);
@@ -34,11 +34,11 @@ Logger *makeDefaultLogger() {
     }
 }
 
-void setLogFormat(const string &logFormatStr) {
+void setLogFormat(const std::string & logFormatStr) {
     setLogFormat(parseLogFormat(logFormatStr));
 }
 
-void setLogFormat(const LogFormat &logFormat) {
+void setLogFormat(const LogFormat & logFormat) {
     defaultLogFormat = logFormat;
     createDefaultLogger();
 }
diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh
index f9ba5ee5e2d..cada0311049 100644
--- a/src/libmain/loggers.hh
+++ b/src/libmain/loggers.hh
@@ -12,8 +12,8 @@ enum class LogFormat {
   barWithLogs,
 };
 
-void setLogFormat(const string &logFormatStr);
-void setLogFormat(const LogFormat &logFormat);
+void setLogFormat(const std::string & logFormatStr);
+void setLogFormat(const LogFormat & logFormat);
 
 void createDefaultLogger();
 
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index c9ba8971486..b287de8a3bc 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -461,7 +461,7 @@ class ProgressBar : public Logger
     }
 };
 
-Logger *makeProgressBar(bool printBuildLogs)
+Logger * makeProgressBar(bool printBuildLogs)
 {
     return new ProgressBar(
         printBuildLogs,
diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh
index 60d0a2076cc..7f0dafecf8c 100644
--- a/src/libmain/progress-bar.hh
+++ b/src/libmain/progress-bar.hh
@@ -4,7 +4,7 @@
 
 namespace nix {
 
-Logger* makeProgressBar(bool printBuildLogs = false);
+Logger * makeProgressBar(bool printBuildLogs = false);
 
 void startProgressBar(bool printBuildLogs = false);
 

From 95eb0640620f0704b4ded6ed6c7509024c00273e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 Jun 2020 15:39:30 +0200
Subject: [PATCH 2177/2196] Shut up warning

---
 src/libmain/loggers.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index 350f90266cd..c44bb640869 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -31,6 +31,8 @@ Logger * makeDefaultLogger() {
         return makeProgressBar();
     case LogFormat::barWithLogs:
         return makeProgressBar(true);
+    default:
+        abort();
     }
 }
 

From dd9bb11d0d38139bb32411170403171c4c92f8cf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 11 Jun 2020 15:42:18 +0200
Subject: [PATCH 2178/2196] Move names.{cc,hh} to libstore

---
 src/{libutil => libstore}/names.cc | 0
 src/{libutil => libstore}/names.hh | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename src/{libutil => libstore}/names.cc (100%)
 rename src/{libutil => libstore}/names.hh (100%)

diff --git a/src/libutil/names.cc b/src/libstore/names.cc
similarity index 100%
rename from src/libutil/names.cc
rename to src/libstore/names.cc
diff --git a/src/libutil/names.hh b/src/libstore/names.hh
similarity index 100%
rename from src/libutil/names.hh
rename to src/libstore/names.hh

From 045b07200c77bf1fe19c0a986aafb531e7e1ba54 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jun 2020 12:46:33 +0200
Subject: [PATCH 2179/2196] Remove Store::queryDerivationOutputNames()

This function was used in only one place, where it could easily be
replaced by readDerivation() since it's not
performance-critical. (This function appears to have been modelled
after queryDerivationOutputs(), which exists only to make the garbage
collector faster.)
---
 src/libexpr/primops.cc          |  2 +-
 src/libstore/build.cc           |  3 ---
 src/libstore/daemon.cc          |  3 +--
 src/libstore/derivations.cc     |  9 +++++++++
 src/libstore/derivations.hh     |  2 ++
 src/libstore/local-store.cc     | 17 -----------------
 src/libstore/local-store.hh     |  2 --
 src/libstore/remote-store.cc    |  9 ---------
 src/libstore/remote-store.hh    |  2 --
 src/libstore/store-api.hh       |  4 ----
 src/libstore/worker-protocol.hh |  2 +-
 11 files changed, 14 insertions(+), 41 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d458ab2721f..9efb8eae5b6 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -688,7 +688,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             for (auto & j : refs) {
                 drv.inputSrcs.insert(j.clone());
                 if (j.isDerivation())
-                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
+                    drv.inputDrvs[j.clone()] = readDerivation(*state.store, state.store->toRealPath(j)).outputNames();
             }
         }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index bdf03ff944f..3ab6220e315 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2723,9 +2723,6 @@ struct RestrictedStore : public LocalFSStore
     StorePathSet queryDerivationOutputs(const StorePath & path) override
     { throw Error("queryDerivationOutputs"); }
 
-    StringSet queryDerivationOutputNames(const StorePath & path) override
-    { throw Error("queryDerivationOutputNames"); }
-
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 5cff170dd93..dc201557943 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -329,8 +329,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        StringSet names;
-        names = store->queryDerivationOutputNames(path);
+        auto names = readDerivation(*store, store->toRealPath(path)).outputNames();
         logger->stopWork();
         to << names;
         break;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index c68e7b16b1a..6de91ec973d 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -410,6 +410,15 @@ StorePathSet BasicDerivation::outputPaths() const
 }
 
 
+StringSet BasicDerivation::outputNames() const
+{
+    StringSet names;
+    for (auto & i : outputs)
+        names.insert(i.first);
+    return names;
+}
+
+
 Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 {
     drv.outputs.clear();
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index b1224b93bbb..88aed66bf29 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -58,6 +58,8 @@ struct BasicDerivation
     /* Return the output paths of a derivation. */
     StorePathSet outputPaths() const;
 
+    /* Return the output names of a derivation. */
+    StringSet outputNames() const;
 };
 
 struct Derivation : BasicDerivation
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1c3795eb1c0..e379db42605 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -785,23 +785,6 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
-{
-    return retrySQLite<StringSet>([&]() {
-        auto state(_state.lock());
-
-        auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
-            (queryValidPathId(*state, path)));
-
-        StringSet outputNames;
-        while (useQueryDerivationOutputs.next())
-            outputNames.insert(useQueryDerivationOutputs.getStr(0));
-
-        return outputNames;
-    });
-}
-
-
 std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
 {
     if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index c1e75390c5d..e17cc45aead 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -135,8 +135,6 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const StorePath & path) override;
-
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5c36693e699..735f59a91ad 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 }
 
 
-PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path)
-{
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputNames << printStorePath(path);
-    conn.processStderr();
-    return readStrings<PathSet>(conn->from);
-}
-
-
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 3c86b4524eb..80c8e9f1168 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -51,8 +51,6 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    StringSet queryDerivationOutputNames(const StorePath & path) override;
-
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index b1e25fc7d66..5ba17e0bc9d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -430,10 +430,6 @@ public:
     virtual StorePathSet queryDerivationOutputs(const StorePath & path)
     { unsupported("queryDerivationOutputs"); }
 
-    /* Query the output names of the derivation denoted by `path'. */
-    virtual StringSet queryDerivationOutputNames(const StorePath & path)
-    { unsupported("queryDerivationOutputNames"); }
-
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */
     virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 857d54d993f..ac42457fc41 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -36,7 +36,7 @@ typedef enum {
     wopClearFailedPaths = 25,
     wopQueryPathInfo = 26,
     wopImportPaths = 27, // obsolete
-    wopQueryDerivationOutputNames = 28,
+    wopQueryDerivationOutputNames = 28, // obsolete
     wopQueryPathFromHashPart = 29,
     wopQuerySubstitutablePathInfos = 30,
     wopQueryValidPaths = 31,

From 9f736dd89d064718a378703a9557de56fdd1a734 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jun 2020 13:04:52 +0200
Subject: [PATCH 2180/2196] Add Store::readDerivation() convenience function

---
 src/build-remote/build-remote.cc | 2 +-
 src/libexpr/primops.cc           | 2 +-
 src/libstore/daemon.cc           | 2 +-
 src/libstore/derivations.cc      | 8 +++++++-
 src/libstore/local-store.cc      | 5 ++---
 src/libstore/store-api.hh        | 3 +++
 src/nix/show-derivation.cc       | 6 ++----
 7 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 00340b7872a..d2ea6c956b8 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -241,7 +241,7 @@ static int _main(int argc, char * * argv)
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        auto drv = store->readDerivation(*drvPath);
         drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9efb8eae5b6..3a27527a04d 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -688,7 +688,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             for (auto & j : refs) {
                 drv.inputSrcs.insert(j.clone());
                 if (j.isDerivation())
-                    drv.inputDrvs[j.clone()] = readDerivation(*state.store, state.store->toRealPath(j)).outputNames();
+                    drv.inputDrvs[j.clone()] = state.store->readDerivation(j).outputNames();
             }
         }
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index dc201557943..877c70a03fc 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -329,7 +329,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputNames: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        auto names = readDerivation(*store, store->toRealPath(path)).outputNames();
+        auto names = store->readDerivation(path).outputNames();
         logger->stopWork();
         to << names;
         break;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 6de91ec973d..e268c65ff61 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -204,6 +204,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
 Derivation Store::derivationFromPath(const StorePath & drvPath)
 {
     ensurePath(drvPath);
+    return readDerivation(drvPath);
+}
+
+
+Derivation Store::readDerivation(const StorePath & drvPath)
+{
     auto accessor = getFSAccessor();
     try {
         return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
@@ -378,7 +384,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
         if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
             h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
-                readDerivation(store, store.toRealPath(i.first)), false)).first;
+                store.readDerivation(i.first), false)).first;
         }
         inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e379db42605..c2c45b4c277 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -595,7 +595,7 @@ uint64_t LocalStore::addValidPath(State & state,
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
+        auto drv = readDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -911,8 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & i : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                checkDerivationOutputs(i.path,
-                    readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
+                checkDerivationOutputs(i.path, readDerivation(i.path));
             }
 
         /* Do a topological sort of the paths.  This will throw an
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5ba17e0bc9d..5ef5063266c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -583,6 +583,9 @@ public:
        ensurePath(). */
     Derivation derivationFromPath(const StorePath & drvPath);
 
+    /* Read a derivation (which must already be valid). */
+    Derivation readDerivation(const StorePath & drvPath);
+
     /* Place in `out' the set of all store paths in the file system
        closure of `storePath'; that is, all paths than can be directly
        or indirectly reached from it.  `out' is not cleared.  If
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 22c569f3cdd..2d31894c271 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -61,11 +61,9 @@ struct CmdShowDerivation : InstallablesCommand
         for (auto & drvPath : drvPaths) {
             if (!drvPath.isDerivation()) continue;
 
-            auto drvPathS = store->printStorePath(drvPath);
+            auto drvObj(jsonRoot.object(store->printStorePath(drvPath)));
 
-            auto drvObj(jsonRoot.object(drvPathS));
-
-            auto drv = readDerivation(*store, drvPathS);
+            auto drv = store->readDerivation(drvPath);
 
             {
                 auto outputsObj(drvObj.object("outputs"));

From b260c9ee0325d7a4c78d91e47bd6e2afbde16e28 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 10:09:42 -0500
Subject: [PATCH 2181/2196] Add newHashAllowEmpty helper function

This replaces the copy&paste with a helper function in hash.hh.
---
 src/libexpr/primops.cc           | 18 ++++--------------
 src/libexpr/primops/fetchTree.cc | 11 +++--------
 src/libfetchers/fetchers.cc      | 11 +++--------
 src/libfetchers/tarball.cc       | 10 ++--------
 src/libutil/hash.cc              | 10 ++++++++++
 src/libutil/hash.hh              |  2 ++
 6 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index df3d4a45917..083c7d39809 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -720,12 +720,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
 
-        Hash h;
-        if (outputHash->empty()) {
-            h = Hash(ht);
-            printError("warning: found empty hash, assuming you wanted '%s'", h.to_string());
-        } else
-            h = Hash(*outputHash, ht);
+        Hash h = newHashAllowEmpty(*outputHash, ht);
 
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
@@ -1131,14 +1126,9 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             filterFun = attr.value;
         } else if (n == "recursive")
             method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
-        else if (n == "sha256") {
-            auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos);
-            if (hashStr == "") {
-                expectedHash = Hash(htSHA256);
-                printError("warning: found empty hash, assuming you wanted '%s'", expectedHash.to_string());
-            } else
-                expectedHash = Hash(hashStr, htSHA256);
-        } else
+        else if (n == "sha256")
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+        else
             throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
     }
     if (path.empty())
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 745f65adfcb..1464aa8b4c4 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -102,14 +102,9 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             string n(attr.name);
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "sha256") {
-                auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos);
-                if (hashStr == "") {
-                    expectedHash = Hash(htSHA256);
-                    printError("warning: found empty hash, assuming you wanted '%s'", expectedHash->to_string());
-                } else
-                    expectedHash = Hash(hashStr, htSHA256);
-            } else if (n == "name")
+            else if (n == "sha256")
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+            else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to '%s', at %s",
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 988a8dc6923..91b897e62c6 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -34,14 +34,9 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
     for (auto & inputScheme : *inputSchemes) {
         auto res = inputScheme->inputFromAttrs(attrs2);
         if (res) {
-            if (auto narHash = maybeGetStrAttr(attrs, "narHash")) {
-                if (narHash->empty()) {
-                    res->narHash = Hash(htUnknown);
-                    printError("warning: found empty hash, assuming you wanted '%s'", res->narHash->to_string());
-                } else
-                    // FIXME: require SRI hash.
-                    res->narHash = Hash(*narHash);
-            }
+            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
+                // FIXME: require SRI hash.
+                res->narHash = newHashAllowEmpty(*narHash, htUnknown);
             return res;
         }
     }
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index e4dafec0b48..937f86bc688 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -263,14 +263,8 @@ struct TarballInputScheme : InputScheme
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto hash = maybeGetStrAttr(attrs, "hash")) {
-            if (hash->empty()) {
-                input->hash = Hash(htUnknown);
-                printError("warning: found empty hash, assuming you wanted '%s'", input->hash->to_string());
-            } else
-                // FIXME: require SRI hash.
-                input->hash = Hash(*hash);
-        }
+        if (auto hash = maybeGetStrAttr(attrs, "hash"))
+            input->hash = newHashAllowEmpty(*hash, htUnknown);
 
         return input;
     }
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 7caee1da7f8..2c8085e4291 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -205,6 +205,16 @@ Hash::Hash(const std::string & s, HashType type)
         throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
 }
 
+Hash newHashAllowEmpty(std::string hashStr, HashType ht)
+{
+    if (hashStr.empty())
+    {
+        Hash h(ht);
+        warn("found empty hash, assuming you wanted '%s'", h.to_string());
+    } else
+        return Hash(hashStr, ht);
+}
+
 
 union Ctx
 {
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index ea9fca3e74f..449cb1b8656 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -94,6 +94,8 @@ struct Hash
     }
 };
 
+/* Helper that defaults empty hashes to the 0 hash. */
+Hash newHashAllowEmpty(std::string hashStr, HashType ht);
 
 /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
 string printHash16or32(const Hash & hash);

From ea0d29d99a400c328fa0ca05ba5e639351673ebc Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Fri, 12 Jun 2020 10:18:27 -0500
Subject: [PATCH 2182/2196] Provide base argument to to_string

---
 src/libutil/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 2c8085e4291..3f6d4c0c903 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -210,7 +210,7 @@ Hash newHashAllowEmpty(std::string hashStr, HashType ht)
     if (hashStr.empty())
     {
         Hash h(ht);
-        warn("found empty hash, assuming you wanted '%s'", h.to_string());
+        warn("found empty hash, assuming you wanted '%s'", h.to_string(SRI));
     } else
         return Hash(hashStr, ht);
 }

From 2853ba4ab26c254d564aee9e75fe8f9f664b94fc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jun 2020 19:00:48 +0200
Subject: [PATCH 2183/2196] Fix build

---
 src/libutil/hash.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 3f6d4c0c903..c935f05eb53 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -207,10 +207,10 @@ Hash::Hash(const std::string & s, HashType type)
 
 Hash newHashAllowEmpty(std::string hashStr, HashType ht)
 {
-    if (hashStr.empty())
-    {
+    if (hashStr.empty()) {
         Hash h(ht);
-        warn("found empty hash, assuming you wanted '%s'", h.to_string(SRI));
+        warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
+        return h;
     } else
         return Hash(hashStr, ht);
 }

From f6f01416b71f239c0d9594f5c6201dfd208fe8c4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Jun 2020 21:12:36 +0000
Subject: [PATCH 2184/2196] Use `std::string_view` in a few more places

---
 src/libfetchers/github.cc | 2 +-
 src/libstore/store-api.cc | 4 ++--
 src/libutil/hash.cc       | 4 ++--
 src/libutil/hash.hh       | 2 +-
 src/libutil/util.cc       | 6 +++---
 src/libutil/util.hh       | 6 +++---
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8675a5a662d..0bee1d6b3f9 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -76,7 +76,7 @@ struct GitHubInput : Input
                 readFile(
                     store->toRealPath(
                         downloadFile(store, url, "source", false).storePath)));
-            rev = Hash(json["sha"], htSHA1);
+            rev = Hash(std::string { json["sha"] }, htSHA1);
             debug("HEAD revision for '%s' is %s", url, rev->gitRev());
         }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0645fca8416..f6901bf422c 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -779,7 +779,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     };
 
     if (hasPrefix(ca, "text:")) {
-        Hash hash(std::string(ca, 5));
+        Hash hash(ca.substr(5));
         if (store.makeTextPath(path.name(), hash, references) == path)
             return true;
         else
@@ -788,7 +788,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
 
     else if (hasPrefix(ca, "fixed:")) {
         FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
-        Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6));
+        Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
         auto refs = cloneStorePathSet(references);
         bool hasSelfReference = false;
         if (refs.count(path)) {
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index c935f05eb53..460d479a32f 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -125,7 +125,7 @@ std::string Hash::to_string(Base base, bool includeType) const
 }
 
 
-Hash::Hash(const std::string & s, HashType type)
+Hash::Hash(std::string_view s, HashType type)
     : type(type)
 {
     size_t pos = 0;
@@ -194,7 +194,7 @@ Hash::Hash(const std::string & s, HashType type)
     }
 
     else if (isSRI || size == base64Len()) {
-        auto d = base64Decode(std::string(s, pos));
+        auto d = base64Decode(s.substr(pos));
         if (d.size() != hashSize)
             throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
         assert(hashSize);
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 5080539b2d8..180fb76334d 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -42,7 +42,7 @@ struct Hash
        Subresource Integrity hash expression). If the 'type' argument
        is htUnknown, then the hash type must be specified in the
        string. */
-    Hash(const std::string & s, HashType type = htUnknown);
+    Hash(std::string_view s, HashType type = htUnknown);
 
     void init();
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index e0a99152b56..b66447e08cc 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1314,7 +1314,7 @@ bool statusOk(int status)
 }
 
 
-bool hasPrefix(const string & s, const string & prefix)
+bool hasPrefix(std::string_view s, std::string_view prefix)
 {
     return s.compare(0, prefix.size(), prefix) == 0;
 }
@@ -1408,7 +1408,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
 static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 
-string base64Encode(const string & s)
+string base64Encode(std::string_view s)
 {
     string res;
     int data = 0, nbits = 0;
@@ -1429,7 +1429,7 @@ string base64Encode(const string & s)
 }
 
 
-string base64Decode(const string & s)
+string base64Decode(std::string_view s)
 {
     bool init = false;
     char decode[256];
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 4b117f9bc05..c95232317c3 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -416,7 +416,7 @@ template<class N> bool string2Float(const string & s, N & n)
 
 
 /* Return true iff `s' starts with `prefix'. */
-bool hasPrefix(const string & s, const string & prefix);
+bool hasPrefix(std::string_view s, std::string_view prefix);
 
 
 /* Return true iff `s' ends in `suffix'. */
@@ -455,8 +455,8 @@ std::string filterANSIEscapes(const std::string & s,
 
 
 /* Base64 encoding/decoding. */
-string base64Encode(const string & s);
-string base64Decode(const string & s);
+string base64Encode(std::string_view s);
+string base64Decode(std::string_view s);
 
 
 /* Get a value for the specified key from an associate container. */

From 340d0b055afb02ec7292d6cd9dd1661640158210 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 Jun 2020 14:57:24 +0200
Subject: [PATCH 2185/2196] upload-release.pl: Fix nix-fallback-paths.nix
 generation

---
 maintainers/upload-release.pl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index cb584d4278b..baefe0f123b 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -142,8 +142,12 @@ sub downloadFile {
 sub getStorePath {
     my ($jobName) = @_;
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build";
-    return $buildInfo->{buildproducts}->{1}->{path};
+    for my $product (values %{$buildInfo->{buildproducts}}) {
+        next unless $product->{type} eq "nix-build";
+        next if $product->{path} =~ /[a-z]+$/;
+        return $product->{path};
+    }
+    die;
 }
 
 write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",

From fd64e4fb96f814440dc337ce664cdbd22e0eabb2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 13:50:33 +0200
Subject: [PATCH 2186/2196] Disambiguate BaseError(Args) constructor

This means that 'throw Error({ ... ErrorInfo ... })' now works.
---
 src/libexpr/eval.cc  | 9 ++++-----
 src/libutil/types.hh | 3 ---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7bf25ea17cf..75e6a23a1b9 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -506,11 +506,10 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
 
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
 {
-    throw EvalError(
-        ErrorInfo { 
-            .hint = hintfmt(s, s2),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw EvalError({
+        .hint = hintfmt(s, s2),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 89ae108f9cb..3af485fa0fe 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -1,6 +1,5 @@
 #pragma once
 
-
 #include "ref.hh"
 
 #include <list>
@@ -25,7 +24,6 @@ typedef string Path;
 typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
-
 /* Helper class to run code at startup. */
 template<typename T>
 struct OnStartup
@@ -33,5 +31,4 @@ struct OnStartup
     OnStartup(T && t) { t(); }
 };
 
-
 }

From 1fb762d11fadc659ef41b79eaddddcce5fbbc192 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 14:06:58 +0200
Subject: [PATCH 2187/2196] Get rid of explicit ErrorInfo constructors

---
 src/libexpr/attr-set.hh               |   9 +-
 src/libexpr/eval-inline.hh            |  18 +-
 src/libexpr/eval.cc                   |  65 ++--
 src/libexpr/nixexpr.cc                |   9 +-
 src/libexpr/nixexpr.hh                |   9 +-
 src/libexpr/parser.y                  |  86 +++--
 src/libexpr/primops.cc                | 442 ++++++++++++--------------
 src/libexpr/primops/context.cc        |  27 +-
 src/libexpr/primops/fetchGit.cc       |  18 +-
 src/libexpr/primops/fetchMercurial.cc |  18 +-
 src/libexpr/primops/fetchTree.cc      |  28 +-
 src/libexpr/primops/fromTOML.cc       |   9 +-
 src/libstore/builtins/buildenv.cc     |  14 +-
 src/libstore/gc.cc                    |  11 +-
 src/libstore/sqlite.cc                |   8 +-
 src/libutil/error.hh                  |  14 +-
 src/libutil/serialise.cc              |   4 +-
 src/nix-build/nix-build.cc            |  11 +-
 src/nix/upgrade-nix.cc                |   7 +-
 19 files changed, 360 insertions(+), 447 deletions(-)

diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index f5651891f04..c601d09c2ad 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -76,11 +76,10 @@ public:
     {
         auto a = get(name);
         if (!a)
-            throw Error(
-                ErrorInfo { 
-                    .hint = hintfmt("attribute '%s' missing", name),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw Error({
+                .hint = hintfmt("attribute '%s' missing", name),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
         return *a;
     }
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index eee49e02e3e..3d544c903fb 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -9,11 +9,10 @@ namespace nix {
 
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
-    throw EvalError(
-        ErrorInfo { 
-            .hint = hintfmt(s),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw EvalError({
+        .hint = hintfmt(s),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
@@ -24,11 +23,10 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
 
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
-    throw TypeError(
-        ErrorInfo { 
-            .hint = hintfmt(s, showType(v)),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw TypeError({
+        .hint = hintfmt(s, showType(v)),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 75e6a23a1b9..8e71db2b872 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -519,30 +519,27 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
 
 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
 {
-    throw EvalError(
-        ErrorInfo { 
-            .hint = hintfmt(s, s2, s3),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw EvalError({
+        .hint = hintfmt(s, s2, s3),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
 {
     // p1 is where the error occurred; p2 is a position mentioned in the message.
-    throw EvalError(
-        ErrorInfo { 
-            .hint = hintfmt(s, sym, p2),
-            .nixCode = NixCode { .errPos = p1 }
-        });
+    throw EvalError({
+        .hint = hintfmt(s, sym, p2),
+        .nixCode = NixCode { .errPos = p1 }
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
-    throw TypeError(
-        ErrorInfo { 
-            .hint = hintfmt(s),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw TypeError({
+        .hint = hintfmt(s),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
@@ -552,29 +549,26 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
 
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
-    throw TypeError(
-        ErrorInfo { 
-            .hint = hintfmt(s, fun.showNamePos(), s2),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw TypeError({
+        .hint = hintfmt(s, fun.showNamePos(), s2),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
 {
-    throw AssertionError(
-        ErrorInfo { 
-            .hint = hintfmt(s, s1),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw AssertionError({
+        .hint = hintfmt(s, s1),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
 {
-    throw UndefinedVarError(
-        ErrorInfo { 
-            .hint = hintfmt(s, s1),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw UndefinedVarError({
+        .hint = hintfmt(s, s1),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@@ -1593,7 +1587,7 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
     string s = forceString(v, pos);
     if (v.string.context) {
         if (pos)
-            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", 
+            throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
                 v.string.s, v.string.context[0]);
         else
             throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
@@ -1919,11 +1913,10 @@ void EvalState::printStats()
 
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
-    throw TypeError(
-        ErrorInfo { 
-            .hint = hintfmt("cannot coerce %1% to a string", showType()),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw TypeError({
+        .hint = hintfmt("cannot coerce %1% to a string", showType()),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 91a50830542..5b2dd9751e3 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -268,11 +268,10 @@ void ExprVar::bindVars(const StaticEnv & env)
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
     if (withLevel == -1) 
-        throw UndefinedVarError(
-            ErrorInfo {
-                .hint = hintfmt("undefined variable '%1%'", name),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw UndefinedVarError({
+            .hint = hintfmt("undefined variable '%1%'", name),
+            .nixCode = NixCode { .errPos = pos }
+        });
     fromWith = true;
     this->level = withLevel;
 }
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 47d0e85ec7f..ec6fd3190be 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -236,11 +236,10 @@ struct ExprLambda : Expr
         : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError(
-                ErrorInfo {
-                    .hint = hintfmt("duplicate formal function argument '%1%'", arg),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw ParseError({
+                .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+                .nixCode = NixCode { .errPos = pos }
+            });
     };
     void setName(Symbol & name);
     string showNamePos() const;
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 0417a3c217f..a1188dec03e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -64,23 +64,20 @@ namespace nix {
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(
-        ErrorInfo {
-            .hint = hintfmt("attribute '%1%' already defined at %2%",
-                showAttrPath(attrPath), prevPos),
-            .nixCode = NixCode { .errPos = pos },
-        });
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%",
+            showAttrPath(attrPath), prevPos),
+        .nixCode = NixCode { .errPos = pos },
+    });
 }
 
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    throw ParseError(
-        ErrorInfo {
-            .hint = hintfmt("attribute '%1%' already defined at %2%",
-                attr, prevPos),
-            .nixCode = NixCode { .errPos = pos },
-        });
+    throw ParseError({
+        .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+        .nixCode = NixCode { .errPos = pos },
+    });
 }
 
 
@@ -148,12 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
     if (!formals->argNames.insert(formal.name).second)
-        throw ParseError(
-            ErrorInfo {
-                .hint = hintfmt("duplicate formal function argument '%1%'",
-                    formal.name),
-                .nixCode = NixCode { .errPos = pos },
-            });
+        throw ParseError({
+            .hint = hintfmt("duplicate formal function argument '%1%'",
+                formal.name),
+            .nixCode = NixCode { .errPos = pos },
+        });
     formals->formals.push_front(formal);
 }
 
@@ -261,10 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = ErrorInfo { 
+    data->error = {
         .hint = hintfmt(error),
         .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
-        };
+    };
 }
 
 
@@ -341,11 +337,10 @@ expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
     { if (!$2->dynamicAttrs.empty())
-        throw ParseError(
-            ErrorInfo {
-                .hint = hintfmt("dynamic attributes not allowed in let"),
-                .nixCode = NixCode { .errPos = CUR_POS },
-            });
+        throw ParseError({
+            .hint = hintfmt("dynamic attributes not allowed in let"),
+            .nixCode = NixCode { .errPos = CUR_POS },
+        });
       $$ = new ExprLet($2, $4);
     }
   | expr_if
@@ -422,11 +417,10 @@ expr_simple
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
       if (noURLLiterals)
-          throw ParseError(
-              ErrorInfo { 
-                  .hint = hintfmt("URL literals are disabled"),
-                  .nixCode = NixCode { .errPos = CUR_POS }
-              });
+          throw ParseError({
+              .hint = hintfmt("URL literals are disabled"),
+              .nixCode = NixCode { .errPos = CUR_POS }
+          });
       $$ = new ExprString(data->symbols.create($1));
   }
   | '(' expr ')' { $$ = $2; }
@@ -496,11 +490,10 @@ attrs
           $$->push_back(AttrName(str->s));
           delete str;
       } else
-          throw ParseError(
-              ErrorInfo {
-                  .hint = hintfmt("dynamic attributes not allowed in inherit"),
-                  .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
-              });
+          throw ParseError({
+              .hint = hintfmt("dynamic attributes not allowed in inherit"),
+              .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+          });
     }
   | { $$ = new AttrPath; }
   ;
@@ -695,11 +688,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
-    throw ThrownError(
-        ErrorInfo { 
-            .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path),
-            .nixCode = NixCode { .errPos = pos }
-        });
+    throw ThrownError({
+        .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path),
+        .nixCode = NixCode { .errPos = pos }
+    });
 }
 
 
@@ -715,10 +707,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, store->toRealPath(fetchers::downloadTarball(
                         store, resolveUri(elem.second), "source", false).storePath) };
         } catch (FileTransferError & e) {
-            logWarning(
-                ErrorInfo { 
-                    .name = "Entry download",
-                    .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+            logWarning({
+                .name = "Entry download",
+                .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
             });
             res = { false, "" };
         }
@@ -727,10 +718,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
         if (pathExists(path))
             res = { true, path };
         else {
-            logWarning(
-                ErrorInfo { 
-                    .name = "Entry not found",
-                    .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+            logWarning({
+                .name = "Entry not found",
+                .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 23ab7dec656..62e5163c902 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -93,12 +93,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid",
-                        path, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -174,13 +172,12 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt(
-                    "cannot import '%1%', since path '%2%' is not valid",
-                    path, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot import '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 
     path = state.checkSourcePath(path);
@@ -215,11 +212,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     auto elems = args[0]->listElems();
     auto count = args[0]->listSize();
     if (count == 0) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("at least one argument to 'exec' required"),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("at least one argument to 'exec' required"),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
@@ -230,12 +226,12 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
-                    program, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });}
+        throw EvalError({
+            .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+                program, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
+    }
 
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
@@ -386,11 +382,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator startSet =
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("attribute 'startSet' required"),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("attribute 'startSet' required"),
+            .nixCode = NixCode { .errPos = pos }
+        });
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
@@ -401,11 +396,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("attribute 'operator' required"),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("attribute 'operator' required"),
+            .nixCode = NixCode { .errPos = pos }
+        });
     state.forceValue(*op->value, pos);
 
     /* Construct the closure by applying the operator to element of
@@ -424,11 +418,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         Bindings::iterator key =
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("attribute 'key' required"),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("attribute 'key' required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
         state.forceValue(*key->value, pos);
 
         if (!doneKeys.insert(key->value).second) continue;
@@ -560,11 +553,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Figure out the name first (for stack backtraces). */
     Bindings::iterator attr = args[0]->attrs->find(state.sName);
     if (attr == args[0]->attrs->end())
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("required attribute 'name' missing"),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("required attribute 'name' missing"),
+            .nixCode = NixCode { .errPos = pos }
+        });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
@@ -607,42 +599,38 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         auto handleHashMode = [&](const std::string & s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else 
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
-                        .nixCode = NixCode { .errPos = posDrvName }
-                    });
+            else
+                throw EvalError({
+                    .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+                    .nixCode = NixCode { .errPos = posDrvName }
+                });
         };
 
         auto handleOutputs = [&](const Strings & ss) {
             outputs.clear();
             for (auto & j : ss) {
                 if (outputs.find(j) != outputs.end())
-                    throw EvalError(
-                        ErrorInfo { 
-                            .hint = hintfmt("duplicate derivation output '%1%'", j),
-                            .nixCode = NixCode { .errPos = posDrvName }
-                        });
+                    throw EvalError({
+                        .hint = hintfmt("duplicate derivation output '%1%'", j),
+                        .nixCode = NixCode { .errPos = posDrvName }
+                    });
                 /* !!! Check whether j is a valid attribute
                    name. */
                 /* Derivations cannot be named ‘drv’, because
                    then we'd have an attribute ‘drvPath’ in
                    the resulting set. */
                 if (j == "drv")
-                    throw EvalError(
-                        ErrorInfo { 
-                            .hint = hintfmt("invalid derivation output name 'drv'" ),
-                            .nixCode = NixCode { .errPos = posDrvName }
-                        });
+                    throw EvalError({
+                        .hint = hintfmt("invalid derivation output name 'drv'" ),
+                        .nixCode = NixCode { .errPos = posDrvName }
+                    });
                 outputs.insert(j);
             }
             if (outputs.empty())
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("derivation cannot have an empty set of outputs"),
-                        .nixCode = NixCode { .errPos = posDrvName }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("derivation cannot have an empty set of outputs"),
+                    .nixCode = NixCode { .errPos = posDrvName }
+                });
         };
 
         try {
@@ -754,35 +742,31 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     /* Do we have all required attributes? */
     if (drv.builder == "")
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("required attribute 'builder' missing"),
-                .nixCode = NixCode { .errPos = posDrvName }
-            });
+        throw EvalError({
+            .hint = hintfmt("required attribute 'builder' missing"),
+            .nixCode = NixCode { .errPos = posDrvName }
+        });
 
     if (drv.platform == "")
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("required attribute 'system' missing"),
-                .nixCode = NixCode { .errPos = posDrvName }
-            });
+        throw EvalError({
+            .hint = hintfmt("required attribute 'system' missing"),
+            .nixCode = NixCode { .errPos = posDrvName }
+        });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
-                .nixCode = NixCode { .errPos = posDrvName }
-            });
+        throw EvalError({
+            .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+            .nixCode = NixCode { .errPos = posDrvName }
+        });
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
-            throw Error(
-                ErrorInfo { 
-                    .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
-                    .nixCode = NixCode { .errPos = posDrvName }
-                });
+            throw Error({
+                .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+                .nixCode = NixCode { .errPos = posDrvName }
+            });
 
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
 
@@ -887,11 +871,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
        e.g. nix-push does the right thing. */
     if (!state.store->isStorePath(path)) path = canonPath(path, true);
     if (!state.store->isInStore(path))
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("path '%1%' is not in the Nix store", path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("path '%1%' is not in the Nix store", path),
+            .nixCode = NixCode { .errPos = pos }
+        });
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
         state.store->ensurePath(state.store->parseStorePath(path2));
@@ -907,13 +890,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt(
-                    "cannot check the existence of '%1%', since path '%2%' is not valid", 
-                    path, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt(
+                "cannot check the existence of '%1%', since path '%2%' is not valid",
+                path, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 
     try {
@@ -956,13 +938,11 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid"
-                     , path, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });
- }
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
+    }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
     if (s.find((char) 0) != string::npos)
         throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path);
@@ -989,11 +969,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("attribute 'path' missing"),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("attribute 'path' missing"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
         PathSet context;
         string path = state.coerceToString(pos, *i->value, context, false, false);
@@ -1001,12 +980,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         try {
             state.realiseContext(context);
         } catch (InvalidPathError & e) {
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid",
-                        path, e.path),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
+                .nixCode = NixCode { .errPos = pos }
+            });
         }
 
         searchPath.emplace_back(prefix, path);
@@ -1023,11 +1000,10 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-      throw Error(
-          ErrorInfo {
-              .hint = hintfmt("unknown hash type '%1%'", type),
-              .nixCode = NixCode { .errPos = pos }
-          });
+      throw Error({
+          .hint = hintfmt("unknown hash type '%1%'", type),
+          .nixCode = NixCode { .errPos = pos }
+      });
 
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
@@ -1043,12 +1019,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     try {
         state.realiseContext(ctx);
     } catch (InvalidPathError & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid",
-                        path, e.path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 
     DirEntries entries = readDirectory(state.checkSourcePath(path));
@@ -1118,15 +1092,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 
     for (auto path : context) {
         if (path.at(0) != '/')
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt(
-                        "in 'toFile': the file named '%1%' must not contain a reference "
-                        "to a derivation but contains (%2%)",
-                        name,
-                        path),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError( {
+                .hint = hintfmt(
+                    "in 'toFile': the file named '%1%' must not contain a reference "
+                    "to a derivation but contains (%2%)",
+                    name, path),
+                .nixCode = NixCode { .errPos = pos }
+            });
         refs.insert(state.store->parseStorePath(path));
     }
 
@@ -1194,21 +1166,19 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
     if (!context.empty())
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(
-            ErrorInfo { 
-                .hint = hintfmt(
-                    "first argument in call to 'filterSource' is not a function but %1%", 
-                    showType(*args[0])),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw TypeError({
+            .hint = hintfmt(
+                "first argument in call to 'filterSource' is not a function but %1%",
+                showType(*args[0])),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
 }
@@ -1228,12 +1198,10 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             PathSet context;
             path = state.coerceToPath(*attr.pos, *attr.value, context);
             if (!context.empty())
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("string '%1%' cannot refer to other paths", 
-                            path),
-                        .nixCode = NixCode { .errPos = *attr.pos }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
@@ -1244,19 +1212,16 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else if (n == "sha256")
             expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
         else
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("unsupported argument '%1%' to 'addPath'",
-                        attr.name),
-                    .nixCode = NixCode { .errPos = *attr.pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
+                .nixCode = NixCode { .errPos = *attr.pos }
+            });
     }
     if (path.empty())
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("'path' required"),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("'path' required"),
+            .nixCode = NixCode { .errPos = pos }
+        });
     if (name.empty())
         name = baseNameOf(path);
 
@@ -1314,11 +1279,10 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     // !!! Should we create a symbol here or just do a lookup?
     Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
     if (i == args[1]->attrs->end())
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("attribute '%1%' missing", attr),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("attribute '%1%' missing", attr),
+            .nixCode = NixCode { .errPos = pos }
+        });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
@@ -1398,22 +1362,20 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
-            throw TypeError(
-                ErrorInfo { 
-                    .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), 
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw TypeError({
+                .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
+                .nixCode = NixCode { .errPos = pos }
+            });
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
         if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
-                throw TypeError(
-                    ErrorInfo { 
-                        .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), 
-                        .nixCode = NixCode { .errPos = pos }
-                    });
+                throw TypeError({
+                    .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
+                    .nixCode = NixCode { .errPos = pos }
+                });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
     }
@@ -1486,11 +1448,10 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
 {
     state.forceValue(*args[0], pos);
     if (args[0]->type != tLambda)
-        throw TypeError(
-            ErrorInfo { 
-                .hint = hintfmt("'functionArgs' requires a function"), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw TypeError({
+            .hint = hintfmt("'functionArgs' requires a function"),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     if (!args[0]->lambda.fun->matchAttrs) {
         state.mkAttrs(v, 0);
@@ -1543,11 +1504,10 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
 {
     state.forceList(list, pos);
     if (n < 0 || (unsigned int) n >= list.listSize())
-        throw Error(
-            ErrorInfo { 
-                .hint = hintfmt("list index %1% is out of bounds", n), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw Error({
+            .hint = hintfmt("list index %1% is out of bounds", n),
+            .nixCode = NixCode { .errPos = pos }
+        });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
 }
@@ -1574,11 +1534,10 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
 {
     state.forceList(*args[0], pos);
     if (args[0]->listSize() == 0)
-        throw Error(
-            ErrorInfo { 
-                .hint = hintfmt("'tail' called on an empty list"), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw Error({
+            .hint = hintfmt("'tail' called on an empty list"),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     state.mkList(v, args[0]->listSize() - 1);
     for (unsigned int n = 0; n < v.listSize(); ++n)
@@ -1720,12 +1679,10 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     auto len = state.forceInt(*args[1], pos);
 
     if (len < 0)
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("cannot create list of size %1%", len), 
-                .nixCode = NixCode { .errPos = pos }
-            });
-
+        throw EvalError({
+            .hint = hintfmt("cannot create list of size %1%", len),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     state.mkList(v, len);
 
@@ -1883,12 +1840,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     state.forceValue(*args[1], pos);
 
     NixFloat f2 = state.forceFloat(*args[1], pos);
-    if (f2 == 0) 
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("division by zero"), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+    if (f2 == 0)
+        throw EvalError({
+            .hint = hintfmt("division by zero"),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
@@ -1897,11 +1853,10 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         NixInt i2 = state.forceInt(*args[1], pos);
         /* Avoid division overflow as it might raise SIGFPE. */
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("overflow in integer division"), 
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("overflow in integer division"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
         mkInt(v, i1 / i2);
     }
@@ -1958,12 +1913,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     PathSet context;
     string s = state.coerceToString(pos, *args[2], context);
 
-    if (start < 0) 
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("negative start position in 'substring'"), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+    if (start < 0)
+        throw EvalError({
+            .hint = hintfmt("negative start position in 'substring'"),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
 }
@@ -1983,11 +1937,10 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     string type = state.forceStringNoCtx(*args[0], pos);
     HashType ht = parseHashType(type);
     if (ht == htUnknown)
-        throw Error(
-            ErrorInfo { 
-                .hint = hintfmt("unknown hash type '%1%'", type), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw Error({
+            .hint = hintfmt("unknown hash type '%1%'", type),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
@@ -2030,17 +1983,15 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .nixCode = NixCode { .errPos = pos }
+            });
         } else {
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("invalid regular expression '%s'", re),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .nixCode = NixCode { .errPos = pos }
+            });
         }
     }
 }
@@ -2105,17 +2056,15 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
     } catch (std::regex_error &e) {
         if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+                .nixCode = NixCode { .errPos = pos }
+            });
         } else {
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("invalid regular expression '%s'", re),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("invalid regular expression '%s'", re),
+                .nixCode = NixCode { .errPos = pos }
+            });
         }
     }
 }
@@ -2146,11 +2095,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*args[0], pos);
     state.forceList(*args[1], pos);
     if (args[0]->listSize() != args[1]->listSize())
-        throw EvalError(
-            ErrorInfo { 
-                .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), 
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
+            .nixCode = NixCode { .errPos = pos }
+        });
 
     vector<string> from;
     from.reserve(args[0]->listSize());
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 7f895fc0159..efa2e9576f7 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -146,11 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (auto & i : *args[1]->attrs) {
         if (!state.store->isStorePath(i.name))
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("Context key '%s' is not a store path", i.name),
-                    .nixCode = NixCode { .errPos = *i.pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("Context key '%s' is not a store path", i.name),
+                .nixCode = NixCode { .errPos = *i.pos }
+            });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
@@ -164,11 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError(
-                        ErrorInfo { 
-                            .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
-                            .nixCode = NixCode { .errPos = *i.pos }
-                        });
+                    throw EvalError({
+                        .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+                        .nixCode = NixCode { .errPos = *i.pos }
+                    });
                 }
                 context.insert("=" + string(i.name));
             }
@@ -178,11 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
-                        .nixCode = NixCode { .errPos = *i.pos }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+                    .nixCode = NixCode { .errPos = *i.pos }
+                });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
index 52826b56ccf..dd7229a3d54 100644
--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -35,19 +35,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else if (n == "submodules")
                 fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                        .nixCode = NixCode { .errPos = *attr.pos }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
         }
 
         if (url.empty())
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("'url' argument required"),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index bb008ba6b36..9bace8f8925 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -38,19 +38,17 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError(
-                    ErrorInfo { 
-                        .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
-                        .nixCode = NixCode { .errPos = *attr.pos }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
         }
 
         if (url.empty())
-            throw EvalError(
-                ErrorInfo { 
-                    .hint = hintfmt("'url' argument required"),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
     } else
         url = state.coerceToString(pos, *args[0], context, false, false);
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 62bc4f4336f..9be93710ad3 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -66,11 +66,10 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         }
 
         if (!attrs.count("type"))
-            throw Error(
-                ErrorInfo {
-                    .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                    .nixCode = NixCode { .errPos = pos }
-                    });
+            throw Error({
+                .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .nixCode = NixCode { .errPos = pos }
+            });
 
         input = fetchers::inputFromAttrs(attrs);
     } else
@@ -111,20 +110,17 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
             else
-                throw EvalError(
-                    ErrorInfo {
-                        .hint = hintfmt("unsupported argument '%s' to '%s'",
-                            attr.name, who),
-                        .nixCode = NixCode { .errPos = *attr.pos }
-                    });
+                throw EvalError({
+                    .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+                    .nixCode = NixCode { .errPos = *attr.pos }
+                });
             }
 
         if (!url)
-            throw EvalError(
-                ErrorInfo {
-                    .hint = hintfmt("'url' argument required"),
-                    .nixCode = NixCode { .errPos = pos }
-                });
+            throw EvalError({
+                .hint = hintfmt("'url' argument required"),
+                .nixCode = NixCode { .errPos = pos }
+            });
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 94806940130..7615d1379f7 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -81,11 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     try {
         visit(v, parser(tomlStream).parse());
     } catch (std::runtime_error & e) {
-        throw EvalError(
-            ErrorInfo {
-                .hint = hintfmt("while parsing a TOML string: %s", e.what()),
-                .nixCode = NixCode { .errPos = pos }
-            });
+        throw EvalError({
+            .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+            .nixCode = NixCode { .errPos = pos }
+        });
     }
 }
 
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 6c493ed770b..802fb87bceb 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -22,10 +22,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
         srcFiles = readDirectory(srcDir);
     } catch (SysError & e) {
         if (e.errNo == ENOTDIR) {
-            logWarning(
-                ErrorInfo { 
-                    .name = "Create links - directory",
-                    .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
+            logWarning({
+                .name = "Create links - directory",
+                .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
             });
             return;
         }
@@ -45,10 +44,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                 throw SysError("getting status of '%1%'", srcFile);
         } catch (SysError & e) {
             if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
-                logWarning(
-                    ErrorInfo { 
-                        .name = "Create links - skipping symlink",
-                        .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
+                logWarning({
+                    .name = "Create links - skipping symlink",
+                    .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
                 });
                 continue;
             }
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 92f9328c3b8..04e3849f7f4 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -129,12 +129,11 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
     if (settings.checkRootReachability) {
         auto roots = findRoots(false);
         if (roots[storePath.clone()].count(gcRoot) == 0)
-            logWarning(
-                ErrorInfo { 
-                    .name = "GC root",
-                    .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                "therefore, '%2%' might be removed by the garbage collector",
-                gcRoot, printStorePath(storePath))
+            logWarning({
+                .name = "GC root",
+                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
+                    "therefore, '%2%' might be removed by the garbage collector",
+                    gcRoot, printStorePath(storePath))
             });
     }
 
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 16cdb6619b4..76c822c4e9c 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -204,10 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e)
 
     if (now > lastWarned + 10) {
         lastWarned = now;
-        logWarning(
-            ErrorInfo { .name = "Sqlite busy",
-                        .hint = hintfmt(e.what())
-            });
+        logWarning({
+            .name = "Sqlite busy",
+            .hint = hintfmt(e.what())
+        });
     }
 
     /* Sleep for a while since retrying the transaction right away
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 8a48fa105b1..2ba3a3b338b 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -22,7 +22,7 @@
 
 namespace nix {
 
-/* 
+/*
 
 This file defines two main structs/classes used in nix error handling.
 
@@ -106,7 +106,7 @@ protected:
 
     mutable std::optional<string> what_;
     const string& calcWhat() const;
-    
+
 public:
     unsigned int status = 1; // exit status
 
@@ -119,9 +119,9 @@ public:
     { }
 
     template<typename... Args>
-    BaseError(const Args & ... args)
+    BaseError(const std::string & fs, const Args & ... args)
         : err { .level = lvlError,
-                .hint = hintfmt(args...)
+                .hint = hintfmt(fs, args...)
               }
     { }
 
@@ -131,7 +131,11 @@ public:
               }
     { }
 
-    BaseError(ErrorInfo e)
+    BaseError(ErrorInfo && e)
+        : err(std::move(e))
+    { }
+
+    BaseError(const ErrorInfo & e)
         : err(e)
     { }
 
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 35f7ee917cd..c8b71188fe0 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -52,10 +52,10 @@ size_t threshold = 256 * 1024 * 1024;
 
 static void warnLargeDump()
 {
-    logWarning(ErrorInfo {
+    logWarning({
         .name = "Large path",
         .description = "dumping very large path (> 256 MiB); this may run out of memory"
-        });
+    });
 }
 
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 591fff99980..da6f005a314 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -368,12 +368,11 @@ static void _main(int argc, char * * argv)
                 shell = drv->queryOutPath() + "/bin/bash";
 
             } catch (Error & e) {
-                logWarning(
-                    ErrorInfo {
-                        .name = "bashInteractive",
-                        .hint = hintfmt("%s; will use bash from your environment", 
-                            (e.info().hint ? e.info().hint->str() : ""))
-                    });
+                logWarning({
+                    .name = "bashInteractive",
+                    .hint = hintfmt("%s; will use bash from your environment",
+                        (e.info().hint ? e.info().hint->str() : ""))
+                });
                 shell = "bash";
             }
         }
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index fdf94e5a37d..a880bdae097 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -68,10 +68,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         if (dryRun) {
             stopProgressBar();
-            logWarning(
-                ErrorInfo { 
-                    .name = "Version update",
-                    .hint = hintfmt("would upgrade to version %s", version)
+            logWarning({
+                .name = "Version update",
+                .hint = hintfmt("would upgrade to version %s", version)
             });
             return;
         }

From e14e62fddde3b00ee82a8da29725f571ad8ecee1 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 14:12:39 +0200
Subject: [PATCH 2188/2196] Remove trailing whitespace

---
 src/build-remote/build-remote.cc              |  2 +-
 src/libexpr/attr-path.cc                      |  4 +--
 src/libexpr/lexer.l                           |  1 -
 src/libexpr/nixexpr.cc                        |  2 +-
 src/libmain/progress-bar.cc                   |  2 +-
 src/libstore/daemon.cc                        |  2 +-
 src/libstore/filetransfer.cc                  |  4 +--
 src/libstore/local-store.cc                   | 10 +++----
 src/libstore/optimise-store.cc                |  6 ++--
 src/libstore/store-api.cc                     |  2 +-
 src/libutil/error.cc                          |  2 +-
 src/libutil/logging.cc                        |  2 +-
 src/libutil/util.cc                           |  2 +-
 src/nix-daemon/nix-daemon.cc                  | 30 +++++++++----------
 src/nix-env/nix-env.cc                        |  8 ++---
 src/nix-store/nix-store.cc                    |  4 +--
 src/nix/verify.cc                             |  6 ++--
 .../resolve-system-dependencies.cc            |  8 ++---
 18 files changed, 48 insertions(+), 49 deletions(-)

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 49b35a4e915..e071174960a 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -200,7 +200,7 @@ static int _main(int argc, char * * argv)
 
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
-                    logError({ 
+                    logError({
                         .name = "Remote build",
                         .hint = hintfmt("cannot build on '%s': %s%s",
                             bestMachine->storeUri, e.what(),
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 9a9531a3f83..8980bc09d55 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -62,7 +62,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
-                    showType(*v)); 
+                    showType(*v));
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
@@ -79,7 +79,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a list but is %2%",
                     attrPath,
-                    showType(*v)); 
+                    showType(*v));
             if (attrIndex >= v->listSize())
                 throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 85376a08f1b..f6e83926bef 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -219,4 +219,3 @@ or          { return OR_KW; }
             }
 
 %%
-
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 5b2dd9751e3..b4b65883dbd 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -267,7 +267,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) 
+    if (withLevel == -1)
         throw UndefinedVarError({
             .hint = hintfmt("undefined variable '%1%'", name),
             .nixCode = NixCode { .errPos = pos }
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 20d9915a089..8d0421c2f92 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -133,7 +133,7 @@ class ProgressBar : public Logger
     {
         auto state(state_.lock());
 
-        std::stringstream oss; 
+        std::stringstream oss;
         oss << ei;
 
         log(*state, ei.level, oss.str());
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 172cfe3cb48..6207225161b 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -77,7 +77,7 @@ struct TunnelLogger : public Logger
     {
         if (ei.level > verbosity) return;
 
-        std::stringstream oss; 
+        std::stringstream oss;
         oss << ei;
 
         StringSink buf;
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 6ca3393abf7..601cb858ae2 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -599,9 +599,9 @@ struct curlFileTransfer : public FileTransfer
             workerThreadMain();
         } catch (nix::Interrupted & e) {
         } catch (std::exception & e) {
-            logError({ 
+            logError({
                 .name = "File transfer",
-                .hint = hintfmt("unexpected error in download thread: %s", 
+                .hint = hintfmt("unexpected error in download thread: %s",
                                 e.what())
             });
         }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5521b8633f6..f5c5bd9b786 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -87,7 +87,7 @@ LocalStore::LocalStore(const Params & params)
 
         struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
         if (!gr)
-            logError({ 
+            logError({
                 .name = "'build-users-group' not found",
                 .hint = hintfmt(
                     "warning: the group '%1%' specified in 'build-users-group' does not exist",
@@ -1223,7 +1223,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             Path linkPath = linksDir + "/" + link.name;
             string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
             if (hash != link.name) {
-                logError({ 
+                logError({
                     .name = "Invalid hash",
                     .hint = hintfmt(
                         "link '%s' was modified! expected hash '%s', got '%s'",
@@ -1261,7 +1261,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
-                    logError({ 
+                    logError({
                         .name = "Invalid hash - path modified",
                         .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
                         printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
@@ -1316,7 +1316,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
     if (!done.insert(pathS).second) return;
 
     if (!isStorePath(pathS)) {
-        logError({ 
+        logError({
             .name = "Nix path not found",
             .hint = hintfmt("path '%s' is not in the Nix store", pathS)
         });
@@ -1342,7 +1342,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
             auto state(_state.lock());
             invalidatePath(*state, path);
         } else {
-            logError({ 
+            logError({
                 .name = "Missing path with referrers",
                 .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
             });
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index d760d110c7e..b2b2412a318 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -130,7 +130,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        NixOS (example: $fontconfig/var/cache being modified).  Skip
        those files.  FIXME: check the modification time. */
     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
-        logWarning({ 
+        logWarning({
             .name = "Suspicious file",
             .hint = hintfmt("skipping suspicious writable file '%1%'", path)
         });
@@ -197,7 +197,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     if (st.st_size != stLink.st_size) {
-        logWarning({ 
+        logWarning({
             .name = "Corrupted link",
             .hint = hintfmt("removing corrupted link '%1%'", linkPath)
         });
@@ -235,7 +235,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
         if (unlink(tempLink.c_str()) == -1)
-            logError({ 
+            logError({
                 .name = "Unlink error",
                 .hint = hintfmt("unable to unlink '%1%'", tempLink)
             });
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 4642610105d..e23a9ca50f9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -777,7 +777,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     auto warn = [&]() {
         logWarning(
             ErrorInfo{
-                .name = "Path not content-addressed", 
+                .name = "Path not content-addressed",
                 .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
             });
     };
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 1fcb8111c5e..cb5fe70bfe5 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -61,7 +61,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 {
     // previous line of code.
     if (nixCode.prevLineOfCode.has_value()) {
-        out << std::endl 
+        out << std::endl
             << fmt("%1% %|2$5d|| %3%",
                 prefix,
                 (nixCode.errPos.line - 1),
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 108dc3bd107..105fadb1584 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -273,7 +273,7 @@ bool handleJSONLogMessage(const std::string & msg,
         }
 
     } catch (std::exception & e) {
-        logError({ 
+        logError({
             .name = "Json log message",
             .hint = hintfmt("bad log message from builder: %s", e.what())
         });
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index a2281237b22..667dd2edb83 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -314,7 +314,7 @@ string readFile(const Path & path)
 void readFile(const Path & path, Sink & sink)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)  
+    if (!fd)
         throw SysError("opening file '%s'", path);
     drainFD(fd.get(), sink);
 }
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index 582c78d14ed..bcb86cbce65 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -36,7 +36,7 @@ using namespace nix::daemon;
 #define SPLICE_F_MOVE 0
 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags)
 {
-    // We ignore most parameters, we just have them for conformance with the linux syscall 
+    // We ignore most parameters, we just have them for conformance with the linux syscall
     std::vector<char> buf(8192);
     auto read_count = read(fd_in, buf.data(), buf.size());
     if (read_count == -1)
@@ -57,7 +57,7 @@ static void sigChldHandler(int sigNo)
 {
     // Ensure we don't modify errno of whatever we've interrupted
     auto saved_errno = errno;
-    //  Reap all dead children. 
+    //  Reap all dead children.
     while (waitpid(-1, 0, WNOHANG) > 0) ;
     errno = saved_errno;
 }
@@ -106,7 +106,7 @@ struct PeerInfo
 };
 
 
-//  Get the identity of the caller, if possible. 
+//  Get the identity of the caller, if possible.
 static PeerInfo getPeerInfo(int remote)
 {
     PeerInfo peer = { false, 0, false, 0, false, 0 };
@@ -154,12 +154,12 @@ static void daemonLoop(char * * argv)
     if (chdir("/") == -1)
         throw SysError("cannot change current directory");
 
-    //  Get rid of children automatically; don't let them become zombies. 
+    //  Get rid of children automatically; don't let them become zombies.
     setSigChldAction(true);
 
     AutoCloseFD fdSocket;
 
-    //  Handle socket-based activation by systemd. 
+    //  Handle socket-based activation by systemd.
     auto listenFds = getEnv("LISTEN_FDS");
     if (listenFds) {
         if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1")
@@ -168,17 +168,17 @@ static void daemonLoop(char * * argv)
         closeOnExec(fdSocket.get());
     }
 
-    //  Otherwise, create and bind to a Unix domain socket. 
+    //  Otherwise, create and bind to a Unix domain socket.
     else {
         createDirs(dirOf(settings.nixDaemonSocketFile));
         fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
     }
 
-    //  Loop accepting connections. 
+    //  Loop accepting connections.
     while (1) {
 
         try {
-            //  Accept a connection. 
+            //  Accept a connection.
             struct sockaddr_un remoteAddr;
             socklen_t remoteAddrLen = sizeof(remoteAddr);
 
@@ -214,7 +214,7 @@ static void daemonLoop(char * * argv)
                 % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>")
                 % (peer.uidKnown ? user : "<unknown>"));
 
-            //  Fork a child to handle the connection. 
+            //  Fork a child to handle the connection.
             ProcessOptions options;
             options.errorPrefix = "unexpected Nix daemon error: ";
             options.dieWithParent = false;
@@ -223,20 +223,20 @@ static void daemonLoop(char * * argv)
             startProcess([&]() {
                 fdSocket = -1;
 
-                //  Background the daemon. 
+                //  Background the daemon.
                 if (setsid() == -1)
                     throw SysError("creating a new session");
 
-                //  Restore normal handling of SIGCHLD. 
+                //  Restore normal handling of SIGCHLD.
                 setSigChldAction(false);
 
-                //  For debugging, stuff the pid into argv[1]. 
+                //  For debugging, stuff the pid into argv[1].
                 if (peer.pidKnown && argv[1]) {
                     string processName = std::to_string(peer.pid);
                     strncpy(argv[1], processName.c_str(), strlen(argv[1]));
                 }
 
-                //  Handle the connection. 
+                //  Handle the connection.
                 FdSource from(remote.get());
                 FdSink to(remote.get());
                 processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid);
@@ -263,7 +263,7 @@ static int _main(int argc, char * * argv)
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
-                ; //  ignored for backwards compatibility 
+                ; //  ignored for backwards compatibility
             else if (*arg == "--help")
                 showManPage("nix-daemon");
             else if (*arg == "--version")
@@ -278,7 +278,7 @@ static int _main(int argc, char * * argv)
 
         if (stdio) {
             if (getStoreType() == tDaemon) {
-                //  Forward on this connection to the real daemon 
+                //  Forward on this connection to the real daemon
                 auto socketPath = settings.nixDaemonSocketFile;
                 auto s = socket(PF_UNIX, SOCK_STREAM, 0);
                 if (s == -1)
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 7ab5ba5000c..2c27d97f5bb 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -123,7 +123,7 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             if (!attrs.insert(attrName).second) {
-                logError({ 
+                logError({
                     .name = "Name collision",
                     .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
                 });
@@ -874,7 +874,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
             auto placeholder = metaObj.placeholder(j);
             Value * v = i.queryMeta(j);
             if (!v) {
-                logError({ 
+                logError({
                     .name = "Invalid meta attribute",
                     .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
                         i.queryName(), j)
@@ -1128,8 +1128,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                             XMLAttrs attrs2;
                             attrs2["name"] = j;
                             Value * v = i.queryMeta(j);
-                            if (!v) 
-                                logError({ 
+                            if (!v)
+                                logError({
                                     .name = "Invalid meta attribute",
                                     .hint = hintfmt(
                                         "derivation '%s' has invalid meta attribute '%s'",
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index dea53b52f8f..6d21395261d 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -704,7 +704,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", i);
 
     if (store->verifyStore(checkContents, repair)) {
-        logWarning({ 
+        logWarning({
             .name = "Store consistency",
             .description = "not all errors were fixed"
             });
@@ -729,7 +729,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
-            logError({ 
+            logError({
                 .name = "Hash mismatch",
                 .hint = hintfmt(
                     "path '%s' was modified! expected hash '%s', got '%s'",
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 246b4b48021..001401ac231 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -99,12 +99,12 @@ struct CmdVerify : StorePathsCommand
                     if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
-                        logError({ 
+                        logError({
                             .name = "Hash error - path modified",
                             .hint = hintfmt(
                                 "path '%s' was modified! expected hash '%s', got '%s'",
                                 store->printStorePath(info->path),
-                                info->narHash.to_string(Base32, true), 
+                                info->narHash.to_string(Base32, true),
                                 hash.first.to_string(Base32, true))
                         });
                     }
@@ -154,7 +154,7 @@ struct CmdVerify : StorePathsCommand
                     if (!good) {
                         untrusted++;
                         act2.result(resUntrustedPath, store->printStorePath(info->path));
-                        logError({ 
+                        logError({
                             .name = "Untrusted path",
                             .hint = hintfmt("path '%s' is untrusted",
                                 store->printStorePath(info->path))
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index 82feacb3dbb..434ad80a662 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -39,7 +39,7 @@ std::set<std::string> runResolver(const Path & filename)
         throw SysError("statting '%s'", filename);
 
     if (!S_ISREG(st.st_mode)) {
-        logError({ 
+        logError({
             .name = "Regular MACH file",
             .hint = hintfmt("file '%s' is not a regular file", filename)
         });
@@ -47,7 +47,7 @@ std::set<std::string> runResolver(const Path & filename)
     }
 
     if (st.st_size < sizeof(mach_header_64)) {
-        logError({ 
+        logError({
             .name = "File too short",
             .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
         });
@@ -72,7 +72,7 @@ std::set<std::string> runResolver(const Path & filename)
             }
         }
         if (mach64_offset == 0) {
-            logError({ 
+            logError({
                 .name = "No mach64 blobs",
                 .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
             });
@@ -81,7 +81,7 @@ std::set<std::string> runResolver(const Path & filename)
     } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
         mach64_offset = 0;
     } else {
-        logError({ 
+        logError({
             .name = "Magic number",
             .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
         });

From 5ed5d7acbd00298df7b4e3a638d731143084da2e Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 16:03:29 +0200
Subject: [PATCH 2189/2196] Improve "waiting for locks" messages

These are now shown in the progress bar.

Closes #3577.
---
 src/libmain/progress-bar.cc |  2 +-
 src/libstore/build.cc       | 22 +++++++++++++++++++---
 src/libutil/logging.hh      |  1 +
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 8d0421c2f92..6dfc3a4d306 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -156,7 +156,7 @@ class ProgressBar : public Logger
     {
         auto state(state_.lock());
 
-        if (lvl <= verbosity && !s.empty())
+        if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
             log(*state, lvl, s + "...");
 
         state->activities.emplace_back(ActInfo());
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index de393e837fa..2eb78591bc6 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -869,6 +869,9 @@ class DerivationGoal : public Goal
 
     std::unique_ptr<Activity> act;
 
+    /* Activity that denotes waiting for a lock. */
+    std::unique_ptr<Activity> actLock;
+
     std::map<ActivityId, Activity> builderActivities;
 
     /* The remote machine on which we're building. */
@@ -1439,10 +1442,15 @@ void DerivationGoal::tryToBuild()
         lockFiles.insert(worker.store.Store::toRealPath(outPath));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
+        if (!actLock)
+            actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
         worker.waitForAWhile(shared_from_this());
         return;
     }
 
+    actLock.reset();
+
     /* Now check again whether the outputs are valid.  This is because
        another process may have started building in parallel.  After
        it has finished and released the locks, we can (and should)
@@ -1481,6 +1489,7 @@ void DerivationGoal::tryToBuild()
             case rpAccept:
                 /* Yes, it has started doing so.  Wait until we get
                    EOF from the hook. */
+                actLock.reset();
                 result.startTime = time(0); // inexact
                 state = &DerivationGoal::buildDone;
                 started();
@@ -1488,6 +1497,9 @@ void DerivationGoal::tryToBuild()
             case rpPostpone:
                 /* Not now; wait until at least one child finishes or
                    the wake-up timeout expires. */
+                if (!actLock)
+                    actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                        fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
                 worker.waitForAWhile(shared_from_this());
                 outputLocks.unlock();
                 return;
@@ -1497,6 +1509,8 @@ void DerivationGoal::tryToBuild()
         }
     }
 
+    actLock.reset();
+
     /* Make sure that we are allowed to start a build.  If this
        derivation prefers to be done locally, do it even if
        maxBuildJobs is 0. */
@@ -1524,7 +1538,9 @@ void DerivationGoal::tryLocalBuild() {
                uid. */
             buildUser->kill();
         } else {
-            debug("waiting for build users");
+            if (!actLock)
+                actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+                    fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
             worker.waitForAWhile(shared_from_this());
             return;
         }
@@ -1535,6 +1551,8 @@ void DerivationGoal::tryLocalBuild() {
 #endif
     }
 
+    actLock.reset();
+
     try {
 
         /* Okay, we have to build. */
@@ -4863,8 +4881,6 @@ void Worker::waitForInput()
        up after a few seconds at most. */
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
-        if (lastWokenUp == steady_time_point::min())
-            printInfo("waiting for locks, build slots or build users...");
         if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
         timeout = std::max(1L,
             (long) std::chrono::duration_cast<std::chrono::seconds>(
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index b99b246c34c..b1583ecede6 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -18,6 +18,7 @@ typedef enum {
     actSubstitute = 108,
     actQueryPathInfo = 109,
     actPostBuildHook = 110,
+    actBuildWaiting = 111,
 } ActivityType;
 
 typedef enum {

From ccfa6b3eee9f5e613152a4b373f67ff05696adf8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 16:12:27 +0200
Subject: [PATCH 2190/2196] Give better error message about <...> in pure eval
 mode

---
 src/libexpr/parser.y | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index a1188dec03e..a639be64e3e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -689,7 +689,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         if (pathExists(res)) return canonPath(res);
     }
     throw ThrownError({
-        .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path),
+        .hint = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
         .nixCode = NixCode { .errPos = pos }
     });
 }

From 31707735b699db6dfd78e8bb8e655c3d5e6e0ba5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 16:47:21 +0200
Subject: [PATCH 2191/2196] Remove unnecessary amDone() overrides

---
 src/libstore/build.cc | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2eb78591bc6..2a34cfbb7de 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -187,7 +187,7 @@ class Goal : public std::enable_shared_from_this<Goal>
 
 protected:
 
-    virtual void amDone(ExitCode result);
+    void amDone(ExitCode result);
 };
 
 
@@ -1009,11 +1009,6 @@ class DerivationGoal : public Goal
 
     void repairClosure();
 
-    void amDone(ExitCode result) override
-    {
-        Goal::amDone(result);
-    }
-
     void started();
 
     void done(BuildResult::Status status, const string & msg = "");
@@ -4335,11 +4330,6 @@ class SubstitutionGoal : public Goal
     void handleEOF(int fd) override;
 
     StorePath getStorePath() { return storePath.clone(); }
-
-    void amDone(ExitCode result) override
-    {
-        Goal::amDone(result);
-    }
 };
 
 

From 4e995bc8a66e5486be83c935bd939726ac39cab3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 18:01:05 +0200
Subject: [PATCH 2192/2196] Always hide the progress bar on exit

---
 src/libmain/progress-bar.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 6dfc3a4d306..95a9187de69 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -111,10 +111,7 @@ class ProgressBar : public Logger
         auto state(state_.lock());
         if (!state->active) return;
         state->active = false;
-        std::string status = getStatus(*state);
         writeToStderr("\r\e[K");
-        if (status != "")
-            writeToStderr("[" + status + "]\n");
         updateCV.notify_one();
         quitCV.notify_one();
     }

From f20bb983cab168bd9fe4f4ad560a64aeb4b19f07 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 18:16:03 +0200
Subject: [PATCH 2193/2196] Cleanup

---
 src/libutil/error.cc | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index cb5fe70bfe5..da11d1ffd73 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -105,7 +105,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 
 std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 {
-    int errwidth = 80;
+    size_t errwidth = 80;
     string prefix = "";
 
     string levelString;
@@ -158,12 +158,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         }
     }
 
-    int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
-    int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+    auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
 
-    string dashes;
-    for (int i = 0; i < dashwidth; ++i)
-        dashes.append("-");
+    std::string dashes(dashwidth, '-');
 
     // divider.
     if (einfo.name != "")

From 8b099812eab7ccad9ebc6906450b637044de9369 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 18:18:02 +0200
Subject: [PATCH 2194/2196] Respect terminal width printing error messages

---
 src/libutil/error.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index da11d1ffd73..0fad9ae423e 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -105,7 +105,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 
 std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
 {
-    size_t errwidth = 80;
+    auto errwidth = std::max<size_t>(getWindowSize().second, 20);
     string prefix = "";
 
     string levelString;

From 24a3208247c248956c5815554804da1bf9763ece Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 18:20:05 +0200
Subject: [PATCH 2195/2196] Include only the base name of the program in error
 messages

---
 src/libmain/shared.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 0f2c189a6de..dc6d5e41394 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -290,7 +290,7 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
 
-    ErrorInfo::programName = programName;
+    ErrorInfo::programName = baseNameOf(programName);
 
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {

From a588b6b19d93fe80deb8518562b7a483fd4c50bc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 15 Jun 2020 19:25:35 +0200
Subject: [PATCH 2196/2196] Print only one error message if a build fails

E.g. instead of

  error: --- BuildError ----------------------------------------------- nix
  builder for '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed with exit code 1
  error: --- Error ---------------------------------------------------- nix
  build of '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed

we now get

  error: --- Error ---------------------------------------------------- nix
  builder for '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed with exit code 1
---
 src/libstore/build.cc | 138 ++++++++++++++++++++----------------------
 1 file changed, 67 insertions(+), 71 deletions(-)

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 2a34cfbb7de..8ad2ca4f32c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -104,13 +104,10 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
 
 
-class Goal : public std::enable_shared_from_this<Goal>
+struct Goal : public std::enable_shared_from_this<Goal>
 {
-public:
     typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
 
-protected:
-
     /* Backlink to the worker. */
     Worker & worker;
 
@@ -138,6 +135,9 @@ class Goal : public std::enable_shared_from_this<Goal>
     /* Whether the goal is finished. */
     ExitCode exitCode;
 
+    /* Exception containing an error message, if any. */
+    std::optional<Error> ex;
+
     Goal(Worker & worker) : worker(worker)
     {
         nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
@@ -149,7 +149,6 @@ class Goal : public std::enable_shared_from_this<Goal>
         trace("goal destroyed");
     }
 
-public:
     virtual void work() = 0;
 
     void addWaitee(GoalPtr waitee);
@@ -173,21 +172,14 @@ class Goal : public std::enable_shared_from_this<Goal>
         return name;
     }
 
-    ExitCode getExitCode()
-    {
-        return exitCode;
-    }
-
     /* Callback in case of a timeout.  It should wake up its waiters,
        get rid of any running child processes that are being monitored
        by the worker (important!), etc. */
-    virtual void timedOut() = 0;
+    virtual void timedOut(Error && ex) = 0;
 
     virtual string key() = 0;
 
-protected:
-
-    void amDone(ExitCode result);
+    void amDone(ExitCode result, std::optional<Error> ex = {});
 };
 
 
@@ -392,8 +384,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     assert(waitees.find(waitee) != waitees.end());
     waitees.erase(waitee);
 
-    trace(format("waitee '%1%' done; %2% left") %
-        waitee->name % waitees.size());
+    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
 
     if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
 
@@ -418,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
 }
 
 
-void Goal::amDone(ExitCode result)
+void Goal::amDone(ExitCode result, std::optional<Error> ex)
 {
     trace("done");
     assert(exitCode == ecBusy);
     assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
     exitCode = result;
+
+    if (ex) {
+        if (!waiters.empty())
+            logError(ex->info());
+        else
+            this->ex = std::move(*ex);
+    }
+
     for (auto & i : waiters) {
         GoalPtr goal = i.lock();
         if (goal) goal->waiteeDone(shared_from_this(), result);
@@ -910,7 +909,7 @@ class DerivationGoal : public Goal
     /* Whether we need to perform hash rewriting if there are valid output paths. */
     bool needsHashRewrite();
 
-    void timedOut() override;
+    void timedOut(Error && ex) override;
 
     string key() override
     {
@@ -1011,7 +1010,9 @@ class DerivationGoal : public Goal
 
     void started();
 
-    void done(BuildResult::Status status, const string & msg = "");
+    void done(
+        BuildResult::Status status,
+        std::optional<Error> ex = {});
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
@@ -1105,10 +1106,10 @@ void DerivationGoal::killChild()
 }
 
 
-void DerivationGoal::timedOut()
+void DerivationGoal::timedOut(Error && ex)
 {
     killChild();
-    done(BuildResult::TimedOut);
+    done(BuildResult::TimedOut, ex);
 }
 
 
@@ -1156,11 +1157,7 @@ void DerivationGoal::loadDerivation()
     trace("loading derivation");
 
     if (nrFailed != 0) {
-        logError({
-            .name = "missing derivation during build",
-            .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))
-        });
-        done(BuildResult::MiscFailure);
+        done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
         return;
     }
 
@@ -1349,13 +1346,9 @@ void DerivationGoal::inputsRealised()
     if (nrFailed != 0) {
         if (!useDerivation)
             throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
-        logError({
-            .name = "Dependencies could not be built",
-            .hint = hintfmt(
-                "cannot build derivation '%s': %s dependencies couldn't be built",
-                worker.store.printStorePath(drvPath), nrFailed)
-        });
-        done(BuildResult::DependencyFailed);
+        done(BuildResult::DependencyFailed, Error(
+                "%s dependencies of derivation '%s' failed to build",
+                nrFailed, worker.store.printStorePath(drvPath)));
         return;
     }
 
@@ -1554,11 +1547,10 @@ void DerivationGoal::tryLocalBuild() {
         startBuilder();
 
     } catch (BuildError & e) {
-        logError(e.info());
         outputLocks.unlock();
         buildUser.reset();
         worker.permanentFailure = true;
-        done(BuildResult::InputRejected, e.msg());
+        done(BuildResult::InputRejected, e);
         return;
     }
 
@@ -1670,7 +1662,7 @@ void DerivationGoal::buildDone()
                 }
 
             auto msg = fmt("builder for '%s' %s",
-                worker.store.printStorePath(drvPath),
+                yellowtxt(worker.store.printStorePath(drvPath)),
                 statusToString(status));
 
             if (!logger->isVerbose() && !logTail.empty()) {
@@ -1771,8 +1763,6 @@ void DerivationGoal::buildDone()
         outputLocks.unlock();
 
     } catch (BuildError & e) {
-        logError(e.info());
-
         outputLocks.unlock();
 
         BuildResult::Status st = BuildResult::MiscFailure;
@@ -1791,7 +1781,7 @@ void DerivationGoal::buildDone()
                 BuildResult::PermanentFailure;
         }
 
-        done(st, e.msg());
+        done(st, e);
         return;
     }
 
@@ -3881,7 +3871,6 @@ void DerivationGoal::registerOutputs()
                     .hint = hint
                 });
 
-
                 curRound = nrRounds; // we know enough, bail out early
             }
     }
@@ -4145,14 +4134,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     {
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
-            logError({
-                .name = "Max log size exceeded",
-                .hint = hintfmt(
-                    "%1% killed after writing more than %2% bytes of log output",
-                    getName(), settings.maxLogSize)
-            });
             killChild();
-            done(BuildResult::LogLimitExceeded);
+            done(
+                BuildResult::LogLimitExceeded,
+                Error("%s killed after writing more than %d bytes of log output",
+                    getName(), settings.maxLogSize));
             return;
         }
 
@@ -4233,11 +4219,12 @@ void DerivationGoal::addHashRewrite(const StorePath & path)
 }
 
 
-void DerivationGoal::done(BuildResult::Status status, const string & msg)
+void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
 {
     result.status = status;
-    result.errorMsg = msg;
-    amDone(result.success() ? ecSuccess : ecFailed);
+    if (ex)
+        result.errorMsg = ex->what();
+    amDone(result.success() ? ecSuccess : ecFailed, ex);
     if (result.status == BuildResult::TimedOut)
         worker.timedOut = true;
     if (result.status == BuildResult::PermanentFailure)
@@ -4306,7 +4293,7 @@ class SubstitutionGoal : public Goal
     SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair);
     ~SubstitutionGoal();
 
-    void timedOut() override { abort(); };
+    void timedOut(Error && ex) override { abort(); };
 
     string key() override
     {
@@ -4693,7 +4680,7 @@ void Worker::removeGoal(GoalPtr goal)
         topGoals.erase(goal);
         /* If a top-level goal failed, then kill all other goals
            (unless keepGoing was set). */
-        if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing)
+        if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
             topGoals.clear();
     }
 
@@ -4935,32 +4922,24 @@ void Worker::waitForInput()
             }
         }
 
-        if (goal->getExitCode() == Goal::ecBusy &&
+        if (goal->exitCode == Goal::ecBusy &&
             0 != settings.maxSilentTime &&
             j->respectTimeouts &&
             after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
-            logError({
-                .name = "Silent build timeout",
-                .hint = hintfmt(
+            goal->timedOut(Error(
                     "%1% timed out after %2% seconds of silence",
-                    goal->getName(), settings.maxSilentTime)
-            });
-            goal->timedOut();
+                    goal->getName(), settings.maxSilentTime));
         }
 
-        else if (goal->getExitCode() == Goal::ecBusy &&
+        else if (goal->exitCode == Goal::ecBusy &&
             0 != settings.buildTimeout &&
             j->respectTimeouts &&
             after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
-            logError({
-                .name = "Build timeout",
-                .hint = hintfmt(
+            goal->timedOut(Error(
                     "%1% timed out after %2% seconds",
-                    goal->getName(), settings.buildTimeout)
-            });
-            goal->timedOut();
+                    goal->getName(), settings.buildTimeout));
         }
     }
 
@@ -5066,16 +5045,28 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
     worker.run(goals);
 
     StorePathSet failed;
+    std::optional<Error> ex;
     for (auto & i : goals) {
-        if (i->getExitCode() != Goal::ecSuccess) {
+        if (i->ex) {
+            if (ex)
+                logError(i->ex->info());
+            else
+                ex = i->ex;
+        }
+        if (i->exitCode != Goal::ecSuccess) {
             DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
             if (i2) failed.insert(i2->getDrvPath());
             else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
         }
     }
 
-    if (!failed.empty())
+    if (failed.size() == 1 && ex) {
+        ex->status = worker.exitStatus();
+        throw *ex;
+    } else if (!failed.empty()) {
+        if (ex) logError(ex->info());
         throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
+    }
 }
 
 BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -5111,8 +5102,13 @@ void LocalStore::ensurePath(const StorePath & path)
 
     worker.run(goals);
 
-    if (goal->getExitCode() != Goal::ecSuccess)
-        throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
+    if (goal->exitCode != Goal::ecSuccess) {
+        if (goal->ex) {
+            goal->ex->status = worker.exitStatus();
+            throw *goal->ex;
+        } else
+            throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
+    }
 }
 
 
@@ -5124,7 +5120,7 @@ void LocalStore::repairPath(const StorePath & path)
 
     worker.run(goals);
 
-    if (goal->getExitCode() != Goal::ecSuccess) {
+    if (goal->exitCode != Goal::ecSuccess) {
         /* Since substituting the path didn't work, if we have a valid
            deriver, then rebuild the deriver. */
         auto info = queryPathInfo(path);